Adding script to remove duplicates

diff --git a/.github/workflows/run-automations.yml b/.github/workflows/run-automations.yml
index df32e0b466075dd94ec3f59b9bb017d7a9289247..eeaf99216d1556d9ddc5ceaae0927d4b54974dc1 100644 (file)
--- a/.github/workflows/run-automations.yml
+++ b/.github/workflows/run-automations.yml
@@ -13,6 +13,7 @@ jobs:
         uses: actions/setup-node@v1
         with:
           node-version: 15.x
+      - run: node scripts/remove-duplicates.js
       - run: node scripts/generate-noip.js
       - run: node scripts/generate-dnsmasq.js
       - name: Commit & Push

diff --git a/scripts/remove-duplicates.js b/scripts/remove-duplicates.js
new file mode 100644 (file)
index 0000000..d9e6ef3
--- /dev/null
+++ b/scripts/remove-duplicates.js
@@ -0,0 +1,24 @@
+const fs = require("fs").promises;
+const path = require("path");
+
+(async () => {
+       const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
+
+       await Promise.all(files.map(async (file) => { // For each file
+               const existingDomains = new Set();
+
+               let fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
+
+               fileContents.split("\n").forEach((line) => {
+                       if (line.startsWith("0.0.0.0 ")) {
+                               const domain = line.replace("0.0.0.0 ", "");
+                               if (existingDomains.has(domain)) {
+                                       fileContents = fileContents.replace(`${line}\n`, "");
+                               }
+                               existingDomains.add(domain);
+                       }
+               });
+
+               await fs.writeFile(path.join(__dirname, "..", file), fileContents, "utf8");
+       }));
+})();