From f26b42e491b11c798ea530851cacc015c708c1ec Mon Sep 17 00:00:00 2001
From: Charlie Fish <fishcharlie@me.com>
Date: Sat, 19 Jun 2021 18:57:16 -0600
Subject: [PATCH] Adding script to remove duplicates

---
 .github/workflows/run-automations.yml |  1 +
 scripts/remove-duplicates.js          | 24 ++++++++++++++++++++++++
 2 files changed, 25 insertions(+)
 create mode 100644 scripts/remove-duplicates.js

diff --git a/.github/workflows/run-automations.yml b/.github/workflows/run-automations.yml
index df32e0b..eeaf992 100644
--- a/.github/workflows/run-automations.yml
+++ b/.github/workflows/run-automations.yml
@@ -13,6 +13,7 @@ jobs:
         uses: actions/setup-node@v1
         with:
           node-version: 15.x
+      - run: node scripts/remove-duplicates.js
       - run: node scripts/generate-noip.js
       - run: node scripts/generate-dnsmasq.js
       - name: Commit & Push
diff --git a/scripts/remove-duplicates.js b/scripts/remove-duplicates.js
new file mode 100644
index 0000000..d9e6ef3
--- /dev/null
+++ b/scripts/remove-duplicates.js
@@ -0,0 +1,24 @@
+const fs = require("fs").promises;
+const path = require("path");
+
+(async () => {
+	const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
+
+	await Promise.all(files.map(async (file) => { // For each file
+		const existingDomains = new Set();
+
+		let fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
+
+		fileContents.split("\n").forEach((line) => {
+			if (line.startsWith("0.0.0.0 ")) {
+				const domain = line.replace("0.0.0.0 ", "");
+				if (existingDomains.has(domain)) {
+					fileContents = fileContents.replace(`${line}\n`, "");
+				}
+				existingDomains.add(domain);
+			}
+		});
+
+		await fs.writeFile(path.join(__dirname, "..", file), fileContents, "utf8");
+	}));
+})();
-- 
2.47.2