From: Charlie Fish <fishcharlie@me.com>
Date: Sun, 20 Jun 2021 00:57:16 +0000 (-0600)
Subject: Adding script to remove duplicates
X-Git-Tag: aggregated-20250518~548^2~1^2
X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=refs%2Fpull%2F342%2Fhead;p=thirdparty%2Fblocklistproject%2Flists.git

Adding script to remove duplicates
---

diff --git a/.github/workflows/run-automations.yml b/.github/workflows/run-automations.yml
index df32e0b..eeaf992 100644
--- a/.github/workflows/run-automations.yml
+++ b/.github/workflows/run-automations.yml
@@ -13,6 +13,7 @@ jobs:
         uses: actions/setup-node@v1
         with:
           node-version: 15.x
+      - run: node scripts/remove-duplicates.js
       - run: node scripts/generate-noip.js
       - run: node scripts/generate-dnsmasq.js
       - name: Commit & Push
diff --git a/scripts/remove-duplicates.js b/scripts/remove-duplicates.js
new file mode 100644
index 0000000..d9e6ef3
--- /dev/null
+++ b/scripts/remove-duplicates.js
@@ -0,0 +1,24 @@
+const fs = require("fs").promises;
+const path = require("path");
+
+(async () => {
+	const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
+
+	await Promise.all(files.map(async (file) => { // For each file
+		const existingDomains = new Set();
+
+		let fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
+
+		fileContents.split("\n").forEach((line) => {
+			if (line.startsWith("0.0.0.0 ")) {
+				const domain = line.replace("0.0.0.0 ", "");
+				if (existingDomains.has(domain)) {
+					fileContents = fileContents.replace(`${line}\n`, "");
+				}
+				existingDomains.add(domain);
+			}
+		});
+
+		await fs.writeFile(path.join(__dirname, "..", file), fileContents, "utf8");
+	}));
+})();