Update remove-duplicates.js

diff --git a/scripts/remove-duplicates.js b/scripts/remove-duplicates.js
index d9e6ef37ae321c0d21ffbfcb19f5179c0400ab7e..5392acad9ce9601ac2a474fa4a1f730c4583f979 100644 (file)
--- a/scripts/remove-duplicates.js
+++ b/scripts/remove-duplicates.js
@@ -2,23 +2,36 @@ const fs = require("fs").promises;
 const path = require("path");
 
 (async () => {
-       const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
+       try {
+               const directoryPath = path.join(__dirname, "..");
+               const files = (await fs.readdir(directoryPath)).filter(file => file.endsWith(".txt"));
 
-       await Promise.all(files.map(async (file) => { // For each file
-               const existingDomains = new Set();
+               await Promise.all(files.map(async file => {
+                       const filePath = path.join(directoryPath, file);
+                       let fileContents = await fs.readFile(filePath, "utf8");
 
-               let fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
+                       const lines = fileContents.split("\n");
+                       const existingDomains = new Set();
+                       const filteredLines = [];
 
-               fileContents.split("\n").forEach((line) => {
-                       if (line.startsWith("0.0.0.0 ")) {
-                               const domain = line.replace("0.0.0.0 ", "");
-                               if (existingDomains.has(domain)) {
-                                       fileContents = fileContents.replace(`${line}\n`, "");
+                       lines.forEach(line => {
+                               if (line.startsWith("0.0.0.0 ")) {
+                                       const domain = line.replace("0.0.0.0 ", "");
+                                       if (!existingDomains.has(domain)) {
+                                               existingDomains.add(domain);
+                                               filteredLines.push(line);
+                                       }
+                               } else {
+                                       filteredLines.push(line);
                                }
-                               existingDomains.add(domain);
-                       }
-               });
+                       });
 
-               await fs.writeFile(path.join(__dirname, "..", file), fileContents, "utf8");
-       }));
+                       // Combine the filtered lines back into a single string
+                       const updatedContents = filteredLines.join("\n");
+
+                       await fs.writeFile(filePath, updatedContents, "utf8");
+               }));
+       } catch (error) {
+               console.error("Error processing files:", error);
+       }
 })();