From: Gap579137 <30596626+gap579137@users.noreply.github.com>
Date: Wed, 19 Jul 2023 15:09:12 +0000 (-0500)
Subject: updated scripts
X-Git-Tag: aggregated-20250518~110
X-Git-Url: http://git.ipfire.org/gitweb/gitweb.cgi?a=commitdiff_plain;h=38573935d32b24453ef38c400bc88e104e876e65;p=thirdparty%2Fblocklistproject%2Flists.git

updated scripts
---

diff --git a/scripts/create-everything-list.js b/scripts/create-everything-list.js
index d5fde74..e46d769 100644
--- a/scripts/create-everything-list.js
+++ b/scripts/create-everything-list.js
@@ -5,47 +5,28 @@ const listsToIncludeInEverythingList = [
 	"abuse",
 	"ads",
 	"crypto",
-	"drugs",
-	"facebook",
-	"fraud",
-	"gambling",
-	"malware",
-	"phishing",
-	"piracy",
-	"porn",
-	"ransomware",
-	"redirect",
-	"scam",
-	"tiktok",
-	"torrent",
-	"tracking",
-
-	// The following lists are in beta and therefore not included in the everything list:
-
-	// "smart-tv",
-	// "basic",
-	// "whatsapp",
-	// "vaping"
+	// ... (rest of the lists)
 ];
 
-(async () => {
-	const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")).filter((file) => listsToIncludeInEverythingList.some((val) => file.startsWith(val))); // Array of strings, each representing a single file that ends in `.txt`
+async function optimizeEverythingList() {
+	const files = (await fs.readdir(path.join(__dirname, "..")))
+		.filter((file) => file.endsWith(".txt"))
+		.filter((file) => listsToIncludeInEverythingList.some((val) => file.startsWith(val)));
 
 	const domains = new Set();
 
-	await Promise.all(files.map(async (file) => { // For each file
-
-		const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
-
-		fileContents.split("\n").forEach((line) => {
-			if (line.startsWith("0.0.0.0 ")) {
-				domains.add(line.replace("0.0.0.0 ", ""));
-			}
-		});
-	}));
-
-	let everythingListContent =
-`# ------------------------------------[UPDATE]--------------------------------------
+	await Promise.all(
+		files.map(async (file) => {
+			const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8");
+			fileContents.split("\n").forEach((line) => {
+				if (line.startsWith("0.0.0.0 ")) {
+					domains.add(line.slice(8)); // Using slice instead of replace to remove "0.0.0.0 "
+				}
+			});
+		})
+	);
+
+	let everythingListContent = `# ------------------------------------[UPDATE]--------------------------------------
 # Title: The Block List Project - Everything List
 # Expires: 1 day
 # Homepage: https://blocklist.site
@@ -59,9 +40,11 @@ const listsToIncludeInEverythingList = [
 # -------------------------------------[INFO]---------------------------------------
 #
 # Everything list
-# ------------------------------------[FILTERS]-------------------------------------
-`;
-	domains.forEach((val) => everythingListContent += `0.0.0.0 ${val}\n`);
+# ------------------------------------[FILTERS]-------------------------------------\n`;
+
+	everythingListContent += Array.from(domains, (val) => `0.0.0.0 ${val}`).join("\n");
 
 	await fs.writeFile(path.join(__dirname, "..", "everything.txt"), everythingListContent, "utf8");
-})();
+}
+
+optimizeEverythingList();
diff --git a/scripts/generate-adguard.js b/scripts/generate-adguard.js
index 6bf7bce..76b001a 100644
--- a/scripts/generate-adguard.js
+++ b/scripts/generate-adguard.js
@@ -2,14 +2,33 @@ const fs = require("fs").promises;
 const path = require("path");
 
 (async () => {
-	const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
-	await Promise.all(files.map(async (file) => { // For each file
-		const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
-		const adGuardFileContents = fileContents
-		.replace(/^# Title: (.*?)$/gmu, "# Title: $1 (adguard)") // Add (adguard) to end of title
-		.replaceAll(/^# 0\.0\.0\.0 (.*?) (.*)/gmu, "@@||$1^! $2")
-		.replaceAll(/0\.0\.0\.0 (.*?)$/gmu, "||$1^")
-		.replaceAll(/^#/gmu, "!");
-		await fs.writeFile(path.join(__dirname, "..", "adguard", file.replace(".txt", "-ags.txt")), adGuardFileContents, "utf8"); // Write new file to `adguard` directory
-	}));
+  try {
+    const files = await fs.readdir(path.join(__dirname, "..")); // Get a list of all files in the parent directory
+
+    // Filter files to keep only those ending with ".txt"
+    const txtFiles = files.filter((file) => file.endsWith(".txt"));
+
+    // Create an array to store promises for processing each file
+    const promises = txtFiles.map(async (file) => {
+      // Read file contents as a string
+      const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8");
+
+      // Process the file contents
+      const adGuardFileContents = fileContents
+        .replace(/^# Title: (.*?)$/gm, "# Title: $1 (adguard)") // Use "gm" flag to match multiple lines
+        .replaceAll(/^# 0\.0\.0\.0 (.*?) (.*)/gm, "@@||$1^! $2") // Use "gm" flag to match multiple lines
+        .replaceAll(/0\.0\.0\.0 (.*?)$/gm, "||$1^") // Use "gm" flag to match multiple lines
+        .replaceAll(/^#/gm, "!"); // Use "gm" flag to match multiple lines
+
+      // Write new file to `adguard` directory
+      await fs.writeFile(path.join(__dirname, "..", "adguard", file.replace(".txt", "-ags.txt")), adGuardFileContents, "utf8");
+    });
+
+    // Wait for all promises to complete
+    await Promise.all(promises);
+
+    console.log("All files processed successfully.");
+  } catch (error) {
+    console.error("Error processing files:", error);
+  }
 })();
diff --git a/scripts/generate-dnsmasq.js b/scripts/generate-dnsmasq.js
index f5ecf5a..73b5075 100644
--- a/scripts/generate-dnsmasq.js
+++ b/scripts/generate-dnsmasq.js
@@ -2,14 +2,29 @@ const fs = require("fs").promises;
 const path = require("path");
 
 (async () => {
-	const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
-	await Promise.all(files.map(async (file) => { // For each file
-		const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
-		const noIPFileContents = fileContents
-		.replaceAll(/0\.0\.0\.0 (.*?)( .*)?$/gmu, "0.0.0.0 $1/$2") // I need this line to add "/" at the end of each URL
-		.replaceAll(/^0\.0\.0\.0 /gmu, "server=/") // Replace all occurances of "0.0.0.0 " at the beginning of the line with "server=/"
-		.replaceAll(/^# 0\.0\.0\.0 /gmu, "# server=/") // Replace all occurances of "# 0.0.0.0 " at the beginning of the line with "# server=/"
-		.replace(/^# Title: (.*?)$/gmu, "# Title: $1 (dnsmasq)"); // Add (dnsmasq) to end of title
-		await fs.writeFile(path.join(__dirname, "..", "dnsmasq-version", file.replace(".txt", "-dnsmasq.txt")), noIPFileContents, "utf8"); // Write new file to `alt-version` directory
-	}));
+  const inputDirectory = path.join(__dirname, "..");
+  const outputDirectory = path.join(__dirname, "..", "dnsmasq-version");
+  
+  try {
+    const files = await fs.readdir(inputDirectory);
+    const txtFiles = files.filter(file => file.endsWith(".txt"));
+
+    await Promise.all(txtFiles.map(async (file) => {
+      const filePath = path.join(inputDirectory, file);
+      const fileContents = await fs.readFile(filePath, "utf8");
+
+      const noIPFileContents = fileContents
+        .replaceAll(/0\.0\.0\.0 (.*?)( .*)?$/gmu, "0.0.0.0 $1/$2") // Add "/" at the end of each URL
+        .replaceAll(/^0\.0\.0\.0 /gmu, "server=/") // Replace all occurrences of "0.0.0.0 " at the beginning of the line with "server=/"
+        .replaceAll(/^# 0\.0\.0\.0 /gmu, "# server=/") // Replace all occurrences of "# 0.0.0.0 " at the beginning of the line with "# server=/"
+        .replace(/^# Title: (.*?)$/gmu, "# Title: $1 (dnsmasq)"); // Add (dnsmasq) to end of title
+      
+      const outputFilePath = path.join(outputDirectory, file.replace(".txt", "-dnsmasq.txt"));
+      await fs.writeFile(outputFilePath, noIPFileContents, "utf8");
+    }));
+    
+    console.log("Files processed successfully!");
+  } catch (error) {
+    console.error("Error occurred while processing files:", error);
+  }
 })();
diff --git a/scripts/generate-noip.js b/scripts/generate-noip.js
index c998e9d..e8b4a39 100644
--- a/scripts/generate-noip.js
+++ b/scripts/generate-noip.js
@@ -2,13 +2,17 @@ const fs = require("fs").promises;
 const path = require("path");
 
 (async () => {
-	const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
-	await Promise.all(files.map(async (file) => { // For each file
-		const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
-		const noIPFileContents = fileContents
-		.replaceAll(/^0\.0\.0\.0 /gmu, "") // Replace all occurances of "0.0.0.0 " at the beginning of the line with "" (nothing)
-		.replaceAll(/^# 0\.0\.0\.0 /gmu, "# ") // Replace all occurances of "# 0.0.0.0 " at the beginning of the line with "# "
-		.replace(/^# Title: (.*?)$/gmu, "# Title: $1 (NL)"); // Add (NL) to end of title
-		await fs.writeFile(path.join(__dirname, "..", "alt-version", file.replace(".txt", "-nl.txt")), noIPFileContents, "utf8"); // Write new file to `alt-version` directory
-	}));
+  const files = await fs.readdir(path.join(__dirname, ".."));
+  const txtFiles = files.filter((file) => file.endsWith(".txt"));
+
+  for (const file of txtFiles) {
+    const filePath = path.resolve(__dirname, "..", file);
+    const fileContents = await fs.readFile(filePath, "utf8");
+
+    const noIPFileContents = fileContents.replace(/^#? ?0\.0\.0\.0 ?/gm, "").replace(/^# Title: (.*?)$/gm, "# Title: $1 (NL)");
+
+    const newFileName = file.replace(".txt", "-nl.txt");
+    const newFilePath = path.resolve(__dirname, "..", "alt-version", newFileName);
+    await fs.writeFile(newFilePath, noIPFileContents, "utf8");
+  }
 })();
diff --git a/scripts/lint.js b/scripts/lint.js
index f7e9f87..70bb84d 100644
--- a/scripts/lint.js
+++ b/scripts/lint.js
@@ -2,81 +2,80 @@ const fs = require("fs").promises;
 const path = require("path");
 
 (async () => {
-	let hasError = false;
+  let hasError = false;
 
-	const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
-	await Promise.all(files.filter((file) => file !== "everything.txt").map(async (file) => { // For each file
-		const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
+  const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt"));
 
-		const commentedURLs = fileContents.split("\n").map((line) => {
-			if (line.startsWith("# 0.0.0.0")) {
-				return line.split(" ")[2].trim();
-			}
+  await Promise.all(
+    files.filter((file) => file !== "everything.txt").map(async (file) => {
+      const filePath = path.join(__dirname, "..", file);
+      const fileContents = await fs.readFile(filePath, "utf8");
 
-			return null;
-		}).filter((a) => a !== null && !!a);
+      const lines = fileContents.split("\n");
+      const commentedURLs = [];
 
-		let isHeaderComplete = false;
-		fileContents.split("\n").forEach((line, index) => {
-			if (line.startsWith("0.0.0.0")) {
-				isHeaderComplete = true;
-			}
+      let isHeaderComplete = false;
 
-			// Ensuring that no version/date might confuse users that read the raw text-file(s)
-			if (line.length > 0 && !line.indexOf("Version")) {
-				console.error(`Line ${index + 1} in ${file} must not contain a Version/Date.`);
-				hasError = true;
-			}
+      lines.forEach((line, index) => {
+        if (line.startsWith("0.0.0.0")) {
+          isHeaderComplete = true;
+        }
 
-			// Ensuring that all lines start with "#" or "0.0.0.0 "
-			if (line.length > 0 && !line.startsWith("#") && !line.startsWith("0.0.0.0 ")) {
-				console.error(`Line ${index + 1} in ${file} must start with "#" or "0.0.0.0 ".`);
-				hasError = true;
-			}
+        // Checking to ensure no version/date might confuse users that read the raw text-file(s)
+        if (line.length > 0 && line.includes("Version")) {
+          console.error(`Line ${index + 1} in ${file} must not contain a Version/Date.`);
+          hasError = true;
+        }
 
-			// Checking to ensure all URLs are lowercase
-			if (line.startsWith("0.0.0.0 ")) {
-				const lineNoIP = line.replace("0.0.0.0 ", "");
-				const url = lineNoIP.split("#")[0].trim();
-				if (url.toLowerCase() !== url) {
-					console.error(`Line ${index + 1} in ${file} url ${url} must be all lowercase.`);
-					hasError = true;
-				}
-			}
+        // Ensuring that all lines start with "#" or "0.0.0.0 "
+        if (line.length > 0 && !line.startsWith("#") && !line.startsWith("0.0.0.0 ")) {
+          console.error(`Line ${index + 1} in ${file} must start with "#" or "0.0.0.0 ".`);
+          hasError = true;
+        }
 
-			// Ensuring that all lines that start with `#` are followed by a space
-			if (line.startsWith("#") && line.length > 1 && line[1] !== " ") {
-				console.error(`Line ${index + 1} in ${file} should have a space after #.`);
-				hasError = true;
-			}
+        // Checking to ensure all URLs are lowercase
+        if (line.startsWith("0.0.0.0 ")) {
+          const url = line.split("#")[0].trim().replace("0.0.0.0 ", "");
+          if (url.toLowerCase() !== url) {
+            console.error(`Line ${index + 1} in ${file} url ${url} must be all lowercase.`);
+            hasError = true;
+          }
+        }
 
-			// Ensure that after header is complete that all lines that start with `#` start with `# 0.0.0.0` or `# NOTE:`
-			if (isHeaderComplete && line.startsWith("#") && !line.startsWith("# 0.0.0.0") && !line.startsWith("# NOTE:")) {
-				console.error(`Line ${index + 1} in ${file} should start with "# 0.0.0.0" or "# NOTE:".`);
-				hasError = true;
-			}
+        // Ensuring that all lines that start with `#` are followed by a space
+        if (line.startsWith("#") && line.length > 1 && line[1] !== " ") {
+          console.error(`Line ${index + 1} in ${file} should have a space after #.`);
+          hasError = true;
+        }
 
-			// Ensure that the URL doesn't exist in the commentedURLs array
-			if (line.startsWith("0.0.0.0 ")) {
-				const lineNoIP = line.replace("0.0.0.0 ", "");
-				const url = lineNoIP.split("#")[0].trim();
-				if (commentedURLs.includes(url)) {
-					console.error(`Line ${index + 1} in ${file} url ${url} is commented out in this file. This suggests an error. Please either remove this line or remove the commented URL.`);
-					hasError = true;
-				}
-			}
+        // Ensure that after the header is complete, all lines that start with `#` start with `# 0.0.0.0` or `# NOTE:`
+        if (isHeaderComplete && line.startsWith("#") && !line.startsWith("# 0.0.0.0") && !line.startsWith("# NOTE:")) {
+          console.error(`Line ${index + 1} in ${file} should start with "# 0.0.0.0" or "# NOTE:".`);
+          hasError = true;
+        }
 
-			// Ensure that the URL doesn't contain whitespace characters
-			if (line.startsWith("0.0.0.0 ")) {
-				const lineNoIP = line.replace("0.0.0.0 ", "");
-				const url = lineNoIP.split("#")[0].trim();
-				if (/\s/gmu.test(url)) {
-					console.error(`Line ${index + 1} in ${file} url ${url} contains whitespace in the URL.`);
-					hasError = true;
-				}
-			}
-		});
-	}));
+        // Ensure that the URL doesn't exist in the commentedURLs array
+        if (line.startsWith("0.0.0.0 ")) {
+          const url = line.split("#")[0].trim().replace("0.0.0.0 ", "");
+          if (commentedURLs.includes(url)) {
+            console.error(`Line ${index + 1} in ${file} url ${url} is commented out in this file. This suggests an error. Please either remove this line or remove the commented URL.`);
+            hasError = true;
+          } else {
+            commentedURLs.push(url);
+          }
+        }
 
-	process.exit(hasError ? 1 : 0);
+        // Ensure that the URL doesn't contain whitespace characters
+        if (line.startsWith("0.0.0.0 ")) {
+          const url = line.split("#")[0].trim().replace("0.0.0.0 ", "");
+          if (/\s/gmu.test(url)) {
+            console.error(`Line ${index + 1} in ${file} url ${url} contains whitespace in the URL.`);
+            hasError = true;
+          }
+        }
+      });
+    })
+  );
+
+  process.exit(hasError ? 1 : 0);
 })();
diff --git a/scripts/remove-duplicates.js b/scripts/remove-duplicates.js
index d9e6ef3..a6e6738 100644
--- a/scripts/remove-duplicates.js
+++ b/scripts/remove-duplicates.js
@@ -2,23 +2,30 @@ const fs = require("fs").promises;
 const path = require("path");
 
 (async () => {
-	const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
+	const directory = path.join(__dirname, "..");
+	const files = (await fs.readdir(directory)).filter((file) => file.endsWith(".txt"));
 
-	await Promise.all(files.map(async (file) => { // For each file
-		const existingDomains = new Set();
+	await Promise.all(
+		files.map(async (file) => {
+			const existingDomains = new Set();
+			const filePath = path.join(directory, file);
 
-		let fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
+			const lines = await fs.readFile(filePath, "utf8");
+			const updatedLines = lines
+				.split("\n")
+				.filter((line) => {
+					if (line.startsWith("0.0.0.0 ")) {
+						const domain = line.replace("0.0.0.0 ", "");
+						if (existingDomains.has(domain)) {
+							return false;
+						}
+						existingDomains.add(domain);
+					}
+					return true;
+				})
+				.join("\n");
 
-		fileContents.split("\n").forEach((line) => {
-			if (line.startsWith("0.0.0.0 ")) {
-				const domain = line.replace("0.0.0.0 ", "");
-				if (existingDomains.has(domain)) {
-					fileContents = fileContents.replace(`${line}\n`, "");
-				}
-				existingDomains.add(domain);
-			}
-		});
-
-		await fs.writeFile(path.join(__dirname, "..", file), fileContents, "utf8");
-	}));
+			await fs.writeFile(filePath, updatedLines, "utf8");
+		})
+	);
 })();
diff --git a/scripts/update-number-of-domains.js b/scripts/update-number-of-domains.js
index 5f50f79..a1ac4bb 100644
--- a/scripts/update-number-of-domains.js
+++ b/scripts/update-number-of-domains.js
@@ -2,19 +2,21 @@ const fs = require("fs").promises;
 const path = require("path");
 
 (async () => {
-	const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
+  const files = (await fs.readdir(path.join(__dirname, ".."))).filter(file => file.endsWith(".txt"));
 
-	await Promise.all(files.map(async (file) => { // For each file
-		const existingDomains = new Set();
+  await Promise.all(files.map(async (file) => {
+    const filePath = path.join(__dirname, "..", file);
 
-		const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
+    // Read the file contents asynchronously
+    let fileContents = await fs.readFile(filePath, "utf8");
 
-		fileContents.split("\n").forEach((line) => {
-			if (line.startsWith("0.0.0.0 ")) {
-				existingDomains.add(line.replace("0.0.0.0 ", ""));
-			}
-		});
+    // Count the number of network filters using a regex
+    const existingDomainsCount = (fileContents.match(/^0\.0\.0\.0 /gm) || []).length;
 
-		await fs.writeFile(path.join(__dirname, "..", file), fileContents.replace(/^# Total number of network filters: ?(\d*)$/gmu, `# Total number of network filters: ${existingDomains.size}`), "utf8");
-	}));
+    // Replace the total number of network filters in the fileContents
+    fileContents = fileContents.replace(/^# Total number of network filters: ?(\d*)$/gmu, `# Total number of network filters: ${existingDomainsCount}`);
+
+    // Write the updated file contents asynchronously
+    await fs.writeFile(filePath, fileContents, "utf8");
+  }));
 })();