From: gap579137 <gap579137@gmail.com>
Date: Thu, 20 Jul 2023 13:42:29 +0000 (-0500)
Subject: Revert "updated scripts"
X-Git-Tag: aggregated-20250518~106
X-Git-Url: http://git.ipfire.org/gitweb/gitweb.cgi?a=commitdiff_plain;h=4ea8176105be5a3db0d35e7ee29e6657ee0d64fa;p=thirdparty%2Fblocklistproject%2Flists.git

Revert "updated scripts"

This reverts commit 38573935d32b24453ef38c400bc88e104e876e65.
---

diff --git a/scripts/create-everything-list.js b/scripts/create-everything-list.js
index e46d769..d5fde74 100644
--- a/scripts/create-everything-list.js
+++ b/scripts/create-everything-list.js
@@ -5,28 +5,47 @@ const listsToIncludeInEverythingList = [
 	"abuse",
 	"ads",
 	"crypto",
-	// ... (rest of the lists)
+	"drugs",
+	"facebook",
+	"fraud",
+	"gambling",
+	"malware",
+	"phishing",
+	"piracy",
+	"porn",
+	"ransomware",
+	"redirect",
+	"scam",
+	"tiktok",
+	"torrent",
+	"tracking",
+
+	// The following lists are in beta and therefore not included in the everything list:
+
+	// "smart-tv",
+	// "basic",
+	// "whatsapp",
+	// "vaping"
 ];
 
-async function optimizeEverythingList() {
-	const files = (await fs.readdir(path.join(__dirname, "..")))
-		.filter((file) => file.endsWith(".txt"))
-		.filter((file) => listsToIncludeInEverythingList.some((val) => file.startsWith(val)));
+(async () => {
+	const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")).filter((file) => listsToIncludeInEverythingList.some((val) => file.startsWith(val))); // Array of strings, each representing a single file that ends in `.txt`
 
 	const domains = new Set();
 
-	await Promise.all(
-		files.map(async (file) => {
-			const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8");
-			fileContents.split("\n").forEach((line) => {
-				if (line.startsWith("0.0.0.0 ")) {
-					domains.add(line.slice(8)); // Using slice instead of replace to remove "0.0.0.0 "
-				}
-			});
-		})
-	);
-
-	let everythingListContent = `# ------------------------------------[UPDATE]--------------------------------------
+	await Promise.all(files.map(async (file) => { // For each file
+
+		const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
+
+		fileContents.split("\n").forEach((line) => {
+			if (line.startsWith("0.0.0.0 ")) {
+				domains.add(line.replace("0.0.0.0 ", ""));
+			}
+		});
+	}));
+
+	let everythingListContent =
+`# ------------------------------------[UPDATE]--------------------------------------
 # Title: The Block List Project - Everything List
 # Expires: 1 day
 # Homepage: https://blocklist.site
@@ -40,11 +59,9 @@ async function optimizeEverythingList() {
 # -------------------------------------[INFO]---------------------------------------
 #
 # Everything list
-# ------------------------------------[FILTERS]-------------------------------------\n`;
-
-	everythingListContent += Array.from(domains, (val) => `0.0.0.0 ${val}`).join("\n");
+# ------------------------------------[FILTERS]-------------------------------------
+`;
+	domains.forEach((val) => everythingListContent += `0.0.0.0 ${val}\n`);
 
 	await fs.writeFile(path.join(__dirname, "..", "everything.txt"), everythingListContent, "utf8");
-}
-
-optimizeEverythingList();
+})();
diff --git a/scripts/generate-adguard.js b/scripts/generate-adguard.js
index 76b001a..6bf7bce 100644
--- a/scripts/generate-adguard.js
+++ b/scripts/generate-adguard.js
@@ -2,33 +2,14 @@ const fs = require("fs").promises;
 const path = require("path");
 
 (async () => {
-  try {
-    const files = await fs.readdir(path.join(__dirname, "..")); // Get a list of all files in the parent directory
-
-    // Filter files to keep only those ending with ".txt"
-    const txtFiles = files.filter((file) => file.endsWith(".txt"));
-
-    // Create an array to store promises for processing each file
-    const promises = txtFiles.map(async (file) => {
-      // Read file contents as a string
-      const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8");
-
-      // Process the file contents
-      const adGuardFileContents = fileContents
-        .replace(/^# Title: (.*?)$/gm, "# Title: $1 (adguard)") // Use "gm" flag to match multiple lines
-        .replaceAll(/^# 0\.0\.0\.0 (.*?) (.*)/gm, "@@||$1^! $2") // Use "gm" flag to match multiple lines
-        .replaceAll(/0\.0\.0\.0 (.*?)$/gm, "||$1^") // Use "gm" flag to match multiple lines
-        .replaceAll(/^#/gm, "!"); // Use "gm" flag to match multiple lines
-
-      // Write new file to `adguard` directory
-      await fs.writeFile(path.join(__dirname, "..", "adguard", file.replace(".txt", "-ags.txt")), adGuardFileContents, "utf8");
-    });
-
-    // Wait for all promises to complete
-    await Promise.all(promises);
-
-    console.log("All files processed successfully.");
-  } catch (error) {
-    console.error("Error processing files:", error);
-  }
+	const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
+	await Promise.all(files.map(async (file) => { // For each file
+		const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
+		const adGuardFileContents = fileContents
+		.replace(/^# Title: (.*?)$/gmu, "# Title: $1 (adguard)") // Add (adguard) to end of title
+		.replaceAll(/^# 0\.0\.0\.0 (.*?) (.*)/gmu, "@@||$1^! $2")
+		.replaceAll(/0\.0\.0\.0 (.*?)$/gmu, "||$1^")
+		.replaceAll(/^#/gmu, "!");
+		await fs.writeFile(path.join(__dirname, "..", "adguard", file.replace(".txt", "-ags.txt")), adGuardFileContents, "utf8"); // Write new file to `adguard` directory
+	}));
 })();
diff --git a/scripts/generate-dnsmasq.js b/scripts/generate-dnsmasq.js
index 73b5075..f5ecf5a 100644
--- a/scripts/generate-dnsmasq.js
+++ b/scripts/generate-dnsmasq.js
@@ -2,29 +2,14 @@ const fs = require("fs").promises;
 const path = require("path");
 
 (async () => {
-  const inputDirectory = path.join(__dirname, "..");
-  const outputDirectory = path.join(__dirname, "..", "dnsmasq-version");
-  
-  try {
-    const files = await fs.readdir(inputDirectory);
-    const txtFiles = files.filter(file => file.endsWith(".txt"));
-
-    await Promise.all(txtFiles.map(async (file) => {
-      const filePath = path.join(inputDirectory, file);
-      const fileContents = await fs.readFile(filePath, "utf8");
-
-      const noIPFileContents = fileContents
-        .replaceAll(/0\.0\.0\.0 (.*?)( .*)?$/gmu, "0.0.0.0 $1/$2") // Add "/" at the end of each URL
-        .replaceAll(/^0\.0\.0\.0 /gmu, "server=/") // Replace all occurrences of "0.0.0.0 " at the beginning of the line with "server=/"
-        .replaceAll(/^# 0\.0\.0\.0 /gmu, "# server=/") // Replace all occurrences of "# 0.0.0.0 " at the beginning of the line with "# server=/"
-        .replace(/^# Title: (.*?)$/gmu, "# Title: $1 (dnsmasq)"); // Add (dnsmasq) to end of title
-      
-      const outputFilePath = path.join(outputDirectory, file.replace(".txt", "-dnsmasq.txt"));
-      await fs.writeFile(outputFilePath, noIPFileContents, "utf8");
-    }));
-    
-    console.log("Files processed successfully!");
-  } catch (error) {
-    console.error("Error occurred while processing files:", error);
-  }
+	const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
+	await Promise.all(files.map(async (file) => { // For each file
+		const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
+		const noIPFileContents = fileContents
+		.replaceAll(/0\.0\.0\.0 (.*?)( .*)?$/gmu, "0.0.0.0 $1/$2") // I need this line to add "/" at the end of each URL
+		.replaceAll(/^0\.0\.0\.0 /gmu, "server=/") // Replace all occurances of "0.0.0.0 " at the beginning of the line with "server=/"
+		.replaceAll(/^# 0\.0\.0\.0 /gmu, "# server=/") // Replace all occurances of "# 0.0.0.0 " at the beginning of the line with "# server=/"
+		.replace(/^# Title: (.*?)$/gmu, "# Title: $1 (dnsmasq)"); // Add (dnsmasq) to end of title
+		await fs.writeFile(path.join(__dirname, "..", "dnsmasq-version", file.replace(".txt", "-dnsmasq.txt")), noIPFileContents, "utf8"); // Write new file to `alt-version` directory
+	}));
 })();
diff --git a/scripts/generate-noip.js b/scripts/generate-noip.js
index e8b4a39..c998e9d 100644
--- a/scripts/generate-noip.js
+++ b/scripts/generate-noip.js
@@ -2,17 +2,13 @@ const fs = require("fs").promises;
 const path = require("path");
 
 (async () => {
-  const files = await fs.readdir(path.join(__dirname, ".."));
-  const txtFiles = files.filter((file) => file.endsWith(".txt"));
-
-  for (const file of txtFiles) {
-    const filePath = path.resolve(__dirname, "..", file);
-    const fileContents = await fs.readFile(filePath, "utf8");
-
-    const noIPFileContents = fileContents.replace(/^#? ?0\.0\.0\.0 ?/gm, "").replace(/^# Title: (.*?)$/gm, "# Title: $1 (NL)");
-
-    const newFileName = file.replace(".txt", "-nl.txt");
-    const newFilePath = path.resolve(__dirname, "..", "alt-version", newFileName);
-    await fs.writeFile(newFilePath, noIPFileContents, "utf8");
-  }
+	const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
+	await Promise.all(files.map(async (file) => { // For each file
+		const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
+		const noIPFileContents = fileContents
+		.replaceAll(/^0\.0\.0\.0 /gmu, "") // Replace all occurances of "0.0.0.0 " at the beginning of the line with "" (nothing)
+		.replaceAll(/^# 0\.0\.0\.0 /gmu, "# ") // Replace all occurances of "# 0.0.0.0 " at the beginning of the line with "# "
+		.replace(/^# Title: (.*?)$/gmu, "# Title: $1 (NL)"); // Add (NL) to end of title
+		await fs.writeFile(path.join(__dirname, "..", "alt-version", file.replace(".txt", "-nl.txt")), noIPFileContents, "utf8"); // Write new file to `alt-version` directory
+	}));
 })();
diff --git a/scripts/lint.js b/scripts/lint.js
index 70bb84d..f7e9f87 100644
--- a/scripts/lint.js
+++ b/scripts/lint.js
@@ -2,80 +2,81 @@ const fs = require("fs").promises;
 const path = require("path");
 
 (async () => {
-  let hasError = false;
+	let hasError = false;
 
-  const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt"));
+	const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
+	await Promise.all(files.filter((file) => file !== "everything.txt").map(async (file) => { // For each file
+		const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
 
-  await Promise.all(
-    files.filter((file) => file !== "everything.txt").map(async (file) => {
-      const filePath = path.join(__dirname, "..", file);
-      const fileContents = await fs.readFile(filePath, "utf8");
+		const commentedURLs = fileContents.split("\n").map((line) => {
+			if (line.startsWith("# 0.0.0.0")) {
+				return line.split(" ")[2].trim();
+			}
 
-      const lines = fileContents.split("\n");
-      const commentedURLs = [];
+			return null;
+		}).filter((a) => a !== null && !!a);
 
-      let isHeaderComplete = false;
+		let isHeaderComplete = false;
+		fileContents.split("\n").forEach((line, index) => {
+			if (line.startsWith("0.0.0.0")) {
+				isHeaderComplete = true;
+			}
 
-      lines.forEach((line, index) => {
-        if (line.startsWith("0.0.0.0")) {
-          isHeaderComplete = true;
-        }
+			// Ensuring that no version/date might confuse users that read the raw text-file(s)
+			if (line.length > 0 && !line.indexOf("Version")) {
+				console.error(`Line ${index + 1} in ${file} must not contain a Version/Date.`);
+				hasError = true;
+			}
 
-        // Checking to ensure no version/date might confuse users that read the raw text-file(s)
-        if (line.length > 0 && line.includes("Version")) {
-          console.error(`Line ${index + 1} in ${file} must not contain a Version/Date.`);
-          hasError = true;
-        }
+			// Ensuring that all lines start with "#" or "0.0.0.0 "
+			if (line.length > 0 && !line.startsWith("#") && !line.startsWith("0.0.0.0 ")) {
+				console.error(`Line ${index + 1} in ${file} must start with "#" or "0.0.0.0 ".`);
+				hasError = true;
+			}
 
-        // Ensuring that all lines start with "#" or "0.0.0.0 "
-        if (line.length > 0 && !line.startsWith("#") && !line.startsWith("0.0.0.0 ")) {
-          console.error(`Line ${index + 1} in ${file} must start with "#" or "0.0.0.0 ".`);
-          hasError = true;
-        }
+			// Checking to ensure all URLs are lowercase
+			if (line.startsWith("0.0.0.0 ")) {
+				const lineNoIP = line.replace("0.0.0.0 ", "");
+				const url = lineNoIP.split("#")[0].trim();
+				if (url.toLowerCase() !== url) {
+					console.error(`Line ${index + 1} in ${file} url ${url} must be all lowercase.`);
+					hasError = true;
+				}
+			}
 
-        // Checking to ensure all URLs are lowercase
-        if (line.startsWith("0.0.0.0 ")) {
-          const url = line.split("#")[0].trim().replace("0.0.0.0 ", "");
-          if (url.toLowerCase() !== url) {
-            console.error(`Line ${index + 1} in ${file} url ${url} must be all lowercase.`);
-            hasError = true;
-          }
-        }
+			// Ensuring that all lines that start with `#` are followed by a space
+			if (line.startsWith("#") && line.length > 1 && line[1] !== " ") {
+				console.error(`Line ${index + 1} in ${file} should have a space after #.`);
+				hasError = true;
+			}
 
-        // Ensuring that all lines that start with `#` are followed by a space
-        if (line.startsWith("#") && line.length > 1 && line[1] !== " ") {
-          console.error(`Line ${index + 1} in ${file} should have a space after #.`);
-          hasError = true;
-        }
+			// Ensure that after header is complete that all lines that start with `#` start with `# 0.0.0.0` or `# NOTE:`
+			if (isHeaderComplete && line.startsWith("#") && !line.startsWith("# 0.0.0.0") && !line.startsWith("# NOTE:")) {
+				console.error(`Line ${index + 1} in ${file} should start with "# 0.0.0.0" or "# NOTE:".`);
+				hasError = true;
+			}
 
-        // Ensure that after the header is complete, all lines that start with `#` start with `# 0.0.0.0` or `# NOTE:`
-        if (isHeaderComplete && line.startsWith("#") && !line.startsWith("# 0.0.0.0") && !line.startsWith("# NOTE:")) {
-          console.error(`Line ${index + 1} in ${file} should start with "# 0.0.0.0" or "# NOTE:".`);
-          hasError = true;
-        }
+			// Ensure that the URL doesn't exist in the commentedURLs array
+			if (line.startsWith("0.0.0.0 ")) {
+				const lineNoIP = line.replace("0.0.0.0 ", "");
+				const url = lineNoIP.split("#")[0].trim();
+				if (commentedURLs.includes(url)) {
+					console.error(`Line ${index + 1} in ${file} url ${url} is commented out in this file. This suggests an error. Please either remove this line or remove the commented URL.`);
+					hasError = true;
+				}
+			}
 
-        // Ensure that the URL doesn't exist in the commentedURLs array
-        if (line.startsWith("0.0.0.0 ")) {
-          const url = line.split("#")[0].trim().replace("0.0.0.0 ", "");
-          if (commentedURLs.includes(url)) {
-            console.error(`Line ${index + 1} in ${file} url ${url} is commented out in this file. This suggests an error. Please either remove this line or remove the commented URL.`);
-            hasError = true;
-          } else {
-            commentedURLs.push(url);
-          }
-        }
+			// Ensure that the URL doesn't contain whitespace characters
+			if (line.startsWith("0.0.0.0 ")) {
+				const lineNoIP = line.replace("0.0.0.0 ", "");
+				const url = lineNoIP.split("#")[0].trim();
+				if (/\s/gmu.test(url)) {
+					console.error(`Line ${index + 1} in ${file} url ${url} contains whitespace in the URL.`);
+					hasError = true;
+				}
+			}
+		});
+	}));
 
-        // Ensure that the URL doesn't contain whitespace characters
-        if (line.startsWith("0.0.0.0 ")) {
-          const url = line.split("#")[0].trim().replace("0.0.0.0 ", "");
-          if (/\s/gmu.test(url)) {
-            console.error(`Line ${index + 1} in ${file} url ${url} contains whitespace in the URL.`);
-            hasError = true;
-          }
-        }
-      });
-    })
-  );
-
-  process.exit(hasError ? 1 : 0);
+	process.exit(hasError ? 1 : 0);
 })();
diff --git a/scripts/remove-duplicates.js b/scripts/remove-duplicates.js
index a6e6738..d9e6ef3 100644
--- a/scripts/remove-duplicates.js
+++ b/scripts/remove-duplicates.js
@@ -2,30 +2,23 @@ const fs = require("fs").promises;
 const path = require("path");
 
 (async () => {
-	const directory = path.join(__dirname, "..");
-	const files = (await fs.readdir(directory)).filter((file) => file.endsWith(".txt"));
+	const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
 
-	await Promise.all(
-		files.map(async (file) => {
-			const existingDomains = new Set();
-			const filePath = path.join(directory, file);
+	await Promise.all(files.map(async (file) => { // For each file
+		const existingDomains = new Set();
 
-			const lines = await fs.readFile(filePath, "utf8");
-			const updatedLines = lines
-				.split("\n")
-				.filter((line) => {
-					if (line.startsWith("0.0.0.0 ")) {
-						const domain = line.replace("0.0.0.0 ", "");
-						if (existingDomains.has(domain)) {
-							return false;
-						}
-						existingDomains.add(domain);
-					}
-					return true;
-				})
-				.join("\n");
+		let fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
 
-			await fs.writeFile(filePath, updatedLines, "utf8");
-		})
-	);
+		fileContents.split("\n").forEach((line) => {
+			if (line.startsWith("0.0.0.0 ")) {
+				const domain = line.replace("0.0.0.0 ", "");
+				if (existingDomains.has(domain)) {
+					fileContents = fileContents.replace(`${line}\n`, "");
+				}
+				existingDomains.add(domain);
+			}
+		});
+
+		await fs.writeFile(path.join(__dirname, "..", file), fileContents, "utf8");
+	}));
 })();
diff --git a/scripts/update-number-of-domains.js b/scripts/update-number-of-domains.js
index a1ac4bb..5f50f79 100644
--- a/scripts/update-number-of-domains.js
+++ b/scripts/update-number-of-domains.js
@@ -2,21 +2,19 @@ const fs = require("fs").promises;
 const path = require("path");
 
 (async () => {
-  const files = (await fs.readdir(path.join(__dirname, ".."))).filter(file => file.endsWith(".txt"));
+	const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
 
-  await Promise.all(files.map(async (file) => {
-    const filePath = path.join(__dirname, "..", file);
+	await Promise.all(files.map(async (file) => { // For each file
+		const existingDomains = new Set();
 
-    // Read the file contents asynchronously
-    let fileContents = await fs.readFile(filePath, "utf8");
+		const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
 
-    // Count the number of network filters using a regex
-    const existingDomainsCount = (fileContents.match(/^0\.0\.0\.0 /gm) || []).length;
+		fileContents.split("\n").forEach((line) => {
+			if (line.startsWith("0.0.0.0 ")) {
+				existingDomains.add(line.replace("0.0.0.0 ", ""));
+			}
+		});
 
-    // Replace the total number of network filters in the fileContents
-    fileContents = fileContents.replace(/^# Total number of network filters: ?(\d*)$/gmu, `# Total number of network filters: ${existingDomainsCount}`);
-
-    // Write the updated file contents asynchronously
-    await fs.writeFile(filePath, fileContents, "utf8");
-  }));
+		await fs.writeFile(path.join(__dirname, "..", file), fileContents.replace(/^# Total number of network filters: ?(\d*)$/gmu, `# Total number of network filters: ${existingDomains.size}`), "utf8");
+	}));
 })();