From: Charlie Fish <fishcharlie@me.com>
Date: Mon, 28 Jun 2021 00:00:25 +0000 (-0600)
Subject: Adding script to create everything list
X-Git-Tag: aggregated-20250518~541^2~1
X-Git-Url: http://git.ipfire.org/gitweb.cgi?a=commitdiff_plain;h=787cb2070b1668291c71b5931f886accddd67095;p=thirdparty%2Fblocklistproject%2Flists.git

Adding script to create everything list
---

diff --git a/.github/workflows/run-automations.yml b/.github/workflows/run-automations.yml
index eeaf992..e594d96 100644
--- a/.github/workflows/run-automations.yml
+++ b/.github/workflows/run-automations.yml
@@ -13,6 +13,7 @@ jobs:
         uses: actions/setup-node@v1
         with:
           node-version: 15.x
+      - run: node scripts/create-everything-list.js
       - run: node scripts/remove-duplicates.js
       - run: node scripts/generate-noip.js
       - run: node scripts/generate-dnsmasq.js
diff --git a/scripts/create-everything-list.js b/scripts/create-everything-list.js
new file mode 100644
index 0000000..f116d84
--- /dev/null
+++ b/scripts/create-everything-list.js
@@ -0,0 +1,66 @@
+const fs = require("fs").promises;
+const path = require("path");
+
+const listsToIncludeInEverythingList = [
+	"abuse",
+	"ads",
+	"crypto",
+	"drugs",
+	"facebook",
+	"fraud",
+	"gambling",
+	"malware",
+	"phishing",
+	"piracy",
+	"porn",
+	"ransomware",
+	"redirect",
+	"scam",
+	"tiktok",
+	"torrent",
+	"tracking",
+
+	// The following lists are in beta and therefore not included in the everything list:
+
+	// "smart-tv",
+	// "basic",
+	// "whatsapp"
+];
+
+(async () => {
+	const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")).filter((file) => listsToIncludeInEverythingList.some((val) => file.startsWith(val))); // Array of strings, each representing a single file that ends in `.txt`
+
+	const domains = new Set();
+
+	await Promise.all(files.map(async (file) => { // For each file
+
+		const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
+
+		fileContents.split("\n").forEach((line) => {
+			if (line.startsWith("0.0.0.0 ")) {
+				domains.add(line.replace("0.0.0.0 ", ""));
+			}
+		});
+	}));
+
+	let everythingListContent =
+`#------------------------------------[UPDATE]--------------------------------------
+# Title: The Block List Project - Everything List
+# Expires: 1 day
+# Homepage: https://blocklist.site
+# Help: https://github.com/blocklistproject/lists/wiki/
+# License: https://unlicense.org
+# Total number of network filters:
+#------------------------------------[SUPPORT]-------------------------------------
+# You can support by:
+# - reporting false positives
+# - making a donation: https://paypal.me/blocklistproject
+#-------------------------------------[INFO]---------------------------------------
+#
+# Everything list
+#------------------------------------[FILTERS]-------------------------------------
+`;
+	domains.forEach((val) => everythingListContent += `0.0.0.0 ${val}\n`);
+
+	await fs.writeFile(path.join(__dirname, "..", "everything.txt"), everythingListContent, "utf8");
+})();