Adding script to create everything list

diff --git a/.github/workflows/run-automations.yml b/.github/workflows/run-automations.yml
index eeaf99216d1556d9ddc5ceaae0927d4b54974dc1..e594d96d5a53653c411ede809a7bdbbb826ff7a8 100644 (file)
--- a/.github/workflows/run-automations.yml
+++ b/.github/workflows/run-automations.yml
@@ -13,6 +13,7 @@ jobs:
         uses: actions/setup-node@v1
         with:
           node-version: 15.x
+      - run: node scripts/create-everything-list.js
       - run: node scripts/remove-duplicates.js
       - run: node scripts/generate-noip.js
       - run: node scripts/generate-dnsmasq.js

diff --git a/scripts/create-everything-list.js b/scripts/create-everything-list.js
new file mode 100644 (file)
index 0000000..f116d84
--- /dev/null
+++ b/scripts/create-everything-list.js
@@ -0,0 +1,66 @@
+const fs = require("fs").promises;
+const path = require("path");
+
+const listsToIncludeInEverythingList = [
+       "abuse",
+       "ads",
+       "crypto",
+       "drugs",
+       "facebook",
+       "fraud",
+       "gambling",
+       "malware",
+       "phishing",
+       "piracy",
+       "porn",
+       "ransomware",
+       "redirect",
+       "scam",
+       "tiktok",
+       "torrent",
+       "tracking",
+
+       // The following lists are in beta and therefore not included in the everything list:
+
+       // "smart-tv",
+       // "basic",
+       // "whatsapp"
+];
+
+(async () => {
+       const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")).filter((file) => listsToIncludeInEverythingList.some((val) => file.startsWith(val))); // Array of strings, each representing a single file that ends in `.txt`
+
+       const domains = new Set();
+
+       await Promise.all(files.map(async (file) => { // For each file
+
+               const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
+
+               fileContents.split("\n").forEach((line) => {
+                       if (line.startsWith("0.0.0.0 ")) {
+                               domains.add(line.replace("0.0.0.0 ", ""));
+                       }
+               });
+       }));
+
+       let everythingListContent =
+`#------------------------------------[UPDATE]--------------------------------------
+# Title: The Block List Project - Everything List
+# Expires: 1 day
+# Homepage: https://blocklist.site
+# Help: https://github.com/blocklistproject/lists/wiki/
+# License: https://unlicense.org
+# Total number of network filters:
+#------------------------------------[SUPPORT]-------------------------------------
+# You can support by:
+# - reporting false positives
+# - making a donation: https://paypal.me/blocklistproject
+#-------------------------------------[INFO]---------------------------------------
+#
+# Everything list
+#------------------------------------[FILTERS]-------------------------------------
+`;
+       domains.forEach((val) => everythingListContent += `0.0.0.0 ${val}\n`);
+
+       await fs.writeFile(path.join(__dirname, "..", "everything.txt"), everythingListContent, "utf8");
+})();