"abuse",
"ads",
"crypto",
- // ... (rest of the lists)
+ "drugs",
+ "facebook",
+ "fraud",
+ "gambling",
+ "malware",
+ "phishing",
+ "piracy",
+ "porn",
+ "ransomware",
+ "redirect",
+ "scam",
+ "tiktok",
+ "torrent",
+ "tracking",
+
+ // The following lists are in beta and therefore not included in the everything list:
+
+ // "smart-tv",
+ // "basic",
+ // "whatsapp",
+ // "vaping"
];
-async function optimizeEverythingList() {
- const files = (await fs.readdir(path.join(__dirname, "..")))
- .filter((file) => file.endsWith(".txt"))
- .filter((file) => listsToIncludeInEverythingList.some((val) => file.startsWith(val)));
+(async () => {
+ const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")).filter((file) => listsToIncludeInEverythingList.some((val) => file.startsWith(val))); // Array of strings, each representing a single file that ends in `.txt`
const domains = new Set();
- await Promise.all(
- files.map(async (file) => {
- const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8");
- fileContents.split("\n").forEach((line) => {
- if (line.startsWith("0.0.0.0 ")) {
- domains.add(line.slice(8)); // Using slice instead of replace to remove "0.0.0.0 "
- }
- });
- })
- );
-
- let everythingListContent = `# ------------------------------------[UPDATE]--------------------------------------
+ await Promise.all(files.map(async (file) => { // For each file
+
+ const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
+
+ fileContents.split("\n").forEach((line) => {
+ if (line.startsWith("0.0.0.0 ")) {
+ domains.add(line.replace("0.0.0.0 ", ""));
+ }
+ });
+ }));
+
+ let everythingListContent =
+`# ------------------------------------[UPDATE]--------------------------------------
# Title: The Block List Project - Everything List
# Expires: 1 day
# Homepage: https://blocklist.site
# -------------------------------------[INFO]---------------------------------------
#
# Everything list
-# ------------------------------------[FILTERS]-------------------------------------\n`;
-
- everythingListContent += Array.from(domains, (val) => `0.0.0.0 ${val}`).join("\n");
+# ------------------------------------[FILTERS]-------------------------------------
+`;
+ domains.forEach((val) => everythingListContent += `0.0.0.0 ${val}\n`);
await fs.writeFile(path.join(__dirname, "..", "everything.txt"), everythingListContent, "utf8");
-}
-
-optimizeEverythingList();
+})();
const path = require("path");
(async () => {
- try {
- const files = await fs.readdir(path.join(__dirname, "..")); // Get a list of all files in the parent directory
-
- // Filter files to keep only those ending with ".txt"
- const txtFiles = files.filter((file) => file.endsWith(".txt"));
-
- // Create an array to store promises for processing each file
- const promises = txtFiles.map(async (file) => {
- // Read file contents as a string
- const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8");
-
- // Process the file contents
- const adGuardFileContents = fileContents
- .replace(/^# Title: (.*?)$/gm, "# Title: $1 (adguard)") // Use "gm" flag to match multiple lines
- .replaceAll(/^# 0\.0\.0\.0 (.*?) (.*)/gm, "@@||$1^! $2") // Use "gm" flag to match multiple lines
- .replaceAll(/0\.0\.0\.0 (.*?)$/gm, "||$1^") // Use "gm" flag to match multiple lines
- .replaceAll(/^#/gm, "!"); // Use "gm" flag to match multiple lines
-
- // Write new file to `adguard` directory
- await fs.writeFile(path.join(__dirname, "..", "adguard", file.replace(".txt", "-ags.txt")), adGuardFileContents, "utf8");
- });
-
- // Wait for all promises to complete
- await Promise.all(promises);
-
- console.log("All files processed successfully.");
- } catch (error) {
- console.error("Error processing files:", error);
- }
+ const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
+ await Promise.all(files.map(async (file) => { // For each file
+ const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
+ const adGuardFileContents = fileContents
+ .replace(/^# Title: (.*?)$/gmu, "# Title: $1 (adguard)") // Add (adguard) to end of title
+ .replaceAll(/^# 0\.0\.0\.0 (.*?) (.*)/gmu, "@@||$1^! $2")
+ .replaceAll(/0\.0\.0\.0 (.*?)$/gmu, "||$1^")
+ .replaceAll(/^#/gmu, "!");
+ await fs.writeFile(path.join(__dirname, "..", "adguard", file.replace(".txt", "-ags.txt")), adGuardFileContents, "utf8"); // Write new file to `adguard` directory
+ }));
})();
const path = require("path");
(async () => {
- const inputDirectory = path.join(__dirname, "..");
- const outputDirectory = path.join(__dirname, "..", "dnsmasq-version");
-
- try {
- const files = await fs.readdir(inputDirectory);
- const txtFiles = files.filter(file => file.endsWith(".txt"));
-
- await Promise.all(txtFiles.map(async (file) => {
- const filePath = path.join(inputDirectory, file);
- const fileContents = await fs.readFile(filePath, "utf8");
-
- const noIPFileContents = fileContents
- .replaceAll(/0\.0\.0\.0 (.*?)( .*)?$/gmu, "0.0.0.0 $1/$2") // Add "/" at the end of each URL
- .replaceAll(/^0\.0\.0\.0 /gmu, "server=/") // Replace all occurrences of "0.0.0.0 " at the beginning of the line with "server=/"
- .replaceAll(/^# 0\.0\.0\.0 /gmu, "# server=/") // Replace all occurrences of "# 0.0.0.0 " at the beginning of the line with "# server=/"
- .replace(/^# Title: (.*?)$/gmu, "# Title: $1 (dnsmasq)"); // Add (dnsmasq) to end of title
-
- const outputFilePath = path.join(outputDirectory, file.replace(".txt", "-dnsmasq.txt"));
- await fs.writeFile(outputFilePath, noIPFileContents, "utf8");
- }));
-
- console.log("Files processed successfully!");
- } catch (error) {
- console.error("Error occurred while processing files:", error);
- }
+ const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
+ await Promise.all(files.map(async (file) => { // For each file
+ const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
+ const noIPFileContents = fileContents
+ .replaceAll(/0\.0\.0\.0 (.*?)( .*)?$/gmu, "0.0.0.0 $1/$2") // I need this line to add "/" at the end of each URL
+ .replaceAll(/^0\.0\.0\.0 /gmu, "server=/") // Replace all occurances of "0.0.0.0 " at the beginning of the line with "server=/"
+ .replaceAll(/^# 0\.0\.0\.0 /gmu, "# server=/") // Replace all occurances of "# 0.0.0.0 " at the beginning of the line with "# server=/"
+ .replace(/^# Title: (.*?)$/gmu, "# Title: $1 (dnsmasq)"); // Add (dnsmasq) to end of title
+ await fs.writeFile(path.join(__dirname, "..", "dnsmasq-version", file.replace(".txt", "-dnsmasq.txt")), noIPFileContents, "utf8"); // Write new file to `alt-version` directory
+ }));
})();
const path = require("path");
(async () => {
- const files = await fs.readdir(path.join(__dirname, ".."));
- const txtFiles = files.filter((file) => file.endsWith(".txt"));
-
- for (const file of txtFiles) {
- const filePath = path.resolve(__dirname, "..", file);
- const fileContents = await fs.readFile(filePath, "utf8");
-
- const noIPFileContents = fileContents.replace(/^#? ?0\.0\.0\.0 ?/gm, "").replace(/^# Title: (.*?)$/gm, "# Title: $1 (NL)");
-
- const newFileName = file.replace(".txt", "-nl.txt");
- const newFilePath = path.resolve(__dirname, "..", "alt-version", newFileName);
- await fs.writeFile(newFilePath, noIPFileContents, "utf8");
- }
+ const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
+ await Promise.all(files.map(async (file) => { // For each file
+ const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
+ const noIPFileContents = fileContents
+ .replaceAll(/^0\.0\.0\.0 /gmu, "") // Replace all occurances of "0.0.0.0 " at the beginning of the line with "" (nothing)
+ .replaceAll(/^# 0\.0\.0\.0 /gmu, "# ") // Replace all occurances of "# 0.0.0.0 " at the beginning of the line with "# "
+ .replace(/^# Title: (.*?)$/gmu, "# Title: $1 (NL)"); // Add (NL) to end of title
+ await fs.writeFile(path.join(__dirname, "..", "alt-version", file.replace(".txt", "-nl.txt")), noIPFileContents, "utf8"); // Write new file to `alt-version` directory
+ }));
})();
const path = require("path");
(async () => {
- let hasError = false;
+ let hasError = false;
- const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt"));
+ const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
+ await Promise.all(files.filter((file) => file !== "everything.txt").map(async (file) => { // For each file
+ const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
- await Promise.all(
- files.filter((file) => file !== "everything.txt").map(async (file) => {
- const filePath = path.join(__dirname, "..", file);
- const fileContents = await fs.readFile(filePath, "utf8");
+ const commentedURLs = fileContents.split("\n").map((line) => {
+ if (line.startsWith("# 0.0.0.0")) {
+ return line.split(" ")[2].trim();
+ }
- const lines = fileContents.split("\n");
- const commentedURLs = [];
+ return null;
+ }).filter((a) => a !== null && !!a);
- let isHeaderComplete = false;
+ let isHeaderComplete = false;
+ fileContents.split("\n").forEach((line, index) => {
+ if (line.startsWith("0.0.0.0")) {
+ isHeaderComplete = true;
+ }
- lines.forEach((line, index) => {
- if (line.startsWith("0.0.0.0")) {
- isHeaderComplete = true;
- }
+ // Ensuring that no version/date might confuse users that read the raw text-file(s)
+ if (line.length > 0 && !line.indexOf("Version")) {
+ console.error(`Line ${index + 1} in ${file} must not contain a Version/Date.`);
+ hasError = true;
+ }
- // Checking to ensure no version/date might confuse users that read the raw text-file(s)
- if (line.length > 0 && line.includes("Version")) {
- console.error(`Line ${index + 1} in ${file} must not contain a Version/Date.`);
- hasError = true;
- }
+ // Ensuring that all lines start with "#" or "0.0.0.0 "
+ if (line.length > 0 && !line.startsWith("#") && !line.startsWith("0.0.0.0 ")) {
+ console.error(`Line ${index + 1} in ${file} must start with "#" or "0.0.0.0 ".`);
+ hasError = true;
+ }
- // Ensuring that all lines start with "#" or "0.0.0.0 "
- if (line.length > 0 && !line.startsWith("#") && !line.startsWith("0.0.0.0 ")) {
- console.error(`Line ${index + 1} in ${file} must start with "#" or "0.0.0.0 ".`);
- hasError = true;
- }
+ // Checking to ensure all URLs are lowercase
+ if (line.startsWith("0.0.0.0 ")) {
+ const lineNoIP = line.replace("0.0.0.0 ", "");
+ const url = lineNoIP.split("#")[0].trim();
+ if (url.toLowerCase() !== url) {
+ console.error(`Line ${index + 1} in ${file} url ${url} must be all lowercase.`);
+ hasError = true;
+ }
+ }
- // Checking to ensure all URLs are lowercase
- if (line.startsWith("0.0.0.0 ")) {
- const url = line.split("#")[0].trim().replace("0.0.0.0 ", "");
- if (url.toLowerCase() !== url) {
- console.error(`Line ${index + 1} in ${file} url ${url} must be all lowercase.`);
- hasError = true;
- }
- }
+ // Ensuring that all lines that start with `#` are followed by a space
+ if (line.startsWith("#") && line.length > 1 && line[1] !== " ") {
+ console.error(`Line ${index + 1} in ${file} should have a space after #.`);
+ hasError = true;
+ }
- // Ensuring that all lines that start with `#` are followed by a space
- if (line.startsWith("#") && line.length > 1 && line[1] !== " ") {
- console.error(`Line ${index + 1} in ${file} should have a space after #.`);
- hasError = true;
- }
+ // Ensure that after header is complete that all lines that start with `#` start with `# 0.0.0.0` or `# NOTE:`
+ if (isHeaderComplete && line.startsWith("#") && !line.startsWith("# 0.0.0.0") && !line.startsWith("# NOTE:")) {
+ console.error(`Line ${index + 1} in ${file} should start with "# 0.0.0.0" or "# NOTE:".`);
+ hasError = true;
+ }
- // Ensure that after the header is complete, all lines that start with `#` start with `# 0.0.0.0` or `# NOTE:`
- if (isHeaderComplete && line.startsWith("#") && !line.startsWith("# 0.0.0.0") && !line.startsWith("# NOTE:")) {
- console.error(`Line ${index + 1} in ${file} should start with "# 0.0.0.0" or "# NOTE:".`);
- hasError = true;
- }
+ // Ensure that the URL doesn't exist in the commentedURLs array
+ if (line.startsWith("0.0.0.0 ")) {
+ const lineNoIP = line.replace("0.0.0.0 ", "");
+ const url = lineNoIP.split("#")[0].trim();
+ if (commentedURLs.includes(url)) {
+ console.error(`Line ${index + 1} in ${file} url ${url} is commented out in this file. This suggests an error. Please either remove this line or remove the commented URL.`);
+ hasError = true;
+ }
+ }
- // Ensure that the URL doesn't exist in the commentedURLs array
- if (line.startsWith("0.0.0.0 ")) {
- const url = line.split("#")[0].trim().replace("0.0.0.0 ", "");
- if (commentedURLs.includes(url)) {
- console.error(`Line ${index + 1} in ${file} url ${url} is commented out in this file. This suggests an error. Please either remove this line or remove the commented URL.`);
- hasError = true;
- } else {
- commentedURLs.push(url);
- }
- }
+ // Ensure that the URL doesn't contain whitespace characters
+ if (line.startsWith("0.0.0.0 ")) {
+ const lineNoIP = line.replace("0.0.0.0 ", "");
+ const url = lineNoIP.split("#")[0].trim();
+ if (/\s/gmu.test(url)) {
+ console.error(`Line ${index + 1} in ${file} url ${url} contains whitespace in the URL.`);
+ hasError = true;
+ }
+ }
+ });
+ }));
- // Ensure that the URL doesn't contain whitespace characters
- if (line.startsWith("0.0.0.0 ")) {
- const url = line.split("#")[0].trim().replace("0.0.0.0 ", "");
- if (/\s/gmu.test(url)) {
- console.error(`Line ${index + 1} in ${file} url ${url} contains whitespace in the URL.`);
- hasError = true;
- }
- }
- });
- })
- );
-
- process.exit(hasError ? 1 : 0);
+ process.exit(hasError ? 1 : 0);
})();
const path = require("path");
(async () => {
- const directory = path.join(__dirname, "..");
- const files = (await fs.readdir(directory)).filter((file) => file.endsWith(".txt"));
+ const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
- await Promise.all(
- files.map(async (file) => {
- const existingDomains = new Set();
- const filePath = path.join(directory, file);
+ await Promise.all(files.map(async (file) => { // For each file
+ const existingDomains = new Set();
- const lines = await fs.readFile(filePath, "utf8");
- const updatedLines = lines
- .split("\n")
- .filter((line) => {
- if (line.startsWith("0.0.0.0 ")) {
- const domain = line.replace("0.0.0.0 ", "");
- if (existingDomains.has(domain)) {
- return false;
- }
- existingDomains.add(domain);
- }
- return true;
- })
- .join("\n");
+ let fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
- await fs.writeFile(filePath, updatedLines, "utf8");
- })
- );
+ fileContents.split("\n").forEach((line) => {
+ if (line.startsWith("0.0.0.0 ")) {
+ const domain = line.replace("0.0.0.0 ", "");
+ if (existingDomains.has(domain)) {
+ fileContents = fileContents.replace(`${line}\n`, "");
+ }
+ existingDomains.add(domain);
+ }
+ });
+
+ await fs.writeFile(path.join(__dirname, "..", file), fileContents, "utf8");
+ }));
})();
const path = require("path");
(async () => {
- const files = (await fs.readdir(path.join(__dirname, ".."))).filter(file => file.endsWith(".txt"));
+ const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
- await Promise.all(files.map(async (file) => {
- const filePath = path.join(__dirname, "..", file);
+ await Promise.all(files.map(async (file) => { // For each file
+ const existingDomains = new Set();
- // Read the file contents asynchronously
- let fileContents = await fs.readFile(filePath, "utf8");
+ const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
- // Count the number of network filters using a regex
- const existingDomainsCount = (fileContents.match(/^0\.0\.0\.0 /gm) || []).length;
+ fileContents.split("\n").forEach((line) => {
+ if (line.startsWith("0.0.0.0 ")) {
+ existingDomains.add(line.replace("0.0.0.0 ", ""));
+ }
+ });
- // Replace the total number of network filters in the fileContents
- fileContents = fileContents.replace(/^# Total number of network filters: ?(\d*)$/gmu, `# Total number of network filters: ${existingDomainsCount}`);
-
- // Write the updated file contents asynchronously
- await fs.writeFile(filePath, fileContents, "utf8");
- }));
+ await fs.writeFile(path.join(__dirname, "..", file), fileContents.replace(/^# Total number of network filters: ?(\d*)$/gmu, `# Total number of network filters: ${existingDomains.size}`), "utf8");
+ }));
})();
projects / thirdparty / blocklistproject / lists.git / commitdiff
