"abuse",
"ads",
"crypto",
- "drugs",
- "facebook",
- "fraud",
- "gambling",
- "malware",
- "phishing",
- "piracy",
- "porn",
- "ransomware",
- "redirect",
- "scam",
- "tiktok",
- "torrent",
- "tracking",
-
- // The following lists are in beta and therefore not included in the everything list:
-
- // "smart-tv",
- // "basic",
- // "whatsapp",
- // "vaping"
+ // ... (rest of the lists)
];
-(async () => {
- const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")).filter((file) => listsToIncludeInEverythingList.some((val) => file.startsWith(val))); // Array of strings, each representing a single file that ends in `.txt`
+async function optimizeEverythingList() {
+ const files = (await fs.readdir(path.join(__dirname, "..")))
+ .filter((file) => file.endsWith(".txt"))
+ .filter((file) => listsToIncludeInEverythingList.some((val) => file.startsWith(val)));
const domains = new Set();
- await Promise.all(files.map(async (file) => { // For each file
-
- const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
-
- fileContents.split("\n").forEach((line) => {
- if (line.startsWith("0.0.0.0 ")) {
- domains.add(line.replace("0.0.0.0 ", ""));
- }
- });
- }));
-
- let everythingListContent =
-`# ------------------------------------[UPDATE]--------------------------------------
+ await Promise.all(
+ files.map(async (file) => {
+ const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8");
+ fileContents.split("\n").forEach((line) => {
+ if (line.startsWith("0.0.0.0 ")) {
+ domains.add(line.slice(8)); // Using slice instead of replace to remove "0.0.0.0 "
+ }
+ });
+ })
+ );
+
+ let everythingListContent = `# ------------------------------------[UPDATE]--------------------------------------
# Title: The Block List Project - Everything List
# Expires: 1 day
# Homepage: https://blocklist.site
# -------------------------------------[INFO]---------------------------------------
#
# Everything list
-# ------------------------------------[FILTERS]-------------------------------------
-`;
- domains.forEach((val) => everythingListContent += `0.0.0.0 ${val}\n`);
+# ------------------------------------[FILTERS]-------------------------------------\n`;
+
+ everythingListContent += Array.from(domains, (val) => `0.0.0.0 ${val}`).join("\n");
await fs.writeFile(path.join(__dirname, "..", "everything.txt"), everythingListContent, "utf8");
-})();
+}
+
+optimizeEverythingList();
const path = require("path");
(async () => {
- const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
- await Promise.all(files.map(async (file) => { // For each file
- const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
- const adGuardFileContents = fileContents
- .replace(/^# Title: (.*?)$/gmu, "# Title: $1 (adguard)") // Add (adguard) to end of title
- .replaceAll(/^# 0\.0\.0\.0 (.*?) (.*)/gmu, "@@||$1^! $2")
- .replaceAll(/0\.0\.0\.0 (.*?)$/gmu, "||$1^")
- .replaceAll(/^#/gmu, "!");
- await fs.writeFile(path.join(__dirname, "..", "adguard", file.replace(".txt", "-ags.txt")), adGuardFileContents, "utf8"); // Write new file to `adguard` directory
- }));
+ try {
+ const files = await fs.readdir(path.join(__dirname, "..")); // Get a list of all files in the parent directory
+
+ // Filter files to keep only those ending with ".txt"
+ const txtFiles = files.filter((file) => file.endsWith(".txt"));
+
+ // Create an array to store promises for processing each file
+ const promises = txtFiles.map(async (file) => {
+ // Read file contents as a string
+ const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8");
+
+ // Process the file contents
+ const adGuardFileContents = fileContents
+ .replace(/^# Title: (.*?)$/gm, "# Title: $1 (adguard)") // Use "gm" flag to match multiple lines
+ .replaceAll(/^# 0\.0\.0\.0 (.*?) (.*)/gm, "@@||$1^! $2") // Use "gm" flag to match multiple lines
+ .replaceAll(/0\.0\.0\.0 (.*?)$/gm, "||$1^") // Use "gm" flag to match multiple lines
+ .replaceAll(/^#/gm, "!"); // Use "gm" flag to match multiple lines
+
+ // Write new file to `adguard` directory
+ await fs.writeFile(path.join(__dirname, "..", "adguard", file.replace(".txt", "-ags.txt")), adGuardFileContents, "utf8");
+ });
+
+ // Wait for all promises to complete
+ await Promise.all(promises);
+
+ console.log("All files processed successfully.");
+ } catch (error) {
+ console.error("Error processing files:", error);
+ }
})();
const path = require("path");
(async () => {
- const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
- await Promise.all(files.map(async (file) => { // For each file
- const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
- const noIPFileContents = fileContents
- .replaceAll(/0\.0\.0\.0 (.*?)( .*)?$/gmu, "0.0.0.0 $1/$2") // I need this line to add "/" at the end of each URL
- .replaceAll(/^0\.0\.0\.0 /gmu, "server=/") // Replace all occurances of "0.0.0.0 " at the beginning of the line with "server=/"
- .replaceAll(/^# 0\.0\.0\.0 /gmu, "# server=/") // Replace all occurances of "# 0.0.0.0 " at the beginning of the line with "# server=/"
- .replace(/^# Title: (.*?)$/gmu, "# Title: $1 (dnsmasq)"); // Add (dnsmasq) to end of title
- await fs.writeFile(path.join(__dirname, "..", "dnsmasq-version", file.replace(".txt", "-dnsmasq.txt")), noIPFileContents, "utf8"); // Write new file to `alt-version` directory
- }));
+ const inputDirectory = path.join(__dirname, "..");
+ const outputDirectory = path.join(__dirname, "..", "dnsmasq-version");
+
+ try {
+ const files = await fs.readdir(inputDirectory);
+ const txtFiles = files.filter(file => file.endsWith(".txt"));
+
+ await Promise.all(txtFiles.map(async (file) => {
+ const filePath = path.join(inputDirectory, file);
+ const fileContents = await fs.readFile(filePath, "utf8");
+
+ const noIPFileContents = fileContents
+ .replaceAll(/0\.0\.0\.0 (.*?)( .*)?$/gmu, "0.0.0.0 $1/$2") // Add "/" at the end of each URL
+ .replaceAll(/^0\.0\.0\.0 /gmu, "server=/") // Replace all occurrences of "0.0.0.0 " at the beginning of the line with "server=/"
+ .replaceAll(/^# 0\.0\.0\.0 /gmu, "# server=/") // Replace all occurrences of "# 0.0.0.0 " at the beginning of the line with "# server=/"
+ .replace(/^# Title: (.*?)$/gmu, "# Title: $1 (dnsmasq)"); // Add (dnsmasq) to end of title
+
+ const outputFilePath = path.join(outputDirectory, file.replace(".txt", "-dnsmasq.txt"));
+ await fs.writeFile(outputFilePath, noIPFileContents, "utf8");
+ }));
+
+ console.log("Files processed successfully!");
+ } catch (error) {
+ console.error("Error occurred while processing files:", error);
+ }
})();
const path = require("path");
(async () => {
- const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
- await Promise.all(files.map(async (file) => { // For each file
- const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
- const noIPFileContents = fileContents
- .replaceAll(/^0\.0\.0\.0 /gmu, "") // Replace all occurances of "0.0.0.0 " at the beginning of the line with "" (nothing)
- .replaceAll(/^# 0\.0\.0\.0 /gmu, "# ") // Replace all occurances of "# 0.0.0.0 " at the beginning of the line with "# "
- .replace(/^# Title: (.*?)$/gmu, "# Title: $1 (NL)"); // Add (NL) to end of title
- await fs.writeFile(path.join(__dirname, "..", "alt-version", file.replace(".txt", "-nl.txt")), noIPFileContents, "utf8"); // Write new file to `alt-version` directory
- }));
+ const files = await fs.readdir(path.join(__dirname, ".."));
+ const txtFiles = files.filter((file) => file.endsWith(".txt"));
+
+ for (const file of txtFiles) {
+ const filePath = path.resolve(__dirname, "..", file);
+ const fileContents = await fs.readFile(filePath, "utf8");
+
+ const noIPFileContents = fileContents.replace(/^#? ?0\.0\.0\.0 ?/gm, "").replace(/^# Title: (.*?)$/gm, "# Title: $1 (NL)");
+
+ const newFileName = file.replace(".txt", "-nl.txt");
+ const newFilePath = path.resolve(__dirname, "..", "alt-version", newFileName);
+ await fs.writeFile(newFilePath, noIPFileContents, "utf8");
+ }
})();
const path = require("path");
(async () => {
- let hasError = false;
+ let hasError = false;
- const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
- await Promise.all(files.filter((file) => file !== "everything.txt").map(async (file) => { // For each file
- const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
+ const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt"));
- const commentedURLs = fileContents.split("\n").map((line) => {
- if (line.startsWith("# 0.0.0.0")) {
- return line.split(" ")[2].trim();
- }
+ await Promise.all(
+ files.filter((file) => file !== "everything.txt").map(async (file) => {
+ const filePath = path.join(__dirname, "..", file);
+ const fileContents = await fs.readFile(filePath, "utf8");
- return null;
- }).filter((a) => a !== null && !!a);
+ const lines = fileContents.split("\n");
+ const commentedURLs = [];
- let isHeaderComplete = false;
- fileContents.split("\n").forEach((line, index) => {
- if (line.startsWith("0.0.0.0")) {
- isHeaderComplete = true;
- }
+ let isHeaderComplete = false;
- // Ensuring that no version/date might confuse users that read the raw text-file(s)
- if (line.length > 0 && !line.indexOf("Version")) {
- console.error(`Line ${index + 1} in ${file} must not contain a Version/Date.`);
- hasError = true;
- }
+ lines.forEach((line, index) => {
+ if (line.startsWith("0.0.0.0")) {
+ isHeaderComplete = true;
+ }
- // Ensuring that all lines start with "#" or "0.0.0.0 "
- if (line.length > 0 && !line.startsWith("#") && !line.startsWith("0.0.0.0 ")) {
- console.error(`Line ${index + 1} in ${file} must start with "#" or "0.0.0.0 ".`);
- hasError = true;
- }
+ // Checking to ensure no version/date might confuse users that read the raw text-file(s)
+ if (line.length > 0 && line.includes("Version")) {
+ console.error(`Line ${index + 1} in ${file} must not contain a Version/Date.`);
+ hasError = true;
+ }
- // Checking to ensure all URLs are lowercase
- if (line.startsWith("0.0.0.0 ")) {
- const lineNoIP = line.replace("0.0.0.0 ", "");
- const url = lineNoIP.split("#")[0].trim();
- if (url.toLowerCase() !== url) {
- console.error(`Line ${index + 1} in ${file} url ${url} must be all lowercase.`);
- hasError = true;
- }
- }
+ // Ensuring that all lines start with "#" or "0.0.0.0 "
+ if (line.length > 0 && !line.startsWith("#") && !line.startsWith("0.0.0.0 ")) {
+ console.error(`Line ${index + 1} in ${file} must start with "#" or "0.0.0.0 ".`);
+ hasError = true;
+ }
- // Ensuring that all lines that start with `#` are followed by a space
- if (line.startsWith("#") && line.length > 1 && line[1] !== " ") {
- console.error(`Line ${index + 1} in ${file} should have a space after #.`);
- hasError = true;
- }
+ // Checking to ensure all URLs are lowercase
+ if (line.startsWith("0.0.0.0 ")) {
+ const url = line.split("#")[0].trim().replace("0.0.0.0 ", "");
+ if (url.toLowerCase() !== url) {
+ console.error(`Line ${index + 1} in ${file} url ${url} must be all lowercase.`);
+ hasError = true;
+ }
+ }
- // Ensure that after header is complete that all lines that start with `#` start with `# 0.0.0.0` or `# NOTE:`
- if (isHeaderComplete && line.startsWith("#") && !line.startsWith("# 0.0.0.0") && !line.startsWith("# NOTE:")) {
- console.error(`Line ${index + 1} in ${file} should start with "# 0.0.0.0" or "# NOTE:".`);
- hasError = true;
- }
+ // Ensuring that all lines that start with `#` are followed by a space
+ if (line.startsWith("#") && line.length > 1 && line[1] !== " ") {
+ console.error(`Line ${index + 1} in ${file} should have a space after #.`);
+ hasError = true;
+ }
- // Ensure that the URL doesn't exist in the commentedURLs array
- if (line.startsWith("0.0.0.0 ")) {
- const lineNoIP = line.replace("0.0.0.0 ", "");
- const url = lineNoIP.split("#")[0].trim();
- if (commentedURLs.includes(url)) {
- console.error(`Line ${index + 1} in ${file} url ${url} is commented out in this file. This suggests an error. Please either remove this line or remove the commented URL.`);
- hasError = true;
- }
- }
+ // Ensure that after the header is complete, all lines that start with `#` start with `# 0.0.0.0` or `# NOTE:`
+ if (isHeaderComplete && line.startsWith("#") && !line.startsWith("# 0.0.0.0") && !line.startsWith("# NOTE:")) {
+ console.error(`Line ${index + 1} in ${file} should start with "# 0.0.0.0" or "# NOTE:".`);
+ hasError = true;
+ }
- // Ensure that the URL doesn't contain whitespace characters
- if (line.startsWith("0.0.0.0 ")) {
- const lineNoIP = line.replace("0.0.0.0 ", "");
- const url = lineNoIP.split("#")[0].trim();
- if (/\s/gmu.test(url)) {
- console.error(`Line ${index + 1} in ${file} url ${url} contains whitespace in the URL.`);
- hasError = true;
- }
- }
- });
- }));
+ // Ensure that the URL doesn't exist in the commentedURLs array
+ if (line.startsWith("0.0.0.0 ")) {
+ const url = line.split("#")[0].trim().replace("0.0.0.0 ", "");
+ if (commentedURLs.includes(url)) {
+ console.error(`Line ${index + 1} in ${file} url ${url} is commented out in this file. This suggests an error. Please either remove this line or remove the commented URL.`);
+ hasError = true;
+ } else {
+ commentedURLs.push(url);
+ }
+ }
- process.exit(hasError ? 1 : 0);
+ // Ensure that the URL doesn't contain whitespace characters
+ if (line.startsWith("0.0.0.0 ")) {
+ const url = line.split("#")[0].trim().replace("0.0.0.0 ", "");
+ if (/\s/gmu.test(url)) {
+ console.error(`Line ${index + 1} in ${file} url ${url} contains whitespace in the URL.`);
+ hasError = true;
+ }
+ }
+ });
+ })
+ );
+
+ process.exit(hasError ? 1 : 0);
})();
const path = require("path");
(async () => {
- const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
+ const directory = path.join(__dirname, "..");
+ const files = (await fs.readdir(directory)).filter((file) => file.endsWith(".txt"));
- await Promise.all(files.map(async (file) => { // For each file
- const existingDomains = new Set();
+ await Promise.all(
+ files.map(async (file) => {
+ const existingDomains = new Set();
+ const filePath = path.join(directory, file);
- let fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
+ const lines = await fs.readFile(filePath, "utf8");
+ const updatedLines = lines
+ .split("\n")
+ .filter((line) => {
+ if (line.startsWith("0.0.0.0 ")) {
+ const domain = line.replace("0.0.0.0 ", "");
+ if (existingDomains.has(domain)) {
+ return false;
+ }
+ existingDomains.add(domain);
+ }
+ return true;
+ })
+ .join("\n");
- fileContents.split("\n").forEach((line) => {
- if (line.startsWith("0.0.0.0 ")) {
- const domain = line.replace("0.0.0.0 ", "");
- if (existingDomains.has(domain)) {
- fileContents = fileContents.replace(`${line}\n`, "");
- }
- existingDomains.add(domain);
- }
- });
-
- await fs.writeFile(path.join(__dirname, "..", file), fileContents, "utf8");
- }));
+ await fs.writeFile(filePath, updatedLines, "utf8");
+ })
+ );
})();
const path = require("path");
(async () => {
- const files = (await fs.readdir(path.join(__dirname, ".."))).filter((file) => file.endsWith(".txt")); // Array of strings, each representing a single file that ends in `.txt`
+ const files = (await fs.readdir(path.join(__dirname, ".."))).filter(file => file.endsWith(".txt"));
- await Promise.all(files.map(async (file) => { // For each file
- const existingDomains = new Set();
+ await Promise.all(files.map(async (file) => {
+ const filePath = path.join(__dirname, "..", file);
- const fileContents = await fs.readFile(path.join(__dirname, "..", file), "utf8"); // Get file contents as a string
+ // Read the file contents asynchronously
+ let fileContents = await fs.readFile(filePath, "utf8");
- fileContents.split("\n").forEach((line) => {
- if (line.startsWith("0.0.0.0 ")) {
- existingDomains.add(line.replace("0.0.0.0 ", ""));
- }
- });
+ // Count the number of network filters using a regex
+ const existingDomainsCount = (fileContents.match(/^0\.0\.0\.0 /gm) || []).length;
- await fs.writeFile(path.join(__dirname, "..", file), fileContents.replace(/^# Total number of network filters: ?(\d*)$/gmu, `# Total number of network filters: ${existingDomains.size}`), "utf8");
- }));
+ // Replace the total number of network filters in the fileContents
+ fileContents = fileContents.replace(/^# Total number of network filters: ?(\d*)$/gmu, `# Total number of network filters: ${existingDomainsCount}`);
+
+ // Write the updated file contents asynchronously
+ await fs.writeFile(filePath, fileContents, "utf8");
+ }));
})();
projects / thirdparty / blocklistproject / lists.git / commitdiff
