+++ /dev/null
-name: Build & Release Aggregated Blocklists
-
-on:
- schedule:
- - cron: '0 0 * * *' # every night at midnight UTC
- workflow_dispatch: # manual trigger
-
-jobs:
- build:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v3
-
- - name: Set up Python
- uses: actions/setup-python@v4
- with:
- python-version: '3.11'
-
- - name: Generate aggregated lists
- run: python3 scripts/aggregate.py
-
- - name: Verify output files exist
- run: |
- mkdir -p releases
- if [ ! -f releases/aggregated-hosts.txt ] || [ ! -f releases/aggregated-dnsmasq.conf ] || [ ! -f releases/aggregated-adblock.txt ]; then
- echo "Error: One or more output files are missing!"
- exit 1
- fi
-
- - name: Commit outputs
- run: |
- git config user.name "github-actions[bot]"
- git config user.email "github-actions[bot]@users.noreply.github.com"
- git add releases/
- # only commit if there are changes
- git diff --quiet && echo "No changes to commit" || git commit -m "chore: update aggregated lists"
- git push
-
- release:
- needs: build
- runs-on: ubuntu-latest
- steps:
- - name: Checkout code
- uses: actions/checkout@v3
- with:
- fetch-depth: 0 # so tags and history are available
- persist-credentials: true # so we can push
-
- - name: Set release date
- run: echo "RELEASE_DATE=$(date +'%Y%m%d')" >> $GITHUB_ENV
-
- - name: Check if tag exists
- id: tag_check
- run: |
- if git rev-parse "aggregated-${{ env.RELEASE_DATE }}" >/dev/null 2>&1; then
- echo "Tag aggregated-${{ env.RELEASE_DATE }} already exists"
- echo "tag_exists=true" >> $GITHUB_OUTPUT
- else
- echo "Tag does not exist, will create it"
- echo "tag_exists=false" >> $GITHUB_OUTPUT
- fi
-
- - name: Create Git tag
- if: steps.tag_check.outputs.tag_exists == 'false'
- run: |
- git config user.name "github-actions[bot]"
- git config user.email "github-actions[bot]@users.noreply.github.com"
- git tag aggregated-${{ env.RELEASE_DATE }}
- git push origin aggregated-${{ env.RELEASE_DATE }}
-
- - name: Check if release exists
- id: release_check
- run: |
- if gh release view aggregated-${{ env.RELEASE_DATE }} >/dev/null 2>&1; then
- echo "Release aggregated-${{ env.RELEASE_DATE }} already exists"
- echo "release_exists=true" >> $GITHUB_OUTPUT
- else
- echo "Release does not exist, will create it"
- echo "release_exists=false" >> $GITHUB_OUTPUT
- fi
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
- - name: Create GitHub Release
- id: create_release
- if: steps.release_check.outputs.release_exists == 'false'
- uses: actions/create-release@v1
- with:
- tag_name: aggregated-${{ env.RELEASE_DATE }}
- release_name: Aggregated Lists ${{ env.RELEASE_DATE }}
- body: |
- This release contains the aggregated hosts, dnsmasq, and adblock blocklists for ${{ env.RELEASE_DATE }}.
- draft: false
- prerelease: false
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
- - name: Upload release assets
- if: steps.release_check.outputs.release_exists == 'false'
- uses: softprops/action-gh-release@v1
- with:
- files: |
- releases/aggregated-hosts.txt
- releases/aggregated-dnsmasq.conf
- releases/aggregated-adblock.txt
- tag_name: aggregated-${{ env.RELEASE_DATE }}
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
\ No newline at end of file
+++ /dev/null
-name: Build Blocklists
-
-on:
- push:
- branches: [master]
- paths:
- - '*.txt'
- - 'config/**'
- - 'src/**'
- - 'build.py'
- - '.github/workflows/build.yml'
- pull_request:
- branches: [master]
- paths:
- - '*.txt'
- - 'config/**'
- - 'src/**'
- - 'build.py'
- workflow_dispatch:
- inputs:
- lists:
- description: 'Specific lists to build (comma-separated, empty for all)'
- required: false
- default: ''
- validate:
- description: 'Run validation checks'
- required: false
- type: boolean
- default: true
-
-permissions:
- contents: write
- pull-requests: write
-
-jobs:
- test:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
-
- - name: Set up Python
- uses: actions/setup-python@v5
- with:
- python-version: '3.13'
- cache: 'pip'
-
- - name: Install dependencies
- run: |
- python -m pip install --upgrade pip
- pip install -e ".[dev]"
-
- - name: Run tests
- run: pytest -v --tb=short
-
- - name: Run linting
- run: |
- pip install ruff
- ruff check src/ tests/ build.py
-
- build:
- needs: test
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
-
- - name: Set up Python
- uses: actions/setup-python@v5
- with:
- python-version: '3.13'
- cache: 'pip'
-
- - name: Install dependencies
- run: |
- python -m pip install --upgrade pip
- pip install -e .
-
- - name: Build lists (PR - dry run)
- if: github.event_name == 'pull_request'
- run: |
- python build.py --dry-run --validate --verbose
- echo "## Build Preview" >> $GITHUB_STEP_SUMMARY
- echo "Dry run completed successfully. No files were modified." >> $GITHUB_STEP_SUMMARY
-
- - name: Build lists (push to master)
- if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
- run: |
- ARGS="--validate --verbose"
- if [ -n "${{ github.event.inputs.lists }}" ]; then
- for list in $(echo "${{ github.event.inputs.lists }}" | tr ',' ' '); do
- ARGS="$ARGS --list $list"
- done
- fi
- python build.py $ARGS
-
- - name: Verify output consistency
- if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
- run: python build.py verify
-
- - name: Show statistics
- run: |
- echo "## Blocklist Statistics" >> $GITHUB_STEP_SUMMARY
- python build.py stats >> $GITHUB_STEP_SUMMARY
-
- - name: Check for changes
- if: github.event_name == 'push' || github.event_name == 'workflow_dispatch'
- id: changes
- run: |
- if git diff --quiet; then
- echo "has_changes=false" >> $GITHUB_OUTPUT
- else
- echo "has_changes=true" >> $GITHUB_OUTPUT
- fi
-
- - name: Commit and push changes
- if: (github.event_name == 'push' || github.event_name == 'workflow_dispatch') && steps.changes.outputs.has_changes == 'true'
- run: |
- git config user.name "github-actions[bot]"
- git config user.email "github-actions[bot]@users.noreply.github.com"
- git add -A
- git commit -m "chore: regenerate blocklists [skip ci]"
- git push
-
- validate-pr:
- if: github.event_name == 'pull_request'
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
- with:
- fetch-depth: 0
-
- - name: Set up Python
- uses: actions/setup-python@v5
- with:
- python-version: '3.13'
- cache: 'pip'
-
- - name: Install dependencies
- run: |
- python -m pip install --upgrade pip
- pip install -e .
-
- - name: Get changed files
- id: changed
- run: |
- CHANGED=$(git diff --name-only origin/${{ github.base_ref }}...HEAD -- '*.txt' | tr '\n' ' ')
- echo "files=$CHANGED" >> $GITHUB_OUTPUT
-
- - name: Validate changed lists
- if: steps.changed.outputs.files != ''
- run: |
- echo "Validating changed files: ${{ steps.changed.outputs.files }}"
- python -c "
- from pathlib import Path
- from src.normalize import parse_file_to_set
- from src.validate import validate_domain_set
-
- files = '${{ steps.changed.outputs.files }}'.split()
- errors = []
- for f in files:
- if not f.endswith('.txt'):
- continue
- path = Path(f)
- if not path.exists():
- continue
- domains = parse_file_to_set(path)
- valid, errs = validate_domain_set(domains)
- if errs:
- errors.extend([f'{f}: {e}' for e in errs[:5]])
-
- if errors:
- print('Validation errors found:')
- for e in errors:
- print(f' - {e}')
- exit(1)
- print('All changed files validated successfully!')
- "
-
- - name: Comment on PR
- uses: actions/github-script@v7
- with:
- script: |
- const { data: comments } = await github.rest.issues.listComments({
- owner: context.repo.owner,
- repo: context.repo.repo,
- issue_number: context.issue.number,
- });
-
- const botComment = comments.find(comment =>
- comment.user.type === 'Bot' &&
- comment.body.includes('Blocklist Validation')
- );
-
- const body = `## ✅ Blocklist Validation Passed
-
- All domain entries in this PR have been validated:
- - ✓ Domain syntax check
- - ✓ TLD validation
- - ✓ Critical domain protection
-
- The lists will be rebuilt when this PR is merged.`;
-
- if (botComment) {
- await github.rest.issues.updateComment({
- owner: context.repo.owner,
- repo: context.repo.repo,
- comment_id: botComment.id,
- body
- });
- } else {
- await github.rest.issues.createComment({
- owner: context.repo.owner,
- repo: context.repo.repo,
- issue_number: context.issue.number,
- body
- });
- }
+++ /dev/null
-name: CI
-on:
- push:
- branches:
- - master
-
-jobs:
- generate:
- runs-on: ubuntu-latest
- steps:
- - run: git config --global --add safe.directory /github/workspace
- - uses: actions/checkout@v4
- - name: Use Node.js
- uses: actions/setup-node@v4
- with:
- node-version: 18.x
- - run: node scripts/create-everything-list.js
- - run: node scripts/remove-duplicates.js
- - run: node scripts/update-number-of-domains.js
- - run: node scripts/generate-noip.js
- - run: node scripts/generate-dnsmasq.js
- - run: node scripts/generate-adguard.js
- - name: Commit & Push
- uses: actions-x/commit@v6
- with:
- email: noreply@blocklist.site
- name: GitHub Actions
- branch: master
+++ /dev/null
-name: Triage Issues
-
-on:
- issues:
- types: [opened, edited]
-
-permissions:
- issues: write
- contents: read
-
-jobs:
- triage:
- runs-on: ubuntu-latest
- steps:
- - uses: actions/checkout@v4
-
- - name: Set up Python
- uses: actions/setup-python@v5
- with:
- python-version: '3.13'
- cache: 'pip'
-
- - name: Install dependencies
- run: |
- python -m pip install --upgrade pip
- pip install -e .
-
- - name: Extract domain from issue
- id: extract
- uses: actions/github-script@v7
- with:
- script: |
- const body = context.payload.issue.body || '';
- const title = context.payload.issue.title || '';
-
- // Try to extract domain from form field or free text
- const domainRegex = /(?:Domain to (?:add|remove)|URL)[:\s]*([a-zA-Z0-9][-a-zA-Z0-9]*(?:\.[a-zA-Z0-9][-a-zA-Z0-9]*)+)/i;
- const match = body.match(domainRegex) || title.match(/\[(?:Add|Remove)\]\s*(.+)/i);
-
- if (match) {
- const domain = match[1].trim().toLowerCase();
- core.setOutput('domain', domain);
- core.setOutput('found', 'true');
- } else {
- core.setOutput('found', 'false');
- }
-
- - name: Check domain in lists
- if: steps.extract.outputs.found == 'true'
- id: check
- run: |
- DOMAIN="${{ steps.extract.outputs.domain }}"
- echo "Checking for domain: $DOMAIN"
-
- # Search for the domain in all .txt files
- FOUND_IN=""
- for file in *.txt; do
- if [ -f "$file" ] && grep -qi "^0\.0\.0\.0 $DOMAIN$\|^$DOMAIN$" "$file" 2>/dev/null; then
- LIST_NAME=$(basename "$file" .txt)
- if [ -z "$FOUND_IN" ]; then
- FOUND_IN="$LIST_NAME"
- else
- FOUND_IN="$FOUND_IN, $LIST_NAME"
- fi
- fi
- done
-
- if [ -n "$FOUND_IN" ]; then
- echo "found_in=$FOUND_IN" >> $GITHUB_OUTPUT
- echo "is_present=true" >> $GITHUB_OUTPUT
- else
- echo "is_present=false" >> $GITHUB_OUTPUT
- fi
-
- - name: Add labels and comment
- uses: actions/github-script@v7
- with:
- script: |
- const domain = '${{ steps.extract.outputs.domain }}';
- const isPresent = '${{ steps.check.outputs.is_present }}' === 'true';
- const foundIn = '${{ steps.check.outputs.found_in }}';
- const isAddRequest = context.payload.issue.title.toLowerCase().includes('[add]');
- const isRemoveRequest = context.payload.issue.title.toLowerCase().includes('[remove]');
-
- let labels = [];
- let comment = '';
-
- if (isAddRequest) {
- if (isPresent) {
- labels.push('duplicate');
- comment = `## 🔍 Domain Check Result
-
- The domain \`${domain}\` is **already present** in the following list(s):
- - ${foundIn}
-
- This request may be a duplicate. Please verify the domain is not already blocked before proceeding.
-
- If you believe this is a different domain or subdomain, please clarify in a comment.`;
- } else {
- labels.push('verified-new');
- comment = `## ✅ Domain Check Result
-
- The domain \`${domain}\` is **not currently** in any blocklist.
-
- This request is ready for maintainer review.`;
- }
- } else if (isRemoveRequest) {
- if (isPresent) {
- labels.push('verified-exists');
- comment = `## 🔍 Domain Check Result
-
- The domain \`${domain}\` was **found** in the following list(s):
- - ${foundIn}
-
- This removal request is ready for maintainer review.`;
- } else {
- labels.push('not-found');
- comment = `## ⚠️ Domain Check Result
-
- The domain \`${domain}\` was **not found** in any blocklist.
-
- This may be because:
- - The domain was already removed
- - The domain spelling is different from what's in the list
- - A parent domain is blocked instead
-
- Please verify the exact domain that's causing the block.`;
- }
- }
-
- // Remove triage label and add new labels
- try {
- await github.rest.issues.removeLabel({
- owner: context.repo.owner,
- repo: context.repo.repo,
- issue_number: context.issue.number,
- name: 'triage'
- });
- } catch (e) {
- // Label might not exist, that's fine
- }
-
- if (labels.length > 0) {
- await github.rest.issues.addLabels({
- owner: context.repo.owner,
- repo: context.repo.repo,
- issue_number: context.issue.number,
- labels: labels
- });
- }
-
- if (comment) {
- await github.rest.issues.createComment({
- owner: context.repo.owner,
- repo: context.repo.repo,
- issue_number: context.issue.number,
- body: comment
- });
- }
projects / thirdparty / blocklistproject / lists.git / commitdiff
