From: Peter van Dijk Date: Fri, 14 Feb 2025 12:11:04 +0000 (+0100) Subject: revert from rclone back to custom sync script X-Git-Tag: dnsdist-2.0.0-alpha1~88^2 X-Git-Url: http://git.ipfire.org/?a=commitdiff_plain;h=f35c8243bbc0740575851a575603bee3db818fcc;p=thirdparty%2Fpdns.git revert from rclone back to custom sync script --- diff --git a/.github/scripts/publish.js b/.github/scripts/publish.js new file mode 100644 index 0000000000..3012475014 --- /dev/null +++ b/.github/scripts/publish.js @@ -0,0 +1,96 @@ +const { S3Client, PutObjectCommand } = require("@aws-sdk/client-s3"); +const { CloudFrontClient, CreateInvalidationCommand } = require("@aws-sdk/client-cloudfront"); +const fs = require('fs'); +const path = require('path'); + +// Configure AWS SDK +const s3Client = new S3Client({ region: process.env.AWS_REGION }); +const cloudFrontClient = new CloudFrontClient({ region: process.env.AWS_REGION }); + +async function uploadToS3(bucketName, sourceDir, destDir = '') { + const files = fs.readdirSync(sourceDir); + + for (const file of files) { + const filePath = path.join(sourceDir, file); + const key = path.join(destDir, file); + + if (fs.statSync(filePath).isDirectory()) { + await uploadToS3(bucketName, filePath, key); + } else { + const fileContent = fs.readFileSync(filePath); + const command = new PutObjectCommand({ + Bucket: bucketName, + Key: key, + Body: fileContent, + ContentType: getContentType(file), + }); + await s3Client.send(command); + } + } +} + +function getContentType(filename) { + const ext = path.extname(filename).toLowerCase(); + switch (ext) { + case '.html': return 'text/html'; + case '.css': return 'text/css'; + case '.js': return 'application/javascript'; + case '.json': return 'application/json'; + case '.png': return 'image/png'; + case '.jpg': case '.jpeg': return 'image/jpeg'; + default: return 'application/octet-stream'; + } +} + +async function invalidateCloudFront(distributionId, paths) { + const command = new CreateInvalidationCommand({ + DistributionId: distributionId, + InvalidationBatch: { + CallerReference: Date.now().toString(), + Paths: { + Quantity: paths.length, + Items: paths, + }, + }, + }); + await cloudFrontClient.send(command); +} + +async function publishToSite(site, sourceDir, targetDir = '') { + const bucketName = process.env.AWS_S3_BUCKET_DOCS; + let distributionId, siteDir; + + if (site === 'dnsdist.org') { + distributionId = process.env.AWS_CLOUDFRONT_DISTRIBUTION_ID_DNSDIST; + siteDir = 'dnsdist.org'; + } else if (site === 'docs.powerdns.com') { + distributionId = process.env.AWS_CLOUDFRONT_DISTRIBUTION_ID_DOCS; + siteDir = 'docs.powerdns.com'; + } else { + throw new Error('Invalid site specified'); + } + + const fullTargetDir = path.join(siteDir, targetDir); + await uploadToS3(bucketName, sourceDir, fullTargetDir); + + // Invalidate CloudFront cache + await invalidateCloudFront(distributionId, ['/*']); + + console.log(`Published from ${sourceDir} to ${site}${targetDir ? '/' + targetDir : ''}`); +} + +async function main() { + const args = process.argv.slice(2); + if (args[0] === 'publish') { + if (args.length < 3 || args.length > 4) { + console.log('Usage: node publish.js publish [TARGET_DIR]'); + return; + } + const [, site, sourceDir, targetDir] = args; + await publishToSite(site, sourceDir, targetDir); + } else { + console.log('Usage: node publish.js publish [TARGET_DIR]'); + } +} + +main().catch(console.error); diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index ec78569d95..163d9d5a11 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -265,35 +265,20 @@ jobs: exit 1 fi - uses: actions/checkout@v4 - # Configure AWS Credentials - - name: Configure AWS credentials - uses: aws-actions/configure-aws-credentials@v4 + - uses: actions/setup-node@v4 with: - aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} - aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - aws-region: ${{ vars.AWS_REGION }} - # Install rclone - - name: Install rclone - run: | - sudo apt-get update - sudo apt-get install -y rclone - - name: Configure rclone - run: | - mkdir -p ~/.config/rclone - cat > ~/.config/rclone/rclone.conf << EOF - [docs-s3] - type = s3 - provider = AWS - env_auth = true - region = ${{ vars.AWS_REGION }} - location_constraint = ${{ vars.AWS_REGION }} - EOF + node-version: '20' + - name: Install AWS SDK v3 + run: npm install @aws-sdk/client-s3 @aws-sdk/client-cloudfront - name: Download artifacts uses: actions/download-artifact@v4 with: path: artifacts - name: Publish to AWS env: + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_REGION: ${{ vars.AWS_REGION }} AWS_S3_BUCKET_DOCS: ${{ vars.AWS_S3_BUCKET_DOCS }} AWS_CLOUDFRONT_DISTRIBUTION_ID_DNSDIST: ${{ vars.AWS_CLOUDFRONT_DISTRIBUTION_ID_DNSDIST }} AWS_CLOUDFRONT_DISTRIBUTION_ID_DOCS: ${{ vars.AWS_CLOUDFRONT_DISTRIBUTION_ID_DOCS }} @@ -301,31 +286,15 @@ jobs: if [ -n "$AWS_CLOUDFRONT_DISTRIBUTION_ID_DNSDIST" ]; then tar -xf artifacts/dnsdist-html-docs-${{needs.build-docs.outputs.pdns_version}}/dnsdist-html-docs.tar cp artifacts/dnsdist-${{needs.build-docs.outputs.pdns_version}}.pdf/dnsdist.pdf dnsdist-html-docs/ - rclone sync dnsdist-html-docs/ docs-s3:$AWS_S3_BUCKET_DOCS/dnsdist.org/ - # Invalidate CloudFront cache for DNSdist - aws cloudfront create-invalidation \ - --distribution-id $AWS_CLOUDFRONT_DISTRIBUTION_ID_DNSDIST \ - --paths "/*" + node .github/scripts/publish.js publish dnsdist.org dnsdist-html-docs / fi if [ -n "$AWS_CLOUDFRONT_DISTRIBUTION_ID_DOCS" ]; then tar -xf artifacts/recursor-html-docs-${{needs.build-docs.outputs.pdns_version}}/rec-html-docs.tar cp artifacts/PowerDNS-Recursor-${{needs.build-docs.outputs.pdns_version}}.pdf/PowerDNS-Recursor.pdf rec-html-docs/ - rclone sync rec-html-docs/ docs-s3:$AWS_S3_BUCKET_DOCS/docs.powerdns.com/recursor/ - + node .github/scripts/publish.js publish docs.powerdns.com rec-html-docs /recursor tar -xf artifacts/authoritative-html-docs-${{needs.build-docs.outputs.pdns_version}}/auth-html-docs.tar cp artifacts/PowerDNS-Authoritative-${{needs.build-docs.outputs.pdns_version}}.pdf/PowerDNS-Authoritative.pdf auth-html-docs/ - rclone sync auth-html-docs/ docs-s3:$AWS_S3_BUCKET_DOCS/docs.powerdns.com/authoritative/ - + node .github/scripts/publish.js publish docs.powerdns.com auth-html-docs /authoritative tar -xf artifacts/website-${{needs.build-docs.outputs.pdns_version}}/website.tar - rclone sync website/docs.powerdns.com/ docs-s3:$AWS_S3_BUCKET_DOCS/docs.powerdns.com/ - - # Invalidate CloudFront cache for docs.powerdns.com - aws cloudfront create-invalidation \ - --distribution-id $AWS_CLOUDFRONT_DISTRIBUTION_ID_DOCS \ - --paths "/*" + node .github/scripts/publish.js publish docs.powerdns.com website/docs.powerdns.com / fi - # Cleanup credentials - - name: Cleanup rclone config - if: always() - run: | - rm -f ~/.config/rclone/rclone.conf