name: documentation
path: docs/_build/html/
- code-checks-backend:
- name: "Backend Code Checks"
+ ci-backend:
+ uses: ./.github/workflows/reusable-ci-backend.yml
+
+ ci-frontend:
+ uses: ./.github/workflows/reusable-ci-frontend.yml
+
+ prepare-docker-build:
+ name: Prepare Docker Pipeline Data
+ if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || startsWith(github.ref, 'refs/tags/ngx-') || startsWith(github.ref, 'refs/tags/beta-'))
runs-on: ubuntu-20.04
+ needs:
+ - documentation
+ - ci-backend
+ - ci-frontend
steps:
-
name: Checkout
uses: actions/checkout@v3
-
- name: Install checkers
- run: |
- pipx install reorder-python-imports
- pipx install yesqa
- pipx install add-trailing-comma
- pipx install flake8
+ name: Get branch name
+ id: branch-name
+ uses: tj-actions/branch-names@v5
-
- name: Run reorder-python-imports
- run: |
- find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs reorder-python-imports
+ name: Login to Github Container Registry
+ uses: docker/login-action@v1
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
-
- name: Run yesqa
- run: |
- find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs yesqa
+ name: Set up Python
+ uses: actions/setup-python@v3
+ with:
+ python-version: "3.9"
-
- name: Run add-trailing-comma
+ name: Make script executable
run: |
- find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs add-trailing-comma
- # black is placed after add-trailing-comma because it may format differently
- # if a trailing comma is added
+ chmod +x ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py
-
- name: Run black
- uses: psf/black@stable
- with:
- options: "--check --diff"
- version: "22.3.0"
+ name: Setup qpdf image
+ id: qpdf-setup
+ run: |
+ build_json=$(python ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py qpdf)
+
+ echo ${build_json}
+
+ echo ::set-output name=qpdf-json::${build_json}
-
- name: Run flake8 checks
+ name: Setup psycopg2 image
+ id: psycopg2-setup
run: |
- cd src/
- flake8 --max-line-length=88 --ignore=E203,W503
+ build_json=$(python ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py psycopg2)
- code-checks-frontend:
- name: "Frontend Code Checks"
- runs-on: ubuntu-20.04
- steps:
+ echo ${build_json}
+
+ echo ::set-output name=psycopg2-json::${build_json}
-
- name: Checkout
- uses: actions/checkout@v3
- - uses: actions/setup-node@v3
- with:
- node-version: '16'
+ name: Setup pikepdf image
+ id: pikepdf-setup
+ run: |
+ build_json=$(python ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py pikepdf)
+
+ echo ${build_json}
+
+ echo ::set-output name=pikepdf-json::${build_json}
-
- name: Install prettier
+ name: Setup jbig2enc image
+ id: jbig2enc-setup
run: |
- npm install prettier
+ build_json=$(python ${GITHUB_WORKSPACE}/docker-builders/get-build-json.py jbig2enc)
+
+ echo ${build_json}
+
+ echo ::set-output name=jbig2enc-json::${build_json}
-
- name: Run prettier
- run:
- npx prettier --check --ignore-path Pipfile.lock **/*.js **/*.ts *.md **/*.md
+ name: Setup frontend image
+ id: frontend-setup
+ run: |
+ frontend_image=ghcr.io/${{ github.repository }}/ngx-frontend:${{ steps.branch-name.outputs.current_branch }}
- tests-backend:
- needs: [code-checks-backend]
- name: "Backend Tests (${{ matrix.python-version }})"
- runs-on: ubuntu-20.04
- strategy:
- matrix:
- python-version: ['3.8', '3.9', '3.10']
- fail-fast: false
+ echo ${frontend_image}
+
+ echo ::set-output name=frontend-image-tag::${frontend_image}
+
+ outputs:
+
+ frontend-image-tag: ${{ steps.frontend-setup.outputs.frontend-image-tag }}
+
+ qpdf-json: ${{ steps.qpdf-setup.outputs.qpdf-json }}
+
+ pikepdf-json: ${{ steps.pikepdf-setup.outputs.pikepdf-json }}
+
+ psycopg2-json: ${{ steps.psycopg2-setup.outputs.psycopg2-json }}
+
+ jbig2enc-json: ${{ steps.jbig2enc-setup.outputs.jbig2enc-json}}
+
+ build-qpdf-debs:
+ name: qpdf
+ needs:
+ - prepare-docker-build
+ uses: ./.github/workflows/reusable-workflow-builder.yml
+ with:
+ dockerfile: ./docker-builders/Dockerfile.qpdf
+ build-json: ${{ needs.prepare-docker-build.outputs.qpdf-json }}
+ build-args: |
+ QPDF_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).version }}
+
+ build-jbig2enc:
+ name: jbig2enc
+ needs:
+ - prepare-docker-build
+ uses: ./.github/workflows/reusable-workflow-builder.yml
+ with:
+ dockerfile: ./docker-builders/Dockerfile.jbig2enc
+ build-json: ${{ needs.prepare-docker-build.outputs.jbig2enc-json }}
+ build-args: |
+ JBIG2ENC_VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).version }}
+
+ build-psycopg2-wheel:
+ name: psycopg2
+ needs:
+ - prepare-docker-build
+ uses: ./.github/workflows/reusable-workflow-builder.yml
+ with:
+ dockerfile: ./docker-builders/Dockerfile.psycopg2
+ build-json: ${{ needs.prepare-docker-build.outputs.psycopg2-json }}
+ build-args: |
+ GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).git_tag }}
+ VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).version }}
+
+ build-pikepdf-wheel:
+ name: pikepdf
+ needs:
+ - prepare-docker-build
+ - build-qpdf-debs
+ uses: ./.github/workflows/reusable-workflow-builder.yml
+ with:
+ dockerfile: ./docker-builders/Dockerfile.pikepdf
+ build-json: ${{ needs.prepare-docker-build.outputs.pikepdf-json }}
+ build-args: |
+ QPDF_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).image_tag }}
+ GIT_TAG=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).git_tag }}
+ VERSION=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).version }}
+
+ build-frontend:
+ name: Compile frontend
+ concurrency:
+ group: ${{ github.workflow }}-build-frontend-${{ github.ref }}
+ cancel-in-progress: false
+ needs:
+ - prepare-docker-build
+ runs-on: ubuntu-latest
steps:
-
name: Checkout
with:
fetch-depth: 2
-
- name: Install pipenv
- run: pipx install pipenv
+ name: Get changed frontend files
+ id: changed-files-specific
+ uses: tj-actions/changed-files@v18.1
+ with:
+ files: |
+ src-ui/**
-
- name: Set up Python
- uses: actions/setup-python@v3
+ name: Login to Github Container Registry
+ uses: docker/login-action@v1
with:
- python-version: "${{ matrix.python-version }}"
- cache: "pipenv"
- cache-dependency-path: 'Pipfile.lock'
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
-
- name: Install system dependencies
+ name: Determine if build needed
+ id: build-skip-check
+ # Skip building the frontend if the tag exists and no src-ui files changed
run: |
- sudo apt-get update -qq
- sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript optipng libzbar0 poppler-utils
+ if ! docker manifest inspect ${{ needs.prepare-docker-build.outputs.frontend-image-tag }} &> /dev/null ; then
+ echo "Build required, no existing image"
+ echo ::set-output name=frontend-build-needed::true
+ elif ${{ steps.changed-files-specific.outputs.any_changed }} == 'true' ; then
+ echo "Build required, src-ui changes"
+ echo ::set-output name=frontend-build-needed::true
+ else
+ echo "No build required"
+ echo ::set-output name=frontend-build-needed::false
+ fi
-
- name: Install Python dependencies
- run: |
- pipenv sync --dev
+ name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v1
+ if: ${{ steps.build-skip-check.outputs.frontend-build-needed == 'true' }}
-
- name: Tests
- run: |
- cd src/
- pipenv run pytest
+ name: Set up QEMU
+ uses: docker/setup-qemu-action@v1
+ if: ${{ steps.build-skip-check.outputs.frontend-build-needed == 'true' }}
-
- name: Get changed files
- id: changed-files-specific
- uses: tj-actions/changed-files@v18.1
+ name: Compile frontend
+ uses: docker/build-push-action@v2
+ if: ${{ steps.build-skip-check.outputs.frontend-build-needed == 'true' }}
with:
- files: |
- src/**
+ context: .
+ file: ./docker-builders/Dockerfile.frontend
+ tags: ${{ needs.prepare-docker-build.outputs.frontend-image-tag }}
+ platforms: linux/amd64,linux/arm64,linux/arm/v7
+ push: true
+ cache-from: type=gha
+ cache-to: type=gha,mode=max
-
- name: List all changed files
+ name: Export frontend artifact from docker
run: |
- for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
- echo "${file} was changed"
- done
+ docker create --name frontend-extract ${{ needs.prepare-docker-build.outputs.frontend-image-tag }}
+ docker cp frontend-extract:/src/src/documents/static/frontend src/documents/static/frontend/
-
- name: Publish coverage results
- if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
- env:
- GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
- # https://github.com/coveralls-clients/coveralls-python/issues/251
- run: |
- cd src/
- pipenv run coveralls --service=github
-
- tests-frontend:
- needs: [code-checks-frontend]
- name: "Frontend Tests"
- runs-on: ubuntu-20.04
- strategy:
- matrix:
- node-version: [16.x]
- steps:
- - uses: actions/checkout@v3
- - name: Use Node.js ${{ matrix.node-version }}
- uses: actions/setup-node@v3
+ name: Upload frontend artifact
+ uses: actions/upload-artifact@v3
with:
- node-version: ${{ matrix.node-version }}
- - run: cd src-ui && npm ci
- - run: cd src-ui && npm run test
- - run: cd src-ui && npm run e2e:ci
+ name: frontend-compiled
+ path: src/documents/static/frontend/
# build and push image to docker hub.
build-docker-image:
- if: github.event_name == 'push' && (startsWith(github.ref, 'refs/heads/feature-') || github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/beta' || startsWith(github.ref, 'refs/tags/ngx-') || startsWith(github.ref, 'refs/tags/beta-'))
concurrency:
group: ${{ github.workflow }}-build-docker-image-${{ github.ref }}
cancel-in-progress: true
runs-on: ubuntu-20.04
- needs: [tests-backend, tests-frontend]
+ concurrency:
+ group: ${{ github.workflow }}-build-docker-image-${{ github.ref }}
+ cancel-in-progress: true
+ needs:
+ - prepare-docker-build
+ - build-psycopg2-wheel
+ - build-jbig2enc
+ - build-qpdf-debs
+ - build-pikepdf-wheel
+ - build-frontend
steps:
-
name: Gather Docker metadata
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.docker-meta.outputs.tags }}
labels: ${{ steps.docker-meta.outputs.labels }}
+ build-args: |
+ JBIG2ENC_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.jbig2enc-json).image_tag }}
+ QPDF_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.qpdf-json).image_tag }}
+ PIKEPDF_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.pikepdf-json).image_tag }}
+ PSYCOPG2_BASE_IMAGE=${{ fromJSON(needs.prepare-docker-build.outputs.psycopg2-json).image_tag }}
+ FRONTEND_BASE_IMAGE=${{ needs.prepare-docker-build.outputs.frontend-image-tag }}
cache-from: type=gha
cache-to: type=gha,mode=max
-
name: Inspect image
run: |
docker buildx imagetools inspect ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
- -
- name: Export frontend artifact from docker
- run: |
- docker create --name frontend-extract ${{ fromJSON(steps.docker-meta.outputs.json).tags[0] }}
- docker cp frontend-extract:/usr/src/paperless/src/documents/static/frontend src/documents/static/frontend/
- -
- name: Upload frontend artifact
- uses: actions/upload-artifact@v3
- with:
- name: frontend-compiled
- path: src/documents/static/frontend/
build-release:
- needs: [build-docker-image, documentation]
+ needs:
+ - build-docker-image
runs-on: ubuntu-20.04
steps:
-
publish-release:
runs-on: ubuntu-20.04
- needs: build-release
+ needs:
+ - build-release
if: contains(github.ref, 'refs/tags/ngx-') || contains(github.ref, 'refs/tags/beta-')
steps:
-
--- /dev/null
+name: Backend CI Jobs
+
+on:
+ workflow_call:
+
+jobs:
+
+ code-checks-backend:
+ name: "Code Style Checks"
+ runs-on: ubuntu-20.04
+ steps:
+ -
+ name: Checkout
+ uses: actions/checkout@v3
+ -
+ name: Install checkers
+ run: |
+ pipx install reorder-python-imports
+ pipx install yesqa
+ pipx install add-trailing-comma
+ pipx install flake8
+ -
+ name: Run reorder-python-imports
+ run: |
+ find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs reorder-python-imports
+ -
+ name: Run yesqa
+ run: |
+ find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs yesqa
+ -
+ name: Run add-trailing-comma
+ run: |
+ find src/ -type f -name '*.py' ! -path "*/migrations/*" | xargs add-trailing-comma
+ # black is placed after add-trailing-comma because it may format differently
+ # if a trailing comma is added
+ -
+ name: Run black
+ uses: psf/black@stable
+ with:
+ options: "--check --diff"
+ version: "22.3.0"
+ -
+ name: Run flake8 checks
+ run: |
+ cd src/
+ flake8 --max-line-length=88 --ignore=E203,W503
+
+ tests-backend:
+ name: "Tests (${{ matrix.python-version }})"
+ runs-on: ubuntu-20.04
+ needs:
+ - code-checks-backend
+ strategy:
+ matrix:
+ python-version: ['3.8', '3.9', '3.10']
+ fail-fast: false
+ steps:
+ -
+ name: Checkout
+ uses: actions/checkout@v3
+ with:
+ fetch-depth: 2
+ -
+ name: Install pipenv
+ run: pipx install pipenv
+ -
+ name: Set up Python
+ uses: actions/setup-python@v3
+ with:
+ python-version: "${{ matrix.python-version }}"
+ cache: "pipenv"
+ cache-dependency-path: 'Pipfile.lock'
+ -
+ name: Install system dependencies
+ run: |
+ sudo apt-get update -qq
+ sudo apt-get install -qq --no-install-recommends unpaper tesseract-ocr imagemagick ghostscript optipng libzbar0 poppler-utils
+ -
+ name: Install Python dependencies
+ run: |
+ pipenv sync --dev
+ -
+ name: Tests
+ run: |
+ cd src/
+ pipenv run pytest
+ -
+ name: Get changed files
+ id: changed-files-specific
+ uses: tj-actions/changed-files@v18.1
+ with:
+ files: |
+ src/**
+ -
+ name: List all changed files
+ run: |
+ for file in ${{ steps.changed-files-specific.outputs.all_changed_files }}; do
+ echo "${file} was changed"
+ done
+ -
+ name: Publish coverage results
+ if: matrix.python-version == '3.9' && steps.changed-files-specific.outputs.any_changed == 'true'
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ # https://github.com/coveralls-clients/coveralls-python/issues/251
+ run: |
+ cd src/
+ pipenv run coveralls --service=github
--- /dev/null
+name: Frontend CI Jobs
+
+on:
+ workflow_call:
+
+jobs:
+
+ code-checks-frontend:
+ name: "Code Style Checks"
+ runs-on: ubuntu-20.04
+ steps:
+ -
+ name: Checkout
+ uses: actions/checkout@v3
+ - uses: actions/setup-node@v3
+ with:
+ node-version: '16'
+ -
+ name: Install prettier
+ run: |
+ npm install prettier
+ -
+ name: Run prettier
+ run:
+ npx prettier --check --ignore-path Pipfile.lock **/*.js **/*.ts *.md **/*.md
+ tests-frontend:
+ name: "Tests"
+ runs-on: ubuntu-20.04
+ needs:
+ - code-checks-frontend
+ strategy:
+ matrix:
+ node-version: [16.x]
+ steps:
+ - uses: actions/checkout@v3
+ - name: Use Node.js ${{ matrix.node-version }}
+ uses: actions/setup-node@v3
+ with:
+ node-version: ${{ matrix.node-version }}
+ - run: cd src-ui && npm ci
+ - run: cd src-ui && npm run test
+ - run: cd src-ui && npm run e2e:ci
--- /dev/null
+name: Reusable Image Builder
+
+on:
+ workflow_call:
+ inputs:
+ dockerfile:
+ required: true
+ type: string
+ build-json:
+ required: true
+ type: string
+ build-args:
+ required: false
+ default: ""
+ type: string
+
+concurrency:
+ group: ${{ github.workflow }}-${{ fromJSON(inputs.build-json).name }}-${{ fromJSON(inputs.build-json).version }}
+ cancel-in-progress: false
+
+jobs:
+ build-image:
+ name: Build ${{ fromJSON(inputs.build-json).name }} @ ${{ fromJSON(inputs.build-json).version }}
+ runs-on: ubuntu-latest
+ steps:
+ -
+ name: Login to Github Container Registry
+ uses: docker/login-action@v1
+ with:
+ registry: ghcr.io
+ username: ${{ github.actor }}
+ password: ${{ secrets.GITHUB_TOKEN }}
+ -
+ name: Determine if build needed
+ id: build-skip-check
+ run: |
+ if ! docker manifest inspect ${{ fromJSON(inputs.build-json).image_tag }} &> /dev/null ; then
+ echo "Building, no image exists with this version"
+ echo ::set-output name=image-exists::false
+ else
+ echo "Not building, image exists with this version"
+ echo ::set-output name=image-exists::true
+ fi
+ -
+ name: Checkout
+ uses: actions/checkout@v3
+ if: ${{ steps.build-skip-check.outputs.image-exists == 'false' }}
+ -
+ name: Set up Docker Buildx
+ uses: docker/setup-buildx-action@v1
+ if: ${{ steps.build-skip-check.outputs.image-exists == 'false' }}
+ -
+ name: Set up QEMU
+ uses: docker/setup-qemu-action@v1
+ if: ${{ steps.build-skip-check.outputs.image-exists == 'false' }}
+ -
+ name: Build ${{ fromJSON(inputs.build-json).name }}
+ uses: docker/build-push-action@v2
+ if: ${{ steps.build-skip-check.outputs.image-exists == 'false' }}
+ with:
+ context: .
+ file: ${{ inputs.dockerfile }}
+ tags: ${{ fromJSON(inputs.build-json).image_tag }}
+ platforms: linux/amd64,linux/arm64,linux/arm/v7
+ build-args: ${{ inputs.build-args }}
+ push: true
+ cache-from: type=gha
+ cache-to: type=gha,mode=max
hooks:
- id: black
# Dockerfile hooks
- - repo: https://github.com/pryorda/dockerfilelint-precommit-hooks
- rev: "v0.1.0"
+ - repo: https://github.com/AleksaC/hadolint-py
+ rev: v1.19.0
hooks:
- - id: dockerfilelint
+ - id: hadolint
+ args:
+ - --ignore
+ - DL3006 # https://github.com/hadolint/hadolint/wiki/DL3006 (doesn't understand FROM with ARG)
+ - --ignore
+ - DL3008 # https://github.com/hadolint/hadolint/wiki/DL3008 (should probably do this at some point)
+ - --ignore
+ - DL3013 # https://github.com/hadolint/hadolint/wiki/DL3013 (should probably do this too at some point)
# Shell script hooks
- repo: https://github.com/lovesegfault/beautysh
rev: v6.2.1
-FROM node:16 AS compile-frontend
-
-COPY . /src
-
-WORKDIR /src/src-ui
-RUN npm update npm -g && npm ci --no-optional
-RUN ./node_modules/.bin/ng build --configuration production
-
-FROM ghcr.io/paperless-ngx/builder/ngx-base:1.7.0 as main-app
+# These are all built previously in the pipeline
+# They provide either a .deb, .whl or whatever npm outputs
+ARG JBIG2ENC_BASE_IMAGE
+ARG QPDF_BASE_IMAGE
+ARG PIKEPDF_BASE_IMAGE
+ARG PSYCOPG2_BASE_IMAGE
+ARG FRONTEND_BASE_IMAGE
+
+FROM ${JBIG2ENC_BASE_IMAGE} AS jbig2enc-builder
+FROM ${QPDF_BASE_IMAGE} as qpdf-builder
+FROM ${PIKEPDF_BASE_IMAGE} as pikepdf-builder
+FROM ${PSYCOPG2_BASE_IMAGE} as psycopg2-builder
+FROM ${FRONTEND_BASE_IMAGE} as compile-frontend
+
+FROM python:3.9-slim-bullseye as main-app
LABEL org.opencontainers.image.authors="paperless-ngx team <hello@paperless-ngx.com>"
LABEL org.opencontainers.image.documentation="https://paperless-ngx.readthedocs.io/en/latest/"
LABEL org.opencontainers.image.url="https://github.com/paperless-ngx/paperless-ngx"
LABEL org.opencontainers.image.licenses="GPL-3.0-only"
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Packages needed only for building
+ARG BUILD_PACKAGES="\
+ build-essential \
+ git \
+ python3-dev"
+
+# Packages need for running
+ARG RUNTIME_PACKAGES="\
+ curl \
+ file \
+ # fonts for text file thumbnail generation
+ fonts-liberation \
+ gettext \
+ ghostscript \
+ gnupg \
+ gosu \
+ icc-profiles-free \
+ imagemagick \
+ media-types \
+ liblept5 \
+ libpq5 \
+ libxml2 \
+ libxslt1.1 \
+ libgnutls30 \
+ libjpeg62-turbo \
+ optipng \
+ python3 \
+ python3-pip \
+ python3-setuptools \
+ postgresql-client \
+ # For Numpy
+ libatlas3-base \
+ # thumbnail size reduction
+ pngquant \
+ # OCRmyPDF dependencies
+ tesseract-ocr \
+ tesseract-ocr-eng \
+ tesseract-ocr-deu \
+ tesseract-ocr-fra \
+ tesseract-ocr-ita \
+ tesseract-ocr-spa \
+ tzdata \
+ unpaper \
+ # Mime type detection
+ zlib1g \
+ # Barcode splitter
+ libzbar0 \
+ poppler-utils"
+
WORKDIR /usr/src/paperless/src/
+# Copy qpdf and runtime library
+COPY --from=qpdf-builder /usr/src/qpdf/libqpdf28_*.deb .
+COPY --from=qpdf-builder /usr/src/qpdf/qpdf_*.deb .
+
+# Copy pikepdf wheel and dependencies
+COPY --from=pikepdf-builder /usr/src/pikepdf/wheels/*.whl .
+
+# Copy psycopg2 wheel
+COPY --from=psycopg2-builder /usr/src/psycopg2/wheels/psycopg2*.whl .
+
+# copy jbig2enc
+COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/.libs/libjbig2enc* /usr/local/lib/
+COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/jbig2 /usr/local/bin/
+COPY --from=jbig2enc-builder /usr/src/jbig2enc/src/*.h /usr/local/include/
+
COPY requirements.txt ../
# Python dependencies
-RUN apt-get update \
- # python-Levenshtein still needs to be compiled here
- && apt-get -y --no-install-recommends install \
- build-essential \
- && python3 -m pip install --upgrade --no-cache-dir pip wheel \
- && python3 -m pip install --default-timeout=1000 --upgrade --no-cache-dir supervisor \
- && python3 -m pip install --default-timeout=1000 --no-cache-dir -r ../requirements.txt \
- && apt-get -y purge build-essential \
- && apt-get -y autoremove --purge \
- && rm -rf /var/lib/apt/lists/*
+RUN set -eux \
+ && apt-get update \
+ && apt-get install --yes --quiet --no-install-recommends ${RUNTIME_PACKAGES} ${BUILD_PACKAGES} \
+ && python3 -m pip install --no-cache-dir --upgrade wheel \
+ && echo "Installing qpdf" \
+ && apt-get install --yes --no-install-recommends ./libqpdf28_*.deb \
+ && apt-get install --yes --no-install-recommends ./qpdf_*.deb \
+ && echo "Installing pikepdf and dependencies wheel" \
+ && python3 -m pip install --no-cache-dir packaging*.whl \
+ && python3 -m pip install --no-cache-dir lxml*.whl \
+ && python3 -m pip install --no-cache-dir Pillow*.whl \
+ && python3 -m pip install --no-cache-dir pyparsing*.whl \
+ && python3 -m pip install --no-cache-dir pikepdf*.whl \
+ && python -m pip list \
+ && echo "Installing psycopg2 wheel" \
+ && python3 -m pip install --no-cache-dir psycopg2*.whl \
+ && python -m pip list \
+ && echo "Installing supervisor" \
+ && python3 -m pip install --default-timeout=1000 --upgrade --no-cache-dir supervisor \
+ && echo "Installing Python requirements" \
+ && python3 -m pip install --default-timeout=1000 --no-cache-dir -r ../requirements.txt \
+ && echo "Cleaning up image" \
+ && apt-get -y purge ${BUILD_PACKAGES} \
+ && apt-get -y autoremove --purge \
+ && apt-get clean --yes \
+ && rm -rf /var/lib/apt/lists/* \
+ && rm -rf /tmp/* \
+ && rm -rf /var/tmp/* \
+ && rm -rf /var/cache/apt/archives/* \
+ && truncate -s 0 /var/log/*log
# setup docker-specific things
COPY docker/ ./docker/
-RUN cd docker \
- && cp imagemagick-policy.xml /etc/ImageMagick-6/policy.xml \
+WORKDIR /usr/src/paperless/src/docker/
+
+RUN set -eux \
+ && cp imagemagick-policy.xml /etc/ImageMagick-6/policy.xml \
&& mkdir /var/log/supervisord /var/run/supervisord \
&& cp supervisord.conf /etc/supervisord.conf \
&& cp docker-entrypoint.sh /sbin/docker-entrypoint.sh \
&& cp docker-prepare.sh /sbin/docker-prepare.sh \
&& chmod 755 /sbin/docker-prepare.sh \
&& chmod +x install_management_commands.sh \
- && ./install_management_commands.sh \
- && cd .. \
- && rm -rf docker/
+ && ./install_management_commands.sh
+
+WORKDIR /usr/src/paperless/
-COPY gunicorn.conf.py ../
+COPY gunicorn.conf.py .
# copy app
COPY --from=compile-frontend /src/src/ ./
# add users, setup scripts
-RUN addgroup --gid 1000 paperless \
+RUN set -eux \
+ && addgroup --gid 1000 paperless \
&& useradd --uid 1000 --gid paperless --home-dir /usr/src/paperless paperless \
&& chown -R paperless:paperless ../ \
&& gosu paperless python3 manage.py collectstatic --clear --no-input \
--- /dev/null
+# This Dockerfile compiles the frontend
+# Inputs: None
+
+FROM node:16-bullseye-slim AS compile-frontend
+
+COPY . /src
+
+WORKDIR /src/src-ui
+RUN set -eux \
+ && npm update npm -g \
+ && npm ci --no-optional
+RUN set -eux \
+ && ./node_modules/.bin/ng build --configuration production
--- /dev/null
+# This Dockerfile compiles the jbig2enc library
+# Inputs:
+# - JBIG2ENC_VERSION - the Git tag to checkout and build
+
+FROM debian:bullseye-slim
+
+LABEL org.opencontainers.image.description="A intermediate image with jbig2enc built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+ build-essential \
+ automake \
+ libtool \
+ libleptonica-dev \
+ zlib1g-dev \
+ git \
+ ca-certificates"
+
+WORKDIR /usr/src/jbig2enc
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+RUN apt-get update --quiet \
+ && apt-get install --yes --quiet --no-install-recommends ${BUILD_PACKAGES} \
+ && rm -rf /var/lib/apt/lists/*
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+ARG JBIG2ENC_VERSION
+
+RUN set -eux \
+ && git clone --quiet --branch $JBIG2ENC_VERSION https://github.com/agl/jbig2enc .
+RUN set -eux \
+ && ./autogen.sh
+RUN set -eux \
+ && ./configure && make
--- /dev/null
+# This Dockerfile builds the pikepdf wheel
+# Inputs:
+# - QPDF_BASE_IMAGE - The image to copy built qpdf .ded files from
+# - GIT_TAG - The Git tag to clone and build from
+# - VERSION - Used to force the built pikepdf version to match
+
+ARG QPDF_BASE_IMAGE
+FROM ${QPDF_BASE_IMAGE} as qpdf-builder
+
+# This does nothing, except provide a name for a copy below
+
+FROM python:3.9-slim-bullseye
+
+LABEL org.opencontainers.image.description="A intermediate image with pikepdf wheel built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+ build-essential \
+ git \
+ libjpeg62-turbo-dev \
+ zlib1g-dev \
+ libgnutls28-dev \
+ libxml2-dev \
+ libxslt1-dev \
+ python3-dev \
+ python3-pip"
+
+WORKDIR /usr/src
+
+COPY --from=qpdf-builder /usr/src/qpdf/*.deb .
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+
+RUN set -eux \
+ && apt-get update --quiet \
+ && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
+ && dpkg --install libqpdf28_*.deb \
+ && dpkg --install libqpdf-dev_*.deb \
+ && python3 -m pip install --no-cache-dir --upgrade pip wheel pybind11 \
+ && rm -rf /var/lib/apt/lists/*
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+ARG GIT_TAG
+ARG VERSION
+
+RUN set -eux \
+ && echo "building pikepdf wheel" \
+ # Note the v in the tag name here
+ && git clone --quiet --depth 1 --branch "${GIT_TAG}" https://github.com/pikepdf/pikepdf.git \
+ && cd pikepdf \
+ # pikepdf seems to specifciy either a next version when built OR
+ # a post release tag.
+ # In either case, this won't match what we want from requirements.txt
+ # Directly modify the setup.py to set the version we just checked out of Git
+ && sed -i "s/use_scm_version=True/version=\"${VERSION}\"/g" setup.py \
+ # https://github.com/pikepdf/pikepdf/issues/323
+ && rm pyproject.toml \
+ && mkdir wheels \
+ && python3 -m pip wheel . --wheel-dir wheels \
+ && ls -ahl wheels
--- /dev/null
+# This Dockerfile builds the psycopg2 wheel
+# Inputs:
+# - GIT_TAG - The Git tag to clone and build from
+# - VERSION - Unused, kept for future possible usage
+
+FROM python:3.9-slim-bullseye
+
+LABEL org.opencontainers.image.description="A intermediate image with psycopg2 wheel built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+ build-essential \
+ git \
+ libpq-dev \
+ python3-dev \
+ python3-pip"
+
+WORKDIR /usr/src
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+
+RUN set -eux \
+ && apt-get update --quiet \
+ && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
+ && rm -rf /var/lib/apt/lists/* \
+ && python3 -m pip install --upgrade pip wheel
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+ARG GIT_TAG
+ARG VERSION
+
+RUN set -eux \
+ && echo "Building psycopg2 wheel" \
+ && cd /usr/src \
+ && git clone --quiet --depth 1 --branch ${GIT_TAG} https://github.com/psycopg/psycopg2.git \
+ && cd psycopg2 \
+ && mkdir wheels \
+ && python3 -m pip wheel . --wheel-dir wheels \
+ && ls -ahl wheels/
--- /dev/null
+FROM debian:bullseye-slim
+
+LABEL org.opencontainers.image.description="A intermediate image with qpdf built"
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+ARG BUILD_PACKAGES="\
+ build-essential \
+ debhelper \
+ debian-keyring \
+ devscripts \
+ equivs \
+ libtool \
+ libjpeg62-turbo-dev \
+ libgnutls28-dev \
+ packaging-dev \
+ zlib1g-dev"
+
+WORKDIR /usr/src
+
+# As this is an base image for a multi-stage final image
+# the added size of the install is basically irrelevant
+
+RUN set -eux \
+ && apt-get update --quiet \
+ && apt-get install --yes --quiet --no-install-recommends $BUILD_PACKAGES \
+ && rm -rf /var/lib/apt/lists/*
+
+# Layers after this point change according to required version
+# For better caching, seperate the basic installs from
+# the building
+
+# This must match to pikepdf's minimum at least
+ARG QPDF_VERSION
+
+# In order to get the required version of qpdf, it is backported from bookwork
+# and then built from source
+RUN set -eux \
+ && echo "Building qpdf" \
+ && echo "deb-src http://deb.debian.org/debian/ bookworm main" | tee /etc/apt/sources.list.d/bookworm-src.list \
+ && apt-get update \
+ && mkdir qpdf \
+ && cd qpdf \
+ && apt-get source --yes --quiet qpdf=${QPDF_VERSION}-1/bookworm \
+ && rm -rf /var/lib/apt/lists/* \
+ && cd qpdf-$QPDF_VERSION \
+ && DEBEMAIL=hello@paperless-ngx.com debchange --bpo \
+ && export DEB_BUILD_OPTIONS="terse nocheck nodoc parallel=2" \
+ && dpkg-buildpackage --build=binary --unsigned-source --unsigned-changes \
+ && pwd \
+ && ls -ahl ../*.deb
--- /dev/null
+#!/usr/bin/env python3
+"""
+This is a helper script to either parse the JSON of the Pipfile.lock
+or otherwise return a JSON object detailing versioning and image tags
+for the packages we build seperately, then copy into the final Docker image
+"""
+import argparse
+import json
+import os
+from pathlib import Path
+from typing import Final
+
+CONFIG: Final = {
+ # All packages need to be in the dict, even if not configured further
+ # as it is used for the possible choices in the argument
+ "psycopg2": {},
+ # Most information about Python packages comes from the Pipfile.lock
+ "pikepdf": {
+ "qpdf_version": "10.6.3",
+ },
+ # For other packages, it is directly configured, for now
+ # These require manual updates to this file for version updates
+ "qpdf": {
+ "version": "10.6.3",
+ "git_tag": "N/A",
+ },
+ "jbig2enc": {
+ "version": "0.29",
+ "git_tag": "0.29",
+ },
+}
+
+
+def _get_image_tag(
+ repo_name: str,
+ pkg_name: str,
+ pkg_version: str,
+) -> str:
+ return f"ghcr.io/{repo_name}/builder/{pkg_name}:{pkg_version}"
+
+
+def _main():
+ parser = argparse.ArgumentParser(
+ description="Generate a JSON object of information required to build the given package, based on the Pipfile.lock",
+ )
+ parser.add_argument(
+ "package",
+ help="The name of the package to generate JSON for",
+ choices=CONFIG.keys(),
+ )
+
+ args = parser.parse_args()
+
+ pip_lock = Path("Pipfile.lock")
+
+ repo_name = os.environ["GITHUB_REPOSITORY"]
+
+ # The JSON object we'll output
+ output = {"name": args.package}
+
+ # Read Pipfile.lock file
+
+ pipfile_data = json.loads(pip_lock.read_text())
+
+ # Read the version from Pipfile.lock
+
+ if args.package in pipfile_data["default"]:
+
+ pkg_data = pipfile_data["default"][args.package]
+
+ pkg_version = pkg_data["version"].split("==")[-1]
+
+ output["version"] = pkg_version
+
+ # Based on the package, generate the expected Git tag name
+
+ if args.package == "pikepdf":
+ git_tag_name = f"v{pkg_version}"
+ elif args.package == "psycopg2":
+ git_tag_name = pkg_version.replace(".", "_")
+
+ output["git_tag"] = git_tag_name
+
+ # Based on the package and environment, generate the Docker image tag
+
+ image_tag = _get_image_tag(repo_name, args.package, pkg_version)
+
+ output["image_tag"] = image_tag
+
+ # Check for any special configuration, based on package
+
+ if args.package in CONFIG:
+ output.update(CONFIG[args.package])
+
+ elif args.package in CONFIG:
+
+ # This is not a Python package
+
+ output.update(CONFIG[args.package])
+
+ output["image_tag"] = _get_image_tag(repo_name, args.package, output["version"])
+
+ else:
+ raise NotImplementedError(args.package)
+
+ # Output the JSON info to stdout
+
+ print(json.dumps(output, indent=2))
+
+
+if __name__ == "__main__":
+ _main()
#!/usr/bin/env bash
+set -e
+
wait_for_postgres() {
attempt_num=1
max_attempts=5
#!/usr/bin/env bash
+set -eu
+
for command in document_archiver document_exporter document_importer mail_fetcher document_create_classifier document_index document_renamer document_retagger document_thumbnails document_sanity_checker manage_superuser;
do
echo "installing $command..."
projects / thirdparty / paperless-ngx.git / commitdiff
