From: Daniele Varrazzo Date: Fri, 22 Sep 2023 20:36:27 +0000 (+0200) Subject: ci: provision a M1 build server automatically, if needed X-Git-Tag: pool-3.1.8~3 X-Git-Url: http://git.ipfire.org/cgi-bin/gitweb.cgi?a=commitdiff_plain;h=9055e07a79381a8fc8b50a5eddcaef1b5d3cbc98;p=thirdparty%2Fpsycopg.git ci: provision a M1 build server automatically, if needed --- diff --git a/tools/build/build_macos_arm64.sh b/tools/build/build_macos_arm64.sh index 33dff3c4a..45d999af7 100755 --- a/tools/build/build_macos_arm64.sh +++ b/tools/build/build_macos_arm64.sh @@ -8,11 +8,15 @@ # so it can pretty much only be executed by a sudo user as it is. set -euo pipefail -set -x +# set -x python_versions="3.8.10 3.9.13 3.10.5 3.11.0" pg_version=16 +function log { + echo "$@" >&2 +} + # Move to the root of the project dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd "${dir}/../../" @@ -26,6 +30,7 @@ fi if [[ -x /opt/homebrew/bin/brew ]]; then eval "$(/opt/homebrew/bin/brew shellenv)" else + log "installing brew" command -v brew > /dev/null || ( # Not necessary: already installed # xcode-select --install @@ -39,11 +44,12 @@ export PGDATA=/opt/homebrew/var/postgresql@${pg_version} # Install PostgreSQL, if necessary command -v pg_config > /dev/null || ( + log "installing postgres" brew install postgresql@${pg_version} ) -# After PostgreSQL 15, the bin path is not in the path. -export PATH=$(ls -d1 /opt/homebrew/Cellar/postgresql@${pg_version}/*/bin):$PATH +# Starting from PostgreSQL 15, the bin path is not in the path. +export PATH="$(ls -d1 /opt/homebrew/Cellar/postgresql@${pg_version}/*/bin):$PATH" # Make sure the server is running @@ -51,7 +57,8 @@ export PATH=$(ls -d1 /opt/homebrew/Cellar/postgresql@${pg_version}/*/bin):$PATH # brew services start postgresql@${pg_version} if ! pg_ctl status; then - pg_ctl -l /opt/homebrew/var/log/postgresql@${pg_version}.log start + log "starting the server" + pg_ctl -l "/opt/homebrew/var/log/postgresql@${pg_version}.log" start fi @@ -59,6 +66,7 @@ fi for ver3 in $python_versions; do ver2=$(echo $ver3 | sed 's/\([^\.]*\)\(\.[^\.]*\)\(.*\)/\1\2/') command -v python${ver2} > /dev/null || ( + log "installing Python $ver3" (cd /tmp && curl -fsSl -O \ https://www.python.org/ftp/python/${ver3}/python-${ver3}-macos11.pkg) @@ -68,12 +76,16 @@ done # Create a virtualenv where to work if [[ ! -x .venv/bin/python ]]; then + log "creating a virtualenv" python3 -m venv .venv fi +log "installing cibuildwheel" source .venv/bin/activate pip install cibuildwheel +log "building wheels" + # Create the psycopg_binary source package rm -rf psycopg_binary python tools/build/copy_to_binary.py diff --git a/tools/build/run_build_macos_arm64.sh b/tools/build/run_build_macos_arm64.sh index f5ae61745..928459b54 100755 --- a/tools/build/run_build_macos_arm64.sh +++ b/tools/build/run_build_macos_arm64.sh @@ -5,36 +5,61 @@ # This script is designed to run on a local machine: it will clone the repos # remotely and execute the `build_macos_arm64.sh` script remotely, then will # download the built packages. A tag to build must be specified. -# -# In order to run the script, the `m1` host must be specified in -# `~/.ssh/config`; for instance: -# -# Host m1 -# User m1 -# HostName 1.2.3.4 + +# The script requires a Scaleway secret key in the SCW_SECRET_KEY env var: +# It will use scaleway_m1.sh to provision a server and use it. set -euo pipefail # set -x +function log { + echo "$@" >&2 +} +function error { + # Print an error message and exit. + log "ERROR: $@" + exit 1 +} + tag=${1:-} if [[ ! "${tag}" ]]; then - echo "Usage: $0 TAG" >&2 - exit 2 + error "Usage: $0 REF" fi -rdir=psycobuild +dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# Clone the repos -ssh m1 rm -rf "${rdir}" -ssh m1 git clone https://github.com/psycopg/psycopg.git --branch ${tag} "${rdir}" +server=$("${dir}/scaleway_m1.sh" ensure) -# Allow sudoing without password, to allow brew to install -ssh -t m1 bash -c \ - 'test -f /etc/sudoers.d/m1 || echo "m1 ALL=(ALL) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/m1' +status=$(echo "$server" | jq -r .status) +if [[ "$status" != "ready" ]]; then + error "server status is $status" +fi + +# Get user, password, ip from vnc url +tmp=$(echo "$server" | jq -r .vnc_url) # vnc://m1:PASS@1.2.3.4:5900 +tmp=${tmp/vnc:\/\//} # m1:PASS@1.2.3.4:5900 +user=${tmp%%:*} # m1 +tmp=${tmp#*:} # PASS@1.2.3.4:5900 +password=${tmp%%@*} # PASS +tmp=${tmp#*@} # 1.2.3.4:5900 +host=${tmp%%:*} # 1.2.3.4 + +ssh="ssh ${user}@${host} -o StrictHostKeyChecking=no" + +# Allow the user to sudo without asking for password. +echo "$password" | \ + $ssh sh -c "test -f /etc/sudoers.d/${user} \ + || sudo -S --prompt= sh -c \ + 'echo \"${user} ALL=(ALL) NOPASSWD:ALL\" > /etc/sudoers.d/${user}'" + +# Clone the repos +rdir=psycobuild +$ssh rm -rf "${rdir}" +$ssh git clone https://github.com/psycopg/psycopg.git --branch ${tag} "${rdir}" # Build the wheel packages -ssh m1 "${rdir}/tools/build/build_macos_arm64.sh" +$ssh "${rdir}/tools/build/build_macos_arm64.sh" # Transfer the packages locally -scp -r "m1:${rdir}/wheelhouse" . +scp -r "${user}@${host}:${rdir}/wheelhouse" . diff --git a/tools/build/scaleway_m1.sh b/tools/build/scaleway_m1.sh new file mode 100755 index 000000000..3fe288e43 --- /dev/null +++ b/tools/build/scaleway_m1.sh @@ -0,0 +1,106 @@ +#!/bin/bash + +# Implement the following commands: +# +# ensure: +# +# Get data about currently provisioned M1 server on Scaleway. If needed, +# provision one. +# +# The script requires the SCW_SECRET_KEY env var set to a valid secret. +# +# If successful, return the response data on stdout. It may look like: +# +# { +# "id": "8b196119-3cea-4a9d-b916-265037a85e60", +# "type": "M1-M", +# "name": "mac-m1-psycopg", +# "project_id": "4cf7a85e-f21e-40d4-b758-21d1f4ad3dfb", +# "organization_id": "4cf7a85e-f21e-40d4-b758-21d1f4ad3dfb", +# "ip": "1.2.3.4", +# "vnc_url": "vnc://m1:PASSWORD@1.2.3.4:5900", +# "status": "starting", +# "created_at": "2023-09-22T18:00:18.754646Z", +# "updated_at": "2023-09-22T18:00:18.754646Z", +# "deletable_at": "2023-09-23T18:00:18.754646Z", +# "zone": "fr-par-3" +# } +# +# delete: +# +# Delete one provisioned server, if available. +# +# See https://www.scaleway.com/en/developers/api/apple-silicon/ for api docs. + +set -euo pipefail +# set -x + +project_id="4cf7a85e-f21e-40d4-b758-21d1f4ad3dfb" +zone=fr-par-3 +servers_url="https://api.scaleway.com/apple-silicon/v1alpha1/zones/${zone}/servers" + +function log { + echo "$@" >&2 +} +function error { + log "ERROR: $@" + exit 1 +} + +function req { + method=$1 + shift + curl -sSL --fail-with-body -X $method \ + -H "Content-Type: application/json" \ + -H "X-Auth-Token: ${SCW_SECRET_KEY}" \ + "$@" +} +function get { + req GET "$@" +} +function post { + req POST "$@" +} +function delete { + req DELETE "$@" +} + +function server_id { + # Return the id of the first server available, else the emtpy string + servers=$(get $servers_url || error "failed to request servers list") + server_ids=$(echo "$servers" | jq -r ".servers[].id") + for id in $server_ids; do + echo $id + break + done +} + +cmd=${1:-} +case $cmd in + ensure) + id=$(server_id) + if [[ "$id" ]]; then + log "You have servers." + get "$servers_url/$id" + else + log "Creating new server." + post $servers_url -d " + { + \"name\": \"mac-m1-psycopg\", + \"project_id\": \"$project_id\", + \"type\": \"M1-M\" + }" + fi + ;; + delete) + id=$(server_id) + if [[ "$id" ]]; then + log "Deleting server $id." + delete "$servers_url/$id" + else + log "No server found." + fi + ;; + *) + error "Usage: $0 {ensure|delete}" +esac diff --git a/tools/update_errors.py b/tools/update_errors.py index 638d3520f..86dab336a 100755 --- a/tools/update_errors.py +++ b/tools/update_errors.py @@ -22,7 +22,7 @@ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(mess def main(): - classes, errors = fetch_errors("9.6 10 11 12 13 14 15".split()) + classes, errors = fetch_errors("9.6 10 11 12 13 14 15 16".split()) fn = os.path.dirname(__file__) + "/../psycopg/psycopg/errors.py" update_file(fn, generate_module_data(classes, errors))