From 2341fb569426c11efac0572a2b6e9e1fb575dac7 Mon Sep 17 00:00:00 2001 From: Amp Date: Sun, 15 Nov 2026 20:11:32 +0000 Subject: [PATCH 1/2] feat(authentik): upgrade PostgreSQL from 16 to 18 authentik supports PostgreSQL 14 to 18, but 18 cannot read a 16 data directory, so the image tag cannot just be bumped. Move the database service to postgres:18.7-alpine3.23 with the 18 layout (a new postgres volume at /var/lib/postgresql, explicit PGDATA), as twenty and n8n use. A migration on web1 and web3 dumps the 16 database with authentik stopped, restores it into authentik_postgres with a throwaway 18 container, and leaves the containers role to start the new stack. The 16 volume and the dump in /opt/ansible/backups stay as the rollback copy. If a step fails, the half-restored volume is removed and the 16 stack is started again. Amp-Thread-ID: https://ampcode.com/threads/T-01a1d579-3073-73a1-ae2a-13126333974c Co-authored-by: Anatoli Nicolae --- ..._0007_upgrade_authentik_postgres_to_18.yml | 164 ++++++++++++++++++ web1/containers/authentik/docker-compose.yml | 9 +- ..._0006_upgrade_authentik_postgres_to_18.yml | 164 ++++++++++++++++++ web3/containers/authentik/docker-compose.yml | 9 +- 4 files changed, 340 insertions(+), 6 deletions(-) create mode 100644 web1/ansible/migrations/20261115_0007_upgrade_authentik_postgres_to_18.yml create mode 100644 web3/ansible/migrations/20261115_0006_upgrade_authentik_postgres_to_18.yml diff --git a/web1/ansible/migrations/20261115_0007_upgrade_authentik_postgres_to_18.yml b/web1/ansible/migrations/20261115_0007_upgrade_authentik_postgres_to_18.yml new file mode 100644 index 00000000..5b3d4c42 --- /dev/null +++ b/web1/ansible/migrations/20261115_0007_upgrade_authentik_postgres_to_18.yml @@ -0,0 +1,164 @@ +--- +# Migration: 20261115_0007_upgrade_authentik_postgres_to_18 +# Description: Upgrade authentik's PostgreSQL from 16 to 18 +# Problem: PostgreSQL 18 cannot read a 16 data directory, and its image keeps +# data in /var/lib/postgresql/18/docker instead of /var/lib/postgresql/data. +# Changing the image tag alone leaves the database refusing to start +# Solution: dump the 16 database while authentik is stopped, restore it into a +# new authentik_postgres volume with a throwaway 18 container, then let the +# containers role start the stack on the new compose file. The 16 volume +# (authentik_database) and the dump in /opt/ansible/backups are kept as the +# rollback copy; remove them by hand once authentik is verified. If any step +# fails, the half-restored volume is removed and the 16 stack is started again + +- name: Check for the authentik .env + ansible.builtin.stat: + path: /opt/containers/authentik/.env + register: authentik_env + +- name: Upgrade authentik's database to PostgreSQL 18 + when: authentik_env.stat.exists + block: + - name: Inspect the authentik database container + community.docker.docker_container_info: + name: authentik-database-1 + register: authentik_db + + - name: Inspect the PostgreSQL 18 volume + community.docker.docker_volume_info: + name: authentik_postgres + register: authentik_pg18_volume + + - name: Find the volume mounted at /var/lib/postgresql + ansible.builtin.set_fact: + authentik_db_on_18: >- + {{ authentik_db.exists and (authentik_db.container.Mounts + | selectattr('Destination', 'eq', '/var/lib/postgresql') + | selectattr('Name', 'eq', 'authentik_postgres') | list | length > 0) }} + + - name: Refuse to overwrite a PostgreSQL 18 volume authentik does not use yet + ansible.builtin.assert: + that: authentik_db_on_18 or not authentik_pg18_volume.exists + fail_msg: >- + authentik_postgres exists, but authentik-database-1 does not use it, so it is + left over from an earlier attempt. Inspect it, remove it with + `docker volume rm authentik_postgres`, and deploy again. + + - name: Dump, restore and switch to PostgreSQL 18 + when: not authentik_db_on_18 + block: + - name: Require the authentik database container + ansible.builtin.assert: + that: authentik_db.exists + fail_msg: >- + authentik-database-1 does not exist, so there is no running PostgreSQL 16 to + dump. Start the 16 stack, or restore the database into authentik_postgres by hand. + + - name: Read the PostgreSQL 18 image from the new compose file + ansible.builtin.set_fact: + authentik_pg18_image: >- + {{ (lookup('ansible.builtin.file', playbook_dir ~ '/../containers/authentik/docker-compose.yml') + | from_yaml).services.database.image }} + authentik_pg_env: >- + {{ dict(authentik_db.container.Config.Env + | select('match', '^POSTGRES_(USER|PASSWORD|DB)=') + | map('regex_findall', '^([^=]+)=(.*)$') | map('first')) }} + no_log: true + + - name: Create the backup directory + ansible.builtin.file: + path: /opt/ansible/backups + state: directory + owner: root + group: root + mode: "0700" + + - name: Upgrade, or start the 16 stack again + block: + - name: Stop authentik and start its database + ansible.builtin.shell: | + set -eu + cd /opt/containers/authentik + docker compose stop server worker + docker compose start database + for _ in $(seq 60); do + docker compose exec -T database sh -c 'pg_isready -q -U "$POSTGRES_USER" -d "$POSTGRES_DB"' && exit 0 + sleep 2 + done + echo "PostgreSQL 16 did not become ready" >&2 + exit 1 + changed_when: true + + # The dump is the only copy of the database outside its volume: + # PostgreSQL volumes are not in the borg backup. + - name: Dump the PostgreSQL 16 database + ansible.builtin.shell: | + set -eu + umask 077 + cd /opt/containers/authentik + docker compose exec -T database sh -c 'pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB"' \ + > /opt/ansible/backups/authentik-postgres16.sql + tail -n 5 /opt/ansible/backups/authentik-postgres16.sql | grep -q 'PostgreSQL database dump complete' + changed_when: true + + - name: Stop the PostgreSQL 16 database + ansible.builtin.command: + argv: [docker, compose, --project-directory, /opt/containers/authentik, stop] + changed_when: true + + - name: Start a PostgreSQL 18 container on authentik_postgres + community.docker.docker_container: + name: authentik-postgres18-restore + image: "{{ authentik_pg18_image }}" + network_mode: none + env: "{{ authentik_pg_env | combine({'PGDATA': '/var/lib/postgresql/18/docker'}) }}" + volumes: + - authentik_postgres:/var/lib/postgresql + state: started + no_log: true + + # Over TCP, so the temporary server initdb runs on the socket + # does not count as ready. + - name: Wait for PostgreSQL 18 to accept connections + ansible.builtin.command: + argv: [docker, exec, authentik-postgres18-restore, pg_isready, -q, -h, 127.0.0.1] + register: authentik_pg18_ready + until: authentik_pg18_ready.rc == 0 + retries: 30 + delay: 2 + changed_when: false + + - name: Restore the dump into PostgreSQL 18 + ansible.builtin.shell: | + set -eu + docker exec -i authentik-postgres18-restore sh -c \ + 'psql -q -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" && psql -q -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c ANALYZE' \ + < /opt/ansible/backups/authentik-postgres16.sql + changed_when: true + + - name: Remove the restore container + community.docker.docker_container: + name: authentik-postgres18-restore + state: absent + + rescue: + - name: Remove the restore container + community.docker.docker_container: + name: authentik-postgres18-restore + state: absent + + - name: Remove the half-restored PostgreSQL 18 volume + community.docker.docker_volume: + name: authentik_postgres + state: absent + + - name: Start the PostgreSQL 16 stack again + ansible.builtin.command: + argv: [docker, compose, --project-directory, /opt/containers/authentik, start] + changed_when: true + + - name: Fail the migration + ansible.builtin.fail: + msg: >- + The PostgreSQL 18 upgrade failed at "{{ ansible_failed_task.name }}" and authentik + was started again on PostgreSQL 16: {{ ansible_failed_result.stderr | default(ansible_failed_result.msg | default('')) }} diff --git a/web1/containers/authentik/docker-compose.yml b/web1/containers/authentik/docker-compose.yml index e2d7a4c3..fcc367eb 100644 --- a/web1/containers/authentik/docker-compose.yml +++ b/web1/containers/authentik/docker-compose.yml @@ -1,7 +1,7 @@ --- services: database: - image: docker.io/library/postgres:16-alpine@sha256:81e77a4833487b8132fe4b00bedd39dc3f74b8641e387f19312a48f7de218c2e + image: postgres:18.7-alpine3.23@sha256:60aac580d8dbe3093c097bb1d11f9e62999814d3a1da2c1aaf729c0b7db117f2 restart: unless-stopped healthcheck: test: ["CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"] @@ -9,8 +9,11 @@ services: interval: 30s retries: 5 timeout: 5s + environment: + # PG18: version-specific PGDATA; volume mounts at /var/lib/postgresql + PGDATA: /var/lib/postgresql/18/docker volumes: - - database:/var/lib/postgresql/data + - postgres:/var/lib/postgresql env_file: - .env networks: @@ -58,7 +61,7 @@ services: - authentik volumes: - database: + postgres: driver: local networks: diff --git a/web3/ansible/migrations/20261115_0006_upgrade_authentik_postgres_to_18.yml b/web3/ansible/migrations/20261115_0006_upgrade_authentik_postgres_to_18.yml new file mode 100644 index 00000000..e9de187e --- /dev/null +++ b/web3/ansible/migrations/20261115_0006_upgrade_authentik_postgres_to_18.yml @@ -0,0 +1,164 @@ +--- +# Migration: 20261115_0006_upgrade_authentik_postgres_to_18 +# Description: Upgrade authentik's PostgreSQL from 16 to 18 +# Problem: PostgreSQL 18 cannot read a 16 data directory, and its image keeps +# data in /var/lib/postgresql/18/docker instead of /var/lib/postgresql/data. +# Changing the image tag alone leaves the database refusing to start +# Solution: dump the 16 database while authentik is stopped, restore it into a +# new authentik_postgres volume with a throwaway 18 container, then let the +# containers role start the stack on the new compose file. The 16 volume +# (authentik_database) and the dump in /opt/ansible/backups are kept as the +# rollback copy; remove them by hand once authentik is verified. If any step +# fails, the half-restored volume is removed and the 16 stack is started again + +- name: Check for the authentik .env + ansible.builtin.stat: + path: /opt/containers/authentik/.env + register: authentik_env + +- name: Upgrade authentik's database to PostgreSQL 18 + when: authentik_env.stat.exists + block: + - name: Inspect the authentik database container + community.docker.docker_container_info: + name: authentik-database-1 + register: authentik_db + + - name: Inspect the PostgreSQL 18 volume + community.docker.docker_volume_info: + name: authentik_postgres + register: authentik_pg18_volume + + - name: Find the volume mounted at /var/lib/postgresql + ansible.builtin.set_fact: + authentik_db_on_18: >- + {{ authentik_db.exists and (authentik_db.container.Mounts + | selectattr('Destination', 'eq', '/var/lib/postgresql') + | selectattr('Name', 'eq', 'authentik_postgres') | list | length > 0) }} + + - name: Refuse to overwrite a PostgreSQL 18 volume authentik does not use yet + ansible.builtin.assert: + that: authentik_db_on_18 or not authentik_pg18_volume.exists + fail_msg: >- + authentik_postgres exists, but authentik-database-1 does not use it, so it is + left over from an earlier attempt. Inspect it, remove it with + `docker volume rm authentik_postgres`, and deploy again. + + - name: Dump, restore and switch to PostgreSQL 18 + when: not authentik_db_on_18 + block: + - name: Require the authentik database container + ansible.builtin.assert: + that: authentik_db.exists + fail_msg: >- + authentik-database-1 does not exist, so there is no running PostgreSQL 16 to + dump. Start the 16 stack, or restore the database into authentik_postgres by hand. + + - name: Read the PostgreSQL 18 image from the new compose file + ansible.builtin.set_fact: + authentik_pg18_image: >- + {{ (lookup('ansible.builtin.file', playbook_dir ~ '/../containers/authentik/docker-compose.yml') + | from_yaml).services.database.image }} + authentik_pg_env: >- + {{ dict(authentik_db.container.Config.Env + | select('match', '^POSTGRES_(USER|PASSWORD|DB)=') + | map('regex_findall', '^([^=]+)=(.*)$') | map('first')) }} + no_log: true + + - name: Create the backup directory + ansible.builtin.file: + path: /opt/ansible/backups + state: directory + owner: root + group: root + mode: "0700" + + - name: Upgrade, or start the 16 stack again + block: + - name: Stop authentik and start its database + ansible.builtin.shell: | + set -eu + cd /opt/containers/authentik + docker compose stop server worker + docker compose start database + for _ in $(seq 60); do + docker compose exec -T database sh -c 'pg_isready -q -U "$POSTGRES_USER" -d "$POSTGRES_DB"' && exit 0 + sleep 2 + done + echo "PostgreSQL 16 did not become ready" >&2 + exit 1 + changed_when: true + + # The dump is the only copy of the database outside its volume: + # PostgreSQL volumes are not in the borg backup. + - name: Dump the PostgreSQL 16 database + ansible.builtin.shell: | + set -eu + umask 077 + cd /opt/containers/authentik + docker compose exec -T database sh -c 'pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB"' \ + > /opt/ansible/backups/authentik-postgres16.sql + tail -n 5 /opt/ansible/backups/authentik-postgres16.sql | grep -q 'PostgreSQL database dump complete' + changed_when: true + + - name: Stop the PostgreSQL 16 database + ansible.builtin.command: + argv: [docker, compose, --project-directory, /opt/containers/authentik, stop] + changed_when: true + + - name: Start a PostgreSQL 18 container on authentik_postgres + community.docker.docker_container: + name: authentik-postgres18-restore + image: "{{ authentik_pg18_image }}" + network_mode: none + env: "{{ authentik_pg_env | combine({'PGDATA': '/var/lib/postgresql/18/docker'}) }}" + volumes: + - authentik_postgres:/var/lib/postgresql + state: started + no_log: true + + # Over TCP, so the temporary server initdb runs on the socket + # does not count as ready. + - name: Wait for PostgreSQL 18 to accept connections + ansible.builtin.command: + argv: [docker, exec, authentik-postgres18-restore, pg_isready, -q, -h, 127.0.0.1] + register: authentik_pg18_ready + until: authentik_pg18_ready.rc == 0 + retries: 30 + delay: 2 + changed_when: false + + - name: Restore the dump into PostgreSQL 18 + ansible.builtin.shell: | + set -eu + docker exec -i authentik-postgres18-restore sh -c \ + 'psql -q -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" && psql -q -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c ANALYZE' \ + < /opt/ansible/backups/authentik-postgres16.sql + changed_when: true + + - name: Remove the restore container + community.docker.docker_container: + name: authentik-postgres18-restore + state: absent + + rescue: + - name: Remove the restore container + community.docker.docker_container: + name: authentik-postgres18-restore + state: absent + + - name: Remove the half-restored PostgreSQL 18 volume + community.docker.docker_volume: + name: authentik_postgres + state: absent + + - name: Start the PostgreSQL 16 stack again + ansible.builtin.command: + argv: [docker, compose, --project-directory, /opt/containers/authentik, start] + changed_when: true + + - name: Fail the migration + ansible.builtin.fail: + msg: >- + The PostgreSQL 18 upgrade failed at "{{ ansible_failed_task.name }}" and authentik + was started again on PostgreSQL 16: {{ ansible_failed_result.stderr | default(ansible_failed_result.msg | default('')) }} diff --git a/web3/containers/authentik/docker-compose.yml b/web3/containers/authentik/docker-compose.yml index bf8dbb9b..316609c1 100644 --- a/web3/containers/authentik/docker-compose.yml +++ b/web3/containers/authentik/docker-compose.yml @@ -1,7 +1,7 @@ --- services: database: - image: docker.io/library/postgres:16-alpine@sha256:81e77a4833487b8132fe4b00bedd39dc3f74b8641e387f19312a48f7de218c2e + image: postgres:18.7-alpine3.23@sha256:60aac580d8dbe3093c097bb1d11f9e62999814d3a1da2c1aaf729c0b7db117f2 restart: unless-stopped healthcheck: test: ["CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"] @@ -9,8 +9,11 @@ services: interval: 30s retries: 5 timeout: 5s + environment: + # PG18: version-specific PGDATA; volume mounts at /var/lib/postgresql + PGDATA: /var/lib/postgresql/18/docker volumes: - - database:/var/lib/postgresql/data + - postgres:/var/lib/postgresql env_file: - .env networks: @@ -57,7 +60,7 @@ services: - authentik volumes: - database: + postgres: driver: local networks: From b74cdb207bd310435527dc99ba5247e9e41d8507 Mon Sep 17 00:00:00 2001 From: Amp Date: Sun, 15 Nov 2026 23:32:42 +0000 Subject: [PATCH 2/2] fix(authentik): finish the PostgreSQL 18 cutover inside the migration The migration stopped the 16 stack and left starting the 18 stack to the containers role. The roles in between (config, monit, crowdsec, docker-login, ...) could fail and leave authentik down until the next deploy. Install the new compose file and start authentik on 18 in the migration, inside its rescue: both images are already on the host, and the containers role then finds nothing to change. The rescue now puts back the 16 compose file and starts that stack before removing the half-restored volume, and a failing cleanup step no longer skips the restart; its error is reported in the final failure. The 18 volume is created with Compose's labels so compose adopts it. Amp-Thread-ID: https://ampcode.com/threads/T-01a1d579-3073-73a1-ae2a-13126333974c Co-authored-by: Anatoli Nicolae --- ..._0007_upgrade_authentik_postgres_to_18.yml | 77 ++++++++++++++++--- ..._0006_upgrade_authentik_postgres_to_18.yml | 77 ++++++++++++++++--- 2 files changed, 134 insertions(+), 20 deletions(-) diff --git a/web1/ansible/migrations/20261115_0007_upgrade_authentik_postgres_to_18.yml b/web1/ansible/migrations/20261115_0007_upgrade_authentik_postgres_to_18.yml index 5b3d4c42..d006dd4b 100644 --- a/web1/ansible/migrations/20261115_0007_upgrade_authentik_postgres_to_18.yml +++ b/web1/ansible/migrations/20261115_0007_upgrade_authentik_postgres_to_18.yml @@ -5,11 +5,13 @@ # data in /var/lib/postgresql/18/docker instead of /var/lib/postgresql/data. # Changing the image tag alone leaves the database refusing to start # Solution: dump the 16 database while authentik is stopped, restore it into a -# new authentik_postgres volume with a throwaway 18 container, then let the -# containers role start the stack on the new compose file. The 16 volume -# (authentik_database) and the dump in /opt/ansible/backups are kept as the -# rollback copy; remove them by hand once authentik is verified. If any step -# fails, the half-restored volume is removed and the 16 stack is started again +# new authentik_postgres volume with a throwaway 18 container, then install +# the new compose file and start authentik on 18 here, so the stack is never +# left stopped for the roles between migrations and containers. The 16 volume +# (authentik_database), its compose file and the dump in /opt/ansible/backups +# are kept as the rollback copy; remove them by hand once authentik is +# verified. If any step fails, the 16 compose file is put back, the 16 stack +# is started again and the half-restored volume is removed - name: Check for the authentik .env ansible.builtin.stat: @@ -73,6 +75,15 @@ group: root mode: "0700" + - name: Keep the PostgreSQL 16 compose file + ansible.builtin.copy: + src: /opt/containers/authentik/docker-compose.yml + dest: /opt/ansible/backups/authentik-docker-compose.pg16.yml + remote_src: true + owner: root + group: root + mode: "0600" + - name: Upgrade, or start the 16 stack again block: - name: Stop authentik and start its database @@ -106,6 +117,15 @@ argv: [docker, compose, --project-directory, /opt/containers/authentik, stop] changed_when: true + # With Compose's labels, so it adopts the volume as the stack's + # `postgres` volume instead of warning on every `up`. + - name: Create the PostgreSQL 18 volume + community.docker.docker_volume: + name: authentik_postgres + labels: + com.docker.compose.project: authentik + com.docker.compose.volume: postgres + - name: Start a PostgreSQL 18 container on authentik_postgres community.docker.docker_container: name: authentik-postgres18-restore @@ -141,24 +161,61 @@ name: authentik-postgres18-restore state: absent + # The containers role would install this file too, but only after + # other roles: if one of them failed, authentik would stay down. + # Both images are already on the host, so no registry login is + # needed. The containers role then finds nothing to change. + - name: Install the PostgreSQL 18 compose file + ansible.builtin.copy: + src: "{{ playbook_dir }}/../containers/authentik/docker-compose.yml" + dest: /opt/containers/authentik/docker-compose.yml + owner: root + group: root + mode: "0755" + + - name: Start authentik on PostgreSQL 18 + ansible.builtin.command: + argv: [docker, compose, --project-directory, /opt/containers/authentik, up, --detach, --wait, --wait-timeout, "300"] + changed_when: true + rescue: + # Cleanup must not stop the rollback, so its errors are reported + # in the final failure instead. - name: Remove the restore container community.docker.docker_container: name: authentik-postgres18-restore state: absent + register: authentik_cleanup_container + ignore_errors: true + + - name: Put back the PostgreSQL 16 compose file + ansible.builtin.copy: + src: /opt/ansible/backups/authentik-docker-compose.pg16.yml + dest: /opt/containers/authentik/docker-compose.yml + remote_src: true + owner: root + group: root + mode: "0755" + - name: Start the PostgreSQL 16 stack again + ansible.builtin.command: + argv: [docker, compose, --project-directory, /opt/containers/authentik, up, --detach, --wait, --wait-timeout, "300"] + changed_when: true + + # After the 16 stack is up, so no container still uses the volume. - name: Remove the half-restored PostgreSQL 18 volume community.docker.docker_volume: name: authentik_postgres state: absent - - - name: Start the PostgreSQL 16 stack again - ansible.builtin.command: - argv: [docker, compose, --project-directory, /opt/containers/authentik, start] - changed_when: true + register: authentik_cleanup_volume + ignore_errors: true - name: Fail the migration ansible.builtin.fail: msg: >- The PostgreSQL 18 upgrade failed at "{{ ansible_failed_task.name }}" and authentik was started again on PostgreSQL 16: {{ ansible_failed_result.stderr | default(ansible_failed_result.msg | default('')) }} + {{ 'Removing authentik-postgres18-restore also failed: ' ~ authentik_cleanup_container.msg + if authentik_cleanup_container is failed else '' }} + {{ 'Removing authentik_postgres also failed, remove it by hand before deploying again: ' + ~ authentik_cleanup_volume.msg if authentik_cleanup_volume is failed else '' }} diff --git a/web3/ansible/migrations/20261115_0006_upgrade_authentik_postgres_to_18.yml b/web3/ansible/migrations/20261115_0006_upgrade_authentik_postgres_to_18.yml index e9de187e..bcda71bb 100644 --- a/web3/ansible/migrations/20261115_0006_upgrade_authentik_postgres_to_18.yml +++ b/web3/ansible/migrations/20261115_0006_upgrade_authentik_postgres_to_18.yml @@ -5,11 +5,13 @@ # data in /var/lib/postgresql/18/docker instead of /var/lib/postgresql/data. # Changing the image tag alone leaves the database refusing to start # Solution: dump the 16 database while authentik is stopped, restore it into a -# new authentik_postgres volume with a throwaway 18 container, then let the -# containers role start the stack on the new compose file. The 16 volume -# (authentik_database) and the dump in /opt/ansible/backups are kept as the -# rollback copy; remove them by hand once authentik is verified. If any step -# fails, the half-restored volume is removed and the 16 stack is started again +# new authentik_postgres volume with a throwaway 18 container, then install +# the new compose file and start authentik on 18 here, so the stack is never +# left stopped for the roles between migrations and containers. The 16 volume +# (authentik_database), its compose file and the dump in /opt/ansible/backups +# are kept as the rollback copy; remove them by hand once authentik is +# verified. If any step fails, the 16 compose file is put back, the 16 stack +# is started again and the half-restored volume is removed - name: Check for the authentik .env ansible.builtin.stat: @@ -73,6 +75,15 @@ group: root mode: "0700" + - name: Keep the PostgreSQL 16 compose file + ansible.builtin.copy: + src: /opt/containers/authentik/docker-compose.yml + dest: /opt/ansible/backups/authentik-docker-compose.pg16.yml + remote_src: true + owner: root + group: root + mode: "0600" + - name: Upgrade, or start the 16 stack again block: - name: Stop authentik and start its database @@ -106,6 +117,15 @@ argv: [docker, compose, --project-directory, /opt/containers/authentik, stop] changed_when: true + # With Compose's labels, so it adopts the volume as the stack's + # `postgres` volume instead of warning on every `up`. + - name: Create the PostgreSQL 18 volume + community.docker.docker_volume: + name: authentik_postgres + labels: + com.docker.compose.project: authentik + com.docker.compose.volume: postgres + - name: Start a PostgreSQL 18 container on authentik_postgres community.docker.docker_container: name: authentik-postgres18-restore @@ -141,24 +161,61 @@ name: authentik-postgres18-restore state: absent + # The containers role would install this file too, but only after + # other roles: if one of them failed, authentik would stay down. + # Both images are already on the host, so no registry login is + # needed. The containers role then finds nothing to change. + - name: Install the PostgreSQL 18 compose file + ansible.builtin.copy: + src: "{{ playbook_dir }}/../containers/authentik/docker-compose.yml" + dest: /opt/containers/authentik/docker-compose.yml + owner: root + group: root + mode: "0755" + + - name: Start authentik on PostgreSQL 18 + ansible.builtin.command: + argv: [docker, compose, --project-directory, /opt/containers/authentik, up, --detach, --wait, --wait-timeout, "300"] + changed_when: true + rescue: + # Cleanup must not stop the rollback, so its errors are reported + # in the final failure instead. - name: Remove the restore container community.docker.docker_container: name: authentik-postgres18-restore state: absent + register: authentik_cleanup_container + ignore_errors: true + + - name: Put back the PostgreSQL 16 compose file + ansible.builtin.copy: + src: /opt/ansible/backups/authentik-docker-compose.pg16.yml + dest: /opt/containers/authentik/docker-compose.yml + remote_src: true + owner: root + group: root + mode: "0755" + - name: Start the PostgreSQL 16 stack again + ansible.builtin.command: + argv: [docker, compose, --project-directory, /opt/containers/authentik, up, --detach, --wait, --wait-timeout, "300"] + changed_when: true + + # After the 16 stack is up, so no container still uses the volume. - name: Remove the half-restored PostgreSQL 18 volume community.docker.docker_volume: name: authentik_postgres state: absent - - - name: Start the PostgreSQL 16 stack again - ansible.builtin.command: - argv: [docker, compose, --project-directory, /opt/containers/authentik, start] - changed_when: true + register: authentik_cleanup_volume + ignore_errors: true - name: Fail the migration ansible.builtin.fail: msg: >- The PostgreSQL 18 upgrade failed at "{{ ansible_failed_task.name }}" and authentik was started again on PostgreSQL 16: {{ ansible_failed_result.stderr | default(ansible_failed_result.msg | default('')) }} + {{ 'Removing authentik-postgres18-restore also failed: ' ~ authentik_cleanup_container.msg + if authentik_cleanup_container is failed else '' }} + {{ 'Removing authentik_postgres also failed, remove it by hand before deploying again: ' + ~ authentik_cleanup_volume.msg if authentik_cleanup_volume is failed else '' }}