diff --git a/web1/ansible/migrations/20261115_0007_upgrade_authentik_postgres_to_18.yml b/web1/ansible/migrations/20261115_0007_upgrade_authentik_postgres_to_18.yml new file mode 100644 index 00000000..d006dd4b --- /dev/null +++ b/web1/ansible/migrations/20261115_0007_upgrade_authentik_postgres_to_18.yml @@ -0,0 +1,221 @@ +--- +# Migration: 20261115_0007_upgrade_authentik_postgres_to_18 +# Description: Upgrade authentik's PostgreSQL from 16 to 18 +# Problem: PostgreSQL 18 cannot read a 16 data directory, and its image keeps +# data in /var/lib/postgresql/18/docker instead of /var/lib/postgresql/data. +# Changing the image tag alone leaves the database refusing to start +# Solution: dump the 16 database while authentik is stopped, restore it into a +# new authentik_postgres volume with a throwaway 18 container, then install +# the new compose file and start authentik on 18 here, so the stack is never +# left stopped for the roles between migrations and containers. The 16 volume +# (authentik_database), its compose file and the dump in /opt/ansible/backups +# are kept as the rollback copy; remove them by hand once authentik is +# verified. If any step fails, the 16 compose file is put back, the 16 stack +# is started again and the half-restored volume is removed + +- name: Check for the authentik .env + ansible.builtin.stat: + path: /opt/containers/authentik/.env + register: authentik_env + +- name: Upgrade authentik's database to PostgreSQL 18 + when: authentik_env.stat.exists + block: + - name: Inspect the authentik database container + community.docker.docker_container_info: + name: authentik-database-1 + register: authentik_db + + - name: Inspect the PostgreSQL 18 volume + community.docker.docker_volume_info: + name: authentik_postgres + register: authentik_pg18_volume + + - name: Find the volume mounted at /var/lib/postgresql + ansible.builtin.set_fact: + authentik_db_on_18: >- + {{ authentik_db.exists and (authentik_db.container.Mounts + | selectattr('Destination', 'eq', '/var/lib/postgresql') + | selectattr('Name', 'eq', 'authentik_postgres') | list | length > 0) }} + + - name: Refuse to overwrite a PostgreSQL 18 volume authentik does not use yet + ansible.builtin.assert: + that: authentik_db_on_18 or not authentik_pg18_volume.exists + fail_msg: >- + authentik_postgres exists, but authentik-database-1 does not use it, so it is + left over from an earlier attempt. Inspect it, remove it with + `docker volume rm authentik_postgres`, and deploy again. + + - name: Dump, restore and switch to PostgreSQL 18 + when: not authentik_db_on_18 + block: + - name: Require the authentik database container + ansible.builtin.assert: + that: authentik_db.exists + fail_msg: >- + authentik-database-1 does not exist, so there is no running PostgreSQL 16 to + dump. Start the 16 stack, or restore the database into authentik_postgres by hand. + + - name: Read the PostgreSQL 18 image from the new compose file + ansible.builtin.set_fact: + authentik_pg18_image: >- + {{ (lookup('ansible.builtin.file', playbook_dir ~ '/../containers/authentik/docker-compose.yml') + | from_yaml).services.database.image }} + authentik_pg_env: >- + {{ dict(authentik_db.container.Config.Env + | select('match', '^POSTGRES_(USER|PASSWORD|DB)=') + | map('regex_findall', '^([^=]+)=(.*)$') | map('first')) }} + no_log: true + + - name: Create the backup directory + ansible.builtin.file: + path: /opt/ansible/backups + state: directory + owner: root + group: root + mode: "0700" + + - name: Keep the PostgreSQL 16 compose file + ansible.builtin.copy: + src: /opt/containers/authentik/docker-compose.yml + dest: /opt/ansible/backups/authentik-docker-compose.pg16.yml + remote_src: true + owner: root + group: root + mode: "0600" + + - name: Upgrade, or start the 16 stack again + block: + - name: Stop authentik and start its database + ansible.builtin.shell: | + set -eu + cd /opt/containers/authentik + docker compose stop server worker + docker compose start database + for _ in $(seq 60); do + docker compose exec -T database sh -c 'pg_isready -q -U "$POSTGRES_USER" -d "$POSTGRES_DB"' && exit 0 + sleep 2 + done + echo "PostgreSQL 16 did not become ready" >&2 + exit 1 + changed_when: true + + # The dump is the only copy of the database outside its volume: + # PostgreSQL volumes are not in the borg backup. + - name: Dump the PostgreSQL 16 database + ansible.builtin.shell: | + set -eu + umask 077 + cd /opt/containers/authentik + docker compose exec -T database sh -c 'pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB"' \ + > /opt/ansible/backups/authentik-postgres16.sql + tail -n 5 /opt/ansible/backups/authentik-postgres16.sql | grep -q 'PostgreSQL database dump complete' + changed_when: true + + - name: Stop the PostgreSQL 16 database + ansible.builtin.command: + argv: [docker, compose, --project-directory, /opt/containers/authentik, stop] + changed_when: true + + # With Compose's labels, so it adopts the volume as the stack's + # `postgres` volume instead of warning on every `up`. + - name: Create the PostgreSQL 18 volume + community.docker.docker_volume: + name: authentik_postgres + labels: + com.docker.compose.project: authentik + com.docker.compose.volume: postgres + + - name: Start a PostgreSQL 18 container on authentik_postgres + community.docker.docker_container: + name: authentik-postgres18-restore + image: "{{ authentik_pg18_image }}" + network_mode: none + env: "{{ authentik_pg_env | combine({'PGDATA': '/var/lib/postgresql/18/docker'}) }}" + volumes: + - authentik_postgres:/var/lib/postgresql + state: started + no_log: true + + # Over TCP, so the temporary server initdb runs on the socket + # does not count as ready. + - name: Wait for PostgreSQL 18 to accept connections + ansible.builtin.command: + argv: [docker, exec, authentik-postgres18-restore, pg_isready, -q, -h, 127.0.0.1] + register: authentik_pg18_ready + until: authentik_pg18_ready.rc == 0 + retries: 30 + delay: 2 + changed_when: false + + - name: Restore the dump into PostgreSQL 18 + ansible.builtin.shell: | + set -eu + docker exec -i authentik-postgres18-restore sh -c \ + 'psql -q -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" && psql -q -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c ANALYZE' \ + < /opt/ansible/backups/authentik-postgres16.sql + changed_when: true + + - name: Remove the restore container + community.docker.docker_container: + name: authentik-postgres18-restore + state: absent + + # The containers role would install this file too, but only after + # other roles: if one of them failed, authentik would stay down. + # Both images are already on the host, so no registry login is + # needed. The containers role then finds nothing to change. + - name: Install the PostgreSQL 18 compose file + ansible.builtin.copy: + src: "{{ playbook_dir }}/../containers/authentik/docker-compose.yml" + dest: /opt/containers/authentik/docker-compose.yml + owner: root + group: root + mode: "0755" + + - name: Start authentik on PostgreSQL 18 + ansible.builtin.command: + argv: [docker, compose, --project-directory, /opt/containers/authentik, up, --detach, --wait, --wait-timeout, "300"] + changed_when: true + + rescue: + # Cleanup must not stop the rollback, so its errors are reported + # in the final failure instead. + - name: Remove the restore container + community.docker.docker_container: + name: authentik-postgres18-restore + state: absent + register: authentik_cleanup_container + ignore_errors: true + + - name: Put back the PostgreSQL 16 compose file + ansible.builtin.copy: + src: /opt/ansible/backups/authentik-docker-compose.pg16.yml + dest: /opt/containers/authentik/docker-compose.yml + remote_src: true + owner: root + group: root + mode: "0755" + + - name: Start the PostgreSQL 16 stack again + ansible.builtin.command: + argv: [docker, compose, --project-directory, /opt/containers/authentik, up, --detach, --wait, --wait-timeout, "300"] + changed_when: true + + # After the 16 stack is up, so no container still uses the volume. + - name: Remove the half-restored PostgreSQL 18 volume + community.docker.docker_volume: + name: authentik_postgres + state: absent + register: authentik_cleanup_volume + ignore_errors: true + + - name: Fail the migration + ansible.builtin.fail: + msg: >- + The PostgreSQL 18 upgrade failed at "{{ ansible_failed_task.name }}" and authentik + was started again on PostgreSQL 16: {{ ansible_failed_result.stderr | default(ansible_failed_result.msg | default('')) }} + {{ 'Removing authentik-postgres18-restore also failed: ' ~ authentik_cleanup_container.msg + if authentik_cleanup_container is failed else '' }} + {{ 'Removing authentik_postgres also failed, remove it by hand before deploying again: ' + ~ authentik_cleanup_volume.msg if authentik_cleanup_volume is failed else '' }} diff --git a/web1/containers/authentik/docker-compose.yml b/web1/containers/authentik/docker-compose.yml index e2d7a4c3..fcc367eb 100644 --- a/web1/containers/authentik/docker-compose.yml +++ b/web1/containers/authentik/docker-compose.yml @@ -1,7 +1,7 @@ --- services: database: - image: docker.io/library/postgres:16-alpine@sha256:81e77a4833487b8132fe4b00bedd39dc3f74b8641e387f19312a48f7de218c2e + image: postgres:18.7-alpine3.23@sha256:60aac580d8dbe3093c097bb1d11f9e62999814d3a1da2c1aaf729c0b7db117f2 restart: unless-stopped healthcheck: test: ["CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"] @@ -9,8 +9,11 @@ services: interval: 30s retries: 5 timeout: 5s + environment: + # PG18: version-specific PGDATA; volume mounts at /var/lib/postgresql + PGDATA: /var/lib/postgresql/18/docker volumes: - - database:/var/lib/postgresql/data + - postgres:/var/lib/postgresql env_file: - .env networks: @@ -58,7 +61,7 @@ services: - authentik volumes: - database: + postgres: driver: local networks: diff --git a/web3/ansible/migrations/20261115_0006_upgrade_authentik_postgres_to_18.yml b/web3/ansible/migrations/20261115_0006_upgrade_authentik_postgres_to_18.yml new file mode 100644 index 00000000..bcda71bb --- /dev/null +++ b/web3/ansible/migrations/20261115_0006_upgrade_authentik_postgres_to_18.yml @@ -0,0 +1,221 @@ +--- +# Migration: 20261115_0006_upgrade_authentik_postgres_to_18 +# Description: Upgrade authentik's PostgreSQL from 16 to 18 +# Problem: PostgreSQL 18 cannot read a 16 data directory, and its image keeps +# data in /var/lib/postgresql/18/docker instead of /var/lib/postgresql/data. +# Changing the image tag alone leaves the database refusing to start +# Solution: dump the 16 database while authentik is stopped, restore it into a +# new authentik_postgres volume with a throwaway 18 container, then install +# the new compose file and start authentik on 18 here, so the stack is never +# left stopped for the roles between migrations and containers. The 16 volume +# (authentik_database), its compose file and the dump in /opt/ansible/backups +# are kept as the rollback copy; remove them by hand once authentik is +# verified. If any step fails, the 16 compose file is put back, the 16 stack +# is started again and the half-restored volume is removed + +- name: Check for the authentik .env + ansible.builtin.stat: + path: /opt/containers/authentik/.env + register: authentik_env + +- name: Upgrade authentik's database to PostgreSQL 18 + when: authentik_env.stat.exists + block: + - name: Inspect the authentik database container + community.docker.docker_container_info: + name: authentik-database-1 + register: authentik_db + + - name: Inspect the PostgreSQL 18 volume + community.docker.docker_volume_info: + name: authentik_postgres + register: authentik_pg18_volume + + - name: Find the volume mounted at /var/lib/postgresql + ansible.builtin.set_fact: + authentik_db_on_18: >- + {{ authentik_db.exists and (authentik_db.container.Mounts + | selectattr('Destination', 'eq', '/var/lib/postgresql') + | selectattr('Name', 'eq', 'authentik_postgres') | list | length > 0) }} + + - name: Refuse to overwrite a PostgreSQL 18 volume authentik does not use yet + ansible.builtin.assert: + that: authentik_db_on_18 or not authentik_pg18_volume.exists + fail_msg: >- + authentik_postgres exists, but authentik-database-1 does not use it, so it is + left over from an earlier attempt. Inspect it, remove it with + `docker volume rm authentik_postgres`, and deploy again. + + - name: Dump, restore and switch to PostgreSQL 18 + when: not authentik_db_on_18 + block: + - name: Require the authentik database container + ansible.builtin.assert: + that: authentik_db.exists + fail_msg: >- + authentik-database-1 does not exist, so there is no running PostgreSQL 16 to + dump. Start the 16 stack, or restore the database into authentik_postgres by hand. + + - name: Read the PostgreSQL 18 image from the new compose file + ansible.builtin.set_fact: + authentik_pg18_image: >- + {{ (lookup('ansible.builtin.file', playbook_dir ~ '/../containers/authentik/docker-compose.yml') + | from_yaml).services.database.image }} + authentik_pg_env: >- + {{ dict(authentik_db.container.Config.Env + | select('match', '^POSTGRES_(USER|PASSWORD|DB)=') + | map('regex_findall', '^([^=]+)=(.*)$') | map('first')) }} + no_log: true + + - name: Create the backup directory + ansible.builtin.file: + path: /opt/ansible/backups + state: directory + owner: root + group: root + mode: "0700" + + - name: Keep the PostgreSQL 16 compose file + ansible.builtin.copy: + src: /opt/containers/authentik/docker-compose.yml + dest: /opt/ansible/backups/authentik-docker-compose.pg16.yml + remote_src: true + owner: root + group: root + mode: "0600" + + - name: Upgrade, or start the 16 stack again + block: + - name: Stop authentik and start its database + ansible.builtin.shell: | + set -eu + cd /opt/containers/authentik + docker compose stop server worker + docker compose start database + for _ in $(seq 60); do + docker compose exec -T database sh -c 'pg_isready -q -U "$POSTGRES_USER" -d "$POSTGRES_DB"' && exit 0 + sleep 2 + done + echo "PostgreSQL 16 did not become ready" >&2 + exit 1 + changed_when: true + + # The dump is the only copy of the database outside its volume: + # PostgreSQL volumes are not in the borg backup. + - name: Dump the PostgreSQL 16 database + ansible.builtin.shell: | + set -eu + umask 077 + cd /opt/containers/authentik + docker compose exec -T database sh -c 'pg_dump -U "$POSTGRES_USER" -d "$POSTGRES_DB"' \ + > /opt/ansible/backups/authentik-postgres16.sql + tail -n 5 /opt/ansible/backups/authentik-postgres16.sql | grep -q 'PostgreSQL database dump complete' + changed_when: true + + - name: Stop the PostgreSQL 16 database + ansible.builtin.command: + argv: [docker, compose, --project-directory, /opt/containers/authentik, stop] + changed_when: true + + # With Compose's labels, so it adopts the volume as the stack's + # `postgres` volume instead of warning on every `up`. + - name: Create the PostgreSQL 18 volume + community.docker.docker_volume: + name: authentik_postgres + labels: + com.docker.compose.project: authentik + com.docker.compose.volume: postgres + + - name: Start a PostgreSQL 18 container on authentik_postgres + community.docker.docker_container: + name: authentik-postgres18-restore + image: "{{ authentik_pg18_image }}" + network_mode: none + env: "{{ authentik_pg_env | combine({'PGDATA': '/var/lib/postgresql/18/docker'}) }}" + volumes: + - authentik_postgres:/var/lib/postgresql + state: started + no_log: true + + # Over TCP, so the temporary server initdb runs on the socket + # does not count as ready. + - name: Wait for PostgreSQL 18 to accept connections + ansible.builtin.command: + argv: [docker, exec, authentik-postgres18-restore, pg_isready, -q, -h, 127.0.0.1] + register: authentik_pg18_ready + until: authentik_pg18_ready.rc == 0 + retries: 30 + delay: 2 + changed_when: false + + - name: Restore the dump into PostgreSQL 18 + ansible.builtin.shell: | + set -eu + docker exec -i authentik-postgres18-restore sh -c \ + 'psql -q -v ON_ERROR_STOP=1 -U "$POSTGRES_USER" -d "$POSTGRES_DB" && psql -q -U "$POSTGRES_USER" -d "$POSTGRES_DB" -c ANALYZE' \ + < /opt/ansible/backups/authentik-postgres16.sql + changed_when: true + + - name: Remove the restore container + community.docker.docker_container: + name: authentik-postgres18-restore + state: absent + + # The containers role would install this file too, but only after + # other roles: if one of them failed, authentik would stay down. + # Both images are already on the host, so no registry login is + # needed. The containers role then finds nothing to change. + - name: Install the PostgreSQL 18 compose file + ansible.builtin.copy: + src: "{{ playbook_dir }}/../containers/authentik/docker-compose.yml" + dest: /opt/containers/authentik/docker-compose.yml + owner: root + group: root + mode: "0755" + + - name: Start authentik on PostgreSQL 18 + ansible.builtin.command: + argv: [docker, compose, --project-directory, /opt/containers/authentik, up, --detach, --wait, --wait-timeout, "300"] + changed_when: true + + rescue: + # Cleanup must not stop the rollback, so its errors are reported + # in the final failure instead. + - name: Remove the restore container + community.docker.docker_container: + name: authentik-postgres18-restore + state: absent + register: authentik_cleanup_container + ignore_errors: true + + - name: Put back the PostgreSQL 16 compose file + ansible.builtin.copy: + src: /opt/ansible/backups/authentik-docker-compose.pg16.yml + dest: /opt/containers/authentik/docker-compose.yml + remote_src: true + owner: root + group: root + mode: "0755" + + - name: Start the PostgreSQL 16 stack again + ansible.builtin.command: + argv: [docker, compose, --project-directory, /opt/containers/authentik, up, --detach, --wait, --wait-timeout, "300"] + changed_when: true + + # After the 16 stack is up, so no container still uses the volume. + - name: Remove the half-restored PostgreSQL 18 volume + community.docker.docker_volume: + name: authentik_postgres + state: absent + register: authentik_cleanup_volume + ignore_errors: true + + - name: Fail the migration + ansible.builtin.fail: + msg: >- + The PostgreSQL 18 upgrade failed at "{{ ansible_failed_task.name }}" and authentik + was started again on PostgreSQL 16: {{ ansible_failed_result.stderr | default(ansible_failed_result.msg | default('')) }} + {{ 'Removing authentik-postgres18-restore also failed: ' ~ authentik_cleanup_container.msg + if authentik_cleanup_container is failed else '' }} + {{ 'Removing authentik_postgres also failed, remove it by hand before deploying again: ' + ~ authentik_cleanup_volume.msg if authentik_cleanup_volume is failed else '' }} diff --git a/web3/containers/authentik/docker-compose.yml b/web3/containers/authentik/docker-compose.yml index bf8dbb9b..316609c1 100644 --- a/web3/containers/authentik/docker-compose.yml +++ b/web3/containers/authentik/docker-compose.yml @@ -1,7 +1,7 @@ --- services: database: - image: docker.io/library/postgres:16-alpine@sha256:81e77a4833487b8132fe4b00bedd39dc3f74b8641e387f19312a48f7de218c2e + image: postgres:18.7-alpine3.23@sha256:60aac580d8dbe3093c097bb1d11f9e62999814d3a1da2c1aaf729c0b7db117f2 restart: unless-stopped healthcheck: test: ["CMD-SHELL", "pg_isready -d $${POSTGRES_DB} -U $${POSTGRES_USER}"] @@ -9,8 +9,11 @@ services: interval: 30s retries: 5 timeout: 5s + environment: + # PG18: version-specific PGDATA; volume mounts at /var/lib/postgresql + PGDATA: /var/lib/postgresql/18/docker volumes: - - database:/var/lib/postgresql/data + - postgres:/var/lib/postgresql env_file: - .env networks: @@ -57,7 +60,7 @@ services: - authentik volumes: - database: + postgres: driver: local networks: