diff --git a/.github/workflows/update_service.yml b/.github/workflows/update_service.yml new file mode 100644 index 00000000..debdd8d6 --- /dev/null +++ b/.github/workflows/update_service.yml @@ -0,0 +1,73 @@ +name: Update Service + +on: + repository_dispatch: + types: [deploy] + +concurrency: + # with this group each combination - can run in parallel. + # Requests to the same - get queued + group: deploy-${{ github.event.client_payload.env }}-${{ github.event.client_payload.service }} + cancel-in-progress: false + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Validate inputs + env: + ENV: ${{ github.event.client_payload.env }} + SERVICE: ${{ github.event.client_payload.service }} + TAG: ${{ github.event.client_payload.tag }} + run: | + case "$ENV" in dev|prod) ;; *) echo "Invalid env: $ENV"; exit 1 ;; esac + case "$SERVICE" in + oonimeasurements|ooniprobe|oonifindings|ooniauth|oonirun) ;; + *) echo "Invalid service: $SERVICE"; exit 1 ;; + esac + if [ -z "$TAG" ]; then echo "Missing tag"; exit 1; fi + + - uses: actions/checkout@v4 + with: + token: ${{ secrets.GH_PAT }} # needs repo write to commit back + ref: nomad-experiment # TODO remove on actual deploy + + - name: Update service_tags.auto.tfvars.json + env: + ENV: ${{ github.event.client_payload.env }} + SERVICE: ${{ github.event.client_payload.service }} + TAG: ${{ github.event.client_payload.tag }} + run: | + FILE="tf/environments/${ENV}/service_tags.auto.tfvars.json" + [ -f "$FILE" ] || echo '{}' > "$FILE" + jq --arg key "${SERVICE}_tag" --arg tag "$TAG" \ + '.[$key] = $tag' "$FILE" > tmp.json && mv tmp.json "$FILE" + + - name: Commit and push + env: + ENV: ${{ github.event.client_payload.env }} + SERVICE: ${{ github.event.client_payload.service }} + TAG: ${{ github.event.client_payload.tag }} + run: | + git config user.name "github-actions" + git config user.email "github-actions@github.com" + git add "tf/environments/${ENV}/service_tags.auto.tfvars.json" + git commit -m "deploy: ${SERVICE} -> ${TAG} (${ENV})" + git push + + - uses: hashicorp/setup-terraform@v3 + + - name: Terraform init + working-directory: tf/environments/${{ github.event.client_payload.env }} + run: terraform init + + - name: Terraform apply + working-directory: tf/environments/${{ github.event.client_payload.env }} + env: + NOMAD_ADDR: ${{ secrets.NOMAD_ADDR }} + NOMAD_TOKEN: ${{ secrets.NOMAD_TOKEN }} + AWS_ACCESS_KEY_ID: ${{ secrets.OONIDEVOPS_AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.OONIDEVOPS_AWS_SECRET_ACCESS_KEY }} + SERVICE: ${{ github.event.client_payload.service }} + run: | + terraform apply -auto-approve -target="nomad_job.${SERVICE}" diff --git a/ansible/deploy-nomad.yml b/ansible/deploy-nomad.yml new file mode 100644 index 00000000..2d35e43f --- /dev/null +++ b/ansible/deploy-nomad.yml @@ -0,0 +1,13 @@ +--- +# This will be in dev mode for the moment, just as a POC +- name: Deploy Nomad agent + hosts: anonc.dev.ooni.io # Use this host in the meanwhile + become: true + vars: + ssl_domains: + - "{{ inventory_hostname }}" + roles: + # - bootstrap + - nginx + - dehydrated + - nomad diff --git a/ansible/roles/bootstrap/tasks/main.yml b/ansible/roles/bootstrap/tasks/main.yml index 2b65c611..e680663b 100644 --- a/ansible/roles/bootstrap/tasks/main.yml +++ b/ansible/roles/bootstrap/tasks/main.yml @@ -68,7 +68,7 @@ - name: install systemd-resolved tags: resolved - ignore_errors: yes # some ubuntu hosts do not have the systemd-resolved package + ignore_errors: true # some ubuntu hosts do not have the systemd-resolved package ansible.builtin.apt: install_recommends: no cache_valid_time: 86400 diff --git a/ansible/roles/nomad/defaults/main.yml b/ansible/roles/nomad/defaults/main.yml new file mode 100644 index 00000000..12987f22 --- /dev/null +++ b/ansible/roles/nomad/defaults/main.yml @@ -0,0 +1,8 @@ +--- +nomad_webui_username: "admin" +nomad_webui_password: "{{ lookup('amazon.aws.aws_ssm', '/oonidevops/secrets/nomad_webui_password', profile='oonidevops_user_dev') }}" +nomad_htpasswd_path: "/etc/nginx/nomad.htpasswd" + +# Choose one that it's outside of system users ranges (0-100) +# and human user ranges > 1000 +podman_uid: 64501 diff --git a/ansible/roles/nomad/handlers/main.yml b/ansible/roles/nomad/handlers/main.yml new file mode 100644 index 00000000..94777c6a --- /dev/null +++ b/ansible/roles/nomad/handlers/main.yml @@ -0,0 +1,32 @@ +- name: test nginx config + command: /usr/sbin/nginx -t -c /etc/nginx/nginx.conf + listen: + - restart nginx + - reload nginx + +- name: restart nginx + service: + name: nginx + state: restarted + +- name: reload nginx + service: + name: nginx + state: reloaded + +- name: reload nftables + tags: nftables + ansible.builtin.systemd_service: + name: nftables + state: reloaded + +- name: restart docker + tags: docker + ansible.builtin.systemd_service: + name: docker + state: restarted + +- name: restart nomad + ansible.builtin.systemd_service: + name: nomad + state: restarted \ No newline at end of file diff --git a/ansible/roles/nomad/tasks/main.yml b/ansible/roles/nomad/tasks/main.yml new file mode 100644 index 00000000..c34738ff --- /dev/null +++ b/ansible/roles/nomad/tasks/main.yml @@ -0,0 +1,294 @@ +--- +# For prometheus scrape requests +- name: Flush all handlers + meta: flush_handlers + +- name: Install acl package (required for Ansible become to unprivileged users) + tags: nomad + ansible.builtin.apt: + name: acl + state: present + update_cache: true + +- name: Allow Nginx HTTPS traffic for Nomad (Port 4645) + become: true + tags: nomad + blockinfile: + path: /etc/ooni/nftables/tcp/4645.nft + create: true + block: | + add rule inet filter input tcp dport 4645 counter accept comment "nomad web ui reverse proxy" + notify: + - Reload nftables + +- name: Install Docker prerequisites + tags: docker + ansible.builtin.apt: + name: + - ca-certificates + - curl + - gpg + - coreutils + state: present + +- name: Create keyring directory + tags: [docker, nomad] + ansible.builtin.file: + path: /usr/share/keyrings + state: directory + mode: '0755' + +- name: Download Docker GPG key + tags: docker + ansible.builtin.get_url: + url: https://download.docker.com/linux/debian/gpg + dest: /tmp/docker.asc + mode: '0644' + +- name: Dearmor Docker GPG key + tags: docker + ansible.builtin.command: + cmd: gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg /tmp/docker.asc + args: + creates: /usr/share/keyrings/docker-archive-keyring.gpg + +- name: Add Docker apt repository + tags: docker + ansible.builtin.apt_repository: + repo: "deb [signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/{{ ansible_distribution | lower }} {{ ansible_distribution_release }} stable" + filename: docker + state: present + +- name: Install Docker + tags: docker + ansible.builtin.apt: + name: + - docker-ce + - docker-ce-cli + - containerd.io + state: present + update_cache: yes + +- name: Ensure Docker service is enabled and started + tags: docker + ansible.builtin.service: + name: docker + state: started + enabled: yes + +- name: Download HashiCorp GPG key + tags: nomad + ansible.builtin.get_url: + url: https://apt.releases.hashicorp.com/gpg + dest: /tmp/hashicorp.asc + mode: '0644' + +- name: Dearmor HashiCorp GPG key + tags: nomad + ansible.builtin.command: + cmd: gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg /tmp/hashicorp.asc + args: + creates: /usr/share/keyrings/hashicorp-archive-keyring.gpg + +- name: Add HashiCorp apt repository + tags: nomad + ansible.builtin.apt_repository: + repo: "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com {{ ansible_distribution_release }} main" + filename: hashicorp + state: present + +- name: Install Nomad + tags: nomad + ansible.builtin.apt: + name: nomad + state: present + update_cache: yes + +- name: Create Nomad plugins directory + tags: nomad + ansible.builtin.file: + path: /opt/nomad/plugins + state: directory + owner: nomad + group: nomad + mode: "0755" + +- name: Install podman + tags: nomad + ansible.builtin.apt: + name: podman + state: present + update_cache: yes + +- name: Create podman-svc group with fixed GID + tags: nomad + ansible.builtin.group: + name: podman-svc + gid: "{{ podman_uid }}" + system: yes + +- name: Create podman-svc user with fixed UID + tags: nomad + ansible.builtin.user: + name: podman-svc + uid: "{{ podman_uid }}" + group: podman-svc + system: yes + shell: /usr/sbin/nologin + create_home: yes + +# The following two actions fix a uid remapping bug +# Error processing tar file(exit status 1): potentially insufficient UIDs or GIDs available in user namespace (requested 0:42 for /etc/gshadow): Check /etc/subuid and /etc/subgid: lchown /etc/gshadow: invalid argument +- name: Set subuid range for podman-svc + tags: nomad + ansible.builtin.lineinfile: + path: /etc/subuid + line: "podman-svc:100000:65536" + create: yes + regexp: "^podman-svc:" + +- name: Set subgid range for podman-svc + tags: nomad + ansible.builtin.lineinfile: + path: /etc/subgid + line: "podman-svc:100000:65536" + create: yes + regexp: "^podman-svc:" + +- name: Enable linger for podman-svc + tags: nomad + ansible.builtin.command: loginctl enable-linger podman-svc + changed_when: false + + +# Required for podman support in nomad +# https://developer.hashicorp.com/nomad/plugins/drivers/podman +- name: Enable rootless podman socket + tags: nomad + ansible.builtin.systemd: + name: podman.socket + scope: user + enabled: true + state: started + become: true + become_user: podman-svc + environment: + XDG_RUNTIME_DIR: "/run/user/{{ podman_uid }}" + +- name: Install podman-driver for Nomad + tags: nomad + ansible.builtin.apt: + name: nomad-driver-podman + state: present + update_cache: yes + +- name: Find nomad-driver-podman binary path + tags: nomad + ansible.builtin.command: which nomad-driver-podman + register: podman_driver_path + changed_when: false + +- name: Symlink podman driver into Nomad plugins directory + tags: nomad + ansible.builtin.file: + src: "{{ podman_driver_path.stdout }}" + dest: /opt/nomad/plugins/nomad-driver-podman + state: link + owner: nomad + group: nomad + +- name: Create Nomad configuration directory + tags: nomad + ansible.builtin.file: + path: /etc/nomad.d + state: directory + mode: '0755' + owner: root + group: root + +- name: Setup Nomad in dev mod + tags: nomad + ansible.builtin.copy: + dest: /etc/nomad.d/nomad.hcl # Main config file + content: | + bind_addr = "127.0.0.1" + data_dir = "/opt/nomad/data" + plugin_dir = "/opt/nomad/plugins" + advertise { + http = "127.0.0.1" + rpc = "127.0.0.1" + serf = "127.0.0.1" + } + + server { + enabled = true + bootstrap_expect = 1 + } + + client { + enabled = true + servers = ["127.0.0.1"] + } + + acl { + enabled = true + } + + tls { + http = false + rpc = false + } + + # TODO change this to use rootless containers instead + plugin "nomad-driver-podman" { + config { + socket_path = "unix:///run/user/{{ podman_uid }}/podman/podman.sock" + } + } + mode: '0644' + owner: root + group: root + notify: restart nomad + +- name: Create basic auth credentials file for Nomad WebUI + tags: nomad + community.general.htpasswd: + path: "{{ nomad_htpasswd_path }}" + name: "{{ nomad_webui_username }}" + password: "{{ nomad_webui_password }}" + owner: root + group: nginx + mode: '0640' + +- name: Configure Nginx reverse proxy for Nomad + tags: nomad + ansible.builtin.template: + src: nginx-nomad.j2 + dest: /etc/nginx/sites-enabled/02-nomad.conf + owner: root + group: root + mode: '0644' + notify: reload nginx + +- name: Configure Nginx reverse proxy for the API + tags: nomad + ansible.builtin.template: + src: nginx-api.j2 + dest: /etc/nginx/sites-enabled/03-api.conf + owner: root + group: root + mode: '0644' + notify: reload nginx + +- name: Remove a file + tags: nomad + ansible.builtin.file: + path: /etc/nginx/sites-enabled/01-prometheus + state: absent + +- name: Ensure Nomad service is enabled and started + tags: nomad + ansible.builtin.service: + name: nomad + state: started + enabled: yes diff --git a/ansible/roles/nomad/templates/nginx-api.j2 b/ansible/roles/nomad/templates/nginx-api.j2 new file mode 100644 index 00000000..5e046c01 --- /dev/null +++ b/ansible/roles/nomad/templates/nginx-api.j2 @@ -0,0 +1,52 @@ +# ansible-managed in ooni/devops.git +upstream oonimeasurements { + server 10.0.0.123:8000; +} + +server { + listen 80; + listen 443 ssl http2; + # this is the correct name, change it in the actual deployment + # server_name api.{{ environment }}.ooni.io; + server_name anonc.dev.ooni.io; + + include /etc/nginx/ssl_intermediate.conf; + ssl_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/fullchain.pem; + ssl_certificate_key /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem; + ssl_trusted_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem; + + # Redirect plain HTTP to HTTPS + if ($scheme = http) { + return 301 https://$host$request_uri; + } + + location /metrics/node_exporter { + auth_basic "Administrator’s Area"; + auth_basic_user_file /etc/ooni/prometheus_passwd; + + proxy_pass http://127.0.0.1:9100/metrics; + + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + # All paths currently route to oonimeasurements. + # Add more location blocks below as you bring up additional services. + location / { + proxy_pass http://oonimeasurements; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + # Example of how you'll extend this: + # location /something { + # proxy_pass http://otherservice; + # proxy_set_header Host $host; + # proxy_set_header X-Real-IP $remote_addr; + # proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + # proxy_set_header X-Forwarded-Proto $scheme; + # } +} diff --git a/ansible/roles/nomad/templates/nginx-nomad.j2 b/ansible/roles/nomad/templates/nginx-nomad.j2 new file mode 100644 index 00000000..5a19385c --- /dev/null +++ b/ansible/roles/nomad/templates/nginx-nomad.j2 @@ -0,0 +1,43 @@ +# ansible-managed in ooni/devops.git +# Nomad-specific settings found here: +# https://developer.hashicorp.com/nomad/docs/deploy/clusters/reverse-proxy-ui#extend-connection-timeout +server { + listen 4645 ssl http2; + server_name {{ inventory_hostname }}; + include /etc/nginx/ssl_intermediate.conf; + ssl_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/fullchain.pem; + ssl_certificate_key /var/lib/dehydrated/certs/{{ inventory_hostname }}/privkey.pem; + ssl_trusted_certificate /var/lib/dehydrated/certs/{{ inventory_hostname }}/chain.pem; + + location / { + auth_basic "Nomad Admin Area"; + auth_basic_user_file {{ nomad_htpasswd_path }}; + + proxy_pass http://127.0.0.1:4646; + proxy_http_version 1.1; + + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + + # Nomad blocking queries will remain open for a default of 5 minutes. + # Increase the proxy timeout to accommodate this timeout with an + # additional grace period. + proxy_read_timeout 319s; + + # Nomad log streaming uses streaming HTTP requests. In order to + # synchronously stream logs from Nomad to NGINX to the browser + # proxy buffering needs to be turned off. + proxy_buffering off; + + # The Upgrade and Connection headers are used to establish + # a WebSockets connection. + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + # The default Origin header will be the proxy address, which + # will be rejected by Nomad. It must be rewritten to be the + # host address instead. + proxy_set_header Origin "${scheme}://$host"; + } +} diff --git a/tf/environments/dev/jobs/base_job.tpl b/tf/environments/dev/jobs/base_job.tpl new file mode 100644 index 00000000..d9a85a5f --- /dev/null +++ b/tf/environments/dev/jobs/base_job.tpl @@ -0,0 +1,66 @@ +job "${service_name}" { + type = "service" + group "${service_name}" { + count = ${desired_count} + network { + port "${service_name}" { + static = ${port} + to = 80 + } + } + service { + name = "${service_name}" + port = "${service_name}" + provider = "nomad" + + check { + type = "http" + path = "/health" + interval = "15s" + timeout = "5s" + } + } + + task "${service_name}-task" { + driver = "docker" + config { + image = "${docker_image}" + ports = ["${service_name}"] + } + +%{ if length(secret_keys) > 0 } + template { + data = <<-EOH + {{ with nomadVar "nomad/jobs/${service_name}" }} +%{ for key in secret_keys ~} + ${key}={{ .${key} }} +%{ endfor ~} + {{ end }} + EOH + destination = "secrets/env.txt" + env = true + } +%{ endif } + + template { + data = <<-EOH + {{ range (nomadService "valkey-svc") }} + VALKEY_URL=valkey://{{ .Address }}:{{ .Port }} + {{ end }} + EOH + destination = "local/valkey.env" + env = true + } + + env { +%{ for key, value in env_vars ~} + ${key} = "${value}" +%{ endfor ~} + } + + resources { + memory = ${task_memory} + } + } + } +} diff --git a/tf/environments/dev/jobs/valkey.hcl b/tf/environments/dev/jobs/valkey.hcl new file mode 100644 index 00000000..3544bddb --- /dev/null +++ b/tf/environments/dev/jobs/valkey.hcl @@ -0,0 +1,47 @@ +job "valkey" { + type = "service" + group "valkey" { + count = 1 + + network { + port "valkey" { + to = 6379 + } + } + + update { + max_parallel = 1 + canary = 1 + min_healthy_time = "10s" + healthy_deadline = "3m" + progress_deadline = "5m" + auto_revert = true + auto_promote = true + } + + task "valkey-task" { + driver = "docker" + config { + image = "docker.io/valkey/valkey:8" + ports = ["valkey"] + } + + service { + name = "valkey-svc" + port = "valkey" + provider = "nomad" + address_mode = "driver" + + check { + type = "tcp" + interval = "10s" + timeout = "2s" + } + } + + resources { + memory = 256 + } + } + } +} diff --git a/tf/environments/dev/main.tf b/tf/environments/dev/main.tf index 17641442..f74317d9 100644 --- a/tf/environments/dev/main.tf +++ b/tf/environments/dev/main.tf @@ -21,6 +21,22 @@ locals { ] } + +data "aws_ssm_parameter" "nomad_address" { + name = "/oonidevops/secrets/nomad_address" + with_decryption = true +} + +data "aws_ssm_parameter" "nomad_token" { + name = "/oonidevops/secrets/nomad_token" + with_decryption = true +} + +data "aws_ssm_parameter" "clickhouse_readonly_url_test" { + name = "/oonidevops/secrets/clickhouse_readonly_test_url" + with_decryption = true +} + ## AWS Setup provider "aws" { @@ -719,8 +735,21 @@ module "ooni_clickhouse_proxy" { from_port = 9000, to_port = 9002, // for several clickhouse instances protocol = "tcp", - cidr_blocks = concat(module.network.vpc_subnet_private[*].cidr_block, ["${module.ooni_fastpath.aws_instance_private_ip}/32", "${module.ooni_fastpath.aws_instance_public_ip}/32"], - ["${module.ooniapi_testlists.aws_instance_private_ip}/32", "${module.ooniapi_testlists.aws_instance_public_ip}/32"]), + cidr_blocks = concat( + module.network.vpc_subnet_private[*].cidr_block, + [ + "${module.ooni_fastpath.aws_instance_private_ip}/32", + "${module.ooni_fastpath.aws_instance_public_ip}/32" + ], + [ + "${module.ooniapi_testlists.aws_instance_private_ip}/32", + "${module.ooniapi_testlists.aws_instance_public_ip}/32" + ], + [ + "${module.ooni_anonc.aws_instance_private_ip}/32", + "${module.ooni_anonc.aws_instance_public_ip}/32", + ] + ), }, { // For the prometheus proxy: from_port = 9200, @@ -1422,6 +1451,11 @@ module "ooni_anonc" { protocol = "tcp", cidr_blocks = ["0.0.0.0/0"], }, { + from_port = 4645, # for the nomad webui + to_port = 4645, + protocol = "tcp", + cidr_blocks = ["0.0.0.0/0"], + }, { from_port = 9100, # for node exporter metrics to_port = 9100, protocol = "tcp" diff --git a/tf/environments/dev/service_tags.auto.tfvars.json b/tf/environments/dev/service_tags.auto.tfvars.json new file mode 100644 index 00000000..213ead24 --- /dev/null +++ b/tf/environments/dev/service_tags.auto.tfvars.json @@ -0,0 +1,3 @@ +{ + "oonimeasurements_tag": "20260520-fe913ea8" +}