feat(release): Move Docker image to ECR and build it on EC2

.github/workflows/cd-production.yml

@@ -3,7 +3,7 @@ name: Deploy Kaapi Guardrails to EC2 Production
 on:
   push:
     tags:
-      - "v*" # Deploy only when tags like v1.0.0, v2.1.0, etc., are created
+      - "v[0-9]+.[0-9]+.[0-9]+" # Deploy only when tags like v1.0.0, v2.1.0, etc., are created
 
 concurrency:
   group: guardrail-production-ec2-deploy
@@ -19,19 +19,40 @@ jobs:
       contents: read
 
     steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v7
+        with:
+          ref: ${{ github.event.release.tag_name }}
+
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v6
         with:
           role-to-assume: ${{ secrets.EC2_DEPLOY_ROLE_ARN }}
           aws-region: ${{ secrets.AWS_REGION }}
 
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Build and Push Docker Image
+        env:
+          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          REPOSITORY: ${{ secrets.AWS_RESOURCE_PREFIX }}-prod-repo
+          IMAGE_TAG: ${{ github.ref_name }}
+        run: |
+          docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG ./backend
+          docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG
+
       - name: Trigger deploy on EC2 via SSM
         id: ssm
         env:
           INSTANCE_ID: ${{ secrets.EC2_INSTANCE_ID }}
           BUILD_DIRECTORY: ${{ secrets.BUILD_DIRECTORY }}
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: ${{ secrets.AWS_RESOURCE_PREFIX }}-prod-repo
+          IMAGE_TAG: ${{ github.ref_name }}
         run: |
-          DEPLOY_CMD="cd ${BUILD_DIRECTORY} && git fetch --tags origin && git checkout --force ${{ github.sha }} && docker compose build && docker compose run --rm --entrypoint '' backend uv run --frozen alembic upgrade head && docker compose up -d --remove-orphans && docker image prune -f && docker builder prune -af"
+          DEPLOY_CMD="cd ${BUILD_DIRECTORY} && git fetch --tags origin && git checkout --force ${IMAGE_TAG} && aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | docker login --username AWS --password-stdin ${ECR_REGISTRY} && export ECR_REGISTRY=${ECR_REGISTRY} ECR_REPOSITORY=${ECR_REPOSITORY} IMAGE_TAG=${IMAGE_TAG} && docker compose pull backend && docker compose --profile prestart run --rm prestart && docker compose up -d --remove-orphans && docker image prune -f"
 
           CMD_ID=$(aws ssm send-command \
             --instance-ids "$INSTANCE_ID" \
@@ -49,6 +70,14 @@ jobs:
           INSTANCE_ID: ${{ secrets.EC2_INSTANCE_ID }}
           CMD_ID: ${{ steps.ssm.outputs.cmd_id }}
         run: |
+          get_output() {
+            aws ssm get-command-invocation \
+              --command-id "$CMD_ID" \
+              --instance-id "$INSTANCE_ID" \
+              --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \
+              --output json
+          }
+
           for i in {1..20}; do
             STATUS=$(aws ssm get-command-invocation \
               --command-id "$CMD_ID" \
@@ -60,37 +89,19 @@ jobs:
 
             if [ "$STATUS" = "Success" ]; then
               echo "Deployment completed successfully."
-
-              aws ssm get-command-invocation \
-                --command-id "$CMD_ID" \
-                --instance-id "$INSTANCE_ID" \
-                --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \
-                --output json
-
+              get_output
               exit 0
             fi
 
             if [ "$STATUS" = "Failed" ] || [ "$STATUS" = "Cancelled" ] || [ "$STATUS" = "TimedOut" ]; then
               echo "Deployment failed."
-
-              aws ssm get-command-invocation \
-                --command-id "$CMD_ID" \
-                --instance-id "$INSTANCE_ID" \
-                --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \
-                --output json
-
+              get_output
               exit 1
             fi
 
             sleep 25
           done
 
           echo "Deployment timed out after waiting too long."
-
-          aws ssm get-command-invocation \
-            --command-id "$CMD_ID" \
-            --instance-id "$INSTANCE_ID" \
-            --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \
-            --output json
-
-          exit 1
+          get_output
+          exit 1

.github/workflows/cd-staging.yml

@@ -18,19 +18,37 @@ jobs:
       contents: read
 
     steps:
+      - name: Checkout the repo
+        uses: actions/checkout@v7
+
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v6
         with:
           role-to-assume: ${{ secrets.EC2_DEPLOY_ROLE_ARN }}
           aws-region: ${{ secrets.AWS_REGION }}
 
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@v2
+
+      - name: Build and Push Docker Image
+        env:
+          REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          REPOSITORY: ${{ secrets.AWS_RESOURCE_PREFIX }}-staging-repo
+        run: |
+          docker build -t $REGISTRY/$REPOSITORY:latest ./backend
+          docker push $REGISTRY/$REPOSITORY:latest
+
       - name: Trigger deploy on EC2 via SSM
         id: ssm
         env:
           INSTANCE_ID: ${{ secrets.STAGING_EC2_INSTANCE_ID }}
           BUILD_DIRECTORY: ${{ secrets.BUILD_DIRECTORY }}
+          ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }}
+          ECR_REPOSITORY: ${{ secrets.AWS_RESOURCE_PREFIX }}-staging-repo
+          IMAGE_TAG: latest
         run: |
-          DEPLOY_CMD='sudo -iu ec2-user bash -lc \"cd '"${BUILD_DIRECTORY}"' && git fetch origin && git reset --hard origin/main && docker compose build && docker compose run --rm --entrypoint \\\"\\\" backend uv run --frozen alembic upgrade head && docker compose up -d --remove-orphans && docker image prune -f && docker builder prune -af\"'
+          DEPLOY_CMD='sudo -iu ec2-user bash -lc \"cd '"${BUILD_DIRECTORY}"' && git fetch origin && git reset --hard origin/main && aws ecr get-login-password --region '"${{ secrets.AWS_REGION }}"' | docker login --username AWS --password-stdin '"${ECR_REGISTRY}"' && export ECR_REGISTRY='"${ECR_REGISTRY}"' ECR_REPOSITORY='"${ECR_REPOSITORY}"' IMAGE_TAG='"${IMAGE_TAG}"' && docker compose pull backend && docker compose --profile prestart run --rm prestart && docker compose up -d --remove-orphans && docker image prune -f\"'
 
           CMD_ID=$(aws ssm send-command \
             --instance-ids "$INSTANCE_ID" \
@@ -48,6 +66,14 @@ jobs:
           INSTANCE_ID: ${{ secrets.STAGING_EC2_INSTANCE_ID }}
           CMD_ID: ${{ steps.ssm.outputs.cmd_id }}
         run: |
+          get_output() {
+            aws ssm get-command-invocation \
+              --command-id "$CMD_ID" \
+              --instance-id "$INSTANCE_ID" \
+              --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \
+              --output json
+          }
+
           for i in {1..20}; do
             STATUS=$(aws ssm get-command-invocation \
               --command-id "$CMD_ID" \
@@ -59,37 +85,19 @@ jobs:
 
             if [ "$STATUS" = "Success" ]; then
               echo "Deployment completed successfully."
-
-              aws ssm get-command-invocation \
-                --command-id "$CMD_ID" \
-                --instance-id "$INSTANCE_ID" \
-                --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \
-                --output json
-
+              get_output
               exit 0
             fi
 
             if [ "$STATUS" = "Failed" ] || [ "$STATUS" = "Cancelled" ] || [ "$STATUS" = "TimedOut" ]; then
               echo "Deployment failed."
-
-              aws ssm get-command-invocation \
-                --command-id "$CMD_ID" \
-                --instance-id "$INSTANCE_ID" \
-                --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \
-                --output json
-
+              get_output
               exit 1
             fi
 
             sleep 25
           done
 
           echo "Deployment timed out after waiting too long."
-
-          aws ssm get-command-invocation \
-            --command-id "$CMD_ID" \
-            --instance-id "$INSTANCE_ID" \
-            --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \
-            --output json
-
-          exit 1
+          get_output
+          exit 1

.github/workflows/continuous-integration.yml

@@ -9,6 +9,8 @@ on:
 jobs:
   checks:
     runs-on: ubuntu-latest
+    env:
+      NLTK_DISABLE_IMPORT_SECURITY: "1"
     services:
       postgres:
         image: postgres:16

backend/Dockerfile

@@ -47,6 +47,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \
 # Install pinned spaCy model in the final environment used at runtime.
 RUN python -m pip install --no-deps "${SPACY_MODEL_WHEEL_URL}"
 
+ENV NLTK_DISABLE_IMPORT_SECURITY=1
+
 # Set HuggingFace cache directory
 ENV HF_HOME=/app/hf_cache
 

backend/scripts/install_guardrails_from_hub.sh

@@ -8,6 +8,8 @@ GUARDRAILS_HUB_API_KEY="${GUARDRAILS_HUB_API_KEY:-}"
 ENABLE_METRICS="${ENABLE_METRICS:-false}"
 ENABLE_REMOTE_INFERENCING="${ENABLE_REMOTE_INFERENCING:-true}"
 
+export NLTK_DISABLE_IMPORT_SECURITY=1
+
 BACKEND_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
 MANIFEST_FILE="${1:-$BACKEND_DIR/app/core/validators/validators.json}"
 
@@ -16,18 +18,43 @@ if [[ ! -f "$MANIFEST_FILE" ]]; then
   exit 1
 fi
 
+retry() {
+  local attempts="$1"
+  shift
+  local delay=5
+  local n=1
+
+  until "$@"; do
+    if (( n >= attempts )); then
+      return 1
+    fi
+    echo "Attempt ${n}/${attempts} failed; retrying in ${delay}s..."
+    sleep "$delay"
+    n=$(( n + 1 ))
+    delay=$(( delay * 2 ))
+  done
+}
+
 #######################################
 # Configure Guardrails (non-interactive)
 #######################################
 
 if [[ -n "$GUARDRAILS_HUB_API_KEY" ]]; then
-  echo "Configuring Guardrails CLI..."
-  guardrails configure \
-    --token "$GUARDRAILS_HUB_API_KEY" \
-    $( [[ "$ENABLE_METRICS" == "true" ]] && echo "--enable-metrics" || echo "--disable-metrics" ) \
-    $( [[ "$ENABLE_REMOTE_INFERENCING" == "true" ]] && echo "--enable-remote-inferencing" || echo "--disable-remote-inferencing" )
+  echo "Writing Guardrails configuration..."
+
+  ANON_ID="$( { python3 -c 'import uuid; print(uuid.uuid4())' \
+    || python -c 'import uuid; print(uuid.uuid4())'; } 2>/dev/null \
+    || echo "00000000-0000-0000-0000-000000000000")"
+
+  cat > "${HOME}/.guardrailsrc" <<EOF
+id=${ANON_ID}
+token=${GUARDRAILS_HUB_API_KEY}
+enable_metrics=$( [[ "$ENABLE_METRICS" == "true" ]] && echo true || echo false )
+use_remote_inferencing=$( [[ "$ENABLE_REMOTE_INFERENCING" == "true" ]] && echo true || echo false )
+EOF
+  chmod 600 "${HOME}/.guardrailsrc"
 else
-  echo "GUARDRAILS_HUB_API_KEY is not set; skipping Guardrails CLI configure."
+  echo "GUARDRAILS_HUB_API_KEY is not set; skipping Guardrails configuration."
 fi
 
 
@@ -51,7 +78,7 @@ fi
 
 for SRC in $HUB_SOURCES; do
   echo "Installing Guardrails hub validator: $SRC"
-  if ! guardrails hub install "$SRC"; then
+  if ! retry 4 guardrails hub install "$SRC"; then
     if [[ -z "$GUARDRAILS_HUB_API_KEY" ]]; then
       echo "Skipping hub validator install for $SRC because GUARDRAILS_HUB_API_KEY is not set."
       continue

docker-compose.yml

@@ -9,7 +9,7 @@ secrets:
 services:
 
   prestart:
-    image: '${DOCKER_IMAGE_BACKEND?Variable not set}:${TAG-latest}'
+    image: '${ECR_REGISTRY}/${ECR_REPOSITORY}:${IMAGE_TAG:-latest}'
     container_name: kaapi-guardrails-prestart
     entrypoint: []
     build:
@@ -25,7 +25,7 @@ services:
     profiles: ["prestart"]
 
   backend:
-    image: '${DOCKER_IMAGE_BACKEND?Variable not set}:${TAG-latest}'
+    image: '${ECR_REGISTRY}/${ECR_REPOSITORY}:${IMAGE_TAG:-latest}'
     container_name: kaapi-guardrails-backend
     restart: always
     build: