diff --git a/.github/workflows/cd-production.yml b/.github/workflows/cd-production.yml index ec019ad..417d940 100644 --- a/.github/workflows/cd-production.yml +++ b/.github/workflows/cd-production.yml @@ -3,7 +3,7 @@ name: Deploy Kaapi Guardrails to EC2 Production on: push: tags: - - "v*" # Deploy only when tags like v1.0.0, v2.1.0, etc., are created + - "v[0-9]+.[0-9]+.[0-9]+" # Deploy only when tags like v1.0.0, v2.1.0, etc., are created concurrency: group: guardrail-production-ec2-deploy @@ -19,19 +19,40 @@ jobs: contents: read steps: + - name: Checkout the repo + uses: actions/checkout@v7 + with: + ref: ${{ github.event.release.tag_name }} + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v6 with: role-to-assume: ${{ secrets.EC2_DEPLOY_ROLE_ARN }} aws-region: ${{ secrets.AWS_REGION }} + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build and Push Docker Image + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + REPOSITORY: ${{ secrets.AWS_RESOURCE_PREFIX }}-prod-repo + IMAGE_TAG: ${{ github.ref_name }} + run: | + docker build -t $REGISTRY/$REPOSITORY:$IMAGE_TAG ./backend + docker push $REGISTRY/$REPOSITORY:$IMAGE_TAG + - name: Trigger deploy on EC2 via SSM id: ssm env: INSTANCE_ID: ${{ secrets.EC2_INSTANCE_ID }} BUILD_DIRECTORY: ${{ secrets.BUILD_DIRECTORY }} + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: ${{ secrets.AWS_RESOURCE_PREFIX }}-prod-repo + IMAGE_TAG: ${{ github.ref_name }} run: | - DEPLOY_CMD="cd ${BUILD_DIRECTORY} && git fetch --tags origin && git checkout --force ${{ github.sha }} && docker compose build && docker compose run --rm --entrypoint '' backend uv run --frozen alembic upgrade head && docker compose up -d --remove-orphans && docker image prune -f && docker builder prune -af" + DEPLOY_CMD="cd ${BUILD_DIRECTORY} && git fetch --tags origin && git checkout --force ${IMAGE_TAG} && aws ecr get-login-password --region ${{ secrets.AWS_REGION }} | docker login --username AWS --password-stdin ${ECR_REGISTRY} && export ECR_REGISTRY=${ECR_REGISTRY} ECR_REPOSITORY=${ECR_REPOSITORY} IMAGE_TAG=${IMAGE_TAG} && docker compose pull backend && docker compose --profile prestart run --rm prestart && docker compose up -d --remove-orphans && docker image prune -f" CMD_ID=$(aws ssm send-command \ --instance-ids "$INSTANCE_ID" \ @@ -49,6 +70,14 @@ jobs: INSTANCE_ID: ${{ secrets.EC2_INSTANCE_ID }} CMD_ID: ${{ steps.ssm.outputs.cmd_id }} run: | + get_output() { + aws ssm get-command-invocation \ + --command-id "$CMD_ID" \ + --instance-id "$INSTANCE_ID" \ + --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \ + --output json + } + for i in {1..20}; do STATUS=$(aws ssm get-command-invocation \ --command-id "$CMD_ID" \ @@ -60,25 +89,13 @@ jobs: if [ "$STATUS" = "Success" ]; then echo "Deployment completed successfully." - - aws ssm get-command-invocation \ - --command-id "$CMD_ID" \ - --instance-id "$INSTANCE_ID" \ - --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \ - --output json - + get_output exit 0 fi if [ "$STATUS" = "Failed" ] || [ "$STATUS" = "Cancelled" ] || [ "$STATUS" = "TimedOut" ]; then echo "Deployment failed." - - aws ssm get-command-invocation \ - --command-id "$CMD_ID" \ - --instance-id "$INSTANCE_ID" \ - --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \ - --output json - + get_output exit 1 fi @@ -86,11 +103,5 @@ jobs: done echo "Deployment timed out after waiting too long." - - aws ssm get-command-invocation \ - --command-id "$CMD_ID" \ - --instance-id "$INSTANCE_ID" \ - --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \ - --output json - - exit 1 \ No newline at end of file + get_output + exit 1 diff --git a/.github/workflows/cd-staging.yml b/.github/workflows/cd-staging.yml index 4becdd1..19690d4 100644 --- a/.github/workflows/cd-staging.yml +++ b/.github/workflows/cd-staging.yml @@ -18,19 +18,37 @@ jobs: contents: read steps: + - name: Checkout the repo + uses: actions/checkout@v7 + - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@v6 with: role-to-assume: ${{ secrets.EC2_DEPLOY_ROLE_ARN }} aws-region: ${{ secrets.AWS_REGION }} + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build and Push Docker Image + env: + REGISTRY: ${{ steps.login-ecr.outputs.registry }} + REPOSITORY: ${{ secrets.AWS_RESOURCE_PREFIX }}-staging-repo + run: | + docker build -t $REGISTRY/$REPOSITORY:latest ./backend + docker push $REGISTRY/$REPOSITORY:latest + - name: Trigger deploy on EC2 via SSM id: ssm env: INSTANCE_ID: ${{ secrets.STAGING_EC2_INSTANCE_ID }} BUILD_DIRECTORY: ${{ secrets.BUILD_DIRECTORY }} + ECR_REGISTRY: ${{ steps.login-ecr.outputs.registry }} + ECR_REPOSITORY: ${{ secrets.AWS_RESOURCE_PREFIX }}-staging-repo + IMAGE_TAG: latest run: | - DEPLOY_CMD='sudo -iu ec2-user bash -lc \"cd '"${BUILD_DIRECTORY}"' && git fetch origin && git reset --hard origin/main && docker compose build && docker compose run --rm --entrypoint \\\"\\\" backend uv run --frozen alembic upgrade head && docker compose up -d --remove-orphans && docker image prune -f && docker builder prune -af\"' + DEPLOY_CMD='sudo -iu ec2-user bash -lc \"cd '"${BUILD_DIRECTORY}"' && git fetch origin && git reset --hard origin/main && aws ecr get-login-password --region '"${{ secrets.AWS_REGION }}"' | docker login --username AWS --password-stdin '"${ECR_REGISTRY}"' && export ECR_REGISTRY='"${ECR_REGISTRY}"' ECR_REPOSITORY='"${ECR_REPOSITORY}"' IMAGE_TAG='"${IMAGE_TAG}"' && docker compose pull backend && docker compose --profile prestart run --rm prestart && docker compose up -d --remove-orphans && docker image prune -f\"' CMD_ID=$(aws ssm send-command \ --instance-ids "$INSTANCE_ID" \ @@ -48,6 +66,14 @@ jobs: INSTANCE_ID: ${{ secrets.STAGING_EC2_INSTANCE_ID }} CMD_ID: ${{ steps.ssm.outputs.cmd_id }} run: | + get_output() { + aws ssm get-command-invocation \ + --command-id "$CMD_ID" \ + --instance-id "$INSTANCE_ID" \ + --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \ + --output json + } + for i in {1..20}; do STATUS=$(aws ssm get-command-invocation \ --command-id "$CMD_ID" \ @@ -59,25 +85,13 @@ jobs: if [ "$STATUS" = "Success" ]; then echo "Deployment completed successfully." - - aws ssm get-command-invocation \ - --command-id "$CMD_ID" \ - --instance-id "$INSTANCE_ID" \ - --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \ - --output json - + get_output exit 0 fi if [ "$STATUS" = "Failed" ] || [ "$STATUS" = "Cancelled" ] || [ "$STATUS" = "TimedOut" ]; then echo "Deployment failed." - - aws ssm get-command-invocation \ - --command-id "$CMD_ID" \ - --instance-id "$INSTANCE_ID" \ - --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \ - --output json - + get_output exit 1 fi @@ -85,11 +99,5 @@ jobs: done echo "Deployment timed out after waiting too long." - - aws ssm get-command-invocation \ - --command-id "$CMD_ID" \ - --instance-id "$INSTANCE_ID" \ - --query '{Status:Status,Stdout:StandardOutputContent,Stderr:StandardErrorContent}' \ - --output json - - exit 1 \ No newline at end of file + get_output + exit 1 diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 8738d11..c9d6c14 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -9,6 +9,8 @@ on: jobs: checks: runs-on: ubuntu-latest + env: + NLTK_DISABLE_IMPORT_SECURITY: "1" services: postgres: image: postgres:16 diff --git a/backend/Dockerfile b/backend/Dockerfile index c6f4a36..eb0c852 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -47,6 +47,8 @@ RUN --mount=type=cache,target=/root/.cache/uv \ # Install pinned spaCy model in the final environment used at runtime. RUN python -m pip install --no-deps "${SPACY_MODEL_WHEEL_URL}" +ENV NLTK_DISABLE_IMPORT_SECURITY=1 + # Set HuggingFace cache directory ENV HF_HOME=/app/hf_cache diff --git a/backend/scripts/install_guardrails_from_hub.sh b/backend/scripts/install_guardrails_from_hub.sh index ffeea3a..fcc5486 100755 --- a/backend/scripts/install_guardrails_from_hub.sh +++ b/backend/scripts/install_guardrails_from_hub.sh @@ -8,6 +8,8 @@ GUARDRAILS_HUB_API_KEY="${GUARDRAILS_HUB_API_KEY:-}" ENABLE_METRICS="${ENABLE_METRICS:-false}" ENABLE_REMOTE_INFERENCING="${ENABLE_REMOTE_INFERENCING:-true}" +export NLTK_DISABLE_IMPORT_SECURITY=1 + BACKEND_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" MANIFEST_FILE="${1:-$BACKEND_DIR/app/core/validators/validators.json}" @@ -16,18 +18,43 @@ if [[ ! -f "$MANIFEST_FILE" ]]; then exit 1 fi +retry() { + local attempts="$1" + shift + local delay=5 + local n=1 + + until "$@"; do + if (( n >= attempts )); then + return 1 + fi + echo "Attempt ${n}/${attempts} failed; retrying in ${delay}s..." + sleep "$delay" + n=$(( n + 1 )) + delay=$(( delay * 2 )) + done +} + ####################################### # Configure Guardrails (non-interactive) ####################################### if [[ -n "$GUARDRAILS_HUB_API_KEY" ]]; then - echo "Configuring Guardrails CLI..." - guardrails configure \ - --token "$GUARDRAILS_HUB_API_KEY" \ - $( [[ "$ENABLE_METRICS" == "true" ]] && echo "--enable-metrics" || echo "--disable-metrics" ) \ - $( [[ "$ENABLE_REMOTE_INFERENCING" == "true" ]] && echo "--enable-remote-inferencing" || echo "--disable-remote-inferencing" ) + echo "Writing Guardrails configuration..." + + ANON_ID="$( { python3 -c 'import uuid; print(uuid.uuid4())' \ + || python -c 'import uuid; print(uuid.uuid4())'; } 2>/dev/null \ + || echo "00000000-0000-0000-0000-000000000000")" + + cat > "${HOME}/.guardrailsrc" <