diff --git a/.gitignore b/.gitignore index fdac443..2bbbb22 100644 --- a/.gitignore +++ b/.gitignore @@ -6,11 +6,8 @@ poetry.lock terraform.tfstate terraform.tfstate.backup .deployml - -data_local -working_Version -square_duckling -data/us_house_Sales_data.csv +.pytest_cache/ +.ruff_cache/ # MkDocs build output site/ @@ -22,13 +19,10 @@ config.yaml mlflow.db *.csv *.house_data ---source_format=PARQUET # Ignore .env file .venv/ .env .env/ local/.env -local/mlflow-artifacts/ - -demo \ No newline at end of file +local/mlflow-artifacts/ \ No newline at end of file diff --git a/config.yaml b/config.yaml deleted file mode 100644 index 28c8cd3..0000000 --- a/config.yaml +++ /dev/null @@ -1,44 +0,0 @@ -name: gcp-mlops-stack-mlflow -provider: - name: gcp - project_id: deployml-489218 - region: us-west1 -deployment: - type: cloud_run -stack: - - experiment_tracking: - name: mlflow - params: - service_name: mlflow-server - - artifact_tracking: - name: mlflow - params: - artifact_bucket: mlflow-artifacts-deployml-489218 - - model_registry: - name: mlflow - params: - backend_store_uri: postgresql - # - feature_store: - # name: feast - # params: - # service_name: feast-server - # backend_store_uri: postgresql - # offline_store: bigquery - # bigquery_dataset: deployml-489218.feast_housing.house_data - - model_serving: - name: fastapi - params: - service_name: fastapi-mlflow-server - - model_monitoring: - name: grafana - params: - service_name: grafana-server - # - workflow_orchestration: - # name: cron - # params: - # jobs: - # - service_name: offline-scoring - # cron_schedule: "0 12 */14 * *" - # bigquery_dataset: feast_housing - # - service_name: metrics-monitoring - # cron_schedule: "0 6 * * *" diff --git a/recycling_bin/AUTO_TEARDOWN.md b/recycling_bin/AUTO_TEARDOWN.md deleted file mode 100644 index 1e4f6d7..0000000 --- a/recycling_bin/AUTO_TEARDOWN.md +++ /dev/null @@ -1,143 +0,0 @@ -# Auto-Teardown Demo - -This demo shows how to test the auto-teardown feature with a 10-minute duration. - -## Quick Start - -### 1. Update Configuration - -Edit `teardown-demo.yaml` and update: -- `project_id`: Your GCP project ID -- `region`: Your preferred GCP region - -### 2. Deploy with Auto-Teardown - -```bash -deployml deploy --config-path demo/teardown-demo.yaml -``` - -You should see output like: -``` -✅ Deployment complete! - -⏰ Auto-teardown scheduled for: 2025-01-15 14:10:00 UTC - (in 24 hours) - To cancel: deployml teardown cancel --config-path demo/teardown-demo.yaml -``` - -### 3. Check Teardown Status - -```bash -deployml teardown status --config-path demo/teardown-demo.yaml -``` - -### 4. Monitor Teardown - -**Watch the countdown:** -```bash -watch -n 60 "deployml teardown status --config-path demo/teardown-demo.yaml" -``` - -**Check Cloud Scheduler:** -```bash -gcloud scheduler jobs describe deployml-teardown-teardown-demo-stack \ - --location us-west1 \ - --project YOUR_PROJECT_ID -``` - -**Check Cloud Function logs:** -```bash -gcloud functions logs read deployml-teardown-teardown-demo-stack \ - --region us-west1 \ - --project YOUR_PROJECT_ID \ - --limit 50 -``` - -### 5. Verify Teardown Executed - -After 10 minutes, check: - -```bash -# Check if infrastructure still exists -gcloud run services list --project YOUR_PROJECT_ID - -# Check Cloud Function logs for teardown execution -gcloud functions logs read deployml-teardown-teardown-demo-stack \ - --region us-west1 \ - --project YOUR_PROJECT_ID \ - --limit 10 -``` - -### 6. Cancel Teardown (Optional) - -If you want to cancel before the scheduled time: - -```bash -deployml teardown cancel --config-path demo/teardown-demo.yaml -``` - -## What Happens - -1. **Deployment**: Infrastructure is deployed normally -2. **Scheduler Created**: Cloud Scheduler job is created to trigger teardown in 10 minutes -3. **Wait**: Wait 10 minutes -4. **Teardown**: Cloud Scheduler triggers Cloud Function -5. **Destroy**: Cloud Function runs `terraform destroy` -6. **Cleanup**: All infrastructure resources are destroyed - -## Configuration Details - -- **duration_hours**: `0.167` = 10 minutes (10/60 hours) -- **time_zone**: `UTC` (you can change this) -- **enabled**: `true` to enable auto-teardown - -## Troubleshooting - -### Teardown didn't execute - -1. Check Cloud Scheduler job status: - ```bash - gcloud scheduler jobs describe deployml-teardown-teardown-demo-stack \ - --location us-west1 \ - --project YOUR_PROJECT_ID - ``` - -2. Check Cloud Function logs: - ```bash - gcloud functions logs read deployml-teardown-teardown-demo-stack \ - --region us-west1 \ - --project YOUR_PROJECT_ID - ``` - -3. Manually trigger the function (for testing): - ```bash - # Get the function URL - FUNCTION_URL=$(gcloud functions describe deployml-teardown-teardown-demo-stack \ - --region us-west1 \ - --project YOUR_PROJECT_ID \ - --format="value(httpsTrigger.url)") - - # Trigger it manually - curl -X POST $FUNCTION_URL \ - -H "Content-Type: application/json" \ - -d '{ - "workspace_name": "teardown-demo-stack", - "project_id": "YOUR_PROJECT_ID" - }' - ``` - -## Cleanup After Demo - -After the demo, clean up any remaining resources: - -```bash -# If teardown didn't execute automatically -deployml destroy --config-path demo/teardown-demo.yaml - -# Clean up the scheduler job if it still exists -gcloud scheduler jobs delete deployml-teardown-teardown-demo-stack \ - --location us-west1 \ - --project YOUR_PROJECT_ID \ - --quiet -``` - diff --git a/recycling_bin/DEPLOYMENT_GUIDE.md b/recycling_bin/DEPLOYMENT_GUIDE.md deleted file mode 100644 index 789c5e0..0000000 --- a/recycling_bin/DEPLOYMENT_GUIDE.md +++ /dev/null @@ -1,1159 +0,0 @@ -# Complete Deployment Guide: MLflow + FastAPI on Kubernetes - -**Demo-ready guide for deploying MLflow and FastAPI to minikube using deployml** - ---- - -## Table of Contents - -1. [Prerequisites](#prerequisites) -2. [Kubeconfig Setup](#kubeconfig-setup) -3. [Minikube Setup](#minikube-setup) -4. [Deploy MLflow](#deploy-mlflow) -5. [Train and Register Models](#train-and-register-models) -6. [Deploy FastAPI](#deploy-fastapi) -7. [Testing and Verification](#testing-and-verification) -8. [Troubleshooting](#troubleshooting) -9. [Cleanup](#cleanup) -10. [Quick Reference](#quick-reference) - ---- - -## Prerequisites - -### Required Software - -Before starting, ensure you have the following installed: - -```bash -# Check installations -minikube version -kubectl version --client -docker --version -deployml --version -python --version - -# Install if missing: -# - Minikube: https://minikube.sigs.k8s.io/docs/start/ -# - kubectl: https://kubernetes.io/docs/tasks/tools/ -# - Docker: https://docs.docker.com/get-docker/ -# - deployml: pip install deployml-core -# - Python 3.9+: https://www.python.org/downloads/ -``` - -### Verify Docker is Running - -```bash -# Check Docker status -docker ps - -# If Docker is not running, start Docker Desktop -# On macOS: Open Docker Desktop application -# On Linux: sudo systemctl start docker -``` - ---- - -## Kubeconfig Setup - -### Understanding Kubeconfig - -**Kubeconfig** is a configuration file that tells `kubectl` which Kubernetes cluster to connect to. It contains: -- Cluster information (server URLs, certificates) -- User credentials -- Contexts (which cluster + namespace to use) - -**Default location:** `~/.kube/config` - -### Step 1: Check Current Kubeconfig - -```bash -# View current kubeconfig location -echo $KUBECONFIG - -# If empty, it's using default: ~/.kube/config -# View current context -kubectl config current-context - -# List all available contexts -kubectl config get-contexts -``` - -### Step 2: Set Kubeconfig for Minikube - -```bash -# Option 1: Use default location (recommended) -# Minikube automatically configures ~/.kube/config when started -# No action needed - just start minikube - -# Option 2: Use custom kubeconfig file -export KUBECONFIG=~/.kube/minikube-config - -# Option 3: Use multiple configs (merged) -export KUBECONFIG=~/.kube/config:~/.kube/minikube-config -``` - -### Step 3: Verify Kubeconfig is Working - -```bash -# Test kubectl connection -kubectl cluster-info - -# Should show cluster information without errors -# If error, minikube may not be started yet -``` - -**Note:** Minikube automatically configures kubeconfig when you run `minikube start`. You don't need to manually set it unless you want a custom location. - ---- - -## Minikube Setup - -### Step 1: Start Minikube - -```bash -# Start minikube cluster -minikube start - -# This will: -# - Create a local Kubernetes cluster -# - Configure kubectl to use minikube -# - Set up networking and storage -``` - -**Expected output:** -``` -😄 minikube v1.37.0 on Darwin 14.5 (arm64) -✨ Using the docker driver based on existing profile -👍 Starting "minikube" primary control-plane node in "minikube" cluster -🚜 Pulling base image... -``` - -### Step 2: Verify Minikube is Running - -```bash -# Check minikube status -minikube status - -# Should show: -# host: Running -# kubelet: Running -# apiserver: Running -# kubeconfig: Configured - -# Verify kubectl can connect -kubectl get nodes - -# Should show: -# NAME STATUS ROLES AGE VERSION -# minikube Ready control-plane Xm v1.XX.X -``` - -### Step 3: Configure Minikube (Optional but Recommended) - -```bash -# Set recommended resources for MLflow + FastAPI -minikube config set memory 4096 # 4GB RAM -minikube config set cpus 4 # 4 CPUs -minikube config set disk-size 20g # 20GB disk - -# View configuration -minikube config view - -# Restart minikube to apply changes -minikube stop -minikube start -``` - -### Step 4: Verify Kubernetes Context - -```bash -# Check current context (should be minikube) -kubectl config current-context - -# Should output: minikube - -# If not minikube, switch context -kubectl config use-context minikube - -# Verify you're on the right cluster -kubectl get nodes - -# Should show: minikube (not AWS/GCP nodes) -``` - -**Important:** Make sure you're using the minikube context, not a remote cloud cluster! - ---- - -## Deploy MLflow - -### Step 1: Build MLflow Docker Image - -**Why:** We need a containerized MLflow server that can run in Kubernetes. - -```bash -# Navigate to MLflow directory -cd demo/mlflow - -# Build Docker image for Linux/amd64 platform -# Note: --platform linux/amd64 is required even on Apple Silicon Macs -docker build --platform linux/amd64 -t mlflow-demo:latest . - -# This creates a Docker image with: -# - Python 3.9 -# - MLflow 2.8.1 -# - Required dependencies -# - MLflow server configured - -# Verify image was created -docker images | grep mlflow-demo - -# Expected output: -# mlflow-demo latest X minutes ago XXX MB - -# Return to project root -cd ../.. -``` - -**What this does:** -- Creates a Docker image containing MLflow server -- Configures MLflow to run on port 5000 -- Sets up health checks -- Includes all necessary dependencies - -### Step 2: Generate MLflow Kubernetes Manifests - -**Why:** We need Kubernetes YAML files (deployment.yaml and service.yaml) to deploy MLflow. - -```bash -# Generate Kubernetes manifests using deployml -deployml mlflow-init \ - --output-dir ./demo/k8s-mlflow \ - --image mlflow-demo:latest - -# This command: -# - Creates deployment.yaml (defines MLflow pod) -# - Creates service.yaml (exposes MLflow on NodePort 30050) -# - Automatically loads Docker image into minikube -# - Configures MLflow with SQLite backend (default) - -# Verify manifests were created -ls -la ./demo/k8s-mlflow/ - -# Should show: -# deployment.yaml -# service.yaml -``` - -**What gets created:** - -**deployment.yaml:** -- Defines MLflow pod with 1 replica -- Sets resource limits (512Mi-1Gi memory, 250m-500m CPU) -- Configures health checks -- Uses SQLite backend by default - -**service.yaml:** -- Exposes MLflow on NodePort 30050 -- Maps port 5000 (container) → 30050 (host) -- Allows access from outside the cluster - -### Step 3: Deploy MLflow to Minikube - -```bash -# Deploy MLflow using the generated manifests -deployml mlflow-deploy --manifest-dir ./demo/k8s-mlflow - -# This command: -# - Applies deployment.yaml (creates MLflow pod) -# - Applies service.yaml (creates MLflow service) -# - Loads Docker image if needed -# - Shows deployment status and URL - -# Verify deployment -kubectl get pods -l app=mlflow - -# Should show: -# NAME READY STATUS RESTARTS AGE -# mlflow-deployment-XXXXX-XXXXX 1/1 Running 0 Xs - -# Check service -kubectl get svc mlflow-service - -# Should show: -# NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -# mlflow-service NodePort 10.XXX.XXX.XXX 5000:30050/TCP Xs -``` - -**Wait for pod to be ready:** -```bash -# Watch pod status -kubectl get pods -l app=mlflow -w - -# Press Ctrl+C when STATUS shows "Running" and READY shows "1/1" -``` - -### Step 4: Get MLflow URL - -**Important:** On macOS with Docker driver, minikube creates a tunneled URL. - -```bash -# Get tunneled URL (works from your Mac) -MLFLOW_URL=$(minikube service mlflow-service --url) -echo "MLflow URL: $MLFLOW_URL" - -# Example output: http://127.0.0.1:64231 - -# Test MLflow health endpoint -curl $MLFLOW_URL/health - -# Should return: {"status":"ok"} - -# Open MLflow UI in browser -open $MLFLOW_URL - -# Or manually navigate to: http://127.0.0.1:64231 -``` - -**Understanding the URLs:** -- **From your Mac:** Use `http://127.0.0.1:XXXXX` (tunneled URL) -- **Inside Kubernetes pods:** Use `http://mlflow-service:5000` (service name) -- **Port may change:** If you restart minikube, get a fresh URL - -### Step 5: Verify MLflow is Working - -```bash -# Check pod logs -kubectl logs -l app=mlflow --tail=50 - -# Should show MLflow server starting messages - -# Check service details -kubectl describe svc mlflow-service - -# Test from inside cluster -kubectl exec -it $(kubectl get pod -l app=mlflow -o jsonpath='{.items[0].metadata.name}') -- curl http://localhost:5000/health -``` - ---- - -## Train and Register Models - -### Step 1: Install Python Dependencies - -```bash -# Install required packages -pip install mlflow pandas numpy scikit-learn - -# Or if you have a requirements.txt -pip install -r requirements.txt - -# Verify MLflow installation -python -c "import mlflow; print(f'MLflow version: {mlflow.__version__}')" -``` - -### Step 2: Understand the Training Script - -The training script (`demo/register_model.py`) does the following: -1. Connects to MLflow tracking server -2. Creates/uses an experiment -3. Trains a RandomForest model -4. Logs metrics and parameters -5. Registers the model -6. Promotes model to Production stage - -### Step 3: Run Training Script - -```bash -# Make sure you have the MLflow URL -MLFLOW_URL=$(minikube service mlflow-service --url) -echo "Using MLflow at: $MLFLOW_URL" - -# Run training script -python demo/register_model.py - -# The script will: -# - Auto-detect MLflow URL from minikube -# - Create experiment: housing-price-prediction -# - Train model on synthetic housing data -# - Register model as: HousingPriceModel -# - Promote to Production stage -``` - -**Expected output:** -``` -✅ Found MLflow URL via minikube service: http://127.0.0.1:64231 -🔗 MLflow Tracking URI: http://127.0.0.1:64231 -✅ Using existing experiment: housing-price-prediction -📊 Preparing data... -✅ Data prepared: 800 training samples, 200 test samples -🤖 Training model... -📊 Model Performance: - Training RMSE: $XX,XXX.XX - Test RMSE: $XX,XXX.XX -✅ Model logged! Run ID: xxxxx -🔄 Promoting model to Production... -✅ Model version 1 promoted to Production! -🎉 Training complete! -``` - -### Step 4: Verify Model Registration - -**Option 1: Check in MLflow UI** -```bash -# Open MLflow UI -MLFLOW_URL=$(minikube service mlflow-service --url) -open $MLFLOW_URL - -# Navigate to: -# 1. "Models" tab (top menu) -# 2. Find "HousingPriceModel" -# 3. Check version 1 is in "Production" stage -``` - -**Option 2: Check via Python** -```python -from mlflow.tracking import MlflowClient - -mlflow.set_tracking_uri("http://127.0.0.1:64231") # Your MLflow URL -client = MlflowClient() - -# List registered models -models = client.search_registered_models() -for model in models: - print(f"Model: {model.name}") - for version in model.latest_versions: - print(f" Version {version.version}: {version.current_stage}") -``` - -**Option 3: Check via Command Line** -```bash -# Set tracking URI -export MLFLOW_TRACKING_URI=$(minikube service mlflow-service --url) - -# List models (if mlflow CLI is available) -mlflow models list -``` - -### Step 5: Test Model Loading - -```python -import mlflow.pyfunc -import pandas as pd - -# Set tracking URI -mlflow.set_tracking_uri("http://127.0.0.1:64231") # Your MLflow URL - -# Load Production model -model = mlflow.pyfunc.load_model("models:/HousingPriceModel/Production") -print("✅ Model loaded successfully!") - -# Make a test prediction -sample = pd.DataFrame({ - 'bedrooms': [3], - 'bathrooms': [2], - 'area_sqft': [2000], - 'lot_size': [5000], - 'year_built': [2010], - 'city': [1], - 'state': [0] -}) - -prediction = model.predict(sample) -print(f"Predicted price: ${prediction[0]:,.2f}") -``` - ---- - -## Deploy FastAPI - -### Step 1: Understand FastAPI Configuration - -The FastAPI application (`demo/fastapi/main.py`) is configured to: -- Connect to MLflow at `http://mlflow-service:5000` (Kubernetes service name) -- Load `HousingPriceModel` from Production stage -- Accept prediction requests with housing features -- Return price predictions - -**Key configuration:** -```python -MLFLOW_TRACKING_URI = "http://mlflow-service:5000" # Kubernetes service name -MODEL_NAME = "HousingPriceModel" -``` - -**Why `mlflow-service:5000`?** -- Inside Kubernetes, pods communicate via service names -- `mlflow-service` is the Kubernetes service name -- Port `5000` is the MLflow container port -- This works from inside the FastAPI pod - -### Step 2: Build FastAPI Docker Image - -```bash -# Navigate to FastAPI directory -cd demo/fastapi - -# Build Docker image -docker build --platform linux/amd64 -t fastapi-mlflow-demo:latest . - -# This creates a Docker image with: -# - Python 3.9 -# - FastAPI and Uvicorn -# - MLflow and scikit-learn -# - Your FastAPI application code - -# Verify image -docker images | grep fastapi-mlflow-demo - -# Return to project root -cd ../.. -``` - -### Step 3: Generate FastAPI Kubernetes Manifests - -**Important:** Use Kubernetes service name for MLflow URI, not the tunneled URL! - -```bash -# Generate manifests with MLflow URI -# Use http://mlflow-service:5000 (Kubernetes service name) -deployml minikube-init \ - --output-dir ./demo/k8s-fastapi \ - --image fastapi-mlflow-demo:latest \ - --mlflow-uri http://mlflow-service:5000 - -# This creates: -# - deployment.yaml (FastAPI pod configuration) -# - service.yaml (exposes FastAPI on NodePort 30080) -# - Sets MLFLOW_TRACKING_URI environment variable -# - Loads Docker image into minikube - -# Verify manifests -ls -la ./demo/k8s-fastapi/ -``` - -**Why `http://mlflow-service:5000`?** -- `mlflow-service` is the Kubernetes service name (DNS resolvable) -- `5000` is the MLflow container port -- This allows FastAPI pod to communicate with MLflow pod internally - -### Step 4: Deploy FastAPI to Minikube - -```bash -# Deploy FastAPI -deployml minikube-deploy --manifest-dir ./demo/k8s-fastapi - -# Verify deployment -kubectl get pods -l app=fastapi - -# Should show: -# NAME READY STATUS RESTARTS AGE -# fastapi-deployment-XXXXX-XXXXX 1/1 Running 0 Xs - -# Check service -kubectl get svc fastapi-service - -# Should show: -# NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -# fastapi-service NodePort 10.XXX.XXX.XXX 8000:30080/TCP Xs -``` - -### Step 5: Verify Model Loading - -```bash -# Check FastAPI logs (should show model loaded) -kubectl logs -l app=fastapi --tail=50 - -# Look for: -# ✓ Loaded model 'HousingPriceModel' from Production stage - -# If you see errors, check: -kubectl describe pod -l app=fastapi -``` - -**Common issues:** -- Model not found → Check model name matches -- Connection refused → Check MLflow service is accessible -- Timeout → Check MLflow pod is running - -### Step 6: Get FastAPI URL - -```bash -# Get tunneled URL -FASTAPI_URL=$(minikube service fastapi-service --url) -echo "FastAPI URL: $FASTAPI_URL" - -# Example: http://127.0.0.1:64750 - -# Test health endpoint -curl $FASTAPI_URL/health - -# Expected response: -# { -# "status": "healthy", -# "mlflow_connected": true, -# "model_loaded": true, -# "timestamp": "2025-12-12T..." -# } -``` - ---- - -## Testing and Verification - -### Test MLflow - -```bash -# Get MLflow URL -MLFLOW_URL=$(minikube service mlflow-service --url) - -# Test health -curl $MLFLOW_URL/health -# Should return: {"status":"ok"} - -# Open UI -open $MLFLOW_URL - -# In the UI, verify: -# 1. Experiment "housing-price-prediction" exists -# 2. Model "HousingPriceModel" is registered -# 3. Version 1 is in "Production" stage -``` - -### Test FastAPI Root Endpoint - -```bash -FASTAPI_URL=$(minikube service fastapi-service --url) - -# Test root endpoint -curl $FASTAPI_URL/ - -# Expected response: -# { -# "service": "FastAPI Demo", -# "version": "1.0.0", -# "model_loaded": true, -# "model_name": "HousingPriceModel", -# "mlflow_uri": "http://mlflow-service:5000", -# "endpoints": {...} -# } -``` - -### Test FastAPI Health Endpoint - -```bash -curl $FASTAPI_URL/health - -# Expected response: -# { -# "status": "healthy", -# "timestamp": "2025-12-12T...", -# "port": 8000, -# "mlflow_connected": true, -# "model_loaded": true -# } -``` - -### Test Prediction Endpoint - -```bash -# Make prediction request -curl -X POST "$FASTAPI_URL/predict" \ - -H "Content-Type: application/json" \ - -d '{ - "features": { - "bedrooms": 3, - "bathrooms": 2, - "area_sqft": 2000, - "lot_size": 5000, - "year_built": 2010, - "city": 1, - "state": 0 - } - }' - -# Expected response: -# { -# "prediction": 557545.45, -# "timestamp": "2025-12-12T06:31:23.450973", -# "model_used": "MLflow: HousingPriceModel" -# } -``` - -**If prediction returns -1:** -- Model not loaded → Check FastAPI logs -- Wrong features → Check feature names match model -- Connection error → Check MLflow service accessibility - -### Test via Swagger UI - -```bash -# Open interactive API documentation -open $FASTAPI_URL/docs - -# Or ReDoc -open $FASTAPI_URL/redoc - -# In Swagger UI: -# 1. Click "POST /predict" -# 2. Click "Try it out" -# 3. Enter features in JSON format -# 4. Click "Execute" -# 5. See prediction result -``` - -### Test Multiple Predictions - -```bash -# Test with different inputs -curl -X POST "$FASTAPI_URL/predict" \ - -H "Content-Type: application/json" \ - -d '{ - "features": { - "bedrooms": 4, - "bathrooms": 3, - "area_sqft": 3000, - "lot_size": 8000, - "year_built": 2015, - "city": 2, - "state": 1 - } - }' -``` - ---- - -## Troubleshooting - -### Issue: Pod shows `ErrImageNeverPull` - -**Symptoms:** -``` -NAME READY STATUS RESTARTS AGE -fastapi-deployment-XXXXX-XXXXX 0/1 ErrImageNeverPull 0 Xs -``` - -**Causes:** -- Wrong Kubernetes context (deploying to remote cluster instead of minikube) -- Image not loaded into minikube -- ImagePullPolicy mismatch - -**Solutions:** - -```bash -# 1. Check current context -kubectl config current-context -# Should show: minikube - -# If not, switch context -kubectl config use-context minikube - -# 2. Reload image into minikube -minikube image load fastapi-mlflow-demo:latest - -# 3. Verify image exists -minikube image ls | grep fastapi-mlflow-demo - -# 4. Delete pods to recreate -kubectl delete pod -l app=fastapi - -# 5. Check new pods -kubectl get pods -l app=fastapi -``` - -### Issue: Model not loading in FastAPI - -**Symptoms:** -- Health endpoint shows `"model_loaded": false` -- Predictions return -1 -- Logs show connection errors - -**Solutions:** - -```bash -# 1. Check FastAPI logs -kubectl logs -l app=fastapi --tail=100 - -# Look for errors like: -# - "Could not load model from MLflow" -# - "Connection refused" -# - "Model not found" - -# 2. Verify MLflow service is accessible from FastAPI pod -kubectl exec -it $(kubectl get pod -l app=fastapi -o jsonpath='{.items[0].metadata.name}') -- curl http://mlflow-service:5000/health - -# Should return: {"status":"ok"} - -# 3. Check environment variables -kubectl exec -it $(kubectl get pod -l app=fastapi -o jsonpath='{.items[0].metadata.name}') -- env | grep MLFLOW - -# Should show: -# MLFLOW_TRACKING_URI=http://mlflow-service:5000 - -# 4. Verify model exists in MLflow -# Open MLflow UI and check Models tab - -# 5. Test model loading manually -python -c " -import mlflow.pyfunc -mlflow.set_tracking_uri('http://127.0.0.1:64231') # Your MLflow URL -model = mlflow.pyfunc.load_model('models:/HousingPriceModel/Production') -print('Model loaded successfully') -" -``` - -### Issue: Wrong Kubernetes Context - -**Symptoms:** -- Pods scheduled on AWS/GCP nodes instead of minikube -- `ErrImageNeverPull` errors -- Cannot access services - -**Solutions:** - -```bash -# 1. List all contexts -kubectl config get-contexts - -# 2. Switch to minikube -kubectl config use-context minikube - -# 3. Verify -kubectl get nodes -# Should show: minikube (not AWS/GCP nodes) - -# 4. Redeploy -deployml minikube-deploy --manifest-dir ./demo/k8s-fastapi -``` - -### Issue: MLflow URL not accessible - -**Symptoms:** -- Cannot connect to MLflow from training script -- Connection timeout errors - -**Solutions:** - -```bash -# 1. Get fresh URL (port may have changed) -MLFLOW_URL=$(minikube service mlflow-service --url) -echo $MLFLOW_URL - -# 2. Test connectivity -curl $MLFLOW_URL/health - -# 3. Check MLflow pod is running -kubectl get pods -l app=mlflow - -# 4. Check MLflow logs -kubectl logs -l app=mlflow --tail=50 - -# 5. Use port-forward for stable port -kubectl port-forward svc/mlflow-service 5000:5000 -# Then use: http://127.0.0.1:5000 -``` - -### Issue: FastAPI returns -1 - -**Symptoms:** -- Prediction endpoint returns `{"prediction": -1}` - -**Causes:** -- Model not loaded -- Wrong feature names -- Model loading error - -**Solutions:** - -```bash -# 1. Check FastAPI logs -kubectl logs -l app=fastapi --tail=100 - -# 2. Verify model is loaded -curl $FASTAPI_URL/health -# Check: "model_loaded": true - -# 3. Verify feature names match -# Check model expects: bedrooms, bathrooms, area_sqft, lot_size, year_built, city, state - -# 4. Test with correct features -curl -X POST "$FASTAPI_URL/predict" \ - -H "Content-Type: application/json" \ - -d '{ - "features": { - "bedrooms": 3, - "bathrooms": 2, - "area_sqft": 2000, - "lot_size": 5000, - "year_built": 2010, - "city": 1, - "state": 0 - } - }' -``` - -### Issue: Port number keeps changing - -**Symptoms:** -- MLflow/FastAPI URL changes after restarting minikube - -**Solutions:** - -```bash -# 1. Always get fresh URL -MLFLOW_URL=$(minikube service mlflow-service --url) -FASTAPI_URL=$(minikube service fastapi-service --url) - -# 2. Use port-forward for stable ports -kubectl port-forward svc/mlflow-service 5000:5000 & -kubectl port-forward svc/fastapi-service 8000:8000 & - -# Then use: -# MLflow: http://127.0.0.1:5000 -# FastAPI: http://127.0.0.1:8000 -``` - ---- - -## Cleanup - -### Delete Deployments - -```bash -# Delete FastAPI -kubectl delete -f ./demo/k8s-fastapi/ - -# Delete MLflow -kubectl delete -f ./demo/k8s-mlflow/ - -# Or delete by label -kubectl delete deployment -l app=fastapi -kubectl delete svc -l app=fastapi -kubectl delete deployment -l app=mlflow -kubectl delete svc -l app=mlflow - -# Verify deletion -kubectl get pods -kubectl get svc -``` - -### Stop Minikube - -```bash -# Stop minikube (keeps data) -minikube stop - -# Delete minikube cluster (removes everything) -minikube delete - -# Confirm deletion -minikube status -``` - -### Remove Docker Images - -```bash -# Remove local images -docker rmi mlflow-demo:latest -docker rmi fastapi-mlflow-demo:latest - -# Remove from minikube (if minikube is running) -minikube image rm mlflow-demo:latest -minikube image rm fastapi-mlflow-demo:latest -``` - ---- - -## Quick Reference - -### Complete Deployment Script - -Save this as `deploy_all.sh`: - -```bash -#!/bin/bash -set -e - -echo "🚀 Complete MLflow + FastAPI Deployment" -echo "========================================" - -# 1. Setup -echo "\n📋 Step 1: Setup" -kubectl config use-context minikube -minikube start -kubectl get nodes - -# 2. Build MLflow -echo "\n📦 Step 2: Building MLflow Image" -cd demo/mlflow -docker build --platform linux/amd64 -t mlflow-demo:latest . -cd ../.. - -# 3. Deploy MLflow -echo "\n🚀 Step 3: Deploying MLflow" -deployml mlflow-init --output-dir ./demo/k8s-mlflow --image mlflow-demo:latest -deployml mlflow-deploy --manifest-dir ./demo/k8s-mlflow - -# 4. Get MLflow URL -echo "\n🔗 Step 4: Getting MLflow URL" -MLFLOW_URL=$(minikube service mlflow-service --url) -echo "✅ MLflow: $MLFLOW_URL" -sleep 5 # Wait for MLflow to be ready - -# 5. Train Model -echo "\n🤖 Step 5: Training Model" -python demo/register_model.py - -# 6. Build FastAPI -echo "\n📦 Step 6: Building FastAPI Image" -cd demo/fastapi -docker build --platform linux/amd64 -t fastapi-mlflow-demo:latest . -cd ../.. - -# 7. Deploy FastAPI -echo "\n🚀 Step 7: Deploying FastAPI" -deployml minikube-init --output-dir ./demo/k8s-fastapi --image fastapi-mlflow-demo:latest --mlflow-uri http://mlflow-service:5000 -deployml minikube-deploy --manifest-dir ./demo/k8s-fastapi - -# 8. Get FastAPI URL -echo "\n🔗 Step 8: Getting FastAPI URL" -sleep 10 # Wait for FastAPI to start -FASTAPI_URL=$(minikube service fastapi-service --url) -echo "✅ FastAPI: $FASTAPI_URL" - -# 9. Test -echo "\n🧪 Step 9: Testing" -curl -f "$FASTAPI_URL/health" && echo "✅ Health check passed!" -curl -X POST "$FASTAPI_URL/predict" \ - -H "Content-Type: application/json" \ - -d '{"features": {"bedrooms": 3, "bathrooms": 2, "area_sqft": 2000, "lot_size": 5000, "year_built": 2010, "city": 1, "state": 0}}' - -echo "\n🎉 Deployment complete!" -echo "📊 MLflow UI: $MLFLOW_URL" -echo "🚀 FastAPI: $FASTAPI_URL" -``` - -### Common Commands Cheat Sheet - -```bash -# Get service URLs -MLFLOW_URL=$(minikube service mlflow-service --url) -FASTAPI_URL=$(minikube service fastapi-service --url) - -# Check pods -kubectl get pods -l app=mlflow -kubectl get pods -l app=fastapi - -# View logs -kubectl logs -l app=mlflow --tail=50 -f -kubectl logs -l app=fastapi --tail=50 -f - -# Restart services -kubectl rollout restart deployment/mlflow-deployment -kubectl rollout restart deployment/fastapi-deployment - -# Check services -kubectl get svc - -# Test endpoints -curl $MLFLOW_URL/health -curl $FASTAPI_URL/health -curl -X POST "$FASTAPI_URL/predict" -H "Content-Type: application/json" -d '{"features": {...}}' -``` - -### Architecture Diagram - -``` -┌─────────────────────────────────────────────────────────┐ -│ Your Mac (Host) │ -│ │ -│ ┌──────────────┐ ┌──────────────┐ │ -│ │ Training │ │ Browser │ │ -│ │ Script │────────▶│ (UI) │ │ -│ └──────────────┘ └──────────────┘ │ -│ │ │ │ -│ │ http://127.0.0.1:64231│ │ -│ │ │ │ -└─────────┼────────────────────────┼──────────────────────┘ - │ │ - │ │ -┌─────────▼────────────────────────▼──────────────────────┐ -│ Minikube Kubernetes Cluster │ -│ │ -│ ┌──────────────────┐ ┌──────────────────┐ │ -│ │ MLflow Pod │ │ FastAPI Pod │ │ -│ │ Port: 5000 │◀─────│ Port: 8000 │ │ -│ │ │ │ │ │ -│ │ Service: │ │ Service: │ │ -│ │ mlflow-service │ │ fastapi-service │ │ -│ │ :5000 │ │ :8000 │ │ -│ └──────────────────┘ └──────────────────┘ │ -│ │ │ -│ │ http://mlflow-service:5000 │ -│ │ │ -│ ┌──────▼────────────────────────────────────┐ │ -│ │ MLflow Model Registry │ │ -│ │ - HousingPriceModel │ │ -│ │ - Production Stage │ │ -│ └───────────────────────────────────────────┘ │ -└───────────────────────────────────────────────────────────┘ -``` - -### Key Points Summary - -1. **Kubeconfig:** Minikube automatically configures it when started -2. **Context:** Always use `minikube` context, not remote clusters -3. **MLflow URL:** - - From host: `http://127.0.0.1:XXXXX` (tunneled) - - From pods: `http://mlflow-service:5000` (service name) -4. **Model:** Registered as `HousingPriceModel` in Production stage -5. **Features:** `bedrooms`, `bathrooms`, `area_sqft`, `lot_size`, `year_built`, `city`, `state` -6. **Ports:** May change after restarting minikube - always get fresh URLs - ---- - -## Demo Flow - -For your demo tomorrow, follow this flow: - -1. **Setup (2 min)** - - Show kubeconfig setup - - Start minikube - - Verify context - -2. **Deploy MLflow (3 min)** - - Build image - - Generate manifests - - Deploy - - Show MLflow UI - -3. **Train Model (2 min)** - - Run training script - - Show model in MLflow UI - - Verify Production stage - -4. **Deploy FastAPI (3 min)** - - Build image - - Deploy with MLflow connection - - Show logs (model loading) - -5. **Test (2 min)** - - Health checks - - Make predictions - - Show Swagger UI - -**Total: ~12 minutes** - ---- - -## Support - -If you encounter issues during the demo: - -1. Check pod status: `kubectl get pods` -2. Check logs: `kubectl logs -l app=` -3. Verify context: `kubectl config current-context` -4. Get fresh URLs: `minikube service --url` - -Good luck with your demo! 🚀 - diff --git a/recycling_bin/GCP_VM_Deployment.md b/recycling_bin/GCP_VM_Deployment.md deleted file mode 100644 index 7cc6316..0000000 --- a/recycling_bin/GCP_VM_Deployment.md +++ /dev/null @@ -1,201 +0,0 @@ -## VM Deployment Guide (GCP VM) - -This guide is designed to help setup and run the deployment of Deployml after the initial Deployml prerequisite have been met. - -### Configuration - -To start create a configuration YAML describing your VM stack. You can start from the example: - -```bash -cp example/config/gcp-cloud-vm-sample.yaml ./vm-stack.yaml -``` - -Key fields in `vm-stack.yaml`: - -- provider: `name`, `project_id`, `region`, `zone` -- deployment: `type: cloud_vm` -- stack stages and params (MLflow, artifact bucket, Feast, Grafana, etc.) - -Minimal VM example (edit values to match your project): - -```yaml -name: gcp-mlops-stack-mlflow-vm -provider: - name: gcp - project_id: YOUR_PROJECT_ID - region: YOUR_CHOSEN_REGION - zone: YOUR_CHOSEN_ZONE -cost_analysis: - enabled: true - warning_threshold: 50.0 - currency: "USD" -deployment: - type: cloud_vm -stack: - - experiment_tracking: - name: mlflow - params: - service_name: mlflow-server-postgres-vm - vm_name: mlflow-postgres-vm-instance - machine_type: e2-medium - disk_size_gb: 20 - mlflow_port: 5000 - allow_public_access: true - fastapi_port: 8000 - fastapi_app_source: "template" - - artifact_tracking: - name: mlflow - params: - artifact_bucket: mlflow-artifact-bucket-postgres-2 - create_artifact_bucket: true - - model_registry: - name: mlflow - params: - -``` - -Notes: - -- If `artifact_bucket` is omitted, the CLI will generate a unique bucket name and set it up. -- If `model_registry.params.backend_store_uri` starts with `postgresql`, the CLI will automatically include Cloud SQL Postgres in the deployment plan and wire dependencies. - -### Deploy - -Run the deploy command. It will generate Terraform files under `.deployml/`, run a Terraform plan, perform optional cost analysis, and then apply: - -```bash -deployml deploy --config-path vm-stack.yaml -``` -- `--config-path` can be replaced with `-c` to simplify the deploymenet command. -Flags: - -- `-y` or `--yes` to skip confirmation prompts. - -What this does: - -- Generates Terraform from templates for `cloud_vm` into `.deployml//terraform` -- Initializes and plans Terraform -- Optionally runs cost analysis (if enabled) -- Applies the plan and prints friendly outputs (URLs, service endpoints, credentials when applicable) - -This process should take an estimated 18 - 20 minutes to complete. -Once the all Google Cloud tools are setup the Virtual Machine will begin downloading and configuring itself which will take another 15 - 18 minutes. - -### Virtual Machine Setup -Get the VM external IP from the outputs or via gcloud: - -```bash -gcloud compute instances list --filter="name~'mlflow'" --format="get(networkInterfaces[0].accessConfigs[0].natIP)" -``` - -SSH to the VM: - -```bash -gcloud compute ssh --zone YOUR_ZONE YOUR_VM_NAME -``` - -Once in the VM you can run: - -```bash -journalctl -f -``` -To see live updates as the VM continues its deployment. -To see if Deployment has been successful run the following Docker Command to see the status of all your Docker Containers. - -```bash -sudo docker ps -``` -If all containers show healthy all tools have been successfully deployed and are ready for use. - -### Outputs and Access - -After a successful deploy, the CLI prints an "DeployML Outputs" section with key values. Typical endpoints and how to access them: - -- MLflow UI: exposed on the VM at port 5000 (use the VM external IP). Example: `http://:5000` -- FastAPI app (template): port 8000 → `http://:8000` -- Feast server (inside VM Docker): port 6566 (access over SSH tunnel or within the VM) -- Grafana (if enabled): port 3000 → `http://:3000` - - -### MLFlow - -Use the link found in the DeployML output to access MLFlow. -To be able to connect to MLFlow all you need to change in your logging files is: -``` -mlflow.set_tracking_uri(EXTERNAL_VM_IP:5000) -``` -### Feast on the VM - -Once the VM is up, you can configure Feast and materialize data. See the detailed guide: - -- `README_vm_FEAST.md` in the repo root (end-to-end walkthrough for setting up registry/online store and loading parquet). - -### Grafana on the VM - -Use the link found in the DeployML output to access Grafana. -To Login the following is the default login and can be changed: -``` -username: Admin | password: Admin -``` -Once logged in you can connect to the postgressql database by going to: -``` -connections -> PostgresSQL -> Add New Datasource -``` -and filling in the following: -``` -Host URL: Found in .tfstate file - -Database name: default is mlflow - -Username: default is mlflow - -Password: found in .tfstate file - -TLS/SSL Mode: Disable -``` - -The .tfstate file will be generated in the .deployml directory upon deployment. -To find the needed information look for: -``` - "db_connection_string": { - "value": "postgresql+psycopg2://mlflow:RANDOMLY_GENERATED_PASSWORD@POSTGRES_IP/mlflow", - "type": "string", - "sensitive": true - }, -``` - -### Airflow - -Use the link found in the DeployML output to access Airflow. -To login to airflow you will need to find the username and password from the deployment logs using while in the VM: -```bash -sudo docker logs airflow-webserver -``` -and look for -``` -Simple auth manager | Password for user 'admin': RANDOM_GENERATED_PASSWORD -``` - - -### Destroy and Cleanup - -To destroy the deployed resources and optionally remove the local workspace: - -```bash -deployml destroy --config-path vm-stack.yaml -``` - -You will be prompted to confirm. To also remove `.deployml/` after destroying, re-run with `--clean-workspace` or confirm when prompted to clean Terraform state files. - -Due to how GCP tears down systems you may need to run the destroy command twice to get a fully tear down. - -### Troubleshooting - -- Authentication: If prompted, the CLI will run `gcloud auth application-default login` during deploy. -- Ports blocked: Ensure your VM firewall rules allow ingress on the ports you need (5000, 8000, 3000, 6566, etc.). The Terraform for `cloud_vm` opens required ports defined by your stack params. -- Artifact bucket missing: If you provide a bucket name, ensure it exists or set `create_artifact_bucket: true`. -- Postgres backend: When using `postgresql` backend for MLflow (and Feast SQL registry), Cloud SQL Postgres resources are created; teardown uses the CLI `destroy` command which also handles Cloud SQL dependencies. - -### Known Bugs -- MLFlow does not successfully deploy due to not finding the right password. -- `deployml destroy --config-path vm-stack.yaml` does not fully tear down the project. diff --git a/recycling_bin/GKE_DEPLOYMENT_GUIDE.md b/recycling_bin/GKE_DEPLOYMENT_GUIDE.md deleted file mode 100644 index 7d4b82f..0000000 --- a/recycling_bin/GKE_DEPLOYMENT_GUIDE.md +++ /dev/null @@ -1,1087 +0,0 @@ -# Complete GKE Deployment Guide: MLflow + FastAPI on Google Kubernetes Engine - -**Step-by-step guide for deploying MLflow and FastAPI to GKE using deployml** - ---- - -## Table of Contents - -1. [Prerequisites](#prerequisites) -2. [GKE Cluster Setup](#gke-cluster-setup) -3. [Build and Push Docker Images](#build-and-push-docker-images) -4. [Configuration File](#configuration-file) -5. [Deploy MLflow](#deploy-mlflow) -6. [Train and Register Models](#train-and-register-models) -7. [Deploy FastAPI](#deploy-fastapi) -8. [Testing and Verification](#testing-and-verification) -9. [Troubleshooting](#troubleshooting) -10. [Resource Management](#resource-management) -11. [Cleanup](#cleanup) -12. [Quick Reference](#quick-reference) - ---- - -## Prerequisites - -### Required Software - -```bash -# Check installations -gcloud --version -kubectl version --client -docker --version -deployml --version - -# Install if missing: -# - gcloud: https://cloud.google.com/sdk/docs/install -# - kubectl: https://kubernetes.io/docs/tasks/tools/ -# - Docker: https://docs.docker.com/get-docker/ -# - deployml: pip install deployml-core -``` - -**Important for users with existing kubeconfig (e.g., company clusters):** -- ✅ **No manual kubeconfig setup needed** - The `deployml deploy` command automatically configures kubectl via `gcloud get-credentials` -- ✅ **Your existing kubeconfig is safe** - The tool adds a new GKE context without deleting your company contexts -- ⚠️ **After deployment**, kubectl will be pointing to the GKE cluster. You can switch back anytime: - ```bash - kubectl config get-contexts # List all contexts - kubectl config use-context YOUR_COMPANY_CONTEXT # Switch back - ``` - -### GCP Authentication - -```bash -# Authenticate with GCP -gcloud auth login -gcloud auth application-default login - -# Set your project -gcloud config set project YOUR_PROJECT_ID - -# Verify authentication -gcloud auth list -``` - -### Enable Required APIs - -```bash -# Enable GKE API -gcloud services enable container.googleapis.com - -# Enable Container Registry API -gcloud services enable containerregistry.googleapis.com - -# Enable Compute Engine API -gcloud services enable compute.googleapis.com -``` - ---- - -## GKE Cluster Setup - -### Option 1: Create New GKE Cluster - -```bash -# Create a GKE cluster -gcloud container clusters create my-gke-cluster \ - --zone us-west1-a \ - --num-nodes 2 \ - --machine-type e2-medium \ - --project YOUR_PROJECT_ID - -# This creates: -# - 2 nodes with e2-medium (2 vCPU, 4GB RAM each) -# - Total: 4 vCPU, 8GB RAM -# - Takes ~3-5 minutes -``` - -### Option 2: Use Existing Cluster - -```bash -# List existing clusters -gcloud container clusters list --project YOUR_PROJECT_ID - -# Connect to existing cluster -gcloud container clusters get-credentials CLUSTER_NAME \ - --zone ZONE \ - --project YOUR_PROJECT_ID - -# Verify connection -kubectl cluster-info -kubectl get nodes -``` - -**Note:** If you have an existing company kubeconfig: -- ✅ **No manual setup needed** - `deployml deploy` handles everything automatically -- ✅ **Your company kubeconfig is safe** - `gcloud get-credentials` adds a new context, it doesn't delete existing ones -- ⚠️ **Context switching** - After deployment, kubectl will point to the GKE cluster. To switch back to your company cluster: - ```bash - kubectl config use-context YOUR_COMPANY_CONTEXT_NAME - ``` -- 📋 **View all contexts**: `kubectl config get-contexts` - -### Cluster Sizing Recommendations - -| Workload | Nodes | Machine Type | Total CPU | Total RAM | -|----------|-------|--------------|-----------|-----------| -| **Small** (MLflow only) | 1 | e2-medium | 2 vCPU | 4GB | -| **Medium** (MLflow + FastAPI) | 2 | e2-medium | 4 vCPU | 8GB | -| **Large** (Production) | 3+ | e2-standard-4 | 12+ vCPU | 16+ GB | - ---- - -## Build and Push Docker Images - -### Step 1: Build MLflow Image - -```bash -# Navigate to MLflow directory -cd demo/mlflow - -# Build for Linux/amd64 (required for GKE) -docker build --platform linux/amd64 -t mlflow-demo:latest . - -# Verify image -docker images | grep mlflow-demo - -cd ../.. -``` - -### Step 2: Build FastAPI Image - -```bash -# Navigate to FastAPI directory -cd demo/fastapi - -# Build for Linux/amd64 -docker build --platform linux/amd64 -t fastapi-mlflow-demo:latest . - -# Verify image -docker images | grep fastapi-mlflow-demo - -cd ../.. -``` - -### Step 3: Push Images to Google Container Registry - -```bash -# Set your project ID -export PROJECT_ID="YOUR_PROJECT_ID" - -# Configure Docker for GCR -gcloud auth configure-docker - -# Tag and push MLflow image -docker tag mlflow-demo:latest gcr.io/$PROJECT_ID/mlflow/mlflow:latest -docker push gcr.io/$PROJECT_ID/mlflow/mlflow:latest - -# Tag and push FastAPI image -docker tag fastapi-mlflow-demo:latest gcr.io/$PROJECT_ID/fastapi/fastapi:latest -docker push gcr.io/$PROJECT_ID/fastapi/fastapi:latest - -# Verify images in GCR -gcloud container images list --project=$PROJECT_ID -``` - -**Note:** The `deployml deploy` command will automatically push images if they're local, but you can also push manually as shown above. - ---- - -## Configuration File - -### Create GKE Config File - -Create `gke-deploy-config.yaml`: - -```yaml -name: gke-mlflow-fastapi -provider: - name: gcp - project_id: YOUR_PROJECT_ID - region: us-west1 - -deployment: - type: gke - -gke: - cluster_name: my-gke-cluster - zone: us-west1-a # For zonal cluster - # OR use region for regional cluster: - # region: us-west1 - -stack: - - experiment_tracking: - name: mlflow - params: - image: mlflow-demo:latest # Local image name (will be pushed to GCR) - backend_store_uri: sqlite:///mlflow.db # or postgresql - - - artifact_tracking: - name: mlflow - params: - artifact_bucket: mlflow-artifacts-bucket - create_artifact_bucket: true - - - model_registry: - name: mlflow - params: - backend_store_uri: sqlite:///mlflow.db - - - model_serving: - name: fastapi - params: - image: fastapi-mlflow-demo:latest # Local image name - mlflow_tracking_uri: http://mlflow-service:5000 -``` - -### Key Configuration Points - -- **`deployment.type: gke`** - Tells deployml to use Kubernetes manifests -- **`gke.cluster_name`** - Your GKE cluster name -- **`gke.zone`** - Cluster zone (or use `region` for regional clusters) -- **`image`** - Local Docker image name (will be converted to GCR format) -- **`mlflow_tracking_uri`** - Use `http://mlflow-service:5000` (Kubernetes internal DNS) - ---- - -## Deploy MLflow - -### Step 1: Deploy Using Config File - -```bash -# Deploy MLflow (and FastAPI if in config) -deployml deploy -c gke-deploy-config.yaml - -# This will: -# 1. Connect to GKE cluster -# 2. Generate Kubernetes manifests -# 3. Push images to GCR (if local) -# 4. Deploy to GKE -# 5. Show LoadBalancer URLs -``` - -### Step 2: Verify Deployment - -```bash -# Check pods -kubectl get pods -l app=mlflow - -# Should show: -# NAME READY STATUS RESTARTS AGE -# mlflow-deployment-XXXXX-XXXXX 1/1 Running 0 Xs - -# Check service -kubectl get svc mlflow-service - -# Should show: -# NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE -# mlflow-service LoadBalancer 10.XXX.XXX.X 34.XXX.XXX.XXX 5000:XXXXX/TCP Xs -``` - -### Step 3: Get MLflow URL - -```bash -# Get LoadBalancer IP -MLFLOW_IP=$(kubectl get svc mlflow-service -o jsonpath='{.status.loadBalancer.ingress[0].ip}') -echo "MLflow URL: http://$MLFLOW_IP:5000" - -# Test health endpoint -curl http://$MLFLOW_IP:5000/health - -# Open in browser -open http://$MLFLOW_IP:5000 -``` - ---- - -## Train and Register Models - -### Update Training Script - -Update `demo/register_model.py` to use your MLflow URL: - -```python -# Get MLflow URL from GKE -MLFLOW_IP = "34.182.23.6" # Your LoadBalancer IP -mlflow.set_tracking_uri(f"http://{MLFLOW_IP}:5000") -``` - -### Run Training - -```bash -# Run training script -python demo/register_model.py - -# This will: -# - Connect to MLflow on GKE -# - Train model -# - Register model as "HousingPriceModel" -# - Promote to Production stage -``` - -### Verify Model Registration - -```bash -# Open MLflow UI -MLFLOW_IP=$(kubectl get svc mlflow-service -o jsonpath='{.status.loadBalancer.ingress[0].ip}') -open http://$MLFLOW_IP:5000 - -# Navigate to: -# 1. "Models" tab -# 2. Find "HousingPriceModel" -# 3. Verify Production stage -``` - ---- - -## Deploy FastAPI - -### Step 1: Update Config File - -Ensure FastAPI is in your config: - -```yaml -stack: - # ... MLflow config ... - - - model_serving: - name: fastapi - params: - image: fastapi-mlflow-demo:latest - mlflow_tracking_uri: http://mlflow-service:5000 # Kubernetes internal DNS -``` - -### Step 2: Deploy FastAPI - -```bash -# Deploy (will deploy both MLflow and FastAPI) -deployml deploy -c gke-deploy-config.yaml - -# Or deploy only FastAPI by commenting out MLflow in config -``` - -### Step 3: Verify FastAPI Deployment - -```bash -# Check FastAPI pod -kubectl get pods -l app=fastapi - -# Check FastAPI service -kubectl get svc fastapi-service - -# Get FastAPI URL -FASTAPI_IP=$(kubectl get svc fastapi-service -o jsonpath='{.status.loadBalancer.ingress[0].ip}') -echo "FastAPI URL: http://34.82.236.4 :8000" - -# Test health endpoint -curl http://34.82.236.4:8000/health - -# Open Swagger UI -open http://$FASTAPI_IP:8000/docs -``` - ---- - -## Testing and Verification - -### Test MLflow - -```bash -# Get MLflow URL -MLFLOW_IP=$(kubectl get svc mlflow-service -o jsonpath='{.status.loadBalancer.ingress[0].ip}') - -# Test health -curl http://$MLFLOW_IP:5000/health - -# Open UI -open http://$MLFLOW_IP:5000 -``` - -### Test FastAPI - -```bash -# Get FastAPI URL -FASTAPI_IP=$(kubectl get svc fastapi-service -o jsonpath='{.status.loadBalancer.ingress[0].ip}') - -# Test health -curl http://$FASTAPI_IP:8000/health - -# Test root endpoint -curl http://$FASTAPI_IP:8000/ - -# Test prediction (if model is registered) -curl -X POST "http://$FASTAPI_IP:8000/predict" \ - -H "Content-Type: application/json" \ - -d '{ - "features": { - "bedrooms": 3, - "bathrooms": 2, - "area_sqft": 2000, - "lot_size": 5000, - "year_built": 2010, - "city": 1, - "state": 0 - } - }' - -# Open Swagger UI -open http://$FASTAPI_IP:8000/docs -``` - -### Check Logs - -```bash -# MLflow logs -kubectl logs -l app=mlflow --tail=50 -f - -# FastAPI logs -kubectl logs -l app=fastapi --tail=50 -f - -# Check if model loaded in FastAPI -kubectl logs -l app=fastapi | grep -i "model" -``` - ---- - -## Troubleshooting - -### Issue: Pod Stuck in `Pending` Status - -**Symptoms:** -``` -NAME READY STATUS RESTARTS AGE -fastapi-deployment-XXXXX-XXXXX 0/1 Pending 0 30m -``` - -**Diagnosis:** - -```bash -# Check why pod is pending -kubectl describe pod POD_NAME - -# Look for Events section - common causes: -# - "Insufficient cpu" or "Insufficient memory" -# - "ImagePullBackOff" or "ErrImagePull" -# - "0/2 nodes are available" -``` - -**Solutions:** - -#### Solution 1: Insufficient CPU/Memory - -**Reduce Resource Requests:** - -```bash -# Patch deployment to reduce CPU/memory requests -kubectl patch deployment DEPLOYMENT_NAME -p '{ - "spec": { - "template": { - "spec": { - "containers": [{ - "name": "CONTAINER_NAME", - "resources": { - "requests": { - "cpu": "100m", - "memory": "256Mi" - }, - "limits": { - "cpu": "500m", - "memory": "1Gi" - } - } - }] - } - } - } -}' - -# Example for FastAPI: -kubectl patch deployment fastapi-deployment -p '{ - "spec": { - "template": { - "spec": { - "containers": [{ - "name": "fastapi", - "resources": { - "requests": { - "cpu": "100m", - "memory": "256Mi" - } - } - }] - } - } - } -}' - -# Delete pending pod (will recreate with new resources) -kubectl delete pod POD_NAME -``` - -**Or Edit Deployment:** - -```bash -# Edit deployment interactively -kubectl edit deployment DEPLOYMENT_NAME - -# Change resources section: -resources: - requests: - cpu: "100m" # Reduced from 250m - memory: "256Mi" # Reduced from 512Mi - limits: - cpu: "500m" - memory: "1Gi" -``` - -**Scale Up Cluster:** - -```bash -# Add more nodes to cluster -gcloud container clusters resize CLUSTER_NAME \ - --num-nodes 3 \ - --zone ZONE \ - --project PROJECT_ID - -# Or use larger machine types -gcloud container node-pools create larger-pool \ - --cluster CLUSTER_NAME \ - --machine-type e2-standard-4 \ - --num-nodes 2 \ - --zone ZONE \ - --project PROJECT_ID -``` - -#### Solution 2: Image Pull Error - -**Check if Image Exists:** - -```bash -# Check if image is in GCR -gcloud container images list-tags gcr.io/PROJECT_ID/SERVICE/IMAGE --project=PROJECT_ID - -# Example: -gcloud container images list-tags gcr.io/mldeploy-468919/fastapi/fastapi --project=mldeploy-468919 -``` - -**Push Image Manually:** - -```bash -# Build image -cd demo/fastapi -docker build --platform linux/amd64 -t fastapi-mlflow-demo:latest . - -# Tag for GCR -docker tag fastapi-mlflow-demo:latest gcr.io/PROJECT_ID/fastapi/fastapi:latest - -# Push to GCR -docker push gcr.io/PROJECT_ID/fastapi/fastapi:latest - -# Delete pod to retry image pull -kubectl delete pod POD_NAME -``` - -#### Solution 3: Check Node Resources - -```bash -# Check node capacity -kubectl describe nodes - -# Check current resource usage -kubectl top nodes -kubectl top pods - -# Check what's using resources -kubectl get pods --all-namespaces -o wide -``` - -### Issue: Pod in `ImagePullBackOff` Status - -**Symptoms:** -``` -NAME READY STATUS RESTARTS AGE -fastapi-deployment-XXXXX-XXXXX 0/1 ImagePullBackOff 0 5m -``` - -**Solutions:** - -```bash -# 1. Check if image exists in GCR -gcloud container images list-tags gcr.io/PROJECT_ID/fastapi/fastapi --project=PROJECT_ID - -# 2. If missing, push image -docker tag fastapi-mlflow-demo:latest gcr.io/PROJECT_ID/fastapi/fastapi:latest -docker push gcr.io/PROJECT_ID/fastapi/fastapi:latest - -# 3. Check image pull secrets (usually not needed for GCR) -kubectl get secrets - -# 4. Delete pod to retry -kubectl delete pod POD_NAME -``` - -### Issue: Pod in `CrashLoopBackOff` Status - -**Symptoms:** -``` -NAME READY STATUS RESTARTS AGE -mlflow-deployment-XXXXX-XXXXX 0/1 CrashLoopBackOff 5 10m -``` - -**Solutions:** - -```bash -# 1. Check pod logs -kubectl logs POD_NAME --tail=100 - -# 2. Check previous container logs (if restarted) -kubectl logs POD_NAME --previous - -# 3. Describe pod for events -kubectl describe pod POD_NAME - -# 4. Common fixes: -# - Increase memory limits (OOM errors) -# - Fix environment variables -# - Check application logs for errors -``` - -### Issue: LoadBalancer IP Stuck in `` - -**Symptoms:** -``` -NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) -fastapi-service LoadBalancer 10.XXX.XXX.X 8000:XXXXX/TCP -``` - -**Solutions:** - -```bash -# 1. Wait (can take 2-5 minutes) -kubectl get svc fastapi-service -w - -# 2. Check service events -kubectl describe svc fastapi-service - -# 3. Verify pod is running first -kubectl get pods -l app=fastapi - -# 4. Check firewall rules (usually auto-created) -gcloud compute firewall-rules list --filter="name~gke" - -# 5. If stuck >10 minutes, delete and recreate service -kubectl delete svc fastapi-service -kubectl apply -f service.yaml -``` - -### Issue: FastAPI Can't Connect to MLflow - -**Symptoms:** -- FastAPI logs show: "Could not load model from MLflow" -- Health endpoint shows `"model_loaded": false` - -**Solutions:** - -```bash -# 1. Verify MLflow service exists -kubectl get svc mlflow-service - -# 2. Test from FastAPI pod -kubectl exec -it FASTAPI_POD_NAME -- curl http://mlflow-service:5000/health - -# 3. Check FastAPI environment variables -kubectl exec FASTAPI_POD_NAME -- env | grep MLFLOW - -# 4. Verify MLflow pod is running -kubectl get pods -l app=mlflow - -# 5. Check network policies (if any) -kubectl get networkpolicies -``` - ---- - -## Resource Management - -### Check Current Resource Usage - -```bash -# Node resources -kubectl top nodes - -# Pod resources -kubectl top pods - -# Detailed node info -kubectl describe nodes -``` - -### Reduce Resource Requests (Fix Pending Pods) - -**Method 1: Patch Deployment** - -```bash -# Reduce CPU and memory requests -kubectl patch deployment DEPLOYMENT_NAME -p '{ - "spec": { - "template": { - "spec": { - "containers": [{ - "name": "CONTAINER_NAME", - "resources": { - "requests": { - "cpu": "100m", - "memory": "256Mi" - }, - "limits": { - "cpu": "500m", - "memory": "1Gi" - } - } - }] - } - } - } -}' -``` - -**Method 2: Edit Deployment** - -```bash -# Edit deployment -kubectl edit deployment DEPLOYMENT_NAME - -# Change resources: -resources: - requests: - cpu: "100m" # Minimum: 100m - memory: "256Mi" # Minimum: 256Mi - limits: - cpu: "500m" - memory: "1Gi" -``` - -**Method 3: Update Manifest File** - -```bash -# Edit generated manifest -nano .deployml/gke-mlflow-test/manifests/fastapi/deployment.yaml - -# Change resources section, then apply: -kubectl apply -f .deployml/gke-mlflow-test/manifests/fastapi/deployment.yaml -``` - -### Increase Resource Requests - -```bash -# Increase CPU and memory -kubectl patch deployment DEPLOYMENT_NAME -p '{ - "spec": { - "template": { - "spec": { - "containers": [{ - "name": "CONTAINER_NAME", - "resources": { - "requests": { - "cpu": "500m", - "memory": "1Gi" - }, - "limits": { - "cpu": "2000m", - "memory": "4Gi" - } - } - }] - } - } - } -}' -``` - -### Scale Up Cluster (Add More Resources) - -```bash -# Add more nodes -gcloud container clusters resize CLUSTER_NAME \ - --num-nodes 3 \ - --zone ZONE \ - --project PROJECT_ID - -# Or create node pool with larger machines -gcloud container node-pools create larger-pool \ - --cluster CLUSTER_NAME \ - --machine-type e2-standard-4 \ - --num-nodes 2 \ - --zone ZONE \ - --project PROJECT_ID -``` - -### Scale Down Cluster (Reduce Costs) - -```bash -# Reduce number of nodes -gcloud container clusters resize CLUSTER_NAME \ - --num-nodes 1 \ - --zone ZONE \ - --project PROJECT_ID -``` - -### Recommended Resource Settings - -**For Small Clusters (2 nodes × e2-medium):** - -```yaml -# MLflow -resources: - requests: - cpu: "250m" - memory: "512Mi" - limits: - cpu: "1000m" - memory: "2Gi" - -# FastAPI -resources: - requests: - cpu: "100m" # Reduced to fit - memory: "256Mi" # Reduced to fit - limits: - cpu: "500m" - memory: "1Gi" -``` - -**For Medium Clusters (3+ nodes):** - -```yaml -# MLflow -resources: - requests: - cpu: "500m" - memory: "1Gi" - limits: - cpu: "2000m" - memory: "4Gi" - -# FastAPI -resources: - requests: - cpu: "250m" - memory: "512Mi" - limits: - cpu: "1000m" - memory: "2Gi" -``` - ---- - -## Cleanup - -### Delete Kubernetes Resources - -```bash -# Delete deployments (also deletes pods and ReplicaSets automatically) -kubectl delete deployment fastapi-deployment -kubectl delete deployment mlflow-deployment - -# Delete services -kubectl delete service fastapi-service -kubectl delete service mlflow-service - -# Verify deletion -kubectl get pods -kubectl get services -kubectl get deployments -``` - -### Delete Using Labels - -```bash -# Delete all FastAPI resources -kubectl delete all -l app=fastapi - -# Delete all MLflow resources -kubectl delete all -l app=mlflow -``` - -### Delete Docker Images from GCR - -```bash -# Delete FastAPI image -gcloud container images delete gcr.io/PROJECT_ID/fastapi/fastapi:latest \ - --project PROJECT_ID - -# Delete MLflow image -gcloud container images delete gcr.io/PROJECT_ID/mlflow/mlflow:latest \ - --project PROJECT_ID - -# List remaining images -gcloud container images list --project PROJECT_ID -``` - -### Delete GKE Cluster (Complete Cleanup) - -```bash -# Delete entire cluster -gcloud container clusters delete CLUSTER_NAME \ - --zone ZONE \ - --project PROJECT_ID - -# This deletes: -# - All pods -# - All services -# - All deployments -# - The cluster itself -``` - -### Clean Up Local Files - -```bash -# Delete generated manifests (optional) -rm -rf .deployml/gke-mlflow-test - -# Delete local Docker images (optional) -docker rmi mlflow-demo:latest -docker rmi fastapi-mlflow-demo:latest -``` - ---- - -## Quick Reference - -### Common Commands - -```bash -# Get service URLs -MLFLOW_IP=$(kubectl get svc mlflow-service -o jsonpath='{.status.loadBalancer.ingress[0].ip}') -FASTAPI_IP=$(kubectl get svc fastapi-service -o jsonpath='{.status.loadBalancer.ingress[0].ip}') - -# Check pods -kubectl get pods -kubectl get pods -l app=mlflow -kubectl get pods -l app=fastapi - -# Check services -kubectl get svc -kubectl get svc mlflow-service -kubectl get svc fastapi-service - -# View logs -kubectl logs -l app=mlflow --tail=50 -f -kubectl logs -l app=fastapi --tail=50 -f - -# Restart deployments -kubectl rollout restart deployment/mlflow-deployment -kubectl rollout restart deployment/fastapi-deployment - -# Describe resources -kubectl describe pod POD_NAME -kubectl describe deployment DEPLOYMENT_NAME -kubectl describe svc SERVICE_NAME -``` - -### Resource Management Commands - -```bash -# Check resource usage -kubectl top nodes -kubectl top pods - -# Reduce resources (fix pending pods) -kubectl patch deployment DEPLOYMENT_NAME -p '{"spec":{"template":{"spec":{"containers":[{"name":"CONTAINER_NAME","resources":{"requests":{"cpu":"100m","memory":"256Mi"}}}]}}}}' - -# Scale cluster -gcloud container clusters resize CLUSTER_NAME --num-nodes 3 --zone ZONE --project PROJECT_ID -``` - -### Deployment Commands - -```bash -# Deploy everything -deployml deploy -c gke-deploy-config.yaml - -# Deploy only MLflow (comment out FastAPI in config) -deployml deploy -c gke-deploy-config.yaml - -# Deploy only FastAPI (comment out MLflow in config) -deployml deploy -c gke-deploy-config.yaml -``` - -### Testing Commands - -```bash -# Test MLflow -curl http://MLFLOW_IP:5000/health -open http://MLFLOW_IP:5000 - -# Test FastAPI -curl http://FASTAPI_IP:8000/health -curl -X POST "http://FASTAPI_IP:8000/predict" -H "Content-Type: application/json" -d '{"features": {...}}' -open http://FASTAPI_IP:8000/docs -``` - ---- - -## Architecture Diagram - -``` -┌─────────────────────────────────────────────────────────┐ -│ Your Local Machine │ -│ │ -│ ┌──────────────┐ ┌──────────────┐ │ -│ │ Training │ │ Browser │ │ -│ │ Script │────────▶│ (UI) │ │ -│ └──────────────┘ └──────────────┘ │ -│ │ │ │ -│ │ http://34.182.23.6:5000│ │ -│ │ │ │ -└─────────┼────────────────────────┼──────────────────────┘ - │ │ - │ │ -┌─────────▼────────────────────────▼──────────────────────┐ -│ GKE Kubernetes Cluster │ -│ │ -│ ┌──────────────────┐ ┌──────────────────┐ │ -│ │ MLflow Pod │ │ FastAPI Pod │ │ -│ │ Port: 5000 │◀─────│ Port: 8000 │ │ -│ │ │ │ │ │ -│ │ Service: │ │ Service: │ │ -│ │ mlflow-service │ │ fastapi-service │ │ -│ │ :5000 │ │ :8000 │ │ -│ └──────────────────┘ └──────────────────┘ │ -│ │ │ -│ │ http://mlflow-service:5000 │ -│ │ │ -│ ┌──────▼────────────────────────────────────┐ │ -│ │ MLflow Model Registry │ │ -│ │ - HousingPriceModel │ │ -│ │ - Production Stage │ │ -│ └──────────────────────────────────────────┘ │ -└───────────────────────────────────────────────────────────┘ -``` - ---- - -## Key Points Summary - -1. **GKE Cluster:** Must exist before deployment (create with `gcloud container clusters create`) -2. **Images:** Must be pushed to GCR (done automatically by deployml or manually) -3. **Service Names:** Use Kubernetes internal DNS (`mlflow-service:5000`) for pod-to-pod communication -4. **LoadBalancer IPs:** Use external IPs (`34.182.23.6:5000`) for browser/local access -5. **Resource Limits:** Adjust if pods are pending due to insufficient CPU/memory -6. **Deployment:** Use `deployml deploy -c config.yaml` (same command for all deployment types) - ---- - -## Support - -If you encounter issues: - -1. Check pod status: `kubectl get pods` -2. Check logs: `kubectl logs POD_NAME` -3. Describe pod: `kubectl describe pod POD_NAME` -4. Check events: `kubectl get events --sort-by='.lastTimestamp'` -5. Verify cluster: `kubectl cluster-info` -6. Check resources: `kubectl top nodes` and `kubectl top pods` - -Good luck with your GKE deployment! 🚀 - diff --git a/recycling_bin/MLOps project plan.pdf b/recycling_bin/MLOps project plan.pdf deleted file mode 100644 index c859c15..0000000 Binary files a/recycling_bin/MLOps project plan.pdf and /dev/null differ diff --git a/recycling_bin/PROJECT_PLAN.md b/recycling_bin/PROJECT_PLAN.md deleted file mode 100644 index 2102af3..0000000 --- a/recycling_bin/PROJECT_PLAN.md +++ /dev/null @@ -1,95 +0,0 @@ -# MLOps Application — Project Plan - -## Project Description - -Easy-to-use, scalable, modular, and lightweight library for deploying MLOps infrastructure in the cloud, targeting academic use (GCP → AWS → Azure). Goal is to reduce friction for instructors and students so they can focus on building rather than debugging environment setup. - -## Goals - -- Software application with documentation and website -- Paper for publication (JOSS, pyopensci, or a journal) - -## Phases - -- **Phase 1** — GCP: one tool per MLOps stage, tested, documented, website with end-to-end example. Target: early September 2025. -- **Phase 2** — AWS: extend, test, update docs/website/example. Target: December 2025. -- **Phase 3** — Azure: lower priority. - -## MLOps Tools - -| Stage | Tool | Notes | -|---|---|---| -| Experiment Tracking | MLflow | Tracking server + Cloud SQL + database | -| Artifact Tracking | MLflow | GCS bucket + tracking server | -| Model Registry | MLflow | GCS bucket + tracking server | -| Model Serving | FastAPI (online) | Container in Cloud Run pulling latest registered model | -| Model Monitoring | From scratch | BigQuery/Postgres tables + Grafana dashboard + scheduled container for drift metrics | -| Operational Monitoring | Grafana | Cloud Run container connected to monitoring tables | -| Feature Store | From scratch | BigQuery tables | -| Data Versioning | Not included | — | - -## App Commands - -| Command | Description | -|---|---| -| `doctor` | System checks for local dependencies | -| `init` | Enable APIs, create empty config + example Dockerfiles | -| `build-images` | Build all images in `docker/` folder in GCP Artifact Registry | -| `deploy` | Render Terraform templates from config.yaml and apply | -| `get-urls` | Extract infra connection info / URLs | -| `destroy` | Terraform destroy wrapper | - -## Repo Structure (relevant parts) - -| Folder | Subfolder | Purpose | -|---|---|---| -| `docs/` | | MKdocs documentation files | -| `example/config/` | | Sample configs — focus on `gcp-sample.yaml` / `gcp-sample-slim.yaml` | -| `notebooks/` | | Test notebooks — focus on `notebook_cli_demo.ipynb` | -| `src/deployml/cli/` | | CLI command implementations | -| `src/deployml/diagnostics/` | | Doctor command | -| `src/deployml/docker/` | | Example Dockerfiles shipped with the app | -| `src/deployml/notebook/` | | Jupyter notebook API (non-CLI interface) | -| `src/deployml/templates/gcp/cloud_run/` | | Jinja2 Terraform templates — main focus | -| `src/deployml/terraform/modules/` | | Terraform modules: `bigquery/`, `cloud_sql_postgres/`, `fastapi/`, `mlflow/`, `grafana/`, `teardown/` | -| `src/deployml/utils/` | | Helper functions for CLI and notebook | - -## What Needs to be Completed - -### Cloud Run Services - -- [x] **1. Grafana in Cloud Run** — working as of last session -- [x] **2. MLflow in Cloud Run** — working as of last session - -### Infrastructure - -- [x] **3. BigQuery table auto-creation** — wired into deploy; creates `mlops` dataset with all 4 tables - - `offline_features` - - `predictions` - - `ground_truth` - - `drift_metrics` -- [x] **4. .env file generation** — `deployml get-urls` prints URLs and writes `.env` - -### End-to-End Example - -- [ ] **5a.** Example training dataset → load into BigQuery -- [ ] **5b.** Training script with MLflow experiment tracking -- [ ] **5c.** Model registration in MLflow -- [ ] **5d.** Feature creation script → store in `offline_features` BigQuery table -- [ ] **5e.** Model scoring script for FastAPI container (pulls latest registered model from MLflow) -- [ ] **5f.** Prediction script → call deployed FastAPI, store results in `predictions` BigQuery table -- [ ] **5g.** Fake ground truth script → populate `ground_truth` BigQuery table -- [ ] **5h.** Drift metrics script → compute and store in `drift_metrics` BigQuery table -- [ ] **5i.** Grafana model monitoring dashboard - -### Documentation - -- [ ] MKdocs site with MKdocstrings for API docs -- [ ] End-to-end example page on website - -## Notes / Known Issues - -- Terraform state lives in `.deployml//`; lock file may need manual deletion between deploys -- Feast, cron, Weights & Biases, VM, and Local options exist in the repo but are deprioritized — may be removed later to streamline -- GCP deploy uses Cloud Run (no K8s) -- `pip install -e .` for local development diff --git a/recycling_bin/README_vm_FEAST.md b/recycling_bin/README_vm_FEAST.md deleted file mode 100644 index 74f5e8c..0000000 --- a/recycling_bin/README_vm_FEAST.md +++ /dev/null @@ -1,319 +0,0 @@ -# Feast Feature Store Setup and Data Upload Guide - -This README documents the complete process of setting up a Feast feature store on a Google Cloud VM and successfully uploading a parquet file (`house_data.parquet`) to it. - -## Overview - -We successfully deployed a Feast feature store on a GCP VM with: -- **PostgreSQL registry and online store** (Cloud SQL) -- **BigQuery offline store** (dataset + table) -- **House sales records** loaded and materialized -- **Features**: price, city, state, bedrooms, bathrooms, area_sqft, lot_size, year_built, days_on_market, property_type, listing_agent, status, zipcode_encoded - -## Prerequisites - -- GCP VM instance running (`mlflow-postgres-vm-instance`) -- Docker and Docker Compose installed on the VM -- Feast server running in a Docker container -- PostgreSQL database accessible (Cloud SQL instance) -- Parquet file (`house_data.parquet`) with house sales data - -## Step-by-Step Process - -### .1 If Using CloudSQL Backend - -1. Navigate to Cloud SQL Instances : In the Google Cloud console, go to the Cloud SQL Instances page. -2. Select your instance : Click on the name of the Cloud SQL instance you want to modify. This will take you to its Overview page. -3. Edit the instance : Click the "Edit" button at the top of the instance's Overview page. -4. Find the "Flags and parameters" section : Scroll down the configuration options until you find the "Flags and parameters" section under "Advanced options." -5. Add or modify the max_connections flag : - - If the max_connections flag is not already listed, click "Add a database flag." - - From the dropdown menu, select max_connections. - - Set max_connections to 20. -6. Save changes : Click "Save" at the bottom of the page to apply your changes. - -### 1. Initial Setup and Troubleshooting - -#### 1.1 Prepare data in BigQuery -```bash -# Load parquet into BigQuery table -bq --project_id= --location=us-west2 load \ - --source_format=PARQUET \ - :feast_offline_store.house_data /path/to/house_data.parquet -``` - -#### 1.2 Connect to the VM -```bash -gcloud compute ssh --zone us-west2-a mlflow-postgres-vm-instance -``` - -#### 1.3 Check Running Containers -```bash -sudo docker ps -# Look for feast-server container -``` - -### 2. Feast Configuration - -#### 2.1 Create Feature Store Configuration -Create `feature_store.yaml` for BigQuery offline store. Choose ONE registry option below. - -Option A) SQL registry (PostgreSQL/Cloud SQL): -```yaml -project: house_sales -provider: gcp -registry: - registry_type: sql - path: postgresql+psycopg://feast:@:5432/feast - cache_ttl_seconds: 60 - sqlalchemy_config_kwargs: - echo: false - pool_pre_ping: true -online_store: - type: postgres - host: - port: 5432 - database: feast - user: feast - password: -offline_store: - type: bigquery - project_id: - dataset: feast_offline_store -entity_key_serialization_version: 3 -``` - -Option B) File registry (no SQL connections): -```yaml -project: house_sales -provider: gcp -registry: - registry_type: file - path: data/registry.db -online_store: - type: postgres - host: - port: 5432 - database: feast - user: feast - password: -offline_store: - type: bigquery - project_id: - dataset: feast_offline_store -entity_key_serialization_version: 3 -``` - -**Key Points:** -- Use the Cloud SQL IP for `` (not localhost) when using SQL registry/online store. -- Offline store is BigQuery; set `` and ensure `feast_offline_store` dataset exists. -- File registry avoids Cloud SQL connection limits during `feast apply`. - -#### 2.2 Copy Configuration to Container -```bash -# Copy config to the Feast container -docker cp feature_store.yaml feast-server:/app/feature_repo/feature_store.yaml -docker cp feature_store.yaml feast-server:/app/feature_store.yaml -``` - -### 3. Feature Definitions - -**⚠️ IMPORTANT: Create these files on the HOST VM first, then copy them to the container** - -#### 3.1 Create Entity Definition (`entities.py`) -Create this file on the VM (e.g., in your working directory): -```python -from feast import Entity, ValueType - -house_entity = Entity( - name="mls_id", - value_type=ValueType.INT64, - description="Unique identifier for a house", - join_keys=["mls_id"] # This matches your parquet column -) -``` - -#### 3.2 Create Data Source (`data_sources.py`) -Create this file on the VM (BigQuery source with only `timestamp_field`): -```python -from feast import BigQuerySource - -house_source = BigQuerySource( - table=".feast_offline_store.house_data", - timestamp_field="event_timestamp", # set to your actual timestamp column - # created_timestamp_column="created # optional; if multiple entries are entered for a row-tntity event -) -``` - -#### 3.3 Create Feature View (`house_features.py`) -Create this file on the VM (fields must match your BigQuery table columns exactly): -```python -from datetime import timedelta -from feast import FeatureView, Field -from feast.types import Float64, Int64 -from feature_repo.entities import house_entity -from feature_repo.data_sources import house_source - -house_features = FeatureView( - name="house_features", - entities=[house_entity], - ttl=timedelta(weeks=52), - schema=[ - Field(name="price", dtype=Float64), - Field(name="city", dtype=Int64), - Field(name="state", dtype=Int64), - Field(name="bedrooms", dtype=Int64), - Field(name="bathrooms", dtype=Int64), - Field(name="area_sqft", dtype=Int64), - Field(name="lot_size", dtype=Int64), - Field(name="year_built", dtype=Int64), - Field(name="days_on_market", dtype=Int64), - Field(name="property_type", dtype=Int64), - Field(name="listing_agent", dtype=Int64), - Field(name="status", dtype=Int64), - Field(name="zipcode_encoded", dtype=Float64), - ], - source=house_source, - tags={"team": "house_sales"}, -) -``` - -**⚠️ IMPORTANT: Repo layout and imports** -- Container path: `/app/feature_repo` -- Place `entities.py` and `data_sources.py` at repo root; place `house_features.py` under `/app/feature_repo/features/`. -- Keep these imports in `features/house_features.py` (absolute from repo root): - - `from entities import house_entity` - - `from data_sources import house_source` - -#### 3.4 Move Feature Files to the Container -After creating the files on the VM, move them into the Feast container with the expected layout and ensure packages are initialized. The deploy now scaffolds `__init__.py` files automatically, but these commands are provided for manual updates: - -```bash -# Create directories and package markers -docker exec -it feast-server mkdir -p /app/feature_repo/features -docker exec -it feast-server sh -lc 'touch /app/feature_repo/__init__.py /app/feature_repo/features/__init__.py' - -# Copy files to the correct locations -docker cp entities.py feast-server:/app/feature_repo/entities.py -docker cp data_sources.py feast-server:/app/feature_repo/data_sources.py -docker cp house_features.py feast-server:/app/feature_repo/features/house_features.py - -# Verify -docker exec -it feast-server ls -la /app/feature_repo/ -docker exec -it feast-server ls -la /app/feature_repo/features/ - -# Optional: Remove the original files from the VM since they're now in the container -rm entities.py data_sources.py house_features.py -``` - -**Key Points:** -- **Entity join key** must match your data column (`mls_id`) -- **Schema fields** must match your parquet columns exactly -- **Data source path** must be accessible from within the container - -### 4. Deploy Features - -#### 4.1 Apply Feature Definitions -```bash -docker exec -it feast-server feast apply -``` - -**Expected Output:** -``` -Applying changes for project house_sales -Deploying infrastructure for house_features -``` - -#### 4.2 Verify Registration -```bash -# List projects -docker exec -it feast-server feast projects list - -# List entities -docker exec -it feast-server feast entities list - -# List features -docker exec -it feast-server feast features list - -# List data sources -docker exec -it feast-server feast data-sources list -``` - -### 5. Data Materialization -#### 5.1 Materialize Data to Online Store -```bash -docker exec -it feast-server feast materialize \ - --views house_features \ - 2017-06-04T00:00:00 2025-08-20T23:59:59 -``` - -**⚠️ CRITICAL: Use the ACTUAL timestamp range from your data!** - -**Expected Output:** -``` -Materializing 1 feature views from 2017-06-04 00:00:00+00:00 to 2025-08-20 23:59:59+00:00 into the postgres online store. - -house_features: -100%|███████████████████████████████████████████████████████████| 3000+/3000+ [00:XX<00:00, XXX.XXit/s] -``` - -**Why This Matters:** -- **Wrong range** (e.g., 2024-01-01 to 2024-12-31): No data loaded, results in `null` values -- **Correct range** (e.g., 2017-06-04 to 2025-08-20): All data loaded, features return actual values -- **Check your data**: Use `df["event_timestamp"].min()` and `df["event_timestamp"].max()` to find the actual range - -### 6. Query Features - -#### 6.1 Get Online Features -```bash -docker exec -it feast-server feast get-online-features \ - --features house_features:price \ - --entities mls_id=112914 -``` - -**Expected Output:** -```json -{ - "mls_id": [112914], - "price": [843750] -} -``` - -#### 6.2 Get Multiple Features -```bash -docker exec -it feast-server python -c " -from feast import FeatureStore -store = FeatureStore('/app/feature_repo') - -# Get multiple features -features = store.get_online_features( - features=['house_features:price', 'house_features:bedrooms', 'house_features:bathrooms'], - entity_rows=[{'mls_id': 112914}] -) -print('Features retrieved successfully:') -print(features.to_dict()) -" -``` - -**Expected Output:** -```json -{ - "mls_id": [112914], - "bedrooms": [1], - "price": [843750.0], - "bathrooms": [3.0] -} -``` - -### 7. Troubleshooting - -- BigQuery table not found: - - Ensure dataset and table exist: `bq ls --project_id=hatchet16 --location=us-west2 feast_offline_store` - - Load data: `bq load --source_format=PARQUET hatchet16:feast_offline_store.house_data /path/to/house_data.parquet` -- Column errors (e.g., `created` not found): - - Remove `created_timestamp_column` from `BigQuerySource` or point to a real column. -- Registry tips: - - If Cloud SQL connections are tight, switch to file registry (`registry_type: file`, `path: data/registry.db`). - - After a successful `feast apply`, restart the Feast server if changes don’t appear immediately. -- Credentials issue: - - Do not set `GOOGLE_APPLICATION_CREDENTIALS` inside the container; rely on VM service account. \ No newline at end of file diff --git a/recycling_bin/config/deployml-489218.yaml b/recycling_bin/config/deployml-489218.yaml deleted file mode 100644 index 1de4c00..0000000 --- a/recycling_bin/config/deployml-489218.yaml +++ /dev/null @@ -1,42 +0,0 @@ -name: gcp-mlops-stack-mlflow -provider: - name: gcp - project_id: deployml-489218 - region: us-west1 -deployment: - type: cloud_run -stack: - - experiment_tracking: - name: mlflow - params: - service_name: mlflow-server - - artifact_tracking: - name: mlflow - - model_registry: - name: mlflow - params: - backend_store_uri: postgresql - # - feature_store: - # name: feast - # params: - # service_name: feast-server - # backend_store_uri: postgresql - # offline_store: bigquery - # bigquery_dataset: deployml-489218.feast_housing.house_data - - model_serving: - name: fastapi - params: - service_name: fastapi-mlflow-server - - model_monitoring: - name: grafana - params: - service_name: grafana-server - # - workflow_orchestration: - # name: cron - # params: - # jobs: - # - service_name: offline-scoring - # cron_schedule: "0 12 */14 * *" - # bigquery_dataset: feast_housing - # - service_name: metrics-monitoring - # cron_schedule: "0 6 * * *" diff --git a/recycling_bin/config/gcp-cloud-vm-sample.yaml b/recycling_bin/config/gcp-cloud-vm-sample.yaml deleted file mode 100644 index 3fb9199..0000000 --- a/recycling_bin/config/gcp-cloud-vm-sample.yaml +++ /dev/null @@ -1,82 +0,0 @@ -name: gcp-mlops-stack-mlflow-vm_TESTING -provider: - name: gcp - project_id: hatchet17 - region: us-west2 - zone: us-west2-a - -cost_analysis: - enabled: true # Enable/disable cost analysis (default: true) - warning_threshold: 50.0 # Warn if monthly cost exceeds this amount (default: 100.0) - currency: "USD" - bucket_amount: 200 # GB stored across GCS buckets - cloudsql_amount: 50 # GB of Cloud SQL storage - -deployment: - type: cloud_vm - -stack: - - experiment_tracking: - name: mlflow - params: - # service_name: mlflow-server-vm - service_name: mlflow-server-postgres-vm # for postgres backend - # vm_name: mlflow-vm-instance - vm_name: mlflow-postgres-vm-instance # for postgres backend - machine_type: e2-medium - disk_size_gb: 20 - mlflow_port: 5000 - allow_public_access: true - # FastAPI configuration - fastapi_port: 8000 - fastapi_app_source: "template" # or "gs://bucket/path.py" or "/local/path.py" - - artifact_tracking: - name: mlflow - params: - # artifact_bucket: mlflow-artifact-bucket-vm-54321 - artifact_bucket: mlflow-artifact-bucket-postgres-2-hatchet16-sacwtaag - create_artifact_bucket: true - - model_registry: - name: mlflow - params: - # backend_store_uri: sqlite:///mlflow.db # SQLite backend (local file) - backend_store_uri: postgresql # for postgres backend - - feature_store: - name: feast - params: - service_name: feast-server - feast_port: 6566 - backend_store_uri: postgresql # Uses same PostgreSQL as MLflow - registry_type: sql # or "file" for development - online_store_type: sqlite # sqlite, datastore, bigtable - offline_store_type: bigquery - bigquery_dataset: feast_offline_store - create_bigquery_dataset: true - sample_data: false # Set to true to create sample sales data table - project: house_sales - - model_monitoring: - name: grafana - params: - service_name: grafana-server - enable_grafana: true - grafana_port: 3000 - grafana_admin_user: admin - grafana_admin_password: admin - - workflow_orchestration: - name: airflow - params: - service_name: airflow-server - airflow_port: 8080 - airflow_worker_port: 8793 - airflow_flower_port: 5555 - backend_store_uri: postgresql - airflow_database_name: airflow - airflow_database_user: airflow - airflow_separate_database: true - airflow_admin_user: admin - airflow_admin_password: admin123 - airflow_executor: LocalExecutor - airflow_parallelism: 4 - airflow_dag_concurrency: 2 - airflow_max_active_runs_per_dag: 2 - airflow_webserver_workers: 1 # Set to 1 worker to fix child process issues diff --git a/recycling_bin/config/gcp-gke-sample.yaml b/recycling_bin/config/gcp-gke-sample.yaml deleted file mode 100644 index 246aa79..0000000 --- a/recycling_bin/config/gcp-gke-sample.yaml +++ /dev/null @@ -1,46 +0,0 @@ -name: gcp-mlops-stack-mlflow-gke -provider: - name: gcp - project_id: YOUR_PROJECT_ID - region: us-west1 - -deployment: - type: gke - -gke: - cluster_name: my-gke-cluster - zone: us-west1-a # For zonal cluster - # OR use region for regional cluster: - # region: us-west1 - -stack: - - experiment_tracking: - name: mlflow - params: - image: mlflow-demo:latest # Local image name, will be pushed to GCR - service_name: mlflow-service - service_type: LoadBalancer - memory_limit: 2Gi - cpu_limit: 1000m - backend_store_uri: postgresql # or sqlite:///mlflow.db - - - artifact_tracking: - name: mlflow - params: - artifact_bucket: mlflow-artifacts-bucket - create_artifact_bucket: true - - - model_registry: - name: mlflow - params: - backend_store_uri: postgresql - - - model_serving: - name: fastapi - params: - image: fastapi-mlflow-demo:latest # Local image name - service_name: fastapi-service - service_type: LoadBalancer - mlflow_tracking_uri: http://mlflow-service:5000 - memory_limit: 1Gi - cpu_limit: 1000m diff --git a/recycling_bin/config/gcp-sample.yaml b/recycling_bin/config/gcp-sample.yaml deleted file mode 100644 index a7c74d9..0000000 --- a/recycling_bin/config/gcp-sample.yaml +++ /dev/null @@ -1,65 +0,0 @@ -name: gcp-mlops-stack-mlflow -provider: - name: gcp - project_id: mlops-intro-461805 - region: us-west1 -deployment: - type: cloud_run -stack: - - experiment_tracking: - name: mlflow - params: - image: gcr.io/mlops-intro-461805/mlflow/mlflow:latest - service_name: mlflow-server - # image: gcr.io/deployml-2025/wandb/wandb:latest - # service_name: wandb-server - - # memory_limit: 1Gi # Add this line - # cpu_limit: 1000m # Add this line - - artifact_tracking: - name: mlflow - params: - image: gcr.io/mlops-intro-461805/mlflow/mlflow:latest - # image: gcr.io/deployml-2025/wandb/wandb:latest - # artifact_bucket: mlflow-artifact-bucket-deployml-2025-v1678 - - model_registry: - name: mlflow - # name: wandb - params: - image: gcr.io/mlops-intro-461805/mlflow/mlflow:latest - backend_store_uri: postgresql - # backend_store_uri: sqlite - # backend_store_uri: postgresql - - feature_store: - name: feast - params: - image: gcr.io/mlops-intro-461805/feast/feast:latest - service_name: feast-server - backend_store_uri: postgresql - offline_store: bigquery - bigquery_dataset: mlops-intro-461805.feast_housing.house_data # Optional: defaults to "feast_offline_store" - - - model_serving: - name: fastapi - params: - image: gcr.io/mlops-intro-461805/fastapi/fastapi:latest - service_name: fastapi-mlflow-server - - model_monitoring: - name: grafana - params: - image: gcr.io/mlops-intro-461805/grafana/grafana:latest - service_name: grafana-server - - workflow_orchestration: - name: cron - params: - jobs: - - service_name: offline-scoring - image: gcr.io/mlops-intro-461805/offline-scoring/offline-scoring:latest - cron_schedule: "0 12 */14 * *" - bigquery_dataset: feast_housing - - service_name: metrics-monitoring - image: gcr.io/mlops-intro-461805/metrics-monitoring/metrics-monitoring:latest - cron_schedule: "0 6 * * *" - - - \ No newline at end of file diff --git a/recycling_bin/config/simple-local.yaml b/recycling_bin/config/simple-local.yaml deleted file mode 100644 index 2d5bde4..0000000 --- a/recycling_bin/config/simple-local.yaml +++ /dev/null @@ -1,10 +0,0 @@ -name: deployml-gcp -provider: - name: gcp - project_id: mlops-intro-461805 - region: us-west2 -deployment: - type: cloud_run -stack: - - experiment_tracking: - name: mlflow diff --git a/recycling_bin/data/house_data.parquet b/recycling_bin/data/house_data.parquet deleted file mode 100644 index 2aeaae3..0000000 Binary files a/recycling_bin/data/house_data.parquet and /dev/null differ diff --git a/recycling_bin/gcp-cloud-vm.md b/recycling_bin/gcp-cloud-vm.md deleted file mode 100644 index f9426ab..0000000 --- a/recycling_bin/gcp-cloud-vm.md +++ /dev/null @@ -1,139 +0,0 @@ -# GCP Cloud VM Deployment - -Deploy MLflow to Google Cloud VM using deployml. - -## Overview - -Cloud VM deployment provides persistent storage and full control over your infrastructure. It's ideal for: - -- Persistent storage -- Full control -- Cost-effective for long-running services -- Custom configurations - -## Quick Start - -### 1. Create Configuration File - -Create `cloud-vm-config.yaml`: - -```yaml -name: mlflow-cloud-vm -provider: - name: gcp - project_id: YOUR_PROJECT_ID - region: us-west1 - zone: us-west1-a - -deployment: - type: cloud_vm - -stack: - - experiment_tracking: - name: mlflow - params: - vm_name: mlflow-vm - machine_type: e2-medium - disk_size_gb: 20 - mlflow_port: 5000 - backend_store_uri: sqlite:///mlflow.db - - - artifact_tracking: - name: mlflow - params: - artifact_bucket: mlflow-artifacts-bucket - create_artifact_bucket: true -``` - -### 2. Initialize GCP Project - -```bash -# Initialize GCP project (first time only) -deployml init --provider gcp --project-id YOUR_PROJECT_ID -``` - -### 3. Deploy - -```bash -# Deploy stack -deployml deploy --config-path cloud-vm-config.yaml -``` - -### 4. Access VM - -After deployment, you'll get the VM's external IP address. Access MLflow via: - -``` -http://VM_EXTERNAL_IP:5000 -``` - -## Configuration Options - -### Custom Machine Type - -```yaml -stack: - - experiment_tracking: - name: mlflow - params: - machine_type: e2-standard-4 # 4 vCPU, 16GB RAM - disk_size_gb: 50 -``` - -### PostgreSQL Backend - -```yaml -stack: - - experiment_tracking: - name: mlflow - params: - backend_store_uri: postgresql://user:pass@host:5432/dbname -``` - -## SSH Access - -```bash -# SSH into VM -gcloud compute ssh mlflow-vm --zone us-west1-a - -# Check MLflow status -sudo systemctl status mlflow - -# View MLflow logs -sudo journalctl -u mlflow -f -``` - -## Cleanup - -```bash -# Destroy infrastructure -deployml destroy --config-path cloud-vm-config.yaml - -# Clean workspace -deployml destroy --config-path cloud-vm-config.yaml --clean-workspace -``` - -## Troubleshooting - -### VM Not Accessible - -```bash -# Check VM status -gcloud compute instances describe mlflow-vm --zone us-west1-a - -# Check firewall rules -gcloud compute firewall-rules list --filter="name~mlflow" -``` - -### MLflow Not Running - -```bash -# SSH into VM -gcloud compute ssh mlflow-vm --zone us-west1-a - -# Check service status -sudo systemctl status mlflow - -# Restart service -sudo systemctl restart mlflow -``` \ No newline at end of file diff --git a/recycling_bin/gcp.md b/recycling_bin/gcp.md deleted file mode 100644 index a377b69..0000000 --- a/recycling_bin/gcp.md +++ /dev/null @@ -1,67 +0,0 @@ -# GCP Deployment Guide - -DeployML supports multiple deployment types on Google Cloud Platform. - -## Deployment Types - -### Cloud Run (Serverless) -Serverless container deployment for MLflow and FastAPI services. - -**Features:** -- Automatic scaling -- Pay per use -- No infrastructure management - -[Get Started →](gcp-cloud-run.md) - -### GKE (Google Kubernetes Engine) -Kubernetes-based deployment for production workloads. - -**Features:** -- Production-ready -- Full control -- Custom configurations -- Two-step workflow (generate manifests, then apply) - -[Get Started →](gke-deployment.md) - -### Cloud VM -Virtual machine deployment for MLflow and other services. - -**Features:** -- Persistent storage -- Full control -- Cost-effective for long-running services - -[Get Started →](gcp-cloud-vm.md) - -## Quick Start - -1. **Install deployml** - ```bash - pip install deployml-core - ``` - -2. **Initialize GCP project** - ```bash - deployml init --provider gcp --project-id YOUR_PROJECT_ID - ``` - -3. **Choose your deployment type** - - [Cloud Run](gcp-cloud-run.md) - Serverless, auto-scaling - - [GKE](gke-deployment.md) - Kubernetes, production-ready - - [Cloud VM](gcp-cloud-vm.md) - Persistent storage, full control - -4. **Deploy** - ```bash - deployml deploy --config-path your-config.yaml - ``` - -## Comparison - -| Feature | Cloud Run | GKE | Cloud VM | -|---------|-----------|-----|----------| -| Scaling | Automatic | Manual | Manual | -| Cost | Pay per use | Per node | Per VM | -| Best For | Production APIs | Production workloads | Long-running services | -| Setup Time | Fast | Medium | Medium | diff --git a/recycling_bin/gke-deploy-test.yaml b/recycling_bin/gke-deploy-test.yaml deleted file mode 100644 index da38d03..0000000 --- a/recycling_bin/gke-deploy-test.yaml +++ /dev/null @@ -1,26 +0,0 @@ -name: gke-mlflow-test -provider: - name: gcp - project_id: mldeploy-468919 # ← Updated from terminal - region: us-west1 - -deployment: - type: gke - -gke: - cluster_name: my-gke-cluster # ← Updated from terminal - zone: us-west1-a # ← Updated from terminal - -stack: - # MLflow already deployed - commenting out - - experiment_tracking: - name: mlflow - params: - image: mlflow-demo:latest - backend_store_uri: sqlite:///mlflow.db - - - model_serving: - name: fastapi - params: - image: fastapi-mlflow-demo:latest - mlflow_tracking_uri: http://mlflow-service:5000 diff --git a/recycling_bin/gke-deployment.md b/recycling_bin/gke-deployment.md deleted file mode 100644 index 685dcab..0000000 --- a/recycling_bin/gke-deployment.md +++ /dev/null @@ -1,335 +0,0 @@ -# GKE Deployment Guide - -Complete guide for deploying MLflow and FastAPI to Google Kubernetes Engine (GKE) using deployml. - -## Overview - -GKE deployment uses Kubernetes manifests (similar to minikube) and automatically pushes Docker images to Google Container Registry (GCR). - -## Prerequisites - -### Required Software - -```bash -# Verify installations -gcloud --version -kubectl version --client -docker --version -deployml --version -``` - -### GCP Setup - -```bash -# Authenticate with GCP -gcloud auth login -gcloud auth application-default login - -# Set your project -gcloud config set project YOUR_PROJECT_ID - -# Enable required APIs -deployml init --provider gcp --project-id YOUR_PROJECT_ID -``` - -## GKE Cluster Setup - -### Create a New Cluster - -```bash -gcloud container clusters create my-gke-cluster \ - --zone us-west1-a \ - --num-nodes 2 \ - --machine-type e2-medium \ - --project YOUR_PROJECT_ID -``` - -### Use Existing Cluster - -```bash -# List existing clusters -gcloud container clusters list --project YOUR_PROJECT_ID -``` - -**Note:** If you have an existing company kubeconfig: -- **No manual setup needed** - `deployml deploy` handles everything automatically -- **Your company kubeconfig is safe** - `gcloud get-credentials` adds a new context, it doesn't delete existing ones -- **Context switching** - After deployment, kubectl will point to the GKE cluster. To switch back: - ```bash - kubectl config use-context YOUR_COMPANY_CONTEXT_NAME - ``` - -### Cluster Sizing Recommendations - -| Workload | Nodes | Machine Type | Total CPU | Total RAM | -|----------|-------|--------------|-----------|-----------| -| **Small** (MLflow only) | 1 | e2-medium | 2 vCPU | 4GB | -| **Medium** (MLflow + FastAPI) | 2 | e2-medium | 4 vCPU | 8GB | -| **Large** (Production) | 3+ | e2-standard-4 | 12+ vCPU | 16+ GB | - -## Build and Push Docker Images - -### Build MLflow Image - -```bash -cd demo/mlflow -docker build --platform linux/amd64 -t mlflow-demo:latest . -cd ../.. -``` - -### Build FastAPI Image - -```bash -cd demo/fastapi -docker build --platform linux/amd64 -t fastapi-mlflow-demo:latest . -cd ../.. -``` - -**Note:** The `deployml deploy` command will automatically push images if they're local, but you can also push manually. - -## Configuration File - -Create `gke-config.yaml`: - -```yaml -name: gke-mlflow-fastapi -provider: - name: gcp - project_id: YOUR_PROJECT_ID - region: us-west1 - -deployment: - type: gke - -gke: - cluster_name: my-gke-cluster - zone: us-west1-a # For zonal cluster - # OR use region for regional cluster: - # region: us-west1 - -stack: - - experiment_tracking: - name: mlflow - params: - image: mlflow-demo:latest # Local image name (will be pushed to GCR) - backend_store_uri: sqlite:///mlflow.db - - - artifact_tracking: - name: mlflow - params: - artifact_bucket: mlflow-artifacts-bucket - create_artifact_bucket: true - - - model_registry: - name: mlflow - params: - backend_store_uri: sqlite:///mlflow.db - - - model_serving: - name: fastapi - params: - image: fastapi-mlflow-demo:latest - mlflow_tracking_uri: http://mlflow-service:5000 -``` - -### Key Configuration Points - -- **`deployment.type: gke`** - Tells deployml to use Kubernetes manifests -- **`gke.cluster_name`** - Your GKE cluster name -- **`gke.zone`** - Cluster zone (or use `region` for regional clusters) -- **`image`** - Local Docker image name (will be converted to GCR format) -- **`mlflow_tracking_uri`** - Use `http://mlflow-service:5000` (Kubernetes internal DNS) - -## Deployment Workflows - -### Option 1: Generate and Deploy in One Step - -```bash -# Deploy everything (generates manifests and applies them) -deployml deploy -c gke-config.yaml - -# This will: -# 1. Connect to GKE cluster -# 2. Generate Kubernetes manifests -# 3. Push images to GCR (if local) -# 4. Deploy to GKE -# 5. Show LoadBalancer URLs -``` - -### Option 2: Two-Step Workflow (Generate, Review & Apply) - -**Step 1: Generate Manifests Only** - -```bash -# Generate manifests without applying -deployml deploy -c gke-config.yaml --generate-only - -# Manifests are saved to: -# .deployml//manifests/mlflow/deployment.yaml -# .deployml//manifests/mlflow/service.yaml -# .deployml//manifests/fastapi/deployment.yaml -# .deployml//manifests/fastapi/service.yaml -``` - -**Step 2: Review and Edit Manifests (Optional)** - -```bash -# Edit manifests if needed (e.g., adjust resource limits) -nano .deployml/gke-mlflow-fastapi/manifests/mlflow/deployment.yaml - -# Common edits: -# - Adjust CPU/memory requests/limits -# - Change replica count -# - Modify environment variables -``` - -**Step 3: Apply Manifests** - -```bash -# Apply manifests to GKE cluster -deployml gke-apply -c gke-config.yaml - -# Or apply manually: -kubectl apply -f .deployml/gke-mlflow-fastapi/manifests/mlflow/ -kubectl apply -f .deployml/gke-mlflow-fastapi/manifests/fastapi/ -``` - -**Benefits of Two-Step Workflow:** -- Review manifests before deployment -- Edit resource limits, replicas, or environment variables -- Version control manifests -- Apply changes incrementally -- Debug issues before deployment - -## Verify Deployment - -```bash -# Check pods -kubectl get pods -l app=mlflow -kubectl get pods -l app=fastapi - -# Check services -kubectl get svc mlflow-service -kubectl get svc fastapi-service - -# Get LoadBalancer IPs -MLFLOW_IP=$(kubectl get svc mlflow-service -o jsonpath='{.status.loadBalancer.ingress[0].ip}') -FASTAPI_IP=$(kubectl get svc fastapi-service -o jsonpath='{.status.loadBalancer.ingress[0].ip}') - -echo "MLflow URL: http://$MLFLOW_IP:5000" -echo "FastAPI URL: http://$FASTAPI_IP:8000" -``` - -## Testing - -```bash -# Test MLflow health -curl http://$MLFLOW_IP:5000/health - -# Test FastAPI health -curl http://$FASTAPI_IP:8000/health - -# View logs -kubectl logs -l app=mlflow --tail=50 -f -kubectl logs -l app=fastapi --tail=50 -f -``` - -## Troubleshooting - -### Pod Stuck in Pending - -```bash -# Check pod status -kubectl describe pod POD_NAME - -# Common fixes: -# 1. Reduce resource requests -kubectl patch deployment mlflow-deployment -p '{"spec":{"template":{"spec":{"containers":[{"name":"mlflow","resources":{"requests":{"cpu":"100m","memory":"256Mi"}}}]}}}}' - -# 2. Scale up cluster -gcloud container clusters resize CLUSTER_NAME --num-nodes 3 --zone ZONE -``` - -### Image Pull Errors - -```bash -# Verify image exists in GCR -gcloud container images list-tags gcr.io/PROJECT_ID/mlflow/mlflow --project=PROJECT_ID - -# Push image manually if needed -docker tag mlflow-demo:latest gcr.io/PROJECT_ID/mlflow/mlflow:latest -docker push gcr.io/PROJECT_ID/mlflow/mlflow:latest -``` - -## Resource Management - -### Check Current Resource Usage - -```bash -kubectl top nodes -kubectl top pods -kubectl describe nodes -``` - -### Reduce Resource Requests - -```bash -# Edit deployment -kubectl edit deployment DEPLOYMENT_NAME - -# Or patch deployment -kubectl patch deployment DEPLOYMENT_NAME -p '{"spec":{"template":{"spec":{"containers":[{"name":"CONTAINER_NAME","resources":{"requests":{"cpu":"100m","memory":"256Mi"}}}]}}}}' -``` - -### Recommended Resource Settings - -**For Small Clusters (2 nodes × e2-medium):** - -```yaml -# MLflow -resources: - requests: - cpu: "250m" - memory: "512Mi" - limits: - cpu: "1000m" - memory: "2Gi" - -# FastAPI -resources: - requests: - cpu: "100m" - memory: "256Mi" - limits: - cpu: "500m" - memory: "1Gi" -``` - -## Cleanup - -```bash -# Delete deployments -kubectl delete deployment mlflow-deployment fastapi-deployment - -# Delete services -kubectl delete service mlflow-service fastapi-service - -# Delete cluster (optional) -gcloud container clusters delete CLUSTER_NAME --zone ZONE -``` - -## Quick Reference - -```bash -# Get service URLs -MLFLOW_IP=$(kubectl get svc mlflow-service -o jsonpath='{.status.loadBalancer.ingress[0].ip}') -FASTAPI_IP=$(kubectl get svc fastapi-service -o jsonpath='{.status.loadBalancer.ingress[0].ip}') - -# Check pods -kubectl get pods -l app=mlflow -kubectl get pods -l app=fastapi - -# View logs -kubectl logs -l app=mlflow --tail=50 -f -kubectl logs -l app=fastapi --tail=50 -f -``` \ No newline at end of file diff --git a/recycling_bin/local/docker-compose.yml b/recycling_bin/local/docker-compose.yml deleted file mode 100644 index e7e44dc..0000000 --- a/recycling_bin/local/docker-compose.yml +++ /dev/null @@ -1,54 +0,0 @@ -services: - mlflow: - build: - context: ../src/deployml/docker/mlflow - ports: - - "5001:5000" - volumes: - - ./mlflow-artifacts:/mlflow-artifacts - networks: - mlops: - ipv4_address: 172.20.0.10 - command: > - mlflow server - --host 0.0.0.0 - --port 5000 - --backend-store-uri sqlite:////mlflow-artifacts/mlflow.db - --artifacts-destination /mlflow-artifacts - --serve-artifacts - --dev - - fastapi: - build: - context: ../src/deployml/docker/fastapi - ports: - - "8000:8000" - volumes: - - ./mlflow-artifacts:/mlflow-artifacts - env_file: - - .env - environment: - - MLFLOW_TRACKING_URI=http://172.20.0.10:5000 - depends_on: - - mlflow - networks: - - mlops - - grafana: - build: - context: ../src/deployml/docker/grafana-container - ports: - - "3000:3000" - environment: - - GF_SERVER_HTTP_PORT=3000 - - GF_SECURITY_ADMIN_USER=${GRAFANA_USER:-admin} - - GF_SECURITY_ADMIN_PASSWORD=${GRAFANA_PASSWORD:-admin} - networks: - - mlops - -networks: - mlops: - driver: bridge - ipam: - config: - - subnet: 172.20.0.0/16 diff --git a/recycling_bin/local/mlflow-artifacts/1/models/m-342f111ca8514da183f93e043676b2e1/artifacts/MLmodel b/recycling_bin/local/mlflow-artifacts/1/models/m-342f111ca8514da183f93e043676b2e1/artifacts/MLmodel deleted file mode 100644 index b10cf2a..0000000 --- a/recycling_bin/local/mlflow-artifacts/1/models/m-342f111ca8514da183f93e043676b2e1/artifacts/MLmodel +++ /dev/null @@ -1,23 +0,0 @@ -artifact_path: /mlflow-artifacts/1/models/m-342f111ca8514da183f93e043676b2e1/artifacts -flavors: - python_function: - env: - conda: conda.yaml - virtualenv: python_env.yaml - loader_module: mlflow.sklearn - model_path: model.pkl - predict_fn: predict - python_version: 3.11.15 - sklearn: - code: null - pickled_model: model.pkl - serialization_format: cloudpickle - sklearn_version: 1.8.0 - skops_trusted_types: null -mlflow_version: 3.10.0 -model_id: m-342f111ca8514da183f93e043676b2e1 -model_size_bytes: 1823266 -model_uuid: m-342f111ca8514da183f93e043676b2e1 -prompts: null -run_id: 022106fada274bc590f44d15d6aca7e7 -utc_time_created: '2026-04-06 23:44:26.120647' diff --git a/recycling_bin/local/mlflow-artifacts/1/models/m-342f111ca8514da183f93e043676b2e1/artifacts/conda.yaml b/recycling_bin/local/mlflow-artifacts/1/models/m-342f111ca8514da183f93e043676b2e1/artifacts/conda.yaml deleted file mode 100644 index 56e3f88..0000000 --- a/recycling_bin/local/mlflow-artifacts/1/models/m-342f111ca8514da183f93e043676b2e1/artifacts/conda.yaml +++ /dev/null @@ -1,14 +0,0 @@ -channels: -- conda-forge -dependencies: -- python=3.11.15 -- pip<=26.0.1 -- pip: - - mlflow==3.10.0 - - cloudpickle==3.1.2 - - numpy==2.4.4 - - pandas==2.3.3 - - pyarrow==23.0.1 - - scikit-learn==1.8.0 - - scipy==1.17.1 -name: mlflow-env diff --git a/recycling_bin/local/mlflow-artifacts/1/models/m-342f111ca8514da183f93e043676b2e1/artifacts/model.pkl b/recycling_bin/local/mlflow-artifacts/1/models/m-342f111ca8514da183f93e043676b2e1/artifacts/model.pkl deleted file mode 100644 index 2b6bacb..0000000 Binary files a/recycling_bin/local/mlflow-artifacts/1/models/m-342f111ca8514da183f93e043676b2e1/artifacts/model.pkl and /dev/null differ diff --git a/recycling_bin/local/mlflow-artifacts/1/models/m-342f111ca8514da183f93e043676b2e1/artifacts/python_env.yaml b/recycling_bin/local/mlflow-artifacts/1/models/m-342f111ca8514da183f93e043676b2e1/artifacts/python_env.yaml deleted file mode 100644 index 211490c..0000000 --- a/recycling_bin/local/mlflow-artifacts/1/models/m-342f111ca8514da183f93e043676b2e1/artifacts/python_env.yaml +++ /dev/null @@ -1,7 +0,0 @@ -python: 3.11.15 -build_dependencies: -- pip==26.0.1 -- setuptools==82.0.1 -- wheel==0.46.3 -dependencies: -- -r requirements.txt diff --git a/recycling_bin/local/mlflow-artifacts/1/models/m-342f111ca8514da183f93e043676b2e1/artifacts/registered_model_meta b/recycling_bin/local/mlflow-artifacts/1/models/m-342f111ca8514da183f93e043676b2e1/artifacts/registered_model_meta deleted file mode 100644 index 228776b..0000000 --- a/recycling_bin/local/mlflow-artifacts/1/models/m-342f111ca8514da183f93e043676b2e1/artifacts/registered_model_meta +++ /dev/null @@ -1,2 +0,0 @@ -model_name: HousingPriceModel -model_version: '1' diff --git a/recycling_bin/local/mlflow-artifacts/1/models/m-342f111ca8514da183f93e043676b2e1/artifacts/requirements.txt b/recycling_bin/local/mlflow-artifacts/1/models/m-342f111ca8514da183f93e043676b2e1/artifacts/requirements.txt deleted file mode 100644 index 75b6c72..0000000 --- a/recycling_bin/local/mlflow-artifacts/1/models/m-342f111ca8514da183f93e043676b2e1/artifacts/requirements.txt +++ /dev/null @@ -1,7 +0,0 @@ -mlflow==3.10.0 -cloudpickle==3.1.2 -numpy==2.4.4 -pandas==2.3.3 -pyarrow==23.0.1 -scikit-learn==1.8.0 -scipy==1.17.1 \ No newline at end of file diff --git a/recycling_bin/local/seed_model.py b/recycling_bin/local/seed_model.py deleted file mode 100644 index f183978..0000000 --- a/recycling_bin/local/seed_model.py +++ /dev/null @@ -1,79 +0,0 @@ -""" -Train a dummy housing price model and register it in local MLflow as Production. -Run with: python seed_model.py -""" -import os -import mlflow -import mlflow.sklearn -import numpy as np -import pandas as pd -from sklearn.ensemble import RandomForestRegressor -from sklearn.model_selection import train_test_split -from mlflow.tracking import MlflowClient - -MLFLOW_URI = os.getenv("MLFLOW_TRACKING_URI", "http://localhost:5001") -MODEL_NAME = "HousingPriceModel" - -FEATURE_COLUMNS = [ - 'bedrooms', 'bathrooms', 'area_sqft', 'lot_size', - 'year_built', 'city', 'state' -] - -def generate_data(n=500): - rng = np.random.default_rng(42) - df = pd.DataFrame({ - 'bedrooms': rng.integers(1, 7, n), - 'bathrooms': rng.integers(1, 5, n), - 'area_sqft': rng.integers(600, 4000, n), - 'lot_size': rng.integers(1000, 10000, n), - 'year_built': rng.integers(1950, 2023, n), - 'city': rng.integers(0, 10, n), - 'state': rng.integers(0, 5, n), - }) - # Simple price formula + noise - df['price'] = ( - df['area_sqft'] * 200 - + df['bedrooms'] * 15000 - + df['bathrooms'] * 10000 - + (2023 - df['year_built']) * -500 - + rng.normal(0, 20000, n) - ) - return df - -def main(): - mlflow.set_tracking_uri(MLFLOW_URI) - mlflow.set_experiment("housing-local") - - df = generate_data() - X = df[FEATURE_COLUMNS] - y = df['price'] - X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42) - - with mlflow.start_run(run_name="dummy-rf") as run: - model = RandomForestRegressor(n_estimators=50, random_state=42) - model.fit(X_train, y_train) - - score = model.score(X_test, y_test) - mlflow.log_param("n_estimators", 50) - mlflow.log_metric("r2", score) - - mlflow.sklearn.log_model(model, artifact_path="model") - run_id = run.info.run_id - print(f"Logged model with R²={score:.3f}") - - # Register and promote to Production - client = MlflowClient(MLFLOW_URI) - mv = mlflow.register_model(f"runs:/{run_id}/model", MODEL_NAME) - - client.transition_model_version_stage( - name=MODEL_NAME, - version=mv.version, - stage="Production", - archive_existing_versions=True, - ) - print(f"Promoted {MODEL_NAME} v{mv.version} to Production") - print(f"\nCheck MLflow UI: {MLFLOW_URI}") - print(f"FastAPI predict: http://localhost:8000/predict") - -if __name__ == "__main__": - main() diff --git a/recycling_bin/minikube.md b/recycling_bin/minikube.md deleted file mode 100644 index 21775f2..0000000 --- a/recycling_bin/minikube.md +++ /dev/null @@ -1,184 +0,0 @@ -# Minikube Local Deployment - -Deploy MLFlow and FastAPI locally using minikube for testing and development. Note that this infrastructure does NOT include all the components of the MLOps pipeline - it is limited to MLFlow and FastAPI and is meant for **practice with Kubernetes** before moving to the cloud. deployml only generates the manifest (which then may need to be edited) and then creates the deployment in minikube. - -## Overview - -Minikube provides a local Kubernetes environment for testing and development. It's ideal for: - -- Local testing -- No cloud costs -- Fast iteration -- Learning Kubernetes - -## Prerequisites - -Install minikube following instructures [here](https://minikube.sigs.k8s.io/docs/start/?arch=%2Fmacos%2Fx86-64%2Fstable%2Fbinary+download). - -```bash -# Verify installation -minikube version -``` - -## Example - -deployml is only used for generating the manifest and then deploying in minikube. In order to deploy MLFlow and FastAPI in minikube, we will need to create the Docker images for both. The corresponding dockerfiles can be found [here](https://github.com/deployml-core/deployml/tree/main/docker-images). - -### 1. Initialize Minikube for MLflow - -```bash -# Generate MLflow manifests -deployml mlflow-init \ - --output-dir ./manifests/mlflow \ - --image mlflow-demo:latest \ - --backend-store-uri sqlite:///mlflow.db - -# Deploy MLflow -deployml mlflow-deploy --manifest-dir ./manifests/mlflow -``` - -### 2. Initialize Minikube for FastAPI - -```bash -# Generate FastAPI manifests -deployml minikube-init \ - --output-dir ./manifests/fastapi \ - --image fastapi-mlflow-demo:latest \ - --mlflow-uri http://mlflow-service:5000 - -# Deploy FastAPI -deployml minikube-deploy --manifest-dir ./manifests/fastapi -``` - -### 3. Access Services - -```bash -# Get service URLs -minikube service mlflow-service --url -minikube service fastapi-service --url - -# Or open in browser -minikube service mlflow-service -minikube service fastapi-service -``` - -## Two-Step Workflow - -### Step 1: Generate Manifests - -```bash -# MLflow -deployml mlflow-init \ - --output-dir ./manifests/mlflow \ - --image mlflow-demo:latest \ - --backend-store-uri sqlite:///mlflow.db - -# FastAPI -deployml minikube-init \ - --output-dir ./manifests/fastapi \ - --image fastapi-mlflow-demo:latest \ - --mlflow-uri http://mlflow-service:5000 -``` - -### Step 2: Edit Manifests (Optional) - -```bash -# Edit deployment.yaml to adjust resources -nano ./manifests/mlflow/deployment.yaml - -# Common edits: -# - Adjust CPU/memory requests/limits -# - Change replica count -# - Modify environment variables -``` - -### Step 3: Deploy - -```bash -# Deploy MLflow -deployml mlflow-deploy --manifest-dir ./manifests/mlflow - -# Deploy FastAPI -deployml minikube-deploy --manifest-dir ./manifests/fastapi -``` - -## Verify Deployment - -```bash -# Check pods -kubectl get pods - -# Check services -kubectl get svc - -# View logs -kubectl logs -l app=mlflow --tail=50 -f -kubectl logs -l app=fastapi --tail=50 -f -``` - -## Testing - -```bash -# Get MLflow URL -MLFLOW_URL=$(minikube service mlflow-service --url) - -# Test health endpoint -curl $MLFLOW_URL/health - -# Get FastAPI URL -FASTAPI_URL=$(minikube service fastapi-service --url) - -# Test health endpoint -curl $FASTAPI_URL/health -``` - -## Cleanup - -```bash -# Delete deployments -kubectl delete deployment mlflow-deployment fastapi-deployment - -# Delete services -kubectl delete service mlflow-service fastapi-service - -# Stop minikube -minikube stop - -# Delete minikube cluster -minikube delete -``` - -## Troubleshooting - -### Pod Not Starting - -```bash -# Check pod status -kubectl get pods -kubectl describe pod POD_NAME - -# Check logs -kubectl logs POD_NAME -``` - -### Image Not Found - -```bash -# Load image into minikube -minikube image load mlflow-demo:latest -minikube image load fastapi-mlflow-demo:latest - -# Or use minikube's Docker daemon -eval $(minikube docker-env) -docker build -t mlflow-demo:latest . -``` - -### Service Not Accessible - -```bash -# Check service status -kubectl get svc - -# Use minikube service command -minikube service SERVICE_NAME --url -``` \ No newline at end of file diff --git a/recycling_bin/notebooks/model_registration.ipynb b/recycling_bin/notebooks/model_registration.ipynb deleted file mode 100644 index 4594b38..0000000 --- a/recycling_bin/notebooks/model_registration.ipynb +++ /dev/null @@ -1,1474 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Model Registration with Feast-Compatible Schema\n", - "\n", - "This notebook registers a housing price prediction model using the cleaned data with proper snake_case feature names that align with the Feast feature store schema." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "MLflow URI: https://mlflow-server-555196125082.us-west1.run.app\n", - "Model Name: HousingModel\n", - "Experiment: housing-feast-compatible\n" - ] - } - ], - "source": [ - "# MLflow Model Registration - Feast Compatible\n", - "import mlflow\n", - "import mlflow.sklearn\n", - "import mlflow.pyfunc\n", - "from mlflow.tracking import MlflowClient\n", - "from mlflow.models import infer_signature\n", - "\n", - "from sklearn.model_selection import train_test_split\n", - "from sklearn.ensemble import RandomForestRegressor\n", - "from sklearn.metrics import root_mean_squared_error, mean_absolute_error, r2_score\n", - "import pandas as pd\n", - "import numpy as np\n", - "from datetime import datetime\n", - "\n", - "# =============================================================================\n", - "# CONFIGURATION - UPDATE THESE VALUES\n", - "# =============================================================================\n", - "MLFLOW_TRACKING_URI = \"https://mlflow-server-555196125082.us-west1.run.app\" # Replace with your MLflow server\n", - "MODEL_NAME = \"HousingModel\" \n", - "EXPERIMENT_NAME = \"housing-feast-compatible\"\n", - "\n", - "# Set up MLflow\n", - "mlflow.set_tracking_uri(MLFLOW_TRACKING_URI)\n", - "# Create experiment if it doesn't exist\n", - "try:\n", - " mlflow.set_experiment(EXPERIMENT_NAME)\n", - "except:\n", - " mlflow.create_experiment(EXPERIMENT_NAME)\n", - " mlflow.set_experiment(EXPERIMENT_NAME)\n", - "\n", - "print(f\"MLflow URI: {MLFLOW_TRACKING_URI}\")\n", - "print(f\"Model Name: {MODEL_NAME}\")\n", - "print(f\"Experiment: {EXPERIMENT_NAME}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "📊 Dataset shape: (3000, 15)\n", - "📋 Columns: ['event_timestamp', 'price', 'city', 'state', 'bedrooms', 'bathrooms', 'area_sqft', 'lot_size', 'year_built', 'days_on_market', 'property_type', 'listing_agent', 'status', 'zipcode_encoded', 'mls_id']\n", - "\n", - "🔍 Data types:\n", - "event_timestamp datetime64[us]\n", - "price int64\n", - "city int64\n", - "state int64\n", - "bedrooms int64\n", - "bathrooms int64\n", - "area_sqft int64\n", - "lot_size int64\n", - "year_built int64\n", - "days_on_market int64\n", - "property_type int64\n", - "listing_agent int64\n", - "status int64\n", - "zipcode_encoded float64\n", - "mls_id int64\n", - "dtype: object\n", - "\n", - "📈 First 5 rows:\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
event_timestamppricecitystatebedroomsbathroomsarea_sqftlot_sizeyear_builtdays_on_marketproperty_typelisting_agentstatuszipcode_encodedmls_id
02017-09-13843750001310693385201043000843750.0112914
12020-09-11274902003421157911197059012274902.0923785
22019-02-22825806203127137477198515032825806.0659459
32019-12-12115382530622619513519661180301153825.0475595
42020-12-1612803483031185447421958120311280348.0647648
\n", - "
" - ], - "text/plain": [ - " event_timestamp price city state bedrooms bathrooms area_sqft \\\n", - "0 2017-09-13 843750 0 0 1 3 1069 \n", - "1 2020-09-11 274902 0 0 3 4 2115 \n", - "2 2019-02-22 825806 2 0 3 1 2713 \n", - "3 2019-12-12 1153825 3 0 6 2 2619 \n", - "4 2020-12-16 1280348 3 0 3 1 1854 \n", - "\n", - " lot_size year_built days_on_market property_type listing_agent status \\\n", - "0 3385 2010 43 0 0 0 \n", - "1 7911 1970 59 0 1 2 \n", - "2 7477 1985 15 0 3 2 \n", - "3 5135 1966 118 0 3 0 \n", - "4 4742 1958 12 0 3 1 \n", - "\n", - " zipcode_encoded mls_id \n", - "0 843750.0 112914 \n", - "1 274902.0 923785 \n", - "2 825806.0 659459 \n", - "3 1153825.0 475595 \n", - "4 1280348.0 647648 " - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# =============================================================================\n", - "# STEP 1: LOAD AND EXAMINE CLEANED DATA\n", - "# =============================================================================\n", - "\n", - "# Load the cleaned parquet file with proper schema\n", - "data_path = 'data/house_data.parquet'\n", - "data = pd.read_parquet(data_path)\n", - "\n", - "print(f\"📊 Dataset shape: {data.shape}\")\n", - "print(f\"📋 Columns: {data.columns.tolist()}\")\n", - "print(f\"\\n🔍 Data types:\")\n", - "print(data.dtypes)\n", - "print(f\"\\n📈 First 5 rows:\")\n", - "data.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "🎯 Target variable: price\n", - "📊 Features (12): ['city', 'state', 'bedrooms', 'bathrooms', 'area_sqft', 'lot_size', 'year_built', 'days_on_market', 'property_type', 'listing_agent', 'status', 'zipcode_encoded']\n", - "\n", - "📈 Target statistics:\n", - " Mean: $810,859.36\n", - " Std: $399,732.02\n", - " Min: $100,283.00\n", - " Max: $1,499,473.00\n", - "\n", - "🔍 Missing values in features:\n", - "None\n", - "\n", - "📊 Data split:\n", - " Training samples: 2400\n", - " Testing samples: 600\n" - ] - } - ], - "source": [ - "# =============================================================================\n", - "# STEP 2: PREPARE FEATURES FOR TRAINING\n", - "# =============================================================================\n", - "\n", - "# Define feature columns (excluding target 'price' and metadata columns)\n", - "feature_columns = [\n", - " 'city', 'state', 'bedrooms', 'bathrooms', 'area_sqft', \n", - " 'lot_size', 'year_built', 'days_on_market', 'property_type', \n", - " 'listing_agent', 'status', 'zipcode_encoded'\n", - "]\n", - "\n", - "# Prepare features and target\n", - "X = data[feature_columns].copy()\n", - "y = data['price'].copy()\n", - "\n", - "print(f\"🎯 Target variable: price\")\n", - "print(f\"📊 Features ({len(feature_columns)}): {feature_columns}\")\n", - "print(f\"\\n📈 Target statistics:\")\n", - "print(f\" Mean: ${y.mean():,.2f}\")\n", - "print(f\" Std: ${y.std():,.2f}\")\n", - "print(f\" Min: ${y.min():,.2f}\")\n", - "print(f\" Max: ${y.max():,.2f}\")\n", - "\n", - "# Check for missing values\n", - "print(f\"\\n🔍 Missing values in features:\")\n", - "missing = X.isnull().sum()\n", - "print(missing[missing > 0] if missing.sum() > 0 else \"None\")\n", - "\n", - "# Split the data\n", - "X_train, X_test, y_train, y_test = train_test_split(\n", - " X, y, test_size=0.2, random_state=42, stratify=None\n", - ")\n", - "\n", - "print(f\"\\n📊 Data split:\")\n", - "print(f\" Training samples: {X_train.shape[0]}\")\n", - "print(f\" Testing samples: {X_test.shape[0]}\")" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "🤖 Training RandomForestRegressor...\n", - "\n", - "📊 Model Performance:\n", - " Training RMSE: $39,728.91\n", - " Testing RMSE: $76,092.37\n", - " Training MAE: $9,495.02\n", - " Testing MAE: $19,398.27\n", - " Training R²: 0.9901\n", - " Testing R²: 0.9645\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/anaconda3/envs/deployml/lib/python3.11/site-packages/mlflow/types/utils.py:452: UserWarning: Hint: Inferred schema contains integer column(s). Integer columns in Python cannot represent missing values. If your input data contains missing values at inference time, it will be encoded as floats and will cause a schema enforcement error. The best way to avoid this problem is to infer the model schema based on a realistic data sample (training dataset) that includes missing values. Alternatively, you can declare integer columns as doubles (float64) whenever these columns may have missing values. See `Handling Integers With Missing Values `_ for more details.\n", - " warnings.warn(\n", - "2025/08/20 16:30:37 WARNING mlflow.models.model: `artifact_path` is deprecated. Please use `name` instead.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "🔄 Registering model as 'HousingModel'...\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Successfully registered model 'HousingModel'.\n", - "2025/08/20 16:30:47 INFO mlflow.store.model_registry.abstract_store: Waiting up to 300 seconds for model version to finish creation. Model name: HousingModel, version 1\n", - "Created version '1' of model 'HousingModel'.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "✅ Model registered! Run ID: 809c50d557e94513b1a8ec930f0c541a\n", - "\n", - "🎯 Top 5 Most Important Features:\n", - " zipcode_encoded: 0.9940\n", - " area_sqft: 0.0012\n", - " lot_size: 0.0011\n", - " days_on_market: 0.0011\n", - " year_built: 0.0008\n", - "🏃 View run feast_compatible_model_20250820_163033 at: https://mlflow-server-555196125082.us-west1.run.app/#/experiments/1/runs/809c50d557e94513b1a8ec930f0c541a\n", - "🧪 View experiment at: https://mlflow-server-555196125082.us-west1.run.app/#/experiments/1\n" - ] - } - ], - "source": [ - "# =============================================================================\n", - "# STEP 3: TRAIN AND REGISTER MODEL WITH FEAST COMPATIBILITY\n", - "# =============================================================================\n", - "\n", - "def train_and_register_model():\n", - " \"\"\"Train model with proper feature schema for Feast compatibility\"\"\"\n", - " \n", - " with mlflow.start_run(run_name=f\"feast_compatible_model_{datetime.now().strftime('%Y%m%d_%H%M%S')}\"):\n", - " \n", - " # 1. Train Random Forest model\n", - " print(\"🤖 Training RandomForestRegressor...\")\n", - " model = RandomForestRegressor(\n", - " n_estimators=200,\n", - " max_depth=15,\n", - " min_samples_split=5,\n", - " min_samples_leaf=2,\n", - " random_state=42,\n", - " n_jobs=-1\n", - " )\n", - " \n", - " model.fit(X_train, y_train)\n", - " \n", - " # 2. Make predictions\n", - " y_pred_train = model.predict(X_train)\n", - " y_pred_test = model.predict(X_test)\n", - " \n", - " # 3. Calculate metrics\n", - " train_rmse = root_mean_squared_error(y_train, y_pred_train)\n", - " test_rmse = root_mean_squared_error(y_test, y_pred_test)\n", - " train_mae = mean_absolute_error(y_train, y_pred_train)\n", - " test_mae = mean_absolute_error(y_test, y_pred_test)\n", - " train_r2 = r2_score(y_train, y_pred_train)\n", - " test_r2 = r2_score(y_test, y_pred_test)\n", - " \n", - " print(f\"\\n📊 Model Performance:\")\n", - " print(f\" Training RMSE: ${train_rmse:,.2f}\")\n", - " print(f\" Testing RMSE: ${test_rmse:,.2f}\")\n", - " print(f\" Training MAE: ${train_mae:,.2f}\")\n", - " print(f\" Testing MAE: ${test_mae:,.2f}\")\n", - " print(f\" Training R²: {train_r2:.4f}\")\n", - " print(f\" Testing R²: {test_r2:.4f}\")\n", - " \n", - " # 4. Log parameters\n", - " mlflow.log_param(\"model_type\", \"RandomForestRegressor\")\n", - " mlflow.log_param(\"n_estimators\", 200)\n", - " mlflow.log_param(\"max_depth\", 15)\n", - " mlflow.log_param(\"min_samples_split\", 5)\n", - " mlflow.log_param(\"min_samples_leaf\", 2)\n", - " mlflow.log_param(\"features\", feature_columns)\n", - " mlflow.log_param(\"feature_count\", len(feature_columns))\n", - " mlflow.log_param(\"data_source\", \"house_data_cleaned.parquet\")\n", - " mlflow.log_param(\"feast_compatible\", True)\n", - " \n", - " # 5. Log metrics\n", - " mlflow.log_metric(\"train_rmse\", train_rmse)\n", - " mlflow.log_metric(\"test_rmse\", test_rmse)\n", - " mlflow.log_metric(\"train_mae\", train_mae)\n", - " mlflow.log_metric(\"test_mae\", test_mae)\n", - " mlflow.log_metric(\"train_r2\", train_r2)\n", - " mlflow.log_metric(\"test_r2\", test_r2)\n", - " \n", - " # 6. Create model signature for input validation\n", - " signature = infer_signature(X_train, y_pred_train)\n", - " \n", - " # 7. Log and register model with signature and input example\n", - " print(f\"\\n🔄 Registering model as '{MODEL_NAME}'...\")\n", - " mlflow.sklearn.log_model(\n", - " sk_model=model,\n", - " artifact_path=\"model\",\n", - " registered_model_name=MODEL_NAME,\n", - " signature=signature,\n", - " input_example=X_train.head(3)\n", - " )\n", - " \n", - " run_id = mlflow.active_run().info.run_id\n", - " print(f\"✅ Model registered! Run ID: {run_id}\")\n", - " \n", - " # 8. Log feature importance\n", - " feature_importance = pd.DataFrame({\n", - " 'feature': feature_columns,\n", - " 'importance': model.feature_importances_\n", - " }).sort_values('importance', ascending=False)\n", - " \n", - " print(f\"\\n🎯 Top 5 Most Important Features:\")\n", - " for idx, row in feature_importance.head().iterrows():\n", - " print(f\" {row['feature']}: {row['importance']:.4f}\")\n", - " \n", - " # Log feature importance as artifact\n", - " feature_importance.to_csv(\"feature_importance.csv\", index=False)\n", - " mlflow.log_artifact(\"feature_importance.csv\")\n", - " \n", - " return run_id, model, feature_importance\n", - "\n", - "# Train and register the model\n", - "run_id, trained_model, feature_importance = train_and_register_model()" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "🔄 Found model version: 1\n", - "✅ Version 1 promoted to Production!\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/var/folders/wl/j0ph6fln0tqbkwlm5_hc43d00000gn/T/ipykernel_32912/4127445802.py:21: FutureWarning: ``mlflow.tracking.client.MlflowClient.transition_model_version_stage`` is deprecated since 2.9.0. Model registry stages will be removed in a future major release. To learn more about the deprecation of model registry stages, see our migration guide here: https://mlflow.org/docs/latest/model-registry.html#migrating-from-stages\n", - " client.transition_model_version_stage(\n" - ] - } - ], - "source": [ - "# =============================================================================\n", - "# STEP 4: PROMOTE TO PRODUCTION\n", - "# =============================================================================\n", - "\n", - "def promote_to_production(model_name, run_id=None):\n", - " \"\"\"Promote latest model version to Production\"\"\"\n", - " \n", - " client = MlflowClient()\n", - " \n", - " try:\n", - " if run_id:\n", - " versions = client.search_model_versions(f\"run_id='{run_id}'\")\n", - " version = versions[0].version\n", - " else:\n", - " latest_versions = client.get_latest_versions(model_name, stages=[\"None\"])\n", - " version = latest_versions[0].version\n", - " \n", - " print(f\"🔄 Found model version: {version}\")\n", - " \n", - " # Transition to Production\n", - " client.transition_model_version_stage(\n", - " name=model_name,\n", - " version=version,\n", - " stage=\"Production\"\n", - " )\n", - " \n", - " print(f\"✅ Version {version} promoted to Production!\")\n", - " return version\n", - " \n", - " except Exception as e:\n", - " print(f\"❌ Error promoting model: {e}\")\n", - " return None\n", - "\n", - "# Promote model to production\n", - "production_version = promote_to_production(MODEL_NAME, run_id)" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "🔄 Testing model loading from: models:/HousingModel/Production\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/anaconda3/envs/deployml/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", - " from .autonotebook import tqdm as notebook_tqdm\n", - "Downloading artifacts: 100%|██████████| 7/7 [00:02<00:00, 2.65it/s]" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "✅ Model loaded successfully!\n", - "\n", - "🧪 Testing with Feast-compatible feature format...\n", - "📊 Sample features:\n", - " city state bedrooms bathrooms area_sqft lot_size year_built \\\n", - "0 1 0 3 2 1500 5000 2000 \n", - "1 2 1 4 3 2200 7500 1995 \n", - "2 0 2 2 1 800 3000 2010 \n", - "\n", - " days_on_market property_type listing_agent status zipcode_encoded \n", - "0 45 0 1 1 450000.0 \n", - "1 30 1 2 0 380000.0 \n", - "2 90 0 3 2 520000.0 \n", - "\n", - "🔮 Predictions: [450484.87331746 380277.08251533 520389.94344048]\n", - "💰 Formatted predictions:\n", - " Sample 1: $450,484.87\n", - " Sample 2: $380,277.08\n", - " Sample 3: $520,389.94\n", - "\n", - "🎯 Single sample prediction: $450,484.87\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\n" - ] - } - ], - "source": [ - "# =============================================================================\n", - "# STEP 5: TEST MODEL LOADING AND FEAST COMPATIBILITY\n", - "# =============================================================================\n", - "\n", - "def test_feast_compatible_model(model_name, stage=\"Production\"):\n", - " \"\"\"Test loading the registered model with Feast-compatible features\"\"\"\n", - " \n", - " model_uri = f\"models:/{model_name}/{stage}\"\n", - " print(f\"🔄 Testing model loading from: {model_uri}\")\n", - " \n", - " try:\n", - " # Load model\n", - " loaded_model = mlflow.pyfunc.load_model(model_uri)\n", - " print(\"✅ Model loaded successfully!\")\n", - " \n", - " # Test prediction with actual feature format that Feast will provide\n", - " print(\"\\n🧪 Testing with Feast-compatible feature format...\")\n", - " \n", - " # Create sample data in the exact format Feast will provide\n", - " sample_features = pd.DataFrame({\n", - " 'city': [1, 2, 0],\n", - " 'state': [0, 1, 2], \n", - " 'bedrooms': [3, 4, 2],\n", - " 'bathrooms': [2, 3, 1],\n", - " 'area_sqft': [1500, 2200, 800],\n", - " 'lot_size': [5000, 7500, 3000],\n", - " 'year_built': [2000, 1995, 2010],\n", - " 'days_on_market': [45, 30, 90],\n", - " 'property_type': [0, 1, 0],\n", - " 'listing_agent': [1, 2, 3],\n", - " 'status': [1, 0, 2],\n", - " 'zipcode_encoded': [450000.0, 380000.0, 520000.0]\n", - " })\n", - " \n", - " print(f\"📊 Sample features:\")\n", - " print(sample_features)\n", - " \n", - " # Make predictions\n", - " predictions = loaded_model.predict(sample_features)\n", - " print(f\"\\n🔮 Predictions: {predictions}\")\n", - " print(f\"💰 Formatted predictions:\")\n", - " for i, pred in enumerate(predictions):\n", - " print(f\" Sample {i+1}: ${pred:,.2f}\")\n", - " \n", - " # Test with a single sample (how FastAPI will call it)\n", - " single_sample = sample_features.iloc[[0]]\n", - " single_prediction = loaded_model.predict(single_sample)\n", - " print(f\"\\n🎯 Single sample prediction: ${single_prediction[0]:,.2f}\")\n", - " \n", - " return True, loaded_model\n", - " \n", - " except Exception as e:\n", - " print(f\"❌ Model loading/prediction failed: {e}\")\n", - " return False, None\n", - "\n", - "# Test the model\n", - "if production_version:\n", - " success, loaded_model = test_feast_compatible_model(MODEL_NAME, \"Production\")\n", - "else:\n", - " print(\"⚠️ Skipping model testing due to promotion failure\")" - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "🔧 Feast FastAPI Service Configuration:\n", - " Service URL: https://feast-server-555196125082.us-west1.run.app\n", - " Health Check: https://feast-server-555196125082.us-west1.run.app/health\n", - " Feature Retrieval: https://feast-server-555196125082.us-west1.run.app/get-online-features\n", - "✅ Feast service is healthy!\n" - ] - } - ], - "source": [ - "# =============================================================================\n", - "# STEP 7: FEAST FASTAPI SERVICE INTEGRATION\n", - "# =============================================================================\n", - "\n", - "import requests\n", - "import json\n", - "\n", - "# Feast FastAPI service configuration\n", - "FEAST_API_URL = \"https://feast-server-555196125082.us-west1.run.app\" # Update with your Feast service URL\n", - "# For production: \"https://feast-service-url.com\"\n", - "\n", - "print(\"🔧 Feast FastAPI Service Configuration:\")\n", - "print(f\" Service URL: {FEAST_API_URL}\")\n", - "print(f\" Health Check: {FEAST_API_URL}/health\")\n", - "print(f\" Feature Retrieval: {FEAST_API_URL}/get-online-features\")\n", - "\n", - "def check_feast_service_health():\n", - " \"\"\"Check if Feast service is running\"\"\"\n", - " try:\n", - " response = requests.get(f\"{FEAST_API_URL}/health\", timeout=5)\n", - " if response.status_code == 200:\n", - " print(\"✅ Feast service is healthy!\")\n", - " return True\n", - " else:\n", - " print(f\"⚠️ Feast service returned status: {response.status_code}\")\n", - " return False\n", - " except requests.exceptions.RequestException as e:\n", - " print(f\"❌ Cannot connect to Feast service: {e}\")\n", - " print(\"📝 Note: Make sure Feast container is running on the specified URL\")\n", - " return False\n", - "\n", - "\n", - "\n", - "# Check service health and get available features\n", - "feast_healthy = check_feast_service_health()\n" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "🔄 Retrieving features via Feast API for MLS IDs: [104635, 535721, 900458]\n", - "{'metadata': {'feature_names': ['mls_id', 'property_type', 'city', 'year_built', 'status', 'bathrooms', 'area_sqft', 'listing_agent', 'lot_size', 'days_on_market', 'state', 'zipcode_encoded', 'bedrooms']}, 'results': [{'values': [104635, 535721, 900458], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['1970-01-01T00:00:00Z', '1970-01-01T00:00:00Z', '1970-01-01T00:00:00Z']}, {'values': [4, 0, 4], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [3, 0, 2], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [1959, 1969, 1990], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [0, 2, 0], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [3, 1, 1], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [772, 2348, 3630], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [0, 1, 4], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [4757, 3615, 9369], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [101, 46, 59], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [0, 2, 4], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [554217.0, 164454.0, 1249331.0], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [1, 1, 6], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}]}\n", - "[[104635, 535721, 900458], [4, 0, 4], [3, 0, 2], [1959, 1969, 1990], [0, 2, 0], [3, 1, 1], [772, 2348, 3630], [0, 1, 4], [4757, 3615, 9369], [101, 46, 59], [0, 2, 4], [554217.0, 164454.0, 1249331.0], [1, 1, 6]]\n", - "\\n📊 Features retrieved via: feast_api\n", - " mls_id property_type city year_built status bathrooms area_sqft \\\n", - "0 104635 4 3 1959 0 3 772 \n", - "1 535721 0 0 1969 2 1 2348 \n", - "2 900458 4 2 1990 0 1 3630 \n", - "\n", - " listing_agent lot_size days_on_market state zipcode_encoded bedrooms \n", - "0 0 4757 101 0 554217.0 1 \n", - "1 1 3615 46 2 164454.0 1 \n", - "2 4 9369 59 4 1249331.0 6 \n" - ] - } - ], - "source": [ - "# =============================================================================\n", - "# STEP 8: ONLINE FEATURE RETRIEVAL VIA FEAST API\n", - "# =============================================================================\n", - "\n", - "def get_features_from_feast_api(mls_ids):\n", - " \"\"\"Retrieve features from Feast FastAPI service\"\"\"\n", - " \n", - " print(f\"🔄 Retrieving features via Feast API for MLS IDs: {mls_ids}\")\n", - " \n", - " # Prepare the request payload for Feast API\n", - " request_payload = {\n", - " \"features\": [\n", - " \"housing_features:city\",\n", - " \"housing_features:state\", \n", - " \"housing_features:bedrooms\",\n", - " \"housing_features:bathrooms\",\n", - " \"housing_features:area_sqft\",\n", - " \"housing_features:lot_size\",\n", - " \"housing_features:year_built\",\n", - " \"housing_features:days_on_market\",\n", - " \"housing_features:property_type\",\n", - " \"housing_features:listing_agent\",\n", - " \"housing_features:status\",\n", - " \"housing_features:zipcode_encoded\"\n", - " ],\n", - " \"entities\": {\n", - " \"mls_id\": mls_ids\n", - " }\n", - " }\n", - " \n", - " try:\n", - " # Make API call to Feast service\n", - " response = requests.post(\n", - " f\"{FEAST_API_URL}/get-online-features\",\n", - " json=request_payload,\n", - " headers={\"Content-Type\": \"application/json\"}\n", - " )\n", - " \n", - " if response.status_code == 200:\n", - " # Parse the response\n", - " feast_response = response.json()\n", - " \n", - " \n", - " print(feast_response)\n", - " \n", - " # Extract features from Feast response\n", - " # Feast typically returns features in a specific format\n", - " values_only = [entry[\"values\"] for entry in feast_response[\"results\"]]\n", - " print(values_only)\n", - "\n", - " rows = list(map(list, zip(*values_only)))\n", - "\n", - " column_names = feast_response[\"metadata\"][\"feature_names\"]\n", - "\n", - " df = pd.DataFrame(rows, columns=column_names)\n", - "\n", - " \n", - " return df \n", - " else:\n", - " print(f\"❌ Feast API request failed with status {response.status_code}\")\n", - " print(f\"Response: {response.text}\")\n", - " return None\n", - " \n", - " except requests.exceptions.RequestException as e:\n", - " print(f\"❌ Error calling Feast API: {e}\")\n", - " return None\n", - "\n", - "\n", - "\n", - "def retrieve_features_with_fallback(mls_ids):\n", - " \"\"\"Try real Feast API first, fallback to simulation if needed\"\"\"\n", - " \n", - " if feast_healthy:\n", - " # Try real Feast API\n", - " features = get_features_from_feast_api(mls_ids)\n", - " if features is not None:\n", - " return features, \"feast_api\"\n", - " \n", - "\n", - "# Test feature retrieval with sample MLS IDs\n", - "sample_mls_ids = [104635, 535721, 900458]\n", - "retrieved_features, source = retrieve_features_with_fallback(sample_mls_ids)\n", - "print(f\"\\\\n📊 Features retrieved via: {source}\")\n", - "print(retrieved_features) " - ] - }, - { - "cell_type": "code", - "execution_count": 24, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "🎯 Testing Complete Online Scoring Pipeline\\n\n", - "🚀 End-to-End Scoring with Feast API\n", - "📋 Input: MLS IDs [104635, 535721, 900458]\n", - "🤖 Model: HousingModel (Production)\n", - "🌐 Feast API: https://feast-server-571828190906.us-west1.run.app\n", - "============================================================\n", - "\\n🔍 STEP 1: Feature Retrieval from Feast API\n", - "🔄 Retrieving features via Feast API for MLS IDs: [104635, 535721, 900458]\n", - "{'metadata': {'feature_names': ['mls_id', 'listing_agent', 'lot_size', 'bedrooms', 'state', 'city', 'status', 'property_type', 'days_on_market', 'area_sqft', 'bathrooms', 'zipcode_encoded', 'year_built']}, 'results': [{'values': [104635, 535721, 900458], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['1970-01-01T00:00:00Z', '1970-01-01T00:00:00Z', '1970-01-01T00:00:00Z']}, {'values': [0, 1, 4], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [4757, 3615, 9369], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [1, 1, 6], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [0, 2, 4], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [3, 0, 2], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [0, 2, 0], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [4, 0, 4], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [101, 46, 59], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [772, 2348, 3630], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [3, 1, 1], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [554217.0, 164454.0, 1249331.0], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}, {'values': [1959, 1969, 1990], 'statuses': ['PRESENT', 'PRESENT', 'PRESENT'], 'event_timestamps': ['2000-01-01T00:00:00Z', '2000-01-02T00:00:00Z', '2000-01-03T00:00:00Z']}]}\n", - "[[104635, 535721, 900458], [0, 1, 4], [4757, 3615, 9369], [1, 1, 6], [0, 2, 4], [3, 0, 2], [0, 2, 0], [4, 0, 4], [101, 46, 59], [772, 2348, 3630], [3, 1, 1], [554217.0, 164454.0, 1249331.0], [1959, 1969, 1990]]\n", - "✅ Features retrieved via: feast_api\n", - "\\n🤖 STEP 2: Loading Production Model from MLflow\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/anaconda3/envs/deployml/lib/python3.11/site-packages/google/auth/_default.py:76: UserWarning: Your application has authenticated using end user credentials from Google Cloud SDK without a quota project. You might receive a \"quota exceeded\" or \"API not enabled\" error. See the following page for troubleshooting: https://cloud.google.com/docs/authentication/adc-troubleshooting/user-creds. \n", - " warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)\n", - "/opt/anaconda3/envs/deployml/lib/python3.11/site-packages/google/auth/_default.py:76: UserWarning: Your application has authenticated using end user credentials from Google Cloud SDK without a quota project. You might receive a \"quota exceeded\" or \"API not enabled\" error. See the following page for troubleshooting: https://cloud.google.com/docs/authentication/adc-troubleshooting/user-creds. \n", - " warnings.warn(_CLOUD_SDK_CREDENTIALS_WARNING)\n", - "Downloading artifacts: 100%|██████████| 7/7 [00:00<00:00, 13.49it/s]\n", - "2025/08/09 04:31:55 WARNING mlflow.models.utils: Found extra inputs in the model input that are not defined in the model signature: `['mls_id']`. These inputs will be ignored.\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "✅ Model loaded from: models:/HousingModel/Production\n", - "\\n🔮 STEP 3: Making Predictions\n", - "✅ Generated 3 predictions\n", - "\\n💰 STEP 4: Formatting Results\n", - " 🏠 MLS 104635: $552,382.04\n", - " 🏠 MLS 535721: $164,622.90\n", - " 🏠 MLS 900458: $1,250,716.93\n", - "\\n📦 STEP 5: Complete API Response\n", - "📋 API Response Summary:\n", - " ✅ Success: True\n", - " 📊 Predictions: 3\n", - " 🎯 Model: HousingModel\n", - " 🌐 Features: feast_api\n", - "🎉 SUCCESS! Complete scoring pipeline working!\n", - "📋 Sample API Response JSON:\n", - "{'success': True, 'predictions': [{'mls_id': 104635, 'predicted_price': 552382.0359126985, 'formatted_price': '$552,382.04', 'features_source': 'feast_api'}, {'mls_id': 535721, 'predicted_price': 164622.90210299424, 'formatted_price': '$164,622.90', 'features_source': 'feast_api'}, {'mls_id': 900458, 'predicted_price': 1250716.9310476193, 'formatted_price': '$1,250,716.93', 'features_source': 'feast_api'}], 'model_info': {'model_name': 'HousingModel', 'model_stage': 'Production', 'model_uri': 'models:/HousingModel/Production', 'mlflow_tracking_uri': 'https://mlflow-server-571828190906.us-west1.run.app'}, 'feast_info': {'feast_api_url': 'https://feast-server-571828190906.us-west1.run.app', 'features_source': 'feast_api', 'feature_count': 13}, 'metadata': {'timestamp': '2025-08-09T04:31:55.474062', 'request_id': 'req_20250809_043155', 'prediction_count': 3}}\n", - "{\n", - " \"success\": true,\n", - " \"predictions\": [\n", - " {\n", - " \"mls_id\": 104635,\n", - " \"predicted_price\": 552382.0359126985,\n", - " \"formatted_price\": \"$552,382.04\",\n", - " \"features_source\": \"feast_api\"\n", - " },\n", - " {\n", - " \"mls_id\": 535721,\n", - " \"predicted_price\": 164622.90210299424,\n", - " \"formatted_price\": \"$164,622.90\",\n", - " \"features_source\": \"feast_api\"\n", - " },\n", - " {\n", - " \"mls_id\": 900458,\n", - " \"predicted_price\": 1250716.9310476193,\n", - " \"formatted_price\": \"$1,250,716.93\",\n", - " \"features_source\": \"feast_api\"\n", - " }\n", - " ],\n", - " \"model_info\": {\n", - " \"model_name\": \"HousingModel\",\n", - " \"model_stage\": \"Production\",\n", - " \"model_uri\": \"models:/HousingModel/Production\",\n", - " \"mlflow_tracking_uri\": \"https://mlflow-server-571828190906.us-west1.run.app\"\n", - " },\n", - " \"feast_info\": {\n", - " \"feast_api_url\": \"https://feast-server-571828190906.us-west1.run.app\",\n", - " \"features_source\": \"feast_api\",\n", - " \"feature_count\": 13\n", - " },\n", - " \"metadata\": {\n", - " \"timestamp\": \"2025-08-09T04:31:55.474062\",\n", - " \"request_id\": \"req_20250809_04...\n" - ] - } - ], - "source": [ - "# =============================================================================\n", - "# STEP 9: END-TO-END SCORING WITH FEAST API\n", - "# =============================================================================\n", - "\n", - "def end_to_end_scoring_with_feast_api(mls_ids, model_name=\"HousingModel\", stage=\"Production\"):\n", - " \"\"\"Complete online scoring workflow using Feast FastAPI service\"\"\"\n", - " \n", - " print(f\"🚀 End-to-End Scoring with Feast API\")\n", - " print(f\"📋 Input: MLS IDs {mls_ids}\")\n", - " print(f\"🤖 Model: {model_name} ({stage})\")\n", - " print(f\"🌐 Feast API: {FEAST_API_URL}\")\n", - " print(\"=\" * 60)\n", - " \n", - " # Step 1: Retrieve features from Feast API\n", - " print(\"\\\\n🔍 STEP 1: Feature Retrieval from Feast API\")\n", - " features, source = retrieve_features_with_fallback(mls_ids)\n", - " \n", - " if features is None:\n", - " print(\"❌ Feature retrieval failed\")\n", - " return None\n", - " \n", - " print(f\"✅ Features retrieved via: {source}\")\n", - " \n", - " # Step 2: Load production model from MLflow\n", - " print(\"\\\\n🤖 STEP 2: Loading Production Model from MLflow\")\n", - " try:\n", - " model_uri = f\"models:/{model_name}/{stage}\"\n", - " loaded_model = mlflow.pyfunc.load_model(model_uri)\n", - " print(f\"✅ Model loaded from: {model_uri}\")\n", - " except Exception as e:\n", - " print(f\"❌ Failed to load model: {e}\")\n", - " return None\n", - " \n", - " # Step 3: Make predictions\n", - " print(\"\\\\n🔮 STEP 3: Making Predictions\")\n", - " try:\n", - " predictions = loaded_model.predict(features)\n", - " print(f\"✅ Generated {len(predictions)} predictions\")\n", - " \n", - " # Step 4: Format results for API response\n", - " print(\"\\\\n💰 STEP 4: Formatting Results\")\n", - " results = []\n", - " for i, (mls_id, prediction) in enumerate(zip(mls_ids, predictions)):\n", - " result = {\n", - " 'mls_id': int(mls_id),\n", - " 'predicted_price': float(prediction),\n", - " 'formatted_price': f\"${prediction:,.2f}\",\n", - " 'features_source': source\n", - " }\n", - " results.append(result)\n", - " print(f\" 🏠 MLS {mls_id}: {result['formatted_price']}\")\n", - " \n", - " # Step 5: Create complete API response\n", - " print(\"\\\\n📦 STEP 5: Complete API Response\")\n", - " api_response = {\n", - " 'success': True,\n", - " 'predictions': results,\n", - " 'model_info': {\n", - " 'model_name': model_name,\n", - " 'model_stage': stage,\n", - " 'model_uri': model_uri,\n", - " 'mlflow_tracking_uri': MLFLOW_TRACKING_URI\n", - " },\n", - " 'feast_info': {\n", - " 'feast_api_url': FEAST_API_URL,\n", - " 'features_source': source,\n", - " 'feature_count': len(features.columns)\n", - " },\n", - " 'metadata': {\n", - " 'timestamp': datetime.now().isoformat(),\n", - " 'request_id': f\"req_{datetime.now().strftime('%Y%m%d_%H%M%S')}\",\n", - " 'prediction_count': len(predictions)\n", - " }\n", - " }\n", - " \n", - " print(\"📋 API Response Summary:\")\n", - " print(f\" ✅ Success: {api_response['success']}\")\n", - " print(f\" 📊 Predictions: {len(api_response['predictions'])}\")\n", - " print(f\" 🎯 Model: {api_response['model_info']['model_name']}\")\n", - " print(f\" 🌐 Features: {api_response['feast_info']['features_source']}\")\n", - " \n", - " return api_response\n", - " \n", - " except Exception as e:\n", - " print(f\"❌ Prediction failed: {e}\")\n", - " return {\n", - " 'success': False,\n", - " 'error': str(e),\n", - " 'metadata': {\n", - " 'timestamp': datetime.now().isoformat()\n", - " }\n", - " }\n", - "\n", - "# Test the complete workflow\n", - "print(\"🎯 Testing Complete Online Scoring Pipeline\\\\n\")\n", - "scoring_result = end_to_end_scoring_with_feast_api([104635, 535721, 900458])\n", - "\n", - "if scoring_result and scoring_result.get('success'):\n", - " print(\"🎉 SUCCESS! Complete scoring pipeline working!\")\n", - " print(\"📋 Sample API Response JSON:\")\n", - " print(scoring_result)\n", - " print(json.dumps(scoring_result, indent=2)[:1000] + \"...\" if len(json.dumps(scoring_result, indent=2)) > 1000 else json.dumps(scoring_result, indent=2))\n", - "else:\n", - " print(\"❌ Scoring pipeline failed\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# =============================================================================\n", - "# STEP 10: PRODUCTION FASTAPI INTEGRATION WITH FEAST API\n", - "# =============================================================================\n", - "\n", - "def generate_production_fastapi_code():\n", - " \"\"\"Generate production FastAPI code that uses Feast API for feature retrieval\"\"\"\n", - " \n", - " print(\"🔧 Production FastAPI + Feast API + MLflow Integration:\\\\n\")\n", - " \n", - " fastapi_code = f'''\n", - "# =============================================================================\n", - "# Production FastAPI with Feast API Integration\n", - "# =============================================================================\n", - "\n", - "from fastapi import FastAPI, HTTPException\n", - "from pydantic import BaseModel\n", - "import mlflow.pyfunc\n", - "import pandas as pd\n", - "import requests\n", - "import json\n", - "from typing import List\n", - "from datetime import datetime\n", - "import os\n", - "import logging\n", - "\n", - "# Setup logging\n", - "logging.basicConfig(level=logging.INFO)\n", - "logger = logging.getLogger(__name__)\n", - "\n", - "app = FastAPI(\n", - " title=\"Housing Price Prediction API\",\n", - " description=\"ML prediction service using Feast features and MLflow models\",\n", - " version=\"2.0.0\"\n", - ")\n", - "\n", - "# =============================================================================\n", - "# CONFIGURATION\n", - "# =============================================================================\n", - "\n", - "# MLflow Configuration\n", - "MLFLOW_TRACKING_URI = \"{MLFLOW_TRACKING_URI}\"\n", - "MODEL_NAME = \"{MODEL_NAME}\"\n", - "MODEL_STAGE = \"Production\"\n", - "\n", - "# Feast API Configuration\n", - "FEAST_API_URL = os.getenv(\"FEAST_API_URL\", \"http://feast-service:8080\")\n", - "FEAST_TIMEOUT = int(os.getenv(\"FEAST_TIMEOUT\", \"10\"))\n", - "\n", - "# Feature configuration - matches your Feast feature view\n", - "REQUIRED_FEATURES = [\n", - " \"housing_features:city\",\n", - " \"housing_features:state\", \n", - " \"housing_features:bedrooms\",\n", - " \"housing_features:bathrooms\",\n", - " \"housing_features:area_sqft\",\n", - " \"housing_features:lot_size\",\n", - " \"housing_features:year_built\",\n", - " \"housing_features:days_on_market\",\n", - " \"housing_features:property_type\",\n", - " \"housing_features:listing_agent\",\n", - " \"housing_features:status\",\n", - " \"housing_features:zipcode_encoded\"\n", - "]\n", - "\n", - "MODEL_FEATURE_ORDER = [\n", - " 'city', 'state', 'bedrooms', 'bathrooms', 'area_sqft', \n", - " 'lot_size', 'year_built', 'days_on_market', 'property_type', \n", - " 'listing_agent', 'status', 'zipcode_encoded'\n", - "]\n", - "\n", - "# =============================================================================\n", - "# STARTUP - LOAD MODEL\n", - "# =============================================================================\n", - "\n", - "mlflow.set_tracking_uri(MLFLOW_TRACKING_URI)\n", - "model_uri = f\"models://{{MODEL_NAME}}/{{MODEL_STAGE}}\"\n", - "model = None\n", - "\n", - "@app.on_event(\"startup\")\n", - "async def startup_event():\n", - " global model\n", - " try:\n", - " logger.info(f\"Loading model from {{model_uri}}\")\n", - " model = mlflow.pyfunc.load_model(model_uri)\n", - " logger.info(\"✅ Model loaded successfully\")\n", - " except Exception as e:\n", - " logger.error(f\"❌ Failed to load model: {{e}}\")\n", - " raise e\n", - "\n", - "# =============================================================================\n", - "# REQUEST/RESPONSE MODELS\n", - "# =============================================================================\n", - "\n", - "class PredictionRequest(BaseModel):\n", - " mls_ids: List[int]\n", - "\n", - "class PredictionResult(BaseModel):\n", - " mls_id: int\n", - " predicted_price: float\n", - " formatted_price: str\n", - "\n", - "class PredictionResponse(BaseModel):\n", - " success: bool\n", - " predictions: List[PredictionResult]\n", - " model_info: dict\n", - " feast_info: dict\n", - " metadata: dict\n", - " error: str = None\n", - "\n", - "# =============================================================================\n", - "# FEAST API INTEGRATION\n", - "# =============================================================================\n", - "\n", - "def get_features_from_feast(mls_ids: List[int]) -> pd.DataFrame:\n", - " \\\"\\\"\\\"Retrieve features from Feast FastAPI service\\\"\\\"\\\"\n", - " \n", - " logger.info(f\"Retrieving features for MLS IDs: {{mls_ids}}\")\n", - " \n", - " # Prepare Feast API request\n", - " feast_request = {{\n", - " \"features\": REQUIRED_FEATURES,\n", - " \"entities\": {{\n", - " \"mls_id\": mls_ids\n", - " }}\n", - " }}\n", - " \n", - " try:\n", - " # Call Feast API\n", - " response = requests.post(\n", - " f\"{{FEAST_API_URL}}/get-online-features\",\n", - " json=feast_request,\n", - " headers={{\"Content-Type\": \"application/json\"}},\n", - " timeout=FEAST_TIMEOUT\n", - " )\n", - " \n", - " if response.status_code != 200:\n", - " raise HTTPException(\n", - " status_code=502,\n", - " detail=f\"Feast API error: {{response.status_code}} - {{response.text}}\"\n", - " )\n", - " \n", - " # Parse Feast response\n", - " feast_response = response.json()\n", - " \n", - " # Convert to DataFrame format for model\n", - " features_data = {{}}\n", - " results = feast_response.get('results', [])\n", - " \n", - " for feature_name in REQUIRED_FEATURES:\n", - " clean_name = feature_name.split(':')[1] # Remove feature_view prefix\n", - " features_data[clean_name] = []\n", - " \n", - " for result in results:\n", - " feature_values = result.get('values', {{}})\n", - " features_data[clean_name].append(feature_values.get(feature_name))\n", - " \n", - " # Create DataFrame with correct feature order\n", - " features_df = pd.DataFrame(features_data)\n", - " features_df = features_df[MODEL_FEATURE_ORDER] # Ensure correct order\n", - " \n", - " logger.info(f\"✅ Retrieved {{len(features_df)}} feature records\")\n", - " return features_df\n", - " \n", - " except requests.exceptions.RequestException as e:\n", - " logger.error(f\"Feast API connection error: {{e}}\")\n", - " raise HTTPException(\n", - " status_code=502, \n", - " detail=f\"Cannot connect to Feast service: {{str(e)}}\"\n", - " )\n", - " except Exception as e:\n", - " logger.error(f\"Feature retrieval error: {{e}}\")\n", - " raise HTTPException(\n", - " status_code=500,\n", - " detail=f\"Feature retrieval failed: {{str(e)}}\"\n", - " )\n", - "\n", - "# =============================================================================\n", - "# PREDICTION ENDPOINTS\n", - "# =============================================================================\n", - "\n", - "@app.post(\"/predict\", response_model=PredictionResponse)\n", - "async def predict_house_prices(request: PredictionRequest):\n", - " \\\"\\\"\\\"Predict house prices using Feast features and MLflow model\\\"\\\"\\\"\n", - " \n", - " if model is None:\n", - " raise HTTPException(status_code=503, detail=\"Model not loaded\")\n", - " \n", - " try:\n", - " start_time = datetime.now()\n", - " \n", - " # Step 1: Get features from Feast\n", - " features_df = get_features_from_feast(request.mls_ids)\n", - " \n", - " # Step 2: Make predictions\n", - " predictions = model.predict(features_df)\n", - " \n", - " # Step 3: Format results\n", - " results = []\n", - " for mls_id, prediction in zip(request.mls_ids, predictions):\n", - " results.append(PredictionResult(\n", - " mls_id=mls_id,\n", - " predicted_price=float(prediction),\n", - " formatted_price=f\"${{prediction:,.2f}}\"\n", - " ))\n", - " \n", - " end_time = datetime.now()\n", - " processing_time = (end_time - start_time).total_seconds() * 1000\n", - " \n", - " return PredictionResponse(\n", - " success=True,\n", - " predictions=results,\n", - " model_info={{\n", - " \"model_name\": MODEL_NAME,\n", - " \"model_stage\": MODEL_STAGE,\n", - " \"model_uri\": model_uri,\n", - " \"mlflow_tracking_uri\": MLFLOW_TRACKING_URI\n", - " }},\n", - " feast_info={{\n", - " \"feast_api_url\": FEAST_API_URL,\n", - " \"feature_count\": len(MODEL_FEATURE_ORDER),\n", - " \"features_retrieved\": len(features_df)\n", - " }},\n", - " metadata={{\n", - " \"timestamp\": datetime.now().isoformat(),\n", - " \"processing_time_ms\": processing_time,\n", - " \"prediction_count\": len(predictions)\n", - " }}\n", - " )\n", - " \n", - " except HTTPException:\n", - " raise # Re-raise HTTP exceptions\n", - " except Exception as e:\n", - " logger.error(f\"Prediction error: {{e}}\")\n", - " raise HTTPException(status_code=500, detail=f\"Prediction failed: {{str(e)}}\")\n", - "\n", - "# =============================================================================\n", - "# HEALTH CHECK ENDPOINTS\n", - "# =============================================================================\n", - "\n", - "@app.get(\"/health\")\n", - "async def health_check():\n", - " \\\"\\\"\\\"Health check for the prediction service\\\"\\\"\\\"\n", - " \n", - " # Check model status\n", - " model_status = model is not None\n", - " \n", - " # Check Feast API status\n", - " feast_status = False\n", - " try:\n", - " response = requests.get(f\"{{FEAST_API_URL}}/health\", timeout=5)\n", - " feast_status = response.status_code == 200\n", - " except:\n", - " pass\n", - " \n", - " overall_status = model_status and feast_status\n", - " \n", - " return {{\n", - " \"status\": \"healthy\" if overall_status else \"unhealthy\",\n", - " \"model_loaded\": model_status,\n", - " \"feast_api_available\": feast_status,\n", - " \"feast_api_url\": FEAST_API_URL,\n", - " \"model_info\": {{\n", - " \"name\": MODEL_NAME,\n", - " \"stage\": MODEL_STAGE,\n", - " \"uri\": model_uri\n", - " }} if model_status else None,\n", - " \"timestamp\": datetime.now().isoformat()\n", - " }}\n", - "\n", - "@app.get(\"/\")\n", - "async def root():\n", - " \\\"\\\"\\\"API information\\\"\\\"\\\"\n", - " return {{\n", - " \"service\": \"Housing Price Prediction API\",\n", - " \"version\": \"2.0.0\",\n", - " \"model\": MODEL_NAME,\n", - " \"endpoints\": {{\n", - " \"predict\": \"/predict\",\n", - " \"health\": \"/health\",\n", - " \"docs\": \"/docs\"\n", - " }},\n", - " \"integration\": {{\n", - " \"feast_api\": FEAST_API_URL,\n", - " \"mlflow\": MLFLOW_TRACKING_URI\n", - " }}\n", - " }}\n", - "\n", - "# =============================================================================\n", - "# MAIN\n", - "# =============================================================================\n", - "\n", - "if __name__ == \"__main__\":\n", - " import uvicorn\n", - " uvicorn.run(app, host=\"0.0.0.0\", port=8080)\n", - "'''\n", - " \n", - " print(fastapi_code)\n", - " \n", - " # Save to file\n", - " with open('production_fastapi_feast_integration.py', 'w') as f:\n", - " f.write(fastapi_code)\n", - " \n", - " print(\"\\\\n💾 Production code saved to 'production_fastapi_feast_integration.py'\")\n", - " \n", - " # Generate requirements\n", - " requirements = '''fastapi==0.104.1\n", - "uvicorn==0.24.0\n", - "mlflow==2.8.1\n", - "pandas==2.1.4\n", - "requests==2.31.0\n", - "pydantic==2.5.2\n", - "numpy==1.24.3\n", - "scikit-learn==1.3.2\n", - "'''\n", - " \n", - " with open('fastapi_requirements.txt', 'w') as f:\n", - " f.write(requirements)\n", - " \n", - " print(\"💾 Requirements saved to 'fastapi_requirements.txt'\")\n", - " \n", - " print(\"\\\\n🚀 Deployment Instructions:\")\n", - " print(\"1. Update FastAPI container Dockerfile to use production_fastapi_feast_integration.py\")\n", - " print(\"2. Update requirements.txt with fastapi_requirements.txt contents\")\n", - " print(\"3. Set environment variables:\")\n", - " print(\" - FEAST_API_URL=http://feast-service:8080\")\n", - " print(\" - FEAST_TIMEOUT=10\")\n", - " print(\"4. Deploy with network connectivity between FastAPI and Feast containers\")\n", - " print(\"5. Test with: POST /predict {\\\\\"mls_ids\\\\\": [104635, 535721]}\")\n", - " \n", - " print(\"\\\\n🔗 API Endpoints:\")\n", - " print(\"- POST /predict - Make predictions\")\n", - " print(\"- GET /health - Service health check\") \n", - " print(\"- GET / - API information\")\n", - " print(\"- GET /docs - Interactive API documentation\")\n", - "\n", - "# Generate the production integration code\n", - "generate_production_fastapi_code()\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# =============================================================================\n", - "# STEP 7: FEAST CLIENT SETUP\n", - "# =============================================================================\n", - "\n", - "from feast import FeatureStore\n", - "import os\n", - "\n", - "# Feast configuration - Update these based on your Feast deployment\n", - "FEAST_REPO_PATH = \"feast-container-housing-postgres-v2\"\n", - "\n", - "# Set environment variables for Feast (these would normally be set in your deployment)\n", - "# For local testing, we'll set them programmatically\n", - "os.environ.setdefault(\"FEAST_REGISTRY_PATH\", \"postgresql://user:password@localhost:5432/feast_registry\")\n", - "os.environ.setdefault(\"FEAST_ONLINE_STORE_HOST\", \"localhost\")\n", - "os.environ.setdefault(\"FEAST_ONLINE_STORE_PORT\", \"5432\")\n", - "os.environ.setdefault(\"FEAST_ONLINE_STORE_DATABASE\", \"feast_online\")\n", - "os.environ.setdefault(\"FEAST_ONLINE_STORE_USER\", \"user\")\n", - "os.environ.setdefault(\"FEAST_ONLINE_STORE_PASSWORD\", \"password\")\n", - "os.environ.setdefault(\"FEAST_OFFLINE_STORE_PROJECT_ID\", \"mlops-intro-461805\")\n", - "os.environ.setdefault(\"FEAST_OFFLINE_STORE_DATASET\", \"feast_housing\")\n", - "\n", - "print(\"🔧 Feast Configuration:\")\n", - "print(f\" Repository Path: {FEAST_REPO_PATH}\")\n", - "print(f\" Registry: PostgreSQL\")\n", - "print(f\" Online Store: PostgreSQL\") \n", - "print(f\" Offline Store: BigQuery\")\n", - "\n", - "# Initialize Feast client\n", - "try:\n", - " store = FeatureStore(repo_path=FEAST_REPO_PATH)\n", - " print(\"✅ Feast client initialized successfully!\")\n", - " \n", - " # List available feature views\n", - " feature_views = store.list_feature_views()\n", - " print(f\"\\n📊 Available Feature Views:\")\n", - " for fv in feature_views:\n", - " print(f\" - {fv.name}: {len(fv.schema)} features\")\n", - " \n", - "except Exception as e:\n", - " print(f\"❌ Feast client initialization failed: {e}\")\n", - " print(\"📝 Note: This is expected if Feast server is not running locally\")\n", - " store = None" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Feast Integration and Model Scoring\n", - "\n", - "This section demonstrates online feature retrieval from Feast and model scoring for production inference." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "deployml", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.0" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/recycling_bin/notebooks/notebook_cli_demo.ipynb b/recycling_bin/notebooks/notebook_cli_demo.ipynb deleted file mode 100644 index 7c5b601..0000000 --- a/recycling_bin/notebooks/notebook_cli_demo.ipynb +++ /dev/null @@ -1,1159 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# DeployML Notebook CLI Demo\n", - "\n", - "Deploy MLOps infrastructure with live CLI logs in Jupyter!\n", - "\n", - "**What this does:** Runs `poetry run deployml deploy -c gcp-sample.yaml -y` and shows all deployment logs directly in the notebook." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 1: Deploy with Live Logs" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "!pip install deployml-core" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "SYSTEM INFORMATION\n", - "================================================================================\n", - "OS: Darwin (64bit)\n", - "Python: 3.14.2\n", - "Location: /Users/rclements/Documents/research/deployml/notebooks\n", - "\n" - ] - }, - { - "ename": "AttributeError", - "evalue": "'Styler' object has no attribute 'applymap'", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mAttributeError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[1]\u001b[39m\u001b[32m, line 4\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28;01mimport\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01mdeployml\u001b[39;00m\n\u001b[32m 3\u001b[39m \u001b[38;5;66;03m# Simple one-liner check\u001b[39;00m\n\u001b[32m----> \u001b[39m\u001b[32m4\u001b[39m \u001b[43mdeployml\u001b[49m\u001b[43m.\u001b[49m\u001b[43mcheck_system\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/research/test_deployml/.venv/lib/python3.14/site-packages/deployml/diagnostics/doctor.py:627\u001b[39m, in \u001b[36mcheck_system\u001b[39m\u001b[34m()\u001b[39m\n\u001b[32m 625\u001b[39m \u001b[38;5;28;01mdef\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34mcheck_system\u001b[39m() -> DeployMLDoctor:\n\u001b[32m 626\u001b[39m \u001b[38;5;250m \u001b[39m\u001b[33;03m\"\"\"Quick system check function for notebooks\"\"\"\u001b[39;00m\n\u001b[32m--> \u001b[39m\u001b[32m627\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mrun_doctor\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/research/test_deployml/.venv/lib/python3.14/site-packages/deployml/diagnostics/doctor.py:621\u001b[39m, in \u001b[36mrun_doctor\u001b[39m\u001b[34m(verbose, show_all)\u001b[39m\n\u001b[32m 619\u001b[39m doctor = DeployMLDoctor(verbose=verbose)\n\u001b[32m 620\u001b[39m doctor.run_all_checks()\n\u001b[32m--> \u001b[39m\u001b[32m621\u001b[39m \u001b[43mdoctor\u001b[49m\u001b[43m.\u001b[49m\u001b[43mprint_results\u001b[49m\u001b[43m(\u001b[49m\u001b[43mshow_all\u001b[49m\u001b[43m=\u001b[49m\u001b[43mshow_all\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 622\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m doctor\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/research/test_deployml/.venv/lib/python3.14/site-packages/deployml/diagnostics/doctor.py:483\u001b[39m, in \u001b[36mDeployMLDoctor.print_results\u001b[39m\u001b[34m(self, show_all)\u001b[39m\n\u001b[32m 481\u001b[39m \u001b[38;5;250m\u001b[39m\u001b[33;03m\"\"\"Print results in a formatted way\"\"\"\u001b[39;00m\n\u001b[32m 482\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m IN_NOTEBOOK:\n\u001b[32m--> \u001b[39m\u001b[32m483\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[43m.\u001b[49m\u001b[43m_print_notebook_results\u001b[49m\u001b[43m(\u001b[49m\u001b[43mshow_all\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 484\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 485\u001b[39m \u001b[38;5;28mself\u001b[39m._print_cli_results(show_all)\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/Documents/research/test_deployml/.venv/lib/python3.14/site-packages/deployml/diagnostics/doctor.py:514\u001b[39m, in \u001b[36mDeployMLDoctor._print_notebook_results\u001b[39m\u001b[34m(self, show_all)\u001b[39m\n\u001b[32m 511\u001b[39m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[32m 512\u001b[39m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[33m'\u001b[39m\u001b[33mcolor: blue\u001b[39m\u001b[33m'\u001b[39m\n\u001b[32m--> \u001b[39m\u001b[32m514\u001b[39m styled_df = \u001b[43mdf\u001b[49m\u001b[43m[\u001b[49m\u001b[43m[\u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mname\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mstatus\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mmessage\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[33;43m'\u001b[39;49m\u001b[33;43mfix_command\u001b[39;49m\u001b[33;43m'\u001b[39;49m\u001b[43m]\u001b[49m\u001b[43m]\u001b[49m\u001b[43m.\u001b[49m\u001b[43mstyle\u001b[49m\u001b[43m.\u001b[49m\u001b[43mapplymap\u001b[49m(\n\u001b[32m 515\u001b[39m color_status, subset=[\u001b[33m'\u001b[39m\u001b[33mstatus\u001b[39m\u001b[33m'\u001b[39m]\n\u001b[32m 516\u001b[39m )\n\u001b[32m 518\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33m\"\u001b[39m\u001b[33mDEPLOYML DOCTOR RESULTS\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m 519\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33m\"\u001b[39m\u001b[33m=\u001b[39m\u001b[33m\"\u001b[39m * \u001b[32m80\u001b[39m)\n", - "\u001b[31mAttributeError\u001b[39m: 'Styler' object has no attribute 'applymap'" - ] - } - ], - "source": [ - "import deployml\n", - "\n", - "# Simple one-liner check\n", - "deployml.check_system()" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "SYSTEM INFORMATION\n", - "================================================================================\n", - "OS: Darwin (64bit)\n", - "Python: 3.14.2\n", - "Location: /Users/rclements/Documents/research/deployml/notebooks\n", - "\n", - "DEPLOYML DOCTOR RESULTS\n", - "================================================================================\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/Users/rclements/tmp/env/deployml/lib/python3.14/site-packages/deployml/diagnostics/doctor.py:504: FutureWarning: Styler.applymap has been deprecated. Use Styler.map instead.\n", - " styled_df = df[['name', 'status', 'message', 'fix_command']].style.applymap(\n" - ] - }, - { - "data": { - "text/html": [ - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
 namestatusmessagefix_command
0Python VersionPASSPython 3.14.2 (>= 3.11)
1Package: typerPASSCLI framework - v0.21.1
2Package: pyyamlFAILMissing required package: pyyaml (YAML configuration parsing)pip install pyyaml
3Package: jinja2PASSTemplate rendering - v3.1.6
4Package: pandasPASSData manipulation - v2.3.3
5Package: requestsPASSHTTP client - v2.32.5
6Package: ipythonFAILMissing required package: ipython (Interactive Python)pip install ipython
7Package: jupyterPASSNotebook support - v1.1.1
8Optional: mlflowINFOOptional package not installed: mlflow (ML experiment tracking)pip install mlflow
9Optional: google-cloud-storageINFOOptional package not installed: google-cloud-storage (GCP storage integration)pip install google-cloud-storage
10Optional: scikit-learnINFOOptional package not installed: scikit-learn (Machine learning)pip install scikit-learn
11Optional: matplotlibINFOOptional package not installed: matplotlib (Plotting)pip install matplotlib
12Optional: seabornINFOOptional package not installed: seaborn (Statistical visualization)pip install seaborn
13DockerPASSDocker version 20.10.23, build 7155243
14TerraformPASSTerraform v1.9.5
15Google Cloud CLIPASSGoogle Cloud SDK 551.0.0
16AWS CLIINFONot installed (optional for cloud deployments)Install from: https://aws.amazon.com/cli/
17Azure CLIINFONot installed (optional for cloud deployments)Install from: https://docs.microsoft.com/en-us/cli/azure/install-azure-cli
18GitPASSgit version 2.50.1 (Apple Git-155)
19InfracostPASSInfracost v0.10.43
20Docker PermissionsPASSCan run Docker commands
21GCP AuthenticationPASSAuthenticated with Google Cloud
22DeployML ConfigINFONo configuration file found (will use defaults)
\n" - ], - "text/plain": [ - "" - ] - }, - "metadata": {}, - "output_type": "display_data" - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "================================================================================\n", - "2 critical issues found. DeployML may not work properly. Run the suggested fix commands above.\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
namestatusmessagerequiredfix_commanddetails
0Python VersionPASSPython 3.14.2 (>= 3.11)True
1Package: typerPASSCLI framework - v0.21.1True
2Package: pyyamlFAILMissing required package: pyyaml (YAML configu...Truepip install pyyaml
3Package: jinja2PASSTemplate rendering - v3.1.6True
4Package: pandasPASSData manipulation - v2.3.3True
5Package: requestsPASSHTTP client - v2.32.5True
6Package: ipythonFAILMissing required package: ipython (Interactive...Truepip install ipython
7Package: jupyterPASSNotebook support - v1.1.1True
8Optional: mlflowINFOOptional package not installed: mlflow (ML exp...Falsepip install mlflow
9Optional: google-cloud-storageINFOOptional package not installed: google-cloud-s...Falsepip install google-cloud-storage
10Optional: scikit-learnINFOOptional package not installed: scikit-learn (...Falsepip install scikit-learn
11Optional: matplotlibINFOOptional package not installed: matplotlib (Pl...Falsepip install matplotlib
12Optional: seabornINFOOptional package not installed: seaborn (Stati...Falsepip install seaborn
13DockerPASSDocker version 20.10.23, build 7155243True
14TerraformPASSTerraform v1.9.5True
15Google Cloud CLIPASSGoogle Cloud SDK 551.0.0False
16AWS CLIINFONot installed (optional for cloud deployments)FalseInstall from: https://aws.amazon.com/cli/
17Azure CLIINFONot installed (optional for cloud deployments)FalseInstall from: https://docs.microsoft.com/en-us...
18GitPASSgit version 2.50.1 (Apple Git-155)False
19InfracostPASSInfracost v0.10.43False
20Docker PermissionsPASSCan run Docker commandsTrue
21GCP AuthenticationPASSAuthenticated with Google CloudFalse
22DeployML ConfigINFONo configuration file found (will use defaults)FalseLooked in: /Users/rclements/.deployml/config.y...
\n", - "
" - ], - "text/plain": [ - " name status \\\n", - "0 Python Version PASS \n", - "1 Package: typer PASS \n", - "2 Package: pyyaml FAIL \n", - "3 Package: jinja2 PASS \n", - "4 Package: pandas PASS \n", - "5 Package: requests PASS \n", - "6 Package: ipython FAIL \n", - "7 Package: jupyter PASS \n", - "8 Optional: mlflow INFO \n", - "9 Optional: google-cloud-storage INFO \n", - "10 Optional: scikit-learn INFO \n", - "11 Optional: matplotlib INFO \n", - "12 Optional: seaborn INFO \n", - "13 Docker PASS \n", - "14 Terraform PASS \n", - "15 Google Cloud CLI PASS \n", - "16 AWS CLI INFO \n", - "17 Azure CLI INFO \n", - "18 Git PASS \n", - "19 Infracost PASS \n", - "20 Docker Permissions PASS \n", - "21 GCP Authentication PASS \n", - "22 DeployML Config INFO \n", - "\n", - " message required \\\n", - "0 Python 3.14.2 (>= 3.11) True \n", - "1 CLI framework - v0.21.1 True \n", - "2 Missing required package: pyyaml (YAML configu... True \n", - "3 Template rendering - v3.1.6 True \n", - "4 Data manipulation - v2.3.3 True \n", - "5 HTTP client - v2.32.5 True \n", - "6 Missing required package: ipython (Interactive... True \n", - "7 Notebook support - v1.1.1 True \n", - "8 Optional package not installed: mlflow (ML exp... False \n", - "9 Optional package not installed: google-cloud-s... False \n", - "10 Optional package not installed: scikit-learn (... False \n", - "11 Optional package not installed: matplotlib (Pl... False \n", - "12 Optional package not installed: seaborn (Stati... False \n", - "13 Docker version 20.10.23, build 7155243 True \n", - "14 Terraform v1.9.5 True \n", - "15 Google Cloud SDK 551.0.0 False \n", - "16 Not installed (optional for cloud deployments) False \n", - "17 Not installed (optional for cloud deployments) False \n", - "18 git version 2.50.1 (Apple Git-155) False \n", - "19 Infracost v0.10.43 False \n", - "20 Can run Docker commands True \n", - "21 Authenticated with Google Cloud False \n", - "22 No configuration file found (will use defaults) False \n", - "\n", - " fix_command \\\n", - "0 \n", - "1 \n", - "2 pip install pyyaml \n", - "3 \n", - "4 \n", - "5 \n", - "6 pip install ipython \n", - "7 \n", - "8 pip install mlflow \n", - "9 pip install google-cloud-storage \n", - "10 pip install scikit-learn \n", - "11 pip install matplotlib \n", - "12 pip install seaborn \n", - "13 \n", - "14 \n", - "15 \n", - "16 Install from: https://aws.amazon.com/cli/ \n", - "17 Install from: https://docs.microsoft.com/en-us... \n", - "18 \n", - "19 \n", - "20 \n", - "21 \n", - "22 \n", - "\n", - " details \n", - "0 \n", - "1 \n", - "2 \n", - "3 \n", - "4 \n", - "5 \n", - "6 \n", - "7 \n", - "8 \n", - "9 \n", - "10 \n", - "11 \n", - "12 \n", - "13 \n", - "14 \n", - "15 \n", - "16 \n", - "17 \n", - "18 \n", - "19 \n", - "20 \n", - "21 \n", - "22 Looked in: /Users/rclements/.deployml/config.y... " - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# Run full diagnostics with all details\n", - "doctor = deployml.run_doctor(show_all=True)\n", - "\n", - "# Get results as DataFrame for analysis\n", - "results_df = doctor.to_dataframe()\n", - "results_df" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "🚀 Starting MLOps Stack Deployment...\n", - "\n", - "============================================================\n", - "🚀 DEPLOYML STACK DEPLOYMENT\n", - "============================================================\n", - "📋 Stack Name: gcp-mlops-stack-mlflow-vm_TESTING\n", - "☁️ Provider: gcp (hatchet17)\n", - "🌍 Region: us-west2\n", - "📄 Configuration: ../example/config/gcp-cloud-vm-sample.yaml\n", - "\n", - "⏳ Initializing deployment...\n", - "🔧 Command: deployml deploy -c ../example/config/gcp-cloud-vm-sample.yaml -y\n", - "\n", - "============================================================\n", - "📝 DEPLOYMENT LOG\n", - "============================================================\n", - "/Users/rclements/tmp/env/deployml/lib/python3.14/site-packages/deployml/diagnostics/doctor.py:10: UserWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html. The pkg_resources package is slated for removal as early as 2025-11-30. Refrain from using this package or pin to Setuptools<81.\n", - " import pkg_resources\n", - "📁 Using workspace: gcp-mlops-stack-mlflow-vm_TESTING\n", - "📍 Workspace path: /Users/rclements/Documents/research/deployml/notebooks/.deployml/gcp-mlops-stack-mlflow-vm_TESTING\n", - "📦 Copying module templates...\n", - "📦 Bucket config: artifact_tracking/mlflow -> mlflow-artifact-bucket-postgres-2-hatchet16-sacwtaag (create: True, exists: False)\n", - "🔧 Unified bucket creation: True\n", - "\n", - "🚀 STARTING DEPLOYMENT\n", - "----------------------------------------\n", - "🚀 Deploying gcp-mlops-stack-mlflow-vm_TESTING to gcp...\n", - "WARNING: Your active project does not match the quota project in your local Application Default Credentials file. This might result in unexpected quota issues.\n", - "To update your Application Default Credentials quota project, use the `gcloud auth application-default set-quota-project` command.\n", - "WARNING: [rclements@usfca.edu] does not have permission to access projects instance [hatchet17] (or it may not exist): The caller does not have permission. This command is authenticated as rclements@usfca.edu which is the active account specified by the [core/account] property\n", - "Updated property [core/project].\n", - "\n", - "🔧 INFRASTRUCTURE SETUP\n", - "----------------------------------------\n", - "📋 Initializing Terraform...\n", - "📊 Planning deployment...\n", - "💰 Running cost analysis...\n", - "============================================================\n", - "\n", - "============================================================\n", - "💰 COST ANALYSIS\n", - "============================================================\n", - "============================================================\n", - "💵 Monthly Cost: $53.48 USD\n", - "💵 Hourly Cost: $0.0733 USD\n", - "Resources: 5 supported, 51 total\n", - "⚠️ WARNING: Monthly cost exceeds $50 threshold!\n", - "📋 Resource Breakdown:\n", - "----------------------------------------\n", - "• google_compute_instance.mlflow_vm\n", - " Type: google_compute_instance\n", - " Monthly Cost: $30.34\n", - " └─ Instance usage (Linux/UNIX, on-demand, e2-medium): $29.38\n", - " └─ Standard provisioned storage (pd-standard): $0.96\n", - "• google_sql_database_instance.postgres\n", - " Type: google_sql_database_instance\n", - " Monthly Cost: $18.54\n", - " └─ SQL instance (db-f1-micro, zonal): $9.20\n", - " └─ Storage (SSD, zonal): $2.04\n", - " └─ IP address (if unused): $7.30\n", - "• google_storage_bucket.artifact_tracking_mlflow_artifact[0]\n", - " Type: google_storage_bucket\n", - " Monthly Cost: $4.60\n", - " └─ Storage (standard): $4.60 (usage-based)\n", - "📊 Note: Some resources have usage-based pricing\n", - " Actual costs may vary based on usage patterns\n", - "\n", - "🏗️ DEPLOYING INFRASTRUCTURE\n", - "----------------------------------------\n", - "🏗️ Applying changes... (Estimated time: ~20 minutes (Cloud SQL/PostgreSQL detected))\n", - "\u001b[?25l\n", - "\u001b[32m⠋\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:00\u001b[0m\n", - "\u001b[32m⠙\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:00\u001b[0m\n", - "\u001b[32m⠹\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:00\u001b[0m\n", - "\u001b[32m⠸\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:00\u001b[0m\n", - "\u001b[32m⠴\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:00\u001b[0m\n", - "\u001b[32m⠦\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:00\u001b[0m\n", - "\u001b[32m⠧\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:00\u001b[0m\n", - "\u001b[32m⠏\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:00\u001b[0m\n", - "\u001b[32m⠋\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:00\u001b[0m\n", - "\u001b[32m⠙\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:00\u001b[0m\n", - "\u001b[32m⠸\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:01\u001b[0m\n", - "\u001b[32m⠼\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:01\u001b[0m\n", - "\u001b[32m⠴\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:01\u001b[0m\n", - "\u001b[32m⠧\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:01\u001b[0m\n", - "\u001b[32m⠇\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:01\u001b[0m\n", - "\u001b[32m⠏\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:01\u001b[0m\n", - "\u001b[32m⠋\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:01\u001b[0m\n", - "\u001b[32m⠹\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:01\u001b[0m\n", - "\u001b[32m⠸\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:01\u001b[0m\n", - "\u001b[32m⠼\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:01\u001b[0m\n", - "\u001b[32m⠦\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:02\u001b[0m\n", - "\u001b[32m⠧\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:02\u001b[0m\n", - "\u001b[32m⠇\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:02\u001b[0m\n", - "\u001b[32m⠋\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:02\u001b[0m\n", - "\u001b[32m⠙\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:02\u001b[0m\n", - "\u001b[32m⠹\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:02\u001b[0m\n", - "\u001b[32m⠸\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:02\u001b[0m\n", - "\u001b[32m⠴\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:02\u001b[0m\n", - "\u001b[32m⠦\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:02\u001b[0m\n", - "\u001b[32m⠧\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:03\u001b[0m\n", - "\u001b[32m⠏\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:03\u001b[0m\n", - "\u001b[32m⠋\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:03\u001b[0m\n", - "\u001b[32m⠙\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:03\u001b[0m\n", - "\u001b[32m⠸\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:03\u001b[0m\n", - "\u001b[32m⠼\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:03\u001b[0m\n", - "\u001b[32m⠴\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:03\u001b[0m\n", - "\u001b[32m⠧\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:03\u001b[0m\n", - "\u001b[32m⠇\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:03\u001b[0m\n", - "\u001b[32m⠏\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:03\u001b[0m\n", - "\u001b[32m⠙\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:04\u001b[0m\n", - "\u001b[32m⠹\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:04\u001b[0m\n", - "\u001b[32m⠸\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:04\u001b[0m\n", - "\u001b[32m⠴\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:04\u001b[0m\n", - "\u001b[32m⠦\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:04\u001b[0m\n", - "\u001b[32m⠧\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:04\u001b[0m\n", - "\u001b[32m⠏\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:04\u001b[0m\n", - "\u001b[32m⠋\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:04\u001b[0m\n", - "\u001b[32m⠙\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:04\u001b[0m\n", - "\u001b[32m⠸\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:05\u001b[0m\n", - "\u001b[32m⠼\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:05\u001b[0m\n", - "\u001b[32m⠴\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:05\u001b[0m\n", - "\u001b[32m⠧\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:05\u001b[0m\n", - "\u001b[32m⠇\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:05\u001b[0m\n", - "\u001b[32m⠏\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:05\u001b[0m\n", - "\u001b[32m⠙\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:05\u001b[0m\n", - "\u001b[32m⠹\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:05\u001b[0m\n", - "\u001b[32m⠸\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:05\u001b[0m\n", - "\u001b[32m⠴\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:06\u001b[0m\n", - "\u001b[32m⠦\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:06\u001b[0m\n", - "\u001b[32m⠧\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:06\u001b[0m\n", - "\u001b[32m⠏\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:06\u001b[0m\n", - "\u001b[32m⠋\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:06\u001b[0m\n", - "\u001b[32m⠙\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:06\u001b[0m\n", - "\u001b[32m⠸\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:06\u001b[0m\n", - "\u001b[32m⠼\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:06\u001b[0m\n", - "\u001b[32m⠴\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:06\u001b[0m\n", - "\u001b[32m⠧\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:06\u001b[0m\n", - "\u001b[32m⠇\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:07\u001b[0m\n", - "\u001b[32m⠏\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:07\u001b[0m\n", - "\u001b[32m⠙\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:07\u001b[0m\n", - "\u001b[32m⠹\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:07\u001b[0m\n", - "\u001b[32m⠸\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:07\u001b[0m\n", - "\u001b[32m⠴\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:07\u001b[0m\n", - "\u001b[32m⠦\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:07\u001b[0m\n", - "\u001b[32m⠧\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:07\u001b[0m\n", - "\u001b[32m⠇\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:07\u001b[0m\n", - "\u001b[32m⠋\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:08\u001b[0m\n", - "\u001b[32m⠙\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:08\u001b[0m\n", - "\u001b[32m⠹\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:08\u001b[0m\n", - "\u001b[32m⠼\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:08\u001b[0m\n", - "\u001b[32m⠴\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:08\u001b[0m\n", - "\u001b[32m⠦\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:08\u001b[0m\n", - "\u001b[32m⠇\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:08\u001b[0m\n", - "\u001b[32m⠏\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:08\u001b[0m\n", - "\u001b[32m⠋\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:08\u001b[0m\n", - "\u001b[32m⠹\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:08\u001b[0m\n", - "\u001b[32m⠸\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:09\u001b[0m\n", - "\u001b[32m⠼\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:09\u001b[0m\n", - "\u001b[32m⠦\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:09\u001b[0m\n", - "\u001b[32m⠧\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:09\u001b[0m\n", - "\u001b[32m⠇\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:09\u001b[0m\n", - "\u001b[32m⠋\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:09\u001b[0m\n", - "\u001b[32m⠙\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:09\u001b[0m\n", - "\u001b[32m⠹\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:09\u001b[0m\n", - "\u001b[32m⠼\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:09\u001b[0m\n", - "\u001b[32m⠴\u001b[0m ⚠️ Terraform apply returned code 1 \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:10\u001b[0m\n", - "\u001b[?25h\n", - "❌ Terraform apply failed with exit code 1\n", - "📋 Check the Terraform log for details:\n", - " /Users/rclements/Documents/research/deployml/notebooks/.deployml/gcp-mlops-stack-mlflow-vm_TESTING/terraform/terraform_apply.log\n", - "💡 Common issues:\n", - " - Required GCP APIs may not be enabled (check log for API activation URLs)\n", - " - Insufficient IAM permissions\n", - " - Resource conflicts or quota limits\n", - "🔍 Last 20 lines of the log:\n", - " │ on main.tf line 259, in resource \"google_project_service\" \"required_apis\":\n", - " │ 259: resource \"google_project_service\" \"required_apis\" {\n", - " │\n", - " ╵\n", - " ╷\n", - " │ Error: Error when reading or editing Project Service : Request `List Project Services hatchet17` returned error: Batch request and retried single request \"List Project Services hatchet17\" both failed. Final error: Failed to list enabled services for project hatchet17: Get \"https://serviceusage.googleapis.com/v1/projects/hatchet17/services?alt=json&fields=services%2Fname%2CnextPageToken&filter=state%3AENABLED&pageSize=200&prettyPrint=false\": oauth2: \"invalid_grant\" \"Bad Request\"\n", - " │\n", - " │ with google_project_service.required_apis[\"serviceusage.googleapis.com\"],\n", - " │ on main.tf line 259, in resource \"google_project_service\" \"required_apis\":\n", - " │ 259: resource \"google_project_service\" \"required_apis\" {\n", - " │\n", - " ╵\n", - " ╷\n", - " │ Error: Error when reading or editing Project Service : Request `List Project Services hatchet17` returned error: Batch request and retried single request \"List Project Services hatchet17\" both failed. Final error: Failed to list enabled services for project hatchet17: Get \"https://serviceusage.googleapis.com/v1/projects/hatchet17/services?alt=json&fields=services%2Fname%2CnextPageToken&filter=state%3AENABLED&pageSize=200&prettyPrint=false\": oauth2: \"invalid_grant\" \"Bad Request\"\n", - " │\n", - " │ with google_project_service.required_apis[\"storage.googleapis.com\"],\n", - " │ on main.tf line 259, in resource \"google_project_service\" \"required_apis\":\n", - " │ 259: resource \"google_project_service\" \"required_apis\" {\n", - " │\n", - " ╵\n", - "\n", - "============================================================\n", - "❌ DEPLOYMENT FAILED (Exit Code: 1)\n", - "============================================================\n", - "\n", - "📋 Last 50 lines of output:\n", - "------------------------------------------------------------\n", - "\u001b[2K\u001b[32m⠹\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:08\u001b[0m\n", - "\u001b[2K\u001b[32m⠼\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:08\u001b[0m\n", - "\u001b[2K\u001b[32m⠴\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:08\u001b[0m\n", - "\u001b[2K\u001b[32m⠦\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:08\u001b[0m\n", - "\u001b[2K\u001b[32m⠇\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:08\u001b[0m\n", - "\u001b[2K\u001b[32m⠏\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:08\u001b[0m\n", - "\u001b[2K\u001b[32m⠋\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:08\u001b[0m\n", - "\u001b[2K\u001b[32m⠹\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:08\u001b[0m\n", - "\u001b[2K\u001b[32m⠸\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:09\u001b[0m\n", - "\u001b[2K\u001b[32m⠼\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:09\u001b[0m\n", - "\u001b[2K\u001b[32m⠦\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:09\u001b[0m\n", - "\u001b[2K\u001b[32m⠧\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:09\u001b[0m\n", - "\u001b[2K\u001b[32m⠇\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:09\u001b[0m\n", - "\u001b[2K\u001b[32m⠋\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:09\u001b[0m\n", - "\u001b[2K\u001b[32m⠙\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:09\u001b[0m\n", - "\u001b[2K\u001b[32m⠹\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:09\u001b[0m\n", - "\u001b[2K\u001b[32m⠼\u001b[0m DeployML: Preparing your cloud environment... \u001b[90m━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:09\u001b[0m\n", - "\u001b[2K\u001b[32m⠴\u001b[0m ⚠️ Terraform apply returned code 1 \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[35m 0%\u001b[0m \u001b[33m0:00:10\u001b[0m\n", - "\u001b[?25h\n", - "❌ Terraform apply failed with exit code 1\n", - "📋 Check the Terraform log for details:\n", - " /Users/rclements/Documents/research/deployml/notebooks/.deployml/gcp-mlops-stack-mlflow-vm_TESTING/terraform/terraform_apply.log\n", - "💡 Common issues:\n", - " - Required GCP APIs may not be enabled (check log for API activation URLs)\n", - " - Insufficient IAM permissions\n", - " - Resource conflicts or quota limits\n", - "🔍 Last 20 lines of the log:\n", - " │ on main.tf line 259, in resource \"google_project_service\" \"required_apis\":\n", - " │ 259: resource \"google_project_service\" \"required_apis\" {\n", - " │\n", - " ╵\n", - " ╷\n", - " │ Error: Error when reading or editing Project Service : Request `List Project Services hatchet17` returned error: Batch request and retried single request \"List Project Services hatchet17\" both failed. Final error: Failed to list enabled services for project hatchet17: Get \"https://serviceusage.googleapis.com/v1/projects/hatchet17/services?alt=json&fields=services%2Fname%2CnextPageToken&filter=state%3AENABLED&pageSize=200&prettyPrint=false\": oauth2: \"invalid_grant\" \"Bad Request\"\n", - " │\n", - " │ with google_project_service.required_apis[\"serviceusage.googleapis.com\"],\n", - " │ on main.tf line 259, in resource \"google_project_service\" \"required_apis\":\n", - " │ 259: resource \"google_project_service\" \"required_apis\" {\n", - " │\n", - " ╵\n", - " ╷\n", - " │ Error: Error when reading or editing Project Service : Request `List Project Services hatchet17` returned error: Batch request and retried single request \"List Project Services hatchet17\" both failed. Final error: Failed to list enabled services for project hatchet17: Get \"https://serviceusage.googleapis.com/v1/projects/hatchet17/services?alt=json&fields=services%2Fname%2CnextPageToken&filter=state%3AENABLED&pageSize=200&prettyPrint=false\": oauth2: \"invalid_grant\" \"Bad Request\"\n", - " │\n", - " │ with google_project_service.required_apis[\"storage.googleapis.com\"],\n", - " │ on main.tf line 259, in resource \"google_project_service\" \"required_apis\":\n", - " │ 259: resource \"google_project_service\" \"required_apis\" {\n", - " │\n", - " ╵\n", - "\n", - "🔍 Error Summary:\n", - "------------------------------------------------------------\n", - "❌ Terraform apply failed with exit code 1\n", - " │ Error: Error when reading or editing Project Service : Request `List Project Services hatchet17` returned error: Batch request and retried single request \"List Project Services hatchet17\" both failed. Final error: Failed to list enabled services for project hatchet17: Get \"https://serviceusage.googleapis.com/v1/projects/hatchet17/services?alt=json&fields=services%2Fname%2CnextPageToken&filter=state%3AENABLED&pageSize=200&prettyPrint=false\": oauth2: \"invalid_grant\" \"Bad Request\"\n", - " │ Error: Error when reading or editing Project Service : Request `List Project Services hatchet17` returned error: Batch request and retried single request \"List Project Services hatchet17\" both failed. Final error: Failed to list enabled services for project hatchet17: Get \"https://serviceusage.googleapis.com/v1/projects/hatchet17/services?alt=json&fields=services%2Fname%2CnextPageToken&filter=state%3AENABLED&pageSize=200&prettyPrint=false\": oauth2: \"invalid_grant\" \"Bad Request\"\n", - "============================================================\n" - ] - }, - { - "ename": "RuntimeError", - "evalue": "Deployment failed with exit code 1. Check the output above for details. Total output lines: 180, Error-related lines: 3", - "output_type": "error", - "traceback": [ - "\u001b[31m---------------------------------------------------------------------------\u001b[39m", - "\u001b[31mRuntimeError\u001b[39m Traceback (most recent call last)", - "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[6]\u001b[39m\u001b[32m, line 2\u001b[39m\n\u001b[32m 1\u001b[39m \u001b[38;5;28mprint\u001b[39m(\u001b[33m\"\u001b[39m\u001b[33m🚀 Starting MLOps Stack Deployment...\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m----> \u001b[39m\u001b[32m2\u001b[39m stack = \u001b[43mdeployml\u001b[49m\u001b[43m.\u001b[49m\u001b[43mdeploy\u001b[49m\u001b[43m(\u001b[49m\u001b[33;43m\"\u001b[39;49m\u001b[33;43m../example/config/gcp-cloud-vm-sample.yaml\u001b[39;49m\u001b[33;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/tmp/env/deployml/lib/python3.14/site-packages/deployml/notebook/deployment.py:69\u001b[39m, in \u001b[36mdeploy\u001b[39m\u001b[34m(config_path, show_progress)\u001b[39m\n\u001b[32m 66\u001b[39m workspace_dir = Path.cwd() / \u001b[33m\"\u001b[39m\u001b[33m.deployml\u001b[39m\u001b[33m\"\u001b[39m / workspace_name\n\u001b[32m 68\u001b[39m \u001b[38;5;66;03m# Run deployment using CLI command with logs (use resolved config_file path)\u001b[39;00m\n\u001b[32m---> \u001b[39m\u001b[32m69\u001b[39m \u001b[43m_deploy_with_cli\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mstr\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43mconfig_file\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mworkspace_dir\u001b[49m\u001b[43m)\u001b[49m\n\u001b[32m 71\u001b[39m \u001b[38;5;66;03m# Create and return deployment stack\u001b[39;00m\n\u001b[32m 72\u001b[39m stack = DeploymentStack(config, workspace_dir)\n", - "\u001b[36mFile \u001b[39m\u001b[32m~/tmp/env/deployml/lib/python3.14/site-packages/deployml/notebook/deployment.py:298\u001b[39m, in \u001b[36m_deploy_with_cli\u001b[39m\u001b[34m(config_path, workspace_dir)\u001b[39m\n\u001b[32m 295\u001b[39m sys.stdout.flush()\n\u001b[32m 296\u001b[39m sys.stderr.flush()\n\u001b[32m--> \u001b[39m\u001b[32m298\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\n\u001b[32m 299\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mDeployment failed with exit code \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mprocess.returncode\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m. \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 300\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mCheck the output above for details. \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 301\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mTotal output lines: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(output_lines)\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m, \u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m 302\u001b[39m \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mError-related lines: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m([l\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mfor\u001b[39;00m\u001b[38;5;250m \u001b[39ml\u001b[38;5;250m \u001b[39m\u001b[38;5;129;01min\u001b[39;00m\u001b[38;5;250m \u001b[39moutput_lines\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mif\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28many\u001b[39m(k\u001b[38;5;250m \u001b[39m\u001b[38;5;129;01min\u001b[39;00m\u001b[38;5;250m \u001b[39ml.lower()\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mfor\u001b[39;00m\u001b[38;5;250m \u001b[39mk\u001b[38;5;250m \u001b[39m\u001b[38;5;129;01min\u001b[39;00m\u001b[38;5;250m \u001b[39m[\u001b[33m'\u001b[39m\u001b[33merror\u001b[39m\u001b[33m'\u001b[39m,\u001b[38;5;250m \u001b[39m\u001b[33m'\u001b[39m\u001b[33mfailed\u001b[39m\u001b[33m'\u001b[39m,\u001b[38;5;250m \u001b[39m\u001b[33m'\u001b[39m\u001b[33mfatal\u001b[39m\u001b[33m'\u001b[39m])])\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m 303\u001b[39m )\n", - "\u001b[31mRuntimeError\u001b[39m: Deployment failed with exit code 1. Check the output above for details. Total output lines: 180, Error-related lines: 3" - ] - } - ], - "source": [ - "print(\"🚀 Starting MLOps Stack Deployment...\")\n", - "stack = deployml.deploy(\"../example/config/gcp-cloud-vm-sample.yaml\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 2: View Service URLs as DataFrame" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Extract all service URLs from the deployment (now includes PostgreSQL and cron jobs)\n", - "urls_df = stack.show_urls()\n", - "urls_df" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 3: Access Individual Services" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Direct URL access (now includes all services)\n", - "print(f\"MLflow Tracking: {stack.urls.mlflow}\")\n", - "print(f\"Model Serving: {stack.urls.serving}\")\n", - "print(f\"Feature Store: {stack.urls.feast}\")\n", - "print(f\"Monitoring: {stack.urls.grafana}\")\n", - "print(f\"PostgreSQL Database: {stack.urls.postgresql}\")\n", - "\n", - "# Access cron job URLs\n", - "if stack.urls.cron_jobs: \n", - " print(\"Workflow Orchestration - Cron Jobs:\")\n", - " for job_name, job_url in stack.urls.cron_jobs.items():\n", - " print(f\" {job_name}: {job_url}\")\n", - "else: \n", - " print(\"No cron jobs found in this deployment\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 5: PostgreSQL Connection Details\\n\\n**NEW**: Get detailed PostgreSQL connection information for database access.\"" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "stack.show_postgresql_connection()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "#stack.show_postgresql_connection(show_credentials=True)\n", - "\n", - "stack.show_postgresql_credentials()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Display PostgreSQL connection details\n", - "stack.show_postgresql_connection()\n", - "\n", - "# Get PostgreSQL info programmatically\n", - "\n", - "pg_info = stack.get_postgresql_info()\n", - "if 'connection_name' in pg_info:\n", - " print(f\"Quick access - Connection name: {pg_info['connection_name']}\")\n", - " print(f\"Cloud SQL Proxy command: {pg_info['cloud_sql_proxy']}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Display detailed cron job information\n", - "# \n", - "stack.show_cron_jobs()\n", - "# Access cron job URLs directly for navigation to GCP/AWS consoles\\n\n", - "if stack.urls.cron_jobs:\n", - " print(\"\\nDirect access to cron job console URLs:\")\n", - " for job_name, job_url in stack.urls.cron_jobs.items():\n", - " print(f\"\\n{job_name.replace('-', ' ').title()} Job:\")\n", - " print(f\" Console URL: {job_url}\")\n", - " print(f\" Click to view in {('GCP' if 'console.cloud.google.com' in job_url else 'AWS')} Console\") \n", - "else:\n", - " print(\"No cron jobs found in this deployment\")\n", - " print(\"Add workflow_orchestration section to your YAML config to deploy cron jobs\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "## Summary - Enhanced Features\n", - "# ✅ **What's NEW in this version:**\n", - "# 1. **PostgreSQL Integration**: Direct access to database connection details\\n \n", - "# - Instance connection information\\n \n", - "# - Cloud SQL Proxy commands\n", - "# - Connection string examples\\n\\n2. \n", - "# \n", - "# **Workflow Orchestration**: Cron job management and monitoring\n", - "# - Direct links to GCP/AWS console for job management\n", - "# - Schedule and configuration details\n", - "# - Job status and execution history\n", - "# \n", - "# 3. **Enhanced Service Table**: Professional HTML table with clickable links\n", - "# - All service URLs in one place\n", - "# - PostgreSQL connection info \n", - "# - Cron job console links\n", - "# - Color-coded status indicators\n", - "# ✅ **Previous features:**\n", - "# - **CLI deployment**: \n", - "# `poetry run deployml deploy -c gcp-sample.yaml -y`\\n \n", - "# **Live logs**: All Terraform output, cost analysis, deployment progress\n", - "# - **DataFrame URLs**: Service URLs in pandas DataFrame format\n", - "# - **Pre-configured clients**: MLflow, Feast, etc.\n", - "# \n", - "# **Complete MLOps Stack Management:**\\n\n", - "# \n", - "# - Experiment tracking, feature stores, model serving, monitoring\n", - "# \n", - "# - Database access and workflow orchestration \n", - "# - Professional notebook interface with clickable links\\n\n", - "# - No CLI knowledge required!\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 7: Deployment Status Overview\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Step 6: Workflow Orchestration - Cron Jobs\n", - "\n", - "\n", - "**NEW**: Access and manage your deployed cron jobs for automated workflows.\"" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary\n", - "\n", - "✅ **What just happened:**\n", - "\n", - "1. **Ran CLI deployment**: `poetry run deployml deploy -c gcp-sample.yaml -y`\n", - "2. **Saw live logs**: All Terraform output, cost analysis, deployment progress\n", - "3. **Got URLs**: Extracted service URLs into a pandas DataFrame\n", - "4. **Ready to use**: Pre-configured clients for MLflow, Feast, etc.\n", - "\n", - "**Benefits:**\n", - "- Same reliable CLI deployment\n", - "- Live logs in notebook\n", - "- DataFrame output for URLs\n", - "- No manual URL hunting\n", - "- Pre-configured service clients" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "test_deployml", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.14.2" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/recycling_bin/notebooks/sandbox.ipynb b/recycling_bin/notebooks/sandbox.ipynb deleted file mode 100644 index 619686f..0000000 --- a/recycling_bin/notebooks/sandbox.ipynb +++ /dev/null @@ -1,610 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 2, - "id": "c308cbed", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Downloading artifacts: 100%|██████████| 7/7 [00:00<00:00, 7.25it/s]\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "✅ Model loaded successfully!\n", - "('bedrooms',)\n", - "('city',)\n", - "('area_sqft',)\n", - "('status',)\n", - "('bathrooms',)\n", - "('days_on_market',)\n", - "('price',)\n", - "('year_built',)\n", - "('listing_agent',)\n", - "('property_type',)\n", - "('zipcode_encoded',)\n", - "('state',)\n", - "('lot_size',)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/opt/anaconda3/envs/deployml/lib/python3.11/site-packages/google/cloud/bigquery/table.py:1965: UserWarning: BigQuery Storage module not found, fetch data with the REST endpoint instead.\n", - " warnings.warn(\n" - ] - }, - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
bedroomscityarea_sqftstatusbathroomsdays_on_marketpriceyear_builtlisting_agentproperty_typezipcode_encodedstatelot_sizemls_id
01010690343843750201000843750.003385112914
13021152459274902197010274902.007911923785
23227132115825806198530825806.007477659459
36326190211811538251966301153825.005135475595
4331854111212803481958301280348.004742647648
\n", - "
" - ], - "text/plain": [ - " bedrooms city area_sqft status bathrooms days_on_market price \\\n", - "0 1 0 1069 0 3 43 843750 \n", - "1 3 0 2115 2 4 59 274902 \n", - "2 3 2 2713 2 1 15 825806 \n", - "3 6 3 2619 0 2 118 1153825 \n", - "4 3 3 1854 1 1 12 1280348 \n", - "\n", - " year_built listing_agent property_type zipcode_encoded state lot_size \\\n", - "0 2010 0 0 843750.0 0 3385 \n", - "1 1970 1 0 274902.0 0 7911 \n", - "2 1985 3 0 825806.0 0 7477 \n", - "3 1966 3 0 1153825.0 0 5135 \n", - "4 1958 3 0 1280348.0 0 4742 \n", - "\n", - " mls_id \n", - "0 112914 \n", - "1 923785 \n", - "2 659459 \n", - "3 475595 \n", - "4 647648 " - ] - }, - "execution_count": 2, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "import mlflow\n", - "import mlflow.sklearn\n", - "import mlflow.pyfunc\n", - "from mlflow.tracking import MlflowClient\n", - "from mlflow.models import infer_signature\n", - "from datetime import datetime\n", - "import os\n", - "import json\n", - "from typing import Dict, List, Optional, Any\n", - "\n", - "from sqlalchemy import create_engine, text, Column, Integer, String, Float, DateTime, Text, Boolean, BigInteger, PrimaryKeyConstraint, Index, func\n", - "from sqlalchemy.orm import declarative_base, sessionmaker, Session\n", - "from sqlalchemy.dialects.postgresql import JSON\n", - "\n", - "import pandas as pd\n", - "from google.cloud import bigquery\n", - "import logging\n", - "from pydantic import BaseModel\n", - "\n", - "logging.basicConfig(level=logging.INFO)\n", - "logger = logging.getLogger(__name__)\n", - "\n", - "MLFLOW_TRACKING_URI = \"https://mlflow-server-555196125082.us-west1.run.app\" # Replace with your MLflow server\n", - "\n", - "model_name = \"HousingModel\"\n", - "stage = \"Production\"\n", - "\n", - "# Set up MLflow\n", - "mlflow.set_tracking_uri(MLFLOW_TRACKING_URI)\n", - "\n", - "model_uri = f\"models:/{model_name}/{stage}\"\n", - "\n", - "try:\n", - " # Load model\n", - " loaded_model = mlflow.pyfunc.load_model(model_uri)\n", - " print(\"✅ Model loaded successfully!\")\n", - "except Exception as e:\n", - " print(f\"❌ Model loading/prediction failed: {e}\")\n", - "\n", - "# get data from last two weeks via connecting to BigQuery\n", - "DATABASE_URL = os.getenv(\"DATABASE_URL\", \"postgresql+psycopg2://mlflow:nd1XVyGLcU-AoBF-@34.187.169.166:5432/feast\")\n", - "\n", - "db_engine = create_engine(\n", - " DATABASE_URL,\n", - " pool_size=1,\n", - " max_overflow=2,\n", - " pool_timeout=30,\n", - " pool_recycle=300,\n", - " pool_pre_ping=True,\n", - " echo=False,\n", - " pool_reset_on_return='commit'\n", - ")\n", - "\n", - "column_names = []\n", - "\n", - "with db_engine.connect() as conn:\n", - " result = conn.execute(text(\"SELECT distinct feature_name FROM public.housing_deployml2025_housing_features\"))\n", - " rows = result.fetchall()\n", - " for row in rows:\n", - " print(row)\n", - " column_names.append(row[0])\n", - "\n", - "PROJECT_ID = \"mldeploy-468919\"\n", - "DATASET_ID = \"feast_housing\"\n", - "\n", - "client = bigquery.Client(project=\"mldeploy-468919\", location=\"US\")\n", - "QUERY = f\"\"\"\n", - "SELECT {','.join(column_names)}, mls_id\n", - "FROM {PROJECT_ID}.{DATASET_ID}.house_data\n", - "WHERE (DATE(event_timestamp) < DATE(\"2025-08-06\"))\n", - "\"\"\"\n", - "\n", - "df = client.query(QUERY).to_dataframe()\n", - "\n", - "df.head()" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "9f8138e6", - "metadata": {}, - "outputs": [], - "source": [ - "Base = declarative_base()\n", - "\n", - "class PredictionMetrics(Base):\n", - " __tablename__ = 'prediction_metrics'\n", - " \n", - " id = Column(Integer, primary_key=True, autoincrement=True)\n", - " request_id = Column(String, unique=True, index=True)\n", - " mls_ids = Column(JSON) # Array of MLS IDs\n", - " predictions = Column(JSON) # Array of prediction results\n", - " processing_time_ms = Column(Float)\n", - " feature_source = Column(String)\n", - " model_name = Column(String)\n", - " model_stage = Column(String)\n", - " model_version = Column(String)\n", - " scoring_timestamp = Column(DateTime, default=datetime.utcnow, index=True) # When data was scored\n", - " created_at = Column(DateTime, default=datetime.utcnow) # When record was created\n", - " success = Column(Boolean, default=True)\n", - " error_message = Column(Text, nullable=True)\n", - "\n", - " __table_args__ = (\n", - " Index('idx_prediction_metrics_scoring_ts', 'scoring_timestamp'),\n", - " )\n", - " \n", - "start_time = datetime.now()\n", - "request_id = f\"req_{start_time.strftime('%Y%m%d_%H%M%S_%f')}\" \n", - "feature_source = 'feast_api'\n", - "model_stage = stage\n", - "MODEL_STAGE = model_stage\n", - "model_version=\"1\"\n", - "mls_ids = df['mls_id'].to_list()\n", - "MODEL_NAME = model_name" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "b4106036", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "2025/08/22 14:10:46 WARNING mlflow.models.utils: Found extra inputs in the model input that are not defined in the model signature: `['mls_id']`. These inputs will be ignored.\n" - ] - } - ], - "source": [ - "predictions = loaded_model.predict(df.drop(columns=['price']))\n", - "\n", - "class PredictionResult(BaseModel):\n", - " mls_id: int\n", - " predicted_price: float\n", - " formatted_price: str\n", - "\n", - "def log_prediction_metrics(request_id: str, mls_ids: List[int], predictions: List[dict], \n", - " processing_time: float, feature_source: str, success: bool, \n", - " features_df: pd.DataFrame = None, model_version: str = \"1\",\n", - " error_message: str = None, db_session=None):\n", - " \"\"\"Log comprehensive prediction metrics to database - now optional\"\"\"\n", - " if db_session is None:\n", - " logger.debug(\"Database session not available - skipping metrics logging\")\n", - " return\n", - " \n", - " try:\n", - " scoring_time = datetime.utcnow()\n", - " \n", - " # Log main prediction metrics\n", - " metric = PredictionMetrics(\n", - " request_id=request_id,\n", - " mls_ids=mls_ids,\n", - " predictions=predictions,\n", - " processing_time_ms=processing_time,\n", - " feature_source=feature_source,\n", - " model_name=MODEL_NAME,\n", - " model_stage=MODEL_STAGE,\n", - " model_version=model_version,\n", - " scoring_timestamp=scoring_time,\n", - " success=success,\n", - " error_message=error_message\n", - " )\n", - " db_session.add(metric) \n", - " db_session.commit()\n", - " logger.info(f\"📊 Comprehensive metrics logged for request {request_id} ({len(mls_ids)} predictions)\")\n", - " \n", - " except Exception as e:\n", - " logger.warning(f\"Failed to log metrics (non-critical): {e}\")\n", - " try:\n", - " db_session.rollback()\n", - " except:\n", - " pass\n", - "\n", - "\n", - "results = []\n", - "for mls_id, prediction in zip(df['mls_id'].to_list(), predictions):\n", - " results.append({\n", - " 'mls_id':mls_id,\n", - " 'predicted_price':float(prediction),\n", - " 'formatted_price':f\"${prediction:,.2f}\"}\n", - " )\n", - "\n", - "end_time = datetime.now()\n", - "processing_time = (end_time - start_time).total_seconds() * 1000 " - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "c1880dd6", - "metadata": {}, - "outputs": [], - "source": [ - "MODEL_FEATURE_ORDER = [\n", - " 'city', 'state', 'bedrooms', 'bathrooms', 'area_sqft', \n", - " 'lot_size', 'year_built', 'days_on_market', 'property_type', \n", - " 'listing_agent', 'status', 'zipcode_encoded'\n", - "]\n", - "\n", - "feature_columns = [col for col in MODEL_FEATURE_ORDER if col in df.columns]\n", - "features_df = df[feature_columns]\n", - "DATABASE_URL = \"postgresql+psycopg2://mlflow:nd1XVyGLcU-AoBF-@34.187.169.166:5432/metrics\"\n", - "\n", - "# Create engine\n", - "engine = create_engine(DATABASE_URL, echo=True, future=True)\n", - "\n", - "# Create session factory\n", - "SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False)\n", - "\n", - "# Open a session\n", - "db = SessionLocal()" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "2caa1353", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-08-22 14:13:13,977 INFO sqlalchemy.engine.Engine select pg_catalog.version()\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sqlalchemy.engine.Engine:select pg_catalog.version()\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-08-22 14:13:13,979 INFO sqlalchemy.engine.Engine [raw sql] {}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sqlalchemy.engine.Engine:[raw sql] {}\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-08-22 14:13:14,043 INFO sqlalchemy.engine.Engine select current_schema()\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sqlalchemy.engine.Engine:select current_schema()\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-08-22 14:13:14,044 INFO sqlalchemy.engine.Engine [raw sql] {}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sqlalchemy.engine.Engine:[raw sql] {}\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-08-22 14:13:14,113 INFO sqlalchemy.engine.Engine show standard_conforming_strings\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sqlalchemy.engine.Engine:show standard_conforming_strings\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-08-22 14:13:14,114 INFO sqlalchemy.engine.Engine [raw sql] {}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sqlalchemy.engine.Engine:[raw sql] {}\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-08-22 14:13:14,179 INFO sqlalchemy.engine.Engine BEGIN (implicit)\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sqlalchemy.engine.Engine:BEGIN (implicit)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-08-22 14:13:14,186 INFO sqlalchemy.engine.Engine INSERT INTO prediction_metrics (request_id, mls_ids, predictions, processing_time_ms, feature_source, model_name, model_stage, model_version, scoring_timestamp, created_at, success, error_message) VALUES (%(request_id)s, %(mls_ids)s, %(predictions)s, %(processing_time_ms)s, %(feature_source)s, %(model_name)s, %(model_stage)s, %(model_version)s, %(scoring_timestamp)s, %(created_at)s, %(success)s, %(error_message)s) RETURNING prediction_metrics.id\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sqlalchemy.engine.Engine:INSERT INTO prediction_metrics (request_id, mls_ids, predictions, processing_time_ms, feature_source, model_name, model_stage, model_version, scoring_timestamp, created_at, success, error_message) VALUES (%(request_id)s, %(mls_ids)s, %(predictions)s, %(processing_time_ms)s, %(feature_source)s, %(model_name)s, %(model_stage)s, %(model_version)s, %(scoring_timestamp)s, %(created_at)s, %(success)s, %(error_message)s) RETURNING prediction_metrics.id\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-08-22 14:13:14,187 INFO sqlalchemy.engine.Engine [generated in 0.00429s] {'request_id': 'req_20250822_141031_863131', 'mls_ids': '[112914, 923785, 659459, 475595, 647648, 909359, 907698, 935760, 984897, 476920, 624860, 975333, 966187, 266062, 496240, 172814, 995727, 759953, 4511 ... (23582 characters truncated) ... 8657, 424130, 753104, 871262, 670212, 229012, 984118, 533640, 830005, 736786, 505545, 395094, 637990, 441673, 851475, 893394, 364680, 289803, 900458]', 'predictions': '[{\"mls_id\": 112914, \"predicted_price\": 839599.1578798194, \"formatted_price\": \"$839,599.16\"}, {\"mls_id\": 923785, \"predicted_price\": 275027.2675555555, ... (277452 characters truncated) ... : 467888.971855159, \"formatted_price\": \"$467,888.97\"}, {\"mls_id\": 900458, \"predicted_price\": 1249778.5011230162, \"formatted_price\": \"$1,249,778.50\"}]', 'processing_time_ms': 15125.985999999999, 'feature_source': 'feast_api', 'model_name': 'HousingModel', 'model_stage': 'Production', 'model_version': '1', 'scoring_timestamp': datetime.datetime(2025, 8, 22, 21, 13, 13, 593522), 'created_at': datetime.datetime(2025, 8, 22, 21, 13, 14, 183248), 'success': True, 'error_message': None}\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sqlalchemy.engine.Engine:[generated in 0.00429s] {'request_id': 'req_20250822_141031_863131', 'mls_ids': '[112914, 923785, 659459, 475595, 647648, 909359, 907698, 935760, 984897, 476920, 624860, 975333, 966187, 266062, 496240, 172814, 995727, 759953, 4511 ... (23582 characters truncated) ... 8657, 424130, 753104, 871262, 670212, 229012, 984118, 533640, 830005, 736786, 505545, 395094, 637990, 441673, 851475, 893394, 364680, 289803, 900458]', 'predictions': '[{\"mls_id\": 112914, \"predicted_price\": 839599.1578798194, \"formatted_price\": \"$839,599.16\"}, {\"mls_id\": 923785, \"predicted_price\": 275027.2675555555, ... (277452 characters truncated) ... : 467888.971855159, \"formatted_price\": \"$467,888.97\"}, {\"mls_id\": 900458, \"predicted_price\": 1249778.5011230162, \"formatted_price\": \"$1,249,778.50\"}]', 'processing_time_ms': 15125.985999999999, 'feature_source': 'feast_api', 'model_name': 'HousingModel', 'model_stage': 'Production', 'model_version': '1', 'scoring_timestamp': datetime.datetime(2025, 8, 22, 21, 13, 13, 593522), 'created_at': datetime.datetime(2025, 8, 22, 21, 13, 14, 183248), 'success': True, 'error_message': None}\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "2025-08-22 14:13:14,439 INFO sqlalchemy.engine.Engine COMMIT\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "INFO:sqlalchemy.engine.Engine:COMMIT\n", - "INFO:__main__:📊 Comprehensive metrics logged for request req_20250822_141031_863131 (2985 predictions)\n" - ] - } - ], - "source": [ - "log_prediction_metrics(\n", - " request_id=request_id,\n", - " mls_ids=mls_ids,\n", - " predictions=results,\n", - " processing_time=processing_time,\n", - " feature_source=feature_source,\n", - " success=True,\n", - " features_df=features_df,\n", - " model_version=\"1\",\n", - " db_session=db\n", - ")" - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "id": "da98b3a1", - "metadata": {}, - "outputs": [], - "source": [ - "db.close()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "mlops", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -}