Serve Production Models at Scale with Seldon Core and Prometheus Client

                      
                     
from fastapi import FastAPI, HTTPException
from prometheus_client import start_http_server, Summary, Counter
import os
import time
import requests

# Configuration
model_url = os.getenv('MODEL_URL')  # URL of the Seldon model
metrics_port = int(os.getenv('METRICS_PORT', '8000'))  # Port for metrics

# Prometheus metrics
request_duration = Summary('request_processing_seconds', 'Time spent processing request')
request_count = Counter('model_requests_total', 'Total model requests')

# Initialize FastAPI app
app = FastAPI()

# Start Prometheus metrics server
start_http_server(metrics_port)

@app.get("/predict")
@request_duration.time()
async def predict(input_data: dict):
    try:
        # Increase request count
        request_count.inc()
        # Forward the request to the Seldon model
        response = requests.post(model_url, json=input_data)
        response.raise_for_status()  # Raise an error for bad responses
        return response.json()  # Return the model response
    except requests.exceptions.RequestException as e:
        raise HTTPException(status_code=500, detail=f"Model request failed: {str(e)}")

if __name__ == '__main__':
    import uvicorn
    uvicorn.run(app, host='0.0.0.0', port=8000)