Run Edge LLMs on IoT Devices with Ollama and llama.cpp

                      
                     
import os
import subprocess
from typing import Dict, Any

# Configuration
class Config:
    LLM_MODEL_PATH: str = os.getenv('LLM_MODEL_PATH', '/path/to/ollama/model')
    TIMEOUT: int = 30000  # milliseconds

# Function to run the LLM model
async def run_llm(input_data: str) -> Dict[str, Any]:
    try:
        # Prepare the command to run the model using Ollama
        command = ["ollama", "run", Config.LLM_MODEL_PATH, input_data]
        # Execute the command
        result = subprocess.run(command, capture_output=True, text=True, check=True)
        return {'success': True, 'output': result.stdout}
    except subprocess.CalledProcessError as e:
        return {'success': False, 'error': str(e)}

# Main execution
if __name__ == '__main__':
    input_text = "Hello, how can I use LLM on IoT devices?"
    response = await run_llm(input_text)
    print(response)