Concurrent Request Handling

import asyncio
from concurrent.futures import ThreadPoolExecutor

class ConcurrentLLMHandler:
    def __init__(self, max_concurrent=100):
        self.semaphore = asyncio.Semaphore(max_concurrent)
        self.executor = ThreadPoolExecutor(max_workers=50)
        
    async def handle_request(self, request):
        async with self.semaphore:
            # Rate limiting per model
            await self.rate_limiter.acquire(request.model)
            
            try:
                response = await self.process_with_retry(request)
                return response
            except Exception as e:
                return await self.fallback_handler(request, e)
                
    async def batch_process(self, requests):
        tasks = [self.handle_request(req) for req in requests]
        return await asyncio.gather(*tasks, return_exceptions=True)