The Hybrid AI Architecture: Local Models for 95% of Requests, Cloud for the Rest

                            +------------------+
                            |   User Request   |
                            +--------+---------+
                                     |
                              +------v-------+
                              |  Task Router  |
                              +------+-------+
                                     |
                    +----------------+----------------+
                    |                                  |
          +---------v----------+            +----------v-----------+
          |   LOCAL PATH (95%) |            |   CLOUD PATH (5%)    |
          |                    |            |                      |
          |  Embeddings        |            |  Complex reasoning   |
          |  Classification    |            |  Long-context gen    |
          |  NER               |            |  Creative writing    |
          |  Reranking         |            |  Multi-hop QA        |
          |  Extractive QA     |            |  Rare languages      |
          |  Summarization     |            |                      |
          |  Translation       |            |                      |
          +---------+----------+            +----------+-----------+
                    |                                  |
          +---------v----------+            +----------v-----------+
          | @localmode/core    |            |  Cloud API (fetch)   |
          | @localmode/        |            |  OpenAI / Cohere /   |
          |   transformers     |            |  Google / Anthropic  |
          +---------+----------+            +----------+-----------+
                    |                                  |
                    +----------------+----------------+
                                     |
                              +------v-------+
                              |   Response   |
                              +--------------+

import {
  detectCapabilities,
  isWebGPUSupported,
  checkModelSupport,
} from '@localmode/core';

// Full capability report at app startup
const capabilities = await detectCapabilities();

console.log('WebGPU:', capabilities.features.webgpu);
console.log('WASM SIMD:', capabilities.features.simd);
console.log('Device memory:', capabilities.hardware.memory, 'GB');
console.log('CPU cores:', capabilities.hardware.cores);
console.log('Storage available:', capabilities.storage.availableBytes);

// Check if a specific model can run on this device
const modelCheck = await checkModelSupport({
  modelId: 'Qwen3-4B-Instruct',
  estimatedMemory: 2_200_000_000,  // 2.2GB
  estimatedStorage: 2_200_000_000,
  prefersWebGPU: true,
});

if (!modelCheck.supported) {
  console.log('Cannot run locally:', modelCheck.reason);
  console.log('Fallback models:', modelCheck.fallbackModels);
  // → Route to cloud API instead
}

import { embed, classify, rerank, extractEntities } from '@localmode/core';
import { transformers } from '@localmode/transformers';

// Embeddings - 99% of OpenAI quality, 8-30ms latency, $0
const { embedding } = await embed({
  model: transformers.embedding('Xenova/bge-small-en-v1.5'),
  value: userQuery,
});

// Classification - 95%+ of GPT-4o, 10-50ms, $0
const { label, score } = await classify({
  model: transformers.classifier(
    'Xenova/distilbert-base-uncased-finetuned-sst-2-english'
  ),
  text: userMessage,
});

// NER - 95-98% of GPT-4o, 30-100ms, $0
const { entities } = await extractEntities({
  model: transformers.ner('Xenova/bert-base-NER'),
  text: documentText,
});

// Reranking - 87-93% of Cohere, 20-80ms, $0
const { results } = await rerank({
  model: transformers.reranker('Xenova/ms-marco-MiniLM-L-6-v2'),
  query: searchQuery,
  documents: candidateResults,
});

import {
  generateText,
  streamText,
  wrapLanguageModel,
  detectCapabilities,
} from '@localmode/core';
import { webllm } from '@localmode/webllm';
import type { LanguageModelMiddleware } from '@localmode/core';

// Create a routing middleware that checks task complexity
const hybridRoutingMiddleware: LanguageModelMiddleware = {
  wrapGenerate: async ({ doGenerate, prompt, model }) => {
    // Estimate if this task needs cloud escalation
    const needsCloud =
      prompt.length > 8000 ||              // Long context
      requiresMultiStepReasoning(prompt) || // Complex reasoning
      requiresCreativeWriting(prompt);      // Nuanced generation

    if (needsCloud) {
      // Escalate to cloud API
      const response = await fetch('/api/generate', {
        method: 'POST',
        body: JSON.stringify({ prompt }),
      });
      const data = await response.json();
      return {
        text: data.text,
        finishReason: 'stop' as const,
        usage: {
          inputTokens: data.usage.input_tokens,
          outputTokens: data.usage.output_tokens,
          totalTokens: data.usage.total_tokens,
          durationMs: data.duration_ms,
        },
      };
    }

    // Handle locally - the 95% path
    return doGenerate();
  },
};

// Wrap the local model with routing middleware
const hybridModel = wrapLanguageModel({
  model: webllm.languageModel('Llama-3.2-1B-Instruct-q4f16_1-MLC'),
  middleware: hybridRoutingMiddleware,
});

// Use it exactly like any other model - routing is transparent
const { text } = await generateText({
  model: hybridModel,
  prompt: 'Summarize this customer feedback...',
  maxTokens: 200,
});

import {
  wrapLanguageModel,
  composeLanguageModelMiddleware,
} from '@localmode/core';
import type { LanguageModelMiddleware } from '@localmode/core';

// Cache responses to avoid redundant inference
const cachingMiddleware: LanguageModelMiddleware = {
  wrapGenerate: async ({ doGenerate, prompt }) => {
    const cached = responseCache.get(prompt);
    if (cached) return cached;
    const result = await doGenerate();
    responseCache.set(prompt, result);
    return result;
  },
};

// Log all generation calls for observability
const loggingMiddleware: LanguageModelMiddleware = {
  wrapGenerate: async ({ doGenerate, prompt, model }) => {
    const start = Date.now();
    const result = await doGenerate();
    console.log(
      `[${model.modelId}] ${prompt.slice(0, 40)}... → ${Date.now() - start}ms`
    );
    return result;
  },
};

// Stack: guardrails → cache → routing → logging
const composedMiddleware = composeLanguageModelMiddleware([
  loggingMiddleware,
  cachingMiddleware,
  hybridRoutingMiddleware,
]);

const productionModel = wrapLanguageModel({
  model: webllm.languageModel('Llama-3.2-1B-Instruct-q4f16_1-MLC'),
  middleware: composedMiddleware,
});

import {
  embed,
  classify,
  detectCapabilities,
  checkModelSupport,
  isWebGPUSupported,
} from '@localmode/core';
import { transformers } from '@localmode/transformers';

async function createHybridEmbedder() {
  // Embeddings are lightweight - almost always run locally
  // Even on low-end devices, bge-small is only 33MB
  return {
    embed: async (text: string) => {
      const { embedding } = await embed({
        model: transformers.embedding('Xenova/bge-small-en-v1.5'),
        value: text,
      });
      return embedding;
    },
  };
}

async function createHybridClassifier() {
  const caps = await detectCapabilities();
  const hasEnoughMemory =
    caps.hardware.memory === undefined || caps.hardware.memory >= 2;

  if (hasEnoughMemory) {
    // Run classification locally - 95%+ of cloud quality
    return {
      classify: async (text: string) => {
        const { label, score } = await classify({
          model: transformers.classifier(
            'Xenova/distilbert-base-uncased-finetuned-sst-2-english'
          ),
          text,
        });
        return { label, score, source: 'local' as const };
      },
    };
  }

  // Fallback to cloud API for very low-memory devices
  return {
    classify: async (text: string) => {
      const res = await fetch('/api/classify', {
        method: 'POST',
        body: JSON.stringify({ text }),
      });
      const data = await res.json();
      return { ...data, source: 'cloud' as const };
    },
  };
}

npm install @localmode/core @localmode/transformers @localmode/webllm

import { embed, classify, generateText, wrapLanguageModel } from '@localmode/core';
import { transformers } from '@localmode/transformers';
import { webllm } from '@localmode/webllm';

// Local path: embeddings and classification (the 95%)
const { embedding } = await embed({
  model: transformers.embedding('Xenova/bge-small-en-v1.5'),
  value: 'How do I reset my password?',
});

const { label } = await classify({
  model: transformers.classifier(
    'Xenova/distilbert-base-uncased-finetuned-sst-2-english'
  ),
  text: 'This product is amazing!',
});

// Hybrid path: LLM with middleware routing (the 5%)
const hybridModel = wrapLanguageModel({
  model: webllm.languageModel('Llama-3.2-1B-Instruct-q4f16_1-MLC'),
  middleware: hybridRoutingMiddleware, // See Step 3 above
});

const { text } = await generateText({
  model: hybridModel,
  prompt: 'Summarize the key points from this feedback...',
  maxTokens: 200,
});