Build a Private RAG Chat Over Your Documents - No Backend Required

┌─────────────────────────────────────────────────────────────────┐
│                        Browser Tab                              │
│                                                                 │
│  ┌──────────┐   ┌──────────┐   ┌──────────┐   ┌────────────┐  │
│  │  PDF.js   │──▶│  Chunker  │──▶│ Embedding │──▶│  VectorDB   │  │
│  │ (extract) │   │(recursive)│   │(BGE-small)│   │ (IndexedDB) │  │
│  └──────────┘   └──────────┘   └──────────┘   └─────┬──────┘  │
│                                                      │         │
│                                     ┌────────────────┘         │
│                                     ▼                          │
│  ┌──────────┐   ┌──────────┐   ┌──────────┐                   │
│  │ Streaming │◀──│  Prompt   │◀──│ Semantic  │                   │
│  │   LLM    │   │ Assembly  │   │  Search   │                   │
│  └──────────┘   └──────────┘   └──────────┘                   │
│                                                                 │
│  Models: BGE-small-en-v1.5 (33M params, ~33MB quantized)       │
│          Llama 3.2 1B or Qwen 3 1.7B (712MB-1.1GB)             │
└─────────────────────────────────────────────────────────────────┘

import { extractPDFText } from '@localmode/pdfjs';

// Accept a File from an <input type="file">
async function loadPDF(file: File) {
  const arrayBuffer = await file.arrayBuffer();

  const result = await extractPDFText(arrayBuffer, {
    includePageNumbers: true,
  });

  console.log(`Extracted ${result.pageCount} pages, ${result.text.length} characters`);
  return result;
}

import { chunk } from '@localmode/core';

const chunks = chunk(result.text, {
  strategy: 'recursive',
  size: 512,
  overlap: 50,
});

console.log(`Created ${chunks.length} chunks`);
// Each chunk: { text, start, end, index }

import { embedMany } from '@localmode/core';
import { transformers } from '@localmode/transformers';

// Create the embedding model (downloaded and cached on first use)
const embeddingModel = transformers.embedding('Xenova/bge-small-en-v1.5', {
  quantized: true,  // ~33MB download instead of ~127MB
});

// Embed all chunks in one call (auto-batched internally)
const { embeddings, usage } = await embedMany({
  model: embeddingModel,
  values: chunks.map((c) => c.text),
});

console.log(`Generated ${embeddings.length} embeddings, ${usage.tokens} tokens`);
// embeddings[0] is a Float32Array of length 384

import { streamEmbedMany } from '@localmode/core';

for await (const { embedding, index } of streamEmbedMany({
  model: embeddingModel,
  values: chunks.map((c) => c.text),
  batchSize: 32,
  onBatch: ({ index, count, total }) => {
    console.log(`Embedded ${index + count}/${total}`);
  },
})) {
  // Process each embedding as it arrives
}

import { createVectorDB } from '@localmode/core';

const db = await createVectorDB({
  name: 'rag-docs',
  dimensions: 384,
  compression: { type: 'sq8' }, // 4x storage reduction via scalar quantization
});

// Store all chunks with their embeddings and metadata
const documents = chunks.map((c, i) => ({
  id: `chunk-${i}`,
  vector: embeddings[i],
  metadata: {
    text: c.text,
    start: c.start,
    end: c.end,
    chunkIndex: c.index,
  },
}));

await db.addMany(documents);
console.log(`Indexed ${documents.length} vectors`);

import { semanticSearch } from '@localmode/core';

const { results, usage } = await semanticSearch({
  db,
  model: embeddingModel,
  query: 'What are the key findings of the report?',
  k: 5,
});

for (const result of results) {
  console.log(`[${result.score.toFixed(3)}] ${result.text?.substring(0, 80)}...`);
}

console.log(
  `Query embedded in ${usage.embedDurationMs.toFixed(0)}ms, ` +
  `searched in ${usage.searchDurationMs.toFixed(0)}ms`
);

import { rerank } from '@localmode/core';

const rerankerModel = transformers.reranker('Xenova/bge-reranker-base');

const reranked = await rerank({
  model: rerankerModel,
  query: 'What are the key findings?',
  documents: results.map((r) => r.text ?? ''),
  topK: 3,
});
// reranked.results: [{ index, score, document }]

import { streamText } from '@localmode/core';
import { webllm } from '@localmode/webllm';

// Create the LLM (downloaded and cached on first use)
const llm = webllm.languageModel('Qwen3-1.7B-q4f16_1-MLC');

// Build the prompt with retrieved context
const context = results
  .map((r, i) => `[${i + 1}] ${r.text}`)
  .join('\n\n');

const result = await streamText({
  model: llm,
  systemPrompt:
    'You are a helpful assistant. Answer the user\'s question based only on the ' +
    'provided context. If the context does not contain the answer, say so.',
  prompt: `Context:\n${context}\n\nQuestion: What are the key findings of the report?`,
  maxTokens: 500,
  temperature: 0.3,
});

// Stream the response token by token
for await (const chunk of result.stream) {
  process.stdout.write(chunk.text); // or append to your UI
}

const fullText = await result.text;
const usage = await result.usage;
console.log(`\nGenerated ${usage.outputTokens} tokens in ${usage.durationMs.toFixed(0)}ms`);

import { wllama } from '@localmode/wllama';

const llm = wllama.languageModel(
  'bartowski/Llama-3.2-1B-Instruct-GGUF:Llama-3.2-1B-Instruct-Q4_K_M.gguf'
);
// Same streamText() call works identically

import {
  createPipeline,
  pipelineEmbedStep,
  pipelineSearchStep,
} from '@localmode/core';

const ragPipeline = createPipeline('rag-query')
  .step('embed-query', pipelineEmbedStep(embeddingModel))
  .step('search', pipelineSearchStep(db, { k: 5 }))
  .build();

const { result } = await ragPipeline.run(userQuery, {
  abortSignal: controller.signal,
  onProgress: (p) => console.log(`${p.currentStep}: ${p.completed}/${p.total}`),
});

import { ingest } from '@localmode/core';

const result = await ingest(db, [{ text: pdfText, id: 'report.pdf' }], {
  chunking: { strategy: 'recursive', size: 512, overlap: 50 },
  generateEmbeddings: true,
  embedder: async (texts) => {
    const { embeddings } = await embedMany({ model: embeddingModel, values: texts });
    return embeddings;
  },
  onProgress: (p) => console.log(`${p.phase}: ${p.chunksProcessed}/${p.totalChunks}`),
});

console.log(`Ingested ${result.documentsProcessed} docs, ${result.chunksCreated} chunks`);