vector-db-rag-expert v4.4.2A

Immutable. This exact content never changes and is served at /api/v1/blob/51053a64604cee31.

---
name: vector-db-rag-expert
description: "Expert guide for high-performance Vector Databases, Deep RAG architectures, pgvector 0.8+ HNSW, Reciprocal Rank Fusion (RRF), Cross-Encoder Re-ranking, and Late Chunking / Panduan ahli Vector DB, arsitektur Deep RAG, pgvector HNSW, RRF, dan Re-ranking."
author: "Roedy Rustam"
version: "4.4.2"

---

# Vector DB & Deep RAG Expert (2026 Edition)

[English](#english) | [Bahasa Indonesia](#bahasa-indonesia)

---

<a name="english"></a>
## English

### Orchestration & Integration
- **Upstream AI Models & Embeddings**: `ai-llm-integration-expert`, `frontier-ai-models-expert`, `vercel-ai-sdk-expert`
- **Database & Storage Engines**: `database-orm-expert`, `graph-rag-knowledge-expert`, `search-engine-expert`
- **Retrieval & Pipeline Tooling**: `xerj-search-expert`, `llm-finops-router`, `context-window-engineer`
- **Security & Quality**: `prompt-injection-firewall`, `ai-safety-governance-expert`, `anti-slop`
- **Lifecycle & Governance**: `zero-to-prod-orchestrator`, `session-memory-manager`

### Purpose & Overview
Production-grade architectural guide for Vector Databases (PostgreSQL `pgvector 0.8+`, Qdrant, LanceDB, Pinecone), Deep RAG indexing strategies, HNSW iterative search, **Contextual Retrieval** (Anthropic 2026 standard), **Reciprocal Rank Fusion (RRF)** hybrid retrieval, **Cross-Encoder Re-ranking** (FlashRank, BGE-Reranker-v2), and **Late Chunking** to eliminate context fragmentation.

### Key Capabilities
1. **pgvector 0.8+ & Halfvec / Sparsevec**: High-dimensional vector storage with 50% RAM savings using `halfvec`, native BM25 sparse vectors via `sparsevec`, and iterative HNSW index scans with metadata filtering.
2. **Contextual Retrieval (Anthropic 2026 Standard)**: Prepend 50-100 token LLM-generated document context to each chunk before embedding and BM25 indexing, reducing failed retrieval rates by up to 67%.
3. **Reciprocal Rank Fusion (RRF)**: Combining sparse keyword BM25 ranks with dense semantic vector ranks using $RRF(d) = \sum \frac{1}{k + rank(d)}$ ($k = 60$).
4. **Cross-Encoder Re-ranking**: Two-stage retrieval pipeline: retrieve Top-50 candidates via fast hybrid search, then re-rank down to Top-5 using a cross-encoder model to maximize NDCG@10.
5. **Late Chunking**: Embed long-context documents in full before pooling token embeddings into individual chunks, preserving document-level semantics across boundaries.

---

### Production Implementation Recipes

#### Recipe 1: Contextual Chunk Prepending before Vector Indexing
```typescript
import { generateText } from 'ai';
import { google } from '@ai-sdk/google';

export interface ChunkWithContext {
  id: string;
  originalText: string;
  contextualizedText: string;
  contextSummary: string;
}

/**
 * Generates chunk-specific context using a fast model (Gemini 2.5 Flash)
 * following the Anthropic Contextual Retrieval methodology
 */
export async function generateContextualChunk(
  documentText: string,
  chunkText: string,
  chunkId: string
): Promise<ChunkWithContext> {
  const prompt = `
<document>
${documentText}
</document>

Here is the chunk we want to situate within the whole document:
<chunk>
${chunkText}
</chunk>

Please give a short, succinct context (1-2 sentences) to situate this chunk within the overall document for search retrieval. Answer only with the context.`;

  const { text: contextSummary } = await generateText({
    model: google('gemini-2.5-flash'),
    prompt,
  });

  const contextualizedText = `${contextSummary.trim()}\n\n${chunkText}`;

  return {
    id: chunkId,
    originalText: chunkText,
    contextualizedText,
    contextSummary: contextSummary.trim(),
  };
}
```

#### Recipe 2: Reciprocal Rank Fusion (RRF) Hybrid Search with Drizzle ORM
```typescript
import { sql } from 'drizzle-orm';
import { db } from '@/lib/db';

export interface SearchResult {
  id: string;
  content: string;
  rrfScore: number;
}

/**
 * Executes Reciprocal Rank Fusion (RRF) combining BM25 keyword search and pgvector HNSW
 * k = 60 is the industry standard constant
 */
export async function reciprocalRankFusionSearch(
  queryVector: number[],
  queryText: string,
  limit = 10,
  k = 60
): Promise<SearchResult[]> {
  const formattedVector = JSON.stringify(queryVector);

  const results = await db.execute(sql`
    WITH vector_matches AS (
      SELECT id, ROW_NUMBER() OVER (ORDER BY embedding <=> ${formattedVector}::halfvec) AS rank
      FROM documents
      WHERE status = 'published'
      ORDER BY embedding <=> ${formattedVector}::halfvec
      LIMIT 50
    ),
    text_matches AS (
      SELECT id, ROW_NUMBER() OVER (ORDER BY ts_rank_cd(fts, websearch_to_tsquery('english', ${queryText})) DESC) AS rank
      FROM documents
      WHERE fts @@ websearch_to_tsquery('english', ${queryText})
      LIMIT 50
    )
    SELECT
      d.id,
      d.content,
      COALESCE(1.0 / (${k} + v.rank), 0.0) +
      COALESCE(1.0 / (${k} + t.rank), 0.0) AS rrf_score
    FROM documents d
    LEFT JOIN vector_matches v ON d.id = v.id
    LEFT JOIN text_matches t ON d.id = t.id
    WHERE v.id IS NOT NULL OR t.id IS NOT NULL
    ORDER BY rrf_score DESC
    LIMIT ${limit};
  `);

  return (results.rows as any[]).map(row => ({
    id: row.id,
    content: row.content,
    rrfScore: Number(row.rrf_score),
  }));
}
```

#### Recipe 3: Cross-Encoder Re-Ranking Pipeline with FlashRank
```typescript
import { FlashRankRegistry } from 'flashrank';

const ranker = new FlashRankRegistry();

export async function rerankCandidates(
  query: string,
  candidates: { id: string; text: string }[],
  topN = 5
) {
  const passages = candidates.map(c => ({ id: c.id, text: c.text }));

  // Ultra-fast client/server cross-encoder re-ranking (<5ms)
  const reranked = await ranker.rerank({
    query,
    passages,
    model: 'ms-marco-TinyBERT-L-2-v2',
  });

  return reranked.slice(0, topN);
}
```

---

<a name="bahasa-indonesia"></a>
## Bahasa Indonesia

### Integrasi Orkestrasi
- **Model AI & Embedding Hulu**: `ai-llm-integration-expert`, `frontier-ai-models-expert`, `vercel-ai-sdk-expert`
- **Engine Database & Penyimpanan**: `database-orm-expert`, `graph-rag-knowledge-expert`, `search-engine-expert`
- **Tooling Retrieval & Pipeline**: `xerj-search-expert`, `llm-finops-router`, `context-window-engineer`
- **Keamanan & Kualitas**: `prompt-injection-firewall`, `ai-safety-governance-expert`, `anti-slop`
- **Siklus Hidup & Tata Kelola**: `zero-to-prod-orchestrator`, `session-memory-manager`

### Deskripsi
Panduan arsitektur tingkat produksi untuk Vector Database (PostgreSQL `pgvector 0.8+`, Qdrant, LanceDB, Pinecone), arsitektur Deep RAG modern, pencarian HNSW iteratif, **Contextual Retrieval** (standar Anthropic 2026), **Reciprocal Rank Fusion (RRF)** hybrid retrieval, **Cross-Encoder Re-ranking** (FlashRank, BGE-Reranker-v2), dan **Late Chunking** untuk mencegah fragmentasi konteks.

### Fitur Utama
1. **pgvector 0.8+ & Halfvec / Sparsevec**: Penyimpanan vektor dimensi tinggi dengan efisiensi RAM 50% via `halfvec`, dukungan native vektor renggang BM25 via `sparsevec`, dan pemindaian HNSW iteratif.
2. **Contextual Retrieval (Metodologi Anthropic 2026)**: Menyematkan konteks ringkas (50-100 token) yang di-generate model LLM pada setiap chunk sebelum proses embedding dan indeks BM25, memangkas kegagalan retrieval hingga 67%.
3. **Reciprocal Rank Fusion (RRF)**: Menggabungkan peringkat kata kunci BM25 dengan peringkat semantik vektor menggunakan rumus $RRF(d) = \sum \frac{1}{k + rank(d)}$ ($k = 60$).
4. **Cross-Encoder Re-ranking**: Pipeline retrieval 2 tahap: ambil 50 kandidat teratas melalui pencarian hybrid, lalu urutkan ulang menjadi 5 dokumen paling relevan menggunakan model cross-encoder berlatensi rendah.
5. **Late Chunking**: Melakukan embedding dokumen secara utuh dalam transformer sebelum memecahnya menjadi chunk-chunk terpisah, mempertahankan makna global dokumen.

---

### Resep Implementasi Produksi

#### Resep 1: Pembuatan Contextual Chunk Sebelum Indeks Vektor
```typescript
import { generateText } from 'ai';
import { google } from '@ai-sdk/google';

export async function buatChunkKontekstual(
  teksDokumen: string,
  teksChunk: string,
  chunkId: string
) {
  const prompt = `
<document>
${teksDokumen}
</document>

Berikut adalah potongan (chunk) yang ingin ditempatkan dalam dokumen:
<chunk>
${teksChunk}
</chunk>

Berikan konteks singkat (1-2 kalimat) untuk menempatkan chunk ini dalam dokumen secara utuh demi akurasi pencarian. Jawab hanya dengan kalimat konteks.`;

  const { text: ringkasanKonteks } = await generateText({
    model: google('gemini-2.5-flash'),
    prompt,
  });

  const teksKontekstual = `${ringkasanKonteks.trim()}\n\n${teksChunk}`;

  return {
    id: chunkId,
    teksAsli: teksChunk,
    teksKontekstual,
    ringkasanKonteks: ringkasanKonteks.trim(),
  };
}
```

#### Resep 2: Pencarian Hybrid RRF dengan Drizzle ORM
```typescript
import { sql } from 'drizzle-orm';
import { db } from '@/lib/db';

export async function cariDokumenRRF(
  queryVector: number[],
  queryText: string,
  limit = 10,
  k = 60
) {
  const vectorStr = JSON.stringify(queryVector);

  const hasil = await db.execute(sql`
    WITH vector_matches AS (
      SELECT id, ROW_NUMBER() OVER (ORDER BY embedding <=> ${vectorStr}::halfvec) AS rank
      FROM documents
      WHERE status = 'published'
      ORDER BY embedding <=> ${vectorStr}::halfvec
      LIMIT 50
    ),
    text_matches AS (
      SELECT id, ROW_NUMBER() OVER (ORDER BY ts_rank_cd(fts, websearch_to_tsquery('english', ${queryText})) DESC) AS rank
      FROM documents
      WHERE fts @@ websearch_to_tsquery('english', ${queryText})
      LIMIT 50
    )
    SELECT
      d.id,
      d.content,
      COALESCE(1.0 / (${k} + v.rank), 0.0) +
      COALESCE(1.0 / (${k} + t.rank), 0.0) AS skor_rrf
    FROM documents d
    LEFT JOIN vector_matches v ON d.id = v.id
    LEFT JOIN text_matches t ON d.id = t.id
    WHERE v.id IS NOT NULL OR t.id IS NOT NULL
    ORDER BY skor_rrf DESC
    LIMIT ${limit};
  `);

  return hasil.rows;
}
```