RAG 구현: 문서 임베딩 + 벡터 검색

1. 사용자 문서 업로드 (PDF·텍스트)
2. Chunking (500-1000 토큰 단위 분할)
3. Embedding (각 chunk → 1536 차원 벡터)
4. pgvector에 저장
5. 사용자 질문 → embedding → 유사 chunk top-5 검색
6. 검색 결과 + 질문 → LLM prompt
7. 답변 + 출처 표시

-- 마이그레이션
create extension if not exists vector;

create table documents (
  id uuid primary key default gen_random_uuid(),
  user_id uuid references users(id) on delete cascade,
  filename text not null,
  content text,
  created_at timestamptz default now()
);

create table chunks (
  id uuid primary key default gen_random_uuid(),
  document_id uuid references documents(id) on delete cascade,
  content text not null,
  embedding vector(1536),  -- OpenAI text-embedding-3-small
  metadata jsonb
);

-- HNSW 인덱스 (빠른 KNN 검색)
create index idx_chunks_embedding on chunks
using hnsw (embedding vector_cosine_ops);

-- RLS
alter table documents enable row level security;
alter table chunks enable row level security;

create policy "users own docs" on documents for all using (auth.uid() = user_id);
create policy "users own chunks" on chunks for all using (
  exists (select 1 from documents where documents.id = chunks.document_id and documents.user_id = auth.uid())
);

// src/lib/rag/chunk.ts
export function chunkText(text: string, size = 800, overlap = 100): string[] {
  const chunks: string[] = []
  let start = 0
  while (start < text.length) {
    chunks.push(text.slice(start, start + size))
    start += size - overlap  // 100자 오버랩 (문맥 유지)
  }
  return chunks
}

// 더 좋은 방법: 문장 경계 기반 (langchain RecursiveCharacterTextSplitter)

// src/lib/rag/embed.ts
import { openai } from '@ai-sdk/openai'
import { embed, embedMany } from 'ai'

export async function embedText(text: string) {
  const { embedding } = await embed({
    model: openai.embedding('text-embedding-3-small'),
    value: text,
  })
  return embedding  // 1536-dim 벡터
}

export async function embedBatch(texts: string[]) {
  const { embeddings } = await embedMany({
    model: openai.embedding('text-embedding-3-small'),
    values: texts,
  })
  return embeddings
}

// 문서 업로드 시
async function indexDocument(userId: string, filename: string, content: string) {
  const supabase = createServiceClient()

  // 1. 문서 저장
  const { data: doc } = await supabase
    .from('documents')
    .insert({ user_id: userId, filename, content })
    .select()
    .single()

  // 2. 청킹
  const chunks = chunkText(content)

  // 3. 임베딩 (배치)
  const embeddings = await embedBatch(chunks)

  // 4. 저장
  await supabase.from('chunks').insert(
    chunks.map((c, i) => ({
      document_id: doc.id,
      content: c,
      embedding: embeddings[i],
      metadata: { chunk_index: i },
    }))
  )
}

// SQL 함수 생성
create or replace function match_chunks(
  query_embedding vector(1536),
  match_user_id uuid,
  match_count int
)
returns table (id uuid, content text, similarity float)
language sql stable as $$
  select c.id, c.content, 1 - (c.embedding <=> query_embedding) as similarity
  from chunks c
  join documents d on c.document_id = d.id
  where d.user_id = match_user_id
  order by c.embedding <=> query_embedding
  limit match_count
$$;

// TypeScript에서 호출
async function searchChunks(userId: string, query: string, k = 5) {
  const supabase = await createClient()
  const queryEmbedding = await embedText(query)

  const { data } = await supabase.rpc('match_chunks', {
    query_embedding: queryEmbedding,
    match_user_id: userId,
    match_count: k,
  })

  return data
}

// /api/chat/rag
import { streamText } from 'ai'
import { anthropic } from '@ai-sdk/anthropic'

export async function POST(request: Request) {
  const { question, userId } = await request.json()

  // 1. 검색
  const chunks = await searchChunks(userId, question, 5)
  const context = chunks.map(c => c.content).join('\n\n---\n\n')

  // 2. LLM
  const result = streamText({
    model: anthropic('claude-sonnet-4-6'),
    system: `당신은 사용자 문서 기반 어시스턴트입니다.
다음 문서를 참고하여 답하세요:

${context}

문서에 없는 내용은 "문서에 없습니다"라고 답하세요.`,
    prompt: question,
  })

  return result.toDataStreamResponse()
}

const result = streamText({
  ...,
  toolCalls: [{
    name: 'cite_source',
    parameters: z.object({
      chunk_ids: z.array(z.string()),
    }),
  }],
})

// UI
<div>
  {answer}
  <p>출처:</p>
  <ul>
    {chunks.map(c => <li key={c.id}>{c.metadata.filename} (chunk {c.metadata.chunk_index})</li>)}
  </ul>
</div>