벡터 DB 비교: Pinecone vs Chroma vs Weaviate

# pip install chromadb langchain-chroma langchain-openai
from langchain_chroma import Chroma
from langchain_openai import OpenAIEmbeddings
from langchain_core.documents import Document

embeddings = OpenAIEmbeddings(model="text-embedding-3-small")

# 영구 저장
vectorstore = Chroma(
    collection_name="my_docs",
    embedding_function=embeddings,
    persist_directory="./chroma_db",
)

# 문서 추가
docs = [
    Document(page_content="React는 UI 라이브러리", metadata={"track": "react"}),
    Document(page_content="Vue는 진화하는 프레임워크", metadata={"track": "vue"}),
    Document(page_content="PostgreSQL 인덱스", metadata={"track": "postgresql"}),
]
vectorstore.add_documents(docs)

# 검색
results = vectorstore.similarity_search("UI 라이브러리", k=2)
for r in results:
    print(r.page_content, r.metadata)


# 메타데이터 필터
results = vectorstore.similarity_search(
    "프레임워크",
    k=5,
    filter={"track": {"$in": ["react", "vue"]}},
)

# pip install pinecone-client langchain-pinecone
import os
from pinecone import Pinecone, ServerlessSpec
from langchain_pinecone import PineconeVectorStore
from langchain_openai import OpenAIEmbeddings

# 인덱스 생성 (한 번만)
pc = Pinecone(api_key=os.environ["PINECONE_API_KEY"])
pc.create_index(
    name="my-index",
    dimension=1536,  # OpenAI text-embedding-3-small
    metric="cosine",
    spec=ServerlessSpec(cloud="aws", region="us-east-1"),
)

embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
vectorstore = PineconeVectorStore(
    index_name="my-index",
    embedding=embeddings,
)

# 사용은 Chroma와 동일
vectorstore.add_documents(docs)
results = vectorstore.similarity_search("질문", k=4)

# Docker 실행
# docker run -p 8080:8080 cr.weaviate.io/semitechnologies/weaviate:1.24

# pip install langchain-weaviate weaviate-client
import weaviate
from langchain_weaviate import WeaviateVectorStore

client = weaviate.connect_to_local()

vectorstore = WeaviateVectorStore(
    client=client,
    index_name="MyDocs",
    text_key="content",
    embedding=embeddings,
)

# Weaviate 강점 — 그래프 관계
# 문서 ↔ 저자 ↔ 카테고리 같은 관계 모델링 가능

# OpenAI text-embedding-3-small
# - 차원 1536, 가격 $0.02/1M tokens
# - 한국어 OK, 일반 용도

# OpenAI text-embedding-3-large
# - 차원 3072, 가격 $0.13/1M tokens
# - 더 정확, 비용 6배

# Cohere embed-multilingual-v3
# - 100개 언어, 차원 1024
# - 다국어 RAG에 강함

# 로컬 — sentence-transformers/multilingual-e5
# - 무료, 단 GPU 권장
# - 차원 768~1024
from langchain_community.embeddings import HuggingFaceEmbeddings

embeddings_local = HuggingFaceEmbeddings(
    model_name="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
)

import os
from pathlib import Path
from langchain_text_splitters import RecursiveCharacterTextSplitter

# 1. 문서 로드 + 청크 분할
splitter = RecursiveCharacterTextSplitter(
    chunk_size=1000,
    chunk_overlap=200,
    separators=["\n\n", "\n", ". ", " "],
)

documents = []
for path in Path("./docs").rglob("*.md"):
    text = path.read_text()
    chunks = splitter.split_text(text)
    for i, chunk in enumerate(chunks):
        documents.append(Document(
            page_content=chunk,
            metadata={
                "source": str(path),
                "chunk": i,
            },
        ))

print(f"총 청크: {len(documents)}")

# 2. 배치 인덱싱 (메모리 절약)
batch_size = 100
for i in range(0, len(documents), batch_size):
    batch = documents[i:i+batch_size]
    vectorstore.add_documents(batch)
    print(f"진행: {min(i+batch_size, len(documents))}/{len(documents)}")