文档索引构建#

1
# pip install langchain-community pypdf beautifulsoup4
2
from langchain_community.document_loaders import (
3
    TextLoader, PyPDFLoader, CSVLoader, WebBaseLoader, DirectoryLoader,
4
)
5

6
# --- TextLoader：纯文本文件 ---
7
loader = TextLoader("./docs/readme.txt", encoding="utf-8")
8
documents = loader.load()  # 整个文件 = 1 个 Document
9

10
# --- PyPDFLoader：PDF 文件（每页 = 1 个 Document） ---
11
loader = PyPDFLoader("./docs/report.pdf")
12
documents = loader.load()
13
print(f"PDF 共 {len(documents)} 页, 元信息: {documents[0].metadata}")
14
# 输出: {'source': './docs/report.pdf', 'page': 0}
15

16
# --- CSVLoader：CSV 文件（每行 = 1 个 Document） ---
17
loader = CSVLoader("./docs/products.csv", encoding="utf-8")
18
documents = loader.load()
19

20
# --- WebBaseLoader：网页抓取 ---
21
loader = WebBaseLoader("https://python.langchain.com/docs/introduction/")
22
documents = loader.load()
23

24
# --- DirectoryLoader：批量加载整个目录 ---
25
loader = DirectoryLoader(
26
    "./docs/", glob="**/*.txt",
27
    loader_cls=TextLoader,
28
    loader_kwargs={"encoding": "utf-8"},
29
    show_progress=True,
30
)
31
documents = loader.load()
32
print(f"共加载 {len(documents)} 个文档")

1
chunk_size 太小                                 chunk_size 太大
2
┌──────────────┐                    ┌──────────────────────────────┐
3
│ 语义不完整               │                    │ 噪声多，不够聚焦                                            │
4
│ 上下文缺失               │                    │ 浪费上下文窗口 Token                                     │
5
│ 检索结果碎片化        │                    │ 嵌入向量被稀释                                                │
6
└──────────────┘                    └──────────────────────────────┘
7
                \                          /
8
                \    推荐区间      /
9
                 \  500 - 1000 /
10
                   \  (中文)     /
11
                   ──────────

1
# pip install langchain-openai numpy
2
import numpy as np
3
from langchain_openai import OpenAIEmbeddings
4

5
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
6

7
# 准备三个语义不同的句子
8
texts = [
9
    "Python 是一种编程语言",      # 文本 A
10
    "Python 是流行的开发语言",     # 文本 B（与 A 语义相近）
11
    "今天的天气非常晴朗",          # 文本 C（与 A/B 语义无关）
12
]
13

14
vectors = embeddings.embed_documents(texts)
15

16
# 计算余弦相似度
17
def cosine_similarity(v1, v2):
18
    return np.dot(v1, v2) / (np.linalg.norm(v1) * np.linalg.norm(v2))
19

20
sim_ab = cosine_similarity(vectors[0], vectors[1])
21
sim_ac = cosine_similarity(vectors[0], vectors[2])
22
sim_bc = cosine_similarity(vectors[1], vectors[2])
23

24
print(f"A vs B (语义相近): {sim_ab:.4f}")  # ≈ 0.92
25
print(f"A vs C (语义无关): {sim_ac:.4f}")  # ≈ 0.65
26
print(f"B vs C (语义无关): {sim_bc:.4f}")  # ≈ 0.63
27
# A 和 B 的相似度远高于 A/C 和 B/C

1
# pip install langchain-openai langchain-community faiss-cpu
2
from langchain_openai import OpenAIEmbeddings
3
from langchain_community.vectorstores import FAISS
4
from langchain_core.documents import Document
5

6
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
7

8
documents = [
9
    Document(page_content="LangChain 是构建 LLM 应用的框架。", metadata={"source": "intro.md"}),
10
    Document(page_content="RAG 通过检索外部文档增强生成。", metadata={"source": "rag.md"}),
11
    Document(page_content="FAISS 是 Meta 开源的向量搜索库。", metadata={"source": "vectordb.md"}),
12
]
13

14
# 创建向量数据库 + 相似度搜索
15
# documents: 存入数据库的**原始内容**
16
# embedding: 用于将documents转化为向量
17
vectorstore = FAISS.from_documents(documents, embeddings)
18
results = vectorstore.similarity_search("什么是检索增强生成？", k=2)
19
for doc in results:
20
    print(f"[{doc.metadata['source']}] {doc.page_content}")
21

22
# 带分数的搜索（L2 距离，分数越低越相似）
23
results_with_scores = vectorstore.similarity_search_with_score("LLM 框架", k=2)
24

25
# 保存 / 加载
26
vectorstore.save_local("./faiss_index")
27
loaded_vectorstore = FAISS.load_local(
28
    "./faiss_index", embeddings, allow_dangerous_deserialization=True,
29
)

1
# pip install langchain-chroma langchain-openai
2
from langchain_openai import OpenAIEmbeddings
3
from langchain_chroma import Chroma
4
from langchain_core.documents import Document
5

6
embeddings = OpenAIEmbeddings(model="text-embedding-3-small")
7
documents = [
8
    Document(page_content="Python 是一种通用编程语言。", metadata={"lang": "zh"}),
9
    Document(page_content="Go 语言适合编写高并发服务。", metadata={"lang": "zh"}),
10
]
11

12
# 创建并持久化
13
vectorstore = Chroma.from_documents(
14
    documents, embeddings,
15
    persist_directory="./chroma_db",
16
    collection_name="programming_langs",
17
)
18

19
# 相似度搜索 + 元数据过滤
20
results = vectorstore.similarity_search("后端开发", k=2, filter={"lang": "zh"})
21

22
# 重新加载 + 增量添加
23
loaded_vs = Chroma(
24
    persist_directory="./chroma_db",
25
    embedding_function=embeddings,
26
    collection_name="programming_langs",
27
)
28
loaded_vs.add_documents([
29
    Document(page_content="Rust 以内存安全著称。", metadata={"lang": "zh"}),
30
])