Initialize Monorepo
This commit is contained in:
commit
f80792d739
106 files changed
+17076
No files matched your search
@@ -0,0 +1,21 @@
|
||||
# Chunker
|
||||
|
||||
Extract text, chunk it, and save images from a PDF.
|
||||
|
||||
chunks is a List[str] of ~800-token strings (100-token overlap).
|
||||
Outputs (text files and images) are written under extracted_content/<pdf_basename>/.
|
||||
## Usage
|
||||
|
||||
```python
|
||||
from chunker import Chunker
|
||||
|
||||
chunker = Chunker("path/to/file.pdf")
|
||||
chunks = chunker.run()
|
||||
|
||||
|
||||
|
||||
Setup:
|
||||
pip install -r requirements.txt
|
||||
python -m spacy download xx_ent_wiki_sm
|
||||
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
[project]
|
||||
name = "librarian-chunker"
|
||||
version = "0.1.0"
|
||||
description = "Chunker for Librarian"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"pdfplumber",
|
||||
"pymupdf",
|
||||
"tiktoken",
|
||||
"spacy",
|
||||
"sentence-transformers",
|
||||
"pydantic",
|
||||
"prefect",
|
||||
"librarian-core",
|
||||
"python-pptx",
|
||||
"python-docx",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling>=1.21"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/librarian_chunker"]
|
||||
|
||||
[tool.hatch.metadata]
|
||||
allow-direct-references = true
|
||||
|
||||
[tool.uv.sources]
|
||||
#librarian-core = { git = "https://github.com/DotNaos/librarian-core", rev = "dev" }
|
||||
|
||||
[project.entry-points."librarian.workers"]
|
||||
chunker = "librarian_chunker.chunker:Chunker"
|
||||
|
||||
|
||||
# ───────── optional: dev / test extras ─────────
|
||||
[project.optional-dependencies]
|
||||
dev = ["ruff", "pytest", "mypy"]
|
||||
@@ -0,0 +1,3 @@
|
||||
from .chunker import Chunker
|
||||
|
||||
__all__ = ["Chunker"]
|
||||
@@ -0,0 +1,217 @@
|
||||
import os
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
import pdfplumber
|
||||
import pymupdf
|
||||
import spacy
|
||||
import tiktoken
|
||||
from librarian_core.utils.path_utils import get_temp_path
|
||||
from librarian_core.workers.base import Worker
|
||||
from librarian_extractor.models.extract_data import ExtractData, ExtractedFile
|
||||
from prefect import get_run_logger, task
|
||||
from prefect.cache_policies import NO_CACHE
|
||||
from prefect.futures import wait
|
||||
|
||||
from librarian_chunker.models.chunk_data import (
|
||||
Chunk,
|
||||
ChunkData,
|
||||
ChunkedCourse,
|
||||
ChunkedTerm,
|
||||
)
|
||||
|
||||
MAX_TOKENS = 800
|
||||
OVERLAP_TOKENS = 100
|
||||
|
||||
|
||||
class Chunker(Worker[ExtractData, ChunkData]):
|
||||
input_model = ExtractData
|
||||
output_model = ChunkData
|
||||
|
||||
async def __run__(self, payload: ExtractData) -> ChunkData: # noqa: D401
|
||||
lg = get_run_logger()
|
||||
lg.info("Chunker started")
|
||||
|
||||
working_dir = get_temp_path()
|
||||
|
||||
# load NLP and tokenizer
|
||||
Chunker.nlp = spacy.load("xx_ent_wiki_sm")
|
||||
Chunker.nlp.add_pipe("sentencizer")
|
||||
Chunker.enc = tiktoken.get_encoding("cl100k_base")
|
||||
|
||||
# chunk parameters
|
||||
Chunker.max_tokens = MAX_TOKENS
|
||||
Chunker.overlap_tokens = OVERLAP_TOKENS
|
||||
|
||||
result = ChunkData(terms=[])
|
||||
|
||||
# Loading files
|
||||
for term in payload.terms:
|
||||
chunked_term = ChunkedTerm(id=term.id, name=term.name)
|
||||
in_term_dir = self.entry / term.name
|
||||
|
||||
out_term_dir = working_dir / term.name
|
||||
out_term_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for course in term.courses:
|
||||
chunked_course = ChunkedCourse(
|
||||
id=course.id, name=course.name, chunks=[]
|
||||
)
|
||||
in_course_dir = in_term_dir / course.name
|
||||
|
||||
out_course_dir = out_term_dir / course.name
|
||||
out_course_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
futs = []
|
||||
# All chunks are just in the course dir, so no new dir
|
||||
for chap in course.chapters:
|
||||
chapter_path = in_course_dir / chap.name
|
||||
|
||||
for f in chap.content_files:
|
||||
futs.append(
|
||||
self._chunk_file.submit(f, chapter_path, out_course_dir)
|
||||
)
|
||||
wait(futs)
|
||||
for fut in futs:
|
||||
chunks, images = fut.result()
|
||||
chunked_course.chunks.extend(chunks)
|
||||
chunked_course.images.extend(images)
|
||||
|
||||
chunked_term.courses.append(chunked_course)
|
||||
|
||||
# Add the chunked term to the result
|
||||
result.terms.append(chunked_term)
|
||||
self.stage(out_term_dir)
|
||||
|
||||
return result
|
||||
@staticmethod
|
||||
@task(log_prints=True)
|
||||
def _chunk_file(
|
||||
f: ExtractedFile, chapter_path: Path, out_course_dir: Path
|
||||
) -> tuple[list[Chunk], list[str]]:
|
||||
lg = get_run_logger()
|
||||
lg.info(f"Chunking file {f.name}")
|
||||
lg.info(f"Chapter path: {chapter_path}")
|
||||
lg.info(f"Out course dir: {out_course_dir}")
|
||||
|
||||
# Extract the Text
|
||||
file_text = Chunker._extract_text(chapter_path / f.name)
|
||||
|
||||
# Chunk the Text
|
||||
chunks = Chunker._chunk_text(file_text, f.name, out_course_dir)
|
||||
|
||||
images_dir = out_course_dir / "images"
|
||||
images_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Extract the Images
|
||||
images = Chunker._extract_images(chapter_path / f.name, images_dir)
|
||||
|
||||
return chunks, images
|
||||
|
||||
|
||||
@staticmethod
|
||||
def _extract_text(file_path: Path) -> str:
|
||||
if not file_path.suffix == ".pdf":
|
||||
return ""
|
||||
|
||||
extracted_text = ""
|
||||
|
||||
with pdfplumber.open(file_path) as pdf:
|
||||
for i in range(len(pdf.pages)):
|
||||
current_page = pdf.pages[i]
|
||||
text = current_page.extract_text() or ""
|
||||
extracted_text += text
|
||||
|
||||
return extracted_text
|
||||
|
||||
@staticmethod
|
||||
def _chunk_text(text: str, f_name: str, out_course_dir: Path) -> list[Chunk]:
|
||||
lg = get_run_logger()
|
||||
lg.info(f"Chunking text for file {f_name}")
|
||||
# split text into sentences and get tokens
|
||||
nlp_doc = Chunker.nlp(text)
|
||||
sentences = [sent.text.strip() for sent in nlp_doc.sents]
|
||||
sentence_token_counts = [len(Chunker.enc.encode(s)) for s in sentences]
|
||||
lg.info(f"Extracted {len(sentences)} sentences with token counts: {sentence_token_counts}")
|
||||
|
||||
# Buffers
|
||||
chunks: list[Chunk] = []
|
||||
current_chunk = []
|
||||
current_token_total = 0
|
||||
|
||||
chunk_id = 0
|
||||
|
||||
for s, tc in zip(sentences, sentence_token_counts): # Pair sentences and tokens
|
||||
if tc + current_token_total <= MAX_TOKENS: # Check Token limit
|
||||
# Add sentences to chunk
|
||||
current_chunk.append(s)
|
||||
current_token_total += tc
|
||||
else:
|
||||
# Flush Chunk
|
||||
chunk_text = "\n\n".join(current_chunk)
|
||||
|
||||
chunk_name = f"{f_name}_{chunk_id}"
|
||||
with open(
|
||||
out_course_dir / f"{chunk_name}.md", "w", encoding="utf-8"
|
||||
) as f:
|
||||
f.write(chunk_text)
|
||||
chunk_id += 1
|
||||
|
||||
chunks.append(
|
||||
Chunk(
|
||||
id=f"{f_name}_{chunk_id}",
|
||||
name=f"{f_name}_{chunk_id}.md",
|
||||
tokens=len(Chunker.enc.encode(chunk_text)),
|
||||
)
|
||||
)
|
||||
|
||||
# Get Overlap from Chunk
|
||||
token_ids = Chunker.enc.encode(chunk_text)
|
||||
overlap_ids = token_ids[-OVERLAP_TOKENS :]
|
||||
overlap_text = Chunker.enc.decode(overlap_ids)
|
||||
overlap_doc = Chunker.nlp(overlap_text)
|
||||
overlap_sents = [sent.text for sent in overlap_doc.sents]
|
||||
|
||||
# Start new Chunk
|
||||
current_chunk = overlap_sents + [s]
|
||||
current_token_total = sum(
|
||||
len(Chunker.enc.encode(s)) for s in current_chunk
|
||||
)
|
||||
|
||||
if current_chunk:
|
||||
chunk_text = "\n\n".join(current_chunk)
|
||||
chunk_name = f"{f_name}_{chunk_id}"
|
||||
with open(out_course_dir / f"{chunk_name}.md", "w", encoding="utf-8") as f:
|
||||
f.write(chunk_text)
|
||||
chunks.append(
|
||||
Chunk(
|
||||
id=f"{f_name}_{chunk_id}",
|
||||
name=f"{f_name}_{chunk_id}",
|
||||
tokens=len(Chunker.enc.encode(chunk_text)),
|
||||
)
|
||||
)
|
||||
lg.info(f"Created {len(chunks)} chunks for file {f_name}")
|
||||
return chunks
|
||||
|
||||
@staticmethod
|
||||
def _extract_images(file: Path, img_folder: Path) -> list[str]:
|
||||
images_list = []
|
||||
if not file.suffix == ".pdf":
|
||||
return []
|
||||
|
||||
with pymupdf.open(file) as doc:
|
||||
for i in range(len(doc)):
|
||||
images = doc.get_page_images(i)
|
||||
|
||||
for img in images:
|
||||
img_xref = img[0]
|
||||
image = doc.extract_image(img_xref)
|
||||
img_content = image["image"]
|
||||
img_ext = image["ext"]
|
||||
img_name = f"img_page{i + 1}_{img_xref}.{img_ext}"
|
||||
img_file_path = img_folder / img_name
|
||||
|
||||
with open(img_file_path, "wb") as img_file:
|
||||
img_file.write(img_content)
|
||||
images_list.append(img_name)
|
||||
return images_list
|
||||
@@ -0,0 +1,3 @@
|
||||
from .chunk_data import Chunk, ChunkedCourse, ChunkedTerm, ChunkData
|
||||
|
||||
__all__ = ["Chunk", "ChunkedCourse", "ChunkedTerm", "ChunkData"]
|
||||
@@ -0,0 +1,29 @@
|
||||
from typing import List
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Output models #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class Chunk(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
tokens: int
|
||||
|
||||
class ChunkedCourse(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
chunks: List[Chunk] = Field(default_factory=list)
|
||||
images: List[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ChunkedTerm(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
courses: List[ChunkedCourse] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ChunkData(BaseModel):
|
||||
terms: List[ChunkedTerm]
|
||||
Whitespace-only changes.
Whitespace-only changes.
+3544
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user