Initialize Monorepo

This commit is contained in:
DotNaos committed 2025-05-24 12:15:48 +02:00
commit f80792d739
106 files changed
+17076

No files matched your search

@@ -0,0 +1,21 @@
# Chunker
Extract text, chunk it, and save images from a PDF.
chunks is a List[str] of ~800-token strings (100-token overlap).
Outputs (text files and images) are written under extracted_content/<pdf_basename>/.
## Usage
```python
from chunker import Chunker
chunker = Chunker("path/to/file.pdf")
chunks = chunker.run()
Setup:
pip install -r requirements.txt
python -m spacy download xx_ent_wiki_sm
@@ -0,0 +1,40 @@
[project]
name = "librarian-chunker"
version = "0.1.0"
description = "Chunker for Librarian"
readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"pdfplumber",
"pymupdf",
"tiktoken",
"spacy",
"sentence-transformers",
"pydantic",
"prefect",
"librarian-core",
"python-pptx",
"python-docx",
]
[build-system]
requires = ["hatchling>=1.21"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src/librarian_chunker"]
[tool.hatch.metadata]
allow-direct-references = true
[tool.uv.sources]
#librarian-core = { git = "https://github.com/DotNaos/librarian-core", rev = "dev" }
[project.entry-points."librarian.workers"]
chunker = "librarian_chunker.chunker:Chunker"
# ───────── optional: dev / test extras ─────────
[project.optional-dependencies]
dev = ["ruff", "pytest", "mypy"]
@@ -0,0 +1,3 @@
from .chunker import Chunker
__all__ = ["Chunker"]
@@ -0,0 +1,217 @@
import os
import shutil
from pathlib import Path
import pdfplumber
import pymupdf
import spacy
import tiktoken
from librarian_core.utils.path_utils import get_temp_path
from librarian_core.workers.base import Worker
from librarian_extractor.models.extract_data import ExtractData, ExtractedFile
from prefect import get_run_logger, task
from prefect.cache_policies import NO_CACHE
from prefect.futures import wait
from librarian_chunker.models.chunk_data import (
Chunk,
ChunkData,
ChunkedCourse,
ChunkedTerm,
)
MAX_TOKENS = 800
OVERLAP_TOKENS = 100
class Chunker(Worker[ExtractData, ChunkData]):
input_model = ExtractData
output_model = ChunkData
async def __run__(self, payload: ExtractData) -> ChunkData: # noqa: D401
lg = get_run_logger()
lg.info("Chunker started")
working_dir = get_temp_path()
# load NLP and tokenizer
Chunker.nlp = spacy.load("xx_ent_wiki_sm")
Chunker.nlp.add_pipe("sentencizer")
Chunker.enc = tiktoken.get_encoding("cl100k_base")
# chunk parameters
Chunker.max_tokens = MAX_TOKENS
Chunker.overlap_tokens = OVERLAP_TOKENS
result = ChunkData(terms=[])
# Loading files
for term in payload.terms:
chunked_term = ChunkedTerm(id=term.id, name=term.name)
in_term_dir = self.entry / term.name
out_term_dir = working_dir / term.name
out_term_dir.mkdir(parents=True, exist_ok=True)
for course in term.courses:
chunked_course = ChunkedCourse(
id=course.id, name=course.name, chunks=[]
)
in_course_dir = in_term_dir / course.name
out_course_dir = out_term_dir / course.name
out_course_dir.mkdir(parents=True, exist_ok=True)
futs = []
# All chunks are just in the course dir, so no new dir
for chap in course.chapters:
chapter_path = in_course_dir / chap.name
for f in chap.content_files:
futs.append(
self._chunk_file.submit(f, chapter_path, out_course_dir)
)
wait(futs)
for fut in futs:
chunks, images = fut.result()
chunked_course.chunks.extend(chunks)
chunked_course.images.extend(images)
chunked_term.courses.append(chunked_course)
# Add the chunked term to the result
result.terms.append(chunked_term)
self.stage(out_term_dir)
return result
@staticmethod
@task(log_prints=True)
def _chunk_file(
f: ExtractedFile, chapter_path: Path, out_course_dir: Path
) -> tuple[list[Chunk], list[str]]:
lg = get_run_logger()
lg.info(f"Chunking file {f.name}")
lg.info(f"Chapter path: {chapter_path}")
lg.info(f"Out course dir: {out_course_dir}")
# Extract the Text
file_text = Chunker._extract_text(chapter_path / f.name)
# Chunk the Text
chunks = Chunker._chunk_text(file_text, f.name, out_course_dir)
images_dir = out_course_dir / "images"
images_dir.mkdir(parents=True, exist_ok=True)
# Extract the Images
images = Chunker._extract_images(chapter_path / f.name, images_dir)
return chunks, images
@staticmethod
def _extract_text(file_path: Path) -> str:
if not file_path.suffix == ".pdf":
return ""
extracted_text = ""
with pdfplumber.open(file_path) as pdf:
for i in range(len(pdf.pages)):
current_page = pdf.pages[i]
text = current_page.extract_text() or ""
extracted_text += text
return extracted_text
@staticmethod
def _chunk_text(text: str, f_name: str, out_course_dir: Path) -> list[Chunk]:
lg = get_run_logger()
lg.info(f"Chunking text for file {f_name}")
# split text into sentences and get tokens
nlp_doc = Chunker.nlp(text)
sentences = [sent.text.strip() for sent in nlp_doc.sents]
sentence_token_counts = [len(Chunker.enc.encode(s)) for s in sentences]
lg.info(f"Extracted {len(sentences)} sentences with token counts: {sentence_token_counts}")
# Buffers
chunks: list[Chunk] = []
current_chunk = []
current_token_total = 0
chunk_id = 0
for s, tc in zip(sentences, sentence_token_counts): # Pair sentences and tokens
if tc + current_token_total <= MAX_TOKENS: # Check Token limit
# Add sentences to chunk
current_chunk.append(s)
current_token_total += tc
else:
# Flush Chunk
chunk_text = "\n\n".join(current_chunk)
chunk_name = f"{f_name}_{chunk_id}"
with open(
out_course_dir / f"{chunk_name}.md", "w", encoding="utf-8"
) as f:
f.write(chunk_text)
chunk_id += 1
chunks.append(
Chunk(
id=f"{f_name}_{chunk_id}",
name=f"{f_name}_{chunk_id}.md",
tokens=len(Chunker.enc.encode(chunk_text)),
)
)
# Get Overlap from Chunk
token_ids = Chunker.enc.encode(chunk_text)
overlap_ids = token_ids[-OVERLAP_TOKENS :]
overlap_text = Chunker.enc.decode(overlap_ids)
overlap_doc = Chunker.nlp(overlap_text)
overlap_sents = [sent.text for sent in overlap_doc.sents]
# Start new Chunk
current_chunk = overlap_sents + [s]
current_token_total = sum(
len(Chunker.enc.encode(s)) for s in current_chunk
)
if current_chunk:
chunk_text = "\n\n".join(current_chunk)
chunk_name = f"{f_name}_{chunk_id}"
with open(out_course_dir / f"{chunk_name}.md", "w", encoding="utf-8") as f:
f.write(chunk_text)
chunks.append(
Chunk(
id=f"{f_name}_{chunk_id}",
name=f"{f_name}_{chunk_id}",
tokens=len(Chunker.enc.encode(chunk_text)),
)
)
lg.info(f"Created {len(chunks)} chunks for file {f_name}")
return chunks
@staticmethod
def _extract_images(file: Path, img_folder: Path) -> list[str]:
images_list = []
if not file.suffix == ".pdf":
return []
with pymupdf.open(file) as doc:
for i in range(len(doc)):
images = doc.get_page_images(i)
for img in images:
img_xref = img[0]
image = doc.extract_image(img_xref)
img_content = image["image"]
img_ext = image["ext"]
img_name = f"img_page{i + 1}_{img_xref}.{img_ext}"
img_file_path = img_folder / img_name
with open(img_file_path, "wb") as img_file:
img_file.write(img_content)
images_list.append(img_name)
return images_list
@@ -0,0 +1,3 @@
from .chunk_data import Chunk, ChunkedCourse, ChunkedTerm, ChunkData
__all__ = ["Chunk", "ChunkedCourse", "ChunkedTerm", "ChunkData"]
@@ -0,0 +1,29 @@
from typing import List
from pydantic import BaseModel, Field
# --------------------------------------------------------------------------- #
# Output models #
# --------------------------------------------------------------------------- #
class Chunk(BaseModel):
id: str
name: str
tokens: int
class ChunkedCourse(BaseModel):
id: str
name: str
chunks: List[Chunk] = Field(default_factory=list)
images: List[str] = Field(default_factory=list)
class ChunkedTerm(BaseModel):
id: str
name: str
courses: List[ChunkedCourse] = Field(default_factory=list)
class ChunkData(BaseModel):
terms: List[ChunkedTerm]
Whitespace-only changes.
Whitespace-only changes.
File diff suppressed because it is too large. Load diff