Update Readme and cleanup

This commit is contained in:
Oliver Schütz authored and GitHub committed 2025-06-01 17:32:50 +02:00
1 parent 52d087ce90
commit b6789cb5f7
34 files changed
+104 -471

No files matched your search

@@ -34,18 +34,15 @@ class Chunker(Worker[ExtractData, ChunkData]):
working_dir = get_temp_path()
# load NLP and tokenizer
Chunker.nlp = spacy.load("xx_ent_wiki_sm")
Chunker.nlp.add_pipe("sentencizer")
Chunker.enc = tiktoken.get_encoding("cl100k_base")
# chunk parameters
Chunker.max_tokens = MAX_TOKENS
Chunker.overlap_tokens = OVERLAP_TOKENS
result = ChunkData(terms=[])
# Loading files
for term in payload.terms:
chunked_term = ChunkedTerm(id=term.id, name=term.name)
in_term_dir = self.entry / term.name
@@ -79,7 +76,6 @@ class Chunker(Worker[ExtractData, ChunkData]):
chunked_term.courses.append(chunked_course)
# Add the chunked term to the result
result.terms.append(chunked_term)
self.stage(out_term_dir)
@@ -94,16 +90,11 @@ class Chunker(Worker[ExtractData, ChunkData]):
lg.info(f"Chapter path: {chapter_path}")
lg.info(f"Out course dir: {out_course_dir}")
# Extract the Text
file_text = Chunker._extract_text(chapter_path / f.name)
# Chunk the Text
chunks = Chunker._chunk_text(file_text, f.name, out_course_dir)
images_dir = out_course_dir / "images"
images_dir.mkdir(parents=True, exist_ok=True)
# Extract the Images
images = Chunker._extract_images(chapter_path / f.name, images_dir)
return chunks, images
@@ -128,28 +119,22 @@ class Chunker(Worker[ExtractData, ChunkData]):
def _chunk_text(text: str, f_name: str, out_course_dir: Path) -> list[Chunk]:
lg = get_run_logger()
lg.info(f"Chunking text for file {f_name}")
# split text into sentences and get tokens
nlp_doc = Chunker.nlp(text)
sentences = [sent.text.strip() for sent in nlp_doc.sents]
sentence_token_counts = [len(Chunker.enc.encode(s)) for s in sentences]
lg.info(f"Extracted {len(sentences)} sentences with token counts: {sentence_token_counts}")
# Buffers
chunks: list[Chunk] = []
current_chunk = []
current_token_total = 0
chunk_id = 0
for s, tc in zip(sentences, sentence_token_counts): # Pair sentences and tokens
if tc + current_token_total <= MAX_TOKENS: # Check Token limit
# Add sentences to chunk
for s, tc in zip(sentences, sentence_token_counts):
if tc + current_token_total <= MAX_TOKENS:
current_chunk.append(s)
current_token_total += tc
else:
# Flush Chunk
chunk_text = "\n\n".join(current_chunk)
chunk_name = f"{f_name}_{chunk_id}"
with open(
out_course_dir / f"{chunk_name}.md", "w", encoding="utf-8"
@@ -165,14 +150,12 @@ class Chunker(Worker[ExtractData, ChunkData]):
)
)
# Get Overlap from Chunk
token_ids = Chunker.enc.encode(chunk_text)
overlap_ids = token_ids[-OVERLAP_TOKENS :]
overlap_text = Chunker.enc.decode(overlap_ids)
overlap_doc = Chunker.nlp(overlap_text)
overlap_sents = [sent.text for sent in overlap_doc.sents]
# Start new Chunk
current_chunk = overlap_sents + [s]
current_token_total = sum(
len(Chunker.enc.encode(s)) for s in current_chunk
@@ -2,10 +2,6 @@ from typing import List
from pydantic import BaseModel, Field
# --------------------------------------------------------------------------- #
# Output models #
# --------------------------------------------------------------------------- #
class Chunk(BaseModel):
id: str