Update Readme and cleanup
This commit is contained in:
1 parent
52d087ce90
commit
b6789cb5f7
34 files changed
+104
-471
No files matched your search
@@ -34,18 +34,15 @@ class Chunker(Worker[ExtractData, ChunkData]):
|
||||
|
||||
working_dir = get_temp_path()
|
||||
|
||||
# load NLP and tokenizer
|
||||
Chunker.nlp = spacy.load("xx_ent_wiki_sm")
|
||||
Chunker.nlp.add_pipe("sentencizer")
|
||||
Chunker.enc = tiktoken.get_encoding("cl100k_base")
|
||||
|
||||
# chunk parameters
|
||||
Chunker.max_tokens = MAX_TOKENS
|
||||
Chunker.overlap_tokens = OVERLAP_TOKENS
|
||||
|
||||
result = ChunkData(terms=[])
|
||||
|
||||
# Loading files
|
||||
for term in payload.terms:
|
||||
chunked_term = ChunkedTerm(id=term.id, name=term.name)
|
||||
in_term_dir = self.entry / term.name
|
||||
@@ -79,7 +76,6 @@ class Chunker(Worker[ExtractData, ChunkData]):
|
||||
|
||||
chunked_term.courses.append(chunked_course)
|
||||
|
||||
# Add the chunked term to the result
|
||||
result.terms.append(chunked_term)
|
||||
self.stage(out_term_dir)
|
||||
|
||||
@@ -94,16 +90,11 @@ class Chunker(Worker[ExtractData, ChunkData]):
|
||||
lg.info(f"Chapter path: {chapter_path}")
|
||||
lg.info(f"Out course dir: {out_course_dir}")
|
||||
|
||||
# Extract the Text
|
||||
file_text = Chunker._extract_text(chapter_path / f.name)
|
||||
|
||||
# Chunk the Text
|
||||
chunks = Chunker._chunk_text(file_text, f.name, out_course_dir)
|
||||
|
||||
images_dir = out_course_dir / "images"
|
||||
images_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Extract the Images
|
||||
images = Chunker._extract_images(chapter_path / f.name, images_dir)
|
||||
|
||||
return chunks, images
|
||||
@@ -128,28 +119,22 @@ class Chunker(Worker[ExtractData, ChunkData]):
|
||||
def _chunk_text(text: str, f_name: str, out_course_dir: Path) -> list[Chunk]:
|
||||
lg = get_run_logger()
|
||||
lg.info(f"Chunking text for file {f_name}")
|
||||
# split text into sentences and get tokens
|
||||
nlp_doc = Chunker.nlp(text)
|
||||
sentences = [sent.text.strip() for sent in nlp_doc.sents]
|
||||
sentence_token_counts = [len(Chunker.enc.encode(s)) for s in sentences]
|
||||
lg.info(f"Extracted {len(sentences)} sentences with token counts: {sentence_token_counts}")
|
||||
|
||||
# Buffers
|
||||
chunks: list[Chunk] = []
|
||||
current_chunk = []
|
||||
current_token_total = 0
|
||||
|
||||
chunk_id = 0
|
||||
|
||||
for s, tc in zip(sentences, sentence_token_counts): # Pair sentences and tokens
|
||||
if tc + current_token_total <= MAX_TOKENS: # Check Token limit
|
||||
# Add sentences to chunk
|
||||
for s, tc in zip(sentences, sentence_token_counts):
|
||||
if tc + current_token_total <= MAX_TOKENS:
|
||||
current_chunk.append(s)
|
||||
current_token_total += tc
|
||||
else:
|
||||
# Flush Chunk
|
||||
chunk_text = "\n\n".join(current_chunk)
|
||||
|
||||
chunk_name = f"{f_name}_{chunk_id}"
|
||||
with open(
|
||||
out_course_dir / f"{chunk_name}.md", "w", encoding="utf-8"
|
||||
@@ -165,14 +150,12 @@ class Chunker(Worker[ExtractData, ChunkData]):
|
||||
)
|
||||
)
|
||||
|
||||
# Get Overlap from Chunk
|
||||
token_ids = Chunker.enc.encode(chunk_text)
|
||||
overlap_ids = token_ids[-OVERLAP_TOKENS :]
|
||||
overlap_text = Chunker.enc.decode(overlap_ids)
|
||||
overlap_doc = Chunker.nlp(overlap_text)
|
||||
overlap_sents = [sent.text for sent in overlap_doc.sents]
|
||||
|
||||
# Start new Chunk
|
||||
current_chunk = overlap_sents + [s]
|
||||
current_token_total = sum(
|
||||
len(Chunker.enc.encode(s)) for s in current_chunk
|
||||
|
||||
@@ -2,10 +2,6 @@ from typing import List
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Output models #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
class Chunk(BaseModel):
|
||||
id: str
|
||||
|
||||
Reference in new issue
Block a user