Update Readme and cleanup

This commit is contained in:
Oliver Schütz authored and GitHub committed 2025-06-01 17:32:50 +02:00
1 parent 52d087ce90
commit b6789cb5f7
34 files changed
+104 -471

No files matched your search

@@ -32,9 +32,6 @@ from librarian_extractor.models.extract_data import (
)
# --------------------------------------------------------------------------- #
# helpers #
# --------------------------------------------------------------------------- #
def _clean_json(txt: str) -> str:
txt = txt.strip()
if txt.startswith("```"):
@@ -51,7 +48,7 @@ def _safe_json_load(txt: str) -> dict:
def _merge_with_original(src: ExtractedCourse, patch: dict, lg) -> ExtractedCourse:
"""Return *patch* merged with *src* so every id is preserved."""
"""Merges LLM patch with source, preserving IDs."""
try:
tgt = ExtractedCourse.model_validate(patch)
except ValidationError as err:
@@ -73,9 +70,6 @@ def _merge_with_original(src: ExtractedCourse, patch: dict, lg) -> ExtractedCour
return tgt
# --------------------------------------------------------------------------- #
# OpenAI call – Prefect task #
# --------------------------------------------------------------------------- #
@task(
name="sanitize_course_json",
retries=2,
@@ -100,9 +94,6 @@ def sanitize_course_json(course_json: str, model: str, temperature: float) -> di
return _safe_json_load(rsp.choices[0].message.content or "{}")
# --------------------------------------------------------------------------- #
# Worker #
# --------------------------------------------------------------------------- #
class AISanitizer(Worker[ExtractData, ExtractData]):
input_model = ExtractData
output_model = ExtractData
@@ -112,14 +103,12 @@ class AISanitizer(Worker[ExtractData, ExtractData]):
self.model_name = model_name or os.getenv("OPENAI_MODEL", "gpt-4o-mini")
self.temperature = temperature
# ------------------------------------------------------------------ #
def __run__(self, data: ExtractData) -> ExtractData:
lg = get_run_logger()
futures: List[PrefectFuture] = []
originals: List[ExtractedCourse] = []
# 1) submit all courses to the LLM
for term in data.terms:
for course in term.courses:
futures.append(
@@ -133,7 +122,6 @@ class AISanitizer(Worker[ExtractData, ExtractData]):
wait(futures)
# 2) build new graph with merged results
terms_out: List[ExtractedTerm] = []
idx = 0
for term in data.terms:
@@ -149,16 +137,12 @@ class AISanitizer(Worker[ExtractData, ExtractData]):
renamed = ExtractData(terms=terms_out)
# 3) stage files with their new names
self._export_with_new_names(data, renamed, lg)
return renamed
# ------------------------------------------------------------------ #
# staging helpers #
# ------------------------------------------------------------------ #
def _stage_or_warn(self, src: Path, dst: Path, lg):
"""Copy *src* → *dst* (via self.stage). Warn if src missing."""
"""Stages file, warns if source missing."""
if not src.exists():
lg.warning("Source missing – skipped %s", src)
return
@@ -175,7 +159,6 @@ class AISanitizer(Worker[ExtractData, ExtractData]):
for term_old, term_new in zip(original.terms, renamed.terms):
for course_old, course_new in zip(term_old.courses, term_new.courses):
# ---------- content files (per chapter) -----------------
for chap_old, chap_new in zip(course_old.chapters, course_new.chapters):
n = min(len(chap_old.content_files), len(chap_new.content_files))
for i in range(n):
@@ -196,7 +179,6 @@ class AISanitizer(Worker[ExtractData, ExtractData]):
)
self._stage_or_warn(src, dst, lg)
# ---------- media files (course-level “media”) ----------
src_media_dir = (
entry / term_old.name / course_old.name / "media"
) # <─ fixed!
@@ -204,7 +186,6 @@ class AISanitizer(Worker[ExtractData, ExtractData]):
if not src_media_dir.is_dir():
continue
# build a flat list of (old, new) media filenames
media_pairs: List[tuple[ExtractedFile, ExtractedFile]] = []
for ch_o, ch_n in zip(course_old.chapters, course_new.chapters):
media_pairs.extend(zip(ch_o.media_files, ch_n.media_files))
@@ -2,9 +2,6 @@
Shared lists and prompts
"""
# -------------------------------------------------------------------- #
# file selection – keep only real documents we can show / convert #
# -------------------------------------------------------------------- #
CONTENT_FILE_EXTENSIONS = [
"*.pdf",
"*.doc",
@@ -26,9 +23,6 @@ MEDIA_FILE_EXTENSIONS = [
"*.mp3",
]
# -------------------------------------------------------------------- #
# naming rules #
# -------------------------------------------------------------------- #
SANITIZE_REGEX = {
"base": [r"\s*\(\d+\)$"],
"course": [
@@ -51,9 +51,6 @@ ALL_EXTS = CONTENT_EXTS | MEDIA_EXTS
_id_rx = re.compile(r"\.(\d{4,})[./]") # 1172180 from “..._.1172180/index.html”
# --------------------------------------------------------------------------- #
# helpers #
# --------------------------------------------------------------------------- #
def _hash_id(fname: str) -> str:
return hashlib.sha1(fname.encode()).hexdigest()[:10]
@@ -80,17 +77,14 @@ def _best_payload(node: Path) -> Path | None: # noqa: C901
• File_xxx/dir → search inside /content or dir itself
• File_xxx/index.html stub → parse to find linked file
"""
# 1) immediate hit
if node.is_file() and node.suffix.lower() in ALL_EXTS:
return node
# 2) if html stub try to parse inner link
if node.is_file() and node.suffix.lower() in {".html", ".htm"}:
hinted = _html_stub_target(node)
if hinted:
return _best_payload(hinted) # recurse
return _best_payload(hinted)
# 3) directories to search
roots: list[Path] = []
if node.is_dir():
roots.append(node)
@@ -120,9 +114,6 @@ def task_(**kw):
return task(**kw)
# --------------------------------------------------------------------------- #
# Worker #
# --------------------------------------------------------------------------- #
class Extractor(Worker[DownloadData, ExtractData]):
input_model = DownloadData
output_model = ExtractData
@@ -158,7 +149,6 @@ class Extractor(Worker[DownloadData, ExtractData]):
lg.info("Extractor finished – %d terms", len(result.terms))
return result
# ------------------------------------------------------------------ #
@staticmethod
@task_()
def _extract_course( # noqa: C901
@@ -238,9 +228,6 @@ class Extractor(Worker[DownloadData, ExtractData]):
finally:
shutil.rmtree(tmp, ignore_errors=True)
# ------------------------------------------------------------------ #
# internal helpers #
# ------------------------------------------------------------------ #
@staticmethod
def _copy_all(
root: Path, dst_root: Path, c_meta: ExtractedCourse, media_dir: Path, lg
@@ -5,24 +5,24 @@ from pydantic import BaseModel, Field
class ExtractedFile(BaseModel):
id: str
name: str # Name of the file, relative to ExtractedChapter.name
name: str
class ExtractedChapter(BaseModel):
name: str # Name of the chapter directory, relative to ExtractedCourse.name
name: str
content_files: List[ExtractedFile] = Field(default_factory=list)
media_files: List[ExtractedFile] = Field(default_factory=list)
class ExtractedCourse(BaseModel):
id: str
name: str # Name of the course directory, relative to ExtractedTerm.name
name: str
chapters: List[ExtractedChapter] = Field(default_factory=list)
class ExtractedTerm(BaseModel):
id: str
name: str # Name of the term directory, relative to ExtractMeta.dir
name: str
courses: List[ExtractedCourse] = Field(default_factory=list)
@@ -1,7 +1,3 @@
# -------------------------------------------------------------------- #
# LLM prompts #
# -------------------------------------------------------------------- #
PROMPT_COURSE = """
General naming rules
====================