Update Readme and cleanup
This commit is contained in:
1 parent
52d087ce90
commit
b6789cb5f7
34 files changed
+104
-471
No files matched your search
+2
-21
@@ -32,9 +32,6 @@ from librarian_extractor.models.extract_data import (
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# helpers #
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _clean_json(txt: str) -> str:
|
||||
txt = txt.strip()
|
||||
if txt.startswith("```"):
|
||||
@@ -51,7 +48,7 @@ def _safe_json_load(txt: str) -> dict:
|
||||
|
||||
|
||||
def _merge_with_original(src: ExtractedCourse, patch: dict, lg) -> ExtractedCourse:
|
||||
"""Return *patch* merged with *src* so every id is preserved."""
|
||||
"""Merges LLM patch with source, preserving IDs."""
|
||||
try:
|
||||
tgt = ExtractedCourse.model_validate(patch)
|
||||
except ValidationError as err:
|
||||
@@ -73,9 +70,6 @@ def _merge_with_original(src: ExtractedCourse, patch: dict, lg) -> ExtractedCour
|
||||
return tgt
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# OpenAI call – Prefect task #
|
||||
# --------------------------------------------------------------------------- #
|
||||
@task(
|
||||
name="sanitize_course_json",
|
||||
retries=2,
|
||||
@@ -100,9 +94,6 @@ def sanitize_course_json(course_json: str, model: str, temperature: float) -> di
|
||||
return _safe_json_load(rsp.choices[0].message.content or "{}")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Worker #
|
||||
# --------------------------------------------------------------------------- #
|
||||
class AISanitizer(Worker[ExtractData, ExtractData]):
|
||||
input_model = ExtractData
|
||||
output_model = ExtractData
|
||||
@@ -112,14 +103,12 @@ class AISanitizer(Worker[ExtractData, ExtractData]):
|
||||
self.model_name = model_name or os.getenv("OPENAI_MODEL", "gpt-4o-mini")
|
||||
self.temperature = temperature
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
def __run__(self, data: ExtractData) -> ExtractData:
|
||||
lg = get_run_logger()
|
||||
|
||||
futures: List[PrefectFuture] = []
|
||||
originals: List[ExtractedCourse] = []
|
||||
|
||||
# 1) submit all courses to the LLM
|
||||
for term in data.terms:
|
||||
for course in term.courses:
|
||||
futures.append(
|
||||
@@ -133,7 +122,6 @@ class AISanitizer(Worker[ExtractData, ExtractData]):
|
||||
|
||||
wait(futures)
|
||||
|
||||
# 2) build new graph with merged results
|
||||
terms_out: List[ExtractedTerm] = []
|
||||
idx = 0
|
||||
for term in data.terms:
|
||||
@@ -149,16 +137,12 @@ class AISanitizer(Worker[ExtractData, ExtractData]):
|
||||
|
||||
renamed = ExtractData(terms=terms_out)
|
||||
|
||||
# 3) stage files with their new names
|
||||
self._export_with_new_names(data, renamed, lg)
|
||||
|
||||
return renamed
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# staging helpers #
|
||||
# ------------------------------------------------------------------ #
|
||||
def _stage_or_warn(self, src: Path, dst: Path, lg):
|
||||
"""Copy *src* → *dst* (via self.stage). Warn if src missing."""
|
||||
"""Stages file, warns if source missing."""
|
||||
if not src.exists():
|
||||
lg.warning("Source missing – skipped %s", src)
|
||||
return
|
||||
@@ -175,7 +159,6 @@ class AISanitizer(Worker[ExtractData, ExtractData]):
|
||||
|
||||
for term_old, term_new in zip(original.terms, renamed.terms):
|
||||
for course_old, course_new in zip(term_old.courses, term_new.courses):
|
||||
# ---------- content files (per chapter) -----------------
|
||||
for chap_old, chap_new in zip(course_old.chapters, course_new.chapters):
|
||||
n = min(len(chap_old.content_files), len(chap_new.content_files))
|
||||
for i in range(n):
|
||||
@@ -196,7 +179,6 @@ class AISanitizer(Worker[ExtractData, ExtractData]):
|
||||
)
|
||||
self._stage_or_warn(src, dst, lg)
|
||||
|
||||
# ---------- media files (course-level “media”) ----------
|
||||
src_media_dir = (
|
||||
entry / term_old.name / course_old.name / "media"
|
||||
) # <─ fixed!
|
||||
@@ -204,7 +186,6 @@ class AISanitizer(Worker[ExtractData, ExtractData]):
|
||||
if not src_media_dir.is_dir():
|
||||
continue
|
||||
|
||||
# build a flat list of (old, new) media filenames
|
||||
media_pairs: List[tuple[ExtractedFile, ExtractedFile]] = []
|
||||
for ch_o, ch_n in zip(course_old.chapters, course_new.chapters):
|
||||
media_pairs.extend(zip(ch_o.media_files, ch_n.media_files))
|
||||
|
||||
@@ -2,9 +2,6 @@
|
||||
Shared lists and prompts
|
||||
"""
|
||||
|
||||
# -------------------------------------------------------------------- #
|
||||
# file selection – keep only real documents we can show / convert #
|
||||
# -------------------------------------------------------------------- #
|
||||
CONTENT_FILE_EXTENSIONS = [
|
||||
"*.pdf",
|
||||
"*.doc",
|
||||
@@ -26,9 +23,6 @@ MEDIA_FILE_EXTENSIONS = [
|
||||
"*.mp3",
|
||||
]
|
||||
|
||||
# -------------------------------------------------------------------- #
|
||||
# naming rules #
|
||||
# -------------------------------------------------------------------- #
|
||||
SANITIZE_REGEX = {
|
||||
"base": [r"\s*\(\d+\)$"],
|
||||
"course": [
|
||||
|
||||
+1
-14
@@ -51,9 +51,6 @@ ALL_EXTS = CONTENT_EXTS | MEDIA_EXTS
|
||||
_id_rx = re.compile(r"\.(\d{4,})[./]") # 1172180 from “..._.1172180/index.html”
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# helpers #
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _hash_id(fname: str) -> str:
|
||||
return hashlib.sha1(fname.encode()).hexdigest()[:10]
|
||||
|
||||
@@ -80,17 +77,14 @@ def _best_payload(node: Path) -> Path | None: # noqa: C901
|
||||
• File_xxx/dir → search inside /content or dir itself
|
||||
• File_xxx/index.html stub → parse to find linked file
|
||||
"""
|
||||
# 1) immediate hit
|
||||
if node.is_file() and node.suffix.lower() in ALL_EXTS:
|
||||
return node
|
||||
|
||||
# 2) if html stub try to parse inner link
|
||||
if node.is_file() and node.suffix.lower() in {".html", ".htm"}:
|
||||
hinted = _html_stub_target(node)
|
||||
if hinted:
|
||||
return _best_payload(hinted) # recurse
|
||||
return _best_payload(hinted)
|
||||
|
||||
# 3) directories to search
|
||||
roots: list[Path] = []
|
||||
if node.is_dir():
|
||||
roots.append(node)
|
||||
@@ -120,9 +114,6 @@ def task_(**kw):
|
||||
return task(**kw)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Worker #
|
||||
# --------------------------------------------------------------------------- #
|
||||
class Extractor(Worker[DownloadData, ExtractData]):
|
||||
input_model = DownloadData
|
||||
output_model = ExtractData
|
||||
@@ -158,7 +149,6 @@ class Extractor(Worker[DownloadData, ExtractData]):
|
||||
lg.info("Extractor finished – %d terms", len(result.terms))
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
@staticmethod
|
||||
@task_()
|
||||
def _extract_course( # noqa: C901
|
||||
@@ -238,9 +228,6 @@ class Extractor(Worker[DownloadData, ExtractData]):
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# internal helpers #
|
||||
# ------------------------------------------------------------------ #
|
||||
@staticmethod
|
||||
def _copy_all(
|
||||
root: Path, dst_root: Path, c_meta: ExtractedCourse, media_dir: Path, lg
|
||||
|
||||
+4
-4
@@ -5,24 +5,24 @@ from pydantic import BaseModel, Field
|
||||
|
||||
class ExtractedFile(BaseModel):
|
||||
id: str
|
||||
name: str # Name of the file, relative to ExtractedChapter.name
|
||||
name: str
|
||||
|
||||
|
||||
class ExtractedChapter(BaseModel):
|
||||
name: str # Name of the chapter directory, relative to ExtractedCourse.name
|
||||
name: str
|
||||
content_files: List[ExtractedFile] = Field(default_factory=list)
|
||||
media_files: List[ExtractedFile] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ExtractedCourse(BaseModel):
|
||||
id: str
|
||||
name: str # Name of the course directory, relative to ExtractedTerm.name
|
||||
name: str
|
||||
chapters: List[ExtractedChapter] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ExtractedTerm(BaseModel):
|
||||
id: str
|
||||
name: str # Name of the term directory, relative to ExtractMeta.dir
|
||||
name: str
|
||||
courses: List[ExtractedCourse] = Field(default_factory=list)
|
||||
|
||||
|
||||
|
||||
@@ -1,7 +1,3 @@
|
||||
# -------------------------------------------------------------------- #
|
||||
# LLM prompts #
|
||||
# -------------------------------------------------------------------- #
|
||||
|
||||
PROMPT_COURSE = """
|
||||
General naming rules
|
||||
====================
|
||||
|
||||
Reference in new issue
Block a user