Initialize Monorepo
This commit is contained in:
commit
f80792d739
106 files changed
+17076
No files matched your search
Whitespace-only changes.
@@ -0,0 +1,40 @@
|
||||
[project]
|
||||
name = "librarian-extractor"
|
||||
version = "0.1.0"
|
||||
description = "Librarian extractor plugin"
|
||||
readme = "README.md"
|
||||
authors = [
|
||||
{ name = "DotNaos", email = "schuetzoliver00@gmail.com" },
|
||||
]
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"librarian-core",
|
||||
"importlib_metadata; python_version<'3.10'",
|
||||
"ollama>=0.4.8",
|
||||
"parsel>=1.10.0",
|
||||
"prefect>=3.4.1",
|
||||
"openai>=1.78.1",
|
||||
]
|
||||
|
||||
#[tool.uv.sources]
|
||||
#librarian-core = { git = "https://github.com/DotNaos/librarian-core", rev = "main" }
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling>=1.21"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/librarian_extractor/"]
|
||||
|
||||
[tool.hatch.metadata]
|
||||
allow-direct-references = true
|
||||
|
||||
|
||||
# ───────── optional: dev / test extras ─────────
|
||||
[project.optional-dependencies]
|
||||
dev = ["ruff", "pytest", "mypy"]
|
||||
|
||||
[project.entry-points."librarian.workers"]
|
||||
extractor = "librarian_extractor.extractor:Extractor"
|
||||
ai_sanitizer = "librarian_extractor.ai_sanitizer:AISanitizer"
|
||||
@@ -0,0 +1,4 @@
|
||||
from librarian_extractor.ai_sanitizer.ai_sanitizer import AISanitizer
|
||||
from librarian_extractor.extractor.extractor import Extractor
|
||||
|
||||
__all__ = ["Extractor", "AISanitizer"]
|
||||
@@ -0,0 +1,3 @@
|
||||
from librarian_extractor.ai_sanitizer.ai_sanitizer import AISanitizer
|
||||
|
||||
__all__ = ["AISanitizer"]
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
"""
|
||||
AI-powered sanitizer
|
||||
====================
|
||||
• in : ExtractData (tree from Extractor)
|
||||
• out : ExtractData (same graph but with prettier names)
|
||||
|
||||
Changes vs. previous revision
|
||||
-----------------------------
|
||||
✓ Media files resolved at course-level `media/` folder
|
||||
✓ Missing sources are warned, not raised
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import List
|
||||
|
||||
import openai
|
||||
from prefect import get_run_logger, task
|
||||
from prefect.futures import PrefectFuture, wait
|
||||
from pydantic import ValidationError
|
||||
|
||||
from librarian_core.workers.base import Worker
|
||||
from librarian_extractor.prompts import PROMPT_COURSE
|
||||
from librarian_extractor.models.extract_data import (
|
||||
ExtractData,
|
||||
ExtractedCourse,
|
||||
ExtractedFile,
|
||||
ExtractedTerm,
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# helpers #
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _clean_json(txt: str) -> str:
|
||||
txt = txt.strip()
|
||||
if txt.startswith("```"):
|
||||
txt = txt.lstrip("`")
|
||||
if "\n" in txt:
|
||||
txt = txt.split("\n", 1)[1]
|
||||
if txt.rstrip().endswith("```"):
|
||||
txt = txt.rstrip()[:-3]
|
||||
return txt.strip()
|
||||
|
||||
|
||||
def _safe_json_load(txt: str) -> dict:
|
||||
return json.loads(_clean_json(txt))
|
||||
|
||||
|
||||
def _merge_with_original(src: ExtractedCourse, patch: dict, lg) -> ExtractedCourse:
|
||||
"""Return *patch* merged with *src* so every id is preserved."""
|
||||
try:
|
||||
tgt = ExtractedCourse.model_validate(patch)
|
||||
except ValidationError as err:
|
||||
lg.warning("LLM payload invalid – keeping original (%s)", err)
|
||||
return src
|
||||
|
||||
if not tgt.id:
|
||||
tgt.id = src.id
|
||||
|
||||
for ch_src, ch_tgt in zip(src.chapters, tgt.chapters):
|
||||
if not ch_tgt.name:
|
||||
ch_tgt.name = ch_src.name
|
||||
for f_src, f_tgt in zip(ch_src.content_files, ch_tgt.content_files):
|
||||
if not f_tgt.id:
|
||||
f_tgt.id = f_src.id
|
||||
for f_src, f_tgt in zip(ch_src.media_files, ch_tgt.media_files):
|
||||
if not f_tgt.id:
|
||||
f_tgt.id = f_src.id
|
||||
return tgt
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# OpenAI call – Prefect task #
|
||||
# --------------------------------------------------------------------------- #
|
||||
@task(
|
||||
name="sanitize_course_json",
|
||||
retries=2,
|
||||
retry_delay_seconds=5,
|
||||
log_prints=True,
|
||||
)
|
||||
def sanitize_course_json(course_json: str, model: str, temperature: float) -> dict:
|
||||
rsp = openai.chat.completions.create(
|
||||
model=model,
|
||||
temperature=temperature,
|
||||
messages=[
|
||||
{"role": "system", "content": PROMPT_COURSE},
|
||||
{"role": "user", "content": course_json},
|
||||
],
|
||||
)
|
||||
usage = rsp.usage
|
||||
get_run_logger().info(
|
||||
"LLM tokens – prompt: %s, completion: %s",
|
||||
usage.prompt_tokens,
|
||||
usage.completion_tokens,
|
||||
)
|
||||
return _safe_json_load(rsp.choices[0].message.content or "{}")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Worker #
|
||||
# --------------------------------------------------------------------------- #
|
||||
class AISanitizer(Worker[ExtractData, ExtractData]):
|
||||
input_model = ExtractData
|
||||
output_model = ExtractData
|
||||
|
||||
def __init__(self, model_name: str | None = None, temperature: float = 0.0):
|
||||
super().__init__()
|
||||
self.model_name = model_name or os.getenv("OPENAI_MODEL", "gpt-4o-mini")
|
||||
self.temperature = temperature
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
def __run__(self, data: ExtractData) -> ExtractData:
|
||||
lg = get_run_logger()
|
||||
|
||||
futures: List[PrefectFuture] = []
|
||||
originals: List[ExtractedCourse] = []
|
||||
|
||||
# 1) submit all courses to the LLM
|
||||
for term in data.terms:
|
||||
for course in term.courses:
|
||||
futures.append(
|
||||
sanitize_course_json.submit(
|
||||
json.dumps(course.model_dump(), ensure_ascii=False),
|
||||
self.model_name,
|
||||
self.temperature,
|
||||
)
|
||||
)
|
||||
originals.append(course)
|
||||
|
||||
wait(futures)
|
||||
|
||||
# 2) build new graph with merged results
|
||||
terms_out: List[ExtractedTerm] = []
|
||||
idx = 0
|
||||
for term in data.terms:
|
||||
new_courses: List[ExtractedCourse] = []
|
||||
for _ in term.courses:
|
||||
clean_dict = futures[idx].result()
|
||||
merged = _merge_with_original(originals[idx], clean_dict, lg)
|
||||
new_courses.append(merged)
|
||||
idx += 1
|
||||
terms_out.append(
|
||||
ExtractedTerm(id=term.id, name=term.name, courses=new_courses)
|
||||
)
|
||||
|
||||
renamed = ExtractData(terms=terms_out)
|
||||
|
||||
# 3) stage files with their new names
|
||||
self._export_with_new_names(data, renamed, lg)
|
||||
|
||||
return renamed
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# staging helpers #
|
||||
# ------------------------------------------------------------------ #
|
||||
def _stage_or_warn(self, src: Path, dst: Path, lg):
|
||||
"""Copy *src* → *dst* (via self.stage). Warn if src missing."""
|
||||
if not src.exists():
|
||||
lg.warning("Source missing – skipped %s", src)
|
||||
return
|
||||
self.stage(src, new_name=str(dst), sanitize=False)
|
||||
lg.debug("Stage %s → %s", src.name, dst)
|
||||
|
||||
def _export_with_new_names(
|
||||
self,
|
||||
original: ExtractData,
|
||||
renamed: ExtractData,
|
||||
lg,
|
||||
):
|
||||
entry = Path(self.entry)
|
||||
|
||||
for term_old, term_new in zip(original.terms, renamed.terms):
|
||||
for course_old, course_new in zip(term_old.courses, term_new.courses):
|
||||
# ---------- content files (per chapter) -----------------
|
||||
for chap_old, chap_new in zip(course_old.chapters, course_new.chapters):
|
||||
n = min(len(chap_old.content_files), len(chap_new.content_files))
|
||||
for i in range(n):
|
||||
fo = chap_old.content_files[i]
|
||||
fn = chap_new.content_files[i]
|
||||
src = (
|
||||
entry
|
||||
/ term_old.name
|
||||
/ course_old.name
|
||||
/ chap_old.name
|
||||
/ fo.name
|
||||
)
|
||||
dst = (
|
||||
Path(term_new.name)
|
||||
/ course_new.name
|
||||
/ chap_new.name
|
||||
/ fn.name
|
||||
)
|
||||
self._stage_or_warn(src, dst, lg)
|
||||
|
||||
# ---------- media files (course-level “media”) ----------
|
||||
src_media_dir = (
|
||||
entry / term_old.name / course_old.name / "media"
|
||||
) # <─ fixed!
|
||||
dst_media_dir = Path(term_new.name) / course_new.name / "media"
|
||||
if not src_media_dir.is_dir():
|
||||
continue
|
||||
|
||||
# build a flat list of (old, new) media filenames
|
||||
media_pairs: List[tuple[ExtractedFile, ExtractedFile]] = []
|
||||
for ch_o, ch_n in zip(course_old.chapters, course_new.chapters):
|
||||
media_pairs.extend(zip(ch_o.media_files, ch_n.media_files))
|
||||
|
||||
for fo, fn in media_pairs:
|
||||
src = src_media_dir / fo.name
|
||||
dst = dst_media_dir / fn.name
|
||||
self._stage_or_warn(src, dst, lg)
|
||||
Whitespace-only changes.
@@ -0,0 +1,66 @@
|
||||
"""
|
||||
Shared lists and prompts
|
||||
"""
|
||||
|
||||
# -------------------------------------------------------------------- #
|
||||
# file selection – keep only real documents we can show / convert #
|
||||
# -------------------------------------------------------------------- #
|
||||
CONTENT_FILE_EXTENSIONS = [
|
||||
"*.pdf",
|
||||
"*.doc",
|
||||
"*.docx",
|
||||
"*.ppt",
|
||||
"*.pptx",
|
||||
"*.txt",
|
||||
"*.rtf",
|
||||
]
|
||||
|
||||
MEDIA_FILE_EXTENSIONS = [
|
||||
"*.jpg",
|
||||
"*.jpeg",
|
||||
"*.png",
|
||||
"*.gif",
|
||||
"*.svg",
|
||||
"*.mp4",
|
||||
"*.mov",
|
||||
"*.mp3",
|
||||
]
|
||||
|
||||
# -------------------------------------------------------------------- #
|
||||
# naming rules #
|
||||
# -------------------------------------------------------------------- #
|
||||
SANITIZE_REGEX = {
|
||||
"base": [r"\s*\(\d+\)$"],
|
||||
"course": [
|
||||
r"^\d+\.\s*",
|
||||
r"\s*\([^)]*\)",
|
||||
r"\s*(?:FS|HS)\d{2}$",
|
||||
],
|
||||
"chapter": [
|
||||
r"^\d+\.?\s*",
|
||||
r"\s*SW_\d+\s*(?:___)?\s*KW_\d+\s*",
|
||||
r"\bKapitel[_\s]*\d+\b",
|
||||
],
|
||||
"file": [
|
||||
r",", # ← new : drop commas
|
||||
r",?\s*inkl\.?\s*",
|
||||
r"\(File\)",
|
||||
r"```json",
|
||||
],
|
||||
}
|
||||
|
||||
BLACKLIST_REGEX = {
|
||||
"chapter": [r"^allgemeine informationen$"],
|
||||
"ressource_types": [
|
||||
"(Forum)",
|
||||
"(URL)",
|
||||
"(External tool)",
|
||||
"(Text and media area)",
|
||||
],
|
||||
}
|
||||
|
||||
RESSOURCE_TYPES = BLACKLIST_REGEX["ressource_types"]
|
||||
BASE_BLACKLIST_REGEX = SANITIZE_REGEX["base"]
|
||||
|
||||
MAX_FILENAME_LENGTH = 100
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
from librarian_extractor.extractor.extractor import Extractor
|
||||
|
||||
__all__ = ["Extractor"]
|
||||
@@ -0,0 +1,301 @@
|
||||
"""
|
||||
Extractor Worker – resilient version
|
||||
------------------------------------
|
||||
* Finds the real payload even when the link goes to
|
||||
File_…/index.html first.
|
||||
* No `iterdir` on non-directories.
|
||||
* Keeps all earlier features: id parsing, allowed-suffix filter,
|
||||
media-folder, sanitising.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
import shutil
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
from typing import Tuple
|
||||
|
||||
import lxml.html
|
||||
import parsel
|
||||
from librarian_core.utils.path_utils import get_temp_path
|
||||
from librarian_core.workers.base import Worker
|
||||
from librarian_scraper.models.download_data import DownloadData
|
||||
from prefect import get_run_logger, task
|
||||
from prefect.futures import wait
|
||||
|
||||
from librarian_extractor.constants import (
|
||||
CONTENT_FILE_EXTENSIONS,
|
||||
MEDIA_FILE_EXTENSIONS,
|
||||
)
|
||||
from librarian_extractor.models.extract_data import (
|
||||
ExtractData,
|
||||
ExtractedChapter,
|
||||
ExtractedCourse,
|
||||
ExtractedFile,
|
||||
ExtractedTerm,
|
||||
)
|
||||
from librarian_extractor.sanitizers import (
|
||||
annotate_chapter_name,
|
||||
is_chapter_allowed,
|
||||
sanitize_chapter_name,
|
||||
sanitize_course_name,
|
||||
sanitize_file_name,
|
||||
)
|
||||
|
||||
CONTENT_EXTS = {Path(p).suffix.lower() for p in CONTENT_FILE_EXTENSIONS}
|
||||
MEDIA_EXTS = {Path(p).suffix.lower() for p in MEDIA_FILE_EXTENSIONS}
|
||||
ALL_EXTS = CONTENT_EXTS | MEDIA_EXTS
|
||||
|
||||
_id_rx = re.compile(r"\.(\d{4,})[./]") # 1172180 from “..._.1172180/index.html”
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# helpers #
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _hash_id(fname: str) -> str:
|
||||
return hashlib.sha1(fname.encode()).hexdigest()[:10]
|
||||
|
||||
|
||||
def _html_stub_target(html_file: Path) -> Path | None:
|
||||
"""Parse a Moodle *index.html* stub and return the first file link."""
|
||||
try:
|
||||
tree = lxml.html.parse(html_file) # type: ignore[arg-type]
|
||||
hrefs = tree.xpath("//ul/li/a/@href")
|
||||
for h in hrefs:
|
||||
h = h.split("?")[0].split("#")[0]
|
||||
p = html_file.parent / h
|
||||
if p.exists():
|
||||
return p
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
def _best_payload(node: Path) -> Path | None: # noqa: C901
|
||||
"""
|
||||
Return the real document given *node* which may be:
|
||||
• the actual file → return it
|
||||
• File_xxx/dir → search inside /content or dir itself
|
||||
• File_xxx/index.html stub → parse to find linked file
|
||||
"""
|
||||
# 1) immediate hit
|
||||
if node.is_file() and node.suffix.lower() in ALL_EXTS:
|
||||
return node
|
||||
|
||||
# 2) if html stub try to parse inner link
|
||||
if node.is_file() and node.suffix.lower() in {".html", ".htm"}:
|
||||
hinted = _html_stub_target(node)
|
||||
if hinted:
|
||||
return _best_payload(hinted) # recurse
|
||||
|
||||
# 3) directories to search
|
||||
roots: list[Path] = []
|
||||
if node.is_dir():
|
||||
roots.append(node)
|
||||
elif node.is_file():
|
||||
roots.append(node.parent)
|
||||
|
||||
for r in list(roots):
|
||||
if r.is_dir() and (r / "content").is_dir():
|
||||
roots.insert(0, r / "content") # prefer content folder
|
||||
|
||||
for r in roots:
|
||||
if not r.is_dir():
|
||||
continue
|
||||
files = [p for p in r.iterdir() if p.is_file() and p.suffix.lower() in ALL_EXTS]
|
||||
if len(files) == 1:
|
||||
return files[0]
|
||||
return None
|
||||
|
||||
|
||||
def _file_id_from_href(href: str) -> str:
|
||||
m = _id_rx.search(href)
|
||||
return m.group(1) if m else ""
|
||||
|
||||
|
||||
def task_(**kw):
|
||||
kw.setdefault("log_prints", True)
|
||||
return task(**kw)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Worker #
|
||||
# --------------------------------------------------------------------------- #
|
||||
class Extractor(Worker[DownloadData, ExtractData]):
|
||||
input_model = DownloadData
|
||||
output_model = ExtractData
|
||||
|
||||
async def __run__(self, downloads: DownloadData) -> ExtractData:
|
||||
lg = get_run_logger()
|
||||
work_root = Path(get_temp_path()) / "extract"
|
||||
work_root.mkdir(parents=True, exist_ok=True)
|
||||
self.out_dir = work_root
|
||||
|
||||
result = ExtractData()
|
||||
futs = []
|
||||
entry_dir = self.entry
|
||||
|
||||
for t in downloads.terms:
|
||||
(work_root / t.name).mkdir(exist_ok=True)
|
||||
result.terms.append(ExtractedTerm(id=t.id, name=t.name))
|
||||
for c in t.courses:
|
||||
futs.append(
|
||||
self._extract_course.submit(t.name, c.id, work_root, entry_dir)
|
||||
)
|
||||
|
||||
done, _ = wait(futs)
|
||||
for fut in done:
|
||||
term, meta = fut.result()
|
||||
if meta:
|
||||
next(t for t in result.terms if t.name == term).courses.append(meta)
|
||||
|
||||
for term in result.terms:
|
||||
self.stage(
|
||||
work_root / term.name, new_name=term.name, sanitize=False, move=True
|
||||
)
|
||||
lg.info("Extractor finished – %d terms", len(result.terms))
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
@staticmethod
|
||||
@task_()
|
||||
def _extract_course( # noqa: C901
|
||||
term: str, cid: str, out_root: Path, entry_dir: Path
|
||||
) -> Tuple[str, ExtractedCourse | None]:
|
||||
lg = get_run_logger()
|
||||
z = entry_dir / term / f"{cid}.zip"
|
||||
if not z.is_file():
|
||||
lg.warning("ZIP missing %s", z)
|
||||
return term, None
|
||||
|
||||
tmp = Path(get_temp_path()) / f"u{cid}"
|
||||
tmp.mkdir(exist_ok=True)
|
||||
try:
|
||||
with zipfile.ZipFile(z) as zf:
|
||||
zf.extractall(tmp)
|
||||
|
||||
html, root = Extractor._index_html(tmp)
|
||||
if not html:
|
||||
lg.warning("index.html missing for %s", cid)
|
||||
return term, None
|
||||
|
||||
cname = Extractor._course_name(html) or cid
|
||||
c_meta = ExtractedCourse(id=cid, name=cname)
|
||||
media_dir = out_root / term / cname / "media"
|
||||
|
||||
structure = Extractor._outline(html)
|
||||
if not structure:
|
||||
Extractor._copy_all(
|
||||
root, out_root / term / cname, c_meta, media_dir, lg # type: ignore
|
||||
)
|
||||
return term, c_meta
|
||||
|
||||
chap_no = 0
|
||||
for title, links in structure:
|
||||
if not is_chapter_allowed(title):
|
||||
continue
|
||||
chap_no += 1
|
||||
chap_name = annotate_chapter_name(sanitize_chapter_name(title), chap_no)
|
||||
chap_dir = out_root / term / cname / chap_name
|
||||
chap_dir.mkdir(parents=True, exist_ok=True)
|
||||
chap_meta = ExtractedChapter(name=chap_name)
|
||||
|
||||
for text, href in links:
|
||||
target = _best_payload(root / href.lstrip("./"))
|
||||
if not target:
|
||||
lg.debug("payload not found %s", href)
|
||||
continue
|
||||
|
||||
base = sanitize_file_name(text)
|
||||
if not Path(base).suffix:
|
||||
base += target.suffix # ensure extension
|
||||
|
||||
dst = (
|
||||
media_dir / base
|
||||
if target.suffix.lower() in MEDIA_EXTS
|
||||
else chap_dir / base
|
||||
)
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(target, dst)
|
||||
|
||||
fid = _file_id_from_href(href) or _hash_id(dst.name)
|
||||
meta_obj = ExtractedFile(id=fid, name=dst.name)
|
||||
(
|
||||
chap_meta.media_files
|
||||
if dst.is_relative_to(media_dir)
|
||||
else chap_meta.content_files
|
||||
).append(meta_obj)
|
||||
|
||||
if chap_meta.content_files or chap_meta.media_files:
|
||||
c_meta.chapters.append(chap_meta)
|
||||
|
||||
if c_meta.chapters:
|
||||
lg.info("Extracted %s (%d chap.)", cname, len(c_meta.chapters))
|
||||
return term, c_meta
|
||||
return term, None
|
||||
finally:
|
||||
shutil.rmtree(tmp, ignore_errors=True)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# internal helpers #
|
||||
# ------------------------------------------------------------------ #
|
||||
@staticmethod
|
||||
def _copy_all(
|
||||
root: Path, dst_root: Path, c_meta: ExtractedCourse, media_dir: Path, lg
|
||||
):
|
||||
chap = ExtractedChapter(name="Everything")
|
||||
dst_root.mkdir(parents=True, exist_ok=True)
|
||||
for fp in root.rglob("*"):
|
||||
if fp.is_file() and fp.suffix.lower() in ALL_EXTS:
|
||||
dst = (
|
||||
media_dir if fp.suffix.lower() in MEDIA_EXTS else dst_root
|
||||
) / fp.name
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(fp, dst)
|
||||
chap.content_files.append(
|
||||
ExtractedFile(id=_hash_id(fp.name), name=dst.name)
|
||||
)
|
||||
if chap.content_files:
|
||||
c_meta.chapters.append(chap)
|
||||
lg.info("Fallback copy %d files", len(chap.content_files))
|
||||
|
||||
@staticmethod
|
||||
def _index_html(root: Path) -> Tuple[str, Path | None]:
|
||||
for idx in root.rglob("index.html"):
|
||||
try:
|
||||
return idx.read_text("utf-8", errors="ignore"), idx.parent
|
||||
except Exception:
|
||||
continue
|
||||
return "", None
|
||||
|
||||
@staticmethod
|
||||
def _course_name(html: str) -> str:
|
||||
sel = parsel.Selector(text=html)
|
||||
return sanitize_course_name(sel.css("h1 a::text").get(default="").strip())
|
||||
|
||||
@staticmethod
|
||||
def _outline(html: str):
|
||||
t = lxml.html.fromstring(html)
|
||||
res = []
|
||||
for h3 in t.xpath("//h3"):
|
||||
title = h3.text_content().strip()
|
||||
ul = next((s for s in h3.itersiblings() if s.tag == "ul"), None)
|
||||
if ul is None:
|
||||
continue
|
||||
links = []
|
||||
for a in ul.findall(".//a"):
|
||||
if "(File)" in (a.text_content() or ""):
|
||||
sel = parsel.Selector(
|
||||
text=lxml.html.tostring(a, encoding="unicode") # type: ignore
|
||||
)
|
||||
links.append(
|
||||
(
|
||||
sel.css("::text").get().strip(), # type: ignore
|
||||
sel.css("::attr(href)").get().strip(), # type: ignore
|
||||
)
|
||||
)
|
||||
if links:
|
||||
res.append((title, links))
|
||||
return res
|
||||
Whitespace-only changes.
@@ -0,0 +1,30 @@
|
||||
from typing import List
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class ExtractedFile(BaseModel):
|
||||
id: str
|
||||
name: str # Name of the file, relative to ExtractedChapter.name
|
||||
|
||||
|
||||
class ExtractedChapter(BaseModel):
|
||||
name: str # Name of the chapter directory, relative to ExtractedCourse.name
|
||||
content_files: List[ExtractedFile] = Field(default_factory=list)
|
||||
media_files: List[ExtractedFile] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ExtractedCourse(BaseModel):
|
||||
id: str
|
||||
name: str # Name of the course directory, relative to ExtractedTerm.name
|
||||
chapters: List[ExtractedChapter] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ExtractedTerm(BaseModel):
|
||||
id: str
|
||||
name: str # Name of the term directory, relative to ExtractMeta.dir
|
||||
courses: List[ExtractedCourse] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ExtractData(BaseModel):
|
||||
terms: List[ExtractedTerm] = Field(default_factory=list)
|
||||
@@ -0,0 +1,29 @@
|
||||
# -------------------------------------------------------------------- #
|
||||
# LLM prompts #
|
||||
# -------------------------------------------------------------------- #
|
||||
|
||||
PROMPT_COURSE = """
|
||||
General naming rules
|
||||
====================
|
||||
* Use underscores instead of spaces.
|
||||
* Keep meaningful numbers / IDs.
|
||||
* Remove Date information, except it is absolutely necessary.
|
||||
* -> Normalize dates / months ("Februar" → "02").
|
||||
* Remove redundant semester / university codes (e.g. FS24, HS, FHGR, CDS).
|
||||
* Remove redundancy in general. ( DRY - Don't Repeat Yourself )
|
||||
* Trim superfluous parts like duplicate week information ("1_SW_01_KW_08" → "SW_01").
|
||||
* Only keep have one enumarator at a time, so "1_SW_01" → "SW_01".
|
||||
* Preserve file extensions!
|
||||
* Avoid repeated dots and illegal filesystem characters (colon, slash, …).
|
||||
|
||||
The most important rule is to keep everything as consistent as possible.
|
||||
|
||||
Important – DO NOT:
|
||||
* change the JSON structure,
|
||||
* change or reorder any `id`,
|
||||
* add any keys.
|
||||
|
||||
Return **only** the modified JSON for the course you receive.
|
||||
|
||||
Everything should be in english after the sanitization.
|
||||
""".strip()
|
||||
Whitespace-only changes.
@@ -0,0 +1,71 @@
|
||||
"""
|
||||
Name-sanitising helpers
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
from librarian_extractor.constants import (
|
||||
BASE_BLACKLIST_REGEX,
|
||||
BLACKLIST_REGEX,
|
||||
MAX_FILENAME_LENGTH,
|
||||
RESSOURCE_TYPES,
|
||||
SANITIZE_REGEX,
|
||||
)
|
||||
|
||||
_INVALID_FS_CHARS = re.compile(r'[\\/:*?"<>|]')
|
||||
_WS = re.compile(r"\s+")
|
||||
_DUP_DOTS = re.compile(r"\.\.+")
|
||||
_TRAILING_NUM = re.compile(r"_\(\d+\)$")
|
||||
|
||||
|
||||
def _sanitize_name(name: str, extra_patterns: list[str]) -> str:
|
||||
original = name
|
||||
for rt in RESSOURCE_TYPES:
|
||||
name = name.replace(rt, "")
|
||||
for rx in BASE_BLACKLIST_REGEX + extra_patterns:
|
||||
name = re.sub(rx, "", name, flags=re.IGNORECASE)
|
||||
name = _INVALID_FS_CHARS.sub("_", name)
|
||||
name = _DUP_DOTS.sub(".", name)
|
||||
name = _WS.sub(" ", name).replace(" ", "_")
|
||||
name = re.sub(r"_+", "_", name).strip("_")
|
||||
base, dot, ext = name.rpartition(".")
|
||||
if dot:
|
||||
base = _TRAILING_NUM.sub("", base)
|
||||
dup = re.compile(rf"(?i)[._]{re.escape(ext)}$")
|
||||
base = dup.sub("", base)
|
||||
name = f"{base}.{ext}" if base else f".{ext}"
|
||||
else:
|
||||
name = _TRAILING_NUM.sub("", name)
|
||||
name = name.strip("_.")
|
||||
if len(name) > MAX_FILENAME_LENGTH:
|
||||
if dot and len(ext) < 10:
|
||||
avail = MAX_FILENAME_LENGTH - len(ext) - 1
|
||||
name = f"{base[:avail]}.{ext}"
|
||||
else:
|
||||
name = name[:MAX_FILENAME_LENGTH].rstrip("_")
|
||||
if not name or name == ".":
|
||||
name = re.sub(_INVALID_FS_CHARS, "_", original)[:MAX_FILENAME_LENGTH] or "file"
|
||||
return name
|
||||
|
||||
|
||||
def sanitize_course_name(name: str) -> str:
|
||||
return _sanitize_name(name, SANITIZE_REGEX["course"])
|
||||
|
||||
|
||||
def sanitize_chapter_name(name: str) -> str:
|
||||
return _sanitize_name(name, SANITIZE_REGEX["chapter"])
|
||||
|
||||
|
||||
def sanitize_file_name(name: str) -> str:
|
||||
return _sanitize_name(name, SANITIZE_REGEX["file"])
|
||||
|
||||
|
||||
def annotate_chapter_name(name: str, idx: Optional[int] = None) -> str:
|
||||
return f"{idx}_{name}" if idx is not None else name
|
||||
|
||||
|
||||
def is_chapter_allowed(name: str) -> bool:
|
||||
return name.strip().lower() not in BLACKLIST_REGEX["chapter"]
|
||||
+215
@@ -0,0 +1,215 @@
|
||||
version = 1
|
||||
revision = 1
|
||||
requires-python = ">=3.10"
|
||||
|
||||
[[package]]
|
||||
name = "colorama"
|
||||
version = "0.4.6"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "exceptiongroup"
|
||||
version = "1.2.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/09/35/2495c4ac46b980e4ca1f6ad6db102322ef3ad2410b79fdde159a4b0f3b92/exceptiongroup-1.2.2.tar.gz", hash = "sha256:47c2edf7c6738fafb49fd34290706d1a1a2f4d1c6df275526b62cbb4aa5393cc", size = 28883 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/02/cc/b7e31358aac6ed1ef2bb790a9746ac2c69bcb3c8588b41616914eb106eaf/exceptiongroup-1.2.2-py3-none-any.whl", hash = "sha256:3111b9d131c238bec2f8f516e123e14ba243563fb135d3fe885990585aa7795b", size = 16453 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iniconfig"
|
||||
version = "2.1.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f2/97/ebf4da567aa6827c909642694d71c9fcf53e5b504f2d96afea02718862f3/iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7", size = 4793 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/2c/e1/e6716421ea10d38022b952c159d5161ca1193197fb744506875fbb87ea7b/iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760", size = 6050 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "librarian-core"
|
||||
version = "0.1.0"
|
||||
source = { git = "https://github.com/DotNaos/librarian-core?rev=main#a564a04ad1019cb196af1ee11d654b77839a469b" }
|
||||
|
||||
[[package]]
|
||||
name = "librarian-scraper"
|
||||
version = "0.1.0"
|
||||
source = { editable = "." }
|
||||
dependencies = [
|
||||
{ name = "librarian-core" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
dev = [
|
||||
{ name = "mypy" },
|
||||
{ name = "pytest" },
|
||||
{ name = "ruff" },
|
||||
]
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [
|
||||
{ name = "importlib-metadata", marker = "python_full_version < '3.10'" },
|
||||
{ name = "librarian-core", git = "https://github.com/DotNaos/librarian-core?rev=main" },
|
||||
{ name = "mypy", marker = "extra == 'dev'" },
|
||||
{ name = "pytest", marker = "extra == 'dev'" },
|
||||
{ name = "ruff", marker = "extra == 'dev'" },
|
||||
]
|
||||
provides-extras = ["dev"]
|
||||
|
||||
[[package]]
|
||||
name = "mypy"
|
||||
version = "1.15.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "mypy-extensions" },
|
||||
{ name = "tomli", marker = "python_full_version < '3.11'" },
|
||||
{ name = "typing-extensions" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ce/43/d5e49a86afa64bd3839ea0d5b9c7103487007d728e1293f52525d6d5486a/mypy-1.15.0.tar.gz", hash = "sha256:404534629d51d3efea5c800ee7c42b72a6554d6c400e6a79eafe15d11341fd43", size = 3239717 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/68/f8/65a7ce8d0e09b6329ad0c8d40330d100ea343bd4dd04c4f8ae26462d0a17/mypy-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:979e4e1a006511dacf628e36fadfecbcc0160a8af6ca7dad2f5025529e082c13", size = 10738433 },
|
||||
{ url = "https://files.pythonhosted.org/packages/b4/95/9c0ecb8eacfe048583706249439ff52105b3f552ea9c4024166c03224270/mypy-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:c4bb0e1bd29f7d34efcccd71cf733580191e9a264a2202b0239da95984c5b559", size = 9861472 },
|
||||
{ url = "https://files.pythonhosted.org/packages/84/09/9ec95e982e282e20c0d5407bc65031dfd0f0f8ecc66b69538296e06fcbee/mypy-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:be68172e9fd9ad8fb876c6389f16d1c1b5f100ffa779f77b1fb2176fcc9ab95b", size = 11611424 },
|
||||
{ url = "https://files.pythonhosted.org/packages/78/13/f7d14e55865036a1e6a0a69580c240f43bc1f37407fe9235c0d4ef25ffb0/mypy-1.15.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c7be1e46525adfa0d97681432ee9fcd61a3964c2446795714699a998d193f1a3", size = 12365450 },
|
||||
{ url = "https://files.pythonhosted.org/packages/48/e1/301a73852d40c241e915ac6d7bcd7fedd47d519246db2d7b86b9d7e7a0cb/mypy-1.15.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:2e2c2e6d3593f6451b18588848e66260ff62ccca522dd231cd4dd59b0160668b", size = 12551765 },
|
||||
{ url = "https://files.pythonhosted.org/packages/77/ba/c37bc323ae5fe7f3f15a28e06ab012cd0b7552886118943e90b15af31195/mypy-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:6983aae8b2f653e098edb77f893f7b6aca69f6cffb19b2cc7443f23cce5f4828", size = 9274701 },
|
||||
{ url = "https://files.pythonhosted.org/packages/03/bc/f6339726c627bd7ca1ce0fa56c9ae2d0144604a319e0e339bdadafbbb599/mypy-1.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:2922d42e16d6de288022e5ca321cd0618b238cfc5570e0263e5ba0a77dbef56f", size = 10662338 },
|
||||
{ url = "https://files.pythonhosted.org/packages/e2/90/8dcf506ca1a09b0d17555cc00cd69aee402c203911410136cd716559efe7/mypy-1.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2ee2d57e01a7c35de00f4634ba1bbf015185b219e4dc5909e281016df43f5ee5", size = 9787540 },
|
||||
{ url = "https://files.pythonhosted.org/packages/05/05/a10f9479681e5da09ef2f9426f650d7b550d4bafbef683b69aad1ba87457/mypy-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:973500e0774b85d9689715feeffcc980193086551110fd678ebe1f4342fb7c5e", size = 11538051 },
|
||||
{ url = "https://files.pythonhosted.org/packages/e9/9a/1f7d18b30edd57441a6411fcbc0c6869448d1a4bacbaee60656ac0fc29c8/mypy-1.15.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5a95fb17c13e29d2d5195869262f8125dfdb5c134dc8d9a9d0aecf7525b10c2c", size = 12286751 },
|
||||
{ url = "https://files.pythonhosted.org/packages/72/af/19ff499b6f1dafcaf56f9881f7a965ac2f474f69f6f618b5175b044299f5/mypy-1.15.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1905f494bfd7d85a23a88c5d97840888a7bd516545fc5aaedff0267e0bb54e2f", size = 12421783 },
|
||||
{ url = "https://files.pythonhosted.org/packages/96/39/11b57431a1f686c1aed54bf794870efe0f6aeca11aca281a0bd87a5ad42c/mypy-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:c9817fa23833ff189db061e6d2eff49b2f3b6ed9856b4a0a73046e41932d744f", size = 9265618 },
|
||||
{ url = "https://files.pythonhosted.org/packages/98/3a/03c74331c5eb8bd025734e04c9840532226775c47a2c39b56a0c8d4f128d/mypy-1.15.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:aea39e0583d05124836ea645f412e88a5c7d0fd77a6d694b60d9b6b2d9f184fd", size = 10793981 },
|
||||
{ url = "https://files.pythonhosted.org/packages/f0/1a/41759b18f2cfd568848a37c89030aeb03534411eef981df621d8fad08a1d/mypy-1.15.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:2f2147ab812b75e5b5499b01ade1f4a81489a147c01585cda36019102538615f", size = 9749175 },
|
||||
{ url = "https://files.pythonhosted.org/packages/12/7e/873481abf1ef112c582db832740f4c11b2bfa510e829d6da29b0ab8c3f9c/mypy-1.15.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ce436f4c6d218a070048ed6a44c0bbb10cd2cc5e272b29e7845f6a2f57ee4464", size = 11455675 },
|
||||
{ url = "https://files.pythonhosted.org/packages/b3/d0/92ae4cde706923a2d3f2d6c39629134063ff64b9dedca9c1388363da072d/mypy-1.15.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8023ff13985661b50a5928fc7a5ca15f3d1affb41e5f0a9952cb68ef090b31ee", size = 12410020 },
|
||||
{ url = "https://files.pythonhosted.org/packages/46/8b/df49974b337cce35f828ba6fda228152d6db45fed4c86ba56ffe442434fd/mypy-1.15.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1124a18bc11a6a62887e3e137f37f53fbae476dc36c185d549d4f837a2a6a14e", size = 12498582 },
|
||||
{ url = "https://files.pythonhosted.org/packages/13/50/da5203fcf6c53044a0b699939f31075c45ae8a4cadf538a9069b165c1050/mypy-1.15.0-cp312-cp312-win_amd64.whl", hash = "sha256:171a9ca9a40cd1843abeca0e405bc1940cd9b305eaeea2dda769ba096932bb22", size = 9366614 },
|
||||
{ url = "https://files.pythonhosted.org/packages/6a/9b/fd2e05d6ffff24d912f150b87db9e364fa8282045c875654ce7e32fffa66/mypy-1.15.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:93faf3fdb04768d44bf28693293f3904bbb555d076b781ad2530214ee53e3445", size = 10788592 },
|
||||
{ url = "https://files.pythonhosted.org/packages/74/37/b246d711c28a03ead1fd906bbc7106659aed7c089d55fe40dd58db812628/mypy-1.15.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:811aeccadfb730024c5d3e326b2fbe9249bb7413553f15499a4050f7c30e801d", size = 9753611 },
|
||||
{ url = "https://files.pythonhosted.org/packages/a6/ac/395808a92e10cfdac8003c3de9a2ab6dc7cde6c0d2a4df3df1b815ffd067/mypy-1.15.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98b7b9b9aedb65fe628c62a6dc57f6d5088ef2dfca37903a7d9ee374d03acca5", size = 11438443 },
|
||||
{ url = "https://files.pythonhosted.org/packages/d2/8b/801aa06445d2de3895f59e476f38f3f8d610ef5d6908245f07d002676cbf/mypy-1.15.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c43a7682e24b4f576d93072216bf56eeff70d9140241f9edec0c104d0c515036", size = 12402541 },
|
||||
{ url = "https://files.pythonhosted.org/packages/c7/67/5a4268782eb77344cc613a4cf23540928e41f018a9a1ec4c6882baf20ab8/mypy-1.15.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:baefc32840a9f00babd83251560e0ae1573e2f9d1b067719479bfb0e987c6357", size = 12494348 },
|
||||
{ url = "https://files.pythonhosted.org/packages/83/3e/57bb447f7bbbfaabf1712d96f9df142624a386d98fb026a761532526057e/mypy-1.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:b9378e2c00146c44793c98b8d5a61039a048e31f429fb0eb546d93f4b000bedf", size = 9373648 },
|
||||
{ url = "https://files.pythonhosted.org/packages/09/4e/a7d65c7322c510de2c409ff3828b03354a7c43f5a8ed458a7a131b41c7b9/mypy-1.15.0-py3-none-any.whl", hash = "sha256:5469affef548bd1895d86d3bf10ce2b44e33d86923c29e4d675b3e323437ea3e", size = 2221777 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "mypy-extensions"
|
||||
version = "1.1.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a2/6e/371856a3fb9d31ca8dac321cda606860fa4548858c0cc45d9d1d4ca2628b/mypy_extensions-1.1.0.tar.gz", hash = "sha256:52e68efc3284861e772bbcd66823fde5ae21fd2fdb51c62a211403730b916558", size = 6343 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/79/7b/2c79738432f5c924bef5071f933bcc9efd0473bac3b4aa584a6f7c1c8df8/mypy_extensions-1.1.0-py3-none-any.whl", hash = "sha256:1be4cccdb0f2482337c4743e60421de3a356cd97508abadd57d47403e94f5505", size = 4963 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "packaging"
|
||||
version = "25.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/a1/d4/1fc4078c65507b51b96ca8f8c3ba19e6a61c8253c72794544580a7b6c24d/packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f", size = 165727 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pluggy"
|
||||
version = "1.5.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/96/2d/02d4312c973c6050a18b314a5ad0b3210edb65a906f868e31c111dede4a6/pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1", size = 67955 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/88/5f/e351af9a41f866ac3f1fac4ca0613908d9a41741cfcf2228f4ad853b697d/pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669", size = 20556 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pytest"
|
||||
version = "8.3.5"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "colorama", marker = "sys_platform == 'win32'" },
|
||||
{ name = "exceptiongroup", marker = "python_full_version < '3.11'" },
|
||||
{ name = "iniconfig" },
|
||||
{ name = "packaging" },
|
||||
{ name = "pluggy" },
|
||||
{ name = "tomli", marker = "python_full_version < '3.11'" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/ae/3c/c9d525a414d506893f0cd8a8d0de7706446213181570cdbd766691164e40/pytest-8.3.5.tar.gz", hash = "sha256:f4efe70cc14e511565ac476b57c279e12a855b11f48f212af1080ef2263d3845", size = 1450891 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/30/3d/64ad57c803f1fa1e963a7946b6e0fea4a70df53c1a7fed304586539c2bac/pytest-8.3.5-py3-none-any.whl", hash = "sha256:c69214aa47deac29fad6c2a4f590b9c4a9fdb16a403176fe154b79c0b4d4d820", size = 343634 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ruff"
|
||||
version = "0.11.7"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/5b/89/6f9c9674818ac2e9cc2f2b35b704b7768656e6b7c139064fc7ba8fbc99f1/ruff-0.11.7.tar.gz", hash = "sha256:655089ad3224070736dc32844fde783454f8558e71f501cb207485fe4eee23d4", size = 4054861 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/b4/ec/21927cb906c5614b786d1621dba405e3d44f6e473872e6df5d1a6bca0455/ruff-0.11.7-py3-none-linux_armv6l.whl", hash = "sha256:d29e909d9a8d02f928d72ab7837b5cbc450a5bdf578ab9ebee3263d0a525091c", size = 10245403 },
|
||||
{ url = "https://files.pythonhosted.org/packages/e2/af/fec85b6c2c725bcb062a354dd7cbc1eed53c33ff3aa665165871c9c16ddf/ruff-0.11.7-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:dd1fb86b168ae349fb01dd497d83537b2c5541fe0626e70c786427dd8363aaee", size = 11007166 },
|
||||
{ url = "https://files.pythonhosted.org/packages/31/9a/2d0d260a58e81f388800343a45898fd8df73c608b8261c370058b675319a/ruff-0.11.7-py3-none-macosx_11_0_arm64.whl", hash = "sha256:d3d7d2e140a6fbbc09033bce65bd7ea29d6a0adeb90b8430262fbacd58c38ada", size = 10378076 },
|
||||
{ url = "https://files.pythonhosted.org/packages/c2/c4/9b09b45051404d2e7dd6d9dbcbabaa5ab0093f9febcae664876a77b9ad53/ruff-0.11.7-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4809df77de390a1c2077d9b7945d82f44b95d19ceccf0c287c56e4dc9b91ca64", size = 10557138 },
|
||||
{ url = "https://files.pythonhosted.org/packages/5e/5e/f62a1b6669870a591ed7db771c332fabb30f83c967f376b05e7c91bccd14/ruff-0.11.7-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:f3a0c2e169e6b545f8e2dba185eabbd9db4f08880032e75aa0e285a6d3f48201", size = 10095726 },
|
||||
{ url = "https://files.pythonhosted.org/packages/45/59/a7aa8e716f4cbe07c3500a391e58c52caf665bb242bf8be42c62adef649c/ruff-0.11.7-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:49b888200a320dd96a68e86736cf531d6afba03e4f6cf098401406a257fcf3d6", size = 11672265 },
|
||||
{ url = "https://files.pythonhosted.org/packages/dd/e3/101a8b707481f37aca5f0fcc3e42932fa38b51add87bfbd8e41ab14adb24/ruff-0.11.7-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:2b19cdb9cf7dae00d5ee2e7c013540cdc3b31c4f281f1dacb5a799d610e90db4", size = 12331418 },
|
||||
{ url = "https://files.pythonhosted.org/packages/dd/71/037f76cbe712f5cbc7b852e4916cd3cf32301a30351818d32ab71580d1c0/ruff-0.11.7-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:64e0ee994c9e326b43539d133a36a455dbaab477bc84fe7bfbd528abe2f05c1e", size = 11794506 },
|
||||
{ url = "https://files.pythonhosted.org/packages/ca/de/e450b6bab1fc60ef263ef8fcda077fb4977601184877dce1c59109356084/ruff-0.11.7-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:bad82052311479a5865f52c76ecee5d468a58ba44fb23ee15079f17dd4c8fd63", size = 13939084 },
|
||||
{ url = "https://files.pythonhosted.org/packages/0e/2c/1e364cc92970075d7d04c69c928430b23e43a433f044474f57e425cbed37/ruff-0.11.7-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7940665e74e7b65d427b82bffc1e46710ec7f30d58b4b2d5016e3f0321436502", size = 11450441 },
|
||||
{ url = "https://files.pythonhosted.org/packages/9d/7d/1b048eb460517ff9accd78bca0fa6ae61df2b276010538e586f834f5e402/ruff-0.11.7-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:169027e31c52c0e36c44ae9a9c7db35e505fee0b39f8d9fca7274a6305295a92", size = 10441060 },
|
||||
{ url = "https://files.pythonhosted.org/packages/3a/57/8dc6ccfd8380e5ca3d13ff7591e8ba46a3b330323515a4996b991b10bd5d/ruff-0.11.7-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:305b93f9798aee582e91e34437810439acb28b5fc1fee6b8205c78c806845a94", size = 10058689 },
|
||||
{ url = "https://files.pythonhosted.org/packages/23/bf/20487561ed72654147817885559ba2aa705272d8b5dee7654d3ef2dbf912/ruff-0.11.7-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a681db041ef55550c371f9cd52a3cf17a0da4c75d6bd691092dfc38170ebc4b6", size = 11073703 },
|
||||
{ url = "https://files.pythonhosted.org/packages/9d/27/04f2db95f4ef73dccedd0c21daf9991cc3b7f29901a4362057b132075aa4/ruff-0.11.7-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:07f1496ad00a4a139f4de220b0c97da6d4c85e0e4aa9b2624167b7d4d44fd6b6", size = 11532822 },
|
||||
{ url = "https://files.pythonhosted.org/packages/e1/72/43b123e4db52144c8add336581de52185097545981ff6e9e58a21861c250/ruff-0.11.7-py3-none-win32.whl", hash = "sha256:f25dfb853ad217e6e5f1924ae8a5b3f6709051a13e9dad18690de6c8ff299e26", size = 10362436 },
|
||||
{ url = "https://files.pythonhosted.org/packages/c5/a0/3e58cd76fdee53d5c8ce7a56d84540833f924ccdf2c7d657cb009e604d82/ruff-0.11.7-py3-none-win_amd64.whl", hash = "sha256:0a931d85959ceb77e92aea4bbedfded0a31534ce191252721128f77e5ae1f98a", size = 11566676 },
|
||||
{ url = "https://files.pythonhosted.org/packages/68/ca/69d7c7752bce162d1516e5592b1cc6b6668e9328c0d270609ddbeeadd7cf/ruff-0.11.7-py3-none-win_arm64.whl", hash = "sha256:778c1e5d6f9e91034142dfd06110534ca13220bfaad5c3735f6cb844654f6177", size = 10677936 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "tomli"
|
||||
version = "2.2.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/18/87/302344fed471e44a87289cf4967697d07e532f2421fdaf868a303cbae4ff/tomli-2.2.1.tar.gz", hash = "sha256:cd45e1dc79c835ce60f7404ec8119f2eb06d38b1deba146f07ced3bbc44505ff", size = 17175 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/43/ca/75707e6efa2b37c77dadb324ae7d9571cb424e61ea73fad7c56c2d14527f/tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249", size = 131077 },
|
||||
{ url = "https://files.pythonhosted.org/packages/c7/16/51ae563a8615d472fdbffc43a3f3d46588c264ac4f024f63f01283becfbb/tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6", size = 123429 },
|
||||
{ url = "https://files.pythonhosted.org/packages/f1/dd/4f6cd1e7b160041db83c694abc78e100473c15d54620083dbd5aae7b990e/tomli-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ece47d672db52ac607a3d9599a9d48dcb2f2f735c6c2d1f34130085bb12b112a", size = 226067 },
|
||||
{ url = "https://files.pythonhosted.org/packages/a9/6b/c54ede5dc70d648cc6361eaf429304b02f2871a345bbdd51e993d6cdf550/tomli-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6972ca9c9cc9f0acaa56a8ca1ff51e7af152a9f87fb64623e31d5c83700080ee", size = 236030 },
|
||||
{ url = "https://files.pythonhosted.org/packages/1f/47/999514fa49cfaf7a92c805a86c3c43f4215621855d151b61c602abb38091/tomli-2.2.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c954d2250168d28797dd4e3ac5cf812a406cd5a92674ee4c8f123c889786aa8e", size = 240898 },
|
||||
{ url = "https://files.pythonhosted.org/packages/73/41/0a01279a7ae09ee1573b423318e7934674ce06eb33f50936655071d81a24/tomli-2.2.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8dd28b3e155b80f4d54beb40a441d366adcfe740969820caf156c019fb5c7ec4", size = 229894 },
|
||||
{ url = "https://files.pythonhosted.org/packages/55/18/5d8bc5b0a0362311ce4d18830a5d28943667599a60d20118074ea1b01bb7/tomli-2.2.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:e59e304978767a54663af13c07b3d1af22ddee3bb2fb0618ca1593e4f593a106", size = 245319 },
|
||||
{ url = "https://files.pythonhosted.org/packages/92/a3/7ade0576d17f3cdf5ff44d61390d4b3febb8a9fc2b480c75c47ea048c646/tomli-2.2.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:33580bccab0338d00994d7f16f4c4ec25b776af3ffaac1ed74e0b3fc95e885a8", size = 238273 },
|
||||
{ url = "https://files.pythonhosted.org/packages/72/6f/fa64ef058ac1446a1e51110c375339b3ec6be245af9d14c87c4a6412dd32/tomli-2.2.1-cp311-cp311-win32.whl", hash = "sha256:465af0e0875402f1d226519c9904f37254b3045fc5084697cefb9bdde1ff99ff", size = 98310 },
|
||||
{ url = "https://files.pythonhosted.org/packages/6a/1c/4a2dcde4a51b81be3530565e92eda625d94dafb46dbeb15069df4caffc34/tomli-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:2d0f2fdd22b02c6d81637a3c95f8cd77f995846af7414c5c4b8d0545afa1bc4b", size = 108309 },
|
||||
{ url = "https://files.pythonhosted.org/packages/52/e1/f8af4c2fcde17500422858155aeb0d7e93477a0d59a98e56cbfe75070fd0/tomli-2.2.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4a8f6e44de52d5e6c657c9fe83b562f5f4256d8ebbfe4ff922c495620a7f6cea", size = 132762 },
|
||||
{ url = "https://files.pythonhosted.org/packages/03/b8/152c68bb84fc00396b83e7bbddd5ec0bd3dd409db4195e2a9b3e398ad2e3/tomli-2.2.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8d57ca8095a641b8237d5b079147646153d22552f1c637fd3ba7f4b0b29167a8", size = 123453 },
|
||||
{ url = "https://files.pythonhosted.org/packages/c8/d6/fc9267af9166f79ac528ff7e8c55c8181ded34eb4b0e93daa767b8841573/tomli-2.2.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4e340144ad7ae1533cb897d406382b4b6fede8890a03738ff1683af800d54192", size = 233486 },
|
||||
{ url = "https://files.pythonhosted.org/packages/5c/51/51c3f2884d7bab89af25f678447ea7d297b53b5a3b5730a7cb2ef6069f07/tomli-2.2.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:db2b95f9de79181805df90bedc5a5ab4c165e6ec3fe99f970d0e302f384ad222", size = 242349 },
|
||||
{ url = "https://files.pythonhosted.org/packages/ab/df/bfa89627d13a5cc22402e441e8a931ef2108403db390ff3345c05253935e/tomli-2.2.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:40741994320b232529c802f8bc86da4e1aa9f413db394617b9a256ae0f9a7f77", size = 252159 },
|
||||
{ url = "https://files.pythonhosted.org/packages/9e/6e/fa2b916dced65763a5168c6ccb91066f7639bdc88b48adda990db10c8c0b/tomli-2.2.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:400e720fe168c0f8521520190686ef8ef033fb19fc493da09779e592861b78c6", size = 237243 },
|
||||
{ url = "https://files.pythonhosted.org/packages/b4/04/885d3b1f650e1153cbb93a6a9782c58a972b94ea4483ae4ac5cedd5e4a09/tomli-2.2.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:02abe224de6ae62c19f090f68da4e27b10af2b93213d36cf44e6e1c5abd19fdd", size = 259645 },
|
||||
{ url = "https://files.pythonhosted.org/packages/9c/de/6b432d66e986e501586da298e28ebeefd3edc2c780f3ad73d22566034239/tomli-2.2.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b82ebccc8c8a36f2094e969560a1b836758481f3dc360ce9a3277c65f374285e", size = 244584 },
|
||||
{ url = "https://files.pythonhosted.org/packages/1c/9a/47c0449b98e6e7d1be6cbac02f93dd79003234ddc4aaab6ba07a9a7482e2/tomli-2.2.1-cp312-cp312-win32.whl", hash = "sha256:889f80ef92701b9dbb224e49ec87c645ce5df3fa2cc548664eb8a25e03127a98", size = 98875 },
|
||||
{ url = "https://files.pythonhosted.org/packages/ef/60/9b9638f081c6f1261e2688bd487625cd1e660d0a85bd469e91d8db969734/tomli-2.2.1-cp312-cp312-win_amd64.whl", hash = "sha256:7fc04e92e1d624a4a63c76474610238576942d6b8950a2d7f908a340494e67e4", size = 109418 },
|
||||
{ url = "https://files.pythonhosted.org/packages/04/90/2ee5f2e0362cb8a0b6499dc44f4d7d48f8fff06d28ba46e6f1eaa61a1388/tomli-2.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:f4039b9cbc3048b2416cc57ab3bda989a6fcf9b36cf8937f01a6e731b64f80d7", size = 132708 },
|
||||
{ url = "https://files.pythonhosted.org/packages/c0/ec/46b4108816de6b385141f082ba99e315501ccd0a2ea23db4a100dd3990ea/tomli-2.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:286f0ca2ffeeb5b9bd4fcc8d6c330534323ec51b2f52da063b11c502da16f30c", size = 123582 },
|
||||
{ url = "https://files.pythonhosted.org/packages/a0/bd/b470466d0137b37b68d24556c38a0cc819e8febe392d5b199dcd7f578365/tomli-2.2.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a92ef1a44547e894e2a17d24e7557a5e85a9e1d0048b0b5e7541f76c5032cb13", size = 232543 },
|
||||
{ url = "https://files.pythonhosted.org/packages/d9/e5/82e80ff3b751373f7cead2815bcbe2d51c895b3c990686741a8e56ec42ab/tomli-2.2.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9316dc65bed1684c9a98ee68759ceaed29d229e985297003e494aa825ebb0281", size = 241691 },
|
||||
{ url = "https://files.pythonhosted.org/packages/05/7e/2a110bc2713557d6a1bfb06af23dd01e7dde52b6ee7dadc589868f9abfac/tomli-2.2.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e85e99945e688e32d5a35c1ff38ed0b3f41f43fad8df0bdf79f72b2ba7bc5272", size = 251170 },
|
||||
{ url = "https://files.pythonhosted.org/packages/64/7b/22d713946efe00e0adbcdfd6d1aa119ae03fd0b60ebed51ebb3fa9f5a2e5/tomli-2.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ac065718db92ca818f8d6141b5f66369833d4a80a9d74435a268c52bdfa73140", size = 236530 },
|
||||
{ url = "https://files.pythonhosted.org/packages/38/31/3a76f67da4b0cf37b742ca76beaf819dca0ebef26d78fc794a576e08accf/tomli-2.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:d920f33822747519673ee656a4b6ac33e382eca9d331c87770faa3eef562aeb2", size = 258666 },
|
||||
{ url = "https://files.pythonhosted.org/packages/07/10/5af1293da642aded87e8a988753945d0cf7e00a9452d3911dd3bb354c9e2/tomli-2.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a198f10c4d1b1375d7687bc25294306e551bf1abfa4eace6650070a5c1ae2744", size = 243954 },
|
||||
{ url = "https://files.pythonhosted.org/packages/5b/b9/1ed31d167be802da0fc95020d04cd27b7d7065cc6fbefdd2f9186f60d7bd/tomli-2.2.1-cp313-cp313-win32.whl", hash = "sha256:d3f5614314d758649ab2ab3a62d4f2004c825922f9e370b29416484086b264ec", size = 98724 },
|
||||
{ url = "https://files.pythonhosted.org/packages/c7/32/b0963458706accd9afcfeb867c0f9175a741bf7b19cd424230714d722198/tomli-2.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:a38aa0308e754b0e3c67e344754dff64999ff9b513e691d0e786265c93583c69", size = 109383 },
|
||||
{ url = "https://files.pythonhosted.org/packages/6e/c2/61d3e0f47e2b74ef40a68b9e6ad5984f6241a942f7cd3bbfbdbd03861ea9/tomli-2.2.1-py3-none-any.whl", hash = "sha256:cb55c73c5f4408779d0cf3eef9f762b9c9f147a77de7b258bef0a5628adc85cc", size = 14257 },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "typing-extensions"
|
||||
version = "4.13.2"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f6/37/23083fcd6e35492953e8d2aaaa68b860eb422b34627b13f2ce3eb6106061/typing_extensions-4.13.2.tar.gz", hash = "sha256:e6c81219bd689f51865d9e372991c540bda33a0379d5573cddb9a3a23f7caaef", size = 106967 }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/8b/54/b1ae86c0973cc6f0210b53d508ca3641fb6d0c56823f288d108bc7ab3cc8/typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c", size = 45806 },
|
||||
]
|
||||
Reference in new issue
Block a user