Initialize Monorepo
This commit is contained in:
commit
f80792d739
106 files changed
+17076
No files matched your search
@@ -0,0 +1 @@
|
||||
# Librarian Scraper
|
||||
@@ -0,0 +1,41 @@
|
||||
[project]
|
||||
name = "librarian-scraper"
|
||||
version = "0.2.1"
|
||||
description = "FastAPI gateway and runtime pipeline for Librarian"
|
||||
readme = "README.md"
|
||||
authors = [{ name = "DotNaos", email = "schuetzoliver00@gmail.com" }]
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"importlib_metadata; python_version<'3.10'",
|
||||
"playwright>=1.51.0",
|
||||
"dotenv>=0.9.9",
|
||||
"parsel>=1.10.0",
|
||||
"librarian-core",
|
||||
"httpx>=0.28.1",
|
||||
]
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling>=1.21"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/librarian_scraper"]
|
||||
|
||||
[tool.hatch.metadata]
|
||||
allow-direct-references = true
|
||||
|
||||
[tool.uv.sources]
|
||||
#librarian-core = { git = "https://github.com/DotNaos/librarian-core", rev = "dev" }
|
||||
|
||||
[project.entry-points."librarian.workers"]
|
||||
crawler = "librarian_scraper.crawler:Crawler"
|
||||
downloader = "librarian_scraper.downloader:Downloader"
|
||||
|
||||
|
||||
# ───────── optional: dev / test extras ─────────
|
||||
[project.optional-dependencies]
|
||||
dev = ["ruff", "pytest", "mypy"]
|
||||
|
||||
[project.scripts]
|
||||
example = "examples.app:app"
|
||||
@@ -0,0 +1,12 @@
|
||||
from .crawler import (
|
||||
Crawler,
|
||||
)
|
||||
from .downloader import (
|
||||
Downloader,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"Crawler",
|
||||
"Downloader",
|
||||
"Extractor",
|
||||
]
|
||||
@@ -0,0 +1,29 @@
|
||||
"""
|
||||
URLs used by the scraper.
|
||||
Functions marked as PUBLIC can be accessed without authentication.
|
||||
Functions marked as PRIVATE require authentication.
|
||||
"""
|
||||
|
||||
BASE_URL = "https://moodle.fhgr.ch"
|
||||
|
||||
CRAWLER = {
|
||||
"DELAY_SLOW": 2.0,
|
||||
"DELAY_FAST": 0.5,
|
||||
"BATCH_SLOW": 2,
|
||||
"BATCH_FAST": 8,
|
||||
}
|
||||
|
||||
class PUBLIC_URLS:
|
||||
base_url = BASE_URL
|
||||
login = f"{BASE_URL}/login/index.php"
|
||||
index = f"{BASE_URL}/course/index.php"
|
||||
degree_program = lambda degree_program_id: f"{BASE_URL}/course/index.php?categoryid={degree_program_id}"
|
||||
category = lambda category_id: f"{BASE_URL}/course/index.php?categoryid={category_id}"
|
||||
term = lambda term_id: f"{BASE_URL}/course/index.php?categoryid={term_id}"
|
||||
|
||||
class PRIVATE_URLS:
|
||||
user_courses = f"{BASE_URL}/my/courses.php"
|
||||
dashboard = f"{BASE_URL}/my/"
|
||||
course = lambda course_id: f"{BASE_URL}/course/view.php?id={course_id}"
|
||||
files = lambda context_id: f"{BASE_URL}/course/downloadcontent.php?contextid={context_id}"
|
||||
file = lambda file_id: f"{BASE_URL}/mod/resource/view.php?id={file_id}"
|
||||
@@ -0,0 +1,7 @@
|
||||
from librarian_scraper.crawler.cookie_crawler import CookieCrawler
|
||||
from librarian_scraper.crawler.crawler import Crawler
|
||||
|
||||
__all__ = [
|
||||
"CookieCrawler",
|
||||
"Crawler",
|
||||
]
|
||||
@@ -0,0 +1,138 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from typing import List, Optional
|
||||
|
||||
from httpx import Cookies
|
||||
from playwright.async_api import Browser, Cookie, Page, async_playwright
|
||||
|
||||
from librarian_scraper.constants import PRIVATE_URLS, PUBLIC_URLS
|
||||
|
||||
|
||||
class CookieCrawler:
|
||||
"""
|
||||
Retrieve Moodle session cookies + sesskey via Playwright.
|
||||
|
||||
Usage
|
||||
-----
|
||||
>>> crawler = CookieCrawler()
|
||||
>>> cookies, sesskey = await crawler.crawl() # inside async code
|
||||
# or
|
||||
>>> cookies, sesskey = CookieCrawler.crawl_sync() # plain scripts
|
||||
"""
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# construction #
|
||||
# ------------------------------------------------------------------ #
|
||||
def __init__(self, *, headless: bool = True) -> None:
|
||||
self.headless = headless
|
||||
self.cookies: Optional[List[Cookie]] = None
|
||||
self.sesskey: str = ""
|
||||
|
||||
self.username: str = os.getenv("MOODLE_USERNAME", "")
|
||||
self.password: str = os.getenv("MOODLE_PASSWORD", "")
|
||||
if not self.username or not self.password:
|
||||
raise ValueError(
|
||||
"Set MOODLE_USERNAME and MOODLE_PASSWORD as environment variables."
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# public API #
|
||||
# ------------------------------------------------------------------ #
|
||||
async def crawl(self) -> tuple[Cookies, str]:
|
||||
"""
|
||||
Async entry-point – await this inside FastAPI / Prefect etc.
|
||||
"""
|
||||
async with async_playwright() as p:
|
||||
browser: Browser = await p.chromium.launch(headless=self.headless)
|
||||
page = await browser.new_page()
|
||||
await page.goto(PUBLIC_URLS.login)
|
||||
logging.info("Login page loaded: %s", page.url)
|
||||
|
||||
await self._login(page)
|
||||
await browser.close()
|
||||
|
||||
if not self.cookies:
|
||||
raise RuntimeError("Login failed – no cookies retrieved.")
|
||||
|
||||
return self._to_cookiejar(self.cookies), self.sesskey
|
||||
|
||||
@classmethod
|
||||
def crawl_sync(cls, **kwargs) -> tuple[Cookies, str]:
|
||||
"""
|
||||
Synchronous helper for CLI / notebooks.
|
||||
|
||||
Detects whether an event loop is already running. If so, it
|
||||
schedules the coroutine and waits; otherwise it starts a fresh loop.
|
||||
"""
|
||||
self = cls(**kwargs)
|
||||
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError: # no loop running → safe to create one
|
||||
return asyncio.run(self.crawl())
|
||||
|
||||
# An event loop exists – schedule coroutine
|
||||
return loop.run_until_complete(self.crawl())
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# internal helpers #
|
||||
# ------------------------------------------------------------------ #
|
||||
async def _login(self, page: Page) -> None:
|
||||
"""Fill the SSO form and extract cookies + sesskey."""
|
||||
|
||||
# Select organisation / IdP
|
||||
await page.click("#wayf_submit_button")
|
||||
|
||||
# Wait for the credential form
|
||||
await page.wait_for_selector("form[method='post']", state="visible")
|
||||
|
||||
# Credentials
|
||||
await page.fill("input[id='username']", self.username)
|
||||
await page.fill("input[id='password']", self.password)
|
||||
await page.click("button[class='aai_login_button']")
|
||||
|
||||
# Wait for redirect to /my/ page (dashboard), this means the login is complete
|
||||
await page.wait_for_url(PRIVATE_URLS.dashboard)
|
||||
await page.wait_for_selector("body", state="attached")
|
||||
|
||||
# Navigate to personal course overview
|
||||
await page.goto(PRIVATE_URLS.user_courses)
|
||||
await page.wait_for_selector("body", state="attached")
|
||||
|
||||
# Collect session cookies
|
||||
self.cookies = await page.context.cookies()
|
||||
|
||||
# Extract sesskey from injected Moodle config
|
||||
try:
|
||||
self.sesskey = await page.evaluate(
|
||||
"() => window.M && M.cfg && M.cfg.sesskey"
|
||||
)
|
||||
except Exception as exc:
|
||||
raise RuntimeError("sesskey not found via JS evaluation") from exc
|
||||
|
||||
if not self.sesskey:
|
||||
raise RuntimeError("sesskey is empty after evaluation.")
|
||||
|
||||
logging.debug("sesskey: %s", self.sesskey)
|
||||
logging.debug("cookies: %s", self.cookies)
|
||||
|
||||
# Dev convenience
|
||||
if not self.headless:
|
||||
await page.wait_for_timeout(5000)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# cookie conversion #
|
||||
# ------------------------------------------------------------------ #
|
||||
def _to_cookiejar(self, raw: List[Cookie]) -> Cookies:
|
||||
jar = Cookies()
|
||||
for c in raw:
|
||||
jar.set(
|
||||
name=c.get("name", ""),
|
||||
value=c.get("value", ""),
|
||||
domain=c.get("domain", "").lstrip("."),
|
||||
path=c.get("path", "/"),
|
||||
)
|
||||
return jar
|
||||
@@ -0,0 +1,264 @@
|
||||
"""
|
||||
librarian_scraper.crawler.crawler
|
||||
---------------------------------
|
||||
Scrapes Moodle degree programmes into CrawlData.
|
||||
• Hero images
|
||||
• Polite throttling / batching
|
||||
• Term-filter: only the latest two terms (dev)
|
||||
• USER_SPECIFIC flag to keep / drop inaccessible courses
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from datetime import timedelta
|
||||
from typing import List, Tuple
|
||||
import sys
|
||||
import asyncio
|
||||
|
||||
if sys.platform == "win32":
|
||||
# Switch from Selector to Proactor so asyncio.subprocess works
|
||||
asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
|
||||
|
||||
import httpx
|
||||
import parsel
|
||||
from librarian_core.utils.path_utils import get_cache_root
|
||||
from librarian_core.workers.base import Worker
|
||||
from prefect import get_run_logger, task
|
||||
from prefect.futures import wait
|
||||
|
||||
from librarian_scraper.constants import CRAWLER, PRIVATE_URLS, PUBLIC_URLS
|
||||
from librarian_scraper.crawler.cookie_crawler import CookieCrawler
|
||||
from librarian_scraper.models.crawl_data import (
|
||||
CrawlCourse,
|
||||
CrawlData,
|
||||
CrawlProgram,
|
||||
CrawlTerm,
|
||||
)
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# module-level shared items for static task #
|
||||
# --------------------------------------------------------------------------- #
|
||||
_COOKIE_JAR: httpx.Cookies | None = None
|
||||
_DELAY: float = 0.0
|
||||
|
||||
CACHE_FILE = get_cache_root() / "librarian_no_access_cache.json"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# utility #
|
||||
# --------------------------------------------------------------------------- #
|
||||
def looks_like_enrol(resp: httpx.Response) -> bool:
|
||||
txt = resp.text.lower()
|
||||
return (
|
||||
"login" in str(resp.url).lower()
|
||||
or "#page-enrol" in txt
|
||||
or "you need to enrol" in txt
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# main worker #
|
||||
# --------------------------------------------------------------------------- #
|
||||
class Crawler(Worker[CrawlProgram, CrawlData]):
|
||||
input_model = CrawlProgram
|
||||
output_model = CrawlData
|
||||
|
||||
# toggles (env overrides)
|
||||
RELAXED: bool
|
||||
USER_SPECIFIC: bool
|
||||
CLEAR_CACHE: bool
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# flow entry-point #
|
||||
# ------------------------------------------------------------------ #
|
||||
async def __run__(self, program: CrawlProgram) -> CrawlData:
|
||||
global _COOKIE_JAR, _DELAY
|
||||
lg = get_run_logger()
|
||||
|
||||
self.RELAXED = os.getenv("SCRAPER_RELAXED", "true").lower() == "true"
|
||||
self.USER_SPECIFIC = os.getenv("SCRAPER_USER_SPECIFIC", "true").lower() == "true"
|
||||
self.CLEAR_CACHE = os.getenv("SCRAPER_CLEAR_CACHE", "false").lower() == "true"
|
||||
|
||||
_DELAY = CRAWLER["DELAY_SLOW"] if self.RELAXED else CRAWLER["DELAY_FAST"]
|
||||
batch = CRAWLER["BATCH_SLOW"] if self.RELAXED else CRAWLER["BATCH_FAST"]
|
||||
lg.info(
|
||||
"Mode=%s user_specific=%s delay=%.1fs batch=%s",
|
||||
"RELAXED" if self.RELAXED else "FAST",
|
||||
self.USER_SPECIFIC,
|
||||
_DELAY,
|
||||
batch,
|
||||
)
|
||||
|
||||
# --------------------------- login
|
||||
cookies, _ = await CookieCrawler().crawl()
|
||||
_COOKIE_JAR = cookies
|
||||
self._client = httpx.Client(cookies=cookies, follow_redirects=True)
|
||||
|
||||
if not self._logged_in():
|
||||
lg.error("Guest session detected – aborting crawl.")
|
||||
raise RuntimeError("Login failed")
|
||||
|
||||
# --------------------------- cache
|
||||
no_access: set[str] = set() if self.CLEAR_CACHE else self._load_cache()
|
||||
|
||||
# --------------------------- scrape terms (first two for dev)
|
||||
terms = self._crawl_terms(program.id)[:2]
|
||||
lg.info("Terms discovered: %d", len(terms))
|
||||
|
||||
# --------------------------- scrape courses
|
||||
for term in terms:
|
||||
courses = self._crawl_courses(term.id)
|
||||
lg.info("[%s] raw courses: %d", term.name, len(courses))
|
||||
|
||||
for i in range(0, len(courses), batch):
|
||||
futs = [
|
||||
self._crawl_course_task.submit(course.id)
|
||||
for course in courses[i : i + batch]
|
||||
]
|
||||
done, _ = wait(futs)
|
||||
|
||||
for fut in done:
|
||||
cid, res_id = fut.result()
|
||||
if res_id:
|
||||
next(
|
||||
c for c in courses if c.id == cid
|
||||
).content_ressource_id = res_id
|
||||
else:
|
||||
no_access.add(cid)
|
||||
|
||||
term.courses = (
|
||||
[c for c in courses if c.content_ressource_id]
|
||||
if self.USER_SPECIFIC
|
||||
else courses
|
||||
)
|
||||
lg.info("[%s] kept: %d", term.name, len(term.courses))
|
||||
|
||||
# --------------------------- persist cache
|
||||
self._save_cache(no_access)
|
||||
|
||||
return CrawlData(
|
||||
degree_program=CrawlProgram(
|
||||
id=program.id,
|
||||
name=program.name,
|
||||
terms=[t for t in terms if t.courses],
|
||||
)
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# static task inside class #
|
||||
# ------------------------------------------------------------------ #
|
||||
@staticmethod
|
||||
@task(
|
||||
name="crawl_course",
|
||||
retries=2,
|
||||
retry_delay_seconds=5,
|
||||
log_prints=True,
|
||||
cache_expiration=timedelta(days=1),
|
||||
)
|
||||
def _crawl_course_task(course_id: str) -> Tuple[str, str]:
|
||||
"""
|
||||
Returns (course_id, content_resource_id or "").
|
||||
Never raises; logs reasons instead.
|
||||
"""
|
||||
lg = get_run_logger()
|
||||
assert _COOKIE_JAR is not None
|
||||
|
||||
url = PRIVATE_URLS.course(course_id)
|
||||
for attempt in (1, 2):
|
||||
try:
|
||||
r = httpx.get(
|
||||
url, cookies=_COOKIE_JAR, follow_redirects=True, timeout=30
|
||||
)
|
||||
r.raise_for_status()
|
||||
time.sleep(_DELAY)
|
||||
break
|
||||
except Exception as exc:
|
||||
lg.warning("GET %s failed (%s) attempt %d/2", url, exc, attempt)
|
||||
time.sleep(_DELAY)
|
||||
else:
|
||||
lg.warning("Course %s unreachable.", course_id)
|
||||
return course_id, ""
|
||||
|
||||
if looks_like_enrol(r):
|
||||
lg.info("No access to course %s (enrol / login page).", course_id)
|
||||
return course_id, ""
|
||||
|
||||
href = (
|
||||
parsel.Selector(r.text)
|
||||
.css('a[data-downloadcourse="1"]::attr(href)')
|
||||
.get("")
|
||||
)
|
||||
if not href:
|
||||
lg.info("Course %s has no downloadable content.", course_id)
|
||||
return course_id, ""
|
||||
|
||||
return course_id, href.split("=")[-1]
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# helpers #
|
||||
# ------------------------------------------------------------------ #
|
||||
def _logged_in(self) -> bool:
|
||||
html = self._get_html(PUBLIC_URLS.index)
|
||||
return not parsel.Selector(text=html).css("div.usermenu span.login a")
|
||||
|
||||
def _crawl_terms(self, dp_id: str) -> List[CrawlTerm]:
|
||||
html = self._get_html(PUBLIC_URLS.degree_program(dp_id))
|
||||
sel = parsel.Selector(text=html)
|
||||
out = []
|
||||
for a in sel.css("div.category h3.categoryname a"):
|
||||
name = a.xpath("text()").get("").strip()
|
||||
if re.match(r"^(FS|HS)\d{2}$", name):
|
||||
out.append(
|
||||
CrawlTerm(name=name, id=a.xpath("@href").get("").split("=")[-1])
|
||||
)
|
||||
order = {"FS": 0, "HS": 1}
|
||||
return sorted(
|
||||
out, key=lambda t: (2000 + int(t.name[2:]), order[t.name[:2]]), reverse=True
|
||||
)
|
||||
|
||||
def _crawl_courses(self, term_id: str) -> List[CrawlCourse]:
|
||||
html = self._get_html(PUBLIC_URLS.term(term_id))
|
||||
sel = parsel.Selector(text=html)
|
||||
courses = []
|
||||
for box in sel.css("div.coursebox"):
|
||||
anchor = box.css("h3.coursename a")
|
||||
if not anchor:
|
||||
continue
|
||||
cid = anchor.attrib.get("href", "").split("=")[-1]
|
||||
raw = anchor.xpath("text()").get("").strip()
|
||||
name = re.sub(r"\s*(FS|HS)\d{2}\s*", "", raw)
|
||||
name = re.sub(r"\s*\(.*?\)\s*", "", name).strip()
|
||||
hero = box.css("div.courseimage img::attr(src)").get("") or ""
|
||||
courses.append(CrawlCourse(id=cid, name=name, hero_image=hero))
|
||||
return courses
|
||||
|
||||
def _get_html(self, url: str) -> str:
|
||||
try:
|
||||
r = self._client.get(url, timeout=30)
|
||||
r.raise_for_status()
|
||||
time.sleep(_DELAY)
|
||||
return r.text
|
||||
except Exception as exc:
|
||||
get_run_logger().warning("GET %s failed (%s)", url, exc)
|
||||
return ""
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# cache helpers #
|
||||
# ------------------------------------------------------------------ #
|
||||
@staticmethod
|
||||
def _load_cache() -> set[str]:
|
||||
try:
|
||||
return set(json.loads(CACHE_FILE.read_text()))
|
||||
except Exception:
|
||||
return set()
|
||||
|
||||
@staticmethod
|
||||
def _save_cache(cache: set[str]) -> None:
|
||||
try:
|
||||
CACHE_FILE.write_text(json.dumps(sorted(cache), indent=2))
|
||||
except Exception as exc:
|
||||
get_run_logger().warning("Could not save cache: %s", exc)
|
||||
@@ -0,0 +1,357 @@
|
||||
import concurrent.futures
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
import parsel
|
||||
from librarian_core.model import Course, DegreeProgram, FileEntry, MoodleIndex, Semester
|
||||
|
||||
from . import URLs
|
||||
|
||||
CACHE_FILENAME = "librarian_no_access_cache.json"
|
||||
NO_ACCESS_CACHE_FILE = Path(tempfile.gettempdir()) / CACHE_FILENAME
|
||||
|
||||
|
||||
class IndexCrawler:
|
||||
def __init__(self, degree_program: DegreeProgram, cookies: httpx.Cookies, debug: bool = False, *, max_workers: int = 8) -> None:
|
||||
self.degree_program = degree_program
|
||||
self.debug = debug
|
||||
self.client = httpx.Client(cookies=cookies, follow_redirects=True)
|
||||
self.max_workers = max_workers
|
||||
|
||||
# When True the cached “no-access” set is ignored for this run
|
||||
self._ignore_cache: bool = False
|
||||
|
||||
# Load persisted cache of course-IDs the user cannot access
|
||||
if NO_ACCESS_CACHE_FILE.exists():
|
||||
try:
|
||||
self._no_access_cache: set[str] = set(json.loads(NO_ACCESS_CACHE_FILE.read_text()))
|
||||
except Exception:
|
||||
logging.warning("Failed to read no-access cache, starting fresh.")
|
||||
self._no_access_cache = set()
|
||||
else:
|
||||
self._no_access_cache = set()
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc_val, exc_tb):
|
||||
self.client.close()
|
||||
|
||||
def __del__(self):
|
||||
# Fallback in case the context manager isn’t used
|
||||
if not self.client.is_closed:
|
||||
self.client.close()
|
||||
|
||||
"""
|
||||
Crawl a single instance of MoodleIndex.
|
||||
This returns a MoodleIndex object populated with data.
|
||||
"""
|
||||
|
||||
def crawl_index(self, userSpecific: bool = True, *, use_cache: bool = True) -> MoodleIndex:
|
||||
"""
|
||||
Build and return a `MoodleIndex`.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
userSpecific : bool
|
||||
When True, include only courses that expose a downloadable content resource.
|
||||
use_cache : bool, default True
|
||||
If False, bypass the persisted “no-access” cache so every course is probed
|
||||
afresh. Newly discovered “no-access” courses are still written back to the
|
||||
cache at the end of the crawl.
|
||||
"""
|
||||
# Set runtime flag for has_user_access()
|
||||
self._ignore_cache = not use_cache
|
||||
|
||||
semesters = []
|
||||
# Get all courses for each semester and the courseid and name for each course.
|
||||
semesters = self.crawl_semesters()
|
||||
# Crawl only the latest two semesters to reduce load (remove once caching is implemented)
|
||||
for semester in semesters[:2]:
|
||||
courses = self.crawl_courses(semester)
|
||||
|
||||
# Crawl courses in parallel to speed things up
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as pool:
|
||||
list(pool.map(self.crawl_course, courses))
|
||||
|
||||
# Filter courses once all have been processed
|
||||
for course in courses:
|
||||
if userSpecific:
|
||||
if course.content_ressource_id:
|
||||
semester.courses.append(course)
|
||||
else:
|
||||
semester.courses.append(course)
|
||||
|
||||
# Only add semesters that have at least one course
|
||||
# Filter out semesters that ended up with no courses after crawling
|
||||
semesters: list[Semester] = [
|
||||
semester for semester in semesters if semester.courses
|
||||
]
|
||||
|
||||
created_index = MoodleIndex(
|
||||
degree_program=DegreeProgram(
|
||||
name=self.degree_program.name,
|
||||
id=self.degree_program.id,
|
||||
semesters=semesters,
|
||||
),
|
||||
)
|
||||
# Persist any newly discovered no-access courses
|
||||
self._save_no_access_cache()
|
||||
|
||||
# Restore default behaviour for subsequent calls
|
||||
self._ignore_cache = False
|
||||
|
||||
return created_index
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
# High-level crawling helpers
|
||||
# --------------------------------------------------------------------- #
|
||||
def crawl_semesters(self) -> list[Semester]:
|
||||
"""
|
||||
Crawl the semesters from the Moodle index page.
|
||||
"""
|
||||
url = URLs.get_degree_program_url(self.degree_program.id)
|
||||
res = self.get_with_retries(url)
|
||||
|
||||
if res.status_code == 200:
|
||||
semesters = self.extract_semesters(res.text)
|
||||
logging.debug(f"Found semesters: {semesters}")
|
||||
return semesters
|
||||
|
||||
return []
|
||||
|
||||
def crawl_courses(self, semester: Semester) -> list[Course]:
|
||||
"""
|
||||
Crawl the courses from the Moodle index page.
|
||||
"""
|
||||
url = URLs.get_semester_url(semester_id=semester.id)
|
||||
res = self.get_with_retries(url)
|
||||
|
||||
if res.status_code == 200:
|
||||
courses = self.extract_courses(res.text)
|
||||
logging.debug(f"Found courses: {courses}")
|
||||
return courses
|
||||
|
||||
return []
|
||||
|
||||
def crawl_course(self, course: Course) -> None:
|
||||
"""
|
||||
Crawl a single Moodle course page.
|
||||
"""
|
||||
|
||||
hasAccess = self.has_user_access(course)
|
||||
|
||||
if not hasAccess:
|
||||
return
|
||||
|
||||
# TODO: Cache which courses the user has no access to, to avoid repeated requests
|
||||
|
||||
course.content_ressource_id = self.crawl_content_ressource_id(course)
|
||||
course.files = self.crawl_course_files(course)
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
# Networking utilities
|
||||
# --------------------------------------------------------------------- #
|
||||
def get_with_retries(self, url: str, retries: int = 3, delay: int = 1) -> httpx.Response:
|
||||
"""
|
||||
Simple GET with retries and exponential back-off.
|
||||
"""
|
||||
for attempt in range(1, retries + 1):
|
||||
try:
|
||||
response = self.client.get(url)
|
||||
|
||||
response.raise_for_status()
|
||||
|
||||
return response
|
||||
except Exception as e:
|
||||
logging.warning(f"Request to {url} failed ({e}), attempt {attempt}/{retries}")
|
||||
if attempt < retries:
|
||||
time.sleep(delay * (2 ** (attempt - 1)))
|
||||
raise Exception(f"Failed to GET {url} after {retries} attempts")
|
||||
|
||||
def save_html(self, url: str, response: httpx.Response) -> None:
|
||||
"""
|
||||
Persist raw HTML locally for debugging.
|
||||
"""
|
||||
filename = url.split("/")[-1] + ".html"
|
||||
with open(filename, "w", encoding="utf-8") as f:
|
||||
f.write(response.text)
|
||||
logging.info(f"Saved HTML to {filename}")
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
# Extractors
|
||||
# --------------------------------------------------------------------- #
|
||||
def extract_semesters(self, html: str) -> list[Semester]:
|
||||
selector = parsel.Selector(text=html)
|
||||
|
||||
logging.info("Extracting semesters from the HTML content.")
|
||||
|
||||
semesters: list[Semester] = []
|
||||
|
||||
# Each semester sits in a collapsed container
|
||||
semester_containers = selector.css("div.category.notloaded.with_children.collapsed")
|
||||
|
||||
for container in semester_containers:
|
||||
anchor = container.css("h3.categoryname.aabtn a")
|
||||
if not anchor:
|
||||
continue
|
||||
|
||||
anchor = anchor[0]
|
||||
semester_name = (
|
||||
anchor.xpath("text()").get("").replace("\n", "").replace("\t", "").strip()
|
||||
)
|
||||
semester_id = anchor.attrib.get("href", "").split("=")[-1]
|
||||
|
||||
# Only keep semesters labeled FS or HS
|
||||
if "FS" not in semester_name and "HS" not in semester_name:
|
||||
continue
|
||||
|
||||
semesters.append(Semester(name=semester_name, id=semester_id))
|
||||
|
||||
semester_order = {
|
||||
"FS": 0, # Frühjahrs‐/Spring Semester
|
||||
"HS": 1, # Herbst‐/Fall Semester
|
||||
}
|
||||
# Sort by year and then by FS before HS
|
||||
sorted_semesters = sorted(
|
||||
semesters,
|
||||
key=lambda s: (
|
||||
2000 + int(s.name[2:]), # parse "25" → int 25, add 2000 → 2025
|
||||
semester_order[s.name[:2]] # map "FS" → 0, "HS" → 1
|
||||
),
|
||||
reverse=True,
|
||||
)
|
||||
return sorted_semesters
|
||||
|
||||
def extract_courses(self, html: str) -> list[Course]:
|
||||
"""
|
||||
Parse courses and capture optional “hero_image” (overview image) if present.
|
||||
"""
|
||||
selector = parsel.Selector(text=html)
|
||||
|
||||
logging.info("Extracting courses from the HTML content.")
|
||||
|
||||
courses: list[Course] = []
|
||||
|
||||
for header in selector.css("h3.coursename"):
|
||||
anchor = header.css("a")
|
||||
if not anchor:
|
||||
logging.warning("No course anchor found in the course header.")
|
||||
continue
|
||||
|
||||
anchor = anchor[0]
|
||||
course_name = (
|
||||
anchor.xpath("text()").get("").replace("\n", "").replace("\t", "").strip()
|
||||
)
|
||||
course_id = anchor.attrib.get("href", "").split("=")[-1]
|
||||
|
||||
# Remove trailing semester tag and code patterns
|
||||
course_name = re.sub(r"\s*(FS|HS)\d{2}\s*", "", course_name)
|
||||
course_name = re.sub(r"\s*\(.*?\)\s*", "", course_name).strip()
|
||||
|
||||
# Try to locate a hero/overview image that belongs to this course box
|
||||
# Traverse up to the containing course box, then look for <div class="courseimage"><img ...>
|
||||
course_container = header.xpath('./ancestor::*[contains(@class,"coursebox")][1]')
|
||||
hero_src = (
|
||||
course_container.css("div.courseimage img::attr(src)").get("")
|
||||
if course_container else ""
|
||||
)
|
||||
|
||||
courses.append(
|
||||
Course(
|
||||
id=course_id,
|
||||
name=course_name,
|
||||
activity_type="", # TODO: Make optional
|
||||
hero_image=hero_src or ""
|
||||
)
|
||||
)
|
||||
|
||||
logging.info(f"{len(courses)} courses extracted.")
|
||||
return courses
|
||||
|
||||
def has_user_access(self, course: Course) -> bool:
|
||||
"""
|
||||
Return True only if the authenticated user can access the course (result cached).
|
||||
(i.e. the response is HTTP 200 **and** is not a redirected login/enrol page).
|
||||
"""
|
||||
if not self._ignore_cache and course.id in self._no_access_cache:
|
||||
return False
|
||||
|
||||
url = URLs.get_course_url(course.id)
|
||||
res = self.get_with_retries(url)
|
||||
|
||||
if res.status_code != 200:
|
||||
self._no_access_cache.add(course.id)
|
||||
return False
|
||||
|
||||
# Detect Moodle redirection to a login or enrolment page
|
||||
final_url = str(res.url).lower()
|
||||
if "login" in final_url or "enrol" in final_url:
|
||||
self._no_access_cache.add(course.id)
|
||||
return False
|
||||
|
||||
# Some enrolment pages still return 200; look for HTML markers
|
||||
if "#page-enrol" in res.text or "you need to enrol" in res.text.lower():
|
||||
self._no_access_cache.add(course.id)
|
||||
return False
|
||||
|
||||
# If we got here the user has access; otherwise cache the deny
|
||||
return True
|
||||
|
||||
def crawl_content_ressource_id(self, course: Course) -> str:
|
||||
course_id = course.id
|
||||
url = URLs.get_course_url(course_id)
|
||||
res = self.get_with_retries(url)
|
||||
psl = parsel.Selector(res.text)
|
||||
|
||||
try:
|
||||
logging.info("Searching for 'Download course content' link.")
|
||||
# Use parsel CSS selector to find the anchor tag with the specific data attribute
|
||||
download_link_selector = psl.css('a[data-downloadcourse="1"]')
|
||||
if not download_link_selector:
|
||||
raise ValueError("Download link not found.")
|
||||
|
||||
# Extract the href attribute from the first matching element
|
||||
href = download_link_selector[0].attrib.get("href")
|
||||
if not href:
|
||||
raise ValueError("Href attribute not found on the download link.")
|
||||
|
||||
context_id = href.split("=")[-1]
|
||||
course.content_ressource_id = context_id
|
||||
|
||||
return context_id
|
||||
except Exception as e:
|
||||
logging.error(
|
||||
f"Error extracting content resource ID for course '{course.name}': {e}",
|
||||
exc_info=False,
|
||||
)
|
||||
logging.debug("Debugging info: Error accessing course content.", exc_info=True)
|
||||
return ''
|
||||
|
||||
def crawl_course_files(self, course: Course) -> list[FileEntry]:
|
||||
"""
|
||||
Crawl the course files from the Moodle course page.
|
||||
"""
|
||||
url = URLs.get_course_url(course.id)
|
||||
res = self.get_with_retries(url)
|
||||
|
||||
if res.status_code == 200:
|
||||
files = [] # TODO: either implement this or remove, because files are extracted from the .zip file
|
||||
logging.debug(f"Found files: {files}")
|
||||
return files
|
||||
|
||||
return []
|
||||
|
||||
# ----------------------------------------------------------------- #
|
||||
# Cache persistence helpers
|
||||
# ----------------------------------------------------------------- #
|
||||
def _save_no_access_cache(self) -> None:
|
||||
try:
|
||||
NO_ACCESS_CACHE_FILE.write_text(json.dumps(sorted(self._no_access_cache)))
|
||||
except Exception as exc:
|
||||
logging.warning(f"Could not persist no-access cache: {exc}")
|
||||
Whitespace-only changes.
@@ -0,0 +1,59 @@
|
||||
# TODO: Move to librarian-core
|
||||
"""
|
||||
All URLs used in the crawler.
|
||||
Functions marked as PUBLIC can be accessed without authentication.
|
||||
Functions marked as PRIVATE require authentication.
|
||||
"""
|
||||
class URLs:
|
||||
base_url = "https://moodle.fhgr.ch"
|
||||
|
||||
@classmethod
|
||||
def get_base_url(cls):
|
||||
"""PUBLIC"""
|
||||
return cls.base_url
|
||||
|
||||
# ------------------------- Moodle URLs -------------------------
|
||||
@classmethod
|
||||
def get_login_url(cls):
|
||||
"""PUBLIC"""
|
||||
return f"{cls.base_url}/login/index.php"
|
||||
|
||||
@classmethod
|
||||
def get_index_url(cls):
|
||||
"""PUBLIC"""
|
||||
return f"{cls.base_url}/course/index.php"
|
||||
|
||||
@classmethod
|
||||
def get_degree_program_url(cls, degree_program_id):
|
||||
"""PUBLIC"""
|
||||
return f"{cls.base_url}/course/index.php?categoryid={degree_program_id}"
|
||||
|
||||
@classmethod
|
||||
def get_category_url(cls, category_id):
|
||||
"""PUBLIC"""
|
||||
return f"{cls.base_url}/course/index.php?categoryid={category_id}"
|
||||
|
||||
@classmethod
|
||||
def get_semester_url(cls, semester_id):
|
||||
"""PUBLIC"""
|
||||
return f"{cls.base_url}/course/index.php?categoryid={semester_id}"
|
||||
|
||||
@classmethod
|
||||
def get_user_courses_url(cls):
|
||||
"""PRIVATE"""
|
||||
return f"{cls.base_url}/my/courses.php"
|
||||
|
||||
@classmethod
|
||||
def get_course_url(cls, course_id):
|
||||
"""PRIVATE"""
|
||||
return f"{cls.base_url}/course/view.php?id={course_id}"
|
||||
|
||||
@classmethod
|
||||
def get_files_url(cls, context_id):
|
||||
"""PRIVATE"""
|
||||
return f"{cls.base_url}/course/downloadcontent.php?contextid={context_id}"
|
||||
|
||||
@classmethod
|
||||
def get_file_url(cls, file_id):
|
||||
"""PRIVATE"""
|
||||
return f"{cls.base_url}/mod/resource/view.php?id={file_id}"
|
||||
@@ -0,0 +1,5 @@
|
||||
from .downloader import *
|
||||
|
||||
__all__ = [
|
||||
"Downloader",
|
||||
]
|
||||
@@ -0,0 +1,151 @@
|
||||
"""
|
||||
Downloader Worker
|
||||
=================
|
||||
Input : CrawlData (from the crawler)
|
||||
Output : DownloadData (metadata only; files staged)
|
||||
|
||||
Folder tree after run
|
||||
---------------------
|
||||
export_dir/
|
||||
└─ {TERM_NAME}/
|
||||
├─ {course_id}.zip
|
||||
└─ …
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import List, Tuple
|
||||
|
||||
import httpx
|
||||
from librarian_core.utils.path_utils import get_temp_path
|
||||
from librarian_core.workers.base import Worker
|
||||
from prefect import get_run_logger, task
|
||||
from prefect.futures import wait
|
||||
|
||||
from librarian_scraper.constants import CRAWLER
|
||||
from librarian_scraper.crawler.cookie_crawler import CookieCrawler
|
||||
from librarian_scraper.models.crawl_data import CrawlData
|
||||
from librarian_scraper.models.download_data import (
|
||||
DownloadCourse,
|
||||
DownloadData,
|
||||
DownloadTerm,
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# helper decorator #
|
||||
# --------------------------------------------------------------------------- #
|
||||
def task_(**kw):
|
||||
kw.setdefault("log_prints", True)
|
||||
kw.setdefault("retries", 2)
|
||||
kw.setdefault("retry_delay_seconds", 5)
|
||||
return task(**kw)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# shared state for static task #
|
||||
# --------------------------------------------------------------------------- #
|
||||
_COOKIE_JAR: httpx.Cookies | None = None
|
||||
_SESSKEY: str = ""
|
||||
_LIMIT: int = 2
|
||||
_DELAY: float = 0.0
|
||||
|
||||
|
||||
class Downloader(Worker[CrawlData, DownloadData]):
|
||||
DOWNLOAD_URL = "https://moodle.fhgr.ch/course/downloadcontent.php"
|
||||
|
||||
# tuning
|
||||
CONCURRENCY = 8
|
||||
RELAXED = True # False → faster
|
||||
|
||||
input_model = CrawlData
|
||||
output_model = DownloadData
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
async def __run__(self, crawl: CrawlData) -> DownloadData:
|
||||
global _COOKIE_JAR, _SESSKEY, _LIMIT, _DELAY
|
||||
lg = get_run_logger()
|
||||
|
||||
# ------------ login
|
||||
cookies, sesskey = await CookieCrawler().crawl()
|
||||
_COOKIE_JAR, _SESSKEY = cookies, sesskey
|
||||
|
||||
# ------------ tuning
|
||||
_LIMIT = 1 if self.RELAXED else max(1, min(self.CONCURRENCY, 8))
|
||||
_DELAY = CRAWLER["DELAY_SLOW"] if self.RELAXED else CRAWLER["DELAY_FAST"]
|
||||
|
||||
# ------------ working dir
|
||||
work_root = Path(get_temp_path()) / f"dl_{int(time.time())}"
|
||||
work_root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
result = DownloadData()
|
||||
futures = []
|
||||
term_dirs: List[Tuple[str, Path]] = []
|
||||
|
||||
# schedule downloads
|
||||
for term in crawl.degree_program.terms:
|
||||
term_dir = work_root / term.name
|
||||
term_dir.mkdir(parents=True, exist_ok=True)
|
||||
term_dirs.append((term.name, term_dir))
|
||||
|
||||
dl_term = DownloadTerm(id=term.id, name=term.name)
|
||||
result.terms.append(dl_term)
|
||||
|
||||
for course in term.courses:
|
||||
dest = term_dir / f"{course.id}.zip"
|
||||
dl_term.courses.append(DownloadCourse(id=course.id, name=course.name))
|
||||
futures.append(
|
||||
self._download_task.submit(course.content_ressource_id, dest)
|
||||
)
|
||||
|
||||
wait(futures) # block for all downloads
|
||||
|
||||
# stage term directories
|
||||
for name, dir_path in term_dirs:
|
||||
self.stage(dir_path, new_name=name, sanitize=False, move=True)
|
||||
|
||||
lg.info("Downloader finished – staged %d term folders", len(term_dirs))
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# static task #
|
||||
# ------------------------------------------------------------------ #
|
||||
@staticmethod
|
||||
@task_()
|
||||
def _download_task(context_id: str, dest: Path) -> None:
|
||||
lg = get_run_logger()
|
||||
if not context_id:
|
||||
lg.info("Skip (no context id) → %s", dest.name)
|
||||
return
|
||||
|
||||
async def fetch() -> bool:
|
||||
sem = asyncio.Semaphore(_LIMIT)
|
||||
|
||||
async with sem:
|
||||
data = {"sesskey": _SESSKEY, "download": 1, "contextid": context_id}
|
||||
async with httpx.AsyncClient(cookies=_COOKIE_JAR) as cli:
|
||||
try:
|
||||
async with cli.stream(
|
||||
"POST", Downloader.DOWNLOAD_URL, data=data, timeout=60
|
||||
) as r:
|
||||
r.raise_for_status()
|
||||
with dest.open("wb") as fh:
|
||||
async for chunk in r.aiter_bytes():
|
||||
fh.write(chunk)
|
||||
lg.info("Downloaded %s", dest)
|
||||
return True
|
||||
except httpx.HTTPStatusError as exc:
|
||||
lg.warning(
|
||||
"HTTP %s for %s", exc.response.status_code, dest.name
|
||||
)
|
||||
except Exception as exc:
|
||||
lg.warning("Error downloading %s (%s)", dest.name, exc)
|
||||
return False
|
||||
|
||||
ok = asyncio.run(fetch())
|
||||
if not ok and dest.exists():
|
||||
dest.unlink(missing_ok=True)
|
||||
time.sleep(_DELAY)
|
||||
Whitespace-only changes.
@@ -0,0 +1,24 @@
|
||||
from librarian_scraper.models.crawl_data import (
|
||||
CrawlCourse,
|
||||
CrawlData,
|
||||
CrawlFile,
|
||||
CrawlProgram,
|
||||
CrawlTerm,
|
||||
)
|
||||
|
||||
from librarian_scraper.models.download_data import (
|
||||
DownloadCourse,
|
||||
DownloadData,
|
||||
DownloadTerm,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"CrawlData",
|
||||
"CrawlCourse",
|
||||
"CrawlFile",
|
||||
"CrawlProgram",
|
||||
"CrawlTerm",
|
||||
"DownloadData",
|
||||
"DownloadCourse",
|
||||
"DownloadTerm",
|
||||
]
|
||||
@@ -0,0 +1,188 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
"""
|
||||
Example of a MoodleIndex (JSON):
|
||||
MoodleIndex: {
|
||||
degree_program: {
|
||||
id: '1157',
|
||||
name: 'Computational and Data Science',
|
||||
terms: [
|
||||
{
|
||||
id: '1745',
|
||||
name: 'FS25',
|
||||
courses: [
|
||||
{
|
||||
id: '18863',
|
||||
name: 'Programmierung und Prompt Engineering II',
|
||||
activity_type: '',
|
||||
hero_image:
|
||||
'https://moodle.fhgr.ch/pluginfile.php/1159522/course/overviewfiles/PythonBooks.PNG',
|
||||
content_ressource_id: '1159522',
|
||||
files: [],
|
||||
},
|
||||
{
|
||||
id: '18240',
|
||||
name: 'Effiziente Algorithmen',
|
||||
activity_type: '',
|
||||
hero_image: '',
|
||||
content_ressource_id: '1125554',
|
||||
files: [],
|
||||
},
|
||||
{
|
||||
id: '18237',
|
||||
name: 'Mathematik II',
|
||||
activity_type: '',
|
||||
hero_image:
|
||||
'https://moodle.fhgr.ch/pluginfile.php/1125458/course/overviewfiles/Integration_Differential_b.png',
|
||||
content_ressource_id: '1125458',
|
||||
files: [],
|
||||
},
|
||||
{
|
||||
id: '18236',
|
||||
name: '2025 FS FHGR CDS Numerische Methoden',
|
||||
activity_type: '',
|
||||
hero_image: '',
|
||||
content_ressource_id: '1125426',
|
||||
files: [],
|
||||
},
|
||||
{
|
||||
id: '18228',
|
||||
name: 'Datenbanken und Datenverarbeitung',
|
||||
activity_type: '',
|
||||
hero_image: '',
|
||||
content_ressource_id: '1125170',
|
||||
files: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
{
|
||||
id: '1746',
|
||||
name: 'HS24',
|
||||
courses: [
|
||||
{
|
||||
id: '18030',
|
||||
name: 'Bootcamp Wissenschaftliches Arbeiten',
|
||||
activity_type: '',
|
||||
hero_image: '',
|
||||
content_ressource_id: '1090544',
|
||||
files: [],
|
||||
},
|
||||
{
|
||||
id: '17527',
|
||||
name: 'Einführung in Data Science',
|
||||
activity_type: '',
|
||||
hero_image:
|
||||
'https://moodle.fhgr.ch/pluginfile.php/1059194/course/overviewfiles/cds1010.jpg',
|
||||
content_ressource_id: '1059194',
|
||||
files: [],
|
||||
},
|
||||
{
|
||||
id: '17526',
|
||||
name: 'Einführung in Computational Science',
|
||||
activity_type: '',
|
||||
hero_image:
|
||||
'https://moodle.fhgr.ch/pluginfile.php/1059162/course/overviewfiles/cds_intro_sim.jpg',
|
||||
content_ressource_id: '1059162',
|
||||
files: [],
|
||||
},
|
||||
{
|
||||
id: '17525',
|
||||
name: 'Mathematik I',
|
||||
activity_type: '',
|
||||
hero_image:
|
||||
'https://moodle.fhgr.ch/pluginfile.php/1059130/course/overviewfiles/AdobeStock_452512134.png',
|
||||
content_ressource_id: '1059130',
|
||||
files: [],
|
||||
},
|
||||
{
|
||||
id: '17507',
|
||||
name: 'Programmierung und Prompt Engineering',
|
||||
activity_type: '',
|
||||
hero_image:
|
||||
'https://moodle.fhgr.ch/pluginfile.php/1058554/course/overviewfiles/10714013_33861.jpg',
|
||||
content_ressource_id: '1058554',
|
||||
files: [],
|
||||
},
|
||||
{
|
||||
id: '17505',
|
||||
name: 'Algorithmen und Datenstrukturen',
|
||||
activity_type: '',
|
||||
hero_image:
|
||||
'https://moodle.fhgr.ch/pluginfile.php/1058490/course/overviewfiles/Bild1.png',
|
||||
content_ressource_id: '1058490',
|
||||
files: [],
|
||||
},
|
||||
{
|
||||
id: '17503',
|
||||
name: 'Computer Science',
|
||||
activity_type: '',
|
||||
hero_image:
|
||||
'https://moodle.fhgr.ch/pluginfile.php/1058426/course/overviewfiles/Titelbild.jpg',
|
||||
content_ressource_id: '1058426',
|
||||
files: [],
|
||||
},
|
||||
],
|
||||
},
|
||||
],
|
||||
},
|
||||
timestamp: '2025-04-27T14:20:11.354825+00:00',
|
||||
};
|
||||
"""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Base Model
|
||||
# ---------------------------------------------------------------------------
|
||||
class CrawlData(BaseModel):
|
||||
degree_program: CrawlProgram = Field(
|
||||
default_factory=lambda: CrawlProgram(id="", name="")
|
||||
)
|
||||
timestamp: str = Field(
|
||||
default_factory=lambda: datetime.now(timezone.utc).isoformat()
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Degree Program
|
||||
# ---------------------------------------------------------------------------
|
||||
class CrawlProgram(BaseModel):
|
||||
id: str = Field("1157", description="Unique identifier for the degree program.")
|
||||
name: str = Field("Computational and Data Science", description="Name of the degree program.")
|
||||
terms: list[CrawlTerm] = Field(default_factory=list)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Term
|
||||
# ---------------------------------------------------------------------------
|
||||
_TERM_RE = re.compile(r"^(HS|FS)\d{2}$") # HS24 / FS25 …
|
||||
|
||||
|
||||
class CrawlTerm(BaseModel):
|
||||
id: str
|
||||
name: str = Field(..., pattern=_TERM_RE.pattern) # e.g. “HS24”
|
||||
courses: list[CrawlCourse] = Field(default_factory=list)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Course
|
||||
# ---------------------------------------------------------------------------
|
||||
class CrawlCourse(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
hero_image: str = ""
|
||||
content_ressource_id: str = ""
|
||||
files: list[CrawlFile] = Field(default_factory=list)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Files
|
||||
# ---------------------------------------------------------------------------
|
||||
class CrawlFile(BaseModel):
|
||||
id: str
|
||||
res_id: str
|
||||
name: str
|
||||
@@ -0,0 +1,18 @@
|
||||
from typing import List
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class DownloadCourse(BaseModel):
|
||||
id: str
|
||||
name: str # Stores the name of the zip file inside the term directory
|
||||
|
||||
|
||||
class DownloadTerm(BaseModel):
|
||||
id: str
|
||||
name: str # Stores the name of the term directory inside DownloadMeta.dir
|
||||
courses: List[DownloadCourse] = Field(default_factory=list)
|
||||
|
||||
|
||||
class DownloadData(BaseModel):
|
||||
terms: List[DownloadTerm] = Field(default_factory=list)
|
||||
Whitespace-only changes.
+1426
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user