Initialize Monorepo

This commit is contained in:
DotNaos committed 2025-05-24 12:15:48 +02:00
commit f80792d739
106 files changed
+17076

No files matched your search

@@ -0,0 +1 @@
# Librarian Scraper
@@ -0,0 +1,41 @@
[project]
name = "librarian-scraper"
version = "0.2.1"
description = "FastAPI gateway and runtime pipeline for Librarian"
readme = "README.md"
authors = [{ name = "DotNaos", email = "schuetzoliver00@gmail.com" }]
requires-python = ">=3.10"
dependencies = [
"importlib_metadata; python_version<'3.10'",
"playwright>=1.51.0",
"dotenv>=0.9.9",
"parsel>=1.10.0",
"librarian-core",
"httpx>=0.28.1",
]
[build-system]
requires = ["hatchling>=1.21"]
build-backend = "hatchling.build"
[tool.hatch.build.targets.wheel]
packages = ["src/librarian_scraper"]
[tool.hatch.metadata]
allow-direct-references = true
[tool.uv.sources]
#librarian-core = { git = "https://github.com/DotNaos/librarian-core", rev = "dev" }
[project.entry-points."librarian.workers"]
crawler = "librarian_scraper.crawler:Crawler"
downloader = "librarian_scraper.downloader:Downloader"
# ───────── optional: dev / test extras ─────────
[project.optional-dependencies]
dev = ["ruff", "pytest", "mypy"]
[project.scripts]
example = "examples.app:app"
@@ -0,0 +1,12 @@
from .crawler import (
Crawler,
)
from .downloader import (
Downloader,
)
__all__ = [
"Crawler",
"Downloader",
"Extractor",
]
@@ -0,0 +1,29 @@
"""
URLs used by the scraper.
Functions marked as PUBLIC can be accessed without authentication.
Functions marked as PRIVATE require authentication.
"""
BASE_URL = "https://moodle.fhgr.ch"
CRAWLER = {
"DELAY_SLOW": 2.0,
"DELAY_FAST": 0.5,
"BATCH_SLOW": 2,
"BATCH_FAST": 8,
}
class PUBLIC_URLS:
base_url = BASE_URL
login = f"{BASE_URL}/login/index.php"
index = f"{BASE_URL}/course/index.php"
degree_program = lambda degree_program_id: f"{BASE_URL}/course/index.php?categoryid={degree_program_id}"
category = lambda category_id: f"{BASE_URL}/course/index.php?categoryid={category_id}"
term = lambda term_id: f"{BASE_URL}/course/index.php?categoryid={term_id}"
class PRIVATE_URLS:
user_courses = f"{BASE_URL}/my/courses.php"
dashboard = f"{BASE_URL}/my/"
course = lambda course_id: f"{BASE_URL}/course/view.php?id={course_id}"
files = lambda context_id: f"{BASE_URL}/course/downloadcontent.php?contextid={context_id}"
file = lambda file_id: f"{BASE_URL}/mod/resource/view.php?id={file_id}"
@@ -0,0 +1,7 @@
from librarian_scraper.crawler.cookie_crawler import CookieCrawler
from librarian_scraper.crawler.crawler import Crawler
__all__ = [
"CookieCrawler",
"Crawler",
]
@@ -0,0 +1,138 @@
from __future__ import annotations
import asyncio
import logging
import os
from typing import List, Optional
from httpx import Cookies
from playwright.async_api import Browser, Cookie, Page, async_playwright
from librarian_scraper.constants import PRIVATE_URLS, PUBLIC_URLS
class CookieCrawler:
"""
Retrieve Moodle session cookies + sesskey via Playwright.
Usage
-----
>>> crawler = CookieCrawler()
>>> cookies, sesskey = await crawler.crawl() # inside async code
# or
>>> cookies, sesskey = CookieCrawler.crawl_sync() # plain scripts
"""
# ------------------------------------------------------------------ #
# construction #
# ------------------------------------------------------------------ #
def __init__(self, *, headless: bool = True) -> None:
self.headless = headless
self.cookies: Optional[List[Cookie]] = None
self.sesskey: str = ""
self.username: str = os.getenv("MOODLE_USERNAME", "")
self.password: str = os.getenv("MOODLE_PASSWORD", "")
if not self.username or not self.password:
raise ValueError(
"Set MOODLE_USERNAME and MOODLE_PASSWORD as environment variables."
)
# ------------------------------------------------------------------ #
# public API #
# ------------------------------------------------------------------ #
async def crawl(self) -> tuple[Cookies, str]:
"""
Async entry-point – await this inside FastAPI / Prefect etc.
"""
async with async_playwright() as p:
browser: Browser = await p.chromium.launch(headless=self.headless)
page = await browser.new_page()
await page.goto(PUBLIC_URLS.login)
logging.info("Login page loaded: %s", page.url)
await self._login(page)
await browser.close()
if not self.cookies:
raise RuntimeError("Login failed – no cookies retrieved.")
return self._to_cookiejar(self.cookies), self.sesskey
@classmethod
def crawl_sync(cls, **kwargs) -> tuple[Cookies, str]:
"""
Synchronous helper for CLI / notebooks.
Detects whether an event loop is already running. If so, it
schedules the coroutine and waits; otherwise it starts a fresh loop.
"""
self = cls(**kwargs)
try:
loop = asyncio.get_running_loop()
except RuntimeError: # no loop running → safe to create one
return asyncio.run(self.crawl())
# An event loop exists – schedule coroutine
return loop.run_until_complete(self.crawl())
# ------------------------------------------------------------------ #
# internal helpers #
# ------------------------------------------------------------------ #
async def _login(self, page: Page) -> None:
"""Fill the SSO form and extract cookies + sesskey."""
# Select organisation / IdP
await page.click("#wayf_submit_button")
# Wait for the credential form
await page.wait_for_selector("form[method='post']", state="visible")
# Credentials
await page.fill("input[id='username']", self.username)
await page.fill("input[id='password']", self.password)
await page.click("button[class='aai_login_button']")
# Wait for redirect to /my/ page (dashboard), this means the login is complete
await page.wait_for_url(PRIVATE_URLS.dashboard)
await page.wait_for_selector("body", state="attached")
# Navigate to personal course overview
await page.goto(PRIVATE_URLS.user_courses)
await page.wait_for_selector("body", state="attached")
# Collect session cookies
self.cookies = await page.context.cookies()
# Extract sesskey from injected Moodle config
try:
self.sesskey = await page.evaluate(
"() => window.M && M.cfg && M.cfg.sesskey"
)
except Exception as exc:
raise RuntimeError("sesskey not found via JS evaluation") from exc
if not self.sesskey:
raise RuntimeError("sesskey is empty after evaluation.")
logging.debug("sesskey: %s", self.sesskey)
logging.debug("cookies: %s", self.cookies)
# Dev convenience
if not self.headless:
await page.wait_for_timeout(5000)
# ------------------------------------------------------------------ #
# cookie conversion #
# ------------------------------------------------------------------ #
def _to_cookiejar(self, raw: List[Cookie]) -> Cookies:
jar = Cookies()
for c in raw:
jar.set(
name=c.get("name", ""),
value=c.get("value", ""),
domain=c.get("domain", "").lstrip("."),
path=c.get("path", "/"),
)
return jar
@@ -0,0 +1,264 @@
"""
librarian_scraper.crawler.crawler
---------------------------------
Scrapes Moodle degree programmes into CrawlData.
• Hero images
• Polite throttling / batching
• Term-filter: only the latest two terms (dev)
• USER_SPECIFIC flag to keep / drop inaccessible courses
"""
from __future__ import annotations
import json
import os
import re
import time
from datetime import timedelta
from typing import List, Tuple
import sys
import asyncio
if sys.platform == "win32":
# Switch from Selector to Proactor so asyncio.subprocess works
asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
import httpx
import parsel
from librarian_core.utils.path_utils import get_cache_root
from librarian_core.workers.base import Worker
from prefect import get_run_logger, task
from prefect.futures import wait
from librarian_scraper.constants import CRAWLER, PRIVATE_URLS, PUBLIC_URLS
from librarian_scraper.crawler.cookie_crawler import CookieCrawler
from librarian_scraper.models.crawl_data import (
CrawlCourse,
CrawlData,
CrawlProgram,
CrawlTerm,
)
# --------------------------------------------------------------------------- #
# module-level shared items for static task #
# --------------------------------------------------------------------------- #
_COOKIE_JAR: httpx.Cookies | None = None
_DELAY: float = 0.0
CACHE_FILE = get_cache_root() / "librarian_no_access_cache.json"
# --------------------------------------------------------------------------- #
# utility #
# --------------------------------------------------------------------------- #
def looks_like_enrol(resp: httpx.Response) -> bool:
txt = resp.text.lower()
return (
"login" in str(resp.url).lower()
or "#page-enrol" in txt
or "you need to enrol" in txt
)
# --------------------------------------------------------------------------- #
# main worker #
# --------------------------------------------------------------------------- #
class Crawler(Worker[CrawlProgram, CrawlData]):
input_model = CrawlProgram
output_model = CrawlData
# toggles (env overrides)
RELAXED: bool
USER_SPECIFIC: bool
CLEAR_CACHE: bool
# ------------------------------------------------------------------ #
# flow entry-point #
# ------------------------------------------------------------------ #
async def __run__(self, program: CrawlProgram) -> CrawlData:
global _COOKIE_JAR, _DELAY
lg = get_run_logger()
self.RELAXED = os.getenv("SCRAPER_RELAXED", "true").lower() == "true"
self.USER_SPECIFIC = os.getenv("SCRAPER_USER_SPECIFIC", "true").lower() == "true"
self.CLEAR_CACHE = os.getenv("SCRAPER_CLEAR_CACHE", "false").lower() == "true"
_DELAY = CRAWLER["DELAY_SLOW"] if self.RELAXED else CRAWLER["DELAY_FAST"]
batch = CRAWLER["BATCH_SLOW"] if self.RELAXED else CRAWLER["BATCH_FAST"]
lg.info(
"Mode=%s user_specific=%s delay=%.1fs batch=%s",
"RELAXED" if self.RELAXED else "FAST",
self.USER_SPECIFIC,
_DELAY,
batch,
)
# --------------------------- login
cookies, _ = await CookieCrawler().crawl()
_COOKIE_JAR = cookies
self._client = httpx.Client(cookies=cookies, follow_redirects=True)
if not self._logged_in():
lg.error("Guest session detected – aborting crawl.")
raise RuntimeError("Login failed")
# --------------------------- cache
no_access: set[str] = set() if self.CLEAR_CACHE else self._load_cache()
# --------------------------- scrape terms (first two for dev)
terms = self._crawl_terms(program.id)[:2]
lg.info("Terms discovered: %d", len(terms))
# --------------------------- scrape courses
for term in terms:
courses = self._crawl_courses(term.id)
lg.info("[%s] raw courses: %d", term.name, len(courses))
for i in range(0, len(courses), batch):
futs = [
self._crawl_course_task.submit(course.id)
for course in courses[i : i + batch]
]
done, _ = wait(futs)
for fut in done:
cid, res_id = fut.result()
if res_id:
next(
c for c in courses if c.id == cid
).content_ressource_id = res_id
else:
no_access.add(cid)
term.courses = (
[c for c in courses if c.content_ressource_id]
if self.USER_SPECIFIC
else courses
)
lg.info("[%s] kept: %d", term.name, len(term.courses))
# --------------------------- persist cache
self._save_cache(no_access)
return CrawlData(
degree_program=CrawlProgram(
id=program.id,
name=program.name,
terms=[t for t in terms if t.courses],
)
)
# ------------------------------------------------------------------ #
# static task inside class #
# ------------------------------------------------------------------ #
@staticmethod
@task(
name="crawl_course",
retries=2,
retry_delay_seconds=5,
log_prints=True,
cache_expiration=timedelta(days=1),
)
def _crawl_course_task(course_id: str) -> Tuple[str, str]:
"""
Returns (course_id, content_resource_id or "").
Never raises; logs reasons instead.
"""
lg = get_run_logger()
assert _COOKIE_JAR is not None
url = PRIVATE_URLS.course(course_id)
for attempt in (1, 2):
try:
r = httpx.get(
url, cookies=_COOKIE_JAR, follow_redirects=True, timeout=30
)
r.raise_for_status()
time.sleep(_DELAY)
break
except Exception as exc:
lg.warning("GET %s failed (%s) attempt %d/2", url, exc, attempt)
time.sleep(_DELAY)
else:
lg.warning("Course %s unreachable.", course_id)
return course_id, ""
if looks_like_enrol(r):
lg.info("No access to course %s (enrol / login page).", course_id)
return course_id, ""
href = (
parsel.Selector(r.text)
.css('a[data-downloadcourse="1"]::attr(href)')
.get("")
)
if not href:
lg.info("Course %s has no downloadable content.", course_id)
return course_id, ""
return course_id, href.split("=")[-1]
# ------------------------------------------------------------------ #
# helpers #
# ------------------------------------------------------------------ #
def _logged_in(self) -> bool:
html = self._get_html(PUBLIC_URLS.index)
return not parsel.Selector(text=html).css("div.usermenu span.login a")
def _crawl_terms(self, dp_id: str) -> List[CrawlTerm]:
html = self._get_html(PUBLIC_URLS.degree_program(dp_id))
sel = parsel.Selector(text=html)
out = []
for a in sel.css("div.category h3.categoryname a"):
name = a.xpath("text()").get("").strip()
if re.match(r"^(FS|HS)\d{2}$", name):
out.append(
CrawlTerm(name=name, id=a.xpath("@href").get("").split("=")[-1])
)
order = {"FS": 0, "HS": 1}
return sorted(
out, key=lambda t: (2000 + int(t.name[2:]), order[t.name[:2]]), reverse=True
)
def _crawl_courses(self, term_id: str) -> List[CrawlCourse]:
html = self._get_html(PUBLIC_URLS.term(term_id))
sel = parsel.Selector(text=html)
courses = []
for box in sel.css("div.coursebox"):
anchor = box.css("h3.coursename a")
if not anchor:
continue
cid = anchor.attrib.get("href", "").split("=")[-1]
raw = anchor.xpath("text()").get("").strip()
name = re.sub(r"\s*(FS|HS)\d{2}\s*", "", raw)
name = re.sub(r"\s*\(.*?\)\s*", "", name).strip()
hero = box.css("div.courseimage img::attr(src)").get("") or ""
courses.append(CrawlCourse(id=cid, name=name, hero_image=hero))
return courses
def _get_html(self, url: str) -> str:
try:
r = self._client.get(url, timeout=30)
r.raise_for_status()
time.sleep(_DELAY)
return r.text
except Exception as exc:
get_run_logger().warning("GET %s failed (%s)", url, exc)
return ""
# ------------------------------------------------------------------ #
# cache helpers #
# ------------------------------------------------------------------ #
@staticmethod
def _load_cache() -> set[str]:
try:
return set(json.loads(CACHE_FILE.read_text()))
except Exception:
return set()
@staticmethod
def _save_cache(cache: set[str]) -> None:
try:
CACHE_FILE.write_text(json.dumps(sorted(cache), indent=2))
except Exception as exc:
get_run_logger().warning("Could not save cache: %s", exc)
@@ -0,0 +1,357 @@
import concurrent.futures
import json
import logging
import re
import tempfile
import time
from pathlib import Path
import httpx
import parsel
from librarian_core.model import Course, DegreeProgram, FileEntry, MoodleIndex, Semester
from . import URLs
CACHE_FILENAME = "librarian_no_access_cache.json"
NO_ACCESS_CACHE_FILE = Path(tempfile.gettempdir()) / CACHE_FILENAME
class IndexCrawler:
def __init__(self, degree_program: DegreeProgram, cookies: httpx.Cookies, debug: bool = False, *, max_workers: int = 8) -> None:
self.degree_program = degree_program
self.debug = debug
self.client = httpx.Client(cookies=cookies, follow_redirects=True)
self.max_workers = max_workers
# When True the cached “no-access” set is ignored for this run
self._ignore_cache: bool = False
# Load persisted cache of course-IDs the user cannot access
if NO_ACCESS_CACHE_FILE.exists():
try:
self._no_access_cache: set[str] = set(json.loads(NO_ACCESS_CACHE_FILE.read_text()))
except Exception:
logging.warning("Failed to read no-access cache, starting fresh.")
self._no_access_cache = set()
else:
self._no_access_cache = set()
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.client.close()
def __del__(self):
# Fallback in case the context manager isn’t used
if not self.client.is_closed:
self.client.close()
"""
Crawl a single instance of MoodleIndex.
This returns a MoodleIndex object populated with data.
"""
def crawl_index(self, userSpecific: bool = True, *, use_cache: bool = True) -> MoodleIndex:
"""
Build and return a `MoodleIndex`.
Parameters
----------
userSpecific : bool
When True, include only courses that expose a downloadable content resource.
use_cache : bool, default True
If False, bypass the persisted “no-access” cache so every course is probed
afresh. Newly discovered “no-access” courses are still written back to the
cache at the end of the crawl.
"""
# Set runtime flag for has_user_access()
self._ignore_cache = not use_cache
semesters = []
# Get all courses for each semester and the courseid and name for each course.
semesters = self.crawl_semesters()
# Crawl only the latest two semesters to reduce load (remove once caching is implemented)
for semester in semesters[:2]:
courses = self.crawl_courses(semester)
# Crawl courses in parallel to speed things up
with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as pool:
list(pool.map(self.crawl_course, courses))
# Filter courses once all have been processed
for course in courses:
if userSpecific:
if course.content_ressource_id:
semester.courses.append(course)
else:
semester.courses.append(course)
# Only add semesters that have at least one course
# Filter out semesters that ended up with no courses after crawling
semesters: list[Semester] = [
semester for semester in semesters if semester.courses
]
created_index = MoodleIndex(
degree_program=DegreeProgram(
name=self.degree_program.name,
id=self.degree_program.id,
semesters=semesters,
),
)
# Persist any newly discovered no-access courses
self._save_no_access_cache()
# Restore default behaviour for subsequent calls
self._ignore_cache = False
return created_index
# --------------------------------------------------------------------- #
# High-level crawling helpers
# --------------------------------------------------------------------- #
def crawl_semesters(self) -> list[Semester]:
"""
Crawl the semesters from the Moodle index page.
"""
url = URLs.get_degree_program_url(self.degree_program.id)
res = self.get_with_retries(url)
if res.status_code == 200:
semesters = self.extract_semesters(res.text)
logging.debug(f"Found semesters: {semesters}")
return semesters
return []
def crawl_courses(self, semester: Semester) -> list[Course]:
"""
Crawl the courses from the Moodle index page.
"""
url = URLs.get_semester_url(semester_id=semester.id)
res = self.get_with_retries(url)
if res.status_code == 200:
courses = self.extract_courses(res.text)
logging.debug(f"Found courses: {courses}")
return courses
return []
def crawl_course(self, course: Course) -> None:
"""
Crawl a single Moodle course page.
"""
hasAccess = self.has_user_access(course)
if not hasAccess:
return
# TODO: Cache which courses the user has no access to, to avoid repeated requests
course.content_ressource_id = self.crawl_content_ressource_id(course)
course.files = self.crawl_course_files(course)
# --------------------------------------------------------------------- #
# Networking utilities
# --------------------------------------------------------------------- #
def get_with_retries(self, url: str, retries: int = 3, delay: int = 1) -> httpx.Response:
"""
Simple GET with retries and exponential back-off.
"""
for attempt in range(1, retries + 1):
try:
response = self.client.get(url)
response.raise_for_status()
return response
except Exception as e:
logging.warning(f"Request to {url} failed ({e}), attempt {attempt}/{retries}")
if attempt < retries:
time.sleep(delay * (2 ** (attempt - 1)))
raise Exception(f"Failed to GET {url} after {retries} attempts")
def save_html(self, url: str, response: httpx.Response) -> None:
"""
Persist raw HTML locally for debugging.
"""
filename = url.split("/")[-1] + ".html"
with open(filename, "w", encoding="utf-8") as f:
f.write(response.text)
logging.info(f"Saved HTML to {filename}")
# --------------------------------------------------------------------- #
# Extractors
# --------------------------------------------------------------------- #
def extract_semesters(self, html: str) -> list[Semester]:
selector = parsel.Selector(text=html)
logging.info("Extracting semesters from the HTML content.")
semesters: list[Semester] = []
# Each semester sits in a collapsed container
semester_containers = selector.css("div.category.notloaded.with_children.collapsed")
for container in semester_containers:
anchor = container.css("h3.categoryname.aabtn a")
if not anchor:
continue
anchor = anchor[0]
semester_name = (
anchor.xpath("text()").get("").replace("\n", "").replace("\t", "").strip()
)
semester_id = anchor.attrib.get("href", "").split("=")[-1]
# Only keep semesters labeled FS or HS
if "FS" not in semester_name and "HS" not in semester_name:
continue
semesters.append(Semester(name=semester_name, id=semester_id))
semester_order = {
"FS": 0, # Frühjahrs‐/Spring Semester
"HS": 1, # Herbst‐/Fall Semester
}
# Sort by year and then by FS before HS
sorted_semesters = sorted(
semesters,
key=lambda s: (
2000 + int(s.name[2:]), # parse "25" → int 25, add 2000 → 2025
semester_order[s.name[:2]] # map "FS" → 0, "HS" → 1
),
reverse=True,
)
return sorted_semesters
def extract_courses(self, html: str) -> list[Course]:
"""
Parse courses and capture optional “hero_image” (overview image) if present.
"""
selector = parsel.Selector(text=html)
logging.info("Extracting courses from the HTML content.")
courses: list[Course] = []
for header in selector.css("h3.coursename"):
anchor = header.css("a")
if not anchor:
logging.warning("No course anchor found in the course header.")
continue
anchor = anchor[0]
course_name = (
anchor.xpath("text()").get("").replace("\n", "").replace("\t", "").strip()
)
course_id = anchor.attrib.get("href", "").split("=")[-1]
# Remove trailing semester tag and code patterns
course_name = re.sub(r"\s*(FS|HS)\d{2}\s*", "", course_name)
course_name = re.sub(r"\s*\(.*?\)\s*", "", course_name).strip()
# Try to locate a hero/overview image that belongs to this course box
# Traverse up to the containing course box, then look for <div class="courseimage"><img ...>
course_container = header.xpath('./ancestor::*[contains(@class,"coursebox")][1]')
hero_src = (
course_container.css("div.courseimage img::attr(src)").get("")
if course_container else ""
)
courses.append(
Course(
id=course_id,
name=course_name,
activity_type="", # TODO: Make optional
hero_image=hero_src or ""
)
)
logging.info(f"{len(courses)} courses extracted.")
return courses
def has_user_access(self, course: Course) -> bool:
"""
Return True only if the authenticated user can access the course (result cached).
(i.e. the response is HTTP 200 **and** is not a redirected login/enrol page).
"""
if not self._ignore_cache and course.id in self._no_access_cache:
return False
url = URLs.get_course_url(course.id)
res = self.get_with_retries(url)
if res.status_code != 200:
self._no_access_cache.add(course.id)
return False
# Detect Moodle redirection to a login or enrolment page
final_url = str(res.url).lower()
if "login" in final_url or "enrol" in final_url:
self._no_access_cache.add(course.id)
return False
# Some enrolment pages still return 200; look for HTML markers
if "#page-enrol" in res.text or "you need to enrol" in res.text.lower():
self._no_access_cache.add(course.id)
return False
# If we got here the user has access; otherwise cache the deny
return True
def crawl_content_ressource_id(self, course: Course) -> str:
course_id = course.id
url = URLs.get_course_url(course_id)
res = self.get_with_retries(url)
psl = parsel.Selector(res.text)
try:
logging.info("Searching for 'Download course content' link.")
# Use parsel CSS selector to find the anchor tag with the specific data attribute
download_link_selector = psl.css('a[data-downloadcourse="1"]')
if not download_link_selector:
raise ValueError("Download link not found.")
# Extract the href attribute from the first matching element
href = download_link_selector[0].attrib.get("href")
if not href:
raise ValueError("Href attribute not found on the download link.")
context_id = href.split("=")[-1]
course.content_ressource_id = context_id
return context_id
except Exception as e:
logging.error(
f"Error extracting content resource ID for course '{course.name}': {e}",
exc_info=False,
)
logging.debug("Debugging info: Error accessing course content.", exc_info=True)
return ''
def crawl_course_files(self, course: Course) -> list[FileEntry]:
"""
Crawl the course files from the Moodle course page.
"""
url = URLs.get_course_url(course.id)
res = self.get_with_retries(url)
if res.status_code == 200:
files = [] # TODO: either implement this or remove, because files are extracted from the .zip file
logging.debug(f"Found files: {files}")
return files
return []
# ----------------------------------------------------------------- #
# Cache persistence helpers
# ----------------------------------------------------------------- #
def _save_no_access_cache(self) -> None:
try:
NO_ACCESS_CACHE_FILE.write_text(json.dumps(sorted(self._no_access_cache)))
except Exception as exc:
logging.warning(f"Could not persist no-access cache: {exc}")
@@ -0,0 +1,59 @@
# TODO: Move to librarian-core
"""
All URLs used in the crawler.
Functions marked as PUBLIC can be accessed without authentication.
Functions marked as PRIVATE require authentication.
"""
class URLs:
base_url = "https://moodle.fhgr.ch"
@classmethod
def get_base_url(cls):
"""PUBLIC"""
return cls.base_url
# ------------------------- Moodle URLs -------------------------
@classmethod
def get_login_url(cls):
"""PUBLIC"""
return f"{cls.base_url}/login/index.php"
@classmethod
def get_index_url(cls):
"""PUBLIC"""
return f"{cls.base_url}/course/index.php"
@classmethod
def get_degree_program_url(cls, degree_program_id):
"""PUBLIC"""
return f"{cls.base_url}/course/index.php?categoryid={degree_program_id}"
@classmethod
def get_category_url(cls, category_id):
"""PUBLIC"""
return f"{cls.base_url}/course/index.php?categoryid={category_id}"
@classmethod
def get_semester_url(cls, semester_id):
"""PUBLIC"""
return f"{cls.base_url}/course/index.php?categoryid={semester_id}"
@classmethod
def get_user_courses_url(cls):
"""PRIVATE"""
return f"{cls.base_url}/my/courses.php"
@classmethod
def get_course_url(cls, course_id):
"""PRIVATE"""
return f"{cls.base_url}/course/view.php?id={course_id}"
@classmethod
def get_files_url(cls, context_id):
"""PRIVATE"""
return f"{cls.base_url}/course/downloadcontent.php?contextid={context_id}"
@classmethod
def get_file_url(cls, file_id):
"""PRIVATE"""
return f"{cls.base_url}/mod/resource/view.php?id={file_id}"
@@ -0,0 +1,5 @@
from .downloader import *
__all__ = [
"Downloader",
]
@@ -0,0 +1,151 @@
"""
Downloader Worker
=================
Input : CrawlData (from the crawler)
Output : DownloadData (metadata only; files staged)
Folder tree after run
---------------------
export_dir/
└─ {TERM_NAME}/
├─ {course_id}.zip
└─ …
"""
from __future__ import annotations
import asyncio
import time
from pathlib import Path
from typing import List, Tuple
import httpx
from librarian_core.utils.path_utils import get_temp_path
from librarian_core.workers.base import Worker
from prefect import get_run_logger, task
from prefect.futures import wait
from librarian_scraper.constants import CRAWLER
from librarian_scraper.crawler.cookie_crawler import CookieCrawler
from librarian_scraper.models.crawl_data import CrawlData
from librarian_scraper.models.download_data import (
DownloadCourse,
DownloadData,
DownloadTerm,
)
# --------------------------------------------------------------------------- #
# helper decorator #
# --------------------------------------------------------------------------- #
def task_(**kw):
kw.setdefault("log_prints", True)
kw.setdefault("retries", 2)
kw.setdefault("retry_delay_seconds", 5)
return task(**kw)
# --------------------------------------------------------------------------- #
# shared state for static task #
# --------------------------------------------------------------------------- #
_COOKIE_JAR: httpx.Cookies | None = None
_SESSKEY: str = ""
_LIMIT: int = 2
_DELAY: float = 0.0
class Downloader(Worker[CrawlData, DownloadData]):
DOWNLOAD_URL = "https://moodle.fhgr.ch/course/downloadcontent.php"
# tuning
CONCURRENCY = 8
RELAXED = True # False → faster
input_model = CrawlData
output_model = DownloadData
# ------------------------------------------------------------------ #
async def __run__(self, crawl: CrawlData) -> DownloadData:
global _COOKIE_JAR, _SESSKEY, _LIMIT, _DELAY
lg = get_run_logger()
# ------------ login
cookies, sesskey = await CookieCrawler().crawl()
_COOKIE_JAR, _SESSKEY = cookies, sesskey
# ------------ tuning
_LIMIT = 1 if self.RELAXED else max(1, min(self.CONCURRENCY, 8))
_DELAY = CRAWLER["DELAY_SLOW"] if self.RELAXED else CRAWLER["DELAY_FAST"]
# ------------ working dir
work_root = Path(get_temp_path()) / f"dl_{int(time.time())}"
work_root.mkdir(parents=True, exist_ok=True)
result = DownloadData()
futures = []
term_dirs: List[Tuple[str, Path]] = []
# schedule downloads
for term in crawl.degree_program.terms:
term_dir = work_root / term.name
term_dir.mkdir(parents=True, exist_ok=True)
term_dirs.append((term.name, term_dir))
dl_term = DownloadTerm(id=term.id, name=term.name)
result.terms.append(dl_term)
for course in term.courses:
dest = term_dir / f"{course.id}.zip"
dl_term.courses.append(DownloadCourse(id=course.id, name=course.name))
futures.append(
self._download_task.submit(course.content_ressource_id, dest)
)
wait(futures) # block for all downloads
# stage term directories
for name, dir_path in term_dirs:
self.stage(dir_path, new_name=name, sanitize=False, move=True)
lg.info("Downloader finished – staged %d term folders", len(term_dirs))
return result
# ------------------------------------------------------------------ #
# static task #
# ------------------------------------------------------------------ #
@staticmethod
@task_()
def _download_task(context_id: str, dest: Path) -> None:
lg = get_run_logger()
if not context_id:
lg.info("Skip (no context id) → %s", dest.name)
return
async def fetch() -> bool:
sem = asyncio.Semaphore(_LIMIT)
async with sem:
data = {"sesskey": _SESSKEY, "download": 1, "contextid": context_id}
async with httpx.AsyncClient(cookies=_COOKIE_JAR) as cli:
try:
async with cli.stream(
"POST", Downloader.DOWNLOAD_URL, data=data, timeout=60
) as r:
r.raise_for_status()
with dest.open("wb") as fh:
async for chunk in r.aiter_bytes():
fh.write(chunk)
lg.info("Downloaded %s", dest)
return True
except httpx.HTTPStatusError as exc:
lg.warning(
"HTTP %s for %s", exc.response.status_code, dest.name
)
except Exception as exc:
lg.warning("Error downloading %s (%s)", dest.name, exc)
return False
ok = asyncio.run(fetch())
if not ok and dest.exists():
dest.unlink(missing_ok=True)
time.sleep(_DELAY)
@@ -0,0 +1,24 @@
from librarian_scraper.models.crawl_data import (
CrawlCourse,
CrawlData,
CrawlFile,
CrawlProgram,
CrawlTerm,
)
from librarian_scraper.models.download_data import (
DownloadCourse,
DownloadData,
DownloadTerm,
)
__all__ = [
"CrawlData",
"CrawlCourse",
"CrawlFile",
"CrawlProgram",
"CrawlTerm",
"DownloadData",
"DownloadCourse",
"DownloadTerm",
]
@@ -0,0 +1,188 @@
from __future__ import annotations
import re
from datetime import datetime, timezone
from pydantic import BaseModel, Field
"""
Example of a MoodleIndex (JSON):
MoodleIndex: {
degree_program: {
id: '1157',
name: 'Computational and Data Science',
terms: [
{
id: '1745',
name: 'FS25',
courses: [
{
id: '18863',
name: 'Programmierung und Prompt Engineering II',
activity_type: '',
hero_image:
'https://moodle.fhgr.ch/pluginfile.php/1159522/course/overviewfiles/PythonBooks.PNG',
content_ressource_id: '1159522',
files: [],
},
{
id: '18240',
name: 'Effiziente Algorithmen',
activity_type: '',
hero_image: '',
content_ressource_id: '1125554',
files: [],
},
{
id: '18237',
name: 'Mathematik II',
activity_type: '',
hero_image:
'https://moodle.fhgr.ch/pluginfile.php/1125458/course/overviewfiles/Integration_Differential_b.png',
content_ressource_id: '1125458',
files: [],
},
{
id: '18236',
name: '2025 FS FHGR CDS Numerische Methoden',
activity_type: '',
hero_image: '',
content_ressource_id: '1125426',
files: [],
},
{
id: '18228',
name: 'Datenbanken und Datenverarbeitung',
activity_type: '',
hero_image: '',
content_ressource_id: '1125170',
files: [],
},
],
},
{
id: '1746',
name: 'HS24',
courses: [
{
id: '18030',
name: 'Bootcamp Wissenschaftliches Arbeiten',
activity_type: '',
hero_image: '',
content_ressource_id: '1090544',
files: [],
},
{
id: '17527',
name: 'Einführung in Data Science',
activity_type: '',
hero_image:
'https://moodle.fhgr.ch/pluginfile.php/1059194/course/overviewfiles/cds1010.jpg',
content_ressource_id: '1059194',
files: [],
},
{
id: '17526',
name: 'Einführung in Computational Science',
activity_type: '',
hero_image:
'https://moodle.fhgr.ch/pluginfile.php/1059162/course/overviewfiles/cds_intro_sim.jpg',
content_ressource_id: '1059162',
files: [],
},
{
id: '17525',
name: 'Mathematik I',
activity_type: '',
hero_image:
'https://moodle.fhgr.ch/pluginfile.php/1059130/course/overviewfiles/AdobeStock_452512134.png',
content_ressource_id: '1059130',
files: [],
},
{
id: '17507',
name: 'Programmierung und Prompt Engineering',
activity_type: '',
hero_image:
'https://moodle.fhgr.ch/pluginfile.php/1058554/course/overviewfiles/10714013_33861.jpg',
content_ressource_id: '1058554',
files: [],
},
{
id: '17505',
name: 'Algorithmen und Datenstrukturen',
activity_type: '',
hero_image:
'https://moodle.fhgr.ch/pluginfile.php/1058490/course/overviewfiles/Bild1.png',
content_ressource_id: '1058490',
files: [],
},
{
id: '17503',
name: 'Computer Science',
activity_type: '',
hero_image:
'https://moodle.fhgr.ch/pluginfile.php/1058426/course/overviewfiles/Titelbild.jpg',
content_ressource_id: '1058426',
files: [],
},
],
},
],
},
timestamp: '2025-04-27T14:20:11.354825+00:00',
};
"""
# ---------------------------------------------------------------------------
# Base Model
# ---------------------------------------------------------------------------
class CrawlData(BaseModel):
degree_program: CrawlProgram = Field(
default_factory=lambda: CrawlProgram(id="", name="")
)
timestamp: str = Field(
default_factory=lambda: datetime.now(timezone.utc).isoformat()
)
# ---------------------------------------------------------------------------
# Degree Program
# ---------------------------------------------------------------------------
class CrawlProgram(BaseModel):
id: str = Field("1157", description="Unique identifier for the degree program.")
name: str = Field("Computational and Data Science", description="Name of the degree program.")
terms: list[CrawlTerm] = Field(default_factory=list)
# ---------------------------------------------------------------------------
# Term
# ---------------------------------------------------------------------------
_TERM_RE = re.compile(r"^(HS|FS)\d{2}$") # HS24 / FS25 …
class CrawlTerm(BaseModel):
id: str
name: str = Field(..., pattern=_TERM_RE.pattern) # e.g. “HS24”
courses: list[CrawlCourse] = Field(default_factory=list)
# ---------------------------------------------------------------------------
# Course
# ---------------------------------------------------------------------------
class CrawlCourse(BaseModel):
id: str
name: str
hero_image: str = ""
content_ressource_id: str = ""
files: list[CrawlFile] = Field(default_factory=list)
# ---------------------------------------------------------------------------
# Files
# ---------------------------------------------------------------------------
class CrawlFile(BaseModel):
id: str
res_id: str
name: str
@@ -0,0 +1,18 @@
from typing import List
from pydantic import BaseModel, Field
class DownloadCourse(BaseModel):
id: str
name: str # Stores the name of the zip file inside the term directory
class DownloadTerm(BaseModel):
id: str
name: str # Stores the name of the term directory inside DownloadMeta.dir
courses: List[DownloadCourse] = Field(default_factory=list)
class DownloadData(BaseModel):
terms: List[DownloadTerm] = Field(default_factory=list)
Whitespace-only changes.
File diff suppressed because it is too large. Load diff