Update Readme and cleanup

This commit is contained in:
Oliver Schütz authored and GitHub committed 2025-06-01 17:32:50 +02:00
1 parent 52d087ce90
commit b6789cb5f7
34 files changed
+104 -471

No files matched your search

@@ -1,8 +1,4 @@
"""
URLs used by the scraper.
Functions marked as PUBLIC can be accessed without authentication.
Functions marked as PRIVATE require authentication.
"""
"""Scraper URLs. PUBLIC/PRIVATE indicates auth requirement."""
BASE_URL = "https://moodle.fhgr.ch"
@@ -12,20 +12,8 @@ from librarian_scraper.constants import PRIVATE_URLS, PUBLIC_URLS
class CookieCrawler:
"""
Retrieve Moodle session cookies + sesskey via Playwright.
"""Retrieve Moodle session cookies + sesskey via Playwright."""
Usage
-----
>>> crawler = CookieCrawler()
>>> cookies, sesskey = await crawler.crawl() # inside async code
# or
>>> cookies, sesskey = CookieCrawler.crawl_sync() # plain scripts
"""
# ------------------------------------------------------------------ #
# construction #
# ------------------------------------------------------------------ #
def __init__(self, *, headless: bool = True) -> None:
self.headless = headless
self.cookies: Optional[List[Cookie]] = None
@@ -38,13 +26,8 @@ class CookieCrawler:
"Set MOODLE_USERNAME and MOODLE_PASSWORD as environment variables."
)
# ------------------------------------------------------------------ #
# public API #
# ------------------------------------------------------------------ #
async def crawl(self) -> tuple[Cookies, str]:
"""
Async entry-point – await this inside FastAPI / Prefect etc.
"""
"""Async method to crawl cookies and sesskey."""
async with async_playwright() as p:
browser: Browser = await p.chromium.launch(headless=self.headless)
page = await browser.new_page()
@@ -61,51 +44,34 @@ class CookieCrawler:
@classmethod
def crawl_sync(cls, **kwargs) -> tuple[Cookies, str]:
"""
Synchronous helper for CLI / notebooks.
Detects whether an event loop is already running. If so, it
schedules the coroutine and waits; otherwise it starts a fresh loop.
"""
"""Synchronous version of crawl. Handles event loop."""
self = cls(**kwargs)
try:
loop = asyncio.get_running_loop()
except RuntimeError: # no loop running → safe to create one
except RuntimeError:
return asyncio.run(self.crawl())
# An event loop exists – schedule coroutine
return loop.run_until_complete(self.crawl())
# ------------------------------------------------------------------ #
# internal helpers #
# ------------------------------------------------------------------ #
async def _login(self, page: Page) -> None:
"""Fill the SSO form and extract cookies + sesskey."""
# Select organisation / IdP
await page.click("#wayf_submit_button")
# Wait for the credential form
await page.wait_for_selector("form[method='post']", state="visible")
# Credentials
await page.fill("input[id='username']", self.username)
await page.fill("input[id='password']", self.password)
await page.click("button[class='aai_login_button']")
# Wait for redirect to /my/ page (dashboard), this means the login is complete
await page.wait_for_url(PRIVATE_URLS.dashboard)
await page.wait_for_selector("body", state="attached")
# Navigate to personal course overview
await page.goto(PRIVATE_URLS.user_courses)
await page.wait_for_selector("body", state="attached")
# Collect session cookies
self.cookies = await page.context.cookies()
# Extract sesskey from injected Moodle config
try:
self.sesskey = await page.evaluate(
"() => window.M && M.cfg && M.cfg.sesskey"
@@ -119,13 +85,9 @@ class CookieCrawler:
logging.debug("sesskey: %s", self.sesskey)
logging.debug("cookies: %s", self.cookies)
# Dev convenience
if not self.headless:
await page.wait_for_timeout(5000)
# ------------------------------------------------------------------ #
# cookie conversion #
# ------------------------------------------------------------------ #
def _to_cookiejar(self, raw: List[Cookie]) -> Cookies:
jar = Cookies()
for c in raw:
@@ -39,18 +39,12 @@ from librarian_scraper.models.crawl_data import (
CrawlTerm,
)
# --------------------------------------------------------------------------- #
# module-level shared items for static task #
# --------------------------------------------------------------------------- #
_COOKIE_JAR: httpx.Cookies | None = None
_DELAY: float = 0.0
CACHE_FILE = get_cache_root() / "librarian_no_access_cache.json"
# --------------------------------------------------------------------------- #
# utility #
# --------------------------------------------------------------------------- #
def looks_like_enrol(resp: httpx.Response) -> bool:
txt = resp.text.lower()
return (
@@ -60,21 +54,14 @@ def looks_like_enrol(resp: httpx.Response) -> bool:
)
# --------------------------------------------------------------------------- #
# main worker #
# --------------------------------------------------------------------------- #
class Crawler(Worker[CrawlProgram, CrawlData]):
input_model = CrawlProgram
output_model = CrawlData
# toggles (env overrides)
RELAXED: bool
USER_SPECIFIC: bool
CLEAR_CACHE: bool
# ------------------------------------------------------------------ #
# flow entry-point #
# ------------------------------------------------------------------ #
async def __run__(self, program: CrawlProgram) -> CrawlData:
global _COOKIE_JAR, _DELAY
lg = get_run_logger()
@@ -93,7 +80,6 @@ class Crawler(Worker[CrawlProgram, CrawlData]):
batch,
)
# --------------------------- login
cookies, _ = await CookieCrawler().crawl()
_COOKIE_JAR = cookies
self._client = httpx.Client(cookies=cookies, follow_redirects=True)
@@ -102,14 +88,11 @@ class Crawler(Worker[CrawlProgram, CrawlData]):
lg.error("Guest session detected – aborting crawl.")
raise RuntimeError("Login failed")
# --------------------------- cache
no_access: set[str] = set() if self.CLEAR_CACHE else self._load_cache()
# --------------------------- scrape terms (first two for dev)
terms = self._crawl_terms(program.id)[:2]
lg.info("Terms discovered: %d", len(terms))
# --------------------------- scrape courses
for term in terms:
courses = self._crawl_courses(term.id)
lg.info("[%s] raw courses: %d", term.name, len(courses))
@@ -137,7 +120,6 @@ class Crawler(Worker[CrawlProgram, CrawlData]):
)
lg.info("[%s] kept: %d", term.name, len(term.courses))
# --------------------------- persist cache
self._save_cache(no_access)
return CrawlData(
@@ -148,9 +130,6 @@ class Crawler(Worker[CrawlProgram, CrawlData]):
)
)
# ------------------------------------------------------------------ #
# static task inside class #
# ------------------------------------------------------------------ #
@staticmethod
@task(
name="crawl_course",
@@ -198,9 +177,6 @@ class Crawler(Worker[CrawlProgram, CrawlData]):
return course_id, href.split("=")[-1]
# ------------------------------------------------------------------ #
# helpers #
# ------------------------------------------------------------------ #
def _logged_in(self) -> bool:
html = self._get_html(PUBLIC_URLS.index)
return not parsel.Selector(text=html).css("div.usermenu span.login a")
@@ -246,9 +222,6 @@ class Crawler(Worker[CrawlProgram, CrawlData]):
get_run_logger().warning("GET %s failed (%s)", url, exc)
return ""
# ------------------------------------------------------------------ #
# cache helpers #
# ------------------------------------------------------------------ #
@staticmethod
def _load_cache() -> set[str]:
try:
@@ -22,11 +22,8 @@ class IndexCrawler:
self.debug = debug
self.client = httpx.Client(cookies=cookies, follow_redirects=True)
self.max_workers = max_workers
# When True the cached “no-access” set is ignored for this run
self._ignore_cache: bool = False
# Load persisted cache of course-IDs the user cannot access
if NO_ACCESS_CACHE_FILE.exists():
try:
self._no_access_cache: set[str] = set(json.loads(NO_ACCESS_CACHE_FILE.read_text()))
@@ -54,7 +51,7 @@ class IndexCrawler:
def crawl_index(self, userSpecific: bool = True, *, use_cache: bool = True) -> MoodleIndex:
"""
Build and return a `MoodleIndex`.
Builds and returns a `MoodleIndex`.
Parameters
----------
@@ -65,21 +62,17 @@ class IndexCrawler:
afresh. Newly discovered “no-access” courses are still written back to the
cache at the end of the crawl.
"""
# Set runtime flag for has_user_access()
self._ignore_cache = not use_cache
semesters = []
# Get all courses for each semester and the courseid and name for each course.
semesters = self.crawl_semesters()
# Crawl only the latest two semesters to reduce load (remove once caching is implemented)
for semester in semesters[:2]:
courses = self.crawl_courses(semester)
# Crawl courses in parallel to speed things up
with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as pool:
list(pool.map(self.crawl_course, courses))
# Filter courses once all have been processed
for course in courses:
if userSpecific:
if course.content_ressource_id:
@@ -87,8 +80,6 @@ class IndexCrawler:
else:
semester.courses.append(course)
# Only add semesters that have at least one course
# Filter out semesters that ended up with no courses after crawling
semesters: list[Semester] = [
semester for semester in semesters if semester.courses
]
@@ -100,21 +91,13 @@ class IndexCrawler:
semesters=semesters,
),
)
# Persist any newly discovered no-access courses
self._save_no_access_cache()
# Restore default behaviour for subsequent calls
self._ignore_cache = False
return created_index
# --------------------------------------------------------------------- #
# High-level crawling helpers
# --------------------------------------------------------------------- #
def crawl_semesters(self) -> list[Semester]:
"""
Crawl the semesters from the Moodle index page.
"""
"""Crawls semester data."""
url = URLs.get_degree_program_url(self.degree_program.id)
res = self.get_with_retries(url)
@@ -126,9 +109,7 @@ class IndexCrawler:
return []
def crawl_courses(self, semester: Semester) -> list[Course]:
"""
Crawl the courses from the Moodle index page.
"""
"""Crawls course data for a semester."""
url = URLs.get_semester_url(semester_id=semester.id)
res = self.get_with_retries(url)
@@ -140,10 +121,7 @@ class IndexCrawler:
return []
def crawl_course(self, course: Course) -> None:
"""
Crawl a single Moodle course page.
"""
"""Crawls details for a single course."""
hasAccess = self.has_user_access(course)
if not hasAccess:
@@ -154,13 +132,8 @@ class IndexCrawler:
course.content_ressource_id = self.crawl_content_ressource_id(course)
course.files = self.crawl_course_files(course)
# --------------------------------------------------------------------- #
# Networking utilities
# --------------------------------------------------------------------- #
def get_with_retries(self, url: str, retries: int = 3, delay: int = 1) -> httpx.Response:
"""
Simple GET with retries and exponential back-off.
"""
"""Simple GET with retries and exponential back-off."""
for attempt in range(1, retries + 1):
try:
response = self.client.get(url)
@@ -183,17 +156,11 @@ class IndexCrawler:
f.write(response.text)
logging.info(f"Saved HTML to {filename}")
# --------------------------------------------------------------------- #
# Extractors
# --------------------------------------------------------------------- #
def extract_semesters(self, html: str) -> list[Semester]:
"""Extracts semester names and IDs from HTML."""
selector = parsel.Selector(text=html)
logging.info("Extracting semesters from the HTML content.")
semesters: list[Semester] = []
# Each semester sits in a collapsed container
semester_containers = selector.css("div.category.notloaded.with_children.collapsed")
for container in semester_containers:
@@ -207,7 +174,6 @@ class IndexCrawler:
)
semester_id = anchor.attrib.get("href", "").split("=")[-1]
# Only keep semesters labeled FS or HS
if "FS" not in semester_name and "HS" not in semester_name:
continue
@@ -250,12 +216,9 @@ class IndexCrawler:
)
course_id = anchor.attrib.get("href", "").split("=")[-1]
# Remove trailing semester tag and code patterns
course_name = re.sub(r"\s*(FS|HS)\d{2}\s*", "", course_name)
course_name = re.sub(r"\s*\(.*?\)\s*", "", course_name).strip()
# Try to locate a hero/overview image that belongs to this course box
# Traverse up to the containing course box, then look for <div class="courseimage"><img ...>
course_container = header.xpath('./ancestor::*[contains(@class,"coursebox")][1]')
hero_src = (
course_container.css("div.courseimage img::attr(src)").get("")
@@ -275,10 +238,7 @@ class IndexCrawler:
return courses
def has_user_access(self, course: Course) -> bool:
"""
Return True only if the authenticated user can access the course (result cached).
(i.e. the response is HTTP 200 **and** is not a redirected login/enrol page).
"""
"""Checks if user can access course (caches negative results)."""
if not self._ignore_cache and course.id in self._no_access_cache:
return False
@@ -289,21 +249,19 @@ class IndexCrawler:
self._no_access_cache.add(course.id)
return False
# Detect Moodle redirection to a login or enrolment page
final_url = str(res.url).lower()
if "login" in final_url or "enrol" in final_url:
self._no_access_cache.add(course.id)
return False
# Some enrolment pages still return 200; look for HTML markers
if "#page-enrol" in res.text or "you need to enrol" in res.text.lower():
self._no_access_cache.add(course.id)
return False
# If we got here the user has access; otherwise cache the deny
return True
def crawl_content_ressource_id(self, course: Course) -> str:
"""Extracts content resource ID for a course."""
course_id = course.id
url = URLs.get_course_url(course_id)
res = self.get_with_retries(url)
@@ -311,12 +269,10 @@ class IndexCrawler:
try:
logging.info("Searching for 'Download course content' link.")
# Use parsel CSS selector to find the anchor tag with the specific data attribute
download_link_selector = psl.css('a[data-downloadcourse="1"]')
if not download_link_selector:
raise ValueError("Download link not found.")
# Extract the href attribute from the first matching element
href = download_link_selector[0].attrib.get("href")
if not href:
raise ValueError("Href attribute not found on the download link.")
@@ -334,9 +290,7 @@ class IndexCrawler:
return ''
def crawl_course_files(self, course: Course) -> list[FileEntry]:
"""
Crawl the course files from the Moodle course page.
"""
"""Crawls file entries for a course."""
url = URLs.get_course_url(course.id)
res = self.get_with_retries(url)
@@ -347,10 +301,8 @@ class IndexCrawler:
return []
# ----------------------------------------------------------------- #
# Cache persistence helpers
# ----------------------------------------------------------------- #
def _save_no_access_cache(self) -> None:
"""Saves course IDs the user cannot access to a cache file."""
try:
NO_ACCESS_CACHE_FILE.write_text(json.dumps(sorted(self._no_access_cache)))
except Exception as exc:
@@ -1,9 +1,5 @@
# TODO: Move to librarian-core
"""
All URLs used in the crawler.
Functions marked as PUBLIC can be accessed without authentication.
Functions marked as PRIVATE require authentication.
"""
"""Moodle URLs. PUBLIC/PRIVATE indicates auth requirement."""
class URLs:
base_url = "https://moodle.fhgr.ch"
@@ -12,7 +8,6 @@ class URLs:
"""PUBLIC"""
return cls.base_url
# ------------------------- Moodle URLs -------------------------
@classmethod
def get_login_url(cls):
"""PUBLIC"""
@@ -35,9 +35,6 @@ from librarian_scraper.models.download_data import (
)
# --------------------------------------------------------------------------- #
# helper decorator #
# --------------------------------------------------------------------------- #
def task_(**kw):
kw.setdefault("log_prints", True)
kw.setdefault("retries", 2)
@@ -45,9 +42,6 @@ def task_(**kw):
return task(**kw)
# --------------------------------------------------------------------------- #
# shared state for static task #
# --------------------------------------------------------------------------- #
_COOKIE_JAR: httpx.Cookies | None = None
_SESSKEY: str = ""
_LIMIT: int = 2
@@ -57,27 +51,22 @@ _DELAY: float = 0.0
class Downloader(Worker[CrawlData, DownloadData]):
DOWNLOAD_URL = "https://moodle.fhgr.ch/course/downloadcontent.php"
# tuning
CONCURRENCY = 8
RELAXED = True # False → faster
RELAXED = True
input_model = CrawlData
output_model = DownloadData
# ------------------------------------------------------------------ #
async def __run__(self, crawl: CrawlData) -> DownloadData:
global _COOKIE_JAR, _SESSKEY, _LIMIT, _DELAY
lg = get_run_logger()
# ------------ login
cookies, sesskey = await CookieCrawler().crawl()
_COOKIE_JAR, _SESSKEY = cookies, sesskey
# ------------ tuning
_LIMIT = 1 if self.RELAXED else max(1, min(self.CONCURRENCY, 8))
_DELAY = CRAWLER["DELAY_SLOW"] if self.RELAXED else CRAWLER["DELAY_FAST"]
# ------------ working dir
work_root = Path(get_temp_path()) / f"dl_{int(time.time())}"
work_root.mkdir(parents=True, exist_ok=True)
@@ -85,7 +74,6 @@ class Downloader(Worker[CrawlData, DownloadData]):
futures = []
term_dirs: List[Tuple[str, Path]] = []
# schedule downloads
for term in crawl.degree_program.terms:
term_dir = work_root / term.name
term_dir.mkdir(parents=True, exist_ok=True)
@@ -101,18 +89,14 @@ class Downloader(Worker[CrawlData, DownloadData]):
self._download_task.submit(course.content_ressource_id, dest)
)
wait(futures) # block for all downloads
wait(futures)
# stage term directories
for name, dir_path in term_dirs:
self.stage(dir_path, new_name=name, sanitize=False, move=True)
lg.info("Downloader finished – staged %d term folders", len(term_dirs))
return result
# ------------------------------------------------------------------ #
# static task #
# ------------------------------------------------------------------ #
@staticmethod
@task_()
def _download_task(context_id: str, dest: Path) -> None:
@@ -135,9 +135,6 @@ MoodleIndex: {
"""
# ---------------------------------------------------------------------------
# Base Model
# ---------------------------------------------------------------------------
class CrawlData(BaseModel):
degree_program: CrawlProgram = Field(
default_factory=lambda: CrawlProgram(id="", name="")
@@ -147,18 +144,12 @@ class CrawlData(BaseModel):
)
# ---------------------------------------------------------------------------
# Degree Program
# ---------------------------------------------------------------------------
class CrawlProgram(BaseModel):
id: str = Field("1157", description="Unique identifier for the degree program.")
name: str = Field("Computational and Data Science", description="Name of the degree program.")
terms: list[CrawlTerm] = Field(default_factory=list)
# ---------------------------------------------------------------------------
# Term
# ---------------------------------------------------------------------------
_TERM_RE = re.compile(r"^(HS|FS)\d{2}$") # HS24 / FS25 …
@@ -168,9 +159,6 @@ class CrawlTerm(BaseModel):
courses: list[CrawlCourse] = Field(default_factory=list)
# ---------------------------------------------------------------------------
# Course
# ---------------------------------------------------------------------------
class CrawlCourse(BaseModel):
id: str
name: str
@@ -179,9 +167,6 @@ class CrawlCourse(BaseModel):
files: list[CrawlFile] = Field(default_factory=list)
# ---------------------------------------------------------------------------
# Files
# ---------------------------------------------------------------------------
class CrawlFile(BaseModel):
id: str
res_id: str