Update Readme and cleanup
This commit is contained in:
1 parent
52d087ce90
commit
b6789cb5f7
34 files changed
+104
-471
No files matched your search
@@ -1,8 +1,4 @@
|
||||
"""
|
||||
URLs used by the scraper.
|
||||
Functions marked as PUBLIC can be accessed without authentication.
|
||||
Functions marked as PRIVATE require authentication.
|
||||
"""
|
||||
"""Scraper URLs. PUBLIC/PRIVATE indicates auth requirement."""
|
||||
|
||||
BASE_URL = "https://moodle.fhgr.ch"
|
||||
|
||||
|
||||
+4
-42
@@ -12,20 +12,8 @@ from librarian_scraper.constants import PRIVATE_URLS, PUBLIC_URLS
|
||||
|
||||
|
||||
class CookieCrawler:
|
||||
"""
|
||||
Retrieve Moodle session cookies + sesskey via Playwright.
|
||||
"""Retrieve Moodle session cookies + sesskey via Playwright."""
|
||||
|
||||
Usage
|
||||
-----
|
||||
>>> crawler = CookieCrawler()
|
||||
>>> cookies, sesskey = await crawler.crawl() # inside async code
|
||||
# or
|
||||
>>> cookies, sesskey = CookieCrawler.crawl_sync() # plain scripts
|
||||
"""
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# construction #
|
||||
# ------------------------------------------------------------------ #
|
||||
def __init__(self, *, headless: bool = True) -> None:
|
||||
self.headless = headless
|
||||
self.cookies: Optional[List[Cookie]] = None
|
||||
@@ -38,13 +26,8 @@ class CookieCrawler:
|
||||
"Set MOODLE_USERNAME and MOODLE_PASSWORD as environment variables."
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# public API #
|
||||
# ------------------------------------------------------------------ #
|
||||
async def crawl(self) -> tuple[Cookies, str]:
|
||||
"""
|
||||
Async entry-point – await this inside FastAPI / Prefect etc.
|
||||
"""
|
||||
"""Async method to crawl cookies and sesskey."""
|
||||
async with async_playwright() as p:
|
||||
browser: Browser = await p.chromium.launch(headless=self.headless)
|
||||
page = await browser.new_page()
|
||||
@@ -61,51 +44,34 @@ class CookieCrawler:
|
||||
|
||||
@classmethod
|
||||
def crawl_sync(cls, **kwargs) -> tuple[Cookies, str]:
|
||||
"""
|
||||
Synchronous helper for CLI / notebooks.
|
||||
|
||||
Detects whether an event loop is already running. If so, it
|
||||
schedules the coroutine and waits; otherwise it starts a fresh loop.
|
||||
"""
|
||||
"""Synchronous version of crawl. Handles event loop."""
|
||||
self = cls(**kwargs)
|
||||
|
||||
try:
|
||||
loop = asyncio.get_running_loop()
|
||||
except RuntimeError: # no loop running → safe to create one
|
||||
except RuntimeError:
|
||||
return asyncio.run(self.crawl())
|
||||
|
||||
# An event loop exists – schedule coroutine
|
||||
return loop.run_until_complete(self.crawl())
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# internal helpers #
|
||||
# ------------------------------------------------------------------ #
|
||||
async def _login(self, page: Page) -> None:
|
||||
"""Fill the SSO form and extract cookies + sesskey."""
|
||||
|
||||
# Select organisation / IdP
|
||||
await page.click("#wayf_submit_button")
|
||||
|
||||
# Wait for the credential form
|
||||
await page.wait_for_selector("form[method='post']", state="visible")
|
||||
|
||||
# Credentials
|
||||
await page.fill("input[id='username']", self.username)
|
||||
await page.fill("input[id='password']", self.password)
|
||||
await page.click("button[class='aai_login_button']")
|
||||
|
||||
# Wait for redirect to /my/ page (dashboard), this means the login is complete
|
||||
await page.wait_for_url(PRIVATE_URLS.dashboard)
|
||||
await page.wait_for_selector("body", state="attached")
|
||||
|
||||
# Navigate to personal course overview
|
||||
await page.goto(PRIVATE_URLS.user_courses)
|
||||
await page.wait_for_selector("body", state="attached")
|
||||
|
||||
# Collect session cookies
|
||||
self.cookies = await page.context.cookies()
|
||||
|
||||
# Extract sesskey from injected Moodle config
|
||||
try:
|
||||
self.sesskey = await page.evaluate(
|
||||
"() => window.M && M.cfg && M.cfg.sesskey"
|
||||
@@ -119,13 +85,9 @@ class CookieCrawler:
|
||||
logging.debug("sesskey: %s", self.sesskey)
|
||||
logging.debug("cookies: %s", self.cookies)
|
||||
|
||||
# Dev convenience
|
||||
if not self.headless:
|
||||
await page.wait_for_timeout(5000)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# cookie conversion #
|
||||
# ------------------------------------------------------------------ #
|
||||
def _to_cookiejar(self, raw: List[Cookie]) -> Cookies:
|
||||
jar = Cookies()
|
||||
for c in raw:
|
||||
|
||||
@@ -39,18 +39,12 @@ from librarian_scraper.models.crawl_data import (
|
||||
CrawlTerm,
|
||||
)
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# module-level shared items for static task #
|
||||
# --------------------------------------------------------------------------- #
|
||||
_COOKIE_JAR: httpx.Cookies | None = None
|
||||
_DELAY: float = 0.0
|
||||
|
||||
CACHE_FILE = get_cache_root() / "librarian_no_access_cache.json"
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# utility #
|
||||
# --------------------------------------------------------------------------- #
|
||||
def looks_like_enrol(resp: httpx.Response) -> bool:
|
||||
txt = resp.text.lower()
|
||||
return (
|
||||
@@ -60,21 +54,14 @@ def looks_like_enrol(resp: httpx.Response) -> bool:
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# main worker #
|
||||
# --------------------------------------------------------------------------- #
|
||||
class Crawler(Worker[CrawlProgram, CrawlData]):
|
||||
input_model = CrawlProgram
|
||||
output_model = CrawlData
|
||||
|
||||
# toggles (env overrides)
|
||||
RELAXED: bool
|
||||
USER_SPECIFIC: bool
|
||||
CLEAR_CACHE: bool
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# flow entry-point #
|
||||
# ------------------------------------------------------------------ #
|
||||
async def __run__(self, program: CrawlProgram) -> CrawlData:
|
||||
global _COOKIE_JAR, _DELAY
|
||||
lg = get_run_logger()
|
||||
@@ -93,7 +80,6 @@ class Crawler(Worker[CrawlProgram, CrawlData]):
|
||||
batch,
|
||||
)
|
||||
|
||||
# --------------------------- login
|
||||
cookies, _ = await CookieCrawler().crawl()
|
||||
_COOKIE_JAR = cookies
|
||||
self._client = httpx.Client(cookies=cookies, follow_redirects=True)
|
||||
@@ -102,14 +88,11 @@ class Crawler(Worker[CrawlProgram, CrawlData]):
|
||||
lg.error("Guest session detected – aborting crawl.")
|
||||
raise RuntimeError("Login failed")
|
||||
|
||||
# --------------------------- cache
|
||||
no_access: set[str] = set() if self.CLEAR_CACHE else self._load_cache()
|
||||
|
||||
# --------------------------- scrape terms (first two for dev)
|
||||
terms = self._crawl_terms(program.id)[:2]
|
||||
lg.info("Terms discovered: %d", len(terms))
|
||||
|
||||
# --------------------------- scrape courses
|
||||
for term in terms:
|
||||
courses = self._crawl_courses(term.id)
|
||||
lg.info("[%s] raw courses: %d", term.name, len(courses))
|
||||
@@ -137,7 +120,6 @@ class Crawler(Worker[CrawlProgram, CrawlData]):
|
||||
)
|
||||
lg.info("[%s] kept: %d", term.name, len(term.courses))
|
||||
|
||||
# --------------------------- persist cache
|
||||
self._save_cache(no_access)
|
||||
|
||||
return CrawlData(
|
||||
@@ -148,9 +130,6 @@ class Crawler(Worker[CrawlProgram, CrawlData]):
|
||||
)
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# static task inside class #
|
||||
# ------------------------------------------------------------------ #
|
||||
@staticmethod
|
||||
@task(
|
||||
name="crawl_course",
|
||||
@@ -198,9 +177,6 @@ class Crawler(Worker[CrawlProgram, CrawlData]):
|
||||
|
||||
return course_id, href.split("=")[-1]
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# helpers #
|
||||
# ------------------------------------------------------------------ #
|
||||
def _logged_in(self) -> bool:
|
||||
html = self._get_html(PUBLIC_URLS.index)
|
||||
return not parsel.Selector(text=html).css("div.usermenu span.login a")
|
||||
@@ -246,9 +222,6 @@ class Crawler(Worker[CrawlProgram, CrawlData]):
|
||||
get_run_logger().warning("GET %s failed (%s)", url, exc)
|
||||
return ""
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# cache helpers #
|
||||
# ------------------------------------------------------------------ #
|
||||
@staticmethod
|
||||
def _load_cache() -> set[str]:
|
||||
try:
|
||||
|
||||
+10
-58
@@ -22,11 +22,8 @@ class IndexCrawler:
|
||||
self.debug = debug
|
||||
self.client = httpx.Client(cookies=cookies, follow_redirects=True)
|
||||
self.max_workers = max_workers
|
||||
|
||||
# When True the cached “no-access” set is ignored for this run
|
||||
self._ignore_cache: bool = False
|
||||
|
||||
# Load persisted cache of course-IDs the user cannot access
|
||||
if NO_ACCESS_CACHE_FILE.exists():
|
||||
try:
|
||||
self._no_access_cache: set[str] = set(json.loads(NO_ACCESS_CACHE_FILE.read_text()))
|
||||
@@ -54,7 +51,7 @@ class IndexCrawler:
|
||||
|
||||
def crawl_index(self, userSpecific: bool = True, *, use_cache: bool = True) -> MoodleIndex:
|
||||
"""
|
||||
Build and return a `MoodleIndex`.
|
||||
Builds and returns a `MoodleIndex`.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
@@ -65,21 +62,17 @@ class IndexCrawler:
|
||||
afresh. Newly discovered “no-access” courses are still written back to the
|
||||
cache at the end of the crawl.
|
||||
"""
|
||||
# Set runtime flag for has_user_access()
|
||||
self._ignore_cache = not use_cache
|
||||
|
||||
semesters = []
|
||||
# Get all courses for each semester and the courseid and name for each course.
|
||||
semesters = self.crawl_semesters()
|
||||
# Crawl only the latest two semesters to reduce load (remove once caching is implemented)
|
||||
for semester in semesters[:2]:
|
||||
courses = self.crawl_courses(semester)
|
||||
|
||||
# Crawl courses in parallel to speed things up
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=self.max_workers) as pool:
|
||||
list(pool.map(self.crawl_course, courses))
|
||||
|
||||
# Filter courses once all have been processed
|
||||
for course in courses:
|
||||
if userSpecific:
|
||||
if course.content_ressource_id:
|
||||
@@ -87,8 +80,6 @@ class IndexCrawler:
|
||||
else:
|
||||
semester.courses.append(course)
|
||||
|
||||
# Only add semesters that have at least one course
|
||||
# Filter out semesters that ended up with no courses after crawling
|
||||
semesters: list[Semester] = [
|
||||
semester for semester in semesters if semester.courses
|
||||
]
|
||||
@@ -100,21 +91,13 @@ class IndexCrawler:
|
||||
semesters=semesters,
|
||||
),
|
||||
)
|
||||
# Persist any newly discovered no-access courses
|
||||
self._save_no_access_cache()
|
||||
|
||||
# Restore default behaviour for subsequent calls
|
||||
self._ignore_cache = False
|
||||
|
||||
return created_index
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
# High-level crawling helpers
|
||||
# --------------------------------------------------------------------- #
|
||||
def crawl_semesters(self) -> list[Semester]:
|
||||
"""
|
||||
Crawl the semesters from the Moodle index page.
|
||||
"""
|
||||
"""Crawls semester data."""
|
||||
url = URLs.get_degree_program_url(self.degree_program.id)
|
||||
res = self.get_with_retries(url)
|
||||
|
||||
@@ -126,9 +109,7 @@ class IndexCrawler:
|
||||
return []
|
||||
|
||||
def crawl_courses(self, semester: Semester) -> list[Course]:
|
||||
"""
|
||||
Crawl the courses from the Moodle index page.
|
||||
"""
|
||||
"""Crawls course data for a semester."""
|
||||
url = URLs.get_semester_url(semester_id=semester.id)
|
||||
res = self.get_with_retries(url)
|
||||
|
||||
@@ -140,10 +121,7 @@ class IndexCrawler:
|
||||
return []
|
||||
|
||||
def crawl_course(self, course: Course) -> None:
|
||||
"""
|
||||
Crawl a single Moodle course page.
|
||||
"""
|
||||
|
||||
"""Crawls details for a single course."""
|
||||
hasAccess = self.has_user_access(course)
|
||||
|
||||
if not hasAccess:
|
||||
@@ -154,13 +132,8 @@ class IndexCrawler:
|
||||
course.content_ressource_id = self.crawl_content_ressource_id(course)
|
||||
course.files = self.crawl_course_files(course)
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
# Networking utilities
|
||||
# --------------------------------------------------------------------- #
|
||||
def get_with_retries(self, url: str, retries: int = 3, delay: int = 1) -> httpx.Response:
|
||||
"""
|
||||
Simple GET with retries and exponential back-off.
|
||||
"""
|
||||
"""Simple GET with retries and exponential back-off."""
|
||||
for attempt in range(1, retries + 1):
|
||||
try:
|
||||
response = self.client.get(url)
|
||||
@@ -183,17 +156,11 @@ class IndexCrawler:
|
||||
f.write(response.text)
|
||||
logging.info(f"Saved HTML to {filename}")
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
# Extractors
|
||||
# --------------------------------------------------------------------- #
|
||||
def extract_semesters(self, html: str) -> list[Semester]:
|
||||
"""Extracts semester names and IDs from HTML."""
|
||||
selector = parsel.Selector(text=html)
|
||||
|
||||
logging.info("Extracting semesters from the HTML content.")
|
||||
|
||||
semesters: list[Semester] = []
|
||||
|
||||
# Each semester sits in a collapsed container
|
||||
semester_containers = selector.css("div.category.notloaded.with_children.collapsed")
|
||||
|
||||
for container in semester_containers:
|
||||
@@ -207,7 +174,6 @@ class IndexCrawler:
|
||||
)
|
||||
semester_id = anchor.attrib.get("href", "").split("=")[-1]
|
||||
|
||||
# Only keep semesters labeled FS or HS
|
||||
if "FS" not in semester_name and "HS" not in semester_name:
|
||||
continue
|
||||
|
||||
@@ -250,12 +216,9 @@ class IndexCrawler:
|
||||
)
|
||||
course_id = anchor.attrib.get("href", "").split("=")[-1]
|
||||
|
||||
# Remove trailing semester tag and code patterns
|
||||
course_name = re.sub(r"\s*(FS|HS)\d{2}\s*", "", course_name)
|
||||
course_name = re.sub(r"\s*\(.*?\)\s*", "", course_name).strip()
|
||||
|
||||
# Try to locate a hero/overview image that belongs to this course box
|
||||
# Traverse up to the containing course box, then look for <div class="courseimage"><img ...>
|
||||
course_container = header.xpath('./ancestor::*[contains(@class,"coursebox")][1]')
|
||||
hero_src = (
|
||||
course_container.css("div.courseimage img::attr(src)").get("")
|
||||
@@ -275,10 +238,7 @@ class IndexCrawler:
|
||||
return courses
|
||||
|
||||
def has_user_access(self, course: Course) -> bool:
|
||||
"""
|
||||
Return True only if the authenticated user can access the course (result cached).
|
||||
(i.e. the response is HTTP 200 **and** is not a redirected login/enrol page).
|
||||
"""
|
||||
"""Checks if user can access course (caches negative results)."""
|
||||
if not self._ignore_cache and course.id in self._no_access_cache:
|
||||
return False
|
||||
|
||||
@@ -289,21 +249,19 @@ class IndexCrawler:
|
||||
self._no_access_cache.add(course.id)
|
||||
return False
|
||||
|
||||
# Detect Moodle redirection to a login or enrolment page
|
||||
final_url = str(res.url).lower()
|
||||
if "login" in final_url or "enrol" in final_url:
|
||||
self._no_access_cache.add(course.id)
|
||||
return False
|
||||
|
||||
# Some enrolment pages still return 200; look for HTML markers
|
||||
if "#page-enrol" in res.text or "you need to enrol" in res.text.lower():
|
||||
self._no_access_cache.add(course.id)
|
||||
return False
|
||||
|
||||
# If we got here the user has access; otherwise cache the deny
|
||||
return True
|
||||
|
||||
def crawl_content_ressource_id(self, course: Course) -> str:
|
||||
"""Extracts content resource ID for a course."""
|
||||
course_id = course.id
|
||||
url = URLs.get_course_url(course_id)
|
||||
res = self.get_with_retries(url)
|
||||
@@ -311,12 +269,10 @@ class IndexCrawler:
|
||||
|
||||
try:
|
||||
logging.info("Searching for 'Download course content' link.")
|
||||
# Use parsel CSS selector to find the anchor tag with the specific data attribute
|
||||
download_link_selector = psl.css('a[data-downloadcourse="1"]')
|
||||
if not download_link_selector:
|
||||
raise ValueError("Download link not found.")
|
||||
|
||||
# Extract the href attribute from the first matching element
|
||||
href = download_link_selector[0].attrib.get("href")
|
||||
if not href:
|
||||
raise ValueError("Href attribute not found on the download link.")
|
||||
@@ -334,9 +290,7 @@ class IndexCrawler:
|
||||
return ''
|
||||
|
||||
def crawl_course_files(self, course: Course) -> list[FileEntry]:
|
||||
"""
|
||||
Crawl the course files from the Moodle course page.
|
||||
"""
|
||||
"""Crawls file entries for a course."""
|
||||
url = URLs.get_course_url(course.id)
|
||||
res = self.get_with_retries(url)
|
||||
|
||||
@@ -347,10 +301,8 @@ class IndexCrawler:
|
||||
|
||||
return []
|
||||
|
||||
# ----------------------------------------------------------------- #
|
||||
# Cache persistence helpers
|
||||
# ----------------------------------------------------------------- #
|
||||
def _save_no_access_cache(self) -> None:
|
||||
"""Saves course IDs the user cannot access to a cache file."""
|
||||
try:
|
||||
NO_ACCESS_CACHE_FILE.write_text(json.dumps(sorted(self._no_access_cache)))
|
||||
except Exception as exc:
|
||||
|
||||
@@ -1,9 +1,5 @@
|
||||
# TODO: Move to librarian-core
|
||||
"""
|
||||
All URLs used in the crawler.
|
||||
Functions marked as PUBLIC can be accessed without authentication.
|
||||
Functions marked as PRIVATE require authentication.
|
||||
"""
|
||||
"""Moodle URLs. PUBLIC/PRIVATE indicates auth requirement."""
|
||||
class URLs:
|
||||
base_url = "https://moodle.fhgr.ch"
|
||||
|
||||
@@ -12,7 +8,6 @@ class URLs:
|
||||
"""PUBLIC"""
|
||||
return cls.base_url
|
||||
|
||||
# ------------------------- Moodle URLs -------------------------
|
||||
@classmethod
|
||||
def get_login_url(cls):
|
||||
"""PUBLIC"""
|
||||
|
||||
@@ -35,9 +35,6 @@ from librarian_scraper.models.download_data import (
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# helper decorator #
|
||||
# --------------------------------------------------------------------------- #
|
||||
def task_(**kw):
|
||||
kw.setdefault("log_prints", True)
|
||||
kw.setdefault("retries", 2)
|
||||
@@ -45,9 +42,6 @@ def task_(**kw):
|
||||
return task(**kw)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# shared state for static task #
|
||||
# --------------------------------------------------------------------------- #
|
||||
_COOKIE_JAR: httpx.Cookies | None = None
|
||||
_SESSKEY: str = ""
|
||||
_LIMIT: int = 2
|
||||
@@ -57,27 +51,22 @@ _DELAY: float = 0.0
|
||||
class Downloader(Worker[CrawlData, DownloadData]):
|
||||
DOWNLOAD_URL = "https://moodle.fhgr.ch/course/downloadcontent.php"
|
||||
|
||||
# tuning
|
||||
CONCURRENCY = 8
|
||||
RELAXED = True # False → faster
|
||||
RELAXED = True
|
||||
|
||||
input_model = CrawlData
|
||||
output_model = DownloadData
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
async def __run__(self, crawl: CrawlData) -> DownloadData:
|
||||
global _COOKIE_JAR, _SESSKEY, _LIMIT, _DELAY
|
||||
lg = get_run_logger()
|
||||
|
||||
# ------------ login
|
||||
cookies, sesskey = await CookieCrawler().crawl()
|
||||
_COOKIE_JAR, _SESSKEY = cookies, sesskey
|
||||
|
||||
# ------------ tuning
|
||||
_LIMIT = 1 if self.RELAXED else max(1, min(self.CONCURRENCY, 8))
|
||||
_DELAY = CRAWLER["DELAY_SLOW"] if self.RELAXED else CRAWLER["DELAY_FAST"]
|
||||
|
||||
# ------------ working dir
|
||||
work_root = Path(get_temp_path()) / f"dl_{int(time.time())}"
|
||||
work_root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
@@ -85,7 +74,6 @@ class Downloader(Worker[CrawlData, DownloadData]):
|
||||
futures = []
|
||||
term_dirs: List[Tuple[str, Path]] = []
|
||||
|
||||
# schedule downloads
|
||||
for term in crawl.degree_program.terms:
|
||||
term_dir = work_root / term.name
|
||||
term_dir.mkdir(parents=True, exist_ok=True)
|
||||
@@ -101,18 +89,14 @@ class Downloader(Worker[CrawlData, DownloadData]):
|
||||
self._download_task.submit(course.content_ressource_id, dest)
|
||||
)
|
||||
|
||||
wait(futures) # block for all downloads
|
||||
wait(futures)
|
||||
|
||||
# stage term directories
|
||||
for name, dir_path in term_dirs:
|
||||
self.stage(dir_path, new_name=name, sanitize=False, move=True)
|
||||
|
||||
lg.info("Downloader finished – staged %d term folders", len(term_dirs))
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# static task #
|
||||
# ------------------------------------------------------------------ #
|
||||
@staticmethod
|
||||
@task_()
|
||||
def _download_task(context_id: str, dest: Path) -> None:
|
||||
|
||||
@@ -135,9 +135,6 @@ MoodleIndex: {
|
||||
"""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Base Model
|
||||
# ---------------------------------------------------------------------------
|
||||
class CrawlData(BaseModel):
|
||||
degree_program: CrawlProgram = Field(
|
||||
default_factory=lambda: CrawlProgram(id="", name="")
|
||||
@@ -147,18 +144,12 @@ class CrawlData(BaseModel):
|
||||
)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Degree Program
|
||||
# ---------------------------------------------------------------------------
|
||||
class CrawlProgram(BaseModel):
|
||||
id: str = Field("1157", description="Unique identifier for the degree program.")
|
||||
name: str = Field("Computational and Data Science", description="Name of the degree program.")
|
||||
terms: list[CrawlTerm] = Field(default_factory=list)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Term
|
||||
# ---------------------------------------------------------------------------
|
||||
_TERM_RE = re.compile(r"^(HS|FS)\d{2}$") # HS24 / FS25 …
|
||||
|
||||
|
||||
@@ -168,9 +159,6 @@ class CrawlTerm(BaseModel):
|
||||
courses: list[CrawlCourse] = Field(default_factory=list)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Course
|
||||
# ---------------------------------------------------------------------------
|
||||
class CrawlCourse(BaseModel):
|
||||
id: str
|
||||
name: str
|
||||
@@ -179,9 +167,6 @@ class CrawlCourse(BaseModel):
|
||||
files: list[CrawlFile] = Field(default_factory=list)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Files
|
||||
# ---------------------------------------------------------------------------
|
||||
class CrawlFile(BaseModel):
|
||||
id: str
|
||||
res_id: str
|
||||
|
||||
Reference in new issue
Block a user