From e06d698802885c51f662fe94fb4863803b013f15 Mon Sep 17 00:00:00 2001 From: herzogflorian Date: Mon, 7 Sep 2026 18:49:29 +0200 Subject: [PATCH] Auto-commit 2026-09-07 18:49: 1 file changed, 215 insertions(+), 192 deletions(-) --- moodle/src/fill_mbz.py | 395 ++++++++++++++++++++++------------------- 1 file changed, 209 insertions(+), 186 deletions(-) diff --git a/moodle/src/fill_mbz.py b/moodle/src/fill_mbz.py index 4dde0c5..1263310 100644 --- a/moodle/src/fill_mbz.py +++ b/moodle/src/fill_mbz.py @@ -1,41 +1,38 @@ #!/usr/bin/env python3 -"""Fill an AISE502 Moodle course backup (.mbz, moodle2 format) with the generated section content. +"""Add the AISE502 course-page content to a Moodle course backup (.mbz, moodle2 format, Moodle 4.4+/5.x). - python3 src/fill_mbz.py inspect - lists Moodle release, course format, every section (number, id, name) and the activities in it - python3 src/fill_mbz.py fill [--summary] - week sections 1..14: section name -> "Week N: ...", the three template text areas (Lernziele / - Theorie / Selbststudium) -> blocks 1..3 of week_NN.html; two sections appended for the lecture - script and the project (cloned from the last week section, one text area each). - --summary: write each week's three blocks into the section description instead of text areas. + python3 moodle/src/fill_mbz.py inspect + python3 moodle/src/fill_mbz.py fill [--text-only] + --text-only: text areas only, no file resources (small archive; upload the PDFs by hand) -Only text fields are changed; ids of new sections/labels are fresh unique numbers; everything else in -the archive is copied untouched. The result is validated structurally (all referenced directories exist, -XML parses) but a restore in Moodle is the real test. +fill appends 16 sections after the existing ones (weeks 1-14, lecture script, project exercise): + week N : text area block 1 | text area block 2 | file: slides of lecture N | text area block 3 + script : text area (lecture_script.html) | file: script PDF + project : text area (project_exercise.html) | files: exercise sheet PDF, project slides PDF, UI sketches ZIP +Existing sections and activities stay untouched. New sections are cloned from the template week +("Woche n"), text areas from a plain label, file resources from an existing file resource. +Restore the result into the course with "Zusammenführen" (merge) and untick the existing sections +in the schema step, so that nothing is duplicated. """ -import io, pathlib, re, shutil, sys, tarfile, tempfile, zipfile, html +import hashlib, html, itertools, json, pathlib, re, shutil, sys, tarfile, tempfile, time, zipfile import xml.etree.ElementTree as ET -HERE = pathlib.Path(__file__).resolve().parent -OUT = HERE.parent +REPO = pathlib.Path(__file__).resolve().parents[2] +MOODLE = REPO / "moodle" +BASE_ID = 20_000_000 -# ----------------------------------------------------------------------------- archive helpers +# ----------------------------------------------------------------------------- helpers def unpack(mbz, dest): - data = open(mbz, "rb").read(4) - if data[:2] == b"PK": - with zipfile.ZipFile(mbz) as z: z.extractall(dest); return "zip" - with tarfile.open(mbz, "r:gz") as t: t.extractall(dest); return "tgz" - -def pack(src, out, kind): - src = pathlib.Path(src) - if kind == "zip": - with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as z: - for p in sorted(src.rglob("*")): - if p.is_file(): z.write(p, p.relative_to(src).as_posix()) + if open(mbz, "rb").read(2) == b"PK": + with zipfile.ZipFile(mbz) as z: z.extractall(dest) else: - with tarfile.open(out, "w:gz") as t: - for p in sorted(src.rglob("*")): - if p.is_file(): t.add(p, p.relative_to(src).as_posix()) + with tarfile.open(mbz, "r:gz") as t: t.extractall(dest) + +def pack_zip(src, out): + src = pathlib.Path(src) + with zipfile.ZipFile(out, "w", zipfile.ZIP_DEFLATED) as z: + for p in sorted(src.rglob("*")): + if p.is_file(): z.write(p, p.relative_to(src).as_posix()) def read(p): return pathlib.Path(p).read_text(encoding="utf-8") def write(p, s): pathlib.Path(p).write_text(s, encoding="utf-8") @@ -44,197 +41,223 @@ def tag(xml, name): m = re.search(rf"<{name}>(.*?)", xml, re.S) return html.unescape(m.group(1)) if m else None -def set_tag(xml, name, value, count=1): - esc = html.escape(value, quote=False) - new, n = re.subn(rf"<{name}>.*?", lambda m: f"<{name}>{esc}", xml, count=count, flags=re.S) - if n == 0: # empty tag form or - new, n = re.subn(rf"<{name}\s*/>", f"<{name}>{esc}", xml, count=count) - assert n, f"tag {name} not found" +def set_tag(xml, name, value): + esc = html.escape(str(value), quote=False) + new, n = re.subn(rf"<{name}>.*?", lambda m: f"<{name}>{esc}", xml, count=1, flags=re.S) + if n == 0: + new, n = re.subn(rf"<{name}\s*/>", f"<{name}>{esc}", xml, count=1) + assert n, f"tag <{name}> not found" return new +def text_of(intro, n=60): + t = html.unescape(re.sub(r"<[^>]+>", " ", intro)); t = re.sub(r"\s+", " ", t).strip() + return t[:n] + # ----------------------------------------------------------------------------- model class Backup: def __init__(self, root): self.root = pathlib.Path(root) self.mb = read(self.root / "moodle_backup.xml") self.release = tag(self.mb, "moodle_release") - self.format = tag(self.mb, "format") # course format e.g. topics - # sections: from moodle_backup.xml contents + self.course_format = tag(self.mb, "original_course_format") + contents = re.search(r"(.*?)", self.mb, re.S).group(1) self.sections = [] - for m in re.finditer(r"
\s*(\d+)\s*(.*?)\s*(.*?)\s*
", self.mb, re.S): - sid, title, d = int(m.group(1)), m.group(2), m.group(3) - sx = read(self.root / d / "section.xml") - self.sections.append(dict(id=sid, dir=d, number=int(tag(sx, "number")), name=tag(sx, "name") or "", + for s in re.findall(r"
.*?
", contents, re.S): + d = tag(s, "directory"); sx = read(self.root / d / "section.xml") + self.sections.append(dict(id=int(tag(s, "sectionid")), dir=d, number=int(tag(sx, "number")), name=tag(sx, "name") or "", sequence=[int(x) for x in (tag(sx, "sequence") or "").split(",") if x.strip()], - visible=tag(sx, "visible"))) + visible=tag(sx, "visible"), component=tag(sx, "component"))) self.sections.sort(key=lambda s: s["number"]) self.activities = {} - for m in re.finditer(r"\s*(\d+)\s*(\d+)\s*(.*?)\s*(.*?)\s*(.*?)\s*", self.mb, re.S): - mid, sid, mod, title, d = int(m.group(1)), int(m.group(2)), m.group(3), html.unescape(m.group(4)), m.group(5) - a = dict(moduleid=mid, sectionid=sid, modname=mod, title=title, dir=d) - mx = read(self.root / d / "module.xml") - a["visible"] = tag(mx, "visible") - if mod == "label": - lx = read(self.root / d / "label.xml") - a["intro"] = tag(lx, "intro") or "" - self.activities[mid] = a - - def label_kind(self, a): - t = re.sub(r"<[^>]+>", " ", a.get("intro", "")) - t = html.unescape(t) - if a["visible"] == "0" or "Auftrag Dozierende" in t: return "hidden" - if "Lernziele" in t or "Learning objectives" in t: return "objectives" - if "Theorie" in t or "Theory" in t: return "theory" - if "Selbststudium" in t or "Self-study" in t: return "selfstudy" - return "other" + for a in re.findall(r".*?", contents, re.S): + mid = int(tag(a, "moduleid")); d = tag(a, "directory"); mod = tag(a, "modulename") + rec = dict(moduleid=mid, sectionid=int(tag(a, "sectionid")), modname=mod, title=tag(a, "title"), dir=d) + rec["visible"] = tag(read(self.root / d / "module.xml"), "visible") + fr = re.search(r"(.*?)", read(self.root / d / "inforef.xml"), re.S) + rec["filerefs"] = [int(x) for x in re.findall(r"(\d+)", fr.group(1))] if fr else [] + if mod == "label": rec["intro"] = tag(read(self.root / d / "label.xml"), "intro") or "" + if mod == "resource": + fx = self.files_xml() + rec["files"] = [tag(f, "filename") for f in re.findall(r".*?", fx, re.S) + if int(re.search(r']*-->\n", t) - assert len(parts) == 4, f"week {week}: expected 3 blocks, got {len(parts)-1}" + assert len(parts) == 4, f"week {n}: expected 3 blocks" title = re.search(r"Section name\): (.*)", t).group(1).strip() - return title, [p.strip() + "\n" for p in parts[1:]] + blocks = [p.strip() + "\n" for p in parts[1:]] + if with_files: + blocks[1] = blocks[1].replace("(added below by the lecturer)", "(file below)") + return title, blocks -def label_name(intro): - t = html.unescape(re.sub(r"<[^>]+>", " ", intro)); t = re.sub(r"\s+", " ", t).strip() - return t[:60] +def fixed_section(name, with_files=True): + t = read(MOODLE / name) + t = re.sub(r"^\s*", "", t, count=1, flags=re.S) + if not with_files: + return t.strip() + "\n" + t = t.replace("AISE502_Vorlesung_Skript.pdf – [link]", "AISE502_Vorlesung_Skript.pdf – the file directly below") + t = t.replace(" [link]", " (file below)") + return t.strip() + "\n" -def set_label(b, a, intro_html): - p = b.root / a["dir"] / "label.xml" - lx = read(p) - lx = set_tag(lx, "intro", intro_html) - lx = set_tag(lx, "name", label_name(intro_html)) - lx = set_tag(lx, "introformat", "1") - write(p, lx) - # title in moodle_backup.xml - b.mb = re.sub(rf"({a['moduleid']}\s*\d+\s*label\s*).*?()", - lambda m: m.group(1) + html.escape(label_name(intro_html), quote=False) + m.group(2), b.mb, count=1, flags=re.S) +def sketches_zip(dest): + src = REPO / "project_exercise" / "ui_sketches" + with zipfile.ZipFile(dest, "w", zipfile.ZIP_DEFLATED) as z: + for p in sorted(src.iterdir()): + if p.suffix in (".html", ".png", ".css"): z.write(p, f"ui_sketches/{p.name}") + return dest -def set_section_name(b, s, name): - p = b.root / s["dir"] / "section.xml" - write(p, set_tag(read(p), "name", name)) - s["name"] = name +# ----------------------------------------------------------------------------- building +class Builder: + def __init__(self, b, scratch): + self.b = b; self.ids = itertools.count(BASE_ID); self.now = int(time.time()); self.scratch = scratch + self.fx = b.files_xml() + self.userid = tag(re.search(r".*?", self.fx, re.S).group(0), "userid") or "0" + secs = [s for s in b.sections if s["component"] in (None, "$@NULL@$")] + self.tmpl_section = next((s for s in secs if s["name"].startswith("Woche n")), secs[-1]) + labels = [a for a in b.activities.values() if a["modname"] == "label"] + self.tmpl_label = next((a for a in labels if not a["filerefs"]), labels[0]) + self.tmpl_resource = next(a for a in b.activities.values() if a["modname"] == "resource") + self.next_number = max(s["number"] for s in b.sections) + 1 + self.report = [] -def set_section_summary(b, s, summary_html): - p = b.root / s["dir"] / "section.xml" - sx = set_tag(read(p), "summary", summary_html) - sx = set_tag(sx, "summaryformat", "1") - write(p, sx) + def _register_activity(self, mod, mid, sid, title, adir): + entry = (f" \n {mid}\n {sid}\n" + f" {mod}\n {html.escape(title, quote=False)}\n" + f" {adir}\n \n \n") + self.b.mb = self.b.mb.replace(" ", entry + " ", 1) + for sname, val in ((f"{mod}_{mid}_included", "1"), (f"{mod}_{mid}_userinfo", "0")): + self.b.mb = self.b.mb.replace(" ", f" \n activity\n {mod}_{mid}\n {sname}\n {val}\n \n ", 1) -def next_ids(b): - """fresh ids above everything used in the archive""" - nums = [int(x) for x in re.findall(r'id="(\d+)"|(\d+)|_(\d+)/|(\d+)|(\d+)|contextid="(\d+)"', b.mb) for x in x if x] if False else [] - allx = "\n".join(read(p) for p in b.root.rglob("*.xml")) - nums = [int(n) for n in re.findall(r"\b(\d{1,9})\b", allx)] - base = (max(nums) // 1000 + 1) * 1000 - return base - -def clone_section(b, template, number, name, label_intros, base): - """clone a week section directory as section `number` with len(label_intros) label activities (cloned from the template's first label)""" - tsec = template - sid = base; base += 1 - sdir = f"sections/section_{sid}" - shutil.copytree(b.root / tsec["dir"], b.root / sdir) - sx = read(b.root / sdir / "section.xml") - sx = re.sub(r'
" in mx else mx - write(b.root / adir / "module.xml", mx) - lx = read(b.root / adir / "label.xml") + mx = set_tag(mx, "sectionid", sid); mx = set_tag(mx, "sectionnumber", number) + mx = set_tag(mx, "added", self.now); mx = set_tag(mx, "visible", 1); mx = set_tag(mx, "visibleold", 1) + mx = set_tag(mx, "visibleoncoursepage", 1); mx = set_tag(mx, "showdescription", showdescription) + write(self.b.root / adir / "module.xml", mx) + return adir + + def add_label(self, sid, number, intro): + mid, aid, ctx = next(self.ids), next(self.ids), next(self.ids) + adir = self._clone_module(self.tmpl_label, "label", mid, sid, number, 1) + lx = read(self.b.root / adir / "label.xml") lx = re.sub(r'\n\n') - # register in moodle_backup.xml: contents/activities + settings - entry = (f" \n {mid}\n {sid}\n" - f" label\n {html.escape(label_name(intro), quote=False)}\n" - f" {adir}\n \n") - b.mb = b.mb.replace("", entry + " ", 1) - for sname, val in ((f"label_{mid}_included", "1"), (f"label_{mid}_userinfo", "0")): - b.mb = b.mb.replace("", f" \n activity\n label_{mid}\n {sname}\n {val}\n \n ", 1) - mids.append(mid) - sx = set_tag(sx, "sequence", ",".join(str(m) for m in mids)) - write(b.root / sdir / "section.xml", sx) - write(b.root / sdir / "inforef.xml", '\n\n') - entry = (f"
\n {sid}\n {number}\n {sdir}\n
\n") - b.mb = b.mb.replace("", entry + " ", 1) - for sname, val in ((f"section_{sid}_included", "1"), (f"section_{sid}_userinfo", "0")): - b.mb = b.mb.replace("", f" \n section\n
section_{sid}
\n {sname}\n {val}\n \n ", 1) - return base + lx = set_tag(lx, "name", text_of(intro)); lx = set_tag(lx, "intro", intro) + lx = set_tag(lx, "introformat", 1); lx = set_tag(lx, "timemodified", self.now) + write(self.b.root / adir / "label.xml", lx) + write(self.b.root / adir / "inforef.xml", '\n\n') + self._register_activity("label", mid, sid, text_of(intro), adir) + return mid -def fill(mbz, out, use_summary=False): - tmp = tempfile.mkdtemp() - kind = unpack(mbz, tmp) - b = Backup(tmp) - weeks = {s["number"]: s for s in b.sections if 1 <= s["number"] <= 14} - assert len(weeks) == 14, f"expected sections 1..14, found {sorted(weeks)}" - report = [] + def _file_entry(self, fid, ctx, filename, data, mimetype): + h = hashlib.sha1(data).hexdigest() + if filename != ".": + d = self.b.root / "files" / h[:2]; d.mkdir(exist_ok=True); (d / h).write_bytes(data) + null = "$@NULL@$" + return (f' \n {h}\n {ctx}\n' + f' mod_resource\n content\n 0\n' + f' /\n {html.escape(filename, quote=False)}\n {self.userid}\n' + f' {len(data)}\n {mimetype if filename != "." else null}\n 0\n' + f' {self.now}\n {self.now}\n' + f' {html.escape(filename, quote=False) if filename != "." else null}\n {null}\n' + f' {"allrightsreserved" if filename != "." else null}\n {1 if filename != "." else 0}\n' + f' {null}\n {null}\n {null}\n \n') + + def add_file(self, sid, number, name, path, mimetype): + mid, aid, ctx, fid1, fid2 = (next(self.ids) for _ in range(5)) + adir = self._clone_module(self.tmpl_resource, "resource", mid, sid, number, 0) + rx = read(self.b.root / adir / "resource.xml") + rx = re.sub(r'", entries + "", 1) + write(self.b.root / adir / "inforef.xml", f'\n\n \n \n {fid1}\n \n \n {fid2}\n \n \n') + self._register_activity("resource", mid, sid, name, adir) + return mid + + def add_section(self, name, items): + sid = next(self.ids); number = self.next_number; self.next_number += 1 + sdir = f"sections/section_{sid}" + shutil.copytree(self.b.root / self.tmpl_section["dir"], self.b.root / sdir) + mids = [] + for kind, *args in items: + mids.append(self.add_label(sid, number, *args) if kind == "label" else self.add_file(sid, number, *args)) + sx = read(self.b.root / sdir / "section.xml") + sx = re.sub(r'
\n\n') + entry = (f"
\n {sid}\n {number}\n" + f" {sdir}\n \n \n
\n") + self.b.mb = self.b.mb.replace(" ", entry + " ", 1) + for sname, val in ((f"section_{sid}_included", "1"), (f"section_{sid}_userinfo", "0")): + self.b.mb = self.b.mb.replace(" ", f" \n section\n
section_{sid}
\n {sname}\n {val}\n \n ", 1) + self.report.append(f"section {number}: {name} ({len(items)} items)") + + def finish(self): + write(self.b.root / "moodle_backup.xml", self.b.mb); write(self.b.root / "files.xml", self.fx) + +def fill(mbz, out, with_files=True): + tmp = pathlib.Path(tempfile.mkdtemp()); unpack(mbz, tmp) + b = Backup(tmp); bl = Builder(b, tmp) + files = (lambda *items: list(items)) if with_files else (lambda *items: []) for n in range(1, 15): - s = weeks[n]; title, blocks = blocks_of(n) - set_section_name(b, s, title) - if use_summary: - set_section_summary(b, s, "\n".join(blocks)); report.append(f"week {n}: name + summary"); continue - kinds = {} - for mid in s["sequence"]: - a = b.activities[mid] - if a["modname"] == "label": - k = b.label_kind(a) - if k in ("objectives", "theory", "selfstudy") and k not in kinds: kinds[k] = a - missing = [k for k in ("objectives", "theory", "selfstudy") if k not in kinds] - assert not missing, f"week {n}: no template text area for {missing}; use --summary or adjust label_kind()" - for k, blk in zip(("objectives", "theory", "selfstudy"), blocks): - set_label(b, kinds[k], blk) - report.append(f"week {n}: name + 3 text areas") - # two extra sections - base = next_ids(b) - last = max(b.sections, key=lambda s: s["number"]) - nxt = last["number"] + 1 - script_html = read(OUT / "lecture_script.html"); project_html = read(OUT / "project_exercise.html") - def body(t): return re.sub(r"\s*", "", t, count=1, flags=re.S).strip() + "\n" - base = clone_section(b, weeks[14], nxt, "📘 Lecture script", [body(script_html)], base) - base = clone_section(b, weeks[14], nxt + 1, "🛠️ Project exercise", [body(project_html)], base) - report.append(f"sections {nxt} (script) and {nxt+1} (project) appended") - write(pathlib.Path(tmp) / "moodle_backup.xml", b.mb) - # structural validation + title, blocks = week_blocks(n, with_files) + sub = json.loads(read(MOODLE / "src" / "content" / f"week_{n:02d}.json"))["deck_subtitle"] + slides = REPO / "Folien" / f"AISE502_Vorlesung_{n}_Folien.pdf" + assert slides.exists(), slides + bl.add_section(title, [("label", blocks[0]), ("label", blocks[1]), + *files(("file", f"Slides – {sub}", slides, "application/pdf")), ("label", blocks[2])]) + bl.add_section("📘 Lecture script", [("label", fixed_section("lecture_script.html", with_files)), + *files(("file", "Lecture script – AISE502_Vorlesung_Skript.pdf", REPO / "skript" / "AISE502_Vorlesung_Skript.pdf", "application/pdf"))]) + zpath = sketches_zip(tmp.parent / "ui_sketches.zip") if with_files else None + bl.add_section("🛠️ Project exercise", [("label", fixed_section("project_exercise.html", with_files)), + *files(("file", "Exercise sheet – project_exercise.pdf", REPO / "project_exercise" / "project_exercise.pdf", "application/pdf"), + ("file", "Project slides – AISE502_Projekt_Folien.pdf", REPO / "Folien" / "AISE502_Projekt_Folien.pdf", "application/pdf"), + ("file", "UI sketches – HTML and PNG (ZIP)", zpath, "application/zip"))]) + bl.finish() + # ---- validation b2 = Backup(tmp) for s in b2.sections: - for mid in s["sequence"]: - assert mid in b2.activities, f"section {s['number']} references unknown module {mid}" - for p in pathlib.Path(tmp).rglob("*.xml"): - ET.parse(p) - pack(tmp, out, kind) - shutil.rmtree(tmp) - print("\n".join(report)); print(f"wrote {out} ({kind}); Moodle release in backup: {b2.release}; sections now: {[s['number'] for s in b2.sections]}") + for mid in s["sequence"]: assert mid in b2.activities, f"section {s['number']} references unknown module {mid}" + for p in tmp.rglob("*.xml"): ET.parse(p) + fx = b2.files_xml(); ids = set() + for f in re.findall(r"(.*?)", fx, re.S): + ids.add(int(f[0])); h = tag(f[1], "contenthash") + if tag(f[1], "filename") != ".": assert (tmp / "files" / h[:2] / h).exists(), f"content missing for file {f[0]}" + for a in b2.activities.values(): + for r in a["filerefs"]: assert r in ids, f"{a['dir']} refers to unknown file {r}" + pack_zip(tmp, out); shutil.rmtree(tmp) + print("\n".join(bl.report)); print(f"wrote {out} ({pathlib.Path(out).stat().st_size/1e6:.1f} MB) · sections now {[s['number'] for s in b2.sections]}") if __name__ == "__main__": - if len(sys.argv) >= 3 and sys.argv[1] == "inspect": inspect(sys.argv[2]) - elif len(sys.argv) >= 4 and sys.argv[1] == "fill": fill(sys.argv[2], sys.argv[3], "--summary" in sys.argv) + if len(sys.argv) == 3 and sys.argv[1] == "inspect": inspect(sys.argv[2]) + elif len(sys.argv) >= 4 and sys.argv[1] == "fill": fill(sys.argv[2], sys.argv[3], "--text-only" not in sys.argv) else: print(__doc__)