Initialize Monorepo

This commit is contained in:
DotNaos committed 2025-05-24 12:15:48 +02:00
commit f80792d739
106 files changed
+17076

No files matched your search

@@ -0,0 +1,20 @@
import pkgutil
import importlib
__all__ = []
# Iterate over all modules in this package
for finder, module_name, is_pkg in pkgutil.iter_modules(__path__):
# import the sub-module
module = importlib.import_module(f"{__name__}.{module_name}")
# decide which names to re-export:
# use module.__all__ if it exists, otherwise every non-private attribute
public_names = getattr(
module, "__all__", [n for n in dir(module) if not n.startswith("_")]
)
# bring each name into the package namespace
for name in public_names:
globals()[name] = getattr(module, name)
__all__.append(name) # type: ignore
Whitespace-only changes.
@@ -0,0 +1,5 @@
from librarian_core.storage.worker_store import WorkerStore
__all__ = [
"WorkerStore",
]
@@ -0,0 +1,243 @@
"""
librarian_core.storage.worker_store
===================================
Persistent directory layout
---------------------------
<data_root>/flows/<worker>/<run_id>/
meta.json # worker_name, state, timestamps …
result.json # pydantic-serialised return model
data/ # files staged by the worker
"""
from __future__ import annotations
import json
import shutil
import tempfile
from datetime import datetime, timezone
from pathlib import Path
from typing import Any, Optional, Type
from pydantic import BaseModel, TypeAdapter
from librarian_core.utils import path_utils
class WorkerStore:
"""Never exposed to worker code – all access is via helper methods."""
# ------------------------------------------------------------------ #
# constructors #
# ------------------------------------------------------------------ #
@classmethod
def new(cls, *, worker_name: str, flow_id: str) -> "WorkerStore":
run_dir = path_utils.get_run_dir(worker_name, flow_id, create=True)
store = cls(run_dir, worker_name, flow_id)
store._write_meta(state="RUNNING")
return store
@classmethod
def open(cls, run_id: str) -> "WorkerStore":
"""
Locate `<flows>/<worker>/<run_id>` regardless of worker name.
"""
flows_dir = path_utils.get_flows_dir()
for worker_dir in flows_dir.iterdir():
candidate = worker_dir / run_id
if candidate.exists():
meta_path = candidate / "meta.json"
if not meta_path.is_file():
continue
meta = json.loads(meta_path.read_text())
return cls(candidate, meta["worker_name"], run_id)
raise FileNotFoundError(run_id)
# ------------------------------------------------------------------ #
# life-cycle #
# ------------------------------------------------------------------ #
def __init__(self, run_dir: Path, worker_name: str, flow_id: str):
self._run_dir = run_dir
self._worker_name = worker_name
self._flow_id = flow_id
cache_root = path_utils.get_cache_root()
self._work_dir = Path(
tempfile.mkdtemp(prefix=f"{self._flow_id}-", dir=cache_root)
)
self._entry_dir = self._work_dir / "entry"
self._exit_dir = self._work_dir / "exit"
self._entry_dir.mkdir(parents=True, exist_ok=True)
self._exit_dir.mkdir(parents=True, exist_ok=True)
# ------------------------------------------------------------------ #
# entry / exit handling #
# ------------------------------------------------------------------ #
@property
def entry_dir(self) -> Path:
return self._entry_dir
def prime_with_input(self, src: Optional[Path]) -> None:
if src and src.exists():
shutil.copytree(src, self._entry_dir, dirs_exist_ok=True)
def stage(
self,
src: Path | str,
*,
new_name: str | None = None,
sanitize: bool = True,
move: bool = False,
) -> Path:
src_path = Path(src).expanduser().resolve()
if not src_path.exists():
raise FileNotFoundError(src_path)
name = new_name or src_path.name
if sanitize:
name = path_utils._sanitize(name)
dst = self._exit_dir / name
if dst.exists():
if dst.is_file():
dst.unlink()
else:
shutil.rmtree(dst)
if move:
src_path.rename(dst)
else:
if src_path.is_dir():
shutil.copytree(src_path, dst)
else:
dst.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(src_path, dst)
return dst
# ------------------------------------------------------------------ #
# result persistence #
# ------------------------------------------------------------------ #
def save_model(
self,
model: BaseModel,
*,
filename: str = "result.json",
**json_kwargs: Any,
) -> Path:
json_kwargs.setdefault("indent", 2)
target = self._run_dir / filename
target.write_text(model.model_dump_json(**json_kwargs))
return target
def persist_exit(self) -> Path:
"""
Move the *exit* directory to the persistent *data* slot and mark
the run completed.
"""
data_dir = self.data_dir
if data_dir.exists():
shutil.rmtree(data_dir)
self._exit_dir.rename(data_dir)
self._write_meta(state="COMPLETED")
return data_dir
def cleanup(self) -> None:
shutil.rmtree(self._work_dir, ignore_errors=True)
# ------------------------------------------------------------------ #
# public helpers (API needs these) #
# ------------------------------------------------------------------ #
@property
def data_dir(self) -> Path:
return self._run_dir / "data"
@property
def meta_path(self) -> Path:
return self._run_dir / "meta.json"
@property
def metadata(self) -> dict[str, Any]:
return json.loads(self.meta_path.read_text())
def load_model(self, *, as_dict: bool = False) -> dict | BaseModel | None:
res_file = self._run_dir / "result.json"
if not res_file.is_file():
return None
data = json.loads(res_file.read_text())
if as_dict:
return data
# try to reconstruct a Pydantic model if possible
try:
OutputModel: Type[BaseModel] | None = self._guess_output_model()
if OutputModel:
return TypeAdapter(OutputModel).validate_python(data)
except Exception:
pass
return data
@staticmethod
# TODO: Should return a Flowartifact, but circular import is messing
def load_latest(worker_name: str) -> dict[str, Any] | None:
flows_dir = path_utils.get_flows_dir()
worker_dir = flows_dir / worker_name
if not worker_dir.exists():
return None
runs: list[tuple[datetime, Path]] = []
for run_id in worker_dir.iterdir():
if not run_id.is_dir():
continue
meta_path = run_id / "meta.json"
if not meta_path.is_file():
continue
meta = json.loads(meta_path.read_text())
if meta["state"] == "COMPLETED":
runs.append((datetime.fromisoformat(meta["timestamp"]), run_id))
if not runs:
return None
sorted_runs = sorted(runs, key=lambda x: x[0])
latest_run_dir = sorted_runs[-1][1]
# Load the model
return { # That is a FlowArtifact
"run_id": latest_run_dir.name,
"dir": latest_run_dir / "data",
"data": WorkerStore.open(latest_run_dir.name).load_model(as_dict=True), # type: ignore
}
# ------------------------------------------------------------------ #
# internals #
# ------------------------------------------------------------------ #
def _write_meta(self, *, state: str) -> None:
meta = {
"worker_name": self._worker_name,
"run_id": self._flow_id,
"state": state,
"timestamp": datetime.now(timezone.utc).isoformat(),
}
self.meta_path.write_text(json.dumps(meta, indent=2))
def _guess_output_model(self) -> Optional[Type[BaseModel]]:
"""
Best-effort import of `<worker_name>.output_model`.
"""
try:
from importlib import import_module
# workers are registered with dotted names in plugin_loader
mod = import_module(self._worker_name)
return getattr(mod, "output_model", None)
except Exception:
return None
# ------------------------------------------------------------------ #
# clean-up #
# ------------------------------------------------------------------ #
def __del__(self) -> None:
try:
shutil.rmtree(self._work_dir, ignore_errors=True)
except Exception:
pass
@@ -0,0 +1,3 @@
from .client import get_client, SupabaseGateway
__all__ = ["get_client", "SupabaseGateway"]
@@ -0,0 +1,60 @@
from __future__ import annotations
from typing import Any, Dict
from pydantic import BaseModel
from supabase import create_client, Client
import os, logging
log = logging.getLogger(__name__)
class _Cfg(BaseModel):
url: str
key: str
db_schema: str = "library"
def _load_cfg() -> _Cfg:
return _Cfg(
url=os.getenv("SUPABASE_URL", ""),
key=os.getenv("SUPABASE_API_KEY", ""),
)
_client: Client | None = None
_cfg: _Cfg | None = None
def get_client() -> Client:
global _client, _cfg
if _client:
return _client
_cfg = _load_cfg()
if not _cfg.url or not _cfg.key:
raise RuntimeError("SUPABASE_URL or SUPABASE_API_KEY missing")
_client = create_client(_cfg.url, _cfg.key)
return _client
class SupabaseGateway:
"""
Thin wrapper around Client with `schema()` pre-selected
and a helper `_rpc()` that raises RuntimeError on error.
"""
def __init__(self) -> None:
self.client = get_client()
self.schema = _cfg.db_schema if _cfg else "library"
# ---------- internal ----------
def _rpc(self, fn: str, payload: Dict[str, Any] | None = None):
resp = (
self.client.schema(self.schema)
.rpc(fn, payload or {})
.execute()
.model_dump()
)
if resp.get("error"):
log.error("%s error: %s", fn, resp["error"])
raise RuntimeError(resp["error"])
log.debug("%s OK", fn)
return resp.get("data")
Whitespace-only changes.
@@ -0,0 +1,61 @@
from __future__ import annotations
from typing import List
from librarian_scraper.models import CrawlCourse, CrawlTerm, MoodleIndex
from librarian_core.supabase.client import SupabaseGateway
gw = SupabaseGateway() # singleton gateway
# -------- public API --------
def upload_index(index: MoodleIndex) -> None:
dp = index.degree_program
_upsert_degree_program(dp.id, dp.name)
for term in dp.terms:
_upsert_term(term)
_upsert_courses(term.courses, term_id=term.id, prog_id=dp.id)
def upload_modules(modules_index) -> None:
# TODO – same pattern
...
# -------- helpers --------
def _upsert_degree_program(dp_id: str, name: str):
gw._rpc(
"upsert_degree_program",
{
"p_program_id": dp_id,
"p_program_name": name,
},
)
def _upsert_term(term: CrawlTerm):
# TODO: Change to term, when supabase is updated
gw._rpc(
"upsert_semester",
{
"p_semester_id": term.id,
"p_semester_name": term.name,
},
)
def _upsert_courses(courses: List[CrawlCourse], *, term_id: str, prog_id: str):
# TODO: Change to term, when supabase is updated
for c in courses:
gw._rpc(
"upsert_course",
{
"p_course_id": c.id,
"p_course_name": c.name,
"p_semester_id": term_id,
"p_program_id": prog_id,
"p_hero_image": c.hero_image,
"p_content_ressource_id": c.content_ressource_id,
},
)
@@ -0,0 +1,11 @@
from .chunk_data import (
ChunkCourse,
ChunkFile,
ChunkData,
)
__all__ = [
"ChunkData",
"ChunkCourse",
"ChunkFile",
]
@@ -0,0 +1,19 @@
from typing import List
from pydantic import BaseModel, Field
# TODO: Move to librarian-chunker
class ChunkFile(BaseModel):
name: str = Field(..., description="Name of the file")
id: str = Field(..., description="ID of the file")
class ChunkCourse(BaseModel):
id: str = Field(..., description="ID of the course")
name: str = Field(..., description="Name of the course")
files: List[ChunkFile] = Field(..., description="List of files in the course")
class ChunkData(BaseModel):
courses: List[ChunkCourse] = Field(..., description="List of courses")
Whitespace-only changes.
@@ -0,0 +1,25 @@
from librarian_core.utils.path_utils import (
copy_to_temp_dir,
get_cache_root,
get_config_root,
get_data_root,
get_flow_name_from_id,
get_flows_dir,
get_run_dir,
get_temp_path,
get_workers_dir,
)
from librarian_core.utils.secrets_loader import load_env
__all__ = [
"load_env",
"get_temp_path",
"get_run_dir",
"get_flow_name_from_id",
"copy_to_temp_dir",
"get_cache_root",
"get_data_root",
"get_config_root",
"get_flows_dir",
"get_workers_dir",
]
@@ -0,0 +1,196 @@
"""
librarian_core/utils/path_utils.py
==================================
Unified helpers for every path the Atlas-Librarian project uses.
Key features
------------
* XDG- and ENV-aware roots for **data**, **config**, and **cache**.
* Dedicated sub-trees for *flows* (per-worker run directories) and
*workers* (registrations, static assets, …).
* Convenience helpers:
- `get_run_dir(worker, run_id)`
- `get_flow_name_from_id(run_id)` ← Prefect lookup (lazy import)
- `get_temp_path()` / `copy_to_temp_dir()`
* **Single source of truth** – change the root once, everything follows.
"""
from __future__ import annotations
import os
import shutil
import tempfile
import uuid
from datetime import datetime
from pathlib import Path
from typing import Optional
from platformdirs import (
user_cache_dir,
user_config_dir,
user_data_dir,
)
# --------------------------------------------------------------------------- #
# Root directories (honours $LIBRARIAN_*_DIR, falls back to XDG) #
# --------------------------------------------------------------------------- #
_APP_NAME = "atlas-librarian"
_DATA_ROOT = Path(
os.getenv("LIBRARIAN_DATA_DIR", user_data_dir(_APP_NAME))
).expanduser()
_CONFIG_ROOT = Path(
os.getenv("LIBRARIAN_CONFIG_DIR", user_config_dir(_APP_NAME))
).expanduser()
_CACHE_ROOT = Path(
os.getenv("LIBRARIAN_CACHE_DIR", user_cache_dir(_APP_NAME))
).expanduser()
# Project-specific sub-trees
_FLOWS_DIR = _DATA_ROOT / "flows" # <data>/flows/<worker>/<run_id>/
_WORKERS_DIR = _DATA_ROOT / "workers" # static registration cache, etc.
# Ensure that the basic tree always exists
for p in (_DATA_ROOT, _CONFIG_ROOT, _CACHE_ROOT, _FLOWS_DIR, _WORKERS_DIR):
p.mkdir(parents=True, exist_ok=True)
# --------------------------------------------------------------------------- #
# Public helpers #
# --------------------------------------------------------------------------- #
# -- roots --
def get_data_root() -> Path:
return _DATA_ROOT
def get_config_root() -> Path:
return _CONFIG_ROOT
def get_cache_root() -> Path:
return _CACHE_ROOT
def get_flows_dir() -> Path:
return _FLOWS_DIR
def get_workers_dir() -> Path:
return _WORKERS_DIR
# -- flow-run directories ---------------------------------------------------- #
def get_run_dir(worker_name: str, run_id: str, *, create: bool = True) -> Path:
"""
Absolute path for one specific Prefect flow-run.
Example
-------
>>> get_run_dir("downloader", "1234abcd")
~/.local/share/atlas-librarian/flows/downloader/1234abcd
"""
safe_worker = _sanitize(worker_name)
path = _FLOWS_DIR / safe_worker / run_id
if create:
path.mkdir(parents=True, exist_ok=True)
return path
def get_flow_name_from_id(run_id: str) -> Optional[str]:
"""
Resolve a Prefect *run-id* → *flow name*.
Returns
-------
The flow (worker) name or *None* if the ID cannot be found.
"""
try:
from prefect.client.orchestration import get_client
except ImportError: # Prefect not installed in caller env
return None
try:
import anyio
async def _lookup() -> Optional[str]:
async with get_client() as client:
fr = await client.read_flow_run(uuid.UUID(run_id))
return fr.flow_name # type: ignore[attr-defined]
return anyio.run(_lookup)
except Exception:
return None
# -- temporary workspace helpers -------------------------------------------- #
def get_temp_path(prefix: str = "atlas") -> Path:
"""
Create a *unique* temporary directory inside the user cache.
The directory is **not** deleted automatically – callers decide.
"""
ts = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
rand = uuid.uuid4().hex[:8]
tmp_root = _CACHE_ROOT / "tmp"
tmp_root.mkdir(parents=True, exist_ok=True)
path = Path(
tempfile.mkdtemp(
dir=tmp_root,
prefix=f"{prefix}-{ts}-{rand}-",
)
)
return path
def copy_to_temp_dir(src: Path | str, *, prefix: str = "atlas") -> Path:
"""
Recursively copy *src* into a fresh temporary directory.
Returns the destination path.
"""
src_path = Path(src).expanduser().resolve()
if not src_path.exists():
raise FileNotFoundError(src_path)
dst = get_temp_path(prefix=prefix)
shutil.copytree(src_path, dst, dirs_exist_ok=True)
return dst
# --------------------------------------------------------------------------- #
# internal helpers #
# --------------------------------------------------------------------------- #
def _sanitize(name: str) -> str:
"""Replace path-hostile characters – keeps things safe across OSes."""
return "".join(c if c.isalnum() or c in "-._" else "_" for c in name)
# --------------------------------------------------------------------------- #
# exports #
# --------------------------------------------------------------------------- #
__all__ = [
# roots
"get_data_root",
"get_config_root",
"get_cache_root",
"get_flows_dir",
"get_workers_dir",
# flow-run helpers
"get_run_dir",
"get_flow_name_from_id",
# temporary space
"get_temp_path",
"copy_to_temp_dir",
]
@@ -0,0 +1,25 @@
"""
Secrets live in a classic .env **outside** the JSON settings file.
Load order:
1. ENV LIBRARIAN_CREDENTIALS_PATH (override)
2. ~/.config/atlas-librarian/credentials.env (XDG path)
"""
from pathlib import Path
import os
import logging
import dotenv
from librarian_core.utils.path_utils import get_config_root
log = logging.getLogger(__name__)
def load_env() -> None:
path = Path(os.getenv("LIBRARIAN_CREDENTIALS_PATH", get_config_root() / "credentials.env"))
if path.exists():
dotenv.load_dotenv(path)
log.debug("Secrets loaded from %s", path)
else:
log.debug("No credentials.env found (looked in %s)", path)
@@ -0,0 +1,3 @@
from librarian_core.workers.base import Worker
__all__ = ["Worker"]
@@ -0,0 +1,192 @@
from __future__ import annotations
import inspect
import uuid
from collections.abc import Awaitable, Callable
from pathlib import Path
from typing import ClassVar, Generic, TypeVar
import pandas as pd
import anyio
from prefect import flow, get_run_logger
from prefect.runtime import flow_run
from pydantic import BaseModel, Field, create_model
from prefect.artifacts import acreate_markdown_artifact
from librarian_core.storage.worker_store import WorkerStore
# --------------------------------------------------------------------------- #
# type parameters #
# --------------------------------------------------------------------------- #
InT = TypeVar("InT", bound=BaseModel)
OutT = TypeVar("OutT", bound=BaseModel)
# --------------------------------------------------------------------------- #
# envelope returned by every worker flow #
# --------------------------------------------------------------------------- #
class FlowArtifact(BaseModel, Generic[OutT]):
run_id: str | None = None
dir: Path | None = None
data: OutT | None = None
@classmethod
def new(cls, run_id: str | None = None, dir: Path | None = None, data: OutT | None = None) -> FlowArtifact:
if not data:
raise ValueError("data is required")
# Intermediate Worker
if run_id and dir:
return FlowArtifact(run_id=run_id, dir=dir, data=data)
# Initial Worker
else:
return FlowArtifact(data=data)
# --------------------------------------------------------------------------- #
# metaclass: adds a Prefect flow + envelope to each Worker #
# --------------------------------------------------------------------------- #
class _WorkerMeta(type):
def __new__(mcls, name, bases, ns, **kw):
cls = super().__new__(mcls, name, bases, dict(ns))
if name == "Worker" and cls.__module__ == __name__:
return cls # abstract base
if not (hasattr(cls, "input_model") and hasattr(cls, "output_model")):
raise TypeError(f"{name}: declare 'input_model' / 'output_model'.")
if "__run__" not in cls.__dict__:
raise TypeError(f"{name}: implement async '__run__(payload)'.")
cls.worker_name = name # type: ignore
cls._create_input_artifact()
cls._prefect_flow = mcls._build_prefect_flow(cls) # type: ignore
return cls
# --------------------------------------------------------------------- #
@staticmethod
def _build_prefect_flow(cls_ref):
"""Create the Prefect flow and return it."""
InArt = cls_ref.input_artifact # noqa: F841
OutModel: type[BaseModel] = cls_ref.output_model # noqa: F841
worker_name: str = cls_ref.worker_name
async def _core(in_art: FlowArtifact[InT]): # type: ignore[name-defined]
logger = get_run_logger()
run_id = flow_run.get_id() or uuid.uuid4().hex
logger.info("%s started (run_id=%s)", worker_name, run_id)
store = WorkerStore.new(worker_name=worker_name, flow_id=run_id)
if in_art.dir and in_art.dir.exists() and in_art.dir != Path("."):
store.prime_with_input(in_art.dir)
inst = cls_ref()
inst._inject_store(store)
# run worker ------------------------------------------------
run_res = inst.__run__(in_art.data)
# allow sync or async implementations
if inspect.iscoroutine(run_res):
result = await run_res
else:
result = run_res
store.save_model(result)
store.persist_exit()
store.cleanup()
logger.info("%s finished", worker_name)
artifact = FlowArtifact(run_id=run_id, dir=store.data_dir, data=result)
md_table = await inst._to_markdown(result)
await acreate_markdown_artifact(
key=f"{worker_name.lower()}-artifact",
markdown=md_table,
description=f"{worker_name} output"
)
# save the markdown artifact in the flow directory
md_file = store._run_dir / "artifact.md"
md_file.write_text(md_table)
return artifact
return flow(name=worker_name, log_prints=True)(_core)
# --------------------------------------------------------------------- #
def _create_input_artifact(cls):
"""Create & attach a pydantic model ‹InputArtifact› = {dir?, data}."""
DirField = (Path | None, None)
DataField = (cls.input_model, ...) # type: ignore # required
art_name = f"{cls.__name__}InputArtifact"
artifact = create_model(art_name, dir=DirField, data=DataField) # type: ignore[arg-type]
artifact.__doc__ = f"Artifact for {cls.__name__} input"
cls.input_artifact = artifact # type: ignore[attr-defined]
# --------------------------------------------------------------------------- #
# public Worker base #
# --------------------------------------------------------------------------- #
class Worker(Generic[InT, OutT], metaclass=_WorkerMeta):
"""
Derive from this class, set *input_model* / *output_model*, and implement
an **async** ``__run__(payload: input_model)``.
"""
input_model: ClassVar[type[BaseModel]]
output_model: ClassVar[type[BaseModel]]
input_artifact: ClassVar[type[BaseModel]] # injected by metaclass
worker_name: ClassVar[str]
_prefect_flow: ClassVar[Callable[[FlowArtifact[InT]], Awaitable[FlowArtifact[OutT]]]]
# injected at runtime
entry: Path
_store: WorkerStore
# ------------------------------------------------------------------ #
# internal wiring #
# ------------------------------------------------------------------ #
def _inject_store(self, store: WorkerStore) -> None:
self._store = store
self.entry = store.entry_dir
# ------------------------------------------------------------------ #
# developer helper #
# ------------------------------------------------------------------ #
def stage(
self,
src: Path | str,
*,
new_name: str | None = None,
sanitize: bool = True,
move: bool = False,
) -> Path:
return self._store.stage(src, new_name=new_name, sanitize=sanitize, move=move)
# ------------------------------------------------------------------ #
# convenience wrappers #
# ------------------------------------------------------------------ #
@classmethod
def flow(cls):
"""Return the auto-generated Prefect flow."""
return cls._prefect_flow
# submit variants --------------------------------------------------- #
@classmethod
def submit(cls, payload: FlowArtifact[InT]) -> FlowArtifact[OutT]:
async def _runner():
art = await cls._prefect_flow(payload) # type: ignore[arg-type]
return art
return anyio.run(_runner)
# ------------------------------------------------------------------ #
# abstract #
# ------------------------------------------------------------------ #
async def __run__(self, payload: InT) -> OutT: ...
# Should be overridden by the worker
async def _to_markdown(self, data: OutT) -> str:
md_table = pd.DataFrame([data.dict()]).to_markdown(index=False)
return md_table