Update Readme and cleanup

This commit is contained in:
Oliver Schütz authored and GitHub committed 2025-06-01 17:32:50 +02:00
1 parent 52d087ce90
commit b6789cb5f7
34 files changed
+104 -471

No files matched your search

@@ -27,9 +27,6 @@ from librarian_core.utils import path_utils
class WorkerStore:
"""Never exposed to worker code – all access is via helper methods."""
# ------------------------------------------------------------------ #
# constructors #
# ------------------------------------------------------------------ #
@classmethod
def new(cls, *, worker_name: str, flow_id: str) -> "WorkerStore":
run_dir = path_utils.get_run_dir(worker_name, flow_id, create=True)
@@ -53,9 +50,6 @@ class WorkerStore:
return cls(candidate, meta["worker_name"], run_id)
raise FileNotFoundError(run_id)
# ------------------------------------------------------------------ #
# life-cycle #
# ------------------------------------------------------------------ #
def __init__(self, run_dir: Path, worker_name: str, flow_id: str):
self._run_dir = run_dir
self._worker_name = worker_name
@@ -71,9 +65,6 @@ class WorkerStore:
self._entry_dir.mkdir(parents=True, exist_ok=True)
self._exit_dir.mkdir(parents=True, exist_ok=True)
# ------------------------------------------------------------------ #
# entry / exit handling #
# ------------------------------------------------------------------ #
@property
def entry_dir(self) -> Path:
return self._entry_dir
@@ -115,9 +106,6 @@ class WorkerStore:
shutil.copy2(src_path, dst)
return dst
# ------------------------------------------------------------------ #
# result persistence #
# ------------------------------------------------------------------ #
def save_model(
self,
model: BaseModel,
@@ -145,9 +133,6 @@ class WorkerStore:
def cleanup(self) -> None:
shutil.rmtree(self._work_dir, ignore_errors=True)
# ------------------------------------------------------------------ #
# public helpers (API needs these) #
# ------------------------------------------------------------------ #
@property
def data_dir(self) -> Path:
return self._run_dir / "data"
@@ -201,16 +186,12 @@ class WorkerStore:
latest_run_dir = sorted_runs[-1][1]
# Load the model
return { # That is a FlowArtifact
return {
"run_id": latest_run_dir.name,
"dir": latest_run_dir / "data",
"data": WorkerStore.open(latest_run_dir.name).load_model(as_dict=True), # type: ignore
}
# ------------------------------------------------------------------ #
# internals #
# ------------------------------------------------------------------ #
def _write_meta(self, *, state: str) -> None:
meta = {
"worker_name": self._worker_name,
@@ -233,9 +214,6 @@ class WorkerStore:
except Exception:
return None
# ------------------------------------------------------------------ #
# clean-up #
# ------------------------------------------------------------------ #
def __del__(self) -> None:
try:
shutil.rmtree(self._work_dir, ignore_errors=True)
@@ -45,7 +45,6 @@ class SupabaseGateway:
self.client = get_client()
self.schema = _cfg.db_schema if _cfg else "library"
# ---------- internal ----------
def _rpc(self, fn: str, payload: Dict[str, Any] | None = None):
resp = (
self.client.schema(self.schema)
@@ -15,16 +15,10 @@ from prefect.artifacts import acreate_markdown_artifact
from librarian_core.storage.worker_store import WorkerStore
# --------------------------------------------------------------------------- #
# type parameters #
# --------------------------------------------------------------------------- #
InT = TypeVar("InT", bound=BaseModel)
OutT = TypeVar("OutT", bound=BaseModel)
# --------------------------------------------------------------------------- #
# envelope returned by every worker flow #
# --------------------------------------------------------------------------- #
class FlowArtifact(BaseModel, Generic[OutT]):
run_id: str | None = None
dir: Path | None = None
@@ -34,23 +28,17 @@ class FlowArtifact(BaseModel, Generic[OutT]):
def new(cls, run_id: str | None = None, dir: Path | None = None, data: OutT | None = None) -> FlowArtifact:
if not data:
raise ValueError("data is required")
# Intermediate Worker
if run_id and dir:
return FlowArtifact(run_id=run_id, dir=dir, data=data)
# Initial Worker
else:
return FlowArtifact(data=data)
# --------------------------------------------------------------------------- #
# metaclass: adds a Prefect flow + envelope to each Worker #
# --------------------------------------------------------------------------- #
class _WorkerMeta(type):
def __new__(mcls, name, bases, ns, **kw):
cls = super().__new__(mcls, name, bases, dict(ns))
if name == "Worker" and cls.__module__ == __name__:
return cls # abstract base
return cls
if not (hasattr(cls, "input_model") and hasattr(cls, "output_model")):
raise TypeError(f"{name}: declare 'input_model' / 'output_model'.")
@@ -62,10 +50,9 @@ class _WorkerMeta(type):
cls._prefect_flow = mcls._build_prefect_flow(cls) # type: ignore
return cls
# --------------------------------------------------------------------- #
@staticmethod
def _build_prefect_flow(cls_ref):
"""Create the Prefect flow and return it."""
"""Builds the Prefect flow for the worker."""
InArt = cls_ref.input_artifact # noqa: F841
OutModel: type[BaseModel] = cls_ref.output_model # noqa: F841
worker_name: str = cls_ref.worker_name
@@ -82,9 +69,7 @@ class _WorkerMeta(type):
inst = cls_ref()
inst._inject_store(store)
# run worker ------------------------------------------------
run_res = inst.__run__(in_art.data)
# allow sync or async implementations
if inspect.iscoroutine(run_res):
result = await run_res
else:
@@ -104,7 +89,6 @@ class _WorkerMeta(type):
description=f"{worker_name} output"
)
# save the markdown artifact in the flow directory
md_file = store._run_dir / "artifact.md"
md_file.write_text(md_table)
@@ -112,9 +96,8 @@ class _WorkerMeta(type):
return flow(name=worker_name, log_prints=True)(_core)
# --------------------------------------------------------------------- #
def _create_input_artifact(cls):
"""Create & attach a pydantic model ‹InputArtifact› = {dir?, data}."""
"""Creates the `InputArtifact` model for the worker."""
DirField = (Path | None, None)
DataField = (cls.input_model, ...) # type: ignore # required
art_name = f"{cls.__name__}InputArtifact"
@@ -124,35 +107,27 @@ class _WorkerMeta(type):
cls.input_artifact = artifact # type: ignore[attr-defined]
# --------------------------------------------------------------------------- #
# public Worker base #
# --------------------------------------------------------------------------- #
class Worker(Generic[InT, OutT], metaclass=_WorkerMeta):
"""
Derive from this class, set *input_model* / *output_model*, and implement
an **async** ``__run__(payload: input_model)``.
Base class for workers. Subclasses must define `input_model`,
`output_model`, and implement `__run__`.
"""
input_model: ClassVar[type[BaseModel]]
output_model: ClassVar[type[BaseModel]]
input_artifact: ClassVar[type[BaseModel]] # injected by metaclass
input_artifact: ClassVar[type[BaseModel]]
worker_name: ClassVar[str]
_prefect_flow: ClassVar[Callable[[FlowArtifact[InT]], Awaitable[FlowArtifact[OutT]]]]
# injected at runtime
entry: Path
_store: WorkerStore
entry: Path # The entry directory for the worker's temporary files
_store: WorkerStore # The WorkerStore instance for this run
# ------------------------------------------------------------------ #
# internal wiring #
# ------------------------------------------------------------------ #
def _inject_store(self, store: WorkerStore) -> None:
"""Injects WorkerStore and sets entry directory."""
self._store = store
self.entry = store.entry_dir
# ------------------------------------------------------------------ #
# developer helper #
# ------------------------------------------------------------------ #
# Helper method for staging files
def stage(
self,
src: Path | str,
@@ -163,30 +138,26 @@ class Worker(Generic[InT, OutT], metaclass=_WorkerMeta):
) -> Path:
return self._store.stage(src, new_name=new_name, sanitize=sanitize, move=move)
# ------------------------------------------------------------------ #
# convenience wrappers #
# ------------------------------------------------------------------ #
@classmethod
def flow(cls):
"""Return the auto-generated Prefect flow."""
"""Returns the Prefect flow."""
return cls._prefect_flow
# submit variants --------------------------------------------------- #
@classmethod
def submit(cls, payload: FlowArtifact[InT]) -> FlowArtifact[OutT]:
"""Submits payload to the Prefect flow."""
async def _runner():
art = await cls._prefect_flow(payload) # type: ignore[arg-type]
return art
return anyio.run(_runner)
# ------------------------------------------------------------------ #
# abstract #
# ------------------------------------------------------------------ #
async def __run__(self, payload: InT) -> OutT: ...
async def __run__(self, payload: InT) -> OutT:
"""Core logic, implemented by subclasses."""
...
# Should be overridden by the worker
async def _to_markdown(self, data: OutT) -> str:
"""Converts output to Markdown. Override for custom format."""
md_table = pd.DataFrame([data.dict()]).to_markdown(index=False)
return md_table