Update Readme and cleanup
This commit is contained in:
1 parent
52d087ce90
commit
b6789cb5f7
34 files changed
+104
-471
No files matched your search
@@ -27,9 +27,6 @@ from librarian_core.utils import path_utils
|
||||
class WorkerStore:
|
||||
"""Never exposed to worker code – all access is via helper methods."""
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# constructors #
|
||||
# ------------------------------------------------------------------ #
|
||||
@classmethod
|
||||
def new(cls, *, worker_name: str, flow_id: str) -> "WorkerStore":
|
||||
run_dir = path_utils.get_run_dir(worker_name, flow_id, create=True)
|
||||
@@ -53,9 +50,6 @@ class WorkerStore:
|
||||
return cls(candidate, meta["worker_name"], run_id)
|
||||
raise FileNotFoundError(run_id)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# life-cycle #
|
||||
# ------------------------------------------------------------------ #
|
||||
def __init__(self, run_dir: Path, worker_name: str, flow_id: str):
|
||||
self._run_dir = run_dir
|
||||
self._worker_name = worker_name
|
||||
@@ -71,9 +65,6 @@ class WorkerStore:
|
||||
self._entry_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._exit_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# entry / exit handling #
|
||||
# ------------------------------------------------------------------ #
|
||||
@property
|
||||
def entry_dir(self) -> Path:
|
||||
return self._entry_dir
|
||||
@@ -115,9 +106,6 @@ class WorkerStore:
|
||||
shutil.copy2(src_path, dst)
|
||||
return dst
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# result persistence #
|
||||
# ------------------------------------------------------------------ #
|
||||
def save_model(
|
||||
self,
|
||||
model: BaseModel,
|
||||
@@ -145,9 +133,6 @@ class WorkerStore:
|
||||
def cleanup(self) -> None:
|
||||
shutil.rmtree(self._work_dir, ignore_errors=True)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# public helpers (API needs these) #
|
||||
# ------------------------------------------------------------------ #
|
||||
@property
|
||||
def data_dir(self) -> Path:
|
||||
return self._run_dir / "data"
|
||||
@@ -201,16 +186,12 @@ class WorkerStore:
|
||||
|
||||
latest_run_dir = sorted_runs[-1][1]
|
||||
|
||||
# Load the model
|
||||
return { # That is a FlowArtifact
|
||||
return {
|
||||
"run_id": latest_run_dir.name,
|
||||
"dir": latest_run_dir / "data",
|
||||
"data": WorkerStore.open(latest_run_dir.name).load_model(as_dict=True), # type: ignore
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# internals #
|
||||
# ------------------------------------------------------------------ #
|
||||
def _write_meta(self, *, state: str) -> None:
|
||||
meta = {
|
||||
"worker_name": self._worker_name,
|
||||
@@ -233,9 +214,6 @@ class WorkerStore:
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# clean-up #
|
||||
# ------------------------------------------------------------------ #
|
||||
def __del__(self) -> None:
|
||||
try:
|
||||
shutil.rmtree(self._work_dir, ignore_errors=True)
|
||||
|
||||
@@ -45,7 +45,6 @@ class SupabaseGateway:
|
||||
self.client = get_client()
|
||||
self.schema = _cfg.db_schema if _cfg else "library"
|
||||
|
||||
# ---------- internal ----------
|
||||
def _rpc(self, fn: str, payload: Dict[str, Any] | None = None):
|
||||
resp = (
|
||||
self.client.schema(self.schema)
|
||||
|
||||
@@ -15,16 +15,10 @@ from prefect.artifacts import acreate_markdown_artifact
|
||||
|
||||
from librarian_core.storage.worker_store import WorkerStore
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# type parameters #
|
||||
# --------------------------------------------------------------------------- #
|
||||
InT = TypeVar("InT", bound=BaseModel)
|
||||
OutT = TypeVar("OutT", bound=BaseModel)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# envelope returned by every worker flow #
|
||||
# --------------------------------------------------------------------------- #
|
||||
class FlowArtifact(BaseModel, Generic[OutT]):
|
||||
run_id: str | None = None
|
||||
dir: Path | None = None
|
||||
@@ -34,23 +28,17 @@ class FlowArtifact(BaseModel, Generic[OutT]):
|
||||
def new(cls, run_id: str | None = None, dir: Path | None = None, data: OutT | None = None) -> FlowArtifact:
|
||||
if not data:
|
||||
raise ValueError("data is required")
|
||||
# Intermediate Worker
|
||||
if run_id and dir:
|
||||
return FlowArtifact(run_id=run_id, dir=dir, data=data)
|
||||
|
||||
# Initial Worker
|
||||
else:
|
||||
return FlowArtifact(data=data)
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# metaclass: adds a Prefect flow + envelope to each Worker #
|
||||
# --------------------------------------------------------------------------- #
|
||||
class _WorkerMeta(type):
|
||||
def __new__(mcls, name, bases, ns, **kw):
|
||||
cls = super().__new__(mcls, name, bases, dict(ns))
|
||||
|
||||
if name == "Worker" and cls.__module__ == __name__:
|
||||
return cls # abstract base
|
||||
return cls
|
||||
|
||||
if not (hasattr(cls, "input_model") and hasattr(cls, "output_model")):
|
||||
raise TypeError(f"{name}: declare 'input_model' / 'output_model'.")
|
||||
@@ -62,10 +50,9 @@ class _WorkerMeta(type):
|
||||
cls._prefect_flow = mcls._build_prefect_flow(cls) # type: ignore
|
||||
return cls
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
@staticmethod
|
||||
def _build_prefect_flow(cls_ref):
|
||||
"""Create the Prefect flow and return it."""
|
||||
"""Builds the Prefect flow for the worker."""
|
||||
InArt = cls_ref.input_artifact # noqa: F841
|
||||
OutModel: type[BaseModel] = cls_ref.output_model # noqa: F841
|
||||
worker_name: str = cls_ref.worker_name
|
||||
@@ -82,9 +69,7 @@ class _WorkerMeta(type):
|
||||
|
||||
inst = cls_ref()
|
||||
inst._inject_store(store)
|
||||
# run worker ------------------------------------------------
|
||||
run_res = inst.__run__(in_art.data)
|
||||
# allow sync or async implementations
|
||||
if inspect.iscoroutine(run_res):
|
||||
result = await run_res
|
||||
else:
|
||||
@@ -104,7 +89,6 @@ class _WorkerMeta(type):
|
||||
description=f"{worker_name} output"
|
||||
)
|
||||
|
||||
# save the markdown artifact in the flow directory
|
||||
md_file = store._run_dir / "artifact.md"
|
||||
md_file.write_text(md_table)
|
||||
|
||||
@@ -112,9 +96,8 @@ class _WorkerMeta(type):
|
||||
|
||||
return flow(name=worker_name, log_prints=True)(_core)
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
def _create_input_artifact(cls):
|
||||
"""Create & attach a pydantic model ‹InputArtifact› = {dir?, data}."""
|
||||
"""Creates the `InputArtifact` model for the worker."""
|
||||
DirField = (Path | None, None)
|
||||
DataField = (cls.input_model, ...) # type: ignore # required
|
||||
art_name = f"{cls.__name__}InputArtifact"
|
||||
@@ -124,35 +107,27 @@ class _WorkerMeta(type):
|
||||
cls.input_artifact = artifact # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# public Worker base #
|
||||
# --------------------------------------------------------------------------- #
|
||||
class Worker(Generic[InT, OutT], metaclass=_WorkerMeta):
|
||||
"""
|
||||
Derive from this class, set *input_model* / *output_model*, and implement
|
||||
an **async** ``__run__(payload: input_model)``.
|
||||
Base class for workers. Subclasses must define `input_model`,
|
||||
`output_model`, and implement `__run__`.
|
||||
"""
|
||||
|
||||
input_model: ClassVar[type[BaseModel]]
|
||||
output_model: ClassVar[type[BaseModel]]
|
||||
input_artifact: ClassVar[type[BaseModel]] # injected by metaclass
|
||||
input_artifact: ClassVar[type[BaseModel]]
|
||||
worker_name: ClassVar[str]
|
||||
_prefect_flow: ClassVar[Callable[[FlowArtifact[InT]], Awaitable[FlowArtifact[OutT]]]]
|
||||
|
||||
# injected at runtime
|
||||
entry: Path
|
||||
_store: WorkerStore
|
||||
entry: Path # The entry directory for the worker's temporary files
|
||||
_store: WorkerStore # The WorkerStore instance for this run
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# internal wiring #
|
||||
# ------------------------------------------------------------------ #
|
||||
def _inject_store(self, store: WorkerStore) -> None:
|
||||
"""Injects WorkerStore and sets entry directory."""
|
||||
self._store = store
|
||||
self.entry = store.entry_dir
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# developer helper #
|
||||
# ------------------------------------------------------------------ #
|
||||
# Helper method for staging files
|
||||
def stage(
|
||||
self,
|
||||
src: Path | str,
|
||||
@@ -163,30 +138,26 @@ class Worker(Generic[InT, OutT], metaclass=_WorkerMeta):
|
||||
) -> Path:
|
||||
return self._store.stage(src, new_name=new_name, sanitize=sanitize, move=move)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# convenience wrappers #
|
||||
# ------------------------------------------------------------------ #
|
||||
@classmethod
|
||||
def flow(cls):
|
||||
"""Return the auto-generated Prefect flow."""
|
||||
"""Returns the Prefect flow."""
|
||||
return cls._prefect_flow
|
||||
|
||||
# submit variants --------------------------------------------------- #
|
||||
@classmethod
|
||||
def submit(cls, payload: FlowArtifact[InT]) -> FlowArtifact[OutT]:
|
||||
"""Submits payload to the Prefect flow."""
|
||||
async def _runner():
|
||||
art = await cls._prefect_flow(payload) # type: ignore[arg-type]
|
||||
return art
|
||||
|
||||
return anyio.run(_runner)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# abstract #
|
||||
# ------------------------------------------------------------------ #
|
||||
async def __run__(self, payload: InT) -> OutT: ...
|
||||
async def __run__(self, payload: InT) -> OutT:
|
||||
"""Core logic, implemented by subclasses."""
|
||||
...
|
||||
|
||||
|
||||
# Should be overridden by the worker
|
||||
async def _to_markdown(self, data: OutT) -> str:
|
||||
"""Converts output to Markdown. Override for custom format."""
|
||||
md_table = pd.DataFrame([data.dict()]).to_markdown(index=False)
|
||||
return md_table
|
||||
Reference in new issue
Block a user