Initialize Monorepo
This commit is contained in:
commit
f80792d739
106 files changed
+17076
No files matched your search
@@ -0,0 +1,20 @@
|
||||
import pkgutil
|
||||
import importlib
|
||||
|
||||
__all__ = []
|
||||
|
||||
# Iterate over all modules in this package
|
||||
for finder, module_name, is_pkg in pkgutil.iter_modules(__path__):
|
||||
# import the sub-module
|
||||
module = importlib.import_module(f"{__name__}.{module_name}")
|
||||
|
||||
# decide which names to re-export:
|
||||
# use module.__all__ if it exists, otherwise every non-private attribute
|
||||
public_names = getattr(
|
||||
module, "__all__", [n for n in dir(module) if not n.startswith("_")]
|
||||
)
|
||||
|
||||
# bring each name into the package namespace
|
||||
for name in public_names:
|
||||
globals()[name] = getattr(module, name)
|
||||
__all__.append(name) # type: ignore
|
||||
Whitespace-only changes.
@@ -0,0 +1,5 @@
|
||||
from librarian_core.storage.worker_store import WorkerStore
|
||||
|
||||
__all__ = [
|
||||
"WorkerStore",
|
||||
]
|
||||
@@ -0,0 +1,243 @@
|
||||
"""
|
||||
librarian_core.storage.worker_store
|
||||
===================================
|
||||
|
||||
Persistent directory layout
|
||||
---------------------------
|
||||
<data_root>/flows/<worker>/<run_id>/
|
||||
meta.json # worker_name, state, timestamps …
|
||||
result.json # pydantic-serialised return model
|
||||
data/ # files staged by the worker
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import shutil
|
||||
import tempfile
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, Type
|
||||
|
||||
from pydantic import BaseModel, TypeAdapter
|
||||
|
||||
from librarian_core.utils import path_utils
|
||||
|
||||
|
||||
class WorkerStore:
|
||||
"""Never exposed to worker code – all access is via helper methods."""
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# constructors #
|
||||
# ------------------------------------------------------------------ #
|
||||
@classmethod
|
||||
def new(cls, *, worker_name: str, flow_id: str) -> "WorkerStore":
|
||||
run_dir = path_utils.get_run_dir(worker_name, flow_id, create=True)
|
||||
store = cls(run_dir, worker_name, flow_id)
|
||||
store._write_meta(state="RUNNING")
|
||||
return store
|
||||
|
||||
@classmethod
|
||||
def open(cls, run_id: str) -> "WorkerStore":
|
||||
"""
|
||||
Locate `<flows>/<worker>/<run_id>` regardless of worker name.
|
||||
"""
|
||||
flows_dir = path_utils.get_flows_dir()
|
||||
for worker_dir in flows_dir.iterdir():
|
||||
candidate = worker_dir / run_id
|
||||
if candidate.exists():
|
||||
meta_path = candidate / "meta.json"
|
||||
if not meta_path.is_file():
|
||||
continue
|
||||
meta = json.loads(meta_path.read_text())
|
||||
return cls(candidate, meta["worker_name"], run_id)
|
||||
raise FileNotFoundError(run_id)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# life-cycle #
|
||||
# ------------------------------------------------------------------ #
|
||||
def __init__(self, run_dir: Path, worker_name: str, flow_id: str):
|
||||
self._run_dir = run_dir
|
||||
self._worker_name = worker_name
|
||||
self._flow_id = flow_id
|
||||
|
||||
cache_root = path_utils.get_cache_root()
|
||||
self._work_dir = Path(
|
||||
tempfile.mkdtemp(prefix=f"{self._flow_id}-", dir=cache_root)
|
||||
)
|
||||
|
||||
self._entry_dir = self._work_dir / "entry"
|
||||
self._exit_dir = self._work_dir / "exit"
|
||||
self._entry_dir.mkdir(parents=True, exist_ok=True)
|
||||
self._exit_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# entry / exit handling #
|
||||
# ------------------------------------------------------------------ #
|
||||
@property
|
||||
def entry_dir(self) -> Path:
|
||||
return self._entry_dir
|
||||
|
||||
def prime_with_input(self, src: Optional[Path]) -> None:
|
||||
if src and src.exists():
|
||||
shutil.copytree(src, self._entry_dir, dirs_exist_ok=True)
|
||||
|
||||
def stage(
|
||||
self,
|
||||
src: Path | str,
|
||||
*,
|
||||
new_name: str | None = None,
|
||||
sanitize: bool = True,
|
||||
move: bool = False,
|
||||
) -> Path:
|
||||
src_path = Path(src).expanduser().resolve()
|
||||
if not src_path.exists():
|
||||
raise FileNotFoundError(src_path)
|
||||
|
||||
name = new_name or src_path.name
|
||||
if sanitize:
|
||||
name = path_utils._sanitize(name)
|
||||
|
||||
dst = self._exit_dir / name
|
||||
if dst.exists():
|
||||
if dst.is_file():
|
||||
dst.unlink()
|
||||
else:
|
||||
shutil.rmtree(dst)
|
||||
|
||||
if move:
|
||||
src_path.rename(dst)
|
||||
else:
|
||||
if src_path.is_dir():
|
||||
shutil.copytree(src_path, dst)
|
||||
else:
|
||||
dst.parent.mkdir(parents=True, exist_ok=True)
|
||||
shutil.copy2(src_path, dst)
|
||||
return dst
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# result persistence #
|
||||
# ------------------------------------------------------------------ #
|
||||
def save_model(
|
||||
self,
|
||||
model: BaseModel,
|
||||
*,
|
||||
filename: str = "result.json",
|
||||
**json_kwargs: Any,
|
||||
) -> Path:
|
||||
json_kwargs.setdefault("indent", 2)
|
||||
target = self._run_dir / filename
|
||||
target.write_text(model.model_dump_json(**json_kwargs))
|
||||
return target
|
||||
|
||||
def persist_exit(self) -> Path:
|
||||
"""
|
||||
Move the *exit* directory to the persistent *data* slot and mark
|
||||
the run completed.
|
||||
"""
|
||||
data_dir = self.data_dir
|
||||
if data_dir.exists():
|
||||
shutil.rmtree(data_dir)
|
||||
self._exit_dir.rename(data_dir)
|
||||
self._write_meta(state="COMPLETED")
|
||||
return data_dir
|
||||
|
||||
def cleanup(self) -> None:
|
||||
shutil.rmtree(self._work_dir, ignore_errors=True)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# public helpers (API needs these) #
|
||||
# ------------------------------------------------------------------ #
|
||||
@property
|
||||
def data_dir(self) -> Path:
|
||||
return self._run_dir / "data"
|
||||
|
||||
@property
|
||||
def meta_path(self) -> Path:
|
||||
return self._run_dir / "meta.json"
|
||||
|
||||
@property
|
||||
def metadata(self) -> dict[str, Any]:
|
||||
return json.loads(self.meta_path.read_text())
|
||||
|
||||
def load_model(self, *, as_dict: bool = False) -> dict | BaseModel | None:
|
||||
res_file = self._run_dir / "result.json"
|
||||
if not res_file.is_file():
|
||||
return None
|
||||
data = json.loads(res_file.read_text())
|
||||
if as_dict:
|
||||
return data
|
||||
# try to reconstruct a Pydantic model if possible
|
||||
try:
|
||||
OutputModel: Type[BaseModel] | None = self._guess_output_model()
|
||||
if OutputModel:
|
||||
return TypeAdapter(OutputModel).validate_python(data)
|
||||
except Exception:
|
||||
pass
|
||||
return data
|
||||
|
||||
@staticmethod
|
||||
# TODO: Should return a Flowartifact, but circular import is messing
|
||||
def load_latest(worker_name: str) -> dict[str, Any] | None:
|
||||
flows_dir = path_utils.get_flows_dir()
|
||||
worker_dir = flows_dir / worker_name
|
||||
if not worker_dir.exists():
|
||||
return None
|
||||
|
||||
runs: list[tuple[datetime, Path]] = []
|
||||
for run_id in worker_dir.iterdir():
|
||||
if not run_id.is_dir():
|
||||
continue
|
||||
meta_path = run_id / "meta.json"
|
||||
if not meta_path.is_file():
|
||||
continue
|
||||
meta = json.loads(meta_path.read_text())
|
||||
if meta["state"] == "COMPLETED":
|
||||
runs.append((datetime.fromisoformat(meta["timestamp"]), run_id))
|
||||
|
||||
if not runs:
|
||||
return None
|
||||
sorted_runs = sorted(runs, key=lambda x: x[0])
|
||||
|
||||
latest_run_dir = sorted_runs[-1][1]
|
||||
|
||||
# Load the model
|
||||
return { # That is a FlowArtifact
|
||||
"run_id": latest_run_dir.name,
|
||||
"dir": latest_run_dir / "data",
|
||||
"data": WorkerStore.open(latest_run_dir.name).load_model(as_dict=True), # type: ignore
|
||||
}
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# internals #
|
||||
# ------------------------------------------------------------------ #
|
||||
def _write_meta(self, *, state: str) -> None:
|
||||
meta = {
|
||||
"worker_name": self._worker_name,
|
||||
"run_id": self._flow_id,
|
||||
"state": state,
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
}
|
||||
self.meta_path.write_text(json.dumps(meta, indent=2))
|
||||
|
||||
def _guess_output_model(self) -> Optional[Type[BaseModel]]:
|
||||
"""
|
||||
Best-effort import of `<worker_name>.output_model`.
|
||||
"""
|
||||
try:
|
||||
from importlib import import_module
|
||||
|
||||
# workers are registered with dotted names in plugin_loader
|
||||
mod = import_module(self._worker_name)
|
||||
return getattr(mod, "output_model", None)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# clean-up #
|
||||
# ------------------------------------------------------------------ #
|
||||
def __del__(self) -> None:
|
||||
try:
|
||||
shutil.rmtree(self._work_dir, ignore_errors=True)
|
||||
except Exception:
|
||||
pass
|
||||
@@ -0,0 +1,3 @@
|
||||
from .client import get_client, SupabaseGateway
|
||||
|
||||
__all__ = ["get_client", "SupabaseGateway"]
|
||||
@@ -0,0 +1,60 @@
|
||||
from __future__ import annotations
|
||||
from typing import Any, Dict
|
||||
from pydantic import BaseModel
|
||||
from supabase import create_client, Client
|
||||
import os, logging
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class _Cfg(BaseModel):
|
||||
url: str
|
||||
key: str
|
||||
db_schema: str = "library"
|
||||
|
||||
|
||||
def _load_cfg() -> _Cfg:
|
||||
return _Cfg(
|
||||
url=os.getenv("SUPABASE_URL", ""),
|
||||
key=os.getenv("SUPABASE_API_KEY", ""),
|
||||
)
|
||||
|
||||
|
||||
_client: Client | None = None
|
||||
_cfg: _Cfg | None = None
|
||||
|
||||
|
||||
def get_client() -> Client:
|
||||
global _client, _cfg
|
||||
if _client:
|
||||
return _client
|
||||
_cfg = _load_cfg()
|
||||
if not _cfg.url or not _cfg.key:
|
||||
raise RuntimeError("SUPABASE_URL or SUPABASE_API_KEY missing")
|
||||
_client = create_client(_cfg.url, _cfg.key)
|
||||
return _client
|
||||
|
||||
|
||||
class SupabaseGateway:
|
||||
"""
|
||||
Thin wrapper around Client with `schema()` pre-selected
|
||||
and a helper `_rpc()` that raises RuntimeError on error.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.client = get_client()
|
||||
self.schema = _cfg.db_schema if _cfg else "library"
|
||||
|
||||
# ---------- internal ----------
|
||||
def _rpc(self, fn: str, payload: Dict[str, Any] | None = None):
|
||||
resp = (
|
||||
self.client.schema(self.schema)
|
||||
.rpc(fn, payload or {})
|
||||
.execute()
|
||||
.model_dump()
|
||||
)
|
||||
if resp.get("error"):
|
||||
log.error("%s error: %s", fn, resp["error"])
|
||||
raise RuntimeError(resp["error"])
|
||||
log.debug("%s OK", fn)
|
||||
return resp.get("data")
|
||||
Whitespace-only changes.
@@ -0,0 +1,61 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List
|
||||
|
||||
from librarian_scraper.models import CrawlCourse, CrawlTerm, MoodleIndex
|
||||
|
||||
from librarian_core.supabase.client import SupabaseGateway
|
||||
|
||||
gw = SupabaseGateway() # singleton gateway
|
||||
|
||||
|
||||
# -------- public API --------
|
||||
def upload_index(index: MoodleIndex) -> None:
|
||||
dp = index.degree_program
|
||||
_upsert_degree_program(dp.id, dp.name)
|
||||
for term in dp.terms:
|
||||
_upsert_term(term)
|
||||
_upsert_courses(term.courses, term_id=term.id, prog_id=dp.id)
|
||||
|
||||
|
||||
def upload_modules(modules_index) -> None:
|
||||
# TODO – same pattern
|
||||
...
|
||||
|
||||
|
||||
# -------- helpers --------
|
||||
def _upsert_degree_program(dp_id: str, name: str):
|
||||
gw._rpc(
|
||||
"upsert_degree_program",
|
||||
{
|
||||
"p_program_id": dp_id,
|
||||
"p_program_name": name,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _upsert_term(term: CrawlTerm):
|
||||
# TODO: Change to term, when supabase is updated
|
||||
gw._rpc(
|
||||
"upsert_semester",
|
||||
{
|
||||
"p_semester_id": term.id,
|
||||
"p_semester_name": term.name,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _upsert_courses(courses: List[CrawlCourse], *, term_id: str, prog_id: str):
|
||||
# TODO: Change to term, when supabase is updated
|
||||
for c in courses:
|
||||
gw._rpc(
|
||||
"upsert_course",
|
||||
{
|
||||
"p_course_id": c.id,
|
||||
"p_course_name": c.name,
|
||||
"p_semester_id": term_id,
|
||||
"p_program_id": prog_id,
|
||||
"p_hero_image": c.hero_image,
|
||||
"p_content_ressource_id": c.content_ressource_id,
|
||||
},
|
||||
)
|
||||
@@ -0,0 +1,11 @@
|
||||
from .chunk_data import (
|
||||
ChunkCourse,
|
||||
ChunkFile,
|
||||
ChunkData,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"ChunkData",
|
||||
"ChunkCourse",
|
||||
"ChunkFile",
|
||||
]
|
||||
@@ -0,0 +1,19 @@
|
||||
from typing import List
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# TODO: Move to librarian-chunker
|
||||
|
||||
class ChunkFile(BaseModel):
|
||||
name: str = Field(..., description="Name of the file")
|
||||
id: str = Field(..., description="ID of the file")
|
||||
|
||||
|
||||
class ChunkCourse(BaseModel):
|
||||
id: str = Field(..., description="ID of the course")
|
||||
name: str = Field(..., description="Name of the course")
|
||||
files: List[ChunkFile] = Field(..., description="List of files in the course")
|
||||
|
||||
|
||||
class ChunkData(BaseModel):
|
||||
courses: List[ChunkCourse] = Field(..., description="List of courses")
|
||||
Whitespace-only changes.
@@ -0,0 +1,25 @@
|
||||
from librarian_core.utils.path_utils import (
|
||||
copy_to_temp_dir,
|
||||
get_cache_root,
|
||||
get_config_root,
|
||||
get_data_root,
|
||||
get_flow_name_from_id,
|
||||
get_flows_dir,
|
||||
get_run_dir,
|
||||
get_temp_path,
|
||||
get_workers_dir,
|
||||
)
|
||||
from librarian_core.utils.secrets_loader import load_env
|
||||
|
||||
__all__ = [
|
||||
"load_env",
|
||||
"get_temp_path",
|
||||
"get_run_dir",
|
||||
"get_flow_name_from_id",
|
||||
"copy_to_temp_dir",
|
||||
"get_cache_root",
|
||||
"get_data_root",
|
||||
"get_config_root",
|
||||
"get_flows_dir",
|
||||
"get_workers_dir",
|
||||
]
|
||||
@@ -0,0 +1,196 @@
|
||||
"""
|
||||
librarian_core/utils/path_utils.py
|
||||
==================================
|
||||
|
||||
Unified helpers for every path the Atlas-Librarian project uses.
|
||||
|
||||
Key features
|
||||
------------
|
||||
* XDG- and ENV-aware roots for **data**, **config**, and **cache**.
|
||||
* Dedicated sub-trees for *flows* (per-worker run directories) and
|
||||
*workers* (registrations, static assets, …).
|
||||
* Convenience helpers:
|
||||
- `get_run_dir(worker, run_id)`
|
||||
- `get_flow_name_from_id(run_id)` ← Prefect lookup (lazy import)
|
||||
- `get_temp_path()` / `copy_to_temp_dir()`
|
||||
* **Single source of truth** – change the root once, everything follows.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import tempfile
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from platformdirs import (
|
||||
user_cache_dir,
|
||||
user_config_dir,
|
||||
user_data_dir,
|
||||
)
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Root directories (honours $LIBRARIAN_*_DIR, falls back to XDG) #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
_APP_NAME = "atlas-librarian"
|
||||
|
||||
_DATA_ROOT = Path(
|
||||
os.getenv("LIBRARIAN_DATA_DIR", user_data_dir(_APP_NAME))
|
||||
).expanduser()
|
||||
_CONFIG_ROOT = Path(
|
||||
os.getenv("LIBRARIAN_CONFIG_DIR", user_config_dir(_APP_NAME))
|
||||
).expanduser()
|
||||
_CACHE_ROOT = Path(
|
||||
os.getenv("LIBRARIAN_CACHE_DIR", user_cache_dir(_APP_NAME))
|
||||
).expanduser()
|
||||
|
||||
# Project-specific sub-trees
|
||||
_FLOWS_DIR = _DATA_ROOT / "flows" # <data>/flows/<worker>/<run_id>/
|
||||
_WORKERS_DIR = _DATA_ROOT / "workers" # static registration cache, etc.
|
||||
|
||||
# Ensure that the basic tree always exists
|
||||
for p in (_DATA_ROOT, _CONFIG_ROOT, _CACHE_ROOT, _FLOWS_DIR, _WORKERS_DIR):
|
||||
p.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Public helpers #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
# -- roots --
|
||||
|
||||
|
||||
def get_data_root() -> Path:
|
||||
return _DATA_ROOT
|
||||
|
||||
|
||||
def get_config_root() -> Path:
|
||||
return _CONFIG_ROOT
|
||||
|
||||
|
||||
def get_cache_root() -> Path:
|
||||
return _CACHE_ROOT
|
||||
|
||||
|
||||
def get_flows_dir() -> Path:
|
||||
return _FLOWS_DIR
|
||||
|
||||
|
||||
def get_workers_dir() -> Path:
|
||||
return _WORKERS_DIR
|
||||
|
||||
|
||||
# -- flow-run directories ---------------------------------------------------- #
|
||||
|
||||
|
||||
def get_run_dir(worker_name: str, run_id: str, *, create: bool = True) -> Path:
|
||||
"""
|
||||
Absolute path for one specific Prefect flow-run.
|
||||
|
||||
Example
|
||||
-------
|
||||
>>> get_run_dir("downloader", "1234abcd")
|
||||
~/.local/share/atlas-librarian/flows/downloader/1234abcd
|
||||
"""
|
||||
safe_worker = _sanitize(worker_name)
|
||||
path = _FLOWS_DIR / safe_worker / run_id
|
||||
if create:
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
return path
|
||||
|
||||
|
||||
def get_flow_name_from_id(run_id: str) -> Optional[str]:
|
||||
"""
|
||||
Resolve a Prefect *run-id* → *flow name*.
|
||||
|
||||
Returns
|
||||
-------
|
||||
The flow (worker) name or *None* if the ID cannot be found.
|
||||
"""
|
||||
try:
|
||||
from prefect.client.orchestration import get_client
|
||||
except ImportError: # Prefect not installed in caller env
|
||||
return None
|
||||
|
||||
try:
|
||||
import anyio
|
||||
|
||||
async def _lookup() -> Optional[str]:
|
||||
async with get_client() as client:
|
||||
fr = await client.read_flow_run(uuid.UUID(run_id))
|
||||
return fr.flow_name # type: ignore[attr-defined]
|
||||
|
||||
return anyio.run(_lookup)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
# -- temporary workspace helpers -------------------------------------------- #
|
||||
|
||||
|
||||
def get_temp_path(prefix: str = "atlas") -> Path:
|
||||
"""
|
||||
Create a *unique* temporary directory inside the user cache.
|
||||
|
||||
The directory is **not** deleted automatically – callers decide.
|
||||
"""
|
||||
ts = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
|
||||
rand = uuid.uuid4().hex[:8]
|
||||
tmp_root = _CACHE_ROOT / "tmp"
|
||||
tmp_root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
path = Path(
|
||||
tempfile.mkdtemp(
|
||||
dir=tmp_root,
|
||||
prefix=f"{prefix}-{ts}-{rand}-",
|
||||
)
|
||||
)
|
||||
return path
|
||||
|
||||
|
||||
def copy_to_temp_dir(src: Path | str, *, prefix: str = "atlas") -> Path:
|
||||
"""
|
||||
Recursively copy *src* into a fresh temporary directory.
|
||||
|
||||
Returns the destination path.
|
||||
"""
|
||||
src_path = Path(src).expanduser().resolve()
|
||||
if not src_path.exists():
|
||||
raise FileNotFoundError(src_path)
|
||||
|
||||
dst = get_temp_path(prefix=prefix)
|
||||
shutil.copytree(src_path, dst, dirs_exist_ok=True)
|
||||
return dst
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# internal helpers #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _sanitize(name: str) -> str:
|
||||
"""Replace path-hostile characters – keeps things safe across OSes."""
|
||||
return "".join(c if c.isalnum() or c in "-._" else "_" for c in name)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# exports #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
__all__ = [
|
||||
# roots
|
||||
"get_data_root",
|
||||
"get_config_root",
|
||||
"get_cache_root",
|
||||
"get_flows_dir",
|
||||
"get_workers_dir",
|
||||
# flow-run helpers
|
||||
"get_run_dir",
|
||||
"get_flow_name_from_id",
|
||||
# temporary space
|
||||
"get_temp_path",
|
||||
"copy_to_temp_dir",
|
||||
]
|
||||
@@ -0,0 +1,25 @@
|
||||
"""
|
||||
Secrets live in a classic .env **outside** the JSON settings file.
|
||||
Load order:
|
||||
|
||||
1. ENV LIBRARIAN_CREDENTIALS_PATH (override)
|
||||
2. ~/.config/atlas-librarian/credentials.env (XDG path)
|
||||
"""
|
||||
|
||||
from pathlib import Path
|
||||
import os
|
||||
import logging
|
||||
import dotenv
|
||||
from librarian_core.utils.path_utils import get_config_root
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_env() -> None:
|
||||
path = Path(os.getenv("LIBRARIAN_CREDENTIALS_PATH", get_config_root() / "credentials.env"))
|
||||
|
||||
if path.exists():
|
||||
dotenv.load_dotenv(path)
|
||||
log.debug("Secrets loaded from %s", path)
|
||||
else:
|
||||
log.debug("No credentials.env found (looked in %s)", path)
|
||||
@@ -0,0 +1,3 @@
|
||||
from librarian_core.workers.base import Worker
|
||||
|
||||
__all__ = ["Worker"]
|
||||
@@ -0,0 +1,192 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import inspect
|
||||
import uuid
|
||||
from collections.abc import Awaitable, Callable
|
||||
from pathlib import Path
|
||||
from typing import ClassVar, Generic, TypeVar
|
||||
|
||||
import pandas as pd
|
||||
import anyio
|
||||
from prefect import flow, get_run_logger
|
||||
from prefect.runtime import flow_run
|
||||
from pydantic import BaseModel, Field, create_model
|
||||
from prefect.artifacts import acreate_markdown_artifact
|
||||
|
||||
from librarian_core.storage.worker_store import WorkerStore
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# type parameters #
|
||||
# --------------------------------------------------------------------------- #
|
||||
InT = TypeVar("InT", bound=BaseModel)
|
||||
OutT = TypeVar("OutT", bound=BaseModel)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# envelope returned by every worker flow #
|
||||
# --------------------------------------------------------------------------- #
|
||||
class FlowArtifact(BaseModel, Generic[OutT]):
|
||||
run_id: str | None = None
|
||||
dir: Path | None = None
|
||||
data: OutT | None = None
|
||||
|
||||
@classmethod
|
||||
def new(cls, run_id: str | None = None, dir: Path | None = None, data: OutT | None = None) -> FlowArtifact:
|
||||
if not data:
|
||||
raise ValueError("data is required")
|
||||
# Intermediate Worker
|
||||
if run_id and dir:
|
||||
return FlowArtifact(run_id=run_id, dir=dir, data=data)
|
||||
|
||||
# Initial Worker
|
||||
else:
|
||||
return FlowArtifact(data=data)
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# metaclass: adds a Prefect flow + envelope to each Worker #
|
||||
# --------------------------------------------------------------------------- #
|
||||
class _WorkerMeta(type):
|
||||
def __new__(mcls, name, bases, ns, **kw):
|
||||
cls = super().__new__(mcls, name, bases, dict(ns))
|
||||
|
||||
if name == "Worker" and cls.__module__ == __name__:
|
||||
return cls # abstract base
|
||||
|
||||
if not (hasattr(cls, "input_model") and hasattr(cls, "output_model")):
|
||||
raise TypeError(f"{name}: declare 'input_model' / 'output_model'.")
|
||||
if "__run__" not in cls.__dict__:
|
||||
raise TypeError(f"{name}: implement async '__run__(payload)'.")
|
||||
|
||||
cls.worker_name = name # type: ignore
|
||||
cls._create_input_artifact()
|
||||
cls._prefect_flow = mcls._build_prefect_flow(cls) # type: ignore
|
||||
return cls
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
@staticmethod
|
||||
def _build_prefect_flow(cls_ref):
|
||||
"""Create the Prefect flow and return it."""
|
||||
InArt = cls_ref.input_artifact # noqa: F841
|
||||
OutModel: type[BaseModel] = cls_ref.output_model # noqa: F841
|
||||
worker_name: str = cls_ref.worker_name
|
||||
|
||||
async def _core(in_art: FlowArtifact[InT]): # type: ignore[name-defined]
|
||||
logger = get_run_logger()
|
||||
run_id = flow_run.get_id() or uuid.uuid4().hex
|
||||
logger.info("%s started (run_id=%s)", worker_name, run_id)
|
||||
|
||||
store = WorkerStore.new(worker_name=worker_name, flow_id=run_id)
|
||||
|
||||
if in_art.dir and in_art.dir.exists() and in_art.dir != Path("."):
|
||||
store.prime_with_input(in_art.dir)
|
||||
|
||||
inst = cls_ref()
|
||||
inst._inject_store(store)
|
||||
# run worker ------------------------------------------------
|
||||
run_res = inst.__run__(in_art.data)
|
||||
# allow sync or async implementations
|
||||
if inspect.iscoroutine(run_res):
|
||||
result = await run_res
|
||||
else:
|
||||
result = run_res
|
||||
|
||||
store.save_model(result)
|
||||
store.persist_exit()
|
||||
store.cleanup()
|
||||
logger.info("%s finished", worker_name)
|
||||
|
||||
artifact = FlowArtifact(run_id=run_id, dir=store.data_dir, data=result)
|
||||
|
||||
md_table = await inst._to_markdown(result)
|
||||
await acreate_markdown_artifact(
|
||||
key=f"{worker_name.lower()}-artifact",
|
||||
markdown=md_table,
|
||||
description=f"{worker_name} output"
|
||||
)
|
||||
|
||||
# save the markdown artifact in the flow directory
|
||||
md_file = store._run_dir / "artifact.md"
|
||||
md_file.write_text(md_table)
|
||||
|
||||
return artifact
|
||||
|
||||
return flow(name=worker_name, log_prints=True)(_core)
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
def _create_input_artifact(cls):
|
||||
"""Create & attach a pydantic model ‹InputArtifact› = {dir?, data}."""
|
||||
DirField = (Path | None, None)
|
||||
DataField = (cls.input_model, ...) # type: ignore # required
|
||||
art_name = f"{cls.__name__}InputArtifact"
|
||||
|
||||
artifact = create_model(art_name, dir=DirField, data=DataField) # type: ignore[arg-type]
|
||||
artifact.__doc__ = f"Artifact for {cls.__name__} input"
|
||||
cls.input_artifact = artifact # type: ignore[attr-defined]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# public Worker base #
|
||||
# --------------------------------------------------------------------------- #
|
||||
class Worker(Generic[InT, OutT], metaclass=_WorkerMeta):
|
||||
"""
|
||||
Derive from this class, set *input_model* / *output_model*, and implement
|
||||
an **async** ``__run__(payload: input_model)``.
|
||||
"""
|
||||
|
||||
input_model: ClassVar[type[BaseModel]]
|
||||
output_model: ClassVar[type[BaseModel]]
|
||||
input_artifact: ClassVar[type[BaseModel]] # injected by metaclass
|
||||
worker_name: ClassVar[str]
|
||||
_prefect_flow: ClassVar[Callable[[FlowArtifact[InT]], Awaitable[FlowArtifact[OutT]]]]
|
||||
|
||||
# injected at runtime
|
||||
entry: Path
|
||||
_store: WorkerStore
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# internal wiring #
|
||||
# ------------------------------------------------------------------ #
|
||||
def _inject_store(self, store: WorkerStore) -> None:
|
||||
self._store = store
|
||||
self.entry = store.entry_dir
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# developer helper #
|
||||
# ------------------------------------------------------------------ #
|
||||
def stage(
|
||||
self,
|
||||
src: Path | str,
|
||||
*,
|
||||
new_name: str | None = None,
|
||||
sanitize: bool = True,
|
||||
move: bool = False,
|
||||
) -> Path:
|
||||
return self._store.stage(src, new_name=new_name, sanitize=sanitize, move=move)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# convenience wrappers #
|
||||
# ------------------------------------------------------------------ #
|
||||
@classmethod
|
||||
def flow(cls):
|
||||
"""Return the auto-generated Prefect flow."""
|
||||
return cls._prefect_flow
|
||||
|
||||
# submit variants --------------------------------------------------- #
|
||||
@classmethod
|
||||
def submit(cls, payload: FlowArtifact[InT]) -> FlowArtifact[OutT]:
|
||||
async def _runner():
|
||||
art = await cls._prefect_flow(payload) # type: ignore[arg-type]
|
||||
return art
|
||||
|
||||
return anyio.run(_runner)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# abstract #
|
||||
# ------------------------------------------------------------------ #
|
||||
async def __run__(self, payload: InT) -> OutT: ...
|
||||
|
||||
|
||||
# Should be overridden by the worker
|
||||
async def _to_markdown(self, data: OutT) -> str:
|
||||
md_table = pd.DataFrame([data.dict()]).to_markdown(index=False)
|
||||
return md_table
|
||||
Reference in new issue
Block a user