Update Readme and cleanup

This commit is contained in:
Oliver Schütz authored and GitHub committed 2025-06-01 17:32:50 +02:00
1 parent 52d087ce90
commit b6789cb5f7
34 files changed
+104 -471

No files matched your search

@@ -62,9 +62,6 @@ def _create_hnsw_index(
except Exception:
logger.exception("Failed to run create_or_reindex_hnsw")
# --------------------------------------------------------------------------- #
# single file #
# --------------------------------------------------------------------------- #
def embed_single_file(
*,
course_id: str,
@@ -104,7 +101,6 @@ def embed_single_file(
wf.process()
return chunk_path
# --------------------------------------------------------------------------- #
async def run_embedder(
course: ChunkCourse,
concat_path: Union[str, Path],
@@ -39,7 +39,6 @@ class EmbeddingWorkflow:
# No need to store db_schema/db_function here if inserter handles it
# ---------------- helpers ----------------
def _load_chunk(self) -> Optional[str]:
try:
text = self.chunk_path.read_text(encoding="utf-8").strip()
@@ -85,7 +84,7 @@ class EmbeddingWorkflow:
return False
logger.debug(f"Successfully processed and inserted {self.chunk_path}")
return True # Indicate success
return True
# Keep __all__ if needed
@@ -22,7 +22,6 @@ from librarian_core.workers.base import Worker
from librarian_vspace.vecview.vecview import get_tsne_json
# ------------------------------------------------------------------ #
def _safe_get_logger(name: str):
try:
return get_run_logger()
@@ -30,9 +29,7 @@ def _safe_get_logger(name: str):
return logging.getLogger(name)
# ------------------------------------------------------------------ #
# Pydantic payloads
# ------------------------------------------------------------------ #
class TsneExportInput(BaseModel):
course_id: int
limit: Optional[int] = None
@@ -48,7 +45,6 @@ class TsneExportOutput(BaseModel):
json_path: Path
# ------------------------------------------------------------------ #
class TsneExportWorker(Worker[TsneExportInput, TsneExportOutput]):
"""Runs the t‑SNE export inside a Prefect worker.""" # noqa: D401
@@ -20,9 +20,6 @@ logger = logging.getLogger(__name__)
DEFAULT_N_CLUSTERS = 8
# --------------------------------------------------------------------- #
# Internal helpers (kept minimal – no extra bells & whistles)
# --------------------------------------------------------------------- #
def _run_kmeans(df: pd.DataFrame, *, embedding_column: str, k: int = DEFAULT_N_CLUSTERS) -> pd.DataFrame:
"""Adds a 'cluster' column using K‑means (string labels)."""
if df.empty or embedding_column not in df.columns:
@@ -61,9 +58,6 @@ def _add_hover(df: pd.DataFrame) -> pd.DataFrame:
return df
# --------------------------------------------------------------------- #
# Public helpers
# --------------------------------------------------------------------- #
def get_tsne_dataframe(
db_schema: str,
db_function: str,
@@ -58,9 +58,6 @@ import pandas as pd
from sklearn.cluster import KMeans
from sklearn.metrics import pairwise_distances_argmin_min
# ---------------------------------------------------------------------------
# Map Vectorbase credential names → Supabase names expected by loader code
# ---------------------------------------------------------------------------
_ALIAS_ENV_MAP = {
"VECTORBASE_URL": "SUPABASE_URL",
"VECTORBASE_API_KEY": "SUPABASE_KEY",
@@ -86,11 +83,8 @@ except ImportError as e:
raise ImportError(f"Could not import VectorQueryLoader: {e}") from e
# ---------------------------------------------------------------------------
# Logging setup (used by both script and callable function)
# ---------------------------------------------------------------------------
# This basicConfig runs when the module is imported.
# Callers might want to configure logging before importing.
# Callers might want toconfigure logging before importing.
# If logging is already configured, basicConfig does nothing.
logging.basicConfig(
level=logging.INFO,
@@ -99,17 +93,11 @@ logging.basicConfig(
)
logger = logging.getLogger(__name__) # Use __name__ for module-specific logger
# ---------------------------------------------------------------------------
# Helper – JSON dump for centroid in YAML front‑matter
# ---------------------------------------------------------------------------
def centroid_to_json(vec: np.ndarray) -> str:
"""Converts a numpy vector to a JSON string suitable for YAML frontmatter."""
return json.dumps([float(x) for x in vec], ensure_ascii=False)
# ---------------------------------------------------------------------------
# Main clustering and export logic as a callable function
# ---------------------------------------------------------------------------
def run_cluster_export_job(
course_id: Optional[int] = None, # Added course_id parameter
output_dir: Union[str, Path] = "./cluster_md", # Output directory parameter
@@ -147,9 +135,7 @@ def run_cluster_export_job(
output_path.mkdir(parents=True, exist_ok=True)
logger.info("Writing Markdown files to %s", output_path)
# ---------------------------------------------------------------------------
# Fetch embeddings - Now using VectorQueryLoader with filtering
# ---------------------------------------------------------------------------
try:
# Use parameters for loader config
# --- FIX: Instantiate VectorQueryLoader ---
@@ -249,9 +235,7 @@ def run_cluster_export_job(
# -------------------------------------------------------------
# ---------------------------------------------------------------------------
# Prepare training sample and determine effective k
# ---------------------------------------------------------------------------
# Use the parameter train_sample_size
train_vecs = embeddings[:train_sample_size]
@@ -302,9 +286,7 @@ def run_cluster_export_job(
raise RuntimeError(f"K-means clustering failed: {e}") from e
# ---------------------------------------------------------------------------
# Assign every vector to its nearest centroid (full table)
# ---------------------------------------------------------------------------
logger.info("Assigning vectors to centroids...")
try:
# Use the determined embedding column for assignment as well
@@ -315,9 +297,7 @@ def run_cluster_export_job(
logger.exception("Failed to assign vectors to centroids.")
raise RuntimeError(f"Failed to assign vectors to centroids: {e}") from e
# ---------------------------------------------------------------------------
# Write one Markdown file per cluster
# ---------------------------------------------------------------------------
files_written_count = 0
logger.info("Writing cluster Markdown files to %s", output_path)
@@ -385,9 +365,7 @@ def run_cluster_export_job(
return output_path
# ---------------------------------------------------------------------------
# Script entry point
# ---------------------------------------------------------------------------
if __name__ == "__main__":
# Configuration via environment for script
script_output_dir = Path(os.environ.get("OUTPUT_DIR", "./cluster_md")).expanduser()
@@ -37,15 +37,9 @@ from librarian_vspace.models.query_model import (
logger = logging.getLogger(__name__)
# --------------------------------------------------------------------- #
# Main helper
# --------------------------------------------------------------------- #
class VectorQuery(BaseVectorOperator):
"""High‑level helper for vector searches via Supabase RPC."""
# -----------------------------------------------------------------
# Public – modern API
# -----------------------------------------------------------------
def search(self, request: VectorSearchRequest) -> VectorSearchResponse:
"""Perform a similarity search and return structured results."""
@@ -100,9 +94,6 @@ class VectorQuery(BaseVectorOperator):
logger.exception("RPC 'vector_search' failed: %s", exc)
return VectorSearchResponse(total=0, results=[])
# -----------------------------------------------------------------
# Public – legacy compatibility
# -----------------------------------------------------------------
def get_chucklets_by_vector(
self,
*,
@@ -38,9 +38,6 @@ import sys
from pathlib import Path
from typing import Any, Dict, Literal, Optional
# --------------------------------------------------------------------------- #
# Optional dependencies #
# --------------------------------------------------------------------------- #
try:
import psutil # type: ignore
except ModuleNotFoundError: # pragma: no cover
@@ -51,10 +48,6 @@ try:
except ModuleNotFoundError: # pragma: no cover
torch = None # type: ignore
# --------------------------------------------------------------------------- #
# Hardware discovery helpers #
# --------------------------------------------------------------------------- #
@functools.lru_cache(maxsize=None)
def logical_cores() -> int:
@@ -124,11 +117,6 @@ def cgroup_cpu_limit() -> Optional[int]:
return None
# --------------------------------------------------------------------------- #
# GPU helpers (CUDA only for now) #
# --------------------------------------------------------------------------- #
@functools.lru_cache(maxsize=None)
def gpu_info() -> Optional[Dict[str, Any]]:
"""Return basic info for the first CUDA device via *torch* (or ``None``)."""
@@ -147,11 +135,6 @@ def gpu_info() -> Optional[Dict[str, Any]]:
}
# --------------------------------------------------------------------------- #
# Recommendation logic #
# --------------------------------------------------------------------------- #
def recommended_workers(
*,
kind: Literal["cpu", "io", "gpu"] = "cpu",
@@ -194,11 +177,6 @@ def recommended_workers(
return max(1, base)
# --------------------------------------------------------------------------- #
# Convenience snapshot of system info #
# --------------------------------------------------------------------------- #
def system_snapshot() -> Dict[str, Any]:
"""Return a JSON‑serialisable snapshot of parallelism‑related facts."""
return {
@@ -211,11 +189,6 @@ def system_snapshot() -> Dict[str, Any]:
}
# --------------------------------------------------------------------------- #
# CLI #
# --------------------------------------------------------------------------- #
def _cli() -> None: # pragma: no cover
parser = argparse.ArgumentParser(
prog="parallelism_advisor", description="Rule‑of‑thumb worker estimator"
@@ -24,13 +24,11 @@ class BaseVectorOperator:
self.table: Optional[str] = None
self._resolve_ids()
# ---------------- public helpers ----------------
def table_fqn(self) -> str:
if not self.table:
raise RuntimeError("VectorOperator not initialised – no table")
return f"{self.schema}.{self.table}"
# ---------------- internals ----------------
def _resolve_ids(self) -> None:
self.model_id = self._rpc_get_model_id(self.model)
if self.model_id is None: