Update Readme and cleanup
This commit is contained in:
1 parent
52d087ce90
commit
b6789cb5f7
34 files changed
+104
-471
No files matched your search
@@ -62,9 +62,6 @@ def _create_hnsw_index(
|
||||
except Exception:
|
||||
logger.exception("Failed to run create_or_reindex_hnsw")
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# single file #
|
||||
# --------------------------------------------------------------------------- #
|
||||
def embed_single_file(
|
||||
*,
|
||||
course_id: str,
|
||||
@@ -104,7 +101,6 @@ def embed_single_file(
|
||||
wf.process()
|
||||
return chunk_path
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
async def run_embedder(
|
||||
course: ChunkCourse,
|
||||
concat_path: Union[str, Path],
|
||||
|
||||
+1
-2
@@ -39,7 +39,6 @@ class EmbeddingWorkflow:
|
||||
|
||||
# No need to store db_schema/db_function here if inserter handles it
|
||||
|
||||
# ---------------- helpers ----------------
|
||||
def _load_chunk(self) -> Optional[str]:
|
||||
try:
|
||||
text = self.chunk_path.read_text(encoding="utf-8").strip()
|
||||
@@ -85,7 +84,7 @@ class EmbeddingWorkflow:
|
||||
return False
|
||||
|
||||
logger.debug(f"Successfully processed and inserted {self.chunk_path}")
|
||||
return True # Indicate success
|
||||
return True
|
||||
|
||||
|
||||
# Keep __all__ if needed
|
||||
|
||||
@@ -22,7 +22,6 @@ from librarian_core.workers.base import Worker
|
||||
|
||||
from librarian_vspace.vecview.vecview import get_tsne_json
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
def _safe_get_logger(name: str):
|
||||
try:
|
||||
return get_run_logger()
|
||||
@@ -30,9 +29,7 @@ def _safe_get_logger(name: str):
|
||||
return logging.getLogger(name)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Pydantic payloads
|
||||
# ------------------------------------------------------------------ #
|
||||
class TsneExportInput(BaseModel):
|
||||
course_id: int
|
||||
limit: Optional[int] = None
|
||||
@@ -48,7 +45,6 @@ class TsneExportOutput(BaseModel):
|
||||
json_path: Path
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
class TsneExportWorker(Worker[TsneExportInput, TsneExportOutput]):
|
||||
"""Runs the t‑SNE export inside a Prefect worker.""" # noqa: D401
|
||||
|
||||
|
||||
@@ -20,9 +20,6 @@ logger = logging.getLogger(__name__)
|
||||
DEFAULT_N_CLUSTERS = 8
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
# Internal helpers (kept minimal – no extra bells & whistles)
|
||||
# --------------------------------------------------------------------- #
|
||||
def _run_kmeans(df: pd.DataFrame, *, embedding_column: str, k: int = DEFAULT_N_CLUSTERS) -> pd.DataFrame:
|
||||
"""Adds a 'cluster' column using K‑means (string labels)."""
|
||||
if df.empty or embedding_column not in df.columns:
|
||||
@@ -61,9 +58,6 @@ def _add_hover(df: pd.DataFrame) -> pd.DataFrame:
|
||||
return df
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
# Public helpers
|
||||
# --------------------------------------------------------------------- #
|
||||
def get_tsne_dataframe(
|
||||
db_schema: str,
|
||||
db_function: str,
|
||||
|
||||
@@ -58,9 +58,6 @@ import pandas as pd
|
||||
from sklearn.cluster import KMeans
|
||||
from sklearn.metrics import pairwise_distances_argmin_min
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Map Vectorbase credential names → Supabase names expected by loader code
|
||||
# ---------------------------------------------------------------------------
|
||||
_ALIAS_ENV_MAP = {
|
||||
"VECTORBASE_URL": "SUPABASE_URL",
|
||||
"VECTORBASE_API_KEY": "SUPABASE_KEY",
|
||||
@@ -86,11 +83,8 @@ except ImportError as e:
|
||||
raise ImportError(f"Could not import VectorQueryLoader: {e}") from e
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Logging setup (used by both script and callable function)
|
||||
# ---------------------------------------------------------------------------
|
||||
# This basicConfig runs when the module is imported.
|
||||
# Callers might want to configure logging before importing.
|
||||
# Callers might want toconfigure logging before importing.
|
||||
# If logging is already configured, basicConfig does nothing.
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
@@ -99,17 +93,11 @@ logging.basicConfig(
|
||||
)
|
||||
logger = logging.getLogger(__name__) # Use __name__ for module-specific logger
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper – JSON dump for centroid in YAML front‑matter
|
||||
# ---------------------------------------------------------------------------
|
||||
def centroid_to_json(vec: np.ndarray) -> str:
|
||||
"""Converts a numpy vector to a JSON string suitable for YAML frontmatter."""
|
||||
return json.dumps([float(x) for x in vec], ensure_ascii=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main clustering and export logic as a callable function
|
||||
# ---------------------------------------------------------------------------
|
||||
def run_cluster_export_job(
|
||||
course_id: Optional[int] = None, # Added course_id parameter
|
||||
output_dir: Union[str, Path] = "./cluster_md", # Output directory parameter
|
||||
@@ -147,9 +135,7 @@ def run_cluster_export_job(
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
logger.info("Writing Markdown files to %s", output_path)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fetch embeddings - Now using VectorQueryLoader with filtering
|
||||
# ---------------------------------------------------------------------------
|
||||
try:
|
||||
# Use parameters for loader config
|
||||
# --- FIX: Instantiate VectorQueryLoader ---
|
||||
@@ -249,9 +235,7 @@ def run_cluster_export_job(
|
||||
# -------------------------------------------------------------
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Prepare training sample and determine effective k
|
||||
# ---------------------------------------------------------------------------
|
||||
# Use the parameter train_sample_size
|
||||
train_vecs = embeddings[:train_sample_size]
|
||||
|
||||
@@ -302,9 +286,7 @@ def run_cluster_export_job(
|
||||
raise RuntimeError(f"K-means clustering failed: {e}") from e
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Assign every vector to its nearest centroid (full table)
|
||||
# ---------------------------------------------------------------------------
|
||||
logger.info("Assigning vectors to centroids...")
|
||||
try:
|
||||
# Use the determined embedding column for assignment as well
|
||||
@@ -315,9 +297,7 @@ def run_cluster_export_job(
|
||||
logger.exception("Failed to assign vectors to centroids.")
|
||||
raise RuntimeError(f"Failed to assign vectors to centroids: {e}") from e
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Write one Markdown file per cluster
|
||||
# ---------------------------------------------------------------------------
|
||||
files_written_count = 0
|
||||
|
||||
logger.info("Writing cluster Markdown files to %s", output_path)
|
||||
@@ -385,9 +365,7 @@ def run_cluster_export_job(
|
||||
return output_path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Script entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
if __name__ == "__main__":
|
||||
# Configuration via environment for script
|
||||
script_output_dir = Path(os.environ.get("OUTPUT_DIR", "./cluster_md")).expanduser()
|
||||
|
||||
@@ -37,15 +37,9 @@ from librarian_vspace.models.query_model import (
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
# Main helper
|
||||
# --------------------------------------------------------------------- #
|
||||
class VectorQuery(BaseVectorOperator):
|
||||
"""High‑level helper for vector searches via Supabase RPC."""
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Public – modern API
|
||||
# -----------------------------------------------------------------
|
||||
def search(self, request: VectorSearchRequest) -> VectorSearchResponse:
|
||||
"""Perform a similarity search and return structured results."""
|
||||
|
||||
@@ -100,9 +94,6 @@ class VectorQuery(BaseVectorOperator):
|
||||
logger.exception("RPC 'vector_search' failed: %s", exc)
|
||||
return VectorSearchResponse(total=0, results=[])
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Public – legacy compatibility
|
||||
# -----------------------------------------------------------------
|
||||
def get_chucklets_by_vector(
|
||||
self,
|
||||
*,
|
||||
|
||||
@@ -38,9 +38,6 @@ import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Literal, Optional
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Optional dependencies #
|
||||
# --------------------------------------------------------------------------- #
|
||||
try:
|
||||
import psutil # type: ignore
|
||||
except ModuleNotFoundError: # pragma: no cover
|
||||
@@ -51,10 +48,6 @@ try:
|
||||
except ModuleNotFoundError: # pragma: no cover
|
||||
torch = None # type: ignore
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Hardware discovery helpers #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
def logical_cores() -> int:
|
||||
@@ -124,11 +117,6 @@ def cgroup_cpu_limit() -> Optional[int]:
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# GPU helpers (CUDA only for now) #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
def gpu_info() -> Optional[Dict[str, Any]]:
|
||||
"""Return basic info for the first CUDA device via *torch* (or ``None``)."""
|
||||
@@ -147,11 +135,6 @@ def gpu_info() -> Optional[Dict[str, Any]]:
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Recommendation logic #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def recommended_workers(
|
||||
*,
|
||||
kind: Literal["cpu", "io", "gpu"] = "cpu",
|
||||
@@ -194,11 +177,6 @@ def recommended_workers(
|
||||
return max(1, base)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Convenience snapshot of system info #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def system_snapshot() -> Dict[str, Any]:
|
||||
"""Return a JSON‑serialisable snapshot of parallelism‑related facts."""
|
||||
return {
|
||||
@@ -211,11 +189,6 @@ def system_snapshot() -> Dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# CLI #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _cli() -> None: # pragma: no cover
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="parallelism_advisor", description="Rule‑of‑thumb worker estimator"
|
||||
|
||||
@@ -24,13 +24,11 @@ class BaseVectorOperator:
|
||||
self.table: Optional[str] = None
|
||||
self._resolve_ids()
|
||||
|
||||
# ---------------- public helpers ----------------
|
||||
def table_fqn(self) -> str:
|
||||
if not self.table:
|
||||
raise RuntimeError("VectorOperator not initialised – no table")
|
||||
return f"{self.schema}.{self.table}"
|
||||
|
||||
# ---------------- internals ----------------
|
||||
def _resolve_ids(self) -> None:
|
||||
self.model_id = self._rpc_get_model_id(self.model)
|
||||
if self.model_id is None:
|
||||
|
||||
Reference in new issue
Block a user