Initialize Monorepo

This commit is contained in:
DotNaos committed 2025-05-24 12:15:48 +02:00
commit f80792d739
106 files changed
+17076

No files matched your search

@@ -0,0 +1,22 @@
"""Embedding‑related helpers."""
import pkgutil
import importlib
__all__ = []
# Iterate over all modules in this package
for finder, module_name, is_pkg in pkgutil.iter_modules(__path__):
# import the sub-module
module = importlib.import_module(f"{__name__}.{module_name}")
# decide which names to re-export:
# use module.__all__ if it exists, otherwise every non-private attribute
public_names = getattr(
module, "__all__", [n for n in dir(module) if not n.startswith("_")]
)
# bring each name into the package namespace
for name in public_names:
globals()[name] = getattr(module, name)
__all__.append(name) # type: ignore
@@ -0,0 +1,38 @@
"""Pydantic models for vector search requests and responses."""
from __future__ import annotations
from typing import Any, Dict, List, Optional
from pydantic import BaseModel, Field
class VectorSearchRequest(BaseModel):
"""Input payload for a vector search."""
interface_name: str = Field(..., description="Name of the embedding interface")
model_name: str = Field(..., description="Name of the embedding model")
search_string: str = Field(..., description="The natural language query to embed and search for")
filters: Optional[Dict[str, Any]] = Field(
default=None,
description="Optional key/value filters applied server‑side",
)
top_k: int = Field(10, ge=1, le=100, description="Number of matches to return")
embedding_column: str = Field(
"embedding",
description="Name of the embedding column in the database table",
)
class Chunklet(BaseModel):
"""Single result row returned by the database RPC."""
chunk: Optional[str] = None
file_id: Optional[str | int] = None
class VectorSearchResponse(BaseModel):
"""Output payload wrapping vector‑search results."""
total: int
results: List[Chunklet]
@@ -0,0 +1,31 @@
"""Data models for t‑SNE exports.
These models are used by *vecview* and any endpoint that needs to return or
validate t‑SNE projection data.
"""
from __future__ import annotations
from typing import List, Optional
from pydantic import BaseModel
class TSNEPoint(BaseModel):
"""A single point in a 3‑D t‑SNE projection."""
x: float
y: float
z: float
file_id: str
chunk: str
cluster: Optional[str] = None
hover_text: Optional[str] = None
class TSNEData(BaseModel):
"""Container returned to callers requesting a t‑SNE view."""
course_id: Optional[int] = None
total: int
points: List[TSNEPoint]
Whitespace-only changes.
@@ -0,0 +1,9 @@
"""Embedding‑related helpers."""
from __future__ import annotations
from .vector_inserter import VectorInserter
from .embedding_generator import EmbeddingGenerator
from .embedding_workflow import EmbeddingWorkflow
__all__ = ["VectorInserter", "EmbeddingGenerator", "EmbeddingWorkflow"]
@@ -0,0 +1,155 @@
"""Parallel‑aware embedding helpers.
* **embed_single_file()** – embed one file (sync).
* **run_embedder()** – embed all files in a course (async, kept for back‑compat).
* **_create_hnsw_index()** – helper to (re)build PGVector HNSW index.
This file contains no Prefect code; it’s pure embedding logic.
"""
from __future__ import annotations
import asyncio
import logging
from functools import lru_cache
from pathlib import Path
from types import SimpleNamespace
from typing import Any, List, Union
from postgrest import APIResponse
from librarian_core.temp_payloads.chunk_data import ChunkCourse, ChunkFile
from librarian_vspace.vecembed.embedding_generator import EmbeddingGenerator
from librarian_vspace.vecembed.vector_inserter import VectorInserter
from librarian_vspace.vecembed.embedding_workflow import EmbeddingWorkflow
from librarian_vspace.vutils.supabase_singleton import MySupabase
logger = logging.getLogger(__name__)
@lru_cache(maxsize=1)
def _autodiscover_pg_conn():
supa = MySupabase.get_client() # type: ignore
if supa is None:
raise RuntimeError("MySupabase.get_client() returned None – no DB connection.")
return supa
def _create_hnsw_index(
supa,
table_fqn: str,
*,
column_name: str = "embedding",
query_operator: str = "<=>",
m: int = 16,
ef: int = 64,
) -> None:
if "." not in table_fqn:
raise ValueError("table_fqn must be schema.table")
schema, table = table_fqn.split(".", 1)
try:
supa.schema(schema).rpc(
"create_or_reindex_hnsw",
dict(
p_schema=schema,
p_table=table,
p_column=column_name,
p_operator=query_operator,
p_m=m,
p_ef=ef,
),
).execute()
except Exception:
logger.exception("Failed to run create_or_reindex_hnsw")
# --------------------------------------------------------------------------- #
# single file #
# --------------------------------------------------------------------------- #
def embed_single_file(
*,
course_id: str,
file_entry: dict | ChunkFile | SimpleNamespace,
concat_path: Union[str, Path],
db_schema: str = "librarian",
db_function: str = "pdf_chunking",
interface_name: str = "ollama",
model_name: str = "snowflake-arctic-embed2",
file_type: str = "md",
) -> Path | None:
if isinstance(file_entry, (dict, SimpleNamespace)):
file_name = file_entry["file_name"] if isinstance(file_entry, dict) else file_entry.file_name
file_id = file_entry["file_id"] if isinstance(file_entry, dict) else file_entry.file_id
else:
file_name, file_id = file_entry.file_name, file_entry.file_id
chunk_path = Path(concat_path) / file_name
if not chunk_path.exists():
logger.warning("Missing chunk file %s – skipping", chunk_path)
return None
generator = EmbeddingGenerator()
inserter = VectorInserter(schema=db_schema, function=db_function, model=model_name)
wf = EmbeddingWorkflow(
chunk_path=chunk_path,
course_id=course_id,
file_id=file_id,
file_type=file_type,
interface_name=interface_name,
model_name=model_name,
generator=generator,
inserter=inserter,
)
wf.process()
return chunk_path
# --------------------------------------------------------------------------- #
async def run_embedder(
course: ChunkCourse,
concat_path: Union[str, Path],
*,
db_schema: str = "librarian",
db_function: str = "pdf_chunking",
interface_name: str = "ollama",
model_name: str = "snowflake-arctic-embed2",
file_type: str = "md",
vector_column: str = "embedding",
query_operator: str = "<=>",
hnsw_m: int = 16,
hnsw_ef: int = 64,
max_parallel_files: int | None = None,
) -> Path:
supa_client = _autodiscover_pg_conn()
root = Path(concat_path)
sem = asyncio.Semaphore(max_parallel_files or len(course.files) or 1)
async def _wrapper(cf):
async with sem:
return await asyncio.to_thread(
embed_single_file,
course_id=course.course_id,
file_entry=cf,
concat_path=root,
db_schema=db_schema,
db_function=db_function,
interface_name=interface_name,
model_name=model_name,
file_type=file_type,
)
await asyncio.gather(*[asyncio.create_task(_wrapper(cf)) for cf in course.files])
inserter = VectorInserter(schema=db_schema, function=db_function, model=model_name)
_create_hnsw_index(
supa_client,
inserter.table_fqn(),
column_name=vector_column,
query_operator=query_operator,
m=hnsw_m,
ef=hnsw_ef,
)
return root
__all__ = ["embed_single_file", "run_embedder", "_create_hnsw_index", "_autodiscover_pg_conn"]
@@ -0,0 +1,67 @@
"""EmbedderWorker – Prefect‑mapped per file."""
from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
from typing import Any, List
from prefect import get_run_logger, task, unmapped
from pydantic import BaseModel, Field
from librarian_core.workers.base import Worker
@task(name="embed_file", retries=2, retry_delay_seconds=5, log_prints=True, tags=["embed_file"])
def embed_file_task(course_dict: dict | SimpleNamespace, file_entry: dict, concat_path: Path) -> Path | None:
from librarian_vspace.vecembed.embedder import embed_single_file
cid = course_dict["course_id"] if isinstance(course_dict, dict) else course_dict.course_id
return embed_single_file(course_id=cid, file_entry=file_entry, concat_path=concat_path)
class EmbedderInput(BaseModel):
chunk_courses: List[Any] = Field(default_factory=list, alias="chunk_courses")
concat_path: Path
chunk_course: Any | None = None
def model_post_init(self, _):
if not self.chunk_courses and self.chunk_course is not None:
self.chunk_courses = [self.chunk_course]
model_config = dict(populate_by_name=True, extra="allow")
class EmbedderOutput(BaseModel):
result_paths: List[Path]
class EmbedderWorker(Worker[EmbedderInput, EmbedderOutput]):
input_model = EmbedderInput
output_model = EmbedderOutput
async def __run__(self, payload: EmbedderInput) -> EmbedderOutput:
log = get_run_logger()
total_files = sum(len(c["files"]) if isinstance(c, dict) else len(c.files) for c in payload.chunk_courses)
log.info("Embedding %d files", total_files)
result_paths: List[Path] = []
# constants – could be parameterised later
schema = "librarian"
func = "pdf_chunking"
model_name = "snowflake-arctic-embed2"
for course in payload.chunk_courses:
files = course["files"] if isinstance(course, dict) else course.files
futures = embed_file_task.map(unmapped(course), files, unmapped(payload.concat_path))
for fut in futures:
path = fut.result()
if path:
result_paths.append(path)
# rebuild index once per course
from librarian_vspace.vecembed.embedder import _create_hnsw_index, _autodiscover_pg_conn
from librarian_vspace.vecembed.vector_inserter import VectorInserter
supa = _autodiscover_pg_conn()
inserter = VectorInserter(schema=schema, function=func, model=model_name)
_create_hnsw_index(supa, inserter.table_fqn())
for p in result_paths:
self.stage(p, new_name=p.name)
return EmbedderOutput(result_paths=result_paths)
@@ -0,0 +1,21 @@
"""Factory for embedding back‑ends."""
import logging
from typing import Any, List, Optional, Tuple, Dict, Type
from librarian_vspace.vecembed.embedding_interface import EmbeddingInterface
from librarian_vspace.vecembed.ollama_embedder import OllamaEmbedder
logger = logging.getLogger(__name__)
class EmbeddingGenerator:
_registry: Dict[str, Type[EmbeddingInterface]] = {
"ollama": OllamaEmbedder,
}
def generate_embedding(self, interface_name: str, model_name: str, text_to_embed: str, identifier: Any) -> Tuple[str, Optional[List[float]], Any]:
cls = self._registry.get(interface_name.lower())
if not cls:
raise ValueError(f"Unsupported embedding interface: {interface_name}")
embedder = cls(model_name=model_name)
return embedder.embed(text_to_embed, identifier)
@@ -0,0 +1,14 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any, List, Optional, Tuple
class EmbeddingInterface(ABC):
"""Contract for any embedding service implementation."""
def __init__(self, model_name: str, **kwargs: Any) -> None:
self.model_name = model_name
@abstractmethod
def embed(self, text_or_chunk: str, identifier: Any) -> Tuple[str, Optional[List[float]], Any]:
"""Return (original_text, embedding, identifier) — embedding may be None on failure."""
pass
@@ -0,0 +1,92 @@
"""Orchestrates loading, embedding, and storing a text chunk."""
from __future__ import annotations
import logging
from pathlib import Path
from typing import Any, Optional, Dict, Union
# Import the worker classes for type hinting
from librarian_vspace.vecembed.embedding_generator import EmbeddingGenerator
from librarian_vspace.vecembed.vector_inserter import VectorInserter
logger = logging.getLogger(__name__)
class EmbeddingWorkflow:
# Accept generator and inserter instances in __init__
def __init__(self,
chunk_path: Union[str, Path],
course_id: Any,
file_id: Any,
file_type: str,
interface_name: str, # Still needed for generate_embedding method
model_name: str, # Still needed for generate_embedding method
generator: EmbeddingGenerator, # Accept pre-instantiated generator
inserter: VectorInserter, # Accept pre-instantiated inserter
# db_schema and db_function are now implicit via the inserter
# db_schema: str = "librarian",
# db_function: str = "pdf_chunking",
):
self.chunk_path = Path(chunk_path)
self.course_id = course_id
self.file_id = file_id
self.file_type = file_type
# Keep interface_name and model_name as they are passed to the generator's method
self.interface_name = interface_name
self.model_name = model_name
# Assign the passed instances instead of creating new ones
self.generator = generator
self.inserter = inserter
# No need to store db_schema/db_function here if inserter handles it
# ---------------- helpers ----------------
def _load_chunk(self) -> Optional[str]:
try:
text = self.chunk_path.read_text(encoding="utf-8").strip()
if not text:
logger.warning("Chunk %s is empty", self.chunk_path)
return None
return text
except Exception as exc:
logger.error("Failed to read %s: %s", self.chunk_path, exc)
return None
def process(self) -> bool:
chunk_text = self._load_chunk()
if chunk_text is None:
return False
# Use the shared generator instance
original_text, vector, _ = self.generator.generate_embedding(
interface_name=self.interface_name, # Pass parameters to the method
model_name=self.model_name, # Pass parameters to the method
text_to_embed=chunk_text,
identifier=self.file_id,
)
if vector is None:
# Log failure within generator if not already done, or here
logger.error(f"Failed to generate embedding for {self.chunk_path}")
return False
payload: Dict[str, Any] = {
"chunk": original_text,
"course_id": self.course_id,
"file_id": self.file_id,
"file_type": self.file_type,
"embedding": vector,
}
# Use the shared inserter instance
insert_result = self.inserter.insert_vector(payload)
if insert_result is None:
logger.error(f"Failed to insert vector for {self.chunk_path}")
return False
logger.debug(f"Successfully processed and inserted {self.chunk_path}")
return True # Indicate success
# Keep __all__ if needed
# __all__ = ["EmbeddingWorkflow"]
@@ -0,0 +1,44 @@
"""Ollama-based embedding implementation (env handled at application layer)."""
from __future__ import annotations
import logging
import os
from typing import Any, List, Optional, Tuple
import requests
from librarian_vspace.vecembed.embedding_interface import EmbeddingInterface
logger = logging.getLogger(__name__)
class OllamaEmbedder(EmbeddingInterface):
def __init__(self, model_name: str, **kwargs: Any) -> None:
super().__init__(model_name=model_name)
self.base_url = os.getenv("OLLAMA_BASE_URL")
if not self.base_url:
raise ValueError("OLLAMA_BASE_URL not configured – ensure env is set in the examples layer")
self.api_endpoint = f"{self.base_url.rstrip('/')}/api/embeddings"
def embed(self, text_or_chunk: str, identifier: Any) -> Tuple[str, Optional[List[float]], Any]:
payload = {"model": self.model_name, "prompt": text_or_chunk}
vector: Optional[List[float]] = None
try:
logger.debug("Requesting embedding for id=%s", identifier)
resp = requests.post(self.api_endpoint, json=payload, timeout=3600, headers={"Content-Type": "application/json"})
resp.raise_for_status()
data = resp.json()
if isinstance(data.get("embedding"), list):
vector = data["embedding"]
logger.debug("Received embedding dim=%d for id=%s", len(vector), identifier)
else:
logger.error("Invalid response from Ollama: %s", data)
except requests.exceptions.Timeout:
logger.error("Timeout contacting Ollama at %s", self.api_endpoint)
except requests.exceptions.RequestException as exc:
logger.error("HTTP error contacting Ollama: %s", exc)
except Exception:
logger.exception("Unexpected error during embed for id=%s", identifier)
return text_or_chunk, vector, identifier
Whitespace-only changes.
@@ -0,0 +1,23 @@
from __future__ import annotations
import logging
from typing import Any, Dict, List, Optional
from librarian_vspace.vutils.vector_class import BaseVectorOperator
logger = logging.getLogger(__name__)
class VectorInserter(BaseVectorOperator):
"""High-level write helper for embeddings."""
def insert_vector(self, data: Dict[str, Any]) -> Optional[List[Dict[str, Any]]]:
if not self.table:
logger.error("Table resolution failed earlier")
return None
preview = {k: (f"<vector,len={len(v)}>" if k == "embedding" else v) for k, v in data.items()}
logger.debug("Insert → %s.%s :: %s", self.schema, self.table, preview)
try:
resp = self.spc.schema(self.schema).table(self.table).insert(data).execute()
return resp.data if isinstance(resp.data, list) else []
except Exception:
logger.exception("Insert failed for %s", self.table_fqn())
return None
@@ -0,0 +1,2 @@
def hello() -> str:
return "Hello from librarian_vspace!"
@@ -0,0 +1,264 @@
"""Loads vectors from Supabase, reduces dimensions using t-SNE."""
from __future__ import annotations
import logging
import json # Import json for parsing
from typing import Any, Dict, List, Optional, Tuple
import numpy as np
import pandas as pd
from sklearn.manifold import TSNE
# Assuming vutils is installed or in the python path
try:
from librarian_vspace.vutils.vector_class import BaseVectorOperator
except ImportError as e:
logging.error(f"Failed to import vutils: {e}. Ensure vutils package is installed.")
raise
logger = logging.getLogger(__name__)
class VectorLoaderError(Exception):
"""Custom exception for loader errors."""
pass
class VectorLoader:
"""Fetches vectors and applies t-SNE."""
DEFAULT_TSNE_PARAMS = {
"n_components": 3,
"perplexity": 30.0, # Adjust based on dataset size (5-50 typically)
"n_iter": 1000, # Minimum recommended iterations
"learning_rate": "auto", # Usually a good default
"init": "pca", # PCA initialization is often faster and more stable
"random_state": 42, # For reproducibility
"n_jobs": -1, # Use all available CPU cores
"verbose": 1, # Log progress (controls scikit-learn's verbosity)
}
def __init__(self, schema: str, function: str, model: str, embedding_column: str = "embedding"):
"""
Initializes the loader.
(Constructor remains the same)
"""
logger.info(f"Initializing VectorLoader for {schema=}, {function=}, {model=}")
try:
self.operator = BaseVectorOperator(schema=schema, function=function, model=model)
self.embedding_column = embedding_column
if not self.operator.table:
raise VectorLoaderError("BaseVectorOperator failed to resolve table.")
logger.info(f"Target table resolved to: {self.operator.table_fqn()}")
except (ImportError, ValueError, RuntimeError) as e:
logger.exception("Failed to initialize BaseVectorOperator.")
raise VectorLoaderError(f"Failed to initialize BaseVectorOperator: {e}") from e
def _parse_vector_string(self, vector_str: Any) -> Optional[List[float]]:
"""Safely parses the string representation of a vector into a list of floats."""
if not isinstance(vector_str, str):
# If it's already a list (less likely now, but safe check), return it if valid
if isinstance(vector_str, list) and all(isinstance(n, (int, float)) for n in vector_str):
return vector_str # Assume it's already correctly parsed
logger.debug(f"Unexpected type for vector parsing: {type(vector_str)}. Skipping.")
return None
try:
# Use json.loads which correctly handles [...] syntax
parsed_list = json.loads(vector_str)
if isinstance(parsed_list, list) and all(isinstance(n, (int, float)) for n in parsed_list):
return [float(n) for n in parsed_list] # Ensure elements are floats
else:
logger.warning(f"Parsed vector string '{vector_str[:50]}...' but result is not a list of numbers.")
return None
except json.JSONDecodeError:
logger.warning(f"Failed to JSON decode vector string: '{vector_str[:50]}...'")
return None
except Exception as e:
logger.error(f"Unexpected error parsing vector string '{vector_str[:50]}...': {e}", exc_info=True)
return None
def fetch_all_vectors(self, limit: Optional[int] = None) -> pd.DataFrame:
"""
Fetches all vectors and metadata from the resolved table.
Parses string representations of vectors into lists.
Args:
limit: Optional limit on the number of rows to fetch (for large tables).
Returns:
A pandas DataFrame with columns like 'file_id', 'chunk', 'embedding' (as list).
Raises:
VectorLoaderError: If fetching fails or no data is found.
"""
if not self.operator.table or not self.operator.schema:
raise VectorLoaderError("Operator not initialized, table name or schema is unknown.")
table_name = self.operator.table
schema_name = self.operator.schema
select_columns = f"file_id, chunk, {self.embedding_column}"
logger.info(f"Fetching data from {schema_name}.{table_name} (columns: {select_columns})...")
try:
query = self.operator.spc.schema(schema_name).table(table_name).select(select_columns)
if limit:
logger.info(f"Applying limit: {limit}")
query = query.limit(limit)
response = query.execute()
if not response.data:
logger.warning(f"No data found in table {self.operator.table_fqn()}.")
return pd.DataFrame(columns=['file_id', 'chunk', self.embedding_column])
logger.info(f"Fetched {len(response.data)} rows.")
df = pd.DataFrame(response.data)
# --- FIX: Parse the embedding string into a list ---
logger.info(f"Parsing string representation in '{self.embedding_column}' column...")
parsed_embeddings = df[self.embedding_column].apply(self._parse_vector_string)
# Overwrite the original string column with the parsed list (or None if parsing failed)
df[self.embedding_column] = parsed_embeddings
logger.debug(f"Sample '{self.embedding_column}' data after parsing (first 5 rows):\n{df[[self.embedding_column]].head()}")
# --- END FIX ---
# === Enhanced Debugging for Embedding Column (Now checks the parsed list) ===
logger.info(f"Checking validity of parsed '{self.embedding_column}' column...")
if self.embedding_column not in df.columns:
raise VectorLoaderError(f"Required embedding column '{self.embedding_column}' missing after processing.")
# 1. Check for NULLs (includes rows where parsing failed and returned None)
initial_count = len(df)
null_mask = df[self.embedding_column].isnull()
null_count = null_mask.sum()
if null_count > 0:
logger.warning(f"Found {null_count} rows with NULL or unparsable vectors in '{self.embedding_column}'.")
df_no_nulls = df.dropna(subset=[self.embedding_column])
count_after_null_drop = len(df_no_nulls)
logger.debug(f"{count_after_null_drop} rows remaining after dropping NULLs/unparsable.")
# 2. Check for non-empty list type (This check might be slightly redundant now if parsing worked, but keep for safety)
if not df_no_nulls.empty:
def is_valid_list(x):
# Check should pass if parsing was successful
return isinstance(x, list) and len(x) > 0
valid_list_mask = df_no_nulls[self.embedding_column].apply(is_valid_list)
invalid_list_count = len(df_no_nulls) - valid_list_mask.sum()
if invalid_list_count > 0:
# This indicates an issue with the parsing logic or unexpected data format
logger.error(f"Found {invalid_list_count} rows where '{self.embedding_column}' is not a non-empty list *after parsing*. This should not happen.")
invalid_entries = df_no_nulls[~valid_list_mask][self.embedding_column]
for i, entry in enumerate(invalid_entries.head(5)):
logger.debug(f" Problematic entry example {i+1}: Type={type(entry)}, Value='{str(entry)[:100]}...'")
df_filtered = df_no_nulls[valid_list_mask].copy()
else:
df_filtered = df_no_nulls
final_count = len(df_filtered)
# === End Enhanced Debugging ===
if final_count < initial_count:
logger.warning(f"Filtered out {initial_count - final_count} rows total due to missing/invalid '{self.embedding_column}'.")
if df_filtered.empty:
logger.warning(f"No valid embedding data found after filtering. Check data in table {self.operator.table_fqn()} and parsing logic.")
return pd.DataFrame(columns=['file_id', 'chunk', self.embedding_column])
logger.info(f"Proceeding with {final_count} valid rows.")
# Validate and potentially add placeholder metadata columns AFTER filtering
if 'file_id' not in df_filtered.columns:
logger.warning("'file_id' column missing, using index instead.")
df_filtered['file_id'] = df_filtered.index
if 'chunk' not in df_filtered.columns:
logger.warning("'chunk' column missing, hover text will be limited.")
df_filtered['chunk'] = "N/A"
return df_filtered
except Exception as e:
logger.exception(f"Failed to fetch data from {self.operator.table_fqn()}.")
if 'relation' in str(e) and 'does not exist' in str(e):
raise VectorLoaderError(f"Table/Relation not found error: {e}. Check schema/table name and permissions.") from e
else:
raise VectorLoaderError(f"Database query failed: {e}") from e
# reduce_dimensions and load_and_reduce methods remain the same as the previous version
# (they expect df with a valid list in the embedding column)
def reduce_dimensions(self, df: pd.DataFrame, tsne_params: Optional[Dict[str, Any]] = None) -> pd.DataFrame:
"""
Applies t-SNE to reduce embedding dimensions to 3D.
(Code remains the same as previous correct version)
"""
if df.empty:
logger.warning("Input DataFrame for reduce_dimensions is empty. Returning empty DataFrame.")
empty_df_with_cols = df.copy()
for col in ['x', 'y', 'z']:
if col not in empty_df_with_cols:
empty_df_with_cols[col] = pd.Series(dtype=float)
return empty_df_with_cols
if self.embedding_column not in df.columns:
raise VectorLoaderError(f"Embedding column '{self.embedding_column}' missing in DataFrame passed to reduce_dimensions.")
try:
embeddings = np.array(df[self.embedding_column].tolist(), dtype=float)
except ValueError as ve:
logger.error(f"Failed to convert embedding list to numeric numpy array: {ve}")
raise VectorLoaderError(f"Data in '{self.embedding_column}' could not be converted to numeric vectors.") from ve
if embeddings.ndim != 2:
raise VectorLoaderError(f"Embedding data has unexpected dimensions: {embeddings.ndim} (expected 2). Shape: {embeddings.shape}")
n_samples = embeddings.shape[0]
if n_samples < 2:
logger.warning(f"Found only {n_samples} valid vector(s). t-SNE requires at least 2. Assigning default 3D coordinates.")
default_coords = [[0.0, 0.0, 0.0]] * n_samples
df[['x', 'y', 'z']] = default_coords
return df
logger.info(f"Applying t-SNE to {n_samples} vectors of dimension {embeddings.shape[1]}...")
current_tsne_params = self.DEFAULT_TSNE_PARAMS.copy()
if tsne_params:
current_tsne_params.update(tsne_params)
logger.info(f"Using custom t-SNE params: {tsne_params}")
if n_samples <= current_tsne_params['perplexity']:
new_perplexity = max(5.0, float(n_samples - 1))
logger.warning(f"Adjusting t-SNE perplexity from {current_tsne_params['perplexity']:.1f} "
f"to {new_perplexity:.1f} due to low sample count ({n_samples}).")
current_tsne_params['perplexity'] = new_perplexity
if n_samples * embeddings.shape[1] > 100000 and current_tsne_params['n_iter'] < 1000:
logger.warning(f"Dataset size seems large, increasing t-SNE n_iter from {current_tsne_params['n_iter']} to 1000 for better convergence.")
current_tsne_params['n_iter'] = 1000
try:
logger.debug(f"Final t-SNE parameters: {current_tsne_params}")
tsne = TSNE(**current_tsne_params)
reduced_embeddings = tsne.fit_transform(embeddings)
df[['x', 'y', 'z']] = reduced_embeddings
logger.info("t-SNE reduction complete.")
return df
except Exception as e:
logger.exception("t-SNE dimensionality reduction failed.")
raise VectorLoaderError(f"t-SNE failed: {e}") from e
def load_and_reduce(self, limit: Optional[int] = None, tsne_params: Optional[Dict[str, Any]] = None) -> pd.DataFrame:
"""Orchestrates fetching vectors and reducing dimensions."""
logger.info("Starting vector load and reduction process...")
df_raw_filtered = self.fetch_all_vectors(limit=limit)
df_reduced = self.reduce_dimensions(df_raw_filtered, tsne_params=tsne_params)
logger.info("Vector load and reduction process finished.")
return df_reduced
Whitespace-only changes.
@@ -0,0 +1,776 @@
# --- START OF FILE visualizer.py ---
"""Dash/Plotly based 3D visualizer for vector embeddings with tabs, clustering, filtering, and centroid click interaction."""
from __future__ import annotations
import logging
import os
from io import StringIO
from typing import Any, Dict, List, Optional, Tuple
import dash
import numpy as np
import pandas as pd
import plotly.graph_objects as go
import plotly.express as px
from dash import dcc, html, ctx # Import ctx
from dash.dependencies import Input, Output, State
from dash.exceptions import PreventUpdate
from sklearn.cluster import KMeans
# --- Imports ---
try:
from librarian_vspace.vecembed.embedding_generator import EmbeddingGenerator
except ImportError as e:
logging.error(f"Import vecembed failed: {e}. Using Dummy.")
# Define dummy class correctly indented
class EmbeddingGenerator:
"""Dummy class if vecembed import fails."""
def generate_embedding(*args, **kwargs) -> Tuple[
str, None, Any]: # Match expected output type Optional[List[float]]
logging.error("Dummy EmbeddingGenerator called.")
text_to_embed = kwargs.get("text_to_embed", args[3] if len(args) > 3 else "unknown")
identifier = kwargs.get("identifier", args[4] if len(args) > 4 else "unknown")
logger.debug(f"Dummy generate_embedding called for text='{text_to_embed}', id='{identifier}'")
# Return None for the vector part to match expected type
return text_to_embed, None, identifier
try:
from librarian_vspace.vutils.vector_query_loader import VectorQueryLoader as VectorLoader, VectorQueryLoaderError as VectorLoaderError
except ImportError as e:
logging.error(f"Import loader failed: {e}. Using Dummy.")
# Define dummy classes correctly indented
class VectorLoader:
"""Dummy class if loader import fails."""
def __init__(self, *args, **kwargs):
logging.error("Dummy VectorLoader initialized.")
pass
def load_and_reduce(self, *args, **kwargs) -> pd.DataFrame:
logging.error("Dummy VectorLoader load_and_reduce called.")
return pd.DataFrame() # Return empty DataFrame
class VectorLoaderError(Exception):
"""Dummy exception if loader import fails."""
pass
# --- End Imports ---
logger = logging.getLogger(__name__)
DEFAULT_N_CLUSTERS = 8
# Opacity constants
OPACITY_DEFAULT = 0.8
OPACITY_SEARCH_DIMMED = 0.1 # Reduced from 0.6 to 0.3 for better visual distinction
OPACITY_SELECTED_CLUSTER = 0.9
class VectorVisualizer:
def __init__(self, initial_data: pd.DataFrame,
db_schema: str, db_function: str,
interface_name: str, model_name: str,
embedding_column: str = "embedding",
initial_limit: Optional[int] = None,
initial_perplexity: float = 30.0,
n_clusters: int = DEFAULT_N_CLUSTERS
):
required_cols = ['x', 'y', 'z', 'file_id', 'chunk', embedding_column]
processed_data_json: Optional[str] = None
processed_color_map: Dict = {}
processed_original_embeddings: np.ndarray = np.array([])
processed_cluster_centroids: Dict[str, List[float]] = {}
self.embedding_column = embedding_column
self.n_clusters = n_clusters
self.db_schema = db_schema
self.db_function = db_function
self.model_name = model_name
self.limit = initial_limit
self.perplexity = initial_perplexity
self.interface_name = interface_name
# Use the correctly defined EmbeddingGenerator (either real or dummy)
self.app = dash.Dash(__name__, suppress_callback_exceptions=True)
self.embedding_generator = EmbeddingGenerator() # Instantiated here
if initial_data.empty or not all(col in initial_data.columns for col in required_cols):
logger.warning("Initial DataFrame empty/invalid.")
base_cols = required_cols + ['cluster', 'hover_text']
initial_df_processed = pd.DataFrame(columns=base_cols)
else:
try:
logger.info("Processing initial data...")
df_copy = initial_data.copy()
df_after_kmeans, kmeans_color_map = self._run_kmeans(df_copy, self.n_clusters)
if not isinstance(df_after_kmeans, pd.DataFrame): raise TypeError("KMeans failed.")
processed_color_map = kmeans_color_map
df_after_prepare = self._prepare_plot_data(df_after_kmeans)
if not isinstance(df_after_prepare, pd.DataFrame): raise TypeError("Prep data failed.")
initial_df_processed = df_after_prepare
if not initial_df_processed.empty and all(
c in initial_df_processed for c in ['x', 'y', 'z', 'cluster']):
processed_cluster_centroids = self._calculate_centroids(initial_df_processed)
else:
logger.warning("Could not calculate initial centroids.")
if not initial_df_processed.empty:
processed_data_json = initial_df_processed.to_json(date_format='iso', orient='split')
else:
logger.warning("DataFrame empty after processing.")
if not initial_df_processed.empty and self.embedding_column in initial_df_processed.columns:
try:
emb = initial_df_processed[self.embedding_column].iloc[0]
if isinstance(emb, np.ndarray):
processed_original_embeddings = np.stack(initial_df_processed[self.embedding_column].values)
elif isinstance(emb, list):
processed_original_embeddings = np.array(
initial_df_processed[self.embedding_column].tolist(), dtype=float)
else:
raise TypeError("Unsupported embedding type.")
except Exception as emb_err:
logger.error(f"Embed processing error: {emb_err}"); processed_original_embeddings = np.array([])
else:
logger.warning("Could not extract original embeddings.")
except Exception as e:
logger.error(f"Initial processing error: {e}", exc_info=True)
processed_data_json, processed_color_map, processed_original_embeddings, processed_cluster_centroids = None, {}, np.array(
[]), {}
initial_df_processed = pd.DataFrame()
self.initial_data_json = processed_data_json
self.initial_cluster_color_map = processed_color_map
self.initial_cluster_centroids = processed_cluster_centroids
self.original_embeddings = processed_original_embeddings
# Determine slider limits and elbow‑based default
self.max_clusters = max(1, len(initial_data))
try:
self.optimal_clusters = self._estimate_optimal_clusters(processed_original_embeddings,
max_k=min(10, self.max_clusters))
except Exception:
self.optimal_clusters = self.n_clusters
# Use elbow result as the current cluster count
self.n_clusters = self.optimal_clusters
self._build_layout();
self._register_callbacks()
def _run_kmeans(self, df: pd.DataFrame, n_clusters: int) -> Tuple[pd.DataFrame, Dict[str, str]]:
"""Runs K-Means, assigns string cluster labels."""
default_map = {"-1": "grey"}
if df.empty or self.embedding_column not in df.columns: df['cluster'] = "-1"; return df, default_map
try:
emb_col = df[self.embedding_column]
if isinstance(emb_col.iloc[0], np.ndarray):
embeddings = np.stack(emb_col.values)
elif isinstance(emb_col.iloc[0], list):
embeddings = np.array(emb_col.tolist(), dtype=float)
else:
raise TypeError("Unsupported embedding type.")
if embeddings.ndim != 2: raise ValueError("Embeddings must be 2D.")
eff_clusters = min(n_clusters, embeddings.shape[0])
if embeddings.shape[0] < 2 or eff_clusters < 1:
lbl = "0" if embeddings.shape[0] > 0 else "-1";
df['cluster'] = lbl
colors = px.colors.qualitative.Plotly;
return df, {lbl: colors[0 % len(colors)]} if lbl == "0" else default_map
if eff_clusters == 1: df['cluster'] = "0"; colors = px.colors.qualitative.Plotly; return df, {
"0": colors[0 % len(colors)]}
kmeans = KMeans(n_clusters=eff_clusters, random_state=42, n_init='auto')
df['cluster'] = kmeans.fit_predict(embeddings).astype(str)
unique_labels = sorted(df['cluster'].unique())
colors = px.colors.qualitative.Plotly
color_map = {lbl: colors[i % len(colors)] for i, lbl in enumerate(unique_labels)}
return df, color_map
except (TypeError, ValueError) as e:
logger.error(f"KMeans input error: {e}"); df['cluster'] = "-1"; return df, default_map
except Exception as e:
logger.exception("KMeans failed."); df['cluster'] = "-1"; return df, default_map
def _prepare_plot_data(self, df: pd.DataFrame) -> pd.DataFrame:
"""Prepares hover text."""
if df.empty: return df
if 'cluster' not in df.columns: df['cluster'] = 'N/A'
df_copy = df.copy()
def gen_hover(row):
try:
return f"ID: {row.get('file_id', 'N/A')}<br>Cluster: {str(row.get('cluster', 'N/A'))}<br>Chunk: {str(row.get('chunk', ''))[:200]}{'...' if len(str(row.get('chunk', ''))) > 200 else ''}"
except Exception:
return "Hover gen error"
try:
df_copy['hover_text'] = df_copy.apply(gen_hover, axis=1); return df_copy
except Exception as e:
logger.error(f"Hover gen failed: {e}"); return df
def _calculate_centroids(self, df: pd.DataFrame) -> Dict[str, List[float]]:
"""Calculates 3D centroids."""
centroids = {}
required = ['x', 'y', 'z', 'cluster'];
numeric_cols = ['x', 'y', 'z']
if df.empty or not all(col in df.columns for col in required): return centroids
df_copy = df.copy();
df_copy['cluster'] = df['cluster'].astype(str)
for col in numeric_cols:
if not pd.api.types.is_numeric_dtype(df_copy[col]):
try:
df_copy[col] = pd.to_numeric(df_copy[col], errors='coerce')
except Exception:
logger.error(f"Centroid calc: conv error '{col}'"); return {}
if df_copy[col].isnull().any(): logger.warning(f"Centroid calc: NaNs in '{col}'")
try:
# Calculate mean, drop rows where ALL numeric_cols are NaN, then drop rows where the resulting mean is NaN
centroid_data = df_copy.dropna(subset=numeric_cols, how='all').groupby('cluster')[
numeric_cols].mean().dropna()
return {str(idx): row.tolist() for idx, row in centroid_data.iterrows()}
except Exception as e:
logger.exception("Centroid calc failed."); return {}
def _create_base_figure(self) -> go.Figure:
"""Creates base Plotly figure."""
fig = go.Figure()
fig.update_layout(title='3D t-SNE', margin=dict(l=0, r=0, b=0, t=40),
scene_camera_eye=dict(x=1.5, y=1.5, z=0.5),
scene=dict(xaxis_title='TSNE-1', yaxis_title='TSNE-2', zaxis_title='TSNE-3',
aspectmode='data'),
legend=dict(yanchor="top", y=0.99, xanchor="left", x=0.01, bgcolor='rgba(255,255,255,0.7)'),
hovermode='closest')
return fig
def _build_layout(self) -> None:
"""Builds the Dash layout."""
self.app.layout = html.Div([
dcc.Store(id='stored-data', data=self.initial_data_json),
dcc.Store(id='cluster-color-map-store', data=self.initial_cluster_color_map),
dcc.Store(id='cluster-centroids-store', data=self.initial_cluster_centroids),
dcc.Store(id='search-results-store', data=None),
dcc.Store(id='selected-cluster-store', data=None), # Store for click state
html.H1("Vector Embedding Visualizer"),
dcc.Tabs(id="main-tabs", value='tab-vis', children=[
dcc.Tab(label='Visualization', value='tab-vis', children=[
html.Div([ # Controls
html.Div(
[html.Button('Reload Data', id='reload-button', n_clicks=0, style={'marginRight': '10px'}),
dcc.Input(id='search-input', type='text', placeholder='Search term...', debounce=True,
style={'width': '40%', 'marginRight': '5px'}),
html.Button('Search', id='search-button', n_clicks=0)],
style={'padding': '10px', 'display': 'flex'}),
html.Div([html.Label("Similarity:", style={'marginRight': '10px'}),
dcc.Slider(id='similarity-slider', min=0, max=1, step=0.01, value=0.0,
marks={i / 10: f'{i / 10:.1f}' for i in range(11)},
tooltip={"placement": "bottom", "always_visible": True}, disabled=True)],
id='slider-container', style={'display': 'none', 'padding': '10px 20px'}),
html.Div([
html.Label("Clusters:", style={'marginRight': '10px'}),
dcc.Slider(
id='cluster-slider',
min=1,
max=self.max_clusters,
step=1,
value=self.optimal_clusters,
marks=self._cluster_marks(),
tooltip={'placement': 'bottom', 'always_visible': True}
)
], style={'padding': '10px 20px'}),
html.Div(id='status-output', style={'padding': '10px', 'color': 'blue', 'minHeight': '20px'}),
dcc.Loading(id="loading-graph", type="circle",
children=dcc.Graph(id='vector-graph', style={'height': '70vh'}))
])
]),
dcc.Tab(label='Settings', value='tab-settings', children=[
html.Div([html.H3("Settings"), html.Div([html.Label("Marker Size:", style={'marginRight': '10px'}),
dcc.Slider(id='size-slider', min=1, max=15, step=1,
value=4,
marks={i: str(i) for i in range(1, 16)},
tooltip={'placement': 'bottom',
'always_visible': True})],
style={'padding': '10px 20px'})], style={'padding': '20px'})
]),
]),
])
def _cosine_similarity(self, vec1: np.ndarray, vec2: np.ndarray) -> np.ndarray | float:
"""Calculates cosine similarity."""
if not isinstance(vec1, np.ndarray): vec1 = np.array(vec1, dtype=float)
if not isinstance(vec2, np.ndarray): vec2 = np.array(vec2, dtype=float)
if vec1.ndim == 1: vec1 = vec1.reshape(1, -1)
if vec2.ndim == 1: vec2 = vec2.reshape(1, -1)
if vec1.shape[1] != vec2.shape[1]: raise ValueError("Vector dimension mismatch")
norm1 = np.linalg.norm(vec1, axis=1, keepdims=True);
norm2 = np.linalg.norm(vec2, axis=1, keepdims=True)
z1 = (norm1 == 0).flatten();
z2 = (norm2 == 0).flatten()
# Handle potential division by zero for zero vectors
norm1[z1] = 1.0;
norm2[z2] = 1.0
sim = np.dot(vec1 / norm1, (vec2 / norm2).T)
# Ensure zero vectors result in zero similarity
if np.any(z1): sim[:, :] = 0.0;
sim[:, z2] = 0.0
sim = np.clip(sim, -1.0, 1.0)
return sim.item() if sim.size == 1 else sim.flatten()
def _find_neighbors(self, search_vector: List[float], k: int = 10) -> Optional[Tuple[np.ndarray, np.ndarray]]:
"""Finds k nearest neighbors."""
if not search_vector or not isinstance(search_vector, list): return None
if self.original_embeddings is None or self.original_embeddings.size == 0: return None
try:
vec = np.array(search_vector, dtype=float)
if vec.ndim != 1: raise ValueError("Search vector != 1D.")
if self.original_embeddings.ndim != 2: raise ValueError("Embeddings != 2D.")
if self.original_embeddings.shape[1] != vec.shape[0]: raise ValueError("Dimension mismatch.")
sims = self._cosine_similarity(vec, self.original_embeddings)
if not isinstance(sims, np.ndarray) or sims.ndim != 1 or sims.shape[0] != self.original_embeddings.shape[
0]: raise TypeError("Similarity calc failed.")
k_actual = min(k, len(sims));
if k_actual <= 0: return None
idx = np.argpartition(sims, -k_actual)[-k_actual:] # Get indices of top k
sorted_idx = idx[np.argsort(sims[idx])][::-1] # Sort top k indices by similarity
return sorted_idx, sims[sorted_idx]
except (ValueError, TypeError) as e:
logger.error(f"Neighbor input error: {e}"); return None
except Exception as e:
logger.exception(f"Neighbor search error: {e}"); return None
# --- Callbacks ---
def _register_callbacks(self) -> None:
"""Sets up Dash callbacks."""
# --- Callback 1: Reload Button ---
@self.app.callback(
Output('stored-data', 'data', allow_duplicate=True), Output('cluster-color-map-store', 'data'),
Output('cluster-centroids-store', 'data'),
Output('status-output', 'children'), Output('search-results-store', 'data', allow_duplicate=True),
Output('selected-cluster-store', 'data', allow_duplicate=True),
Input('reload-button', 'n_clicks'), prevent_initial_call=True)
def handle_reload(n_clicks: int) -> Tuple[Optional[str], Dict, Dict, str, None, None]:
if n_clicks == 0: raise PreventUpdate
logger.info("Reload triggered...")
status = "Reloading...";
color_map, centroids, data_json = {}, {}, None;
self.original_embeddings = np.array([])
try:
# Ensure VectorLoader is properly imported or defined (dummy used if import fails)
loader = VectorLoader(self.db_schema, self.db_function, self.model_name, self.embedding_column)
reduced_data = loader.load_and_reduce(limit=self.limit, tsne_params={"perplexity": self.perplexity})
if not isinstance(reduced_data, pd.DataFrame) or reduced_data.empty: raise VectorLoaderError("No data.")
df_clustered, color_map = self._run_kmeans(reduced_data.copy(), self.n_clusters)
if not isinstance(df_clustered, pd.DataFrame): raise TypeError(
"KMeans failed post-reload.") # Add check
df_final = self._prepare_plot_data(df_clustered)
if not isinstance(df_final, pd.DataFrame): raise TypeError(
"Prepare plot failed post-reload.") # Add check
if not df_final.empty and all(c in df_final for c in ['x', 'y', 'z', 'cluster']):
centroids = self._calculate_centroids(df_final)
else:
logger.warning("Could not calculate centroids after reload (missing cols or empty).")
if not reduced_data.empty and self.embedding_column in reduced_data.columns:
try:
emb_col = reduced_data[self.embedding_column]
# Check type of first element before processing
if not emb_col.empty:
first_emb = emb_col.iloc[0]
if isinstance(first_emb, np.ndarray):
self.original_embeddings = np.stack(emb_col.values)
elif isinstance(first_emb, list):
self.original_embeddings = np.array(emb_col.tolist(), dtype=float)
else:
raise TypeError(f"Unsupported reloaded embed type: {type(first_emb)}")
logger.info(f"Stored reloaded embeddings (shape: {self.original_embeddings.shape}).")
else:
logger.warning("Embedding column empty during reload storage.")
except Exception as e:
logger.error(f"Store embed fail: {e}"); self.original_embeddings = np.array([])
else:
logger.warning("Embedding column missing or df empty during reload storage.")
if not df_final.empty:
data_json = df_final.to_json(date_format='iso',
orient='split'); status = f"Reloaded ({len(df_final)} pts)."
else:
status = "Warning: Reload empty post-process."
except (VectorLoaderError, TypeError, Exception) as e:
logger.exception(
f"Reload error: {e}"); status = f"Error: {e}"; data_json, color_map, centroids = None, {}, {}; self.original_embeddings = np.array(
[])
return data_json, color_map, centroids, status, None, None
# --- Callback 1 b: Cluster‑count Slider ---
@self.app.callback(
Output('stored-data', 'data', allow_duplicate=True),
Output('cluster-color-map-store', 'data', allow_duplicate=True),
Output('cluster-centroids-store', 'data', allow_duplicate=True),
Output('status-output', 'children', allow_duplicate=True),
Input('cluster-slider', 'value'),
State('stored-data', 'data'),
prevent_initial_call=True
)
def update_n_clusters(k: int, stored_json: str):
if not stored_json:
raise PreventUpdate
# Update the visualizer state
self.n_clusters = k
try:
df = pd.read_json(StringIO(stored_json), orient='split')
df, color_map = self._run_kmeans(df, k)
df = self._prepare_plot_data(df)
centroids = self._calculate_centroids(df)
status = f"Cluster count set to {k}."
return (df.to_json(date_format='iso', orient='split'),
color_map,
centroids,
status)
except Exception as err:
logger.error(f"Clustering update failed: {err}")
raise PreventUpdate
# --- Callback 2: Search Button ---
@self.app.callback(
Output('search-results-store', 'data', allow_duplicate=True),
Output('status-output', 'children', allow_duplicate=True),
Input('search-button', 'n_clicks'), State('search-input', 'value'), prevent_initial_call=True)
def handle_search(n_clicks: int, term: str) -> Tuple[Optional[Dict], str]:
if n_clicks == 0 or not term: return None, "Enter search term."
logger.info(f"Search: '{term}'");
status = f"Embedding '{term}'..."
try:
if self.original_embeddings is None or self.original_embeddings.size == 0: return None, "Error: No data."
_, vec, _ = self.embedding_generator.generate_embedding(self.interface_name, self.model_name, term,
"search")
if vec is None: return None, f"Error: Embed failed."
status = f"Finding neighbors...";
neighbors = self._find_neighbors(vec, k=20)
if neighbors is None: return None, f"No neighbors found."
idx, sims = neighbors;
results = {"indices": idx.tolist(), "similarities": sims.tolist(), "term": term}
status = f"Found {len(idx)} neighbors.";
return results, status
except Exception as e:
logger.exception("Search error."); return None, f"Error: {e}"
# --- Callback 3: Slider Visibility ---
@self.app.callback(
Output('slider-container', 'style'), Output('similarity-slider', 'disabled'),
Output('similarity-slider', 'value'),
Input('search-results-store', 'data'), prevent_initial_call=True)
def update_slider_visibility(res: Optional[Dict]) -> Tuple[Dict, bool, float]:
show = res and isinstance(res, dict) and "indices" in res
style = {'display': 'block' if show else 'none', 'padding': '10px 20px'}
return style, not show, 0.0
# --- Callback 4: Graph Update (Main Logic with clickData fix and logging) ---
@self.app.callback(
Output('vector-graph', 'figure'), Output('selected-cluster-store', 'data'),
Output('status-output', 'children', allow_duplicate=True),
Input('stored-data', 'data'), Input('cluster-color-map-store', 'data'),
Input('cluster-centroids-store', 'data'),
Input('search-results-store', 'data'), Input('similarity-slider', 'value'), Input('size-slider', 'value'),
Input('vector-graph', 'clickData'), # Input for clicks
State('selected-cluster-store', 'data'), # Get current selection
prevent_initial_call='initial_duplicate' # Allow initial run
)
def update_graph(stored_data_json: Optional[str], cluster_color_map: Optional[Dict],
cluster_centroids: Optional[Dict[str, List[float]]],
search_results: Optional[Dict], similarity_threshold: float, size_value: int,
click_data: Optional[Dict],
current_selected_cluster: Optional[str]) -> Tuple[go.Figure, Optional[str], str]:
fig = self._create_base_figure();
status_msg = "";
new_selected_cluster = current_selected_cluster
trigger = ctx.triggered_id if ctx.triggered else "Initial"
logger.debug(f"--- Graph Update | Trigger: {trigger} | CurrentSel: {current_selected_cluster} ---")
# --- Data Load & Validation ---
if not stored_data_json: return fig, None, "No data."
try:
df = pd.read_json(StringIO(stored_data_json), orient='split')
if df.empty: return fig, None, "Empty data."
required = ['x', 'y', 'z', 'cluster', 'hover_text'];
assert all(col in df.columns for col in required)
df['cluster'] = df['cluster'].astype(str)
color_map = cluster_color_map if isinstance(cluster_color_map, dict) else {}
centroids = cluster_centroids if isinstance(cluster_centroids, dict) else {}
if not color_map:
logger.warning("Missing color map, generating default.")
unique_clusters = df['cluster'].unique();
colors = px.colors.qualitative.Plotly
color_map = {str(c): colors[i % len(colors)] for i, c in enumerate(unique_clusters)} or {
'0': 'grey'}
# Calculate overall data range (handle potential NaNs/Infs in full data)
df_finite = df[['x', 'y', 'z']].replace([np.inf, -np.inf], np.nan).dropna()
if not df_finite.empty:
overall_x_min, overall_x_max = df_finite['x'].min(), df_finite['x'].max()
overall_y_min, overall_y_max = df_finite['y'].min(), df_finite['y'].max()
overall_z_min, overall_z_max = df_finite['z'].min(), df_finite['z'].max()
logger.debug(
f"Overall Finite Range: X=[{overall_x_min:.2f}, {overall_x_max:.2f}], Y=[{overall_y_min:.2f}, {overall_y_max:.2f}], Z=[{overall_z_min:.2f}, {overall_z_max:.2f}]")
else:
logger.warning("No finite data points found in the dataset to calculate overall range.")
overall_x_min, overall_x_max = -10, 10 # Default ranges if no finite data
overall_y_min, overall_y_max = -10, 10
overall_z_min, overall_z_max = -10, 10
except Exception as e:
logger.exception("Graph data error."); return fig, current_selected_cluster, f"Error: {e}"
# --- Click Processing ---
if trigger == 'vector-graph':
logger.debug(f"Click Data Received: {click_data}")
if click_data and 'points' in click_data and click_data['points']:
point_data = click_data['points'][0]
clicked_customdata = point_data.get('customdata');
clicked_text = point_data.get('text', '')
logger.debug(f"Clicked Point Customdata: {clicked_customdata}");
logger.debug(f"Clicked Point Text: '{clicked_text}'")
is_centroid_click = False;
clicked_cluster_id = None
if isinstance(clicked_customdata, list) and len(clicked_customdata) > 0: clicked_customdata = \
clicked_customdata[0]
if isinstance(clicked_customdata, (str, int)): # Accept string or int cluster IDs
is_centroid_click = True;
clicked_cluster_id = str(clicked_customdata);
logger.info(f"Centroid Click Parsed via customdata: Cluster '{clicked_cluster_id}'")
elif isinstance(clicked_text, str) and clicked_text.startswith("Centroid: Cluster "):
try:
clicked_cluster_id = clicked_text.split("Centroid: Cluster ")[
1]; is_centroid_click = True; logger.info(
f"Centroid Click Parsed via text: Cluster '{clicked_cluster_id}'")
except Exception as parse_err:
logger.warning(f"Failed text parse: {parse_err}")
if is_centroid_click and clicked_cluster_id is not None:
if current_selected_cluster == clicked_cluster_id:
new_selected_cluster = None; status_msg = "Cluster view reset."; logger.info("Deselecting.")
else:
new_selected_cluster = clicked_cluster_id; status_msg = f"Showing Cluster {new_selected_cluster}."; logger.info(
f"Selecting {new_selected_cluster}.")
elif not is_centroid_click and current_selected_cluster is not None:
new_selected_cluster = None; status_msg = "Cluster view reset."; logger.info("Deselecting.")
else: # Click background
if current_selected_cluster is not None: new_selected_cluster = None; status_msg = "Cluster view reset."; logger.info(
"Deselecting.")
logger.debug(f"Click Result: new_selected_cluster = {new_selected_cluster}")
else:
logger.debug("No click trigger.")
# --- Data Filtering ---
active_selection_id = new_selected_cluster
df_to_plot = df.copy();
centroids_to_plot = centroids.copy()
logger.debug(f"Filtering based on active_selection_id: {active_selection_id}")
if active_selection_id is not None:
df_to_plot = df_to_plot[df_to_plot['cluster'] == active_selection_id]
centroids_to_plot = {cid: coords for cid, coords in centroids_to_plot.items() if
cid == active_selection_id}
logger.debug(f"Filtered DF rows: {len(df_to_plot)}")
if not df_to_plot.empty:
logger.debug(f"Coordinates of filtered points:\n{df_to_plot[['x', 'y', 'z']]}")
else:
logger.warning("Filtered DataFrame is empty.")
# --- Search Highlighting ---
search_highlight_mask = np.zeros(len(df_to_plot), dtype=bool)
search_term = None;
is_search_active = False;
highlight_sims = {}
if search_results and isinstance(search_results, dict) and "indices" in search_results:
is_search_active = True;
search_term = search_results.get("term", "N/A")
orig_indices = search_results.get("indices", []);
orig_sims = search_results.get("similarities", [])
if not df_to_plot.empty:
orig_to_current_map = {orig_idx: current_idx for current_idx, orig_idx in
enumerate(df_to_plot.index)}
current_indices_hl = [orig_to_current_map[oi] for i, oi in enumerate(orig_indices) if
i < len(orig_sims) and orig_sims[
i] >= similarity_threshold and oi in orig_to_current_map]
if current_indices_hl:
search_highlight_mask[current_indices_hl] = True
for i, orig_idx in enumerate(orig_indices):
if i < len(orig_sims) and orig_sims[
i] >= similarity_threshold and orig_idx in orig_to_current_map:
highlight_sims[orig_to_current_map[orig_idx]] = orig_sims[i]
else:
logger.warning("Cannot apply search highlighting - filtered df empty.")
# --- Plotting ---
df_search_hl = df_to_plot[search_highlight_mask];
df_normal = df_to_plot[~search_highlight_mask]
base_size = size_value;
normal_op = OPACITY_SELECTED_CLUSTER if active_selection_id else (
OPACITY_SEARCH_DIMMED if is_search_active else OPACITY_DEFAULT)
# --- Add Dummy Points if needed ---
num_points_to_plot = len(df_normal) + len(df_search_hl)
if active_selection_id is not None and num_points_to_plot <= 2:
logger.info(
f"Adding dummy invisible points to aid auto-ranging for cluster {active_selection_id} (points={num_points_to_plot}).")
# Use overall range calculated earlier
dummy_x = [overall_x_min, overall_x_max]
dummy_y = [overall_y_min, overall_y_max]
dummy_z = [overall_z_min, overall_z_max]
# Ensure dummy points are valid numbers (in case overall range calc failed)
if np.isfinite(dummy_x + dummy_y + dummy_z).all():
fig.add_trace(go.Scatter3d(
x=dummy_x, y=dummy_y, z=dummy_z,
mode='markers', marker=dict(size=1, opacity=0), # Invisible
hoverinfo='skip', showlegend=False, name='_dummy_'
))
else:
logger.warning("Could not add dummy points because overall range contained non-finite values.")
# Plot Normal Points
if not df_normal.empty:
finite_mask_normal = np.isfinite(df_normal[['x', 'y', 'z']]).all(axis=1)
df_normal_finite = df_normal[finite_mask_normal]
if not df_normal_finite.empty:
logger.debug(f"Plotting df_normal (len={len(df_normal_finite)}).")
colors = df_normal_finite['cluster'].map(color_map).fillna('darkgrey')
name = 'Embeddings' if active_selection_id is None else f'Cluster {active_selection_id}'
fig.add_trace(
go.Scatter3d(x=df_normal_finite['x'], y=df_normal_finite['y'], z=df_normal_finite['z'],
mode='markers',
marker=dict(color=colors, size=base_size, opacity=normal_op, line=dict(width=0.5)),
text=df_normal_finite['hover_text'], hoverinfo='text', name=name))
else:
logger.warning("No finite normal points to plot.")
# Plot Search Highlighted Points
if not df_search_hl.empty:
finite_mask_search = np.isfinite(df_search_hl[['x', 'y', 'z']]).all(axis=1)
df_search_hl_finite = df_search_hl[finite_mask_search]
if not df_search_hl_finite.empty:
hl_size = max(base_size * 1.5, base_size + 2);
hl_texts = []
# Need mapping from df_search_hl_finite index back to df_to_plot positional index for sims
positions_in_df_to_plot = df_to_plot.index.get_indexer_for(df_search_hl_finite.index)
for i, (global_index, row) in enumerate(df_search_hl_finite.iterrows()):
pos = positions_in_df_to_plot[i] # Get original position in df_to_plot
sim = highlight_sims.get(pos, float('nan'))
sim_txt = f"{sim:.4f}" if not np.isnan(sim) else "N/A"
hl_texts.append(f"{row['hover_text']}<br><b>Sim: {sim_txt}</b>")
fig.add_trace(
go.Scatter3d(x=df_search_hl_finite['x'], y=df_search_hl_finite['y'], z=df_search_hl_finite['z'],
mode='markers',
marker=dict(color='red', size=hl_size, opacity=1.0, symbol='diamond',
line=dict(color='black', width=1)), text=hl_texts, hoverinfo='text',
name=f'Search Neighbors'))
if not df_search_hl_finite[['x', 'y', 'z']].isnull().values.any(): # Search Centroid
try:
sc = df_search_hl_finite[['x', 'y', 'z']].mean().values; fig.add_trace(
go.Scatter3d(x=[sc[0]], y=[sc[1]], z=[sc[2]], mode='markers',
marker=dict(color='magenta', size=max(hl_size, 10), symbol='cross',
line=dict(width=1)), text=f"Search: '{search_term}' Centroid",
hoverinfo='text', name='Search Centroid'))
except Exception as e:
logger.warning(f"Search centroid plot fail: {e}")
else:
logger.warning("No finite search highlighted points to plot.")
# Plot Centroids (filtered)
if centroids_to_plot:
cent_size = base_size + 1;
logger.debug(f"Plotting centroids: {list(centroids_to_plot.keys())}")
for cid, coords in centroids_to_plot.items():
if isinstance(coords, list) and len(coords) == 3:
logger.debug(f"Plotting Centroid {cid} at coords: {coords}")
if np.isnan(coords).any() or np.isinf(coords).any(): logger.error(
f"!!! Centroid {cid} NaN/Inf coords !!!"); continue
color = color_map.get(str(cid), 'grey');
name = f"Centroid {cid}";
hover_txt = f"Centroid: Cluster {cid}"
fig.add_trace(go.Scatter3d(
x=[coords[0]], y=[coords[1]], z=[coords[2]], mode='markers',
marker=dict(color=color, size=cent_size, symbol='circle', opacity=0.9,
line=dict(color='black', width=1.5)),
customdata=[str(cid)], text=hover_txt, hoverinfo='text', name=name,
legendgroup="centroids", showlegend=True
))
else:
logger.warning(f"Invalid centroid data for {cid}")
# --- Final Layout & Status ---
title = f"3D t-SNE ({len(df)} points)"
if active_selection_id is not None:
title = f"Cluster {active_selection_id} ({len(df_to_plot)} points)"
elif is_search_active:
title = f"3D t-SNE - Search: '{search_term}'"
if active_selection_id and is_search_active: title += f" - Search: '{search_term}'"
base_layout = self._create_base_figure().layout
fig.update_layout(
title=title, legend_title_text='Legend', legend=base_layout.legend,
scene=base_layout.scene # Use base scene settings (includes aspectmode='data')
# Rely on auto-ranging (potentially helped by dummy points if added)
)
final_status = status_msg
if not final_status: # Default status
base = f"{len(df_to_plot)} points shown."
if active_selection_id: base = f"Cluster {active_selection_id}: {len(df_to_plot)} points."
final_status = base;
if is_search_active: final_status += f" (Search: '{search_term}')"
return fig, new_selected_cluster, final_status
def run(self, host: str = "127.0.0.1", port: int = 8050, debug: bool = False) -> None:
"""Starts the Dash server."""
logger.info(f"Starting Dash server on http://{host}:{port}")
try:
self.app.run(host=host, port=port, debug=debug)
except OSError as e:
logger.error(f"Server start failed: {e}. Port {port} busy?")
except Exception as e:
logger.exception(f"Server error: {e}")
# ──────────────────────────────────────────────────────────────
# >>> Helpers for automatic cluster‐count selection <<<
# ──────────────────────────────────────────────────────────────
def _estimate_optimal_clusters(self, embeddings: np.ndarray, max_k: int = 10) -> int:
"""
Estimate an optimal number of clusters using a quick elbow heuristic.
Computes K‑means inertia for k = 1…max_k and picks the k that is farthest
from the straight line connecting (1, inertia₁) and (max_k, inertiaₘₐₓ).
"""
if embeddings is None or embeddings.size == 0:
return 1
n_samples = embeddings.shape[0]
if n_samples < 3:
return 1
max_k = min(max_k, n_samples)
inertias = []
for k in range(1, max_k + 1):
km = KMeans(n_clusters=k, random_state=42, n_init="auto").fit(embeddings)
inertias.append(km.inertia_)
# distance from each point to the line between first and last
x = np.arange(1, max_k + 1)
x1, y1 = 1, inertias[0]
x2, y2 = max_k, inertias[-1]
numerator = np.abs((y2 - y1) * x - (x2 - x1) * np.array(inertias) + x2 * y1 - y2 * x1)
denominator = np.sqrt((y2 - y1) ** 2 + (x2 - x1) ** 2)
elbow_idx = int(np.argmax(numerator / denominator))
return elbow_idx + 1 # since k starts at 1
def _cluster_marks(self) -> Dict[int, str]:
"""Generate tick marks for the cluster-count slider."""
if self.max_clusters <= 15:
return {i: str(i) for i in range(1, self.max_clusters + 1)}
# Show first, optimal, and max for large data sets
return {1: "1", self.optimal_clusters: str(self.optimal_clusters), self.max_clusters: str(self.max_clusters)}
# --- END OF FILE visualizer.py ---
@@ -0,0 +1,2 @@
def hello() -> str:
return "Hello from librarian_vspace!"
Whitespace-only changes.
@@ -0,0 +1,93 @@
"""TsneExportWorker – Prefect worker that generates a t‑SNE JSON export.
It wraps Vspace.get_tnse → vecview.get_tsne_json, writes the JSON to a file,
stages it, and returns the file path.
Minimal Pydantic payload models are defined locally to avoid extra deps.
"""
from __future__ import annotations
import asyncio
import json
import logging
import tempfile
from pathlib import Path
from typing import Optional
from prefect import get_run_logger
from pydantic import BaseModel
from librarian_core.workers.base import Worker
from librarian_vspace.vecview.vecview import get_tsne_json
# ------------------------------------------------------------------ #
def _safe_get_logger(name: str):
try:
return get_run_logger()
except Exception:
return logging.getLogger(name)
# ------------------------------------------------------------------ #
# Pydantic payloads
# ------------------------------------------------------------------ #
class TsneExportInput(BaseModel):
course_id: int
limit: Optional[int] = None
perplexity: float = 30.0
db_schema: str = "librarian"
rpc_function: str = "pdf_chunking"
embed_model: str = "snowflake-arctic-embed2"
embedding_column: str = "embedding"
base_output_dir: Optional[Path] = None # where to place JSON file
class TsneExportOutput(BaseModel):
json_path: Path
# ------------------------------------------------------------------ #
class TsneExportWorker(Worker[TsneExportInput, TsneExportOutput]):
"""Runs the t‑SNE export inside a Prefect worker.""" # noqa: D401
input_model = TsneExportInput
output_model = TsneExportOutput
async def __run__(self, payload: TsneExportInput) -> TsneExportOutput:
logger = _safe_get_logger(self.worker_name)
logger.info("🔨 %s startet (payload=%r)", self.worker_name, payload)
# Run get_tsne_json in a thread
data_json = await asyncio.to_thread(
get_tsne_json,
db_schema=payload.db_schema,
db_function=payload.rpc_function,
model_name=payload.embed_model,
limit=payload.limit,
course_id=payload.course_id,
perplexity=payload.perplexity,
embedding_column=payload.embedding_column,
)
# Determine output file
if payload.base_output_dir:
out_dir = Path(payload.base_output_dir).expanduser()
out_dir.mkdir(parents=True, exist_ok=True)
json_path = out_dir / f"{payload.course_id}_tsne.json"
else:
tf = tempfile.NamedTemporaryFile(
mode="w+", suffix="_tsne.json", prefix="vspace_", delete=False
)
json_path = Path(tf.name)
# Write JSON to file
json_path.write_text(data_json, encoding="utf-8")
# Stage file for Prefect
self.stage(json_path, new_name=json_path.name)
result = TsneExportOutput(json_path=json_path)
logger.info("✅ %s fertig: %r", self.worker_name, result)
return result
@@ -0,0 +1,104 @@
"""Utility functions to fetch vectors from Supabase, apply t‑SNE, add simple K‑means
clustering and hover text – prepared exactly like the `VectorVisualizer` expects.
"""
from __future__ import annotations
import logging
from typing import Optional, List
import numpy as np
import pandas as pd
from sklearn.cluster import KMeans
from librarian_vspace.vutils.vector_query_loader import VectorQueryLoader, VectorQueryLoaderError
from librarian_vspace.models.tsne_model import TSNEPoint, TSNEData
logger = logging.getLogger(__name__)
DEFAULT_N_CLUSTERS = 8
# --------------------------------------------------------------------- #
# Internal helpers (kept minimal – no extra bells & whistles)
# --------------------------------------------------------------------- #
def _run_kmeans(df: pd.DataFrame, *, embedding_column: str, k: int = DEFAULT_N_CLUSTERS) -> pd.DataFrame:
"""Adds a 'cluster' column using K‑means (string labels)."""
if df.empty or embedding_column not in df.columns:
df['cluster'] = "-1"
return df
embeddings = np.array(df[embedding_column].tolist(), dtype=float)
n_samples = embeddings.shape[0]
k = max(1, min(k, n_samples)) # ensure 1 ≤ k ≤ n_samples
if n_samples < 2:
df['cluster'] = "0"
return df
km = KMeans(n_clusters=k, random_state=42, n_init='auto')
df['cluster'] = km.fit_predict(embeddings).astype(str)
return df
def _add_hover(df: pd.DataFrame) -> pd.DataFrame:
if df.empty:
return df
df = df.copy()
def _hover(row):
preview = str(row.get('chunk', ''))[:200]
if len(str(row.get('chunk', ''))) > 200:
preview += "..."
return (
f"ID: {row.get('file_id', 'N/A')}<br>"
f"Cluster: {row.get('cluster', 'N/A')}<br>"
f"Chunk: {preview}"
)
df['hover_text'] = df.apply(_hover, axis=1)
return df
# --------------------------------------------------------------------- #
# Public helpers
# --------------------------------------------------------------------- #
def get_tsne_dataframe(
db_schema: str,
db_function: str,
model_name: str,
*,
limit: Optional[int] = None,
course_id: Optional[int] = None,
perplexity: float = 30.0,
embedding_column: str = "embedding",
n_clusters: int = DEFAULT_N_CLUSTERS,
) -> pd.DataFrame:
"""Returns a pandas DataFrame with tsne (x,y,z) & metadata ready for plotting."""
loader = VectorQueryLoader(db_schema, db_function, model_name, embedding_column)
df = loader.load_and_reduce(
limit=limit,
course_id=course_id,
tsne_params={"perplexity": perplexity},
)
if df.empty:
return df
df = _run_kmeans(df, embedding_column=embedding_column, k=n_clusters)
df = _add_hover(df)
return df
def get_tsne_json(**kwargs) -> str:
"""Convenience wrapper returning DataFrame as JSON (orient='split')."""
df = get_tsne_dataframe(**kwargs)
return df.to_json(date_format='iso', orient='split')
def get_tsne_response(**kwargs) -> TSNEData:
"""Returns a validated `TSNEResponse` Pydantic model."""
df = get_tsne_dataframe(**kwargs)
points: List[TSNEPoint] = [TSNEPoint(**row.dropna().to_dict()) for _, row in df.iterrows()]
return TSNEData(course_id=kwargs.get('course_id'), total=len(points), points=points)
@@ -0,0 +1,10 @@
"""
vquery package for high-level read operations against vector tables.
"""
from .query import VectorQuery
__all__ = ["VectorQuery"] # Defines the public interface of the package
# Optional: Add package-level logging setup if desired, but often handled by the application
# import logging
# logging.getLogger(__name__).addHandler(logging.NullHandler())
@@ -0,0 +1,438 @@
# --- START OF FILE cluster_export.py (Refactored & Workaround - Import Updated) ---
"""
cluster_export.py – Generate IVFFlat‑equivalent clusters from Supabase/Vectorbase
pgvector data and export each cluster’s chunks to Markdown.
This version fetches vectors filtered by course ID at the database level using
VectorQueryLoader, performs k-means clustering, and exports to Markdown.
Includes automatic k-downsizing.
Environment variables (used by the script entry point)
---------------------
* **Vectorbase credentials** (auto‑mapped to Supabase):
* `VECTORBASE_URL` → `SUPABASE_URL`
* `VECTORBASE_API_KEY` → `SUPABASE_KEY`
* `VECTORBASE_USER_UUID` → `SUPABASE_USER_UUID` (optional)
* **Embedding/table config**
* `VECTOR_SCHEMA` – Postgres schema (default `librarian`)
* `VECTOR_FUNCTION` – RPC / Postgres function name (optional)
* `EMBED_MODEL` – embedding model label (default `snowflake-arctic-embed2`)
* **Clustering hyper‑parameters**
* `K` – requested number of clusters / IVFFlat *nlist* (default 128)
* `TRAIN_SAMPLE` – how many rows to feed into k‑means (default 20 000, but
capped at the table size)
* **Export**
* `OUTPUT_DIR` – directory for the generated Markdown files (default
`./cluster_md`)
* `CLUSTER_COURSE_ID` - Optional course ID to filter vectors (used by script)
Usage
~~~~~
# Via script entry point
export VECTORBASE_URL="https://xyz.vectorbase.co"
export VECTORBASE_API_KEY="service_role_key"
export VECTOR_SCHEMA=librarian
export EMBED_MODEL=snowflake-arctic-embed2
export CLUSTER_COURSE_ID=123 # Optional filtering
export K=64
python -m librarian_vspace.vquery.cluster_export
# As a callable function
from librarian_vspace.vquery.cluster_export import run_cluster_export_job
output_path = run_cluster_export_job(course_id=456, output_dir="/tmp/clusters_456", ...)
"""
from __future__ import annotations
import json
import logging
import os
import sys
from pathlib import Path
from typing import List, Optional, Dict, Any, Union # Added Union
import numpy as np
import pandas as pd
from sklearn.cluster import KMeans
from sklearn.metrics import pairwise_distances_argmin_min
# ---------------------------------------------------------------------------
# Map Vectorbase credential names → Supabase names expected by loader code
# ---------------------------------------------------------------------------
_ALIAS_ENV_MAP = {
"VECTORBASE_URL": "SUPABASE_URL",
"VECTORBASE_API_KEY": "SUPABASE_KEY",
"VECTORBASE_USER_UUID": "SUPABASE_USER_UUID", # optional
}
for src, dest in _ALIAS_ENV_MAP.items():
if dest not in os.environ and src in os.environ:
os.environ[dest] = os.environ[src]
# Import the NEW data‑loading helper with filtering capabilities
try:
# --- FIX: Import VectorQueryLoader from vutils ---
from librarian_vspace.vutils.vector_query_loader import VectorQueryLoader, VectorQueryLoaderError
# VectorLoaderError is now VectorQueryLoaderError
# --- END FIX ---
except ImportError as e:
# Keep the original script's error handling for standalone use
sys.stderr.write(
"\n[ERROR] Could not import VectorQueryLoader – check PYTHONPATH. "
f"Original error: {e}\n"
)
# For callable use, we should raise an ImportError or custom exception
raise ImportError(f"Could not import VectorQueryLoader: {e}") from e
# ---------------------------------------------------------------------------
# Logging setup (used by both script and callable function)
# ---------------------------------------------------------------------------
# This basicConfig runs when the module is imported.
# Callers might want to configure logging before importing.
# If logging is already configured, basicConfig does nothing.
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
logger = logging.getLogger(__name__) # Use __name__ for module-specific logger
# ---------------------------------------------------------------------------
# Helper – JSON dump for centroid in YAML front‑matter
# ---------------------------------------------------------------------------
def centroid_to_json(vec: np.ndarray) -> str:
"""Converts a numpy vector to a JSON string suitable for YAML frontmatter."""
return json.dumps([float(x) for x in vec], ensure_ascii=False)
# ---------------------------------------------------------------------------
# Main clustering and export logic as a callable function
# ---------------------------------------------------------------------------
def run_cluster_export_job(
course_id: Optional[int] = None, # Added course_id parameter
output_dir: Union[str, Path] = "./cluster_md", # Output directory parameter
schema: str = "librarian",
rpc_function: str = "pdf_chunking", # Default to actual function name
model: str = "snowflake-arctic-embed2",
k_clusters: int = 128, # Requested number of clusters (k)
train_sample_size: int = 20000, # Sample size for K-means training
embedding_column: str = "embedding" # Added embedding column parameter
) -> Path:
"""
Fetches vectors, performs K-means clustering, and exports clustered chunks to Markdown.
Args:
course_id: Optional ID to filter vectors belonging to a specific course.
output_dir: Directory path where the cluster Markdown files will be saved.
schema: Postgres schema containing the vector table.
rpc_function: Optional RPC function name used by VectorQueryLoader (needed for table lookup).
model: Embedding model label used by VectorQueryLoader (needed for table lookup).
k_clusters: The requested number of clusters (k). Will be downsized if fewer
vectors are available.
train_sample_size: The maximum number of vectors to use for K-means training.
Capped by the total number of vectors fetched.
embedding_column: The name of the column containing the vector embeddings.
Returns:
The absolute path to the output directory.
Raises:
VectorQueryLoaderError: If vector loading fails.
RuntimeError: If no embeddings are retrieved or training sample is empty after filtering.
Exception: For other errors during clustering or export.
"""
output_path = Path(output_dir).expanduser().resolve() # Resolve path early
output_path.mkdir(parents=True, exist_ok=True)
logger.info("Writing Markdown files to %s", output_path)
# ---------------------------------------------------------------------------
# Fetch embeddings - Now using VectorQueryLoader with filtering
# ---------------------------------------------------------------------------
try:
# Use parameters for loader config
# --- FIX: Instantiate VectorQueryLoader ---
loader = VectorQueryLoader(schema=schema, function=rpc_function, model=model, embedding_column=embedding_column)
# --- END FIX ---
# --- FIX: Call fetch_vectors WITH the course_id argument ---
# VectorQueryLoader.fetch_vectors handles the DB-level filtering
df = loader.fetch_vectors(limit=None, course_id=course_id)
# --- END FIX ---
# --- REMOVE: In-memory filtering logic is no longer needed ---
# initial_rows = len(df)
# if course_id is not None and not df.empty:
# ... (removed filtering code) ...
# elif course_id is not None and df.empty:
# ... (removed warning) ...
# --- END REMOVE ---
# --- FIX: Catch VectorQueryLoaderError ---
except VectorQueryLoaderError as e:
logger.error("Vector loading failed: %s", e)
raise e # Re-raise the specific exception for the caller
# --- END FIX ---
except Exception as e:
# Catch other unexpected errors during loading
logger.exception("An unexpected error occurred during vector loading.")
raise RuntimeError(f"An unexpected error occurred during vector loading: {e}") from e
# --- Check if DataFrame is empty *after* fetching (which includes DB filtering) ---
if df.empty:
logger.error("No embeddings retrieved or found for course_id %s – aborting.", course_id)
# Raise a RuntimeError as no clustering can be done
raise RuntimeError(f"No embeddings retrieved or found for course_id {course_id} – nothing to cluster.")
# ----------------------------------------------------------------------------------
# Use the actual embedding column name from the loader instance
# This check is crucial *after* fetching
if not hasattr(loader, 'embedding_column') or loader.embedding_column not in df.columns:
# This should ideally be caught by VectorQueryLoader's internal checks, but double-check
logger.error("Embedding column '%s' not found in fetched data.", embedding_column) # Use the input param name for error msg
raise RuntimeError(f"Embedding column '{embedding_column}' not found in fetched data.")
# --- Ensure embeddings are numeric lists before stacking ---
# The VectorQueryLoader.fetch_vectors method now handles parsing and dropping invalid rows.
# We just need to safely stack the potentially filtered/cleaned data.
try:
# Ensure data is list of floats before stacking
# This check might be redundant if VectorQueryLoader guarantees cleaned data,
# but it adds safety.
if not all(isinstance(x, list) and all(isinstance(n, float) for n in x) for x in df[embedding_column]):
logger.error(f"Data in '{embedding_column}' is not strictly list[float] format after fetching. Attempting conversion.")
# This might catch issues the loader missed or unexpected data structures
try:
# Attempt robust conversion similar to the loader's parse method
embeddings_list = []
for item in df[embedding_column]:
parsed_item = None
if isinstance(item, str):
try: parsed_item = json.loads(item)
except json.JSONDecodeError: pass
elif isinstance(item, (list, tuple, np.ndarray)):
parsed_item = item
elif isinstance(item, dict) and 'vector' in item and isinstance(item['vector'], (list, tuple, np.ndarray)):
parsed_item = item['vector']
if isinstance(parsed_item, (list, tuple, np.ndarray)) and all(isinstance(val, (int, float, np.number)) for val in parsed_item):
embeddings_list.append([float(n) for n in parsed_item])
else:
logger.debug(f"Skipping problematic embedding during secondary clean: {str(item)[:100]}...")
if not embeddings_list:
logger.error("No valid embeddings remained after secondary cleaning.")
raise ValueError("No valid embeddings for stacking.")
embeddings = np.array(embeddings_list, dtype=float)
logger.warning("Successfully converted problematic embedding data for stacking.")
except Exception as e:
logger.exception(f"Failed secondary attempt to convert embeddings for stacking: {e}")
raise RuntimeError(f"Failed to process embedding data for stacking: {e}") from e
else:
# Data is in the expected list of float format, proceed directly
embeddings = np.stack(df[embedding_column].to_list()).astype(float)
logger.info("Prepared %d embeddings for clustering.", embeddings.shape[0])
except ValueError as ve:
logger.exception(f"Failed to stack embeddings into a numpy array: {ve}. Ensure '{embedding_column}' contains valid vector data.")
raise RuntimeError(f"Failed to process embedding data: {ve}") from ve
except Exception as e:
logger.exception(f"An unexpected error occurred while processing '{embedding_column}' column for stacking.")
raise RuntimeError(f"An unexpected error occurred while processing embedding data for stacking: {e}") from e
# -------------------------------------------------------------
# ---------------------------------------------------------------------------
# Prepare training sample and determine effective k
# ---------------------------------------------------------------------------
# Use the parameter train_sample_size
train_vecs = embeddings[:train_sample_size]
if train_vecs.shape[0] == 0:
# If course_id filtering resulted in 0 vectors, this check prevents the crash
# but the df.empty check earlier should already handle this.
# Keep this check for robustness in case train_sample_size is 0 or negative.
logger.error("Training sample is empty – nothing to cluster.")
raise RuntimeError("Training sample is empty – nothing to cluster.")
# Use the parameter k_clusters
K = min(k_clusters, train_vecs.shape[0])
if K < k_clusters:
logger.warning(
"Requested k=%d but only %d training vectors available; "
"using k=%d.",
k_clusters,
train_vecs.shape[0],
K,
)
# Ensure K is at least 1 if there's any data
if K == 0 and train_vecs.shape[0] > 0:
K = 1
logger.warning("Adjusted k to 1 as requested k resulted in 0 but data exists.")
if K == 0:
# If after adjustments K is still 0 (meaning train_vecs.shape[0] was 0)
logger.error("Effective k is 0. Cannot train k-means.")
raise RuntimeError("Effective k is 0. Cannot train k-means.")
logger.info("Training k‑means (k=%d) on %d vectors", K, train_vecs.shape[0])
try:
kmeans = KMeans(
n_clusters=K,
init="k-means++",
n_init="auto", # Use 'auto' for better handling of small k/n_samples
algorithm="lloyd", # 'lloyd' is the standard, 'elkan' can be faster but has limitations
max_iter=300,
random_state=0,
)
kmeans.fit(train_vecs)
centroids: np.ndarray = kmeans.cluster_centers_
logger.info("K‑means converged in %d iterations", kmeans.n_iter_)
except Exception as e:
logger.exception("K-means clustering failed.")
raise RuntimeError(f"K-means clustering failed: {e}") from e
# ---------------------------------------------------------------------------
# Assign every vector to its nearest centroid (full table)
# ---------------------------------------------------------------------------
logger.info("Assigning vectors to centroids...")
try:
# Use the determined embedding column for assignment as well
labels_full, _ = pairwise_distances_argmin_min(embeddings, centroids, metric="euclidean")
df["cluster_id"] = labels_full
logger.info("Assigned cluster labels to all embeddings.")
except Exception as e:
logger.exception("Failed to assign vectors to centroids.")
raise RuntimeError(f"Failed to assign vectors to centroids: {e}") from e
# ---------------------------------------------------------------------------
# Write one Markdown file per cluster
# ---------------------------------------------------------------------------
files_written_count = 0
logger.info("Writing cluster Markdown files to %s", output_path)
try:
# Only iterate up to the number of actual clusters found by KMeans
# KMeans might return fewer clusters than K if there are issues or identical points
num_actual_clusters = len(centroids)
if num_actual_clusters < K:
logger.warning(f"KMeans returned only {num_actual_clusters} centroids, expected {K}. Iterating over actual centroids.")
for cid in range(num_actual_clusters): # Iterate over actual cluster IDs
# Find all data points assigned to this cluster ID
subset = df[df.cluster_id == cid]
# Ensure centroid_vec corresponds to the centroid of the *current* cluster ID (cid)
# This check is more robust now iterating up to num_actual_clusters
if cid < len(centroids):
centroid_vec = centroids[cid]
else:
# This case should theoretically not be reached with the loop range
logger.error(f"Centroid for cluster ID {cid} missing! Using zero vector.")
centroid_vec = np.zeros(embeddings.shape[1])
# Use .get() and .fillna("") defensively in case 'chunk' column is missing
# Ensure chunk column exists - it should if SELECT * worked
if 'chunk' not in subset.columns:
logger.warning("'chunk' column missing in subset data for cluster %d. Using empty strings.", cid)
chunks = [""] * len(subset)
else:
chunks = subset['chunk'].fillna("").tolist()
md_lines = [
#"---",
#f"cluster_id: {cid}",
#f"centroid: {centroid_to_json(centroid_vec)}",
#"---\n", # Separator between frontmatter and content
]
# Add chunks, ensuring each chunk is on a new line or separated by blank lines
md_lines.extend(chunks)
outfile = output_path / f"cluster_{cid:03d}.md"
# Use a different separator for chunks within the file if needed,
# currently just joins with newline, but chunks might contain newlines.
# Joining with "\n\n" provides separation *between* chunks.
try:
outfile.write_text("\n\n".join(md_lines), encoding="utf-8")
files_written_count += 1
logger.debug("Wrote %s (%d chunks)", outfile.name, len(chunks)) # Use debug for per-file
except Exception as write_exc:
logger.error(f"Failed to write cluster file {outfile}: {write_exc}", exc_info=True)
# Decide whether to continue or raise here. Continuing allows other clusters to be saved.
# For robustness in script, maybe continue. For library function, maybe raise.
# For now, we'll just log and continue.
except Exception as e:
logger.exception("Failed during Markdown file writing loop.")
raise RuntimeError(f"Failed during Markdown file writing: {e}") from e
logger.info("Done. %d Markdown files created in %s", files_written_count, output_path)
return output_path
# ---------------------------------------------------------------------------
# Script entry point
# ---------------------------------------------------------------------------
if __name__ == "__main__":
# Configuration via environment for script
script_output_dir = Path(os.environ.get("OUTPUT_DIR", "./cluster_md")).expanduser()
script_schema = os.environ.get("VECTOR_SCHEMA", "librarian")
script_rpc_function = os.environ.get("VECTOR_FUNCTION", "pdf_chunking") # Default to actual function name
script_model = os.environ.get("EMBED_MODEL", "snowflake-arctic-embed2")
script_k_req = int(os.environ.get("K", "128"))
script_train_sample = int(os.environ.get("TRAIN_SAMPLE", "20000"))
# Added course ID specific to script entry point
script_course_id_str = os.environ.get("CLUSTER_COURSE_ID")
script_course_id = int(script_course_id_str) if script_course_id_str and script_course_id_str.isdigit() else None # Added isdigit check
# Configure basic logging for the script entry point
# (The module-level config above might not run if imported in specific ways)
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s: %(message)s",
datefmt="%Y-%m-%d %H:%M:%S",
)
# Re-get the logger after basicConfig to ensure it's configured
logger = logging.getLogger(__name__)
try:
logger.info("Starting cluster export script...")
final_output_path = run_cluster_export_job(
course_id=script_course_id,
output_dir=script_output_dir,
schema=script_schema,
rpc_function=script_rpc_function,
model=script_model,
k_clusters=script_k_req,
train_sample_size=script_train_sample,
# embedding_column defaults to 'embedding' in the function
)
logger.info("Script finished successfully. Output in %s", final_output_path)
sys.exit(0) # Explicit success exit
# --- FIX: Catch VectorQueryLoaderError ---
except (VectorQueryLoaderError, RuntimeError) as e: # Catch the new error type
# --- END FIX ---
# Specific errors we raised
logger.error("Script failed: %s", e)
sys.exit(1) # Indicate failure
except Exception as e:
# Catch any other unexpected errors
logger.exception("An unhandled error occurred during script execution.")
sys.exit(1) # Indicate failure
@@ -0,0 +1,73 @@
"""ClusterExportWorker – Prefect worker that wraps run_cluster_export_job."""
from __future__ import annotations
import asyncio
import logging
import tempfile
from pathlib import Path
from typing import Optional
from prefect import get_run_logger
from pydantic import BaseModel
from librarian_core.workers.base import Worker
from librarian_vspace.vquery.cluster_export import run_cluster_export_job
def _safe_get_logger(name: str):
try:
return get_run_logger()
except Exception:
return logging.getLogger(name)
class ClusterExportInput(BaseModel):
course_id: int
k_clusters: int = 128
train_sample_size: int = 20_000
db_schema: str = "librarian"
rpc_function: str = "pdf_chunking"
model: str = "snowflake-arctic-embed2"
embedding_column: str = "embedding"
base_output_dir: Optional[Path] = None
class ClusterExportOutput(BaseModel):
output_dir: Path
class ClusterExportWorker(Worker[ClusterExportInput, ClusterExportOutput]):
input_model = ClusterExportInput
output_model = ClusterExportOutput
async def __run__(self, payload: ClusterExportInput) -> ClusterExportOutput:
logger = _safe_get_logger(self.worker_name)
logger.info("🔨 %s startet (payload=%r)", self.worker_name, payload)
# Prepare output directory
if payload.base_output_dir:
base_dir = Path(payload.base_output_dir).expanduser()
base_dir.mkdir(parents=True, exist_ok=True)
tmp_base = tempfile.mkdtemp(dir=base_dir)
else:
tmp_base = tempfile.mkdtemp()
output_dir = Path(tmp_base) / str(payload.course_id)
output_dir.mkdir(parents=True, exist_ok=True)
logger.debug("Output directory: %s", output_dir)
final_dir = await asyncio.to_thread(
run_cluster_export_job,
course_id=payload.course_id,
output_dir=output_dir,
schema=payload.db_schema,
rpc_function=payload.rpc_function,
model=payload.model,
k_clusters=payload.k_clusters,
train_sample_size=payload.train_sample_size,
embedding_column=payload.embedding_column,
)
self.stage(final_dir, new_name=final_dir.name)
result = ClusterExportOutput(output_dir=final_dir)
logger.info("✅ %s fertig: %r", self.worker_name, result)
return result
Whitespace-only changes.
@@ -0,0 +1,127 @@
"""VectorQuery – helper for vector searches against chunklet tables.
This module provides:
* A Pydantic‑powered request / response API (see ``librarian_vspace.models.query_model``).
* A single public method :py:meth:`VectorQuery.search` that returns a
:class:`~librarian_vspace.models.query_model.VectorSearchResponse`.
* A thin legacy wrapper ``get_chucklets_by_vector`` that produces the
historical ``List[Dict[str, Any]]`` format, built on top of ``search``.
"""
from __future__ import annotations
import logging
from typing import Any, Dict, List, Optional
try:
from librarian_vspace.vutils.vector_class import BaseVectorOperator
from librarian_vspace.vecembed.embedding_generator import EmbeddingGenerator
except ImportError as exc: # pragma: no cover
logging.error(
"Failed to import vutils or vecembed sub‑packages: %s. " "Ensure they are on PYTHONPATH.", exc
)
class BaseVectorOperator: # type: ignore
"""Minimal stub if real class is unavailable (runtime error later)."""
class EmbeddingGenerator: # type: ignore
"""Minimal stub; will raise at runtime if used."""
from librarian_vspace.models.query_model import (
VectorSearchRequest,
VectorSearchResponse,
Chunklet,
)
logger = logging.getLogger(__name__)
# --------------------------------------------------------------------- #
# Main helper
# --------------------------------------------------------------------- #
class VectorQuery(BaseVectorOperator):
"""High‑level helper for vector searches via Supabase RPC."""
# -----------------------------------------------------------------
# Public – modern API
# -----------------------------------------------------------------
def search(self, request: VectorSearchRequest) -> VectorSearchResponse:
"""Perform a similarity search and return structured results."""
if not getattr(self, "table", None):
logger.error("VectorQuery: target table not determined (self.table is None).")
return VectorSearchResponse(total=0, results=[])
# 1) Generate query embedding
try:
_tts, query_vec, _ = EmbeddingGenerator().generate_embedding(
interface_name=request.interface_name,
model_name=request.model_name,
text_to_embed=request.search_string,
identifier="query",
)
if query_vec is None:
logger.error("Embedding generation returned None.")
return VectorSearchResponse(total=0, results=[])
except Exception as exc: # pragma: no cover
logger.exception("Embedding generation failed: %s", exc)
return VectorSearchResponse(total=0, results=[])
# 2) Build RPC parameters
rpc_params = {
"p_query_embedding": query_vec,
"p_target_table": self.table,
"p_embedding_column": request.embedding_column,
"p_match_count": request.top_k,
"p_filters": request.filters or {},
}
# 3) Execute RPC
try:
if not getattr(self, "spc", None):
logger.error("Supabase client (self.spc) not available.")
return VectorSearchResponse(total=0, results=[])
resp = (
self.spc
.schema(self.schema)
.rpc("vector_search", rpc_params)
.execute()
)
data = resp.data or []
results = [
Chunklet(chunk=row.get("chunk"), file_id=row.get("file_id")) if isinstance(row.get("file_id"), str) else Chunklet(chunk=row.get("chunk"), file_id=str(row.get("file_id")))
for row in data
]
return VectorSearchResponse(total=len(results), results=results)
except Exception as exc: # pragma: no cover
logger.exception("RPC 'vector_search' failed: %s", exc)
return VectorSearchResponse(total=0, results=[])
# -----------------------------------------------------------------
# Public – legacy compatibility
# -----------------------------------------------------------------
def get_chucklets_by_vector(
self,
*,
interface_name: str,
model_name: str,
search_string: str,
filters: Optional[Dict[str, Any]] = None,
top_k: int = 10,
embedding_column: str = "embedding",
) -> List[Dict[str, Any]]:
"""Backward‑compatible wrapper returning ``{'chunk', 'file_id'}`` dicts."""
req = VectorSearchRequest(
interface_name=interface_name,
model_name=model_name,
search_string=search_string,
filters=filters,
top_k=top_k,
embedding_column=embedding_column,
)
resp = self.search(req)
return [ck.dict() for ck in resp.results]
@@ -0,0 +1,62 @@
"""QueryWorker – Prefect worker that performs a vector search.
It instantiates VectorQuery directly (no vspace dependency) and returns the
VectorSearchResponse.
"""
from __future__ import annotations
import asyncio
import logging
from pathlib import Path
from prefect import get_run_logger
from pydantic import BaseModel
from librarian_core.workers.base import Worker
from librarian_vspace.vquery.query import VectorQuery
from librarian_vspace.models.query_model import VectorSearchRequest, VectorSearchResponse
def _safe_get_logger(name: str):
try:
return get_run_logger()
except Exception:
return logging.getLogger(name)
class QueryInput(BaseModel):
request: VectorSearchRequest
db_schema: str = "librarian"
rpc_function: str = "pdf_chunking"
embed_model: str = "snowflake-arctic-embed2"
embedding_column: str = "embedding"
class QueryWorker(Worker[QueryInput, VectorSearchResponse]):
"""Runs a Supabase vector search via VectorQuery."""
input_model = QueryInput
output_model = VectorSearchResponse
async def __run__(self, payload: QueryInput) -> VectorSearchResponse:
logger = _safe_get_logger(self.worker_name)
logger.info("🔨 %s startet (payload=%r)", self.worker_name, payload)
def _do_search() -> VectorSearchResponse:
try:
vq = VectorQuery(
schema=payload.db_schema,
function=payload.rpc_function,
model=payload.embed_model,
embedding_column=payload.embedding_column,
)
except TypeError:
# fallback to positional signature
vq = VectorQuery(payload.db_schema, payload.rpc_function, payload.embed_model)
return vq.search(payload.request)
response = await asyncio.to_thread(_do_search)
logger.info("✅ %s fertig: %s results", self.worker_name, response.total)
return response
@@ -0,0 +1,2 @@
def hello() -> str:
return "Hello from librarian_vspace!"
@@ -0,0 +1,248 @@
#!/usr/bin/env python3
"""parallelism_advisor.py
=====================================
Cross‑platform helper to inspect the hardware parallelism available on the
current machine *and* to give rule‑of‑thumb recommendations for the number of
workers you should spin up for CPU‑bound, IO‑bound, or GPU workloads.
Highlights
----------
* Works on Linux, macOS (Intel **and** Apple Silicon) and Windows.
* Respects Docker / Kubernetes CPU quotas (cgroup v1 & v2) automatically.
* Falls back gracefully when optional dependencies (``psutil``, ``torch``)
are missing.
* Caches expensive hardware queries.
* Tiny CLI: ``python -m parallelism_advisor --json``
MIT‑licensed. Drop the file somewhere on ``$PYTHONPATH`` or
``pip install .`` if you convert it into a proper package.
Usage
-----
>>> from librarian_vspace.vutils.parallelism_advisor import recommended_workers
>>> recommended_workers(kind="cpu")
8
>>> recommended_workers(kind="io")
64
"""
from __future__ import annotations
import argparse
import functools
import json
import os
import platform
import sys
from pathlib import Path
from typing import Any, Dict, Literal, Optional
# --------------------------------------------------------------------------- #
# Optional dependencies #
# --------------------------------------------------------------------------- #
try:
import psutil # type: ignore
except ModuleNotFoundError: # pragma: no cover
psutil = None # type: ignore
try:
import torch # type: ignore
except ModuleNotFoundError: # pragma: no cover
torch = None # type: ignore
# --------------------------------------------------------------------------- #
# Hardware discovery helpers #
# --------------------------------------------------------------------------- #
@functools.lru_cache(maxsize=None)
def logical_cores() -> int:
"""Return the number of *logical* CPUs visible to the process.
Falls back to 1 if Python or the OS cannot determine the value.
"""
return os.cpu_count() or 1
@functools.lru_cache(maxsize=None)
def physical_cores() -> int:
"""Return the number of *physical* CPU cores (no SMT).
If ``psutil`` is not installed or fails to determine the value, the function
falls back to ``logical_cores()``.
"""
if psutil is not None:
pc = psutil.cpu_count(logical=False)
if pc: # 0 or None means "could not determine"
return pc
# Heuristic fallback
return logical_cores()
def _read_first_line(path: Path) -> Optional[str]:
try:
return path.read_text().strip()
except FileNotFoundError:
return None
except Exception: # pragma: no cover
return None
@functools.lru_cache(maxsize=None)
def cgroup_cpu_limit() -> Optional[int]:
"""Return Docker/Kubernetes CPU quota if present (else ``None``).
Supports both cgroup v1 and v2. *Unit tests mock this function.*
"""
# --- cgroup v2 ---------------------------------------------------------
max_path = Path("/sys/fs/cgroup/cpu.max")
line = _read_first_line(max_path)
if line and " " in line:
quota, period = line.split()[:2]
try:
quota_i = int(quota)
period_i = int(period)
if quota_i > 0 and period_i > 0:
return max(1, quota_i // period_i)
except ValueError: # pragma: no cover
pass
# --- cgroup v1 ---------------------------------------------------------
quota_path = Path("/sys/fs/cgroup/cpu/cpu.cfs_quota_us")
period_path = Path("/sys/fs/cgroup/cpu/cpu.cfs_period_us")
quota_str = _read_first_line(quota_path)
period_str = _read_first_line(period_path)
if quota_str and period_str:
try:
quota_i = int(quota_str)
period_i = int(period_str)
if quota_i > 0 and period_i > 0:
return max(1, quota_i // period_i)
except ValueError: # pragma: no cover
pass
return None
# --------------------------------------------------------------------------- #
# GPU helpers (CUDA only for now) #
# --------------------------------------------------------------------------- #
@functools.lru_cache(maxsize=None)
def gpu_info() -> Optional[Dict[str, Any]]:
"""Return basic info for the first CUDA device via *torch* (or ``None``)."""
if torch is None or not torch.cuda.is_available():
return None
try:
device = torch.cuda.get_device_properties(0)
except Exception: # pragma: no cover
return None
return {
"device_name": device.name,
"sm_count": device.multi_processor_count,
"max_threads_per_sm": device.max_threads_per_multi_processor,
"total_memory_mb": device.total_memory // (1024 ** 2),
}
# --------------------------------------------------------------------------- #
# Recommendation logic #
# --------------------------------------------------------------------------- #
def recommended_workers(
*,
kind: Literal["cpu", "io", "gpu"] = "cpu",
io_factor: int = 4,
io_max: Optional[int] = 512,
) -> int:
"""Return a *rule‑of‑thumb* worker count for the given workload *kind*.
Parameters
----------
kind:
"cpu": CPU‑bound pure‑Python or C‑extension code (GIL matters)
"io" : Network / disk I/O where threads mostly block waiting
"gpu": GPU kernels (CUDA only, requires *torch*)
io_factor:
Multiplier for IO‑bound worker calculation (defaults to ``4``).
io_max:
Hard cap for IO‑bound workers (``None`` = no cap). Defaults to 512.
The function always respects container CPU quotas when present.
"""
if kind == "cpu":
base = physical_cores()
elif kind == "io":
base = logical_cores() * io_factor
if io_max is not None:
base = min(base, io_max)
elif kind == "gpu":
info = gpu_info()
if info is None:
return 0
# A very rough heuristic: one worker per SM
base = info["sm_count"]
else: # pragma: no cover
raise ValueError("kind must be 'cpu', 'io', or 'gpu'") from None
quota = cgroup_cpu_limit()
if quota is not None and kind != "gpu":
return max(1, min(base, quota))
return max(1, base)
# --------------------------------------------------------------------------- #
# Convenience snapshot of system info #
# --------------------------------------------------------------------------- #
def system_snapshot() -> Dict[str, Any]:
"""Return a JSON‑serialisable snapshot of parallelism‑related facts."""
return {
"platform_system": platform.system(),
"platform_machine": platform.machine(),
"logical_cores": logical_cores(),
"physical_cores": physical_cores(),
"docker_cpu_quota": cgroup_cpu_limit(),
"gpu": gpu_info(),
}
# --------------------------------------------------------------------------- #
# CLI #
# --------------------------------------------------------------------------- #
def _cli() -> None: # pragma: no cover
parser = argparse.ArgumentParser(
prog="parallelism_advisor", description="Rule‑of‑thumb worker estimator"
)
parser.add_argument(
"--json",
action="store_true",
help="Print full hardware snapshot as JSON (default: pretty table)",
)
parser.add_argument(
"--kind",
choices=["cpu", "io", "gpu"],
default="cpu",
help="Workload kind to estimate workers for (default: cpu)",
)
args = parser.parse_args()
if args.json:
print(json.dumps(system_snapshot(), indent=2))
else:
snap = system_snapshot()
workers = recommended_workers(kind=args.kind)
print("✦ System snapshot:")
for k, v in snap.items():
print(f" {k:20}: {v}")
print(f"\nRecommended {args.kind} workers: {workers}")
if __name__ == "__main__": # pragma: no cover
_cli()
Whitespace-only changes.
@@ -0,0 +1,50 @@
"""Singleton wrapper around the Supabase Python client.
Lives in **vutils** because it is pure infrastructure that other
packages depend on (vecembed, vecmap, vquery, …). Nothing outside
vutils should create its own Supabase client – always import
:pyclass:`MySupabase` and call :pymeth:`MySupabase.get_client`.
"""
from __future__ import annotations
import logging
import os
from typing import Optional
from supabase import Client, create_client
try:
from .config_utils import load_dotenv_from_root # type: ignore
load_dotenv_from_root()
except ModuleNotFoundError:
pass
logger = logging.getLogger(__name__)
class MySupabase:
"""A tiny singleton around :class:`supabase.Client`."""
_instance: Optional[Client] = None
@classmethod
def _create_client(cls) -> Client:
url = os.getenv("VECTORBASE_URL")
key = os.getenv("VECTORBASE_API_KEY") or os.getenv("SUPABASE_SERVICE_ROLE_KEY")
if not url or not key:
logger.error("VECTORBASE_URL / VECTORBASE_API_KEY missing – check your env")
raise ValueError("Supabase credentials not configured")
logger.info("Creating Supabase client …")
return create_client(url, key)
@classmethod
def get_client(cls) -> Client:
"""Return a singleton client instance."""
if cls._instance is None:
cls._instance = cls._create_client()
return cls._instance
# convenience for ``MySupabase()``
def __new__(cls) -> Client: # pragma: no cover
return cls.get_client()
@@ -0,0 +1,65 @@
"""Core utilities for resolving (schema, function, model) → table mapping."""
from __future__ import annotations
import logging
from typing import Optional
from postgrest import APIResponse # type: ignore
from librarian_vspace.vutils.supabase_singleton import MySupabase
logger = logging.getLogger(__name__)
class BaseVectorOperator:
"""Resolve mapping and expose *table_fqn()* for children."""
def __init__(self, schema: str, function: str, model: str):
self.spc = MySupabase.get_client()
self.schema = schema
self.function = function
self.model = model
self.model_id: Optional[int] = None
self.table: Optional[str] = None
self._resolve_ids()
# ---------------- public helpers ----------------
def table_fqn(self) -> str:
if not self.table:
raise RuntimeError("VectorOperator not initialised – no table")
return f"{self.schema}.{self.table}"
# ---------------- internals ----------------
def _resolve_ids(self) -> None:
self.model_id = self._rpc_get_model_id(self.model)
if self.model_id is None:
raise ValueError(f"Unknown model: {self.model}")
self.table = self._rpc_get_table(self.function, self.model_id)
if not self.table:
raise ValueError(f"Could not map (function={self.function}, model_id={self.model_id}) → table")
logger.info("VectorOperator ready → %s", self.table_fqn())
def _rpc_get_model_id(self, model: str) -> Optional[int]:
try:
resp: APIResponse = self.spc.schema(self.schema).rpc(
"get_model_id_by_model", {"p_model": model}
).execute()
logger.debug("get_model_id_by_model → %s", resp.data)
return int(resp.data) if resp.data not in (None, "") else None
except Exception: # noqa: BLE001
logger.exception("RPC get_model_id_by_model failed for %s", model)
return None
def _rpc_get_table(self, function: str, model_id: int) -> Optional[str]:
try:
resp: APIResponse = self.spc.schema(self.schema).rpc(
"get_table_by_model", {"p_function": function, "p_model_library_id": model_id}
).execute()
logger.debug("get_table_by_model → %s", resp.data)
if isinstance(resp.data, str):
return resp.data.strip() or None
return None
except Exception: # noqa: BLE001
logger.exception("RPC get_table_by_model failed")
return None
@@ -0,0 +1,413 @@
# --- START OF FILE librarian_vspace/vutils/vector_query_loader.py (Fixed Again) ---
"""Loads vectors from Supabase, with filtering capabilities, and handles parsing.
Intended for querying/clustering tasks where filtering is needed.
"""
from __future__ import annotations
import logging
import json
from typing import Any, Dict, List, Optional, Tuple, Union # Added Union
import numpy as np
import pandas as pd
# You might not need TSNE here if this class is only for fetching,
# but keeping it for generality as requested.
from sklearn.manifold import TSNE
# Assuming vutils is installed or in the python path
try:
from librarian_vspace.vutils.vector_class import BaseVectorOperator
except ImportError as e:
logging.error(f"Failed to import vutils: {e}. Ensure vutils package is installed.")
raise
logger = logging.getLogger(__name__)
# Define a specific error for this loader
class VectorQueryLoaderError(Exception):
"""Custom exception for VectorQueryLoader errors."""
pass
class VectorQueryLoader:
"""Fetches vectors with filtering capabilities."""
# --- Define the constant for the course ID column name ---
COURSE_ID_COLUMN_NAME = "course_id"
# ----------------------------------------------------------
# Keep TSNE defaults for generality, though perhaps not used by clustering logic
DEFAULT_TSNE_PARAMS = {
"n_components": 3,
"perplexity": 30.0, # Adjust based on dataset size (5-50 typically)
"n_iter": 1000, # Minimum recommended iterations
"learning_rate": "auto", # Usually a good default
"init": "pca", # PCA initialization is often faster and more stable
"random_state": 42, # For reproducibility
"n_jobs": -1, # Use all available CPU cores
"verbose": 1, # Log progress (controls scikit-learn's verbosity)
}
def __init__(self, schema: str, function: str, model: str, embedding_column: str = "embedding"):
"""
Initializes the loader.
(Constructor remains the same)
"""
logger.info(f"Initializing VectorQueryLoader for {schema=}, {function=}, {model=}")
try:
self.operator = BaseVectorOperator(schema=schema, function=function, model=model)
self.embedding_column = embedding_column
if not self.operator.table:
raise VectorQueryLoaderError("BaseVectorOperator failed to resolve table.")
logger.info(f"Target table resolved to: {self.operator.table_fqn()}")
except (ImportError, ValueError, RuntimeError) as e:
logger.exception("Failed to initialize BaseVectorOperator.")
# Raise the specific error for this loader
raise VectorQueryLoaderError(f"Failed to initialize BaseVectorOperator: {e}") from e
def _parse_vector_string(self, vector_str: Any) -> Optional[List[float]]:
"""Safely parses the string representation of a vector into a list of floats."""
if not isinstance(vector_str, str):
# If it's already a list, tuple, or numpy array, assume it's valid vector data
if isinstance(vector_str, (list, tuple, np.ndarray)) and all(isinstance(n, (int, float, np.number)) for n in vector_str):
return [float(n) for n in vector_str] # Ensure elements are floats
# Handle potential dict representation of vector (e.g., {"vector": [...]})
if isinstance(vector_str, dict) and 'vector' in vector_str and isinstance(vector_str['vector'], (list, tuple, np.ndarray)):
if all(isinstance(n, (int, float, np.number)) for n in vector_str['vector']):
return [float(n) for n in vector_str['vector']]
logger.debug(f"Unexpected type for vector parsing: {type(vector_str)}. Value starts with: {str(vector_str)[:50]}. Skipping.")
return None
try:
# Use json.loads which correctly handles [...] syntax from pgvector/Supabase
parsed_list = json.loads(vector_str)
if isinstance(parsed_list, list) and all(isinstance(n, (int, float)) for n in parsed_list):
return [float(n) for n in parsed_list] # Ensure elements are floats
else:
logger.warning(f"Parsed vector string '{vector_str[:50]}...' but result is not a list of numbers. Result type: {type(parsed_list)}")
return None
except json.JSONDecodeError:
logger.warning(f"Failed to JSON decode vector string: '{vector_str[:50]}...'")
return None
except Exception as e:
logger.error(f"Unexpected error parsing vector string '{vector_str[:50]}...': {e}", exc_info=True)
return None
def fetch_vectors(self, limit: Optional[int] = None, course_id: Optional[int] = None) -> pd.DataFrame:
"""
Fetches vectors and metadata from the resolved table, optionally filtered by course_id.
Parses string representations of vectors into lists.
Args:
limit: Optional limit on the number of rows to fetch (for large tables).
course_id: Optional ID to filter vectors by the COURSE_ID_COLUMN_NAME.
Returns:
A pandas DataFrame with columns like 'file_id', 'chunk', 'course_id',
and the embedding_column (as list). Includes all columns found if not specified.
Raises:
VectorQueryLoaderError: If fetching fails or no data is found after filtering.
"""
if not self.operator.table or not self.operator.schema:
raise VectorQueryLoaderError("Operator not initialized, table name or schema is unknown.")
table_name = self.operator.table
schema_name = self.operator.schema
# Select all columns (*) to ensure we get 'course_id' and 'chunk' if they exist
select_columns = "*" # Changed from fixed columns
logger.info(f"Fetching data from {schema_name}.{table_name} (selecting: {select_columns})...")
try:
query = self.operator.spc.schema(schema_name).table(table_name).select(select_columns)
# --- Add filtering logic using the constant ---
if course_id is not None:
logger.info(f"Filtering by {self.COURSE_ID_COLUMN_NAME}: {course_id}")
# This will raise a Supabase API error if the column doesn't exist, which is caught below
query = query.eq(self.COURSE_ID_COLUMN_NAME, course_id)
# -----------------------------------------------------------------
if limit:
logger.info(f"Applying limit: {limit}")
query = query.limit(limit)
# Execute the query
# --- REMOVE THIS LINE: query.url does not exist ---
# logger.debug(f"Executing query: {query.url}") # Log the actual URL for debugging
# --- END REMOVE ---
response = query.execute()
if not response.data:
logger.warning(f"No data found in table {self.operator.table_fqn()} matching criteria (course_id={course_id}).")
# Return empty DataFrame with expected key columns for consistency
# Use the columns from the initial DataFrame if available, otherwise assume minimal
cols_if_empty = ['file_id', 'chunk', self.COURSE_ID_COLUMN_NAME, self.embedding_column]
# Try to infer column dtypes from potential previous successful fetches or assume basic types
return pd.DataFrame(columns=cols_if_empty) # Simple approach for empty case
logger.info(f"Fetched {len(response.data)} rows.")
df = pd.DataFrame(response.data)
# --- Parse the embedding string into a list ---
logger.info(f"Parsing string representation in '{self.embedding_column}' column...")
if self.embedding_column not in df.columns:
logger.warning(f"Embedding column '{self.embedding_column}' not found in fetched data. Cannot parse vectors.")
# Return the DataFrame as is, but log the issue
# You might want to drop rows missing the embedding column here if they shouldn't exist
else:
# Apply parsing only if the column exists
parsed_embeddings = df[self.embedding_column].apply(self._parse_vector_string)
# Overwrite the original string column with the parsed list (or None if parsing failed)
df[self.embedding_column] = parsed_embeddings
logger.debug(f"Sample '{self.embedding_column}' data after parsing (first 5 rows):\n{df[[self.embedding_column]].head()}")
# === Enhanced Debugging and Filtering for Embedding Column ===
logger.info(f"Checking validity of parsed '{self.embedding_column}' column...")
if self.embedding_column not in df.columns:
# If column wasn't found and parsing wasn't attempted
raise VectorQueryLoaderError(f"Required embedding column '{self.embedding_column}' missing in fetched data.")
# 1. Check for NULLs (includes rows where parsing failed and returned None)
initial_count = len(df) # Count *before* filtering
null_mask = df[self.embedding_column].isnull()
null_count = null_mask.sum()
if null_count > 0:
logger.warning(f"Found {null_count} rows with NULL or unparsable vectors in '{self.embedding_column}'. These will be dropped.")
df_no_nulls = df.dropna(subset=[self.embedding_column]).copy() # Use copy after dropna
count_after_null_drop = len(df_no_nulls)
if count_after_null_drop < initial_count:
logger.debug(f"{count_after_null_drop} rows remaining after dropping NULLs/unparsable.")
# 2. Check for non-empty list type (Ensures parsed data is correct)
if not df_no_nulls.empty:
def is_valid_list_of_floats(x):
return isinstance(x, list) and len(x) > 0 and all(isinstance(n, float) for n in x)
valid_list_mask = df_no_nulls[self.embedding_column].apply(is_valid_list_of_floats)
invalid_list_count = len(df_no_nulls) - valid_list_mask.sum()
if invalid_list_count > 0:
# This indicates an issue with the parsing logic or unexpected data format
logger.error(f"Found {invalid_list_count} rows where '{self.embedding_column}' is not a non-empty list of floats *after parsing*. These will be dropped.")
# Optional: log examples of problematic entries here if needed
# invalid_entries = df_no_nulls[~valid_list_mask][self.embedding_column]
# for i, entry in enumerate(invalid_entries.head(5)):
# logger.debug(f" Problematic entry example {i+1}: Type={type(entry)}, Value='{str(entry)[:100]}...'")
df_filtered_embeddings = df_no_nulls[valid_list_mask].copy() # Use copy after filtering
else:
df_filtered_embeddings = df_no_nulls # Still empty
final_count = len(df_filtered_embeddings)
# === End Enhanced Debugging ===
if final_count == 0 and initial_count > 0:
# This happens if all rows were filtered out due to invalid embeddings
logger.error(f"All {initial_count} rows were filtered out due to missing/invalid '{self.embedding_column}'. Check raw data and parsing.")
# Return empty DF, but potentially raise a more critical error if this is unexpected
# raise VectorQueryLoaderError(f"No valid embedding data found after processing. All {initial_count} rows were invalid.")
# Ensure empty DF has expected columns
# Try to preserve the original columns found before dropping
cols_to_keep = [col for col in df.columns if col != self.embedding_column] + [self.embedding_column]
return pd.DataFrame(columns=cols_to_keep) # Return empty DataFrame with original columns
if final_count < initial_count:
logger.warning(f"Filtered out {initial_count - final_count} rows total due to missing/invalid '{self.embedding_column}'.")
logger.info(f"Proceeding with {final_count} valid rows.")
# Use the DataFrame *after* filtering out invalid embeddings
df_result = df_filtered_embeddings
# Validate and potentially add placeholder metadata columns AFTER filtering
# Use .get() for robustness if columns aren't always present
required_metadata_cols = ['file_id', 'chunk', self.COURSE_ID_COLUMN_NAME]
for col in required_metadata_cols:
if col not in df_result.columns:
# If filtering by course_id was requested and successful,
# the COURSE_ID_COLUMN_NAME must exist. If not, it's an error.
if col == self.COURSE_ID_COLUMN_NAME and course_id is not None:
logger.error(f"Required column '{col}' missing in fetched data despite successful filtering. This is unexpected.")
# Raise a critical error as filtering implies this column should exist
raise VectorQueryLoaderError(f"Required column '{col}' missing in fetched data.")
else:
logger.warning(f"Required metadata column '{col}' missing in fetched data.")
# Add a placeholder column of appropriate type if needed downstream
if col == 'file_id':
df_result['file_id'] = df_result.index.astype(str) # Use index as string ID
elif col == 'chunk':
df_result['chunk'] = "" # Empty string placeholder
elif col == self.COURSE_ID_COLUMN_NAME and course_id is None:
# Only add None if we weren't filtering and the column is missing
df_result[self.COURSE_ID_COLUMN_NAME] = pd.Series([None] * len(df_result), index=df_result.index, dtype=int) # Or object
# Reorder columns for consistency if desired, e.g., [file_id, chunk, course_id, embedding, other_cols]
# Example reordering (optional):
# cols = required_metadata_cols + [col for col in df_result.columns if col not in required_metadata_cols]
# # Ensure embedding column is towards the end if needed
# if self.embedding_column in cols:
# cols.remove(self.embedding_column)
# cols.append(self.embedding_column)
# # Handle duplicate column names if SELECT * returned duplicates (unlikely but possible)
# df_result = df_result[list(dict.fromkeys(cols))] # Use dict to preserve order and get unique columns
return df_result # Return the cleaned and validated DataFrame
except Exception as e:
logger.exception(f"Failed to fetch data from {self.operator.table_fqn()}.")
if 'relation' in str(e) and 'does not exist' in str(e):
raise VectorQueryLoaderError(f"Table/Relation not found error: {e}. Check schema/table name and permissions.") from e
# Catch the specific Supabase/PostgREST error if the column for filtering doesn't exist
elif 'column' in str(e).lower() and self.COURSE_ID_COLUMN_NAME.lower() in str(e).lower() and 'does not exist' in str(e).lower():
raise VectorQueryLoaderError(f"Filtering column '{self.COURSE_ID_COLUMN_NAME}' does not exist in table {self.operator.table_fqn()}. Query failed: {e}") from e
else:
raise VectorQueryLoaderError(f"Database query failed: {e}") from e
# You can keep or remove the TSNE methods depending on whether this
# loader should also handle visualization prep. If not, remove these.
# Keeping for "generalized" request, but they use fetch_vectors now.
def reduce_dimensions(self, df: pd.DataFrame, tsne_params: Optional[Dict[str, Any]] = None) -> pd.DataFrame:
"""
Applies t-SNE to reduce embedding dimensions to 3D.
(Code remains the same as previous correct version, assuming it works)
"""
logger.info("Starting t-SNE dimensionality reduction...")
if df.empty:
logger.warning("Input DataFrame for reduce_dimensions is empty. Returning empty DataFrame.")
empty_df_with_cols = df.copy()
for col in ['x', 'y', 'z']:
if col not in empty_df_with_cols:
empty_df_with_cols[col] = pd.Series(dtype=float)
return empty_df_with_cols
# Use the embedding_column name from the instance
embedding_col = self.embedding_column # Use local variable for clarity
if embedding_col not in df.columns:
# This should be caught by fetch_vectors, but defensive check
raise VectorQueryLoaderError(f"Embedding column '{embedding_col}' missing in DataFrame passed to reduce_dimensions.")
# Ensure embeddings are lists of floats
try:
# Check if all entries in the embedding column are lists of floats
# (This should be true if fetch_vectors's parsing/filtering worked)
# If not strictly list[float], try robust conversion again
if not all(isinstance(x, list) and all(isinstance(n, float) for n in x) for x in df[embedding_col]):
logger.error(f"Data in '{embedding_col}' is not strictly list[float] format after fetching. Attempting conversion before stacking.")
# Attempt robust conversion similar to the loader's parse method
embeddings_list = []
for item in df[embedding_col]:
parsed_item = None
if isinstance(item, str):
try: parsed_item = json.loads(item)
except json.JSONDecodeError: pass
elif isinstance(item, (list, tuple, np.ndarray)):
parsed_item = item
elif isinstance(item, dict) and 'vector' in item and isinstance(item['vector'], (list, tuple, np.ndarray)):
parsed_item = item['vector']
if isinstance(parsed_item, (list, tuple, np.ndarray)) and all(isinstance(val, (int, float, np.number)) for val in parsed_item):
embeddings_list.append([float(n) for n in parsed_item])
else:
logger.debug(f"Skipping problematic embedding during secondary clean for t-SNE: {str(item)[:100]}...")
if not embeddings_list:
logger.error("No valid embeddings remained after secondary cleaning for t-SNE.")
# No embeddings to process, return empty DF with t-SNE cols
empty_df_with_cols = df.copy()
for col in ['x', 'y', 'z']:
if col not in empty_df_with_cols:
empty_df_with_cols[col] = pd.Series(dtype=float)
return empty_df_with_cols
embeddings = np.array(embeddings_list, dtype=float)
logger.warning("Successfully converted problematic embedding data for t-SNE stacking.")
else:
# Data is in the expected list of float format, proceed directly
embeddings = np.array(df[embedding_col].to_list(), dtype=float)
except ValueError as ve:
logger.exception(f"Failed to convert embedding list to numeric numpy array for t-SNE: {ve}. Ensure '{embedding_col}' contains valid vector data.")
raise VectorQueryLoaderError(f"Data in '{embedding_col}' could not be converted to numeric vectors for t-SNE.") from ve
except Exception as e:
logger.exception(f"An unexpected error occurred while processing '{embedding_col}' column for t-SNE.")
raise VectorQueryLoaderError(f"An unexpected error occurred while processing embedding data for t-SNE: {e}") from e
if embeddings.ndim != 2:
raise VectorQueryLoaderError(f"Embedding data has unexpected dimensions for t-SNE: {embeddings.ndim} (expected 2). Shape: {embeddings.shape}")
n_samples = embeddings.shape[0]
if n_samples < 2:
logger.warning(f"Found only {n_samples} valid vector(s). t-SNE requires at least 2. Assigning default 3D coordinates.")
default_coords = [[0.0, 0.0, 0.0]] * n_samples
df_copy = df.copy() # Work on a copy
df_copy[['x', 'y', 'z']] = default_coords
return df_copy
logger.info(f"Applying t-SNE to {n_samples} vectors of dimension {embeddings.shape[1]}...")
current_tsne_params = self.DEFAULT_TSNE_PARAMS.copy()
if tsne_params:
current_tsne_params.update(tsne_params)
logger.info(f"Using custom t-SNE params: {tsne_params}")
# Adjust perplexity if sample size is too small
if n_samples <= current_tsne_params['perplexity']:
new_perplexity = max(5.0, float(n_samples - 1)) # Perplexity must be less than num samples
if new_perplexity < 5.0: # Ensure minimum perplexity if possible
if n_samples > 5: new_perplexity = 5.0 # if samples > 5, use 5
else: new_perplexity = max(1.0, float(n_samples - 1)) # Otherwise just samples - 1 (min 1)
if current_tsne_params['perplexity'] != new_perplexity: # Only log if changed
logger.warning(f"Adjusting t-SNE perplexity from {current_tsne_params['perplexity']:.1f} "
f"to {new_perplexity:.1f} due to low sample count ({n_samples}).")
current_tsne_params['perplexity'] = new_perplexity
# Adjust n_iter for potential large datasets if not already high
# This check is copied from your original loader, keep it consistent
# if n_samples * embeddings.shape[1] > 100000 and current_tsne_params['n_iter'] < 1000:
# logger.warning(f"Dataset size seems large, increasing t-SNE n_iter from {current_tsne_params['n_iter']} to 1000 for better convergence.")
# current_tsne_params['n_iter'] = 1000
try:
logger.debug(f"Final t-SNE parameters: {current_tsne_params}")
tsne = TSNE(**current_tsne_params)
reduced_embeddings = tsne.fit_transform(embeddings)
# Ensure the DataFrame index aligns with the embeddings if rows were dropped
df_copy = df.copy() # Work on a copy to avoid modifying the original index alignment issue if any
df_copy[['x', 'y', 'z']] = reduced_embeddings
logger.info("t-SNE reduction complete.")
return df_copy
except Exception as e:
logger.exception("t-SNE dimensionality reduction failed.")
raise VectorQueryLoaderError(f"t-SNE failed: {e}") from e
# Keep load_and_reduce for generality, though clustering might not use it
def load_and_reduce(self, limit: Optional[int] = None, course_id: Optional[int] = None, tsne_params: Optional[Dict[str, Any]] = None) -> pd.DataFrame:
"""Orchestrates fetching vectors (optionally filtered) and reducing dimensions."""
logger.info("Starting vector load (filtered) and reduction process...")
# Use the fetch_vectors method which handles filtering
df_raw_filtered = self.fetch_vectors(limit=limit, course_id=course_id)
df_reduced = self.reduce_dimensions(df_raw_filtered, tsne_params=tsne_params)
logger.info("Vector load and reduction process finished.")
return df_reduced
# --- END OF FILE librarian_vspace/vutils/vector_query_loader.py ---