Initialize Monorepo
This commit is contained in:
commit
f80792d739
106 files changed
+17076
No files matched your search
@@ -0,0 +1,5 @@
|
||||
# UV Update
|
||||
```shell
|
||||
uv lock --upgrade
|
||||
uv sync
|
||||
```
|
||||
@@ -0,0 +1,47 @@
|
||||
import os
|
||||
|
||||
def chunk_file(input_file, output_dir=None, start_num=1, padding=2):
|
||||
"""
|
||||
Split a file into chunks and save each chunk as a separate file.
|
||||
|
||||
Args:
|
||||
input_file (str): Path to the input file
|
||||
output_dir (str, optional): Directory to save chunk files. Defaults to current directory.
|
||||
start_num (int, optional): Starting number for the chunk files. Defaults to 1.
|
||||
padding (int, optional): Number of digits to pad the incremental numbers. Defaults to 2.
|
||||
"""
|
||||
if output_dir and not os.path.exists(output_dir):
|
||||
os.makedirs(output_dir)
|
||||
|
||||
with open(input_file) as f:
|
||||
content = f.read()
|
||||
chunks = content.split("---")
|
||||
|
||||
chunk_count = start_num
|
||||
for chunk in chunks:
|
||||
chunk = chunk.replace('---', '').strip()
|
||||
if not chunk: # Skip empty chunks
|
||||
continue
|
||||
|
||||
# Define output path with padded incremental number
|
||||
file_name = f'chunk_{chunk_count:0{padding}d}.md'
|
||||
if output_dir:
|
||||
outfile_path = os.path.join(output_dir, file_name)
|
||||
else:
|
||||
outfile_path = file_name
|
||||
|
||||
with open(outfile_path, 'w') as outfile:
|
||||
outfile.write(chunk)
|
||||
|
||||
chunk_count += 1
|
||||
|
||||
return chunk_count - start_num # Return the number of chunks written
|
||||
|
||||
# Example usage
|
||||
if __name__ == "__main__":
|
||||
#input_file = "/home/gra/PycharmProjects/librarian_vspace/examples/chunks/knowledge_chunks_detailed.md"
|
||||
input_file = "/home/gra/PycharmProjects/librarian_vspace/examples/chunks/knowledge_chunks_1500.md"
|
||||
# You can specify an output directory or omit it to use the current directory
|
||||
output_dir = "/examples/chunks/chunk_md_x"
|
||||
chunk_file(input_file, output_dir)
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
|
||||
#!/usr/bin/env python3
|
||||
"""examples/demo_run_cluster_export.py
|
||||
|
||||
Launch ClusterExportWorker via FlowArtifact wrapper, mirroring the embedder demo.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import os
|
||||
|
||||
from librarian_vspace.vquery.cluster_export_worker import ClusterExportWorker, ClusterExportInput
|
||||
from librarian_core.workers.base import FlowArtifact
|
||||
|
||||
COURSE_ID = 15512 # example id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def _load_env(path: Path) -> None:
|
||||
if not path.is_file():
|
||||
return
|
||||
for line in path.read_text().splitlines():
|
||||
if line.strip() and not line.startswith("#") and "=" in line:
|
||||
k, v = [p.strip() for p in line.split("=", 1)]
|
||||
os.environ.setdefault(k, v)
|
||||
|
||||
async def _main() -> None:
|
||||
payload = ClusterExportInput(course_id=COURSE_ID)
|
||||
|
||||
worker = ClusterExportWorker()
|
||||
art = FlowArtifact.new(run_id="", dir=Path.cwd(), data=payload)
|
||||
result_artifact = await worker.flow()(art) # FlowArtifact
|
||||
|
||||
output = result_artifact.data # ClusterExportOutput
|
||||
logger.info("✅ Worker finished – output directory: %s", output.output_dir)
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
|
||||
APP_DIR = Path(__file__).resolve().parent
|
||||
_load_env(APP_DIR / ".env")
|
||||
asyncio.run(_main())
|
||||
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
from pathlib import Path
|
||||
from typing import Any, List, Dict
|
||||
import json
|
||||
|
||||
from librarian_vspace.vecembed.embedder_worker import EmbedderWorker, EmbedderInput
|
||||
from librarian_core.workers.base import FlowArtifact
|
||||
from librarian_core.temp_payloads.chunk_data import ChunkData
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Configuration
|
||||
# ------------------------------------------------------------------ #
|
||||
# Folder with the small sample dataset (3 × .md files)
|
||||
DEMO_PATH: Path = Path("/home/gra/PycharmProjects/librarian_vspace/examples/chunks/moodle_chunks/51cd7cf6-e782-4f17-af00-30852cdcd5fc/51cd7cf6-e782-4f17-af00-30852cdcd5fc/data/FS25/Effiziente_Algorithmen").expanduser()
|
||||
#DEMO_PATH: Path = Path("/home/gra/PycharmProjects/librarian_vspace/examples/chunks/chunk_md").expanduser()
|
||||
|
||||
# Where to write the concatenated text file
|
||||
# (one level above the dataset folder keeps things tidy)
|
||||
COURSE_ID_POOL = [16301, 16091, 17505, 18239, 17503, 15512]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
INPUT_MODEL=Path("/home/gra/PycharmProjects/librarian_vspace/examples/chunks/moodle_chunks/51cd7cf6-e782-4f17-af00-30852cdcd5fc/51cd7cf6-e782-4f17-af00-30852cdcd5fc/result.json")
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
def _load_env(path: Path) -> None:
|
||||
"""Load KEY=VALUE pairs from a .env file if present."""
|
||||
if not path.is_file():
|
||||
return
|
||||
for line in path.read_text().splitlines():
|
||||
if line.strip() and not line.startswith("#") and "=" in line:
|
||||
k, v = [p.strip() for p in line.split("=", 1)]
|
||||
os.environ.setdefault(k, v)
|
||||
|
||||
|
||||
def discover_chunks(root: Path) -> List[Path]:
|
||||
"""Return all markdown files in the dataset folder."""
|
||||
return sorted(root.glob("*.md"))
|
||||
|
||||
|
||||
def build_course(root: Path) -> Dict[str, Any]:
|
||||
"""Minimal dict that satisfies EmbedderWorker's `chunk_course`."""
|
||||
files = [
|
||||
{"file_name": p.name, "file_id": str(random.getrandbits(24))}
|
||||
for p in discover_chunks(root)
|
||||
]
|
||||
if not files:
|
||||
raise FileNotFoundError(f"No .md files found in {root}")
|
||||
return {
|
||||
"path": str(root),
|
||||
"files": files,
|
||||
#"course_id": str(random.choice(COURSE_ID_POOL)),
|
||||
"course_id": "18240"
|
||||
}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
async def _main() -> None:
|
||||
course = build_course(DEMO_PATH)
|
||||
concat_path = DEMO_PATH
|
||||
|
||||
with open(INPUT_MODEL, 'r') as file:
|
||||
json_data = json.load(file)
|
||||
|
||||
#payload = EmbedderInput(chunk_course=course, concat_path=concat_path)
|
||||
payload = ChunkData.model_validate_json(json_data)
|
||||
worker = EmbedderWorker()
|
||||
logger.info("🔨 Launching EmbedderWorker …")
|
||||
art = FlowArtifact.new(run_id="", dir=concat_path, data=payload)
|
||||
result = await worker.flow()(art) # type: ignore[arg-type]
|
||||
|
||||
logger.info("✅ Worker finished: %s", result)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
if __name__ == "__main__":
|
||||
APP_DIR = Path(__file__).resolve().parent
|
||||
_load_env(APP_DIR / ".env")
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
|
||||
asyncio.run(_main())
|
||||
@@ -0,0 +1,66 @@
|
||||
|
||||
#!/usr/bin/env python3
|
||||
"""examples/demo_run_query.py
|
||||
|
||||
Runs QueryWorker via FlowArtifact wrapper (mirrors cluster export demo).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from librarian_vspace.vquery.query_worker import QueryWorker, QueryInput
|
||||
from librarian_vspace.models.query_model import VectorSearchRequest
|
||||
from librarian_core.workers.base import FlowArtifact
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Config
|
||||
# ------------------------------------------------------------------ #
|
||||
SEARCH_STRING = "integration"
|
||||
COURSE_FILTER_GT = 900 # adjust if needed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def _load_env(path: Path) -> None:
|
||||
if not path.is_file():
|
||||
return
|
||||
for line in path.read_text().splitlines():
|
||||
if line.strip() and not line.startswith("#") and "=" in line:
|
||||
k, v = [p.strip() for p in line.split("=", 1)]
|
||||
os.environ.setdefault(k, v)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
async def _main() -> None:
|
||||
# Vector search request
|
||||
vs_req = VectorSearchRequest(
|
||||
interface_name=os.getenv("EMBED_INTERFACE", "ollama"),
|
||||
model_name=os.getenv("EMBED_MODEL", "snowflake-arctic-embed2"),
|
||||
search_string=SEARCH_STRING,
|
||||
filters={"file_id": ("gt", COURSE_FILTER_GT)},
|
||||
top_k=10,
|
||||
)
|
||||
|
||||
payload = QueryInput(
|
||||
request=vs_req,
|
||||
db_schema=os.getenv("VECTOR_SCHEMA", "librarian"),
|
||||
rpc_function=os.getenv("VECTOR_FUNCTION", "pdf_chunking"),
|
||||
embed_model=os.getenv("EMBED_MODEL", "snowflake-arctic-embed2"),
|
||||
)
|
||||
|
||||
worker = QueryWorker()
|
||||
art = FlowArtifact.new(run_id="", dir=Path.cwd(), data=payload)
|
||||
result_artifact = await worker.flow()(art) # FlowArtifact
|
||||
|
||||
response = result_artifact.data # VectorSearchResponse
|
||||
logger.info("✅ Worker finished – received %s results", response.total)
|
||||
for idx, ck in enumerate(response.results, 1):
|
||||
logger.info("• %s: %s", idx, ck.chunk[:80] + ("…" if len(ck.chunk or '') > 80 else ""))
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
|
||||
APP_DIR = Path(__file__).resolve().parent
|
||||
_load_env(APP_DIR / ".env")
|
||||
asyncio.run(_main())
|
||||
@@ -0,0 +1,43 @@
|
||||
|
||||
#!/usr/bin/env python3
|
||||
"""examples/demo_run_tsne_export.py
|
||||
|
||||
Launch TsneExportWorker via FlowArtifact wrapper.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import os
|
||||
|
||||
from librarian_vspace.vecview.tsne_export_worker import TsneExportWorker, TsneExportInput
|
||||
from librarian_core.workers.base import FlowArtifact
|
||||
|
||||
COURSE_ID = 18240 # choose a course with embeddings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def _load_env(path: Path) -> None:
|
||||
if not path.is_file():
|
||||
return
|
||||
for line in path.read_text().splitlines():
|
||||
if line.strip() and not line.startswith("#") and "=" in line:
|
||||
k, v = [p.strip() for p in line.split("=", 1)]
|
||||
os.environ.setdefault(k, v)
|
||||
|
||||
async def _main() -> None:
|
||||
payload = TsneExportInput(course_id=COURSE_ID)
|
||||
|
||||
worker = TsneExportWorker()
|
||||
art = FlowArtifact.new(run_id="", dir=Path.cwd(), data=payload)
|
||||
result_artifact = await worker.flow()(art) # FlowArtifact
|
||||
|
||||
output = result_artifact.data # TsneExportOutput
|
||||
logger.info("✅ Worker finished – JSON file: %s", output.json_path)
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
|
||||
APP_DIR = Path(__file__).resolve().parent
|
||||
_load_env(APP_DIR / ".env")
|
||||
asyncio.run(_main())
|
||||
@@ -0,0 +1,4 @@
|
||||
from librarian_vspace.vutils.parallelism_advisor import recommended_workers
|
||||
print(recommended_workers(kind="cpu"))
|
||||
print(recommended_workers(kind="io"))
|
||||
print(recommended_workers(kind="gpu"))
|
||||
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Loads vector data using vecmap.loader, reduces dimensions via t-SNE,
|
||||
and launches an interactive 3D visualization using vecmap.visualizer (Dash/Plotly).
|
||||
|
||||
Configuration is primarily driven by environment variables.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
import pandas as pd
|
||||
|
||||
# Define application directory relative to this script file
|
||||
APP_DIR = pathlib.Path(__file__).resolve().parent
|
||||
# Define the source directory containing vecmap, vutils, etc.
|
||||
SRC_DIR = APP_DIR.parent / "src"
|
||||
# Define path to .env file relative to APP_DIR
|
||||
DOTENV_PATH = APP_DIR / ".env"
|
||||
|
||||
# --- Explicitly Manage sys.path ---
|
||||
app_dir_str = str(APP_DIR)
|
||||
src_dir_str = str(SRC_DIR)
|
||||
if app_dir_str in sys.path:
|
||||
try: sys.path.remove(app_dir_str)
|
||||
except ValueError: pass
|
||||
if src_dir_str not in sys.path:
|
||||
sys.path.insert(0, src_dir_str)
|
||||
elif sys.path[0] != src_dir_str:
|
||||
try: sys.path.remove(src_dir_str)
|
||||
except ValueError: pass
|
||||
sys.path.insert(0, src_dir_str)
|
||||
print(f"[DEBUG] sys.path start: {sys.path[:3]}")
|
||||
|
||||
# --- .env Loader ---
|
||||
def _load_env_file(path: pathlib.Path) -> None:
|
||||
print(f"Attempting to load .env file from: {path}")
|
||||
if not path.is_file(): print(f".env file not found at {path}, skipping."); return
|
||||
loaded, skipped = 0, 0
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
line = line.strip();
|
||||
if not line or line.startswith("#") or "=" not in line: continue
|
||||
key, val = line.split("=", 1); key, val = key.strip(), val.strip()
|
||||
if key not in os.environ: os.environ[key] = val; loaded += 1
|
||||
else: skipped += 1
|
||||
print(f"Loaded {loaded} new vars, skipped {skipped} existing vars from .env")
|
||||
except Exception as e: print(f"Error reading .env file at {path}: {e}")
|
||||
_load_env_file(DOTENV_PATH)
|
||||
|
||||
# --- Logging Setup ---
|
||||
log_level_str = os.getenv("VECMAP_DEBUG", "false").lower()
|
||||
log_level = logging.DEBUG if log_level_str in ("true", "1") else logging.INFO
|
||||
logging.basicConfig(level=log_level, format='[%(asctime)s] [%(levelname)s] [%(name)s:%(lineno)d] %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
|
||||
if log_level > logging.DEBUG:
|
||||
for logger_name in ["urllib3", "httpx", "supabase"]: logging.getLogger(logger_name).setLevel(logging.WARNING)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# --- Imports ---
|
||||
try:
|
||||
from librarian_vspace.vecmap.loader import VectorLoader, VectorLoaderError
|
||||
from librarian_vspace.vecmap.visualizer import VectorVisualizer # Removed DEFAULT_N_CLUSTERS import
|
||||
import librarian_vspace.vutils
|
||||
import librarian_vspace.vecembed
|
||||
logger.debug("Successfully imported components.")
|
||||
except ImportError as e:
|
||||
logger.error(f"Failed to import necessary modules: {e}", exc_info=True)
|
||||
sys.exit(1)
|
||||
|
||||
# --- Main Logic ---
|
||||
def main() -> None:
|
||||
logger.info("--- Starting VecMap Visualizer ---")
|
||||
|
||||
# --- Configuration ---
|
||||
db_schema = os.getenv("VECTOR_SCHEMA", "librarian")
|
||||
db_function = os.getenv("VECTOR_FUNCTION", "pdf_chunking")
|
||||
model_name = os.getenv("EMBED_MODEL", "snowflake-arctic-embed2")
|
||||
interface_name = os.getenv("EMBED_INTERFACE", "ollama")
|
||||
embedding_column = os.getenv("EMBEDDING_COLUMN", "embedding")
|
||||
try: limit_str = os.getenv("VECMAP_LIMIT"); data_limit = int(limit_str) if limit_str else None
|
||||
except ValueError: logger.warning(f"Invalid VECMAP_LIMIT. Ignoring."); data_limit = None
|
||||
try: perplexity_str = os.getenv("VECMAP_PERPLEXITY", "30.0"); tsne_perplexity = float(perplexity_str)
|
||||
except ValueError: logger.warning(f"Invalid VECMAP_PERPLEXITY. Using 30.0."); tsne_perplexity = 30.0
|
||||
|
||||
# n_clusters configuration removed
|
||||
|
||||
dash_host = os.getenv("VECMAP_HOST", "127.0.0.1")
|
||||
try: port_str = os.getenv("VECMAP_PORT", "8050"); dash_port = int(port_str)
|
||||
except ValueError: logger.warning(f"Invalid VECMAP_PORT. Using 8050."); dash_port = 8050
|
||||
dash_debug = log_level == logging.DEBUG
|
||||
|
||||
logger.info("Effective Configuration:")
|
||||
logger.info(f" Database: schema={db_schema}, function={db_function}")
|
||||
logger.info(f" Model/Interface: model={model_name}, interface={interface_name}")
|
||||
logger.info(f" Data Params: column={embedding_column}, limit={data_limit}")
|
||||
logger.info(f" Processing: perplexity={tsne_perplexity} (n_clusters is now dynamic)") # Updated log
|
||||
logger.info(f" Server: host={dash_host}, port={dash_port}, debug={dash_debug}")
|
||||
|
||||
# --- 1. Initial Load and Reduce ---
|
||||
initial_df_reduced = pd.DataFrame()
|
||||
try:
|
||||
logger.info("Performing initial data load and processing...")
|
||||
loader = VectorLoader(schema=db_schema, function=db_function, model=model_name, embedding_column=embedding_column)
|
||||
tsne_params = {"perplexity": tsne_perplexity}
|
||||
initial_df_reduced = loader.load_and_reduce(limit=data_limit, tsne_params=tsne_params)
|
||||
if initial_df_reduced.empty: logger.warning("Initial data load resulted in an empty dataset.")
|
||||
else: logger.info(f"Successfully loaded and reduced {len(initial_df_reduced)} vectors initially.")
|
||||
except VectorLoaderError as e: logger.error(f"Initial data load failed: {e}", exc_info=dash_debug)
|
||||
except Exception as e: logger.error(f"Unexpected error during initial data load: {e}", exc_info=dash_debug)
|
||||
|
||||
# --- 2. Initialize and Start Visualization ---
|
||||
try:
|
||||
logger.info("Initializing VectorVisualizer...")
|
||||
visualizer = VectorVisualizer(
|
||||
initial_data=initial_df_reduced,
|
||||
db_schema=db_schema,
|
||||
db_function=db_function,
|
||||
interface_name=interface_name,
|
||||
model_name=model_name,
|
||||
embedding_column=embedding_column,
|
||||
initial_limit=data_limit,
|
||||
initial_perplexity=tsne_perplexity
|
||||
# n_clusters argument removed
|
||||
)
|
||||
logger.info("Launching visualizer...")
|
||||
visualizer.run(host=dash_host, port=dash_port, debug=dash_debug)
|
||||
except TypeError as te:
|
||||
logger.error(f"TypeError during VectorVisualizer initialization: {te}", exc_info=True)
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize or run visualizer: {e}", exc_info=dash_debug)
|
||||
sys.exit(1)
|
||||
|
||||
logger.info("--- VecMap Visualizer finished ---")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,52 @@
|
||||
[project]
|
||||
name = "librarian-vspace"
|
||||
version = "0.1.0"
|
||||
description = "Add your description here"
|
||||
readme = "README.md"
|
||||
authors = [
|
||||
{ name = "TheOriginalGraLargeShrimpakaReaper", email = "graber-michael@hotmail.com" }
|
||||
]
|
||||
requires-python = ">=3.10"
|
||||
dependencies = [
|
||||
"librarian-core",
|
||||
"importlib_metadata; python_version<'3.10'",
|
||||
"dotenv>=0.9.9",
|
||||
"psycopg2-binary>=2.9.10",
|
||||
"python-dotenv>=1.1.0",
|
||||
"requests>=2.32.3",
|
||||
"supabase>=2.15.0",
|
||||
"numpy>=2.2.5",
|
||||
"dash>=3.0.4",
|
||||
"scikit-learn>=1.6.1",
|
||||
"plotly>=6.0.1",
|
||||
"pandas>=2.2.3",
|
||||
"pathlib>=1.0.1",
|
||||
"prefect>=3.4.1",
|
||||
]
|
||||
|
||||
[tool.uv.sources]
|
||||
librarian-core = { git = "https://github.com/DotNaos/librarian-core", rev = "dev" }
|
||||
|
||||
[build-system]
|
||||
requires = ["hatchling>=1.21"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = ["src/librarian_vspace"]
|
||||
|
||||
|
||||
[tool.hatch.metadata]
|
||||
allow-direct-references = true
|
||||
|
||||
|
||||
# ───────── optional: dev / test extras ─────────
|
||||
[project.optional-dependencies]
|
||||
dev = ["ruff", "pytest", "mypy"]
|
||||
|
||||
[project.entry-points."librarian.workers"]
|
||||
embedder = "librarian_vspace.vecembed:EmbedderWorker"
|
||||
clusterexporter = "librarian_vspace.vquery:ClusterExportWorker"
|
||||
tnseexport = "librarian_vspace.vecview:TsneExportWorker"
|
||||
vectorquerying = "librarian_vspace.vquery:QueryWorker"
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
|
||||
"""Embedding‑related helpers."""
|
||||
import pkgutil
|
||||
import importlib
|
||||
|
||||
__all__ = []
|
||||
|
||||
# Iterate over all modules in this package
|
||||
for finder, module_name, is_pkg in pkgutil.iter_modules(__path__):
|
||||
# import the sub-module
|
||||
module = importlib.import_module(f"{__name__}.{module_name}")
|
||||
|
||||
# decide which names to re-export:
|
||||
# use module.__all__ if it exists, otherwise every non-private attribute
|
||||
public_names = getattr(
|
||||
module, "__all__", [n for n in dir(module) if not n.startswith("_")]
|
||||
)
|
||||
|
||||
# bring each name into the package namespace
|
||||
for name in public_names:
|
||||
globals()[name] = getattr(module, name)
|
||||
__all__.append(name) # type: ignore
|
||||
Whitespace-only changes.
@@ -0,0 +1,38 @@
|
||||
|
||||
"""Pydantic models for vector search requests and responses."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any, Dict, List, Optional
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class VectorSearchRequest(BaseModel):
|
||||
"""Input payload for a vector search."""
|
||||
|
||||
interface_name: str = Field(..., description="Name of the embedding interface")
|
||||
model_name: str = Field(..., description="Name of the embedding model")
|
||||
search_string: str = Field(..., description="The natural language query to embed and search for")
|
||||
filters: Optional[Dict[str, Any]] = Field(
|
||||
default=None,
|
||||
description="Optional key/value filters applied server‑side",
|
||||
)
|
||||
top_k: int = Field(10, ge=1, le=100, description="Number of matches to return")
|
||||
embedding_column: str = Field(
|
||||
"embedding",
|
||||
description="Name of the embedding column in the database table",
|
||||
)
|
||||
|
||||
|
||||
class Chunklet(BaseModel):
|
||||
"""Single result row returned by the database RPC."""
|
||||
|
||||
chunk: Optional[str] = None
|
||||
file_id: Optional[str | int] = None
|
||||
|
||||
|
||||
class VectorSearchResponse(BaseModel):
|
||||
"""Output payload wrapping vector‑search results."""
|
||||
|
||||
total: int
|
||||
results: List[Chunklet]
|
||||
@@ -0,0 +1,31 @@
|
||||
|
||||
"""Data models for t‑SNE exports.
|
||||
|
||||
These models are used by *vecview* and any endpoint that needs to return or
|
||||
validate t‑SNE projection data.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import List, Optional
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class TSNEPoint(BaseModel):
|
||||
"""A single point in a 3‑D t‑SNE projection."""
|
||||
|
||||
x: float
|
||||
y: float
|
||||
z: float
|
||||
file_id: str
|
||||
chunk: str
|
||||
cluster: Optional[str] = None
|
||||
hover_text: Optional[str] = None
|
||||
|
||||
|
||||
class TSNEData(BaseModel):
|
||||
"""Container returned to callers requesting a t‑SNE view."""
|
||||
|
||||
course_id: Optional[int] = None
|
||||
total: int
|
||||
points: List[TSNEPoint]
|
||||
Whitespace-only changes.
@@ -0,0 +1,9 @@
|
||||
|
||||
"""Embedding‑related helpers."""
|
||||
from __future__ import annotations
|
||||
|
||||
from .vector_inserter import VectorInserter
|
||||
from .embedding_generator import EmbeddingGenerator
|
||||
from .embedding_workflow import EmbeddingWorkflow
|
||||
|
||||
__all__ = ["VectorInserter", "EmbeddingGenerator", "EmbeddingWorkflow"]
|
||||
@@ -0,0 +1,155 @@
|
||||
|
||||
"""Parallel‑aware embedding helpers.
|
||||
|
||||
* **embed_single_file()** – embed one file (sync).
|
||||
* **run_embedder()** – embed all files in a course (async, kept for back‑compat).
|
||||
* **_create_hnsw_index()** – helper to (re)build PGVector HNSW index.
|
||||
|
||||
This file contains no Prefect code; it’s pure embedding logic.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, List, Union
|
||||
|
||||
from postgrest import APIResponse
|
||||
|
||||
from librarian_core.temp_payloads.chunk_data import ChunkCourse, ChunkFile
|
||||
from librarian_vspace.vecembed.embedding_generator import EmbeddingGenerator
|
||||
from librarian_vspace.vecembed.vector_inserter import VectorInserter
|
||||
from librarian_vspace.vecembed.embedding_workflow import EmbeddingWorkflow
|
||||
from librarian_vspace.vutils.supabase_singleton import MySupabase
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _autodiscover_pg_conn():
|
||||
supa = MySupabase.get_client() # type: ignore
|
||||
if supa is None:
|
||||
raise RuntimeError("MySupabase.get_client() returned None – no DB connection.")
|
||||
return supa
|
||||
|
||||
|
||||
def _create_hnsw_index(
|
||||
supa,
|
||||
table_fqn: str,
|
||||
*,
|
||||
column_name: str = "embedding",
|
||||
query_operator: str = "<=>",
|
||||
m: int = 16,
|
||||
ef: int = 64,
|
||||
) -> None:
|
||||
if "." not in table_fqn:
|
||||
raise ValueError("table_fqn must be schema.table")
|
||||
schema, table = table_fqn.split(".", 1)
|
||||
try:
|
||||
supa.schema(schema).rpc(
|
||||
"create_or_reindex_hnsw",
|
||||
dict(
|
||||
p_schema=schema,
|
||||
p_table=table,
|
||||
p_column=column_name,
|
||||
p_operator=query_operator,
|
||||
p_m=m,
|
||||
p_ef=ef,
|
||||
),
|
||||
).execute()
|
||||
except Exception:
|
||||
logger.exception("Failed to run create_or_reindex_hnsw")
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# single file #
|
||||
# --------------------------------------------------------------------------- #
|
||||
def embed_single_file(
|
||||
*,
|
||||
course_id: str,
|
||||
file_entry: dict | ChunkFile | SimpleNamespace,
|
||||
concat_path: Union[str, Path],
|
||||
db_schema: str = "librarian",
|
||||
db_function: str = "pdf_chunking",
|
||||
interface_name: str = "ollama",
|
||||
model_name: str = "snowflake-arctic-embed2",
|
||||
file_type: str = "md",
|
||||
) -> Path | None:
|
||||
|
||||
if isinstance(file_entry, (dict, SimpleNamespace)):
|
||||
file_name = file_entry["file_name"] if isinstance(file_entry, dict) else file_entry.file_name
|
||||
file_id = file_entry["file_id"] if isinstance(file_entry, dict) else file_entry.file_id
|
||||
else:
|
||||
file_name, file_id = file_entry.file_name, file_entry.file_id
|
||||
|
||||
chunk_path = Path(concat_path) / file_name
|
||||
if not chunk_path.exists():
|
||||
logger.warning("Missing chunk file %s – skipping", chunk_path)
|
||||
return None
|
||||
|
||||
generator = EmbeddingGenerator()
|
||||
inserter = VectorInserter(schema=db_schema, function=db_function, model=model_name)
|
||||
|
||||
wf = EmbeddingWorkflow(
|
||||
chunk_path=chunk_path,
|
||||
course_id=course_id,
|
||||
file_id=file_id,
|
||||
file_type=file_type,
|
||||
interface_name=interface_name,
|
||||
model_name=model_name,
|
||||
generator=generator,
|
||||
inserter=inserter,
|
||||
)
|
||||
wf.process()
|
||||
return chunk_path
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
async def run_embedder(
|
||||
course: ChunkCourse,
|
||||
concat_path: Union[str, Path],
|
||||
*,
|
||||
db_schema: str = "librarian",
|
||||
db_function: str = "pdf_chunking",
|
||||
interface_name: str = "ollama",
|
||||
model_name: str = "snowflake-arctic-embed2",
|
||||
file_type: str = "md",
|
||||
vector_column: str = "embedding",
|
||||
query_operator: str = "<=>",
|
||||
hnsw_m: int = 16,
|
||||
hnsw_ef: int = 64,
|
||||
max_parallel_files: int | None = None,
|
||||
) -> Path:
|
||||
|
||||
supa_client = _autodiscover_pg_conn()
|
||||
root = Path(concat_path)
|
||||
sem = asyncio.Semaphore(max_parallel_files or len(course.files) or 1)
|
||||
|
||||
async def _wrapper(cf):
|
||||
async with sem:
|
||||
return await asyncio.to_thread(
|
||||
embed_single_file,
|
||||
course_id=course.course_id,
|
||||
file_entry=cf,
|
||||
concat_path=root,
|
||||
db_schema=db_schema,
|
||||
db_function=db_function,
|
||||
interface_name=interface_name,
|
||||
model_name=model_name,
|
||||
file_type=file_type,
|
||||
)
|
||||
|
||||
await asyncio.gather(*[asyncio.create_task(_wrapper(cf)) for cf in course.files])
|
||||
|
||||
inserter = VectorInserter(schema=db_schema, function=db_function, model=model_name)
|
||||
_create_hnsw_index(
|
||||
supa_client,
|
||||
inserter.table_fqn(),
|
||||
column_name=vector_column,
|
||||
query_operator=query_operator,
|
||||
m=hnsw_m,
|
||||
ef=hnsw_ef,
|
||||
)
|
||||
return root
|
||||
|
||||
__all__ = ["embed_single_file", "run_embedder", "_create_hnsw_index", "_autodiscover_pg_conn"]
|
||||
@@ -0,0 +1,67 @@
|
||||
|
||||
"""EmbedderWorker – Prefect‑mapped per file."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, List
|
||||
|
||||
from prefect import get_run_logger, task, unmapped
|
||||
from pydantic import BaseModel, Field
|
||||
from librarian_core.workers.base import Worker
|
||||
|
||||
@task(name="embed_file", retries=2, retry_delay_seconds=5, log_prints=True, tags=["embed_file"])
|
||||
def embed_file_task(course_dict: dict | SimpleNamespace, file_entry: dict, concat_path: Path) -> Path | None:
|
||||
from librarian_vspace.vecembed.embedder import embed_single_file
|
||||
cid = course_dict["course_id"] if isinstance(course_dict, dict) else course_dict.course_id
|
||||
return embed_single_file(course_id=cid, file_entry=file_entry, concat_path=concat_path)
|
||||
|
||||
class EmbedderInput(BaseModel):
|
||||
chunk_courses: List[Any] = Field(default_factory=list, alias="chunk_courses")
|
||||
concat_path: Path
|
||||
chunk_course: Any | None = None
|
||||
def model_post_init(self, _):
|
||||
if not self.chunk_courses and self.chunk_course is not None:
|
||||
self.chunk_courses = [self.chunk_course]
|
||||
model_config = dict(populate_by_name=True, extra="allow")
|
||||
|
||||
class EmbedderOutput(BaseModel):
|
||||
result_paths: List[Path]
|
||||
|
||||
class EmbedderWorker(Worker[EmbedderInput, EmbedderOutput]):
|
||||
input_model = EmbedderInput
|
||||
output_model = EmbedderOutput
|
||||
|
||||
async def __run__(self, payload: EmbedderInput) -> EmbedderOutput:
|
||||
log = get_run_logger()
|
||||
total_files = sum(len(c["files"]) if isinstance(c, dict) else len(c.files) for c in payload.chunk_courses)
|
||||
log.info("Embedding %d files", total_files)
|
||||
|
||||
result_paths: List[Path] = []
|
||||
|
||||
# constants – could be parameterised later
|
||||
schema = "librarian"
|
||||
func = "pdf_chunking"
|
||||
model_name = "snowflake-arctic-embed2"
|
||||
|
||||
for course in payload.chunk_courses:
|
||||
files = course["files"] if isinstance(course, dict) else course.files
|
||||
futures = embed_file_task.map(unmapped(course), files, unmapped(payload.concat_path))
|
||||
for fut in futures:
|
||||
path = fut.result()
|
||||
if path:
|
||||
result_paths.append(path)
|
||||
|
||||
# rebuild index once per course
|
||||
from librarian_vspace.vecembed.embedder import _create_hnsw_index, _autodiscover_pg_conn
|
||||
from librarian_vspace.vecembed.vector_inserter import VectorInserter
|
||||
|
||||
supa = _autodiscover_pg_conn()
|
||||
inserter = VectorInserter(schema=schema, function=func, model=model_name)
|
||||
_create_hnsw_index(supa, inserter.table_fqn())
|
||||
|
||||
for p in result_paths:
|
||||
self.stage(p, new_name=p.name)
|
||||
|
||||
return EmbedderOutput(result_paths=result_paths)
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
|
||||
"""Factory for embedding back‑ends."""
|
||||
import logging
|
||||
from typing import Any, List, Optional, Tuple, Dict, Type
|
||||
|
||||
from librarian_vspace.vecembed.embedding_interface import EmbeddingInterface
|
||||
from librarian_vspace.vecembed.ollama_embedder import OllamaEmbedder
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class EmbeddingGenerator:
|
||||
_registry: Dict[str, Type[EmbeddingInterface]] = {
|
||||
"ollama": OllamaEmbedder,
|
||||
}
|
||||
|
||||
def generate_embedding(self, interface_name: str, model_name: str, text_to_embed: str, identifier: Any) -> Tuple[str, Optional[List[float]], Any]:
|
||||
cls = self._registry.get(interface_name.lower())
|
||||
if not cls:
|
||||
raise ValueError(f"Unsupported embedding interface: {interface_name}")
|
||||
embedder = cls(model_name=model_name)
|
||||
return embedder.embed(text_to_embed, identifier)
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
from __future__ import annotations
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, List, Optional, Tuple
|
||||
|
||||
class EmbeddingInterface(ABC):
|
||||
"""Contract for any embedding service implementation."""
|
||||
|
||||
def __init__(self, model_name: str, **kwargs: Any) -> None:
|
||||
self.model_name = model_name
|
||||
|
||||
@abstractmethod
|
||||
def embed(self, text_or_chunk: str, identifier: Any) -> Tuple[str, Optional[List[float]], Any]:
|
||||
"""Return (original_text, embedding, identifier) — embedding may be None on failure."""
|
||||
pass
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
"""Orchestrates loading, embedding, and storing a text chunk."""
|
||||
from __future__ import annotations
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, Dict, Union
|
||||
|
||||
# Import the worker classes for type hinting
|
||||
from librarian_vspace.vecembed.embedding_generator import EmbeddingGenerator
|
||||
from librarian_vspace.vecembed.vector_inserter import VectorInserter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class EmbeddingWorkflow:
|
||||
# Accept generator and inserter instances in __init__
|
||||
def __init__(self,
|
||||
chunk_path: Union[str, Path],
|
||||
course_id: Any,
|
||||
file_id: Any,
|
||||
file_type: str,
|
||||
interface_name: str, # Still needed for generate_embedding method
|
||||
model_name: str, # Still needed for generate_embedding method
|
||||
generator: EmbeddingGenerator, # Accept pre-instantiated generator
|
||||
inserter: VectorInserter, # Accept pre-instantiated inserter
|
||||
# db_schema and db_function are now implicit via the inserter
|
||||
# db_schema: str = "librarian",
|
||||
# db_function: str = "pdf_chunking",
|
||||
):
|
||||
self.chunk_path = Path(chunk_path)
|
||||
self.course_id = course_id
|
||||
self.file_id = file_id
|
||||
self.file_type = file_type
|
||||
# Keep interface_name and model_name as they are passed to the generator's method
|
||||
self.interface_name = interface_name
|
||||
self.model_name = model_name
|
||||
|
||||
# Assign the passed instances instead of creating new ones
|
||||
self.generator = generator
|
||||
self.inserter = inserter
|
||||
|
||||
# No need to store db_schema/db_function here if inserter handles it
|
||||
|
||||
# ---------------- helpers ----------------
|
||||
def _load_chunk(self) -> Optional[str]:
|
||||
try:
|
||||
text = self.chunk_path.read_text(encoding="utf-8").strip()
|
||||
if not text:
|
||||
logger.warning("Chunk %s is empty", self.chunk_path)
|
||||
return None
|
||||
return text
|
||||
except Exception as exc:
|
||||
logger.error("Failed to read %s: %s", self.chunk_path, exc)
|
||||
return None
|
||||
|
||||
def process(self) -> bool:
|
||||
chunk_text = self._load_chunk()
|
||||
if chunk_text is None:
|
||||
return False
|
||||
|
||||
# Use the shared generator instance
|
||||
original_text, vector, _ = self.generator.generate_embedding(
|
||||
interface_name=self.interface_name, # Pass parameters to the method
|
||||
model_name=self.model_name, # Pass parameters to the method
|
||||
text_to_embed=chunk_text,
|
||||
identifier=self.file_id,
|
||||
)
|
||||
|
||||
if vector is None:
|
||||
# Log failure within generator if not already done, or here
|
||||
logger.error(f"Failed to generate embedding for {self.chunk_path}")
|
||||
return False
|
||||
|
||||
payload: Dict[str, Any] = {
|
||||
"chunk": original_text,
|
||||
"course_id": self.course_id,
|
||||
"file_id": self.file_id,
|
||||
"file_type": self.file_type,
|
||||
"embedding": vector,
|
||||
}
|
||||
|
||||
# Use the shared inserter instance
|
||||
insert_result = self.inserter.insert_vector(payload)
|
||||
|
||||
if insert_result is None:
|
||||
logger.error(f"Failed to insert vector for {self.chunk_path}")
|
||||
return False
|
||||
|
||||
logger.debug(f"Successfully processed and inserted {self.chunk_path}")
|
||||
return True # Indicate success
|
||||
|
||||
|
||||
# Keep __all__ if needed
|
||||
# __all__ = ["EmbeddingWorkflow"]
|
||||
@@ -0,0 +1,44 @@
|
||||
|
||||
"""Ollama-based embedding implementation (env handled at application layer)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, List, Optional, Tuple
|
||||
|
||||
import requests
|
||||
|
||||
from librarian_vspace.vecembed.embedding_interface import EmbeddingInterface
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OllamaEmbedder(EmbeddingInterface):
|
||||
def __init__(self, model_name: str, **kwargs: Any) -> None:
|
||||
super().__init__(model_name=model_name)
|
||||
self.base_url = os.getenv("OLLAMA_BASE_URL")
|
||||
if not self.base_url:
|
||||
raise ValueError("OLLAMA_BASE_URL not configured – ensure env is set in the examples layer")
|
||||
self.api_endpoint = f"{self.base_url.rstrip('/')}/api/embeddings"
|
||||
|
||||
|
||||
def embed(self, text_or_chunk: str, identifier: Any) -> Tuple[str, Optional[List[float]], Any]:
|
||||
payload = {"model": self.model_name, "prompt": text_or_chunk}
|
||||
vector: Optional[List[float]] = None
|
||||
try:
|
||||
logger.debug("Requesting embedding for id=%s", identifier)
|
||||
resp = requests.post(self.api_endpoint, json=payload, timeout=3600, headers={"Content-Type": "application/json"})
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if isinstance(data.get("embedding"), list):
|
||||
vector = data["embedding"]
|
||||
logger.debug("Received embedding dim=%d for id=%s", len(vector), identifier)
|
||||
else:
|
||||
logger.error("Invalid response from Ollama: %s", data)
|
||||
except requests.exceptions.Timeout:
|
||||
logger.error("Timeout contacting Ollama at %s", self.api_endpoint)
|
||||
except requests.exceptions.RequestException as exc:
|
||||
logger.error("HTTP error contacting Ollama: %s", exc)
|
||||
except Exception:
|
||||
logger.exception("Unexpected error during embed for id=%s", identifier)
|
||||
return text_or_chunk, vector, identifier
|
||||
Whitespace-only changes.
@@ -0,0 +1,23 @@
|
||||
|
||||
from __future__ import annotations
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
from librarian_vspace.vutils.vector_class import BaseVectorOperator
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class VectorInserter(BaseVectorOperator):
|
||||
"""High-level write helper for embeddings."""
|
||||
|
||||
def insert_vector(self, data: Dict[str, Any]) -> Optional[List[Dict[str, Any]]]:
|
||||
if not self.table:
|
||||
logger.error("Table resolution failed earlier")
|
||||
return None
|
||||
preview = {k: (f"<vector,len={len(v)}>" if k == "embedding" else v) for k, v in data.items()}
|
||||
logger.debug("Insert → %s.%s :: %s", self.schema, self.table, preview)
|
||||
try:
|
||||
resp = self.spc.schema(self.schema).table(self.table).insert(data).execute()
|
||||
return resp.data if isinstance(resp.data, list) else []
|
||||
except Exception:
|
||||
logger.exception("Insert failed for %s", self.table_fqn())
|
||||
return None
|
||||
@@ -0,0 +1,2 @@
|
||||
def hello() -> str:
|
||||
return "Hello from librarian_vspace!"
|
||||
@@ -0,0 +1,264 @@
|
||||
"""Loads vectors from Supabase, reduces dimensions using t-SNE."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import json # Import json for parsing
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.manifold import TSNE
|
||||
|
||||
# Assuming vutils is installed or in the python path
|
||||
try:
|
||||
from librarian_vspace.vutils.vector_class import BaseVectorOperator
|
||||
except ImportError as e:
|
||||
logging.error(f"Failed to import vutils: {e}. Ensure vutils package is installed.")
|
||||
raise
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class VectorLoaderError(Exception):
|
||||
"""Custom exception for loader errors."""
|
||||
pass
|
||||
|
||||
|
||||
class VectorLoader:
|
||||
"""Fetches vectors and applies t-SNE."""
|
||||
|
||||
DEFAULT_TSNE_PARAMS = {
|
||||
"n_components": 3,
|
||||
"perplexity": 30.0, # Adjust based on dataset size (5-50 typically)
|
||||
"n_iter": 1000, # Minimum recommended iterations
|
||||
"learning_rate": "auto", # Usually a good default
|
||||
"init": "pca", # PCA initialization is often faster and more stable
|
||||
"random_state": 42, # For reproducibility
|
||||
"n_jobs": -1, # Use all available CPU cores
|
||||
"verbose": 1, # Log progress (controls scikit-learn's verbosity)
|
||||
}
|
||||
|
||||
def __init__(self, schema: str, function: str, model: str, embedding_column: str = "embedding"):
|
||||
"""
|
||||
Initializes the loader.
|
||||
(Constructor remains the same)
|
||||
"""
|
||||
logger.info(f"Initializing VectorLoader for {schema=}, {function=}, {model=}")
|
||||
try:
|
||||
self.operator = BaseVectorOperator(schema=schema, function=function, model=model)
|
||||
self.embedding_column = embedding_column
|
||||
if not self.operator.table:
|
||||
raise VectorLoaderError("BaseVectorOperator failed to resolve table.")
|
||||
logger.info(f"Target table resolved to: {self.operator.table_fqn()}")
|
||||
except (ImportError, ValueError, RuntimeError) as e:
|
||||
logger.exception("Failed to initialize BaseVectorOperator.")
|
||||
raise VectorLoaderError(f"Failed to initialize BaseVectorOperator: {e}") from e
|
||||
|
||||
|
||||
def _parse_vector_string(self, vector_str: Any) -> Optional[List[float]]:
|
||||
"""Safely parses the string representation of a vector into a list of floats."""
|
||||
if not isinstance(vector_str, str):
|
||||
# If it's already a list (less likely now, but safe check), return it if valid
|
||||
if isinstance(vector_str, list) and all(isinstance(n, (int, float)) for n in vector_str):
|
||||
return vector_str # Assume it's already correctly parsed
|
||||
logger.debug(f"Unexpected type for vector parsing: {type(vector_str)}. Skipping.")
|
||||
return None
|
||||
try:
|
||||
# Use json.loads which correctly handles [...] syntax
|
||||
parsed_list = json.loads(vector_str)
|
||||
if isinstance(parsed_list, list) and all(isinstance(n, (int, float)) for n in parsed_list):
|
||||
return [float(n) for n in parsed_list] # Ensure elements are floats
|
||||
else:
|
||||
logger.warning(f"Parsed vector string '{vector_str[:50]}...' but result is not a list of numbers.")
|
||||
return None
|
||||
except json.JSONDecodeError:
|
||||
logger.warning(f"Failed to JSON decode vector string: '{vector_str[:50]}...'")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error parsing vector string '{vector_str[:50]}...': {e}", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
def fetch_all_vectors(self, limit: Optional[int] = None) -> pd.DataFrame:
|
||||
"""
|
||||
Fetches all vectors and metadata from the resolved table.
|
||||
Parses string representations of vectors into lists.
|
||||
|
||||
Args:
|
||||
limit: Optional limit on the number of rows to fetch (for large tables).
|
||||
|
||||
Returns:
|
||||
A pandas DataFrame with columns like 'file_id', 'chunk', 'embedding' (as list).
|
||||
|
||||
Raises:
|
||||
VectorLoaderError: If fetching fails or no data is found.
|
||||
"""
|
||||
if not self.operator.table or not self.operator.schema:
|
||||
raise VectorLoaderError("Operator not initialized, table name or schema is unknown.")
|
||||
|
||||
table_name = self.operator.table
|
||||
schema_name = self.operator.schema
|
||||
select_columns = f"file_id, chunk, {self.embedding_column}"
|
||||
|
||||
logger.info(f"Fetching data from {schema_name}.{table_name} (columns: {select_columns})...")
|
||||
try:
|
||||
query = self.operator.spc.schema(schema_name).table(table_name).select(select_columns)
|
||||
if limit:
|
||||
logger.info(f"Applying limit: {limit}")
|
||||
query = query.limit(limit)
|
||||
response = query.execute()
|
||||
|
||||
if not response.data:
|
||||
logger.warning(f"No data found in table {self.operator.table_fqn()}.")
|
||||
return pd.DataFrame(columns=['file_id', 'chunk', self.embedding_column])
|
||||
|
||||
logger.info(f"Fetched {len(response.data)} rows.")
|
||||
df = pd.DataFrame(response.data)
|
||||
|
||||
# --- FIX: Parse the embedding string into a list ---
|
||||
logger.info(f"Parsing string representation in '{self.embedding_column}' column...")
|
||||
parsed_embeddings = df[self.embedding_column].apply(self._parse_vector_string)
|
||||
# Overwrite the original string column with the parsed list (or None if parsing failed)
|
||||
df[self.embedding_column] = parsed_embeddings
|
||||
logger.debug(f"Sample '{self.embedding_column}' data after parsing (first 5 rows):\n{df[[self.embedding_column]].head()}")
|
||||
# --- END FIX ---
|
||||
|
||||
|
||||
# === Enhanced Debugging for Embedding Column (Now checks the parsed list) ===
|
||||
logger.info(f"Checking validity of parsed '{self.embedding_column}' column...")
|
||||
if self.embedding_column not in df.columns:
|
||||
raise VectorLoaderError(f"Required embedding column '{self.embedding_column}' missing after processing.")
|
||||
|
||||
# 1. Check for NULLs (includes rows where parsing failed and returned None)
|
||||
initial_count = len(df)
|
||||
null_mask = df[self.embedding_column].isnull()
|
||||
null_count = null_mask.sum()
|
||||
if null_count > 0:
|
||||
logger.warning(f"Found {null_count} rows with NULL or unparsable vectors in '{self.embedding_column}'.")
|
||||
|
||||
df_no_nulls = df.dropna(subset=[self.embedding_column])
|
||||
count_after_null_drop = len(df_no_nulls)
|
||||
logger.debug(f"{count_after_null_drop} rows remaining after dropping NULLs/unparsable.")
|
||||
|
||||
# 2. Check for non-empty list type (This check might be slightly redundant now if parsing worked, but keep for safety)
|
||||
if not df_no_nulls.empty:
|
||||
def is_valid_list(x):
|
||||
# Check should pass if parsing was successful
|
||||
return isinstance(x, list) and len(x) > 0
|
||||
|
||||
valid_list_mask = df_no_nulls[self.embedding_column].apply(is_valid_list)
|
||||
invalid_list_count = len(df_no_nulls) - valid_list_mask.sum()
|
||||
|
||||
if invalid_list_count > 0:
|
||||
# This indicates an issue with the parsing logic or unexpected data format
|
||||
logger.error(f"Found {invalid_list_count} rows where '{self.embedding_column}' is not a non-empty list *after parsing*. This should not happen.")
|
||||
invalid_entries = df_no_nulls[~valid_list_mask][self.embedding_column]
|
||||
for i, entry in enumerate(invalid_entries.head(5)):
|
||||
logger.debug(f" Problematic entry example {i+1}: Type={type(entry)}, Value='{str(entry)[:100]}...'")
|
||||
|
||||
df_filtered = df_no_nulls[valid_list_mask].copy()
|
||||
else:
|
||||
df_filtered = df_no_nulls
|
||||
|
||||
final_count = len(df_filtered)
|
||||
# === End Enhanced Debugging ===
|
||||
|
||||
if final_count < initial_count:
|
||||
logger.warning(f"Filtered out {initial_count - final_count} rows total due to missing/invalid '{self.embedding_column}'.")
|
||||
|
||||
if df_filtered.empty:
|
||||
logger.warning(f"No valid embedding data found after filtering. Check data in table {self.operator.table_fqn()} and parsing logic.")
|
||||
return pd.DataFrame(columns=['file_id', 'chunk', self.embedding_column])
|
||||
|
||||
logger.info(f"Proceeding with {final_count} valid rows.")
|
||||
|
||||
# Validate and potentially add placeholder metadata columns AFTER filtering
|
||||
if 'file_id' not in df_filtered.columns:
|
||||
logger.warning("'file_id' column missing, using index instead.")
|
||||
df_filtered['file_id'] = df_filtered.index
|
||||
if 'chunk' not in df_filtered.columns:
|
||||
logger.warning("'chunk' column missing, hover text will be limited.")
|
||||
df_filtered['chunk'] = "N/A"
|
||||
|
||||
return df_filtered
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Failed to fetch data from {self.operator.table_fqn()}.")
|
||||
if 'relation' in str(e) and 'does not exist' in str(e):
|
||||
raise VectorLoaderError(f"Table/Relation not found error: {e}. Check schema/table name and permissions.") from e
|
||||
else:
|
||||
raise VectorLoaderError(f"Database query failed: {e}") from e
|
||||
|
||||
# reduce_dimensions and load_and_reduce methods remain the same as the previous version
|
||||
# (they expect df with a valid list in the embedding column)
|
||||
|
||||
def reduce_dimensions(self, df: pd.DataFrame, tsne_params: Optional[Dict[str, Any]] = None) -> pd.DataFrame:
|
||||
"""
|
||||
Applies t-SNE to reduce embedding dimensions to 3D.
|
||||
(Code remains the same as previous correct version)
|
||||
"""
|
||||
if df.empty:
|
||||
logger.warning("Input DataFrame for reduce_dimensions is empty. Returning empty DataFrame.")
|
||||
empty_df_with_cols = df.copy()
|
||||
for col in ['x', 'y', 'z']:
|
||||
if col not in empty_df_with_cols:
|
||||
empty_df_with_cols[col] = pd.Series(dtype=float)
|
||||
return empty_df_with_cols
|
||||
|
||||
if self.embedding_column not in df.columns:
|
||||
raise VectorLoaderError(f"Embedding column '{self.embedding_column}' missing in DataFrame passed to reduce_dimensions.")
|
||||
|
||||
try:
|
||||
embeddings = np.array(df[self.embedding_column].tolist(), dtype=float)
|
||||
except ValueError as ve:
|
||||
logger.error(f"Failed to convert embedding list to numeric numpy array: {ve}")
|
||||
raise VectorLoaderError(f"Data in '{self.embedding_column}' could not be converted to numeric vectors.") from ve
|
||||
|
||||
if embeddings.ndim != 2:
|
||||
raise VectorLoaderError(f"Embedding data has unexpected dimensions: {embeddings.ndim} (expected 2). Shape: {embeddings.shape}")
|
||||
|
||||
n_samples = embeddings.shape[0]
|
||||
|
||||
if n_samples < 2:
|
||||
logger.warning(f"Found only {n_samples} valid vector(s). t-SNE requires at least 2. Assigning default 3D coordinates.")
|
||||
default_coords = [[0.0, 0.0, 0.0]] * n_samples
|
||||
df[['x', 'y', 'z']] = default_coords
|
||||
return df
|
||||
|
||||
logger.info(f"Applying t-SNE to {n_samples} vectors of dimension {embeddings.shape[1]}...")
|
||||
|
||||
current_tsne_params = self.DEFAULT_TSNE_PARAMS.copy()
|
||||
if tsne_params:
|
||||
current_tsne_params.update(tsne_params)
|
||||
logger.info(f"Using custom t-SNE params: {tsne_params}")
|
||||
|
||||
if n_samples <= current_tsne_params['perplexity']:
|
||||
new_perplexity = max(5.0, float(n_samples - 1))
|
||||
logger.warning(f"Adjusting t-SNE perplexity from {current_tsne_params['perplexity']:.1f} "
|
||||
f"to {new_perplexity:.1f} due to low sample count ({n_samples}).")
|
||||
current_tsne_params['perplexity'] = new_perplexity
|
||||
|
||||
if n_samples * embeddings.shape[1] > 100000 and current_tsne_params['n_iter'] < 1000:
|
||||
logger.warning(f"Dataset size seems large, increasing t-SNE n_iter from {current_tsne_params['n_iter']} to 1000 for better convergence.")
|
||||
current_tsne_params['n_iter'] = 1000
|
||||
|
||||
try:
|
||||
logger.debug(f"Final t-SNE parameters: {current_tsne_params}")
|
||||
tsne = TSNE(**current_tsne_params)
|
||||
reduced_embeddings = tsne.fit_transform(embeddings)
|
||||
|
||||
df[['x', 'y', 'z']] = reduced_embeddings
|
||||
logger.info("t-SNE reduction complete.")
|
||||
return df
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("t-SNE dimensionality reduction failed.")
|
||||
raise VectorLoaderError(f"t-SNE failed: {e}") from e
|
||||
|
||||
|
||||
def load_and_reduce(self, limit: Optional[int] = None, tsne_params: Optional[Dict[str, Any]] = None) -> pd.DataFrame:
|
||||
"""Orchestrates fetching vectors and reducing dimensions."""
|
||||
logger.info("Starting vector load and reduction process...")
|
||||
df_raw_filtered = self.fetch_all_vectors(limit=limit)
|
||||
df_reduced = self.reduce_dimensions(df_raw_filtered, tsne_params=tsne_params)
|
||||
logger.info("Vector load and reduction process finished.")
|
||||
return df_reduced
|
||||
Whitespace-only changes.
@@ -0,0 +1,776 @@
|
||||
# --- START OF FILE visualizer.py ---
|
||||
|
||||
"""Dash/Plotly based 3D visualizer for vector embeddings with tabs, clustering, filtering, and centroid click interaction."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from io import StringIO
|
||||
from typing import Any, Dict, List, Optional, Tuple
|
||||
|
||||
import dash
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import plotly.graph_objects as go
|
||||
import plotly.express as px
|
||||
from dash import dcc, html, ctx # Import ctx
|
||||
from dash.dependencies import Input, Output, State
|
||||
from dash.exceptions import PreventUpdate
|
||||
from sklearn.cluster import KMeans
|
||||
|
||||
# --- Imports ---
|
||||
try:
|
||||
from librarian_vspace.vecembed.embedding_generator import EmbeddingGenerator
|
||||
except ImportError as e:
|
||||
logging.error(f"Import vecembed failed: {e}. Using Dummy.")
|
||||
|
||||
|
||||
# Define dummy class correctly indented
|
||||
class EmbeddingGenerator:
|
||||
"""Dummy class if vecembed import fails."""
|
||||
|
||||
def generate_embedding(*args, **kwargs) -> Tuple[
|
||||
str, None, Any]: # Match expected output type Optional[List[float]]
|
||||
logging.error("Dummy EmbeddingGenerator called.")
|
||||
text_to_embed = kwargs.get("text_to_embed", args[3] if len(args) > 3 else "unknown")
|
||||
identifier = kwargs.get("identifier", args[4] if len(args) > 4 else "unknown")
|
||||
logger.debug(f"Dummy generate_embedding called for text='{text_to_embed}', id='{identifier}'")
|
||||
# Return None for the vector part to match expected type
|
||||
return text_to_embed, None, identifier
|
||||
|
||||
try:
|
||||
from librarian_vspace.vutils.vector_query_loader import VectorQueryLoader as VectorLoader, VectorQueryLoaderError as VectorLoaderError
|
||||
except ImportError as e:
|
||||
logging.error(f"Import loader failed: {e}. Using Dummy.")
|
||||
|
||||
|
||||
# Define dummy classes correctly indented
|
||||
class VectorLoader:
|
||||
"""Dummy class if loader import fails."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
logging.error("Dummy VectorLoader initialized.")
|
||||
pass
|
||||
|
||||
def load_and_reduce(self, *args, **kwargs) -> pd.DataFrame:
|
||||
logging.error("Dummy VectorLoader load_and_reduce called.")
|
||||
return pd.DataFrame() # Return empty DataFrame
|
||||
|
||||
|
||||
class VectorLoaderError(Exception):
|
||||
"""Dummy exception if loader import fails."""
|
||||
pass
|
||||
# --- End Imports ---
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
DEFAULT_N_CLUSTERS = 8
|
||||
|
||||
# Opacity constants
|
||||
OPACITY_DEFAULT = 0.8
|
||||
OPACITY_SEARCH_DIMMED = 0.1 # Reduced from 0.6 to 0.3 for better visual distinction
|
||||
OPACITY_SELECTED_CLUSTER = 0.9
|
||||
|
||||
|
||||
class VectorVisualizer:
|
||||
def __init__(self, initial_data: pd.DataFrame,
|
||||
db_schema: str, db_function: str,
|
||||
interface_name: str, model_name: str,
|
||||
embedding_column: str = "embedding",
|
||||
initial_limit: Optional[int] = None,
|
||||
initial_perplexity: float = 30.0,
|
||||
n_clusters: int = DEFAULT_N_CLUSTERS
|
||||
):
|
||||
required_cols = ['x', 'y', 'z', 'file_id', 'chunk', embedding_column]
|
||||
processed_data_json: Optional[str] = None
|
||||
processed_color_map: Dict = {}
|
||||
processed_original_embeddings: np.ndarray = np.array([])
|
||||
processed_cluster_centroids: Dict[str, List[float]] = {}
|
||||
|
||||
self.embedding_column = embedding_column
|
||||
self.n_clusters = n_clusters
|
||||
self.db_schema = db_schema
|
||||
self.db_function = db_function
|
||||
self.model_name = model_name
|
||||
self.limit = initial_limit
|
||||
self.perplexity = initial_perplexity
|
||||
self.interface_name = interface_name
|
||||
# Use the correctly defined EmbeddingGenerator (either real or dummy)
|
||||
self.app = dash.Dash(__name__, suppress_callback_exceptions=True)
|
||||
self.embedding_generator = EmbeddingGenerator() # Instantiated here
|
||||
|
||||
if initial_data.empty or not all(col in initial_data.columns for col in required_cols):
|
||||
logger.warning("Initial DataFrame empty/invalid.")
|
||||
base_cols = required_cols + ['cluster', 'hover_text']
|
||||
initial_df_processed = pd.DataFrame(columns=base_cols)
|
||||
else:
|
||||
try:
|
||||
logger.info("Processing initial data...")
|
||||
df_copy = initial_data.copy()
|
||||
df_after_kmeans, kmeans_color_map = self._run_kmeans(df_copy, self.n_clusters)
|
||||
if not isinstance(df_after_kmeans, pd.DataFrame): raise TypeError("KMeans failed.")
|
||||
processed_color_map = kmeans_color_map
|
||||
df_after_prepare = self._prepare_plot_data(df_after_kmeans)
|
||||
if not isinstance(df_after_prepare, pd.DataFrame): raise TypeError("Prep data failed.")
|
||||
initial_df_processed = df_after_prepare
|
||||
if not initial_df_processed.empty and all(
|
||||
c in initial_df_processed for c in ['x', 'y', 'z', 'cluster']):
|
||||
processed_cluster_centroids = self._calculate_centroids(initial_df_processed)
|
||||
else:
|
||||
logger.warning("Could not calculate initial centroids.")
|
||||
if not initial_df_processed.empty:
|
||||
processed_data_json = initial_df_processed.to_json(date_format='iso', orient='split')
|
||||
else:
|
||||
logger.warning("DataFrame empty after processing.")
|
||||
if not initial_df_processed.empty and self.embedding_column in initial_df_processed.columns:
|
||||
try:
|
||||
emb = initial_df_processed[self.embedding_column].iloc[0]
|
||||
if isinstance(emb, np.ndarray):
|
||||
processed_original_embeddings = np.stack(initial_df_processed[self.embedding_column].values)
|
||||
elif isinstance(emb, list):
|
||||
processed_original_embeddings = np.array(
|
||||
initial_df_processed[self.embedding_column].tolist(), dtype=float)
|
||||
else:
|
||||
raise TypeError("Unsupported embedding type.")
|
||||
except Exception as emb_err:
|
||||
logger.error(f"Embed processing error: {emb_err}"); processed_original_embeddings = np.array([])
|
||||
else:
|
||||
logger.warning("Could not extract original embeddings.")
|
||||
except Exception as e:
|
||||
logger.error(f"Initial processing error: {e}", exc_info=True)
|
||||
processed_data_json, processed_color_map, processed_original_embeddings, processed_cluster_centroids = None, {}, np.array(
|
||||
[]), {}
|
||||
initial_df_processed = pd.DataFrame()
|
||||
|
||||
self.initial_data_json = processed_data_json
|
||||
self.initial_cluster_color_map = processed_color_map
|
||||
self.initial_cluster_centroids = processed_cluster_centroids
|
||||
self.original_embeddings = processed_original_embeddings
|
||||
# Determine slider limits and elbow‑based default
|
||||
self.max_clusters = max(1, len(initial_data))
|
||||
try:
|
||||
self.optimal_clusters = self._estimate_optimal_clusters(processed_original_embeddings,
|
||||
max_k=min(10, self.max_clusters))
|
||||
except Exception:
|
||||
self.optimal_clusters = self.n_clusters
|
||||
# Use elbow result as the current cluster count
|
||||
self.n_clusters = self.optimal_clusters
|
||||
self._build_layout();
|
||||
self._register_callbacks()
|
||||
|
||||
def _run_kmeans(self, df: pd.DataFrame, n_clusters: int) -> Tuple[pd.DataFrame, Dict[str, str]]:
|
||||
"""Runs K-Means, assigns string cluster labels."""
|
||||
default_map = {"-1": "grey"}
|
||||
if df.empty or self.embedding_column not in df.columns: df['cluster'] = "-1"; return df, default_map
|
||||
try:
|
||||
emb_col = df[self.embedding_column]
|
||||
if isinstance(emb_col.iloc[0], np.ndarray):
|
||||
embeddings = np.stack(emb_col.values)
|
||||
elif isinstance(emb_col.iloc[0], list):
|
||||
embeddings = np.array(emb_col.tolist(), dtype=float)
|
||||
else:
|
||||
raise TypeError("Unsupported embedding type.")
|
||||
if embeddings.ndim != 2: raise ValueError("Embeddings must be 2D.")
|
||||
|
||||
eff_clusters = min(n_clusters, embeddings.shape[0])
|
||||
if embeddings.shape[0] < 2 or eff_clusters < 1:
|
||||
lbl = "0" if embeddings.shape[0] > 0 else "-1";
|
||||
df['cluster'] = lbl
|
||||
colors = px.colors.qualitative.Plotly;
|
||||
return df, {lbl: colors[0 % len(colors)]} if lbl == "0" else default_map
|
||||
if eff_clusters == 1: df['cluster'] = "0"; colors = px.colors.qualitative.Plotly; return df, {
|
||||
"0": colors[0 % len(colors)]}
|
||||
|
||||
kmeans = KMeans(n_clusters=eff_clusters, random_state=42, n_init='auto')
|
||||
df['cluster'] = kmeans.fit_predict(embeddings).astype(str)
|
||||
unique_labels = sorted(df['cluster'].unique())
|
||||
colors = px.colors.qualitative.Plotly
|
||||
color_map = {lbl: colors[i % len(colors)] for i, lbl in enumerate(unique_labels)}
|
||||
return df, color_map
|
||||
except (TypeError, ValueError) as e:
|
||||
logger.error(f"KMeans input error: {e}"); df['cluster'] = "-1"; return df, default_map
|
||||
except Exception as e:
|
||||
logger.exception("KMeans failed."); df['cluster'] = "-1"; return df, default_map
|
||||
|
||||
def _prepare_plot_data(self, df: pd.DataFrame) -> pd.DataFrame:
|
||||
"""Prepares hover text."""
|
||||
if df.empty: return df
|
||||
if 'cluster' not in df.columns: df['cluster'] = 'N/A'
|
||||
df_copy = df.copy()
|
||||
|
||||
def gen_hover(row):
|
||||
try:
|
||||
return f"ID: {row.get('file_id', 'N/A')}<br>Cluster: {str(row.get('cluster', 'N/A'))}<br>Chunk: {str(row.get('chunk', ''))[:200]}{'...' if len(str(row.get('chunk', ''))) > 200 else ''}"
|
||||
except Exception:
|
||||
return "Hover gen error"
|
||||
|
||||
try:
|
||||
df_copy['hover_text'] = df_copy.apply(gen_hover, axis=1); return df_copy
|
||||
except Exception as e:
|
||||
logger.error(f"Hover gen failed: {e}"); return df
|
||||
|
||||
def _calculate_centroids(self, df: pd.DataFrame) -> Dict[str, List[float]]:
|
||||
"""Calculates 3D centroids."""
|
||||
centroids = {}
|
||||
required = ['x', 'y', 'z', 'cluster'];
|
||||
numeric_cols = ['x', 'y', 'z']
|
||||
if df.empty or not all(col in df.columns for col in required): return centroids
|
||||
df_copy = df.copy();
|
||||
df_copy['cluster'] = df['cluster'].astype(str)
|
||||
for col in numeric_cols:
|
||||
if not pd.api.types.is_numeric_dtype(df_copy[col]):
|
||||
try:
|
||||
df_copy[col] = pd.to_numeric(df_copy[col], errors='coerce')
|
||||
except Exception:
|
||||
logger.error(f"Centroid calc: conv error '{col}'"); return {}
|
||||
if df_copy[col].isnull().any(): logger.warning(f"Centroid calc: NaNs in '{col}'")
|
||||
try:
|
||||
# Calculate mean, drop rows where ALL numeric_cols are NaN, then drop rows where the resulting mean is NaN
|
||||
centroid_data = df_copy.dropna(subset=numeric_cols, how='all').groupby('cluster')[
|
||||
numeric_cols].mean().dropna()
|
||||
return {str(idx): row.tolist() for idx, row in centroid_data.iterrows()}
|
||||
except Exception as e:
|
||||
logger.exception("Centroid calc failed."); return {}
|
||||
|
||||
def _create_base_figure(self) -> go.Figure:
|
||||
"""Creates base Plotly figure."""
|
||||
fig = go.Figure()
|
||||
fig.update_layout(title='3D t-SNE', margin=dict(l=0, r=0, b=0, t=40),
|
||||
scene_camera_eye=dict(x=1.5, y=1.5, z=0.5),
|
||||
scene=dict(xaxis_title='TSNE-1', yaxis_title='TSNE-2', zaxis_title='TSNE-3',
|
||||
aspectmode='data'),
|
||||
legend=dict(yanchor="top", y=0.99, xanchor="left", x=0.01, bgcolor='rgba(255,255,255,0.7)'),
|
||||
hovermode='closest')
|
||||
return fig
|
||||
|
||||
def _build_layout(self) -> None:
|
||||
"""Builds the Dash layout."""
|
||||
self.app.layout = html.Div([
|
||||
dcc.Store(id='stored-data', data=self.initial_data_json),
|
||||
dcc.Store(id='cluster-color-map-store', data=self.initial_cluster_color_map),
|
||||
dcc.Store(id='cluster-centroids-store', data=self.initial_cluster_centroids),
|
||||
dcc.Store(id='search-results-store', data=None),
|
||||
dcc.Store(id='selected-cluster-store', data=None), # Store for click state
|
||||
html.H1("Vector Embedding Visualizer"),
|
||||
dcc.Tabs(id="main-tabs", value='tab-vis', children=[
|
||||
dcc.Tab(label='Visualization', value='tab-vis', children=[
|
||||
html.Div([ # Controls
|
||||
html.Div(
|
||||
[html.Button('Reload Data', id='reload-button', n_clicks=0, style={'marginRight': '10px'}),
|
||||
dcc.Input(id='search-input', type='text', placeholder='Search term...', debounce=True,
|
||||
style={'width': '40%', 'marginRight': '5px'}),
|
||||
html.Button('Search', id='search-button', n_clicks=0)],
|
||||
style={'padding': '10px', 'display': 'flex'}),
|
||||
html.Div([html.Label("Similarity:", style={'marginRight': '10px'}),
|
||||
dcc.Slider(id='similarity-slider', min=0, max=1, step=0.01, value=0.0,
|
||||
marks={i / 10: f'{i / 10:.1f}' for i in range(11)},
|
||||
tooltip={"placement": "bottom", "always_visible": True}, disabled=True)],
|
||||
id='slider-container', style={'display': 'none', 'padding': '10px 20px'}),
|
||||
html.Div([
|
||||
html.Label("Clusters:", style={'marginRight': '10px'}),
|
||||
dcc.Slider(
|
||||
id='cluster-slider',
|
||||
min=1,
|
||||
max=self.max_clusters,
|
||||
step=1,
|
||||
value=self.optimal_clusters,
|
||||
marks=self._cluster_marks(),
|
||||
tooltip={'placement': 'bottom', 'always_visible': True}
|
||||
)
|
||||
], style={'padding': '10px 20px'}),
|
||||
html.Div(id='status-output', style={'padding': '10px', 'color': 'blue', 'minHeight': '20px'}),
|
||||
dcc.Loading(id="loading-graph", type="circle",
|
||||
children=dcc.Graph(id='vector-graph', style={'height': '70vh'}))
|
||||
])
|
||||
]),
|
||||
dcc.Tab(label='Settings', value='tab-settings', children=[
|
||||
html.Div([html.H3("Settings"), html.Div([html.Label("Marker Size:", style={'marginRight': '10px'}),
|
||||
dcc.Slider(id='size-slider', min=1, max=15, step=1,
|
||||
value=4,
|
||||
marks={i: str(i) for i in range(1, 16)},
|
||||
tooltip={'placement': 'bottom',
|
||||
'always_visible': True})],
|
||||
style={'padding': '10px 20px'})], style={'padding': '20px'})
|
||||
]),
|
||||
]),
|
||||
])
|
||||
|
||||
def _cosine_similarity(self, vec1: np.ndarray, vec2: np.ndarray) -> np.ndarray | float:
|
||||
"""Calculates cosine similarity."""
|
||||
if not isinstance(vec1, np.ndarray): vec1 = np.array(vec1, dtype=float)
|
||||
if not isinstance(vec2, np.ndarray): vec2 = np.array(vec2, dtype=float)
|
||||
if vec1.ndim == 1: vec1 = vec1.reshape(1, -1)
|
||||
if vec2.ndim == 1: vec2 = vec2.reshape(1, -1)
|
||||
if vec1.shape[1] != vec2.shape[1]: raise ValueError("Vector dimension mismatch")
|
||||
norm1 = np.linalg.norm(vec1, axis=1, keepdims=True);
|
||||
norm2 = np.linalg.norm(vec2, axis=1, keepdims=True)
|
||||
z1 = (norm1 == 0).flatten();
|
||||
z2 = (norm2 == 0).flatten()
|
||||
# Handle potential division by zero for zero vectors
|
||||
norm1[z1] = 1.0;
|
||||
norm2[z2] = 1.0
|
||||
sim = np.dot(vec1 / norm1, (vec2 / norm2).T)
|
||||
# Ensure zero vectors result in zero similarity
|
||||
if np.any(z1): sim[:, :] = 0.0;
|
||||
sim[:, z2] = 0.0
|
||||
sim = np.clip(sim, -1.0, 1.0)
|
||||
return sim.item() if sim.size == 1 else sim.flatten()
|
||||
|
||||
def _find_neighbors(self, search_vector: List[float], k: int = 10) -> Optional[Tuple[np.ndarray, np.ndarray]]:
|
||||
"""Finds k nearest neighbors."""
|
||||
if not search_vector or not isinstance(search_vector, list): return None
|
||||
if self.original_embeddings is None or self.original_embeddings.size == 0: return None
|
||||
try:
|
||||
vec = np.array(search_vector, dtype=float)
|
||||
if vec.ndim != 1: raise ValueError("Search vector != 1D.")
|
||||
if self.original_embeddings.ndim != 2: raise ValueError("Embeddings != 2D.")
|
||||
if self.original_embeddings.shape[1] != vec.shape[0]: raise ValueError("Dimension mismatch.")
|
||||
sims = self._cosine_similarity(vec, self.original_embeddings)
|
||||
if not isinstance(sims, np.ndarray) or sims.ndim != 1 or sims.shape[0] != self.original_embeddings.shape[
|
||||
0]: raise TypeError("Similarity calc failed.")
|
||||
k_actual = min(k, len(sims));
|
||||
if k_actual <= 0: return None
|
||||
idx = np.argpartition(sims, -k_actual)[-k_actual:] # Get indices of top k
|
||||
sorted_idx = idx[np.argsort(sims[idx])][::-1] # Sort top k indices by similarity
|
||||
return sorted_idx, sims[sorted_idx]
|
||||
except (ValueError, TypeError) as e:
|
||||
logger.error(f"Neighbor input error: {e}"); return None
|
||||
except Exception as e:
|
||||
logger.exception(f"Neighbor search error: {e}"); return None
|
||||
|
||||
# --- Callbacks ---
|
||||
def _register_callbacks(self) -> None:
|
||||
"""Sets up Dash callbacks."""
|
||||
|
||||
# --- Callback 1: Reload Button ---
|
||||
@self.app.callback(
|
||||
Output('stored-data', 'data', allow_duplicate=True), Output('cluster-color-map-store', 'data'),
|
||||
Output('cluster-centroids-store', 'data'),
|
||||
Output('status-output', 'children'), Output('search-results-store', 'data', allow_duplicate=True),
|
||||
Output('selected-cluster-store', 'data', allow_duplicate=True),
|
||||
Input('reload-button', 'n_clicks'), prevent_initial_call=True)
|
||||
def handle_reload(n_clicks: int) -> Tuple[Optional[str], Dict, Dict, str, None, None]:
|
||||
if n_clicks == 0: raise PreventUpdate
|
||||
logger.info("Reload triggered...")
|
||||
status = "Reloading...";
|
||||
color_map, centroids, data_json = {}, {}, None;
|
||||
self.original_embeddings = np.array([])
|
||||
try:
|
||||
# Ensure VectorLoader is properly imported or defined (dummy used if import fails)
|
||||
loader = VectorLoader(self.db_schema, self.db_function, self.model_name, self.embedding_column)
|
||||
reduced_data = loader.load_and_reduce(limit=self.limit, tsne_params={"perplexity": self.perplexity})
|
||||
if not isinstance(reduced_data, pd.DataFrame) or reduced_data.empty: raise VectorLoaderError("No data.")
|
||||
|
||||
df_clustered, color_map = self._run_kmeans(reduced_data.copy(), self.n_clusters)
|
||||
if not isinstance(df_clustered, pd.DataFrame): raise TypeError(
|
||||
"KMeans failed post-reload.") # Add check
|
||||
df_final = self._prepare_plot_data(df_clustered)
|
||||
if not isinstance(df_final, pd.DataFrame): raise TypeError(
|
||||
"Prepare plot failed post-reload.") # Add check
|
||||
|
||||
if not df_final.empty and all(c in df_final for c in ['x', 'y', 'z', 'cluster']):
|
||||
centroids = self._calculate_centroids(df_final)
|
||||
else:
|
||||
logger.warning("Could not calculate centroids after reload (missing cols or empty).")
|
||||
|
||||
if not reduced_data.empty and self.embedding_column in reduced_data.columns:
|
||||
try:
|
||||
emb_col = reduced_data[self.embedding_column]
|
||||
# Check type of first element before processing
|
||||
if not emb_col.empty:
|
||||
first_emb = emb_col.iloc[0]
|
||||
if isinstance(first_emb, np.ndarray):
|
||||
self.original_embeddings = np.stack(emb_col.values)
|
||||
elif isinstance(first_emb, list):
|
||||
self.original_embeddings = np.array(emb_col.tolist(), dtype=float)
|
||||
else:
|
||||
raise TypeError(f"Unsupported reloaded embed type: {type(first_emb)}")
|
||||
logger.info(f"Stored reloaded embeddings (shape: {self.original_embeddings.shape}).")
|
||||
else:
|
||||
logger.warning("Embedding column empty during reload storage.")
|
||||
except Exception as e:
|
||||
logger.error(f"Store embed fail: {e}"); self.original_embeddings = np.array([])
|
||||
else:
|
||||
logger.warning("Embedding column missing or df empty during reload storage.")
|
||||
|
||||
if not df_final.empty:
|
||||
data_json = df_final.to_json(date_format='iso',
|
||||
orient='split'); status = f"Reloaded ({len(df_final)} pts)."
|
||||
else:
|
||||
status = "Warning: Reload empty post-process."
|
||||
except (VectorLoaderError, TypeError, Exception) as e:
|
||||
logger.exception(
|
||||
f"Reload error: {e}"); status = f"Error: {e}"; data_json, color_map, centroids = None, {}, {}; self.original_embeddings = np.array(
|
||||
[])
|
||||
return data_json, color_map, centroids, status, None, None
|
||||
|
||||
# --- Callback 1 b: Cluster‑count Slider ---
|
||||
@self.app.callback(
|
||||
Output('stored-data', 'data', allow_duplicate=True),
|
||||
Output('cluster-color-map-store', 'data', allow_duplicate=True),
|
||||
Output('cluster-centroids-store', 'data', allow_duplicate=True),
|
||||
Output('status-output', 'children', allow_duplicate=True),
|
||||
Input('cluster-slider', 'value'),
|
||||
State('stored-data', 'data'),
|
||||
prevent_initial_call=True
|
||||
)
|
||||
def update_n_clusters(k: int, stored_json: str):
|
||||
if not stored_json:
|
||||
raise PreventUpdate
|
||||
|
||||
# Update the visualizer state
|
||||
self.n_clusters = k
|
||||
|
||||
try:
|
||||
df = pd.read_json(StringIO(stored_json), orient='split')
|
||||
df, color_map = self._run_kmeans(df, k)
|
||||
df = self._prepare_plot_data(df)
|
||||
centroids = self._calculate_centroids(df)
|
||||
|
||||
status = f"Cluster count set to {k}."
|
||||
return (df.to_json(date_format='iso', orient='split'),
|
||||
color_map,
|
||||
centroids,
|
||||
status)
|
||||
except Exception as err:
|
||||
logger.error(f"Clustering update failed: {err}")
|
||||
raise PreventUpdate
|
||||
|
||||
# --- Callback 2: Search Button ---
|
||||
@self.app.callback(
|
||||
Output('search-results-store', 'data', allow_duplicate=True),
|
||||
Output('status-output', 'children', allow_duplicate=True),
|
||||
Input('search-button', 'n_clicks'), State('search-input', 'value'), prevent_initial_call=True)
|
||||
def handle_search(n_clicks: int, term: str) -> Tuple[Optional[Dict], str]:
|
||||
if n_clicks == 0 or not term: return None, "Enter search term."
|
||||
logger.info(f"Search: '{term}'");
|
||||
status = f"Embedding '{term}'..."
|
||||
try:
|
||||
if self.original_embeddings is None or self.original_embeddings.size == 0: return None, "Error: No data."
|
||||
_, vec, _ = self.embedding_generator.generate_embedding(self.interface_name, self.model_name, term,
|
||||
"search")
|
||||
if vec is None: return None, f"Error: Embed failed."
|
||||
status = f"Finding neighbors...";
|
||||
neighbors = self._find_neighbors(vec, k=20)
|
||||
if neighbors is None: return None, f"No neighbors found."
|
||||
idx, sims = neighbors;
|
||||
results = {"indices": idx.tolist(), "similarities": sims.tolist(), "term": term}
|
||||
status = f"Found {len(idx)} neighbors.";
|
||||
return results, status
|
||||
except Exception as e:
|
||||
logger.exception("Search error."); return None, f"Error: {e}"
|
||||
|
||||
# --- Callback 3: Slider Visibility ---
|
||||
@self.app.callback(
|
||||
Output('slider-container', 'style'), Output('similarity-slider', 'disabled'),
|
||||
Output('similarity-slider', 'value'),
|
||||
Input('search-results-store', 'data'), prevent_initial_call=True)
|
||||
def update_slider_visibility(res: Optional[Dict]) -> Tuple[Dict, bool, float]:
|
||||
show = res and isinstance(res, dict) and "indices" in res
|
||||
style = {'display': 'block' if show else 'none', 'padding': '10px 20px'}
|
||||
return style, not show, 0.0
|
||||
|
||||
# --- Callback 4: Graph Update (Main Logic with clickData fix and logging) ---
|
||||
@self.app.callback(
|
||||
Output('vector-graph', 'figure'), Output('selected-cluster-store', 'data'),
|
||||
Output('status-output', 'children', allow_duplicate=True),
|
||||
Input('stored-data', 'data'), Input('cluster-color-map-store', 'data'),
|
||||
Input('cluster-centroids-store', 'data'),
|
||||
Input('search-results-store', 'data'), Input('similarity-slider', 'value'), Input('size-slider', 'value'),
|
||||
Input('vector-graph', 'clickData'), # Input for clicks
|
||||
State('selected-cluster-store', 'data'), # Get current selection
|
||||
prevent_initial_call='initial_duplicate' # Allow initial run
|
||||
)
|
||||
def update_graph(stored_data_json: Optional[str], cluster_color_map: Optional[Dict],
|
||||
cluster_centroids: Optional[Dict[str, List[float]]],
|
||||
search_results: Optional[Dict], similarity_threshold: float, size_value: int,
|
||||
click_data: Optional[Dict],
|
||||
current_selected_cluster: Optional[str]) -> Tuple[go.Figure, Optional[str], str]:
|
||||
|
||||
fig = self._create_base_figure();
|
||||
status_msg = "";
|
||||
new_selected_cluster = current_selected_cluster
|
||||
trigger = ctx.triggered_id if ctx.triggered else "Initial"
|
||||
logger.debug(f"--- Graph Update | Trigger: {trigger} | CurrentSel: {current_selected_cluster} ---")
|
||||
|
||||
# --- Data Load & Validation ---
|
||||
if not stored_data_json: return fig, None, "No data."
|
||||
try:
|
||||
df = pd.read_json(StringIO(stored_data_json), orient='split')
|
||||
if df.empty: return fig, None, "Empty data."
|
||||
required = ['x', 'y', 'z', 'cluster', 'hover_text'];
|
||||
assert all(col in df.columns for col in required)
|
||||
df['cluster'] = df['cluster'].astype(str)
|
||||
color_map = cluster_color_map if isinstance(cluster_color_map, dict) else {}
|
||||
centroids = cluster_centroids if isinstance(cluster_centroids, dict) else {}
|
||||
if not color_map:
|
||||
logger.warning("Missing color map, generating default.")
|
||||
unique_clusters = df['cluster'].unique();
|
||||
colors = px.colors.qualitative.Plotly
|
||||
color_map = {str(c): colors[i % len(colors)] for i, c in enumerate(unique_clusters)} or {
|
||||
'0': 'grey'}
|
||||
|
||||
# Calculate overall data range (handle potential NaNs/Infs in full data)
|
||||
df_finite = df[['x', 'y', 'z']].replace([np.inf, -np.inf], np.nan).dropna()
|
||||
if not df_finite.empty:
|
||||
overall_x_min, overall_x_max = df_finite['x'].min(), df_finite['x'].max()
|
||||
overall_y_min, overall_y_max = df_finite['y'].min(), df_finite['y'].max()
|
||||
overall_z_min, overall_z_max = df_finite['z'].min(), df_finite['z'].max()
|
||||
logger.debug(
|
||||
f"Overall Finite Range: X=[{overall_x_min:.2f}, {overall_x_max:.2f}], Y=[{overall_y_min:.2f}, {overall_y_max:.2f}], Z=[{overall_z_min:.2f}, {overall_z_max:.2f}]")
|
||||
else:
|
||||
logger.warning("No finite data points found in the dataset to calculate overall range.")
|
||||
overall_x_min, overall_x_max = -10, 10 # Default ranges if no finite data
|
||||
overall_y_min, overall_y_max = -10, 10
|
||||
overall_z_min, overall_z_max = -10, 10
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Graph data error."); return fig, current_selected_cluster, f"Error: {e}"
|
||||
|
||||
# --- Click Processing ---
|
||||
if trigger == 'vector-graph':
|
||||
logger.debug(f"Click Data Received: {click_data}")
|
||||
if click_data and 'points' in click_data and click_data['points']:
|
||||
point_data = click_data['points'][0]
|
||||
clicked_customdata = point_data.get('customdata');
|
||||
clicked_text = point_data.get('text', '')
|
||||
logger.debug(f"Clicked Point Customdata: {clicked_customdata}");
|
||||
logger.debug(f"Clicked Point Text: '{clicked_text}'")
|
||||
is_centroid_click = False;
|
||||
clicked_cluster_id = None
|
||||
if isinstance(clicked_customdata, list) and len(clicked_customdata) > 0: clicked_customdata = \
|
||||
clicked_customdata[0]
|
||||
if isinstance(clicked_customdata, (str, int)): # Accept string or int cluster IDs
|
||||
is_centroid_click = True;
|
||||
clicked_cluster_id = str(clicked_customdata);
|
||||
logger.info(f"Centroid Click Parsed via customdata: Cluster '{clicked_cluster_id}'")
|
||||
elif isinstance(clicked_text, str) and clicked_text.startswith("Centroid: Cluster "):
|
||||
try:
|
||||
clicked_cluster_id = clicked_text.split("Centroid: Cluster ")[
|
||||
1]; is_centroid_click = True; logger.info(
|
||||
f"Centroid Click Parsed via text: Cluster '{clicked_cluster_id}'")
|
||||
except Exception as parse_err:
|
||||
logger.warning(f"Failed text parse: {parse_err}")
|
||||
if is_centroid_click and clicked_cluster_id is not None:
|
||||
if current_selected_cluster == clicked_cluster_id:
|
||||
new_selected_cluster = None; status_msg = "Cluster view reset."; logger.info("Deselecting.")
|
||||
else:
|
||||
new_selected_cluster = clicked_cluster_id; status_msg = f"Showing Cluster {new_selected_cluster}."; logger.info(
|
||||
f"Selecting {new_selected_cluster}.")
|
||||
elif not is_centroid_click and current_selected_cluster is not None:
|
||||
new_selected_cluster = None; status_msg = "Cluster view reset."; logger.info("Deselecting.")
|
||||
else: # Click background
|
||||
if current_selected_cluster is not None: new_selected_cluster = None; status_msg = "Cluster view reset."; logger.info(
|
||||
"Deselecting.")
|
||||
logger.debug(f"Click Result: new_selected_cluster = {new_selected_cluster}")
|
||||
else:
|
||||
logger.debug("No click trigger.")
|
||||
|
||||
# --- Data Filtering ---
|
||||
active_selection_id = new_selected_cluster
|
||||
df_to_plot = df.copy();
|
||||
centroids_to_plot = centroids.copy()
|
||||
logger.debug(f"Filtering based on active_selection_id: {active_selection_id}")
|
||||
if active_selection_id is not None:
|
||||
df_to_plot = df_to_plot[df_to_plot['cluster'] == active_selection_id]
|
||||
centroids_to_plot = {cid: coords for cid, coords in centroids_to_plot.items() if
|
||||
cid == active_selection_id}
|
||||
logger.debug(f"Filtered DF rows: {len(df_to_plot)}")
|
||||
if not df_to_plot.empty:
|
||||
logger.debug(f"Coordinates of filtered points:\n{df_to_plot[['x', 'y', 'z']]}")
|
||||
else:
|
||||
logger.warning("Filtered DataFrame is empty.")
|
||||
|
||||
# --- Search Highlighting ---
|
||||
search_highlight_mask = np.zeros(len(df_to_plot), dtype=bool)
|
||||
search_term = None;
|
||||
is_search_active = False;
|
||||
highlight_sims = {}
|
||||
if search_results and isinstance(search_results, dict) and "indices" in search_results:
|
||||
is_search_active = True;
|
||||
search_term = search_results.get("term", "N/A")
|
||||
orig_indices = search_results.get("indices", []);
|
||||
orig_sims = search_results.get("similarities", [])
|
||||
if not df_to_plot.empty:
|
||||
orig_to_current_map = {orig_idx: current_idx for current_idx, orig_idx in
|
||||
enumerate(df_to_plot.index)}
|
||||
current_indices_hl = [orig_to_current_map[oi] for i, oi in enumerate(orig_indices) if
|
||||
i < len(orig_sims) and orig_sims[
|
||||
i] >= similarity_threshold and oi in orig_to_current_map]
|
||||
if current_indices_hl:
|
||||
search_highlight_mask[current_indices_hl] = True
|
||||
for i, orig_idx in enumerate(orig_indices):
|
||||
if i < len(orig_sims) and orig_sims[
|
||||
i] >= similarity_threshold and orig_idx in orig_to_current_map:
|
||||
highlight_sims[orig_to_current_map[orig_idx]] = orig_sims[i]
|
||||
else:
|
||||
logger.warning("Cannot apply search highlighting - filtered df empty.")
|
||||
|
||||
# --- Plotting ---
|
||||
df_search_hl = df_to_plot[search_highlight_mask];
|
||||
df_normal = df_to_plot[~search_highlight_mask]
|
||||
base_size = size_value;
|
||||
normal_op = OPACITY_SELECTED_CLUSTER if active_selection_id else (
|
||||
OPACITY_SEARCH_DIMMED if is_search_active else OPACITY_DEFAULT)
|
||||
|
||||
# --- Add Dummy Points if needed ---
|
||||
num_points_to_plot = len(df_normal) + len(df_search_hl)
|
||||
if active_selection_id is not None and num_points_to_plot <= 2:
|
||||
logger.info(
|
||||
f"Adding dummy invisible points to aid auto-ranging for cluster {active_selection_id} (points={num_points_to_plot}).")
|
||||
# Use overall range calculated earlier
|
||||
dummy_x = [overall_x_min, overall_x_max]
|
||||
dummy_y = [overall_y_min, overall_y_max]
|
||||
dummy_z = [overall_z_min, overall_z_max]
|
||||
# Ensure dummy points are valid numbers (in case overall range calc failed)
|
||||
if np.isfinite(dummy_x + dummy_y + dummy_z).all():
|
||||
fig.add_trace(go.Scatter3d(
|
||||
x=dummy_x, y=dummy_y, z=dummy_z,
|
||||
mode='markers', marker=dict(size=1, opacity=0), # Invisible
|
||||
hoverinfo='skip', showlegend=False, name='_dummy_'
|
||||
))
|
||||
else:
|
||||
logger.warning("Could not add dummy points because overall range contained non-finite values.")
|
||||
|
||||
# Plot Normal Points
|
||||
if not df_normal.empty:
|
||||
finite_mask_normal = np.isfinite(df_normal[['x', 'y', 'z']]).all(axis=1)
|
||||
df_normal_finite = df_normal[finite_mask_normal]
|
||||
if not df_normal_finite.empty:
|
||||
logger.debug(f"Plotting df_normal (len={len(df_normal_finite)}).")
|
||||
colors = df_normal_finite['cluster'].map(color_map).fillna('darkgrey')
|
||||
name = 'Embeddings' if active_selection_id is None else f'Cluster {active_selection_id}'
|
||||
fig.add_trace(
|
||||
go.Scatter3d(x=df_normal_finite['x'], y=df_normal_finite['y'], z=df_normal_finite['z'],
|
||||
mode='markers',
|
||||
marker=dict(color=colors, size=base_size, opacity=normal_op, line=dict(width=0.5)),
|
||||
text=df_normal_finite['hover_text'], hoverinfo='text', name=name))
|
||||
else:
|
||||
logger.warning("No finite normal points to plot.")
|
||||
|
||||
# Plot Search Highlighted Points
|
||||
if not df_search_hl.empty:
|
||||
finite_mask_search = np.isfinite(df_search_hl[['x', 'y', 'z']]).all(axis=1)
|
||||
df_search_hl_finite = df_search_hl[finite_mask_search]
|
||||
if not df_search_hl_finite.empty:
|
||||
hl_size = max(base_size * 1.5, base_size + 2);
|
||||
hl_texts = []
|
||||
# Need mapping from df_search_hl_finite index back to df_to_plot positional index for sims
|
||||
positions_in_df_to_plot = df_to_plot.index.get_indexer_for(df_search_hl_finite.index)
|
||||
|
||||
for i, (global_index, row) in enumerate(df_search_hl_finite.iterrows()):
|
||||
pos = positions_in_df_to_plot[i] # Get original position in df_to_plot
|
||||
sim = highlight_sims.get(pos, float('nan'))
|
||||
sim_txt = f"{sim:.4f}" if not np.isnan(sim) else "N/A"
|
||||
hl_texts.append(f"{row['hover_text']}<br><b>Sim: {sim_txt}</b>")
|
||||
|
||||
fig.add_trace(
|
||||
go.Scatter3d(x=df_search_hl_finite['x'], y=df_search_hl_finite['y'], z=df_search_hl_finite['z'],
|
||||
mode='markers',
|
||||
marker=dict(color='red', size=hl_size, opacity=1.0, symbol='diamond',
|
||||
line=dict(color='black', width=1)), text=hl_texts, hoverinfo='text',
|
||||
name=f'Search Neighbors'))
|
||||
if not df_search_hl_finite[['x', 'y', 'z']].isnull().values.any(): # Search Centroid
|
||||
try:
|
||||
sc = df_search_hl_finite[['x', 'y', 'z']].mean().values; fig.add_trace(
|
||||
go.Scatter3d(x=[sc[0]], y=[sc[1]], z=[sc[2]], mode='markers',
|
||||
marker=dict(color='magenta', size=max(hl_size, 10), symbol='cross',
|
||||
line=dict(width=1)), text=f"Search: '{search_term}' Centroid",
|
||||
hoverinfo='text', name='Search Centroid'))
|
||||
except Exception as e:
|
||||
logger.warning(f"Search centroid plot fail: {e}")
|
||||
else:
|
||||
logger.warning("No finite search highlighted points to plot.")
|
||||
|
||||
# Plot Centroids (filtered)
|
||||
if centroids_to_plot:
|
||||
cent_size = base_size + 1;
|
||||
logger.debug(f"Plotting centroids: {list(centroids_to_plot.keys())}")
|
||||
for cid, coords in centroids_to_plot.items():
|
||||
if isinstance(coords, list) and len(coords) == 3:
|
||||
logger.debug(f"Plotting Centroid {cid} at coords: {coords}")
|
||||
if np.isnan(coords).any() or np.isinf(coords).any(): logger.error(
|
||||
f"!!! Centroid {cid} NaN/Inf coords !!!"); continue
|
||||
color = color_map.get(str(cid), 'grey');
|
||||
name = f"Centroid {cid}";
|
||||
hover_txt = f"Centroid: Cluster {cid}"
|
||||
fig.add_trace(go.Scatter3d(
|
||||
x=[coords[0]], y=[coords[1]], z=[coords[2]], mode='markers',
|
||||
marker=dict(color=color, size=cent_size, symbol='circle', opacity=0.9,
|
||||
line=dict(color='black', width=1.5)),
|
||||
customdata=[str(cid)], text=hover_txt, hoverinfo='text', name=name,
|
||||
legendgroup="centroids", showlegend=True
|
||||
))
|
||||
else:
|
||||
logger.warning(f"Invalid centroid data for {cid}")
|
||||
|
||||
# --- Final Layout & Status ---
|
||||
title = f"3D t-SNE ({len(df)} points)"
|
||||
if active_selection_id is not None:
|
||||
title = f"Cluster {active_selection_id} ({len(df_to_plot)} points)"
|
||||
elif is_search_active:
|
||||
title = f"3D t-SNE - Search: '{search_term}'"
|
||||
if active_selection_id and is_search_active: title += f" - Search: '{search_term}'"
|
||||
base_layout = self._create_base_figure().layout
|
||||
fig.update_layout(
|
||||
title=title, legend_title_text='Legend', legend=base_layout.legend,
|
||||
scene=base_layout.scene # Use base scene settings (includes aspectmode='data')
|
||||
# Rely on auto-ranging (potentially helped by dummy points if added)
|
||||
)
|
||||
|
||||
final_status = status_msg
|
||||
if not final_status: # Default status
|
||||
base = f"{len(df_to_plot)} points shown."
|
||||
if active_selection_id: base = f"Cluster {active_selection_id}: {len(df_to_plot)} points."
|
||||
final_status = base;
|
||||
if is_search_active: final_status += f" (Search: '{search_term}')"
|
||||
|
||||
return fig, new_selected_cluster, final_status
|
||||
|
||||
def run(self, host: str = "127.0.0.1", port: int = 8050, debug: bool = False) -> None:
|
||||
"""Starts the Dash server."""
|
||||
logger.info(f"Starting Dash server on http://{host}:{port}")
|
||||
try:
|
||||
self.app.run(host=host, port=port, debug=debug)
|
||||
except OSError as e:
|
||||
logger.error(f"Server start failed: {e}. Port {port} busy?")
|
||||
except Exception as e:
|
||||
logger.exception(f"Server error: {e}")
|
||||
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
# >>> Helpers for automatic cluster‐count selection <<<
|
||||
# ──────────────────────────────────────────────────────────────
|
||||
def _estimate_optimal_clusters(self, embeddings: np.ndarray, max_k: int = 10) -> int:
|
||||
"""
|
||||
Estimate an optimal number of clusters using a quick elbow heuristic.
|
||||
Computes K‑means inertia for k = 1…max_k and picks the k that is farthest
|
||||
from the straight line connecting (1, inertia₁) and (max_k, inertiaₘₐₓ).
|
||||
"""
|
||||
if embeddings is None or embeddings.size == 0:
|
||||
return 1
|
||||
n_samples = embeddings.shape[0]
|
||||
if n_samples < 3:
|
||||
return 1
|
||||
|
||||
max_k = min(max_k, n_samples)
|
||||
inertias = []
|
||||
for k in range(1, max_k + 1):
|
||||
km = KMeans(n_clusters=k, random_state=42, n_init="auto").fit(embeddings)
|
||||
inertias.append(km.inertia_)
|
||||
|
||||
# distance from each point to the line between first and last
|
||||
x = np.arange(1, max_k + 1)
|
||||
x1, y1 = 1, inertias[0]
|
||||
x2, y2 = max_k, inertias[-1]
|
||||
numerator = np.abs((y2 - y1) * x - (x2 - x1) * np.array(inertias) + x2 * y1 - y2 * x1)
|
||||
denominator = np.sqrt((y2 - y1) ** 2 + (x2 - x1) ** 2)
|
||||
elbow_idx = int(np.argmax(numerator / denominator))
|
||||
return elbow_idx + 1 # since k starts at 1
|
||||
|
||||
def _cluster_marks(self) -> Dict[int, str]:
|
||||
"""Generate tick marks for the cluster-count slider."""
|
||||
if self.max_clusters <= 15:
|
||||
return {i: str(i) for i in range(1, self.max_clusters + 1)}
|
||||
# Show first, optimal, and max for large data sets
|
||||
return {1: "1", self.optimal_clusters: str(self.optimal_clusters), self.max_clusters: str(self.max_clusters)}
|
||||
|
||||
# --- END OF FILE visualizer.py ---
|
||||
@@ -0,0 +1,2 @@
|
||||
def hello() -> str:
|
||||
return "Hello from librarian_vspace!"
|
||||
Whitespace-only changes.
@@ -0,0 +1,93 @@
|
||||
|
||||
"""TsneExportWorker – Prefect worker that generates a t‑SNE JSON export.
|
||||
|
||||
It wraps Vspace.get_tnse → vecview.get_tsne_json, writes the JSON to a file,
|
||||
stages it, and returns the file path.
|
||||
|
||||
Minimal Pydantic payload models are defined locally to avoid extra deps.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from prefect import get_run_logger
|
||||
from pydantic import BaseModel
|
||||
from librarian_core.workers.base import Worker
|
||||
|
||||
from librarian_vspace.vecview.vecview import get_tsne_json
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
def _safe_get_logger(name: str):
|
||||
try:
|
||||
return get_run_logger()
|
||||
except Exception:
|
||||
return logging.getLogger(name)
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Pydantic payloads
|
||||
# ------------------------------------------------------------------ #
|
||||
class TsneExportInput(BaseModel):
|
||||
course_id: int
|
||||
limit: Optional[int] = None
|
||||
perplexity: float = 30.0
|
||||
db_schema: str = "librarian"
|
||||
rpc_function: str = "pdf_chunking"
|
||||
embed_model: str = "snowflake-arctic-embed2"
|
||||
embedding_column: str = "embedding"
|
||||
base_output_dir: Optional[Path] = None # where to place JSON file
|
||||
|
||||
|
||||
class TsneExportOutput(BaseModel):
|
||||
json_path: Path
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
class TsneExportWorker(Worker[TsneExportInput, TsneExportOutput]):
|
||||
"""Runs the t‑SNE export inside a Prefect worker.""" # noqa: D401
|
||||
|
||||
input_model = TsneExportInput
|
||||
output_model = TsneExportOutput
|
||||
|
||||
async def __run__(self, payload: TsneExportInput) -> TsneExportOutput:
|
||||
logger = _safe_get_logger(self.worker_name)
|
||||
logger.info("🔨 %s startet (payload=%r)", self.worker_name, payload)
|
||||
|
||||
# Run get_tsne_json in a thread
|
||||
data_json = await asyncio.to_thread(
|
||||
get_tsne_json,
|
||||
db_schema=payload.db_schema,
|
||||
db_function=payload.rpc_function,
|
||||
model_name=payload.embed_model,
|
||||
limit=payload.limit,
|
||||
course_id=payload.course_id,
|
||||
perplexity=payload.perplexity,
|
||||
embedding_column=payload.embedding_column,
|
||||
)
|
||||
|
||||
# Determine output file
|
||||
if payload.base_output_dir:
|
||||
out_dir = Path(payload.base_output_dir).expanduser()
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
json_path = out_dir / f"{payload.course_id}_tsne.json"
|
||||
else:
|
||||
tf = tempfile.NamedTemporaryFile(
|
||||
mode="w+", suffix="_tsne.json", prefix="vspace_", delete=False
|
||||
)
|
||||
json_path = Path(tf.name)
|
||||
|
||||
# Write JSON to file
|
||||
json_path.write_text(data_json, encoding="utf-8")
|
||||
|
||||
# Stage file for Prefect
|
||||
self.stage(json_path, new_name=json_path.name)
|
||||
|
||||
result = TsneExportOutput(json_path=json_path)
|
||||
logger.info("✅ %s fertig: %r", self.worker_name, result)
|
||||
return result
|
||||
@@ -0,0 +1,104 @@
|
||||
|
||||
"""Utility functions to fetch vectors from Supabase, apply t‑SNE, add simple K‑means
|
||||
clustering and hover text – prepared exactly like the `VectorVisualizer` expects.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Optional, List
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.cluster import KMeans
|
||||
|
||||
from librarian_vspace.vutils.vector_query_loader import VectorQueryLoader, VectorQueryLoaderError
|
||||
from librarian_vspace.models.tsne_model import TSNEPoint, TSNEData
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_N_CLUSTERS = 8
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
# Internal helpers (kept minimal – no extra bells & whistles)
|
||||
# --------------------------------------------------------------------- #
|
||||
def _run_kmeans(df: pd.DataFrame, *, embedding_column: str, k: int = DEFAULT_N_CLUSTERS) -> pd.DataFrame:
|
||||
"""Adds a 'cluster' column using K‑means (string labels)."""
|
||||
if df.empty or embedding_column not in df.columns:
|
||||
df['cluster'] = "-1"
|
||||
return df
|
||||
|
||||
embeddings = np.array(df[embedding_column].tolist(), dtype=float)
|
||||
n_samples = embeddings.shape[0]
|
||||
k = max(1, min(k, n_samples)) # ensure 1 ≤ k ≤ n_samples
|
||||
if n_samples < 2:
|
||||
df['cluster'] = "0"
|
||||
return df
|
||||
|
||||
km = KMeans(n_clusters=k, random_state=42, n_init='auto')
|
||||
df['cluster'] = km.fit_predict(embeddings).astype(str)
|
||||
return df
|
||||
|
||||
|
||||
def _add_hover(df: pd.DataFrame) -> pd.DataFrame:
|
||||
if df.empty:
|
||||
return df
|
||||
|
||||
df = df.copy()
|
||||
|
||||
def _hover(row):
|
||||
preview = str(row.get('chunk', ''))[:200]
|
||||
if len(str(row.get('chunk', ''))) > 200:
|
||||
preview += "..."
|
||||
return (
|
||||
f"ID: {row.get('file_id', 'N/A')}<br>"
|
||||
f"Cluster: {row.get('cluster', 'N/A')}<br>"
|
||||
f"Chunk: {preview}"
|
||||
)
|
||||
|
||||
df['hover_text'] = df.apply(_hover, axis=1)
|
||||
return df
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
# Public helpers
|
||||
# --------------------------------------------------------------------- #
|
||||
def get_tsne_dataframe(
|
||||
db_schema: str,
|
||||
db_function: str,
|
||||
model_name: str,
|
||||
*,
|
||||
limit: Optional[int] = None,
|
||||
course_id: Optional[int] = None,
|
||||
perplexity: float = 30.0,
|
||||
embedding_column: str = "embedding",
|
||||
n_clusters: int = DEFAULT_N_CLUSTERS,
|
||||
) -> pd.DataFrame:
|
||||
"""Returns a pandas DataFrame with tsne (x,y,z) & metadata ready for plotting."""
|
||||
loader = VectorQueryLoader(db_schema, db_function, model_name, embedding_column)
|
||||
df = loader.load_and_reduce(
|
||||
limit=limit,
|
||||
course_id=course_id,
|
||||
tsne_params={"perplexity": perplexity},
|
||||
)
|
||||
|
||||
if df.empty:
|
||||
return df
|
||||
|
||||
df = _run_kmeans(df, embedding_column=embedding_column, k=n_clusters)
|
||||
df = _add_hover(df)
|
||||
return df
|
||||
|
||||
|
||||
def get_tsne_json(**kwargs) -> str:
|
||||
"""Convenience wrapper returning DataFrame as JSON (orient='split')."""
|
||||
df = get_tsne_dataframe(**kwargs)
|
||||
return df.to_json(date_format='iso', orient='split')
|
||||
|
||||
|
||||
def get_tsne_response(**kwargs) -> TSNEData:
|
||||
"""Returns a validated `TSNEResponse` Pydantic model."""
|
||||
df = get_tsne_dataframe(**kwargs)
|
||||
points: List[TSNEPoint] = [TSNEPoint(**row.dropna().to_dict()) for _, row in df.iterrows()]
|
||||
return TSNEData(course_id=kwargs.get('course_id'), total=len(points), points=points)
|
||||
@@ -0,0 +1,10 @@
|
||||
"""
|
||||
vquery package for high-level read operations against vector tables.
|
||||
"""
|
||||
from .query import VectorQuery
|
||||
|
||||
__all__ = ["VectorQuery"] # Defines the public interface of the package
|
||||
|
||||
# Optional: Add package-level logging setup if desired, but often handled by the application
|
||||
# import logging
|
||||
# logging.getLogger(__name__).addHandler(logging.NullHandler())
|
||||
@@ -0,0 +1,438 @@
|
||||
# --- START OF FILE cluster_export.py (Refactored & Workaround - Import Updated) ---
|
||||
|
||||
"""
|
||||
cluster_export.py – Generate IVFFlat‑equivalent clusters from Supabase/Vectorbase
|
||||
pgvector data and export each cluster’s chunks to Markdown.
|
||||
|
||||
This version fetches vectors filtered by course ID at the database level using
|
||||
VectorQueryLoader, performs k-means clustering, and exports to Markdown.
|
||||
|
||||
Includes automatic k-downsizing.
|
||||
|
||||
Environment variables (used by the script entry point)
|
||||
---------------------
|
||||
* **Vectorbase credentials** (auto‑mapped to Supabase):
|
||||
* `VECTORBASE_URL` → `SUPABASE_URL`
|
||||
* `VECTORBASE_API_KEY` → `SUPABASE_KEY`
|
||||
* `VECTORBASE_USER_UUID` → `SUPABASE_USER_UUID` (optional)
|
||||
* **Embedding/table config**
|
||||
* `VECTOR_SCHEMA` – Postgres schema (default `librarian`)
|
||||
* `VECTOR_FUNCTION` – RPC / Postgres function name (optional)
|
||||
* `EMBED_MODEL` – embedding model label (default `snowflake-arctic-embed2`)
|
||||
* **Clustering hyper‑parameters**
|
||||
* `K` – requested number of clusters / IVFFlat *nlist* (default 128)
|
||||
* `TRAIN_SAMPLE` – how many rows to feed into k‑means (default 20 000, but
|
||||
capped at the table size)
|
||||
* **Export**
|
||||
* `OUTPUT_DIR` – directory for the generated Markdown files (default
|
||||
`./cluster_md`)
|
||||
* `CLUSTER_COURSE_ID` - Optional course ID to filter vectors (used by script)
|
||||
|
||||
|
||||
Usage
|
||||
~~~~~
|
||||
# Via script entry point
|
||||
export VECTORBASE_URL="https://xyz.vectorbase.co"
|
||||
export VECTORBASE_API_KEY="service_role_key"
|
||||
export VECTOR_SCHEMA=librarian
|
||||
export EMBED_MODEL=snowflake-arctic-embed2
|
||||
export CLUSTER_COURSE_ID=123 # Optional filtering
|
||||
export K=64
|
||||
python -m librarian_vspace.vquery.cluster_export
|
||||
|
||||
# As a callable function
|
||||
from librarian_vspace.vquery.cluster_export import run_cluster_export_job
|
||||
output_path = run_cluster_export_job(course_id=456, output_dir="/tmp/clusters_456", ...)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import List, Optional, Dict, Any, Union # Added Union
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from sklearn.cluster import KMeans
|
||||
from sklearn.metrics import pairwise_distances_argmin_min
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Map Vectorbase credential names → Supabase names expected by loader code
|
||||
# ---------------------------------------------------------------------------
|
||||
_ALIAS_ENV_MAP = {
|
||||
"VECTORBASE_URL": "SUPABASE_URL",
|
||||
"VECTORBASE_API_KEY": "SUPABASE_KEY",
|
||||
"VECTORBASE_USER_UUID": "SUPABASE_USER_UUID", # optional
|
||||
}
|
||||
for src, dest in _ALIAS_ENV_MAP.items():
|
||||
if dest not in os.environ and src in os.environ:
|
||||
os.environ[dest] = os.environ[src]
|
||||
|
||||
# Import the NEW data‑loading helper with filtering capabilities
|
||||
try:
|
||||
# --- FIX: Import VectorQueryLoader from vutils ---
|
||||
from librarian_vspace.vutils.vector_query_loader import VectorQueryLoader, VectorQueryLoaderError
|
||||
# VectorLoaderError is now VectorQueryLoaderError
|
||||
# --- END FIX ---
|
||||
except ImportError as e:
|
||||
# Keep the original script's error handling for standalone use
|
||||
sys.stderr.write(
|
||||
"\n[ERROR] Could not import VectorQueryLoader – check PYTHONPATH. "
|
||||
f"Original error: {e}\n"
|
||||
)
|
||||
# For callable use, we should raise an ImportError or custom exception
|
||||
raise ImportError(f"Could not import VectorQueryLoader: {e}") from e
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Logging setup (used by both script and callable function)
|
||||
# ---------------------------------------------------------------------------
|
||||
# This basicConfig runs when the module is imported.
|
||||
# Callers might want to configure logging before importing.
|
||||
# If logging is already configured, basicConfig does nothing.
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s: %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
logger = logging.getLogger(__name__) # Use __name__ for module-specific logger
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helper – JSON dump for centroid in YAML front‑matter
|
||||
# ---------------------------------------------------------------------------
|
||||
def centroid_to_json(vec: np.ndarray) -> str:
|
||||
"""Converts a numpy vector to a JSON string suitable for YAML frontmatter."""
|
||||
return json.dumps([float(x) for x in vec], ensure_ascii=False)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Main clustering and export logic as a callable function
|
||||
# ---------------------------------------------------------------------------
|
||||
def run_cluster_export_job(
|
||||
course_id: Optional[int] = None, # Added course_id parameter
|
||||
output_dir: Union[str, Path] = "./cluster_md", # Output directory parameter
|
||||
schema: str = "librarian",
|
||||
rpc_function: str = "pdf_chunking", # Default to actual function name
|
||||
model: str = "snowflake-arctic-embed2",
|
||||
k_clusters: int = 128, # Requested number of clusters (k)
|
||||
train_sample_size: int = 20000, # Sample size for K-means training
|
||||
embedding_column: str = "embedding" # Added embedding column parameter
|
||||
) -> Path:
|
||||
"""
|
||||
Fetches vectors, performs K-means clustering, and exports clustered chunks to Markdown.
|
||||
|
||||
Args:
|
||||
course_id: Optional ID to filter vectors belonging to a specific course.
|
||||
output_dir: Directory path where the cluster Markdown files will be saved.
|
||||
schema: Postgres schema containing the vector table.
|
||||
rpc_function: Optional RPC function name used by VectorQueryLoader (needed for table lookup).
|
||||
model: Embedding model label used by VectorQueryLoader (needed for table lookup).
|
||||
k_clusters: The requested number of clusters (k). Will be downsized if fewer
|
||||
vectors are available.
|
||||
train_sample_size: The maximum number of vectors to use for K-means training.
|
||||
Capped by the total number of vectors fetched.
|
||||
embedding_column: The name of the column containing the vector embeddings.
|
||||
|
||||
Returns:
|
||||
The absolute path to the output directory.
|
||||
|
||||
Raises:
|
||||
VectorQueryLoaderError: If vector loading fails.
|
||||
RuntimeError: If no embeddings are retrieved or training sample is empty after filtering.
|
||||
Exception: For other errors during clustering or export.
|
||||
"""
|
||||
output_path = Path(output_dir).expanduser().resolve() # Resolve path early
|
||||
output_path.mkdir(parents=True, exist_ok=True)
|
||||
logger.info("Writing Markdown files to %s", output_path)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Fetch embeddings - Now using VectorQueryLoader with filtering
|
||||
# ---------------------------------------------------------------------------
|
||||
try:
|
||||
# Use parameters for loader config
|
||||
# --- FIX: Instantiate VectorQueryLoader ---
|
||||
loader = VectorQueryLoader(schema=schema, function=rpc_function, model=model, embedding_column=embedding_column)
|
||||
# --- END FIX ---
|
||||
|
||||
# --- FIX: Call fetch_vectors WITH the course_id argument ---
|
||||
# VectorQueryLoader.fetch_vectors handles the DB-level filtering
|
||||
df = loader.fetch_vectors(limit=None, course_id=course_id)
|
||||
# --- END FIX ---
|
||||
|
||||
# --- REMOVE: In-memory filtering logic is no longer needed ---
|
||||
# initial_rows = len(df)
|
||||
# if course_id is not None and not df.empty:
|
||||
# ... (removed filtering code) ...
|
||||
# elif course_id is not None and df.empty:
|
||||
# ... (removed warning) ...
|
||||
# --- END REMOVE ---
|
||||
|
||||
# --- FIX: Catch VectorQueryLoaderError ---
|
||||
except VectorQueryLoaderError as e:
|
||||
logger.error("Vector loading failed: %s", e)
|
||||
raise e # Re-raise the specific exception for the caller
|
||||
# --- END FIX ---
|
||||
except Exception as e:
|
||||
# Catch other unexpected errors during loading
|
||||
logger.exception("An unexpected error occurred during vector loading.")
|
||||
raise RuntimeError(f"An unexpected error occurred during vector loading: {e}") from e
|
||||
|
||||
|
||||
# --- Check if DataFrame is empty *after* fetching (which includes DB filtering) ---
|
||||
if df.empty:
|
||||
logger.error("No embeddings retrieved or found for course_id %s – aborting.", course_id)
|
||||
# Raise a RuntimeError as no clustering can be done
|
||||
raise RuntimeError(f"No embeddings retrieved or found for course_id {course_id} – nothing to cluster.")
|
||||
# ----------------------------------------------------------------------------------
|
||||
|
||||
|
||||
# Use the actual embedding column name from the loader instance
|
||||
# This check is crucial *after* fetching
|
||||
if not hasattr(loader, 'embedding_column') or loader.embedding_column not in df.columns:
|
||||
# This should ideally be caught by VectorQueryLoader's internal checks, but double-check
|
||||
logger.error("Embedding column '%s' not found in fetched data.", embedding_column) # Use the input param name for error msg
|
||||
raise RuntimeError(f"Embedding column '{embedding_column}' not found in fetched data.")
|
||||
|
||||
|
||||
# --- Ensure embeddings are numeric lists before stacking ---
|
||||
# The VectorQueryLoader.fetch_vectors method now handles parsing and dropping invalid rows.
|
||||
# We just need to safely stack the potentially filtered/cleaned data.
|
||||
try:
|
||||
# Ensure data is list of floats before stacking
|
||||
# This check might be redundant if VectorQueryLoader guarantees cleaned data,
|
||||
# but it adds safety.
|
||||
if not all(isinstance(x, list) and all(isinstance(n, float) for n in x) for x in df[embedding_column]):
|
||||
logger.error(f"Data in '{embedding_column}' is not strictly list[float] format after fetching. Attempting conversion.")
|
||||
# This might catch issues the loader missed or unexpected data structures
|
||||
try:
|
||||
# Attempt robust conversion similar to the loader's parse method
|
||||
embeddings_list = []
|
||||
for item in df[embedding_column]:
|
||||
parsed_item = None
|
||||
if isinstance(item, str):
|
||||
try: parsed_item = json.loads(item)
|
||||
except json.JSONDecodeError: pass
|
||||
elif isinstance(item, (list, tuple, np.ndarray)):
|
||||
parsed_item = item
|
||||
elif isinstance(item, dict) and 'vector' in item and isinstance(item['vector'], (list, tuple, np.ndarray)):
|
||||
parsed_item = item['vector']
|
||||
|
||||
if isinstance(parsed_item, (list, tuple, np.ndarray)) and all(isinstance(val, (int, float, np.number)) for val in parsed_item):
|
||||
embeddings_list.append([float(n) for n in parsed_item])
|
||||
else:
|
||||
logger.debug(f"Skipping problematic embedding during secondary clean: {str(item)[:100]}...")
|
||||
|
||||
|
||||
if not embeddings_list:
|
||||
logger.error("No valid embeddings remained after secondary cleaning.")
|
||||
raise ValueError("No valid embeddings for stacking.")
|
||||
embeddings = np.array(embeddings_list, dtype=float)
|
||||
logger.warning("Successfully converted problematic embedding data for stacking.")
|
||||
except Exception as e:
|
||||
logger.exception(f"Failed secondary attempt to convert embeddings for stacking: {e}")
|
||||
raise RuntimeError(f"Failed to process embedding data for stacking: {e}") from e
|
||||
else:
|
||||
# Data is in the expected list of float format, proceed directly
|
||||
embeddings = np.stack(df[embedding_column].to_list()).astype(float)
|
||||
|
||||
logger.info("Prepared %d embeddings for clustering.", embeddings.shape[0])
|
||||
|
||||
|
||||
except ValueError as ve:
|
||||
logger.exception(f"Failed to stack embeddings into a numpy array: {ve}. Ensure '{embedding_column}' contains valid vector data.")
|
||||
raise RuntimeError(f"Failed to process embedding data: {ve}") from ve
|
||||
except Exception as e:
|
||||
logger.exception(f"An unexpected error occurred while processing '{embedding_column}' column for stacking.")
|
||||
raise RuntimeError(f"An unexpected error occurred while processing embedding data for stacking: {e}") from e
|
||||
# -------------------------------------------------------------
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Prepare training sample and determine effective k
|
||||
# ---------------------------------------------------------------------------
|
||||
# Use the parameter train_sample_size
|
||||
train_vecs = embeddings[:train_sample_size]
|
||||
|
||||
if train_vecs.shape[0] == 0:
|
||||
# If course_id filtering resulted in 0 vectors, this check prevents the crash
|
||||
# but the df.empty check earlier should already handle this.
|
||||
# Keep this check for robustness in case train_sample_size is 0 or negative.
|
||||
logger.error("Training sample is empty – nothing to cluster.")
|
||||
raise RuntimeError("Training sample is empty – nothing to cluster.")
|
||||
|
||||
# Use the parameter k_clusters
|
||||
K = min(k_clusters, train_vecs.shape[0])
|
||||
if K < k_clusters:
|
||||
logger.warning(
|
||||
"Requested k=%d but only %d training vectors available; "
|
||||
"using k=%d.",
|
||||
k_clusters,
|
||||
train_vecs.shape[0],
|
||||
K,
|
||||
)
|
||||
# Ensure K is at least 1 if there's any data
|
||||
if K == 0 and train_vecs.shape[0] > 0:
|
||||
K = 1
|
||||
logger.warning("Adjusted k to 1 as requested k resulted in 0 but data exists.")
|
||||
|
||||
if K == 0:
|
||||
# If after adjustments K is still 0 (meaning train_vecs.shape[0] was 0)
|
||||
logger.error("Effective k is 0. Cannot train k-means.")
|
||||
raise RuntimeError("Effective k is 0. Cannot train k-means.")
|
||||
|
||||
|
||||
logger.info("Training k‑means (k=%d) on %d vectors", K, train_vecs.shape[0])
|
||||
|
||||
try:
|
||||
kmeans = KMeans(
|
||||
n_clusters=K,
|
||||
init="k-means++",
|
||||
n_init="auto", # Use 'auto' for better handling of small k/n_samples
|
||||
algorithm="lloyd", # 'lloyd' is the standard, 'elkan' can be faster but has limitations
|
||||
max_iter=300,
|
||||
random_state=0,
|
||||
)
|
||||
kmeans.fit(train_vecs)
|
||||
centroids: np.ndarray = kmeans.cluster_centers_
|
||||
logger.info("K‑means converged in %d iterations", kmeans.n_iter_)
|
||||
except Exception as e:
|
||||
logger.exception("K-means clustering failed.")
|
||||
raise RuntimeError(f"K-means clustering failed: {e}") from e
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Assign every vector to its nearest centroid (full table)
|
||||
# ---------------------------------------------------------------------------
|
||||
logger.info("Assigning vectors to centroids...")
|
||||
try:
|
||||
# Use the determined embedding column for assignment as well
|
||||
labels_full, _ = pairwise_distances_argmin_min(embeddings, centroids, metric="euclidean")
|
||||
df["cluster_id"] = labels_full
|
||||
logger.info("Assigned cluster labels to all embeddings.")
|
||||
except Exception as e:
|
||||
logger.exception("Failed to assign vectors to centroids.")
|
||||
raise RuntimeError(f"Failed to assign vectors to centroids: {e}") from e
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Write one Markdown file per cluster
|
||||
# ---------------------------------------------------------------------------
|
||||
files_written_count = 0
|
||||
|
||||
logger.info("Writing cluster Markdown files to %s", output_path)
|
||||
try:
|
||||
# Only iterate up to the number of actual clusters found by KMeans
|
||||
# KMeans might return fewer clusters than K if there are issues or identical points
|
||||
num_actual_clusters = len(centroids)
|
||||
if num_actual_clusters < K:
|
||||
logger.warning(f"KMeans returned only {num_actual_clusters} centroids, expected {K}. Iterating over actual centroids.")
|
||||
|
||||
|
||||
for cid in range(num_actual_clusters): # Iterate over actual cluster IDs
|
||||
# Find all data points assigned to this cluster ID
|
||||
subset = df[df.cluster_id == cid]
|
||||
|
||||
# Ensure centroid_vec corresponds to the centroid of the *current* cluster ID (cid)
|
||||
# This check is more robust now iterating up to num_actual_clusters
|
||||
if cid < len(centroids):
|
||||
centroid_vec = centroids[cid]
|
||||
else:
|
||||
# This case should theoretically not be reached with the loop range
|
||||
logger.error(f"Centroid for cluster ID {cid} missing! Using zero vector.")
|
||||
centroid_vec = np.zeros(embeddings.shape[1])
|
||||
|
||||
|
||||
# Use .get() and .fillna("") defensively in case 'chunk' column is missing
|
||||
# Ensure chunk column exists - it should if SELECT * worked
|
||||
if 'chunk' not in subset.columns:
|
||||
logger.warning("'chunk' column missing in subset data for cluster %d. Using empty strings.", cid)
|
||||
chunks = [""] * len(subset)
|
||||
else:
|
||||
chunks = subset['chunk'].fillna("").tolist()
|
||||
|
||||
|
||||
md_lines = [
|
||||
#"---",
|
||||
#f"cluster_id: {cid}",
|
||||
#f"centroid: {centroid_to_json(centroid_vec)}",
|
||||
#"---\n", # Separator between frontmatter and content
|
||||
]
|
||||
# Add chunks, ensuring each chunk is on a new line or separated by blank lines
|
||||
md_lines.extend(chunks)
|
||||
|
||||
outfile = output_path / f"cluster_{cid:03d}.md"
|
||||
# Use a different separator for chunks within the file if needed,
|
||||
# currently just joins with newline, but chunks might contain newlines.
|
||||
# Joining with "\n\n" provides separation *between* chunks.
|
||||
try:
|
||||
outfile.write_text("\n\n".join(md_lines), encoding="utf-8")
|
||||
files_written_count += 1
|
||||
logger.debug("Wrote %s (%d chunks)", outfile.name, len(chunks)) # Use debug for per-file
|
||||
except Exception as write_exc:
|
||||
logger.error(f"Failed to write cluster file {outfile}: {write_exc}", exc_info=True)
|
||||
# Decide whether to continue or raise here. Continuing allows other clusters to be saved.
|
||||
# For robustness in script, maybe continue. For library function, maybe raise.
|
||||
# For now, we'll just log and continue.
|
||||
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("Failed during Markdown file writing loop.")
|
||||
raise RuntimeError(f"Failed during Markdown file writing: {e}") from e
|
||||
|
||||
|
||||
logger.info("Done. %d Markdown files created in %s", files_written_count, output_path)
|
||||
return output_path
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Script entry point
|
||||
# ---------------------------------------------------------------------------
|
||||
if __name__ == "__main__":
|
||||
# Configuration via environment for script
|
||||
script_output_dir = Path(os.environ.get("OUTPUT_DIR", "./cluster_md")).expanduser()
|
||||
script_schema = os.environ.get("VECTOR_SCHEMA", "librarian")
|
||||
script_rpc_function = os.environ.get("VECTOR_FUNCTION", "pdf_chunking") # Default to actual function name
|
||||
script_model = os.environ.get("EMBED_MODEL", "snowflake-arctic-embed2")
|
||||
script_k_req = int(os.environ.get("K", "128"))
|
||||
script_train_sample = int(os.environ.get("TRAIN_SAMPLE", "20000"))
|
||||
# Added course ID specific to script entry point
|
||||
script_course_id_str = os.environ.get("CLUSTER_COURSE_ID")
|
||||
script_course_id = int(script_course_id_str) if script_course_id_str and script_course_id_str.isdigit() else None # Added isdigit check
|
||||
|
||||
|
||||
# Configure basic logging for the script entry point
|
||||
# (The module-level config above might not run if imported in specific ways)
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s %(levelname)s: %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S",
|
||||
)
|
||||
# Re-get the logger after basicConfig to ensure it's configured
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
try:
|
||||
logger.info("Starting cluster export script...")
|
||||
final_output_path = run_cluster_export_job(
|
||||
course_id=script_course_id,
|
||||
output_dir=script_output_dir,
|
||||
schema=script_schema,
|
||||
rpc_function=script_rpc_function,
|
||||
model=script_model,
|
||||
k_clusters=script_k_req,
|
||||
train_sample_size=script_train_sample,
|
||||
# embedding_column defaults to 'embedding' in the function
|
||||
)
|
||||
logger.info("Script finished successfully. Output in %s", final_output_path)
|
||||
sys.exit(0) # Explicit success exit
|
||||
# --- FIX: Catch VectorQueryLoaderError ---
|
||||
except (VectorQueryLoaderError, RuntimeError) as e: # Catch the new error type
|
||||
# --- END FIX ---
|
||||
# Specific errors we raised
|
||||
logger.error("Script failed: %s", e)
|
||||
sys.exit(1) # Indicate failure
|
||||
except Exception as e:
|
||||
# Catch any other unexpected errors
|
||||
logger.exception("An unhandled error occurred during script execution.")
|
||||
sys.exit(1) # Indicate failure
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
|
||||
"""ClusterExportWorker – Prefect worker that wraps run_cluster_export_job."""
|
||||
|
||||
from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
from prefect import get_run_logger
|
||||
from pydantic import BaseModel
|
||||
from librarian_core.workers.base import Worker
|
||||
|
||||
from librarian_vspace.vquery.cluster_export import run_cluster_export_job
|
||||
|
||||
def _safe_get_logger(name: str):
|
||||
try:
|
||||
return get_run_logger()
|
||||
except Exception:
|
||||
return logging.getLogger(name)
|
||||
|
||||
class ClusterExportInput(BaseModel):
|
||||
course_id: int
|
||||
k_clusters: int = 128
|
||||
train_sample_size: int = 20_000
|
||||
db_schema: str = "librarian"
|
||||
rpc_function: str = "pdf_chunking"
|
||||
model: str = "snowflake-arctic-embed2"
|
||||
embedding_column: str = "embedding"
|
||||
base_output_dir: Optional[Path] = None
|
||||
|
||||
class ClusterExportOutput(BaseModel):
|
||||
output_dir: Path
|
||||
|
||||
class ClusterExportWorker(Worker[ClusterExportInput, ClusterExportOutput]):
|
||||
input_model = ClusterExportInput
|
||||
output_model = ClusterExportOutput
|
||||
|
||||
async def __run__(self, payload: ClusterExportInput) -> ClusterExportOutput:
|
||||
logger = _safe_get_logger(self.worker_name)
|
||||
logger.info("🔨 %s startet (payload=%r)", self.worker_name, payload)
|
||||
|
||||
# Prepare output directory
|
||||
if payload.base_output_dir:
|
||||
base_dir = Path(payload.base_output_dir).expanduser()
|
||||
base_dir.mkdir(parents=True, exist_ok=True)
|
||||
tmp_base = tempfile.mkdtemp(dir=base_dir)
|
||||
else:
|
||||
tmp_base = tempfile.mkdtemp()
|
||||
|
||||
output_dir = Path(tmp_base) / str(payload.course_id)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
logger.debug("Output directory: %s", output_dir)
|
||||
|
||||
final_dir = await asyncio.to_thread(
|
||||
run_cluster_export_job,
|
||||
course_id=payload.course_id,
|
||||
output_dir=output_dir,
|
||||
schema=payload.db_schema,
|
||||
rpc_function=payload.rpc_function,
|
||||
model=payload.model,
|
||||
k_clusters=payload.k_clusters,
|
||||
train_sample_size=payload.train_sample_size,
|
||||
embedding_column=payload.embedding_column,
|
||||
)
|
||||
|
||||
self.stage(final_dir, new_name=final_dir.name)
|
||||
|
||||
result = ClusterExportOutput(output_dir=final_dir)
|
||||
logger.info("✅ %s fertig: %r", self.worker_name, result)
|
||||
return result
|
||||
Whitespace-only changes.
@@ -0,0 +1,127 @@
|
||||
|
||||
"""VectorQuery – helper for vector searches against chunklet tables.
|
||||
|
||||
This module provides:
|
||||
* A Pydantic‑powered request / response API (see ``librarian_vspace.models.query_model``).
|
||||
* A single public method :py:meth:`VectorQuery.search` that returns a
|
||||
:class:`~librarian_vspace.models.query_model.VectorSearchResponse`.
|
||||
* A thin legacy wrapper ``get_chucklets_by_vector`` that produces the
|
||||
historical ``List[Dict[str, Any]]`` format, built on top of ``search``.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
try:
|
||||
from librarian_vspace.vutils.vector_class import BaseVectorOperator
|
||||
from librarian_vspace.vecembed.embedding_generator import EmbeddingGenerator
|
||||
except ImportError as exc: # pragma: no cover
|
||||
logging.error(
|
||||
"Failed to import vutils or vecembed sub‑packages: %s. " "Ensure they are on PYTHONPATH.", exc
|
||||
)
|
||||
|
||||
class BaseVectorOperator: # type: ignore
|
||||
"""Minimal stub if real class is unavailable (runtime error later)."""
|
||||
|
||||
class EmbeddingGenerator: # type: ignore
|
||||
"""Minimal stub; will raise at runtime if used."""
|
||||
|
||||
from librarian_vspace.models.query_model import (
|
||||
VectorSearchRequest,
|
||||
VectorSearchResponse,
|
||||
Chunklet,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------- #
|
||||
# Main helper
|
||||
# --------------------------------------------------------------------- #
|
||||
class VectorQuery(BaseVectorOperator):
|
||||
"""High‑level helper for vector searches via Supabase RPC."""
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Public – modern API
|
||||
# -----------------------------------------------------------------
|
||||
def search(self, request: VectorSearchRequest) -> VectorSearchResponse:
|
||||
"""Perform a similarity search and return structured results."""
|
||||
|
||||
if not getattr(self, "table", None):
|
||||
logger.error("VectorQuery: target table not determined (self.table is None).")
|
||||
return VectorSearchResponse(total=0, results=[])
|
||||
|
||||
# 1) Generate query embedding
|
||||
try:
|
||||
_tts, query_vec, _ = EmbeddingGenerator().generate_embedding(
|
||||
interface_name=request.interface_name,
|
||||
model_name=request.model_name,
|
||||
text_to_embed=request.search_string,
|
||||
identifier="query",
|
||||
)
|
||||
if query_vec is None:
|
||||
logger.error("Embedding generation returned None.")
|
||||
return VectorSearchResponse(total=0, results=[])
|
||||
except Exception as exc: # pragma: no cover
|
||||
logger.exception("Embedding generation failed: %s", exc)
|
||||
return VectorSearchResponse(total=0, results=[])
|
||||
|
||||
# 2) Build RPC parameters
|
||||
rpc_params = {
|
||||
"p_query_embedding": query_vec,
|
||||
"p_target_table": self.table,
|
||||
"p_embedding_column": request.embedding_column,
|
||||
"p_match_count": request.top_k,
|
||||
"p_filters": request.filters or {},
|
||||
}
|
||||
|
||||
# 3) Execute RPC
|
||||
try:
|
||||
if not getattr(self, "spc", None):
|
||||
logger.error("Supabase client (self.spc) not available.")
|
||||
return VectorSearchResponse(total=0, results=[])
|
||||
|
||||
resp = (
|
||||
self.spc
|
||||
.schema(self.schema)
|
||||
.rpc("vector_search", rpc_params)
|
||||
.execute()
|
||||
)
|
||||
data = resp.data or []
|
||||
results = [
|
||||
Chunklet(chunk=row.get("chunk"), file_id=row.get("file_id")) if isinstance(row.get("file_id"), str) else Chunklet(chunk=row.get("chunk"), file_id=str(row.get("file_id")))
|
||||
for row in data
|
||||
]
|
||||
return VectorSearchResponse(total=len(results), results=results)
|
||||
|
||||
except Exception as exc: # pragma: no cover
|
||||
logger.exception("RPC 'vector_search' failed: %s", exc)
|
||||
return VectorSearchResponse(total=0, results=[])
|
||||
|
||||
# -----------------------------------------------------------------
|
||||
# Public – legacy compatibility
|
||||
# -----------------------------------------------------------------
|
||||
def get_chucklets_by_vector(
|
||||
self,
|
||||
*,
|
||||
interface_name: str,
|
||||
model_name: str,
|
||||
search_string: str,
|
||||
filters: Optional[Dict[str, Any]] = None,
|
||||
top_k: int = 10,
|
||||
embedding_column: str = "embedding",
|
||||
) -> List[Dict[str, Any]]:
|
||||
"""Backward‑compatible wrapper returning ``{'chunk', 'file_id'}`` dicts."""
|
||||
|
||||
req = VectorSearchRequest(
|
||||
interface_name=interface_name,
|
||||
model_name=model_name,
|
||||
search_string=search_string,
|
||||
filters=filters,
|
||||
top_k=top_k,
|
||||
embedding_column=embedding_column,
|
||||
)
|
||||
resp = self.search(req)
|
||||
return [ck.dict() for ck in resp.results]
|
||||
@@ -0,0 +1,62 @@
|
||||
|
||||
"""QueryWorker – Prefect worker that performs a vector search.
|
||||
|
||||
It instantiates VectorQuery directly (no vspace dependency) and returns the
|
||||
VectorSearchResponse.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
from prefect import get_run_logger
|
||||
from pydantic import BaseModel
|
||||
from librarian_core.workers.base import Worker
|
||||
|
||||
from librarian_vspace.vquery.query import VectorQuery
|
||||
from librarian_vspace.models.query_model import VectorSearchRequest, VectorSearchResponse
|
||||
|
||||
def _safe_get_logger(name: str):
|
||||
try:
|
||||
return get_run_logger()
|
||||
except Exception:
|
||||
return logging.getLogger(name)
|
||||
|
||||
|
||||
class QueryInput(BaseModel):
|
||||
request: VectorSearchRequest
|
||||
db_schema: str = "librarian"
|
||||
rpc_function: str = "pdf_chunking"
|
||||
embed_model: str = "snowflake-arctic-embed2"
|
||||
embedding_column: str = "embedding"
|
||||
|
||||
|
||||
class QueryWorker(Worker[QueryInput, VectorSearchResponse]):
|
||||
"""Runs a Supabase vector search via VectorQuery."""
|
||||
|
||||
input_model = QueryInput
|
||||
output_model = VectorSearchResponse
|
||||
|
||||
async def __run__(self, payload: QueryInput) -> VectorSearchResponse:
|
||||
logger = _safe_get_logger(self.worker_name)
|
||||
logger.info("🔨 %s startet (payload=%r)", self.worker_name, payload)
|
||||
|
||||
def _do_search() -> VectorSearchResponse:
|
||||
try:
|
||||
vq = VectorQuery(
|
||||
schema=payload.db_schema,
|
||||
function=payload.rpc_function,
|
||||
model=payload.embed_model,
|
||||
embedding_column=payload.embedding_column,
|
||||
)
|
||||
except TypeError:
|
||||
# fallback to positional signature
|
||||
vq = VectorQuery(payload.db_schema, payload.rpc_function, payload.embed_model)
|
||||
return vq.search(payload.request)
|
||||
|
||||
response = await asyncio.to_thread(_do_search)
|
||||
|
||||
logger.info("✅ %s fertig: %s results", self.worker_name, response.total)
|
||||
return response
|
||||
@@ -0,0 +1,2 @@
|
||||
def hello() -> str:
|
||||
return "Hello from librarian_vspace!"
|
||||
+248
@@ -0,0 +1,248 @@
|
||||
|
||||
#!/usr/bin/env python3
|
||||
"""parallelism_advisor.py
|
||||
=====================================
|
||||
Cross‑platform helper to inspect the hardware parallelism available on the
|
||||
current machine *and* to give rule‑of‑thumb recommendations for the number of
|
||||
workers you should spin up for CPU‑bound, IO‑bound, or GPU workloads.
|
||||
|
||||
Highlights
|
||||
----------
|
||||
* Works on Linux, macOS (Intel **and** Apple Silicon) and Windows.
|
||||
* Respects Docker / Kubernetes CPU quotas (cgroup v1 & v2) automatically.
|
||||
* Falls back gracefully when optional dependencies (``psutil``, ``torch``)
|
||||
are missing.
|
||||
* Caches expensive hardware queries.
|
||||
* Tiny CLI: ``python -m parallelism_advisor --json``
|
||||
|
||||
MIT‑licensed. Drop the file somewhere on ``$PYTHONPATH`` or
|
||||
``pip install .`` if you convert it into a proper package.
|
||||
|
||||
Usage
|
||||
-----
|
||||
>>> from librarian_vspace.vutils.parallelism_advisor import recommended_workers
|
||||
>>> recommended_workers(kind="cpu")
|
||||
8
|
||||
>>> recommended_workers(kind="io")
|
||||
64
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import functools
|
||||
import json
|
||||
import os
|
||||
import platform
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any, Dict, Literal, Optional
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Optional dependencies #
|
||||
# --------------------------------------------------------------------------- #
|
||||
try:
|
||||
import psutil # type: ignore
|
||||
except ModuleNotFoundError: # pragma: no cover
|
||||
psutil = None # type: ignore
|
||||
|
||||
try:
|
||||
import torch # type: ignore
|
||||
except ModuleNotFoundError: # pragma: no cover
|
||||
torch = None # type: ignore
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Hardware discovery helpers #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
def logical_cores() -> int:
|
||||
"""Return the number of *logical* CPUs visible to the process.
|
||||
|
||||
Falls back to 1 if Python or the OS cannot determine the value.
|
||||
"""
|
||||
return os.cpu_count() or 1
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
def physical_cores() -> int:
|
||||
"""Return the number of *physical* CPU cores (no SMT).
|
||||
|
||||
If ``psutil`` is not installed or fails to determine the value, the function
|
||||
falls back to ``logical_cores()``.
|
||||
"""
|
||||
if psutil is not None:
|
||||
pc = psutil.cpu_count(logical=False)
|
||||
if pc: # 0 or None means "could not determine"
|
||||
return pc
|
||||
# Heuristic fallback
|
||||
return logical_cores()
|
||||
|
||||
|
||||
def _read_first_line(path: Path) -> Optional[str]:
|
||||
try:
|
||||
return path.read_text().strip()
|
||||
except FileNotFoundError:
|
||||
return None
|
||||
except Exception: # pragma: no cover
|
||||
return None
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
def cgroup_cpu_limit() -> Optional[int]:
|
||||
"""Return Docker/Kubernetes CPU quota if present (else ``None``).
|
||||
|
||||
Supports both cgroup v1 and v2. *Unit tests mock this function.*
|
||||
"""
|
||||
# --- cgroup v2 ---------------------------------------------------------
|
||||
max_path = Path("/sys/fs/cgroup/cpu.max")
|
||||
line = _read_first_line(max_path)
|
||||
if line and " " in line:
|
||||
quota, period = line.split()[:2]
|
||||
try:
|
||||
quota_i = int(quota)
|
||||
period_i = int(period)
|
||||
if quota_i > 0 and period_i > 0:
|
||||
return max(1, quota_i // period_i)
|
||||
except ValueError: # pragma: no cover
|
||||
pass
|
||||
|
||||
# --- cgroup v1 ---------------------------------------------------------
|
||||
quota_path = Path("/sys/fs/cgroup/cpu/cpu.cfs_quota_us")
|
||||
period_path = Path("/sys/fs/cgroup/cpu/cpu.cfs_period_us")
|
||||
quota_str = _read_first_line(quota_path)
|
||||
period_str = _read_first_line(period_path)
|
||||
if quota_str and period_str:
|
||||
try:
|
||||
quota_i = int(quota_str)
|
||||
period_i = int(period_str)
|
||||
if quota_i > 0 and period_i > 0:
|
||||
return max(1, quota_i // period_i)
|
||||
except ValueError: # pragma: no cover
|
||||
pass
|
||||
return None
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# GPU helpers (CUDA only for now) #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=None)
|
||||
def gpu_info() -> Optional[Dict[str, Any]]:
|
||||
"""Return basic info for the first CUDA device via *torch* (or ``None``)."""
|
||||
if torch is None or not torch.cuda.is_available():
|
||||
return None
|
||||
try:
|
||||
device = torch.cuda.get_device_properties(0)
|
||||
except Exception: # pragma: no cover
|
||||
return None
|
||||
|
||||
return {
|
||||
"device_name": device.name,
|
||||
"sm_count": device.multi_processor_count,
|
||||
"max_threads_per_sm": device.max_threads_per_multi_processor,
|
||||
"total_memory_mb": device.total_memory // (1024 ** 2),
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Recommendation logic #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def recommended_workers(
|
||||
*,
|
||||
kind: Literal["cpu", "io", "gpu"] = "cpu",
|
||||
io_factor: int = 4,
|
||||
io_max: Optional[int] = 512,
|
||||
) -> int:
|
||||
"""Return a *rule‑of‑thumb* worker count for the given workload *kind*.
|
||||
|
||||
Parameters
|
||||
----------
|
||||
kind:
|
||||
"cpu": CPU‑bound pure‑Python or C‑extension code (GIL matters)
|
||||
"io" : Network / disk I/O where threads mostly block waiting
|
||||
"gpu": GPU kernels (CUDA only, requires *torch*)
|
||||
io_factor:
|
||||
Multiplier for IO‑bound worker calculation (defaults to ``4``).
|
||||
io_max:
|
||||
Hard cap for IO‑bound workers (``None`` = no cap). Defaults to 512.
|
||||
|
||||
The function always respects container CPU quotas when present.
|
||||
"""
|
||||
if kind == "cpu":
|
||||
base = physical_cores()
|
||||
elif kind == "io":
|
||||
base = logical_cores() * io_factor
|
||||
if io_max is not None:
|
||||
base = min(base, io_max)
|
||||
elif kind == "gpu":
|
||||
info = gpu_info()
|
||||
if info is None:
|
||||
return 0
|
||||
# A very rough heuristic: one worker per SM
|
||||
base = info["sm_count"]
|
||||
else: # pragma: no cover
|
||||
raise ValueError("kind must be 'cpu', 'io', or 'gpu'") from None
|
||||
|
||||
quota = cgroup_cpu_limit()
|
||||
if quota is not None and kind != "gpu":
|
||||
return max(1, min(base, quota))
|
||||
return max(1, base)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Convenience snapshot of system info #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def system_snapshot() -> Dict[str, Any]:
|
||||
"""Return a JSON‑serialisable snapshot of parallelism‑related facts."""
|
||||
return {
|
||||
"platform_system": platform.system(),
|
||||
"platform_machine": platform.machine(),
|
||||
"logical_cores": logical_cores(),
|
||||
"physical_cores": physical_cores(),
|
||||
"docker_cpu_quota": cgroup_cpu_limit(),
|
||||
"gpu": gpu_info(),
|
||||
}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# CLI #
|
||||
# --------------------------------------------------------------------------- #
|
||||
|
||||
|
||||
def _cli() -> None: # pragma: no cover
|
||||
parser = argparse.ArgumentParser(
|
||||
prog="parallelism_advisor", description="Rule‑of‑thumb worker estimator"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--json",
|
||||
action="store_true",
|
||||
help="Print full hardware snapshot as JSON (default: pretty table)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--kind",
|
||||
choices=["cpu", "io", "gpu"],
|
||||
default="cpu",
|
||||
help="Workload kind to estimate workers for (default: cpu)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.json:
|
||||
print(json.dumps(system_snapshot(), indent=2))
|
||||
else:
|
||||
snap = system_snapshot()
|
||||
workers = recommended_workers(kind=args.kind)
|
||||
print("✦ System snapshot:")
|
||||
for k, v in snap.items():
|
||||
print(f" {k:20}: {v}")
|
||||
print(f"\nRecommended {args.kind} workers: {workers}")
|
||||
|
||||
|
||||
if __name__ == "__main__": # pragma: no cover
|
||||
_cli()
|
||||
Whitespace-only changes.
@@ -0,0 +1,50 @@
|
||||
|
||||
"""Singleton wrapper around the Supabase Python client.
|
||||
|
||||
Lives in **vutils** because it is pure infrastructure that other
|
||||
packages depend on (vecembed, vecmap, vquery, …). Nothing outside
|
||||
vutils should create its own Supabase client – always import
|
||||
:pyclass:`MySupabase` and call :pymeth:`MySupabase.get_client`.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
from supabase import Client, create_client
|
||||
|
||||
try:
|
||||
from .config_utils import load_dotenv_from_root # type: ignore
|
||||
load_dotenv_from_root()
|
||||
except ModuleNotFoundError:
|
||||
pass
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class MySupabase:
|
||||
"""A tiny singleton around :class:`supabase.Client`."""
|
||||
|
||||
_instance: Optional[Client] = None
|
||||
|
||||
@classmethod
|
||||
def _create_client(cls) -> Client:
|
||||
url = os.getenv("VECTORBASE_URL")
|
||||
key = os.getenv("VECTORBASE_API_KEY") or os.getenv("SUPABASE_SERVICE_ROLE_KEY")
|
||||
if not url or not key:
|
||||
logger.error("VECTORBASE_URL / VECTORBASE_API_KEY missing – check your env")
|
||||
raise ValueError("Supabase credentials not configured")
|
||||
logger.info("Creating Supabase client …")
|
||||
return create_client(url, key)
|
||||
|
||||
@classmethod
|
||||
def get_client(cls) -> Client:
|
||||
"""Return a singleton client instance."""
|
||||
if cls._instance is None:
|
||||
cls._instance = cls._create_client()
|
||||
return cls._instance
|
||||
|
||||
# convenience for ``MySupabase()``
|
||||
def __new__(cls) -> Client: # pragma: no cover
|
||||
return cls.get_client()
|
||||
@@ -0,0 +1,65 @@
|
||||
|
||||
"""Core utilities for resolving (schema, function, model) → table mapping."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Optional
|
||||
|
||||
from postgrest import APIResponse # type: ignore
|
||||
|
||||
from librarian_vspace.vutils.supabase_singleton import MySupabase
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class BaseVectorOperator:
|
||||
"""Resolve mapping and expose *table_fqn()* for children."""
|
||||
|
||||
def __init__(self, schema: str, function: str, model: str):
|
||||
self.spc = MySupabase.get_client()
|
||||
self.schema = schema
|
||||
self.function = function
|
||||
self.model = model
|
||||
self.model_id: Optional[int] = None
|
||||
self.table: Optional[str] = None
|
||||
self._resolve_ids()
|
||||
|
||||
# ---------------- public helpers ----------------
|
||||
def table_fqn(self) -> str:
|
||||
if not self.table:
|
||||
raise RuntimeError("VectorOperator not initialised – no table")
|
||||
return f"{self.schema}.{self.table}"
|
||||
|
||||
# ---------------- internals ----------------
|
||||
def _resolve_ids(self) -> None:
|
||||
self.model_id = self._rpc_get_model_id(self.model)
|
||||
if self.model_id is None:
|
||||
raise ValueError(f"Unknown model: {self.model}")
|
||||
self.table = self._rpc_get_table(self.function, self.model_id)
|
||||
if not self.table:
|
||||
raise ValueError(f"Could not map (function={self.function}, model_id={self.model_id}) → table")
|
||||
logger.info("VectorOperator ready → %s", self.table_fqn())
|
||||
|
||||
def _rpc_get_model_id(self, model: str) -> Optional[int]:
|
||||
try:
|
||||
resp: APIResponse = self.spc.schema(self.schema).rpc(
|
||||
"get_model_id_by_model", {"p_model": model}
|
||||
).execute()
|
||||
logger.debug("get_model_id_by_model → %s", resp.data)
|
||||
return int(resp.data) if resp.data not in (None, "") else None
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("RPC get_model_id_by_model failed for %s", model)
|
||||
return None
|
||||
|
||||
def _rpc_get_table(self, function: str, model_id: int) -> Optional[str]:
|
||||
try:
|
||||
resp: APIResponse = self.spc.schema(self.schema).rpc(
|
||||
"get_table_by_model", {"p_function": function, "p_model_library_id": model_id}
|
||||
).execute()
|
||||
logger.debug("get_table_by_model → %s", resp.data)
|
||||
if isinstance(resp.data, str):
|
||||
return resp.data.strip() or None
|
||||
return None
|
||||
except Exception: # noqa: BLE001
|
||||
logger.exception("RPC get_table_by_model failed")
|
||||
return None
|
||||
+413
@@ -0,0 +1,413 @@
|
||||
# --- START OF FILE librarian_vspace/vutils/vector_query_loader.py (Fixed Again) ---
|
||||
|
||||
"""Loads vectors from Supabase, with filtering capabilities, and handles parsing.
|
||||
Intended for querying/clustering tasks where filtering is needed.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional, Tuple, Union # Added Union
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
# You might not need TSNE here if this class is only for fetching,
|
||||
# but keeping it for generality as requested.
|
||||
from sklearn.manifold import TSNE
|
||||
|
||||
# Assuming vutils is installed or in the python path
|
||||
try:
|
||||
from librarian_vspace.vutils.vector_class import BaseVectorOperator
|
||||
except ImportError as e:
|
||||
logging.error(f"Failed to import vutils: {e}. Ensure vutils package is installed.")
|
||||
raise
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Define a specific error for this loader
|
||||
class VectorQueryLoaderError(Exception):
|
||||
"""Custom exception for VectorQueryLoader errors."""
|
||||
pass
|
||||
|
||||
|
||||
class VectorQueryLoader:
|
||||
"""Fetches vectors with filtering capabilities."""
|
||||
|
||||
# --- Define the constant for the course ID column name ---
|
||||
COURSE_ID_COLUMN_NAME = "course_id"
|
||||
# ----------------------------------------------------------
|
||||
|
||||
# Keep TSNE defaults for generality, though perhaps not used by clustering logic
|
||||
DEFAULT_TSNE_PARAMS = {
|
||||
"n_components": 3,
|
||||
"perplexity": 30.0, # Adjust based on dataset size (5-50 typically)
|
||||
"n_iter": 1000, # Minimum recommended iterations
|
||||
"learning_rate": "auto", # Usually a good default
|
||||
"init": "pca", # PCA initialization is often faster and more stable
|
||||
"random_state": 42, # For reproducibility
|
||||
"n_jobs": -1, # Use all available CPU cores
|
||||
"verbose": 1, # Log progress (controls scikit-learn's verbosity)
|
||||
}
|
||||
|
||||
|
||||
def __init__(self, schema: str, function: str, model: str, embedding_column: str = "embedding"):
|
||||
"""
|
||||
Initializes the loader.
|
||||
(Constructor remains the same)
|
||||
"""
|
||||
logger.info(f"Initializing VectorQueryLoader for {schema=}, {function=}, {model=}")
|
||||
try:
|
||||
self.operator = BaseVectorOperator(schema=schema, function=function, model=model)
|
||||
self.embedding_column = embedding_column
|
||||
if not self.operator.table:
|
||||
raise VectorQueryLoaderError("BaseVectorOperator failed to resolve table.")
|
||||
logger.info(f"Target table resolved to: {self.operator.table_fqn()}")
|
||||
except (ImportError, ValueError, RuntimeError) as e:
|
||||
logger.exception("Failed to initialize BaseVectorOperator.")
|
||||
# Raise the specific error for this loader
|
||||
raise VectorQueryLoaderError(f"Failed to initialize BaseVectorOperator: {e}") from e
|
||||
|
||||
|
||||
def _parse_vector_string(self, vector_str: Any) -> Optional[List[float]]:
|
||||
"""Safely parses the string representation of a vector into a list of floats."""
|
||||
if not isinstance(vector_str, str):
|
||||
# If it's already a list, tuple, or numpy array, assume it's valid vector data
|
||||
if isinstance(vector_str, (list, tuple, np.ndarray)) and all(isinstance(n, (int, float, np.number)) for n in vector_str):
|
||||
return [float(n) for n in vector_str] # Ensure elements are floats
|
||||
# Handle potential dict representation of vector (e.g., {"vector": [...]})
|
||||
if isinstance(vector_str, dict) and 'vector' in vector_str and isinstance(vector_str['vector'], (list, tuple, np.ndarray)):
|
||||
if all(isinstance(n, (int, float, np.number)) for n in vector_str['vector']):
|
||||
return [float(n) for n in vector_str['vector']]
|
||||
logger.debug(f"Unexpected type for vector parsing: {type(vector_str)}. Value starts with: {str(vector_str)[:50]}. Skipping.")
|
||||
return None
|
||||
|
||||
try:
|
||||
# Use json.loads which correctly handles [...] syntax from pgvector/Supabase
|
||||
parsed_list = json.loads(vector_str)
|
||||
if isinstance(parsed_list, list) and all(isinstance(n, (int, float)) for n in parsed_list):
|
||||
return [float(n) for n in parsed_list] # Ensure elements are floats
|
||||
else:
|
||||
logger.warning(f"Parsed vector string '{vector_str[:50]}...' but result is not a list of numbers. Result type: {type(parsed_list)}")
|
||||
return None
|
||||
except json.JSONDecodeError:
|
||||
logger.warning(f"Failed to JSON decode vector string: '{vector_str[:50]}...'")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.error(f"Unexpected error parsing vector string '{vector_str[:50]}...': {e}", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
def fetch_vectors(self, limit: Optional[int] = None, course_id: Optional[int] = None) -> pd.DataFrame:
|
||||
"""
|
||||
Fetches vectors and metadata from the resolved table, optionally filtered by course_id.
|
||||
Parses string representations of vectors into lists.
|
||||
|
||||
Args:
|
||||
limit: Optional limit on the number of rows to fetch (for large tables).
|
||||
course_id: Optional ID to filter vectors by the COURSE_ID_COLUMN_NAME.
|
||||
|
||||
Returns:
|
||||
A pandas DataFrame with columns like 'file_id', 'chunk', 'course_id',
|
||||
and the embedding_column (as list). Includes all columns found if not specified.
|
||||
|
||||
Raises:
|
||||
VectorQueryLoaderError: If fetching fails or no data is found after filtering.
|
||||
"""
|
||||
if not self.operator.table or not self.operator.schema:
|
||||
raise VectorQueryLoaderError("Operator not initialized, table name or schema is unknown.")
|
||||
|
||||
table_name = self.operator.table
|
||||
schema_name = self.operator.schema
|
||||
# Select all columns (*) to ensure we get 'course_id' and 'chunk' if they exist
|
||||
select_columns = "*" # Changed from fixed columns
|
||||
|
||||
logger.info(f"Fetching data from {schema_name}.{table_name} (selecting: {select_columns})...")
|
||||
try:
|
||||
query = self.operator.spc.schema(schema_name).table(table_name).select(select_columns)
|
||||
|
||||
# --- Add filtering logic using the constant ---
|
||||
if course_id is not None:
|
||||
logger.info(f"Filtering by {self.COURSE_ID_COLUMN_NAME}: {course_id}")
|
||||
# This will raise a Supabase API error if the column doesn't exist, which is caught below
|
||||
query = query.eq(self.COURSE_ID_COLUMN_NAME, course_id)
|
||||
# -----------------------------------------------------------------
|
||||
|
||||
if limit:
|
||||
logger.info(f"Applying limit: {limit}")
|
||||
query = query.limit(limit)
|
||||
|
||||
# Execute the query
|
||||
# --- REMOVE THIS LINE: query.url does not exist ---
|
||||
# logger.debug(f"Executing query: {query.url}") # Log the actual URL for debugging
|
||||
# --- END REMOVE ---
|
||||
response = query.execute()
|
||||
|
||||
if not response.data:
|
||||
logger.warning(f"No data found in table {self.operator.table_fqn()} matching criteria (course_id={course_id}).")
|
||||
# Return empty DataFrame with expected key columns for consistency
|
||||
# Use the columns from the initial DataFrame if available, otherwise assume minimal
|
||||
cols_if_empty = ['file_id', 'chunk', self.COURSE_ID_COLUMN_NAME, self.embedding_column]
|
||||
# Try to infer column dtypes from potential previous successful fetches or assume basic types
|
||||
return pd.DataFrame(columns=cols_if_empty) # Simple approach for empty case
|
||||
|
||||
logger.info(f"Fetched {len(response.data)} rows.")
|
||||
df = pd.DataFrame(response.data)
|
||||
|
||||
# --- Parse the embedding string into a list ---
|
||||
logger.info(f"Parsing string representation in '{self.embedding_column}' column...")
|
||||
if self.embedding_column not in df.columns:
|
||||
logger.warning(f"Embedding column '{self.embedding_column}' not found in fetched data. Cannot parse vectors.")
|
||||
# Return the DataFrame as is, but log the issue
|
||||
# You might want to drop rows missing the embedding column here if they shouldn't exist
|
||||
else:
|
||||
# Apply parsing only if the column exists
|
||||
parsed_embeddings = df[self.embedding_column].apply(self._parse_vector_string)
|
||||
# Overwrite the original string column with the parsed list (or None if parsing failed)
|
||||
df[self.embedding_column] = parsed_embeddings
|
||||
logger.debug(f"Sample '{self.embedding_column}' data after parsing (first 5 rows):\n{df[[self.embedding_column]].head()}")
|
||||
|
||||
# === Enhanced Debugging and Filtering for Embedding Column ===
|
||||
logger.info(f"Checking validity of parsed '{self.embedding_column}' column...")
|
||||
if self.embedding_column not in df.columns:
|
||||
# If column wasn't found and parsing wasn't attempted
|
||||
raise VectorQueryLoaderError(f"Required embedding column '{self.embedding_column}' missing in fetched data.")
|
||||
|
||||
# 1. Check for NULLs (includes rows where parsing failed and returned None)
|
||||
initial_count = len(df) # Count *before* filtering
|
||||
null_mask = df[self.embedding_column].isnull()
|
||||
null_count = null_mask.sum()
|
||||
if null_count > 0:
|
||||
logger.warning(f"Found {null_count} rows with NULL or unparsable vectors in '{self.embedding_column}'. These will be dropped.")
|
||||
|
||||
df_no_nulls = df.dropna(subset=[self.embedding_column]).copy() # Use copy after dropna
|
||||
count_after_null_drop = len(df_no_nulls)
|
||||
if count_after_null_drop < initial_count:
|
||||
logger.debug(f"{count_after_null_drop} rows remaining after dropping NULLs/unparsable.")
|
||||
|
||||
# 2. Check for non-empty list type (Ensures parsed data is correct)
|
||||
if not df_no_nulls.empty:
|
||||
def is_valid_list_of_floats(x):
|
||||
return isinstance(x, list) and len(x) > 0 and all(isinstance(n, float) for n in x)
|
||||
|
||||
valid_list_mask = df_no_nulls[self.embedding_column].apply(is_valid_list_of_floats)
|
||||
invalid_list_count = len(df_no_nulls) - valid_list_mask.sum()
|
||||
|
||||
if invalid_list_count > 0:
|
||||
# This indicates an issue with the parsing logic or unexpected data format
|
||||
logger.error(f"Found {invalid_list_count} rows where '{self.embedding_column}' is not a non-empty list of floats *after parsing*. These will be dropped.")
|
||||
# Optional: log examples of problematic entries here if needed
|
||||
# invalid_entries = df_no_nulls[~valid_list_mask][self.embedding_column]
|
||||
# for i, entry in enumerate(invalid_entries.head(5)):
|
||||
# logger.debug(f" Problematic entry example {i+1}: Type={type(entry)}, Value='{str(entry)[:100]}...'")
|
||||
|
||||
|
||||
df_filtered_embeddings = df_no_nulls[valid_list_mask].copy() # Use copy after filtering
|
||||
else:
|
||||
df_filtered_embeddings = df_no_nulls # Still empty
|
||||
|
||||
final_count = len(df_filtered_embeddings)
|
||||
# === End Enhanced Debugging ===
|
||||
|
||||
if final_count == 0 and initial_count > 0:
|
||||
# This happens if all rows were filtered out due to invalid embeddings
|
||||
logger.error(f"All {initial_count} rows were filtered out due to missing/invalid '{self.embedding_column}'. Check raw data and parsing.")
|
||||
# Return empty DF, but potentially raise a more critical error if this is unexpected
|
||||
# raise VectorQueryLoaderError(f"No valid embedding data found after processing. All {initial_count} rows were invalid.")
|
||||
# Ensure empty DF has expected columns
|
||||
# Try to preserve the original columns found before dropping
|
||||
cols_to_keep = [col for col in df.columns if col != self.embedding_column] + [self.embedding_column]
|
||||
return pd.DataFrame(columns=cols_to_keep) # Return empty DataFrame with original columns
|
||||
|
||||
|
||||
if final_count < initial_count:
|
||||
logger.warning(f"Filtered out {initial_count - final_count} rows total due to missing/invalid '{self.embedding_column}'.")
|
||||
|
||||
logger.info(f"Proceeding with {final_count} valid rows.")
|
||||
# Use the DataFrame *after* filtering out invalid embeddings
|
||||
df_result = df_filtered_embeddings
|
||||
|
||||
# Validate and potentially add placeholder metadata columns AFTER filtering
|
||||
# Use .get() for robustness if columns aren't always present
|
||||
required_metadata_cols = ['file_id', 'chunk', self.COURSE_ID_COLUMN_NAME]
|
||||
for col in required_metadata_cols:
|
||||
if col not in df_result.columns:
|
||||
# If filtering by course_id was requested and successful,
|
||||
# the COURSE_ID_COLUMN_NAME must exist. If not, it's an error.
|
||||
if col == self.COURSE_ID_COLUMN_NAME and course_id is not None:
|
||||
logger.error(f"Required column '{col}' missing in fetched data despite successful filtering. This is unexpected.")
|
||||
# Raise a critical error as filtering implies this column should exist
|
||||
raise VectorQueryLoaderError(f"Required column '{col}' missing in fetched data.")
|
||||
else:
|
||||
logger.warning(f"Required metadata column '{col}' missing in fetched data.")
|
||||
# Add a placeholder column of appropriate type if needed downstream
|
||||
if col == 'file_id':
|
||||
df_result['file_id'] = df_result.index.astype(str) # Use index as string ID
|
||||
elif col == 'chunk':
|
||||
df_result['chunk'] = "" # Empty string placeholder
|
||||
elif col == self.COURSE_ID_COLUMN_NAME and course_id is None:
|
||||
# Only add None if we weren't filtering and the column is missing
|
||||
df_result[self.COURSE_ID_COLUMN_NAME] = pd.Series([None] * len(df_result), index=df_result.index, dtype=int) # Or object
|
||||
|
||||
# Reorder columns for consistency if desired, e.g., [file_id, chunk, course_id, embedding, other_cols]
|
||||
# Example reordering (optional):
|
||||
# cols = required_metadata_cols + [col for col in df_result.columns if col not in required_metadata_cols]
|
||||
# # Ensure embedding column is towards the end if needed
|
||||
# if self.embedding_column in cols:
|
||||
# cols.remove(self.embedding_column)
|
||||
# cols.append(self.embedding_column)
|
||||
# # Handle duplicate column names if SELECT * returned duplicates (unlikely but possible)
|
||||
# df_result = df_result[list(dict.fromkeys(cols))] # Use dict to preserve order and get unique columns
|
||||
|
||||
|
||||
return df_result # Return the cleaned and validated DataFrame
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Failed to fetch data from {self.operator.table_fqn()}.")
|
||||
if 'relation' in str(e) and 'does not exist' in str(e):
|
||||
raise VectorQueryLoaderError(f"Table/Relation not found error: {e}. Check schema/table name and permissions.") from e
|
||||
# Catch the specific Supabase/PostgREST error if the column for filtering doesn't exist
|
||||
elif 'column' in str(e).lower() and self.COURSE_ID_COLUMN_NAME.lower() in str(e).lower() and 'does not exist' in str(e).lower():
|
||||
raise VectorQueryLoaderError(f"Filtering column '{self.COURSE_ID_COLUMN_NAME}' does not exist in table {self.operator.table_fqn()}. Query failed: {e}") from e
|
||||
else:
|
||||
raise VectorQueryLoaderError(f"Database query failed: {e}") from e
|
||||
|
||||
|
||||
# You can keep or remove the TSNE methods depending on whether this
|
||||
# loader should also handle visualization prep. If not, remove these.
|
||||
# Keeping for "generalized" request, but they use fetch_vectors now.
|
||||
|
||||
def reduce_dimensions(self, df: pd.DataFrame, tsne_params: Optional[Dict[str, Any]] = None) -> pd.DataFrame:
|
||||
"""
|
||||
Applies t-SNE to reduce embedding dimensions to 3D.
|
||||
(Code remains the same as previous correct version, assuming it works)
|
||||
"""
|
||||
logger.info("Starting t-SNE dimensionality reduction...")
|
||||
if df.empty:
|
||||
logger.warning("Input DataFrame for reduce_dimensions is empty. Returning empty DataFrame.")
|
||||
empty_df_with_cols = df.copy()
|
||||
for col in ['x', 'y', 'z']:
|
||||
if col not in empty_df_with_cols:
|
||||
empty_df_with_cols[col] = pd.Series(dtype=float)
|
||||
return empty_df_with_cols
|
||||
|
||||
# Use the embedding_column name from the instance
|
||||
embedding_col = self.embedding_column # Use local variable for clarity
|
||||
if embedding_col not in df.columns:
|
||||
# This should be caught by fetch_vectors, but defensive check
|
||||
raise VectorQueryLoaderError(f"Embedding column '{embedding_col}' missing in DataFrame passed to reduce_dimensions.")
|
||||
|
||||
# Ensure embeddings are lists of floats
|
||||
try:
|
||||
# Check if all entries in the embedding column are lists of floats
|
||||
# (This should be true if fetch_vectors's parsing/filtering worked)
|
||||
# If not strictly list[float], try robust conversion again
|
||||
if not all(isinstance(x, list) and all(isinstance(n, float) for n in x) for x in df[embedding_col]):
|
||||
logger.error(f"Data in '{embedding_col}' is not strictly list[float] format after fetching. Attempting conversion before stacking.")
|
||||
# Attempt robust conversion similar to the loader's parse method
|
||||
embeddings_list = []
|
||||
for item in df[embedding_col]:
|
||||
parsed_item = None
|
||||
if isinstance(item, str):
|
||||
try: parsed_item = json.loads(item)
|
||||
except json.JSONDecodeError: pass
|
||||
elif isinstance(item, (list, tuple, np.ndarray)):
|
||||
parsed_item = item
|
||||
elif isinstance(item, dict) and 'vector' in item and isinstance(item['vector'], (list, tuple, np.ndarray)):
|
||||
parsed_item = item['vector']
|
||||
|
||||
if isinstance(parsed_item, (list, tuple, np.ndarray)) and all(isinstance(val, (int, float, np.number)) for val in parsed_item):
|
||||
embeddings_list.append([float(n) for n in parsed_item])
|
||||
else:
|
||||
logger.debug(f"Skipping problematic embedding during secondary clean for t-SNE: {str(item)[:100]}...")
|
||||
|
||||
|
||||
if not embeddings_list:
|
||||
logger.error("No valid embeddings remained after secondary cleaning for t-SNE.")
|
||||
# No embeddings to process, return empty DF with t-SNE cols
|
||||
empty_df_with_cols = df.copy()
|
||||
for col in ['x', 'y', 'z']:
|
||||
if col not in empty_df_with_cols:
|
||||
empty_df_with_cols[col] = pd.Series(dtype=float)
|
||||
return empty_df_with_cols
|
||||
|
||||
|
||||
embeddings = np.array(embeddings_list, dtype=float)
|
||||
logger.warning("Successfully converted problematic embedding data for t-SNE stacking.")
|
||||
else:
|
||||
# Data is in the expected list of float format, proceed directly
|
||||
embeddings = np.array(df[embedding_col].to_list(), dtype=float)
|
||||
|
||||
except ValueError as ve:
|
||||
logger.exception(f"Failed to convert embedding list to numeric numpy array for t-SNE: {ve}. Ensure '{embedding_col}' contains valid vector data.")
|
||||
raise VectorQueryLoaderError(f"Data in '{embedding_col}' could not be converted to numeric vectors for t-SNE.") from ve
|
||||
except Exception as e:
|
||||
logger.exception(f"An unexpected error occurred while processing '{embedding_col}' column for t-SNE.")
|
||||
raise VectorQueryLoaderError(f"An unexpected error occurred while processing embedding data for t-SNE: {e}") from e
|
||||
|
||||
|
||||
if embeddings.ndim != 2:
|
||||
raise VectorQueryLoaderError(f"Embedding data has unexpected dimensions for t-SNE: {embeddings.ndim} (expected 2). Shape: {embeddings.shape}")
|
||||
|
||||
n_samples = embeddings.shape[0]
|
||||
|
||||
if n_samples < 2:
|
||||
logger.warning(f"Found only {n_samples} valid vector(s). t-SNE requires at least 2. Assigning default 3D coordinates.")
|
||||
default_coords = [[0.0, 0.0, 0.0]] * n_samples
|
||||
df_copy = df.copy() # Work on a copy
|
||||
df_copy[['x', 'y', 'z']] = default_coords
|
||||
return df_copy
|
||||
|
||||
|
||||
logger.info(f"Applying t-SNE to {n_samples} vectors of dimension {embeddings.shape[1]}...")
|
||||
|
||||
current_tsne_params = self.DEFAULT_TSNE_PARAMS.copy()
|
||||
if tsne_params:
|
||||
current_tsne_params.update(tsne_params)
|
||||
logger.info(f"Using custom t-SNE params: {tsne_params}")
|
||||
|
||||
# Adjust perplexity if sample size is too small
|
||||
if n_samples <= current_tsne_params['perplexity']:
|
||||
new_perplexity = max(5.0, float(n_samples - 1)) # Perplexity must be less than num samples
|
||||
if new_perplexity < 5.0: # Ensure minimum perplexity if possible
|
||||
if n_samples > 5: new_perplexity = 5.0 # if samples > 5, use 5
|
||||
else: new_perplexity = max(1.0, float(n_samples - 1)) # Otherwise just samples - 1 (min 1)
|
||||
if current_tsne_params['perplexity'] != new_perplexity: # Only log if changed
|
||||
logger.warning(f"Adjusting t-SNE perplexity from {current_tsne_params['perplexity']:.1f} "
|
||||
f"to {new_perplexity:.1f} due to low sample count ({n_samples}).")
|
||||
current_tsne_params['perplexity'] = new_perplexity
|
||||
|
||||
# Adjust n_iter for potential large datasets if not already high
|
||||
# This check is copied from your original loader, keep it consistent
|
||||
# if n_samples * embeddings.shape[1] > 100000 and current_tsne_params['n_iter'] < 1000:
|
||||
# logger.warning(f"Dataset size seems large, increasing t-SNE n_iter from {current_tsne_params['n_iter']} to 1000 for better convergence.")
|
||||
# current_tsne_params['n_iter'] = 1000
|
||||
|
||||
|
||||
try:
|
||||
logger.debug(f"Final t-SNE parameters: {current_tsne_params}")
|
||||
tsne = TSNE(**current_tsne_params)
|
||||
reduced_embeddings = tsne.fit_transform(embeddings)
|
||||
|
||||
# Ensure the DataFrame index aligns with the embeddings if rows were dropped
|
||||
df_copy = df.copy() # Work on a copy to avoid modifying the original index alignment issue if any
|
||||
df_copy[['x', 'y', 'z']] = reduced_embeddings
|
||||
logger.info("t-SNE reduction complete.")
|
||||
return df_copy
|
||||
|
||||
except Exception as e:
|
||||
logger.exception("t-SNE dimensionality reduction failed.")
|
||||
raise VectorQueryLoaderError(f"t-SNE failed: {e}") from e
|
||||
|
||||
|
||||
# Keep load_and_reduce for generality, though clustering might not use it
|
||||
def load_and_reduce(self, limit: Optional[int] = None, course_id: Optional[int] = None, tsne_params: Optional[Dict[str, Any]] = None) -> pd.DataFrame:
|
||||
"""Orchestrates fetching vectors (optionally filtered) and reducing dimensions."""
|
||||
logger.info("Starting vector load (filtered) and reduction process...")
|
||||
# Use the fetch_vectors method which handles filtering
|
||||
df_raw_filtered = self.fetch_vectors(limit=limit, course_id=course_id)
|
||||
df_reduced = self.reduce_dimensions(df_raw_filtered, tsne_params=tsne_params)
|
||||
logger.info("Vector load and reduction process finished.")
|
||||
return df_reduced
|
||||
|
||||
|
||||
# --- END OF FILE librarian_vspace/vutils/vector_query_loader.py ---
|
||||
+3315
File diff suppressed because it is too large.
Load diff
Reference in new issue
Block a user