Initialize Monorepo

This commit is contained in:
DotNaos committed 2025-05-24 12:15:48 +02:00
commit f80792d739
106 files changed
+17076

No files matched your search

@@ -0,0 +1,47 @@
import os
def chunk_file(input_file, output_dir=None, start_num=1, padding=2):
"""
Split a file into chunks and save each chunk as a separate file.
Args:
input_file (str): Path to the input file
output_dir (str, optional): Directory to save chunk files. Defaults to current directory.
start_num (int, optional): Starting number for the chunk files. Defaults to 1.
padding (int, optional): Number of digits to pad the incremental numbers. Defaults to 2.
"""
if output_dir and not os.path.exists(output_dir):
os.makedirs(output_dir)
with open(input_file) as f:
content = f.read()
chunks = content.split("---")
chunk_count = start_num
for chunk in chunks:
chunk = chunk.replace('---', '').strip()
if not chunk: # Skip empty chunks
continue
# Define output path with padded incremental number
file_name = f'chunk_{chunk_count:0{padding}d}.md'
if output_dir:
outfile_path = os.path.join(output_dir, file_name)
else:
outfile_path = file_name
with open(outfile_path, 'w') as outfile:
outfile.write(chunk)
chunk_count += 1
return chunk_count - start_num # Return the number of chunks written
# Example usage
if __name__ == "__main__":
#input_file = "/home/gra/PycharmProjects/librarian_vspace/examples/chunks/knowledge_chunks_detailed.md"
input_file = "/home/gra/PycharmProjects/librarian_vspace/examples/chunks/knowledge_chunks_1500.md"
# You can specify an output directory or omit it to use the current directory
output_dir = "/examples/chunks/chunk_md_x"
chunk_file(input_file, output_dir)
@@ -0,0 +1,43 @@
#!/usr/bin/env python3
"""examples/demo_run_cluster_export.py
Launch ClusterExportWorker via FlowArtifact wrapper, mirroring the embedder demo.
"""
from __future__ import annotations
import asyncio
import logging
from pathlib import Path
import os
from librarian_vspace.vquery.cluster_export_worker import ClusterExportWorker, ClusterExportInput
from librarian_core.workers.base import FlowArtifact
COURSE_ID = 15512 # example id
logger = logging.getLogger(__name__)
def _load_env(path: Path) -> None:
if not path.is_file():
return
for line in path.read_text().splitlines():
if line.strip() and not line.startswith("#") and "=" in line:
k, v = [p.strip() for p in line.split("=", 1)]
os.environ.setdefault(k, v)
async def _main() -> None:
payload = ClusterExportInput(course_id=COURSE_ID)
worker = ClusterExportWorker()
art = FlowArtifact.new(run_id="", dir=Path.cwd(), data=payload)
result_artifact = await worker.flow()(art) # FlowArtifact
output = result_artifact.data # ClusterExportOutput
logger.info("✅ Worker finished – output directory: %s", output.output_dir)
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
APP_DIR = Path(__file__).resolve().parent
_load_env(APP_DIR / ".env")
asyncio.run(_main())
@@ -0,0 +1,86 @@
#!/usr/bin/env python3
from __future__ import annotations
import asyncio
import logging
import os
import random
from pathlib import Path
from typing import Any, List, Dict
import json
from librarian_vspace.vecembed.embedder_worker import EmbedderWorker, EmbedderInput
from librarian_core.workers.base import FlowArtifact
from librarian_core.temp_payloads.chunk_data import ChunkData
# ------------------------------------------------------------------ #
# Configuration
# ------------------------------------------------------------------ #
# Folder with the small sample dataset (3 × .md files)
DEMO_PATH: Path = Path("/home/gra/PycharmProjects/librarian_vspace/examples/chunks/moodle_chunks/51cd7cf6-e782-4f17-af00-30852cdcd5fc/51cd7cf6-e782-4f17-af00-30852cdcd5fc/data/FS25/Effiziente_Algorithmen").expanduser()
#DEMO_PATH: Path = Path("/home/gra/PycharmProjects/librarian_vspace/examples/chunks/chunk_md").expanduser()
# Where to write the concatenated text file
# (one level above the dataset folder keeps things tidy)
COURSE_ID_POOL = [16301, 16091, 17505, 18239, 17503, 15512]
logger = logging.getLogger(__name__)
INPUT_MODEL=Path("/home/gra/PycharmProjects/librarian_vspace/examples/chunks/moodle_chunks/51cd7cf6-e782-4f17-af00-30852cdcd5fc/51cd7cf6-e782-4f17-af00-30852cdcd5fc/result.json")
# ------------------------------------------------------------------ #
def _load_env(path: Path) -> None:
"""Load KEY=VALUE pairs from a .env file if present."""
if not path.is_file():
return
for line in path.read_text().splitlines():
if line.strip() and not line.startswith("#") and "=" in line:
k, v = [p.strip() for p in line.split("=", 1)]
os.environ.setdefault(k, v)
def discover_chunks(root: Path) -> List[Path]:
"""Return all markdown files in the dataset folder."""
return sorted(root.glob("*.md"))
def build_course(root: Path) -> Dict[str, Any]:
"""Minimal dict that satisfies EmbedderWorker's `chunk_course`."""
files = [
{"file_name": p.name, "file_id": str(random.getrandbits(24))}
for p in discover_chunks(root)
]
if not files:
raise FileNotFoundError(f"No .md files found in {root}")
return {
"path": str(root),
"files": files,
#"course_id": str(random.choice(COURSE_ID_POOL)),
"course_id": "18240"
}
# ------------------------------------------------------------------ #
async def _main() -> None:
course = build_course(DEMO_PATH)
concat_path = DEMO_PATH
with open(INPUT_MODEL, 'r') as file:
json_data = json.load(file)
#payload = EmbedderInput(chunk_course=course, concat_path=concat_path)
payload = ChunkData.model_validate_json(json_data)
worker = EmbedderWorker()
logger.info("🔨 Launching EmbedderWorker …")
art = FlowArtifact.new(run_id="", dir=concat_path, data=payload)
result = await worker.flow()(art) # type: ignore[arg-type]
logger.info("✅ Worker finished: %s", result)
# ------------------------------------------------------------------ #
if __name__ == "__main__":
APP_DIR = Path(__file__).resolve().parent
_load_env(APP_DIR / ".env")
logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
asyncio.run(_main())
@@ -0,0 +1,66 @@
#!/usr/bin/env python3
"""examples/demo_run_query.py
Runs QueryWorker via FlowArtifact wrapper (mirrors cluster export demo).
"""
from __future__ import annotations
import asyncio
import logging
import os
from pathlib import Path
from librarian_vspace.vquery.query_worker import QueryWorker, QueryInput
from librarian_vspace.models.query_model import VectorSearchRequest
from librarian_core.workers.base import FlowArtifact
# ------------------------------------------------------------------ #
# Config
# ------------------------------------------------------------------ #
SEARCH_STRING = "integration"
COURSE_FILTER_GT = 900 # adjust if needed
logger = logging.getLogger(__name__)
def _load_env(path: Path) -> None:
if not path.is_file():
return
for line in path.read_text().splitlines():
if line.strip() and not line.startswith("#") and "=" in line:
k, v = [p.strip() for p in line.split("=", 1)]
os.environ.setdefault(k, v)
# ------------------------------------------------------------------ #
async def _main() -> None:
# Vector search request
vs_req = VectorSearchRequest(
interface_name=os.getenv("EMBED_INTERFACE", "ollama"),
model_name=os.getenv("EMBED_MODEL", "snowflake-arctic-embed2"),
search_string=SEARCH_STRING,
filters={"file_id": ("gt", COURSE_FILTER_GT)},
top_k=10,
)
payload = QueryInput(
request=vs_req,
db_schema=os.getenv("VECTOR_SCHEMA", "librarian"),
rpc_function=os.getenv("VECTOR_FUNCTION", "pdf_chunking"),
embed_model=os.getenv("EMBED_MODEL", "snowflake-arctic-embed2"),
)
worker = QueryWorker()
art = FlowArtifact.new(run_id="", dir=Path.cwd(), data=payload)
result_artifact = await worker.flow()(art) # FlowArtifact
response = result_artifact.data # VectorSearchResponse
logger.info("✅ Worker finished – received %s results", response.total)
for idx, ck in enumerate(response.results, 1):
logger.info("• %s: %s", idx, ck.chunk[:80] + ("…" if len(ck.chunk or '') > 80 else ""))
# ------------------------------------------------------------------ #
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
APP_DIR = Path(__file__).resolve().parent
_load_env(APP_DIR / ".env")
asyncio.run(_main())
@@ -0,0 +1,43 @@
#!/usr/bin/env python3
"""examples/demo_run_tsne_export.py
Launch TsneExportWorker via FlowArtifact wrapper.
"""
from __future__ import annotations
import asyncio
import logging
from pathlib import Path
import os
from librarian_vspace.vecview.tsne_export_worker import TsneExportWorker, TsneExportInput
from librarian_core.workers.base import FlowArtifact
COURSE_ID = 18240 # choose a course with embeddings
logger = logging.getLogger(__name__)
def _load_env(path: Path) -> None:
if not path.is_file():
return
for line in path.read_text().splitlines():
if line.strip() and not line.startswith("#") and "=" in line:
k, v = [p.strip() for p in line.split("=", 1)]
os.environ.setdefault(k, v)
async def _main() -> None:
payload = TsneExportInput(course_id=COURSE_ID)
worker = TsneExportWorker()
art = FlowArtifact.new(run_id="", dir=Path.cwd(), data=payload)
result_artifact = await worker.flow()(art) # FlowArtifact
output = result_artifact.data # TsneExportOutput
logger.info("✅ Worker finished – JSON file: %s", output.json_path)
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
APP_DIR = Path(__file__).resolve().parent
_load_env(APP_DIR / ".env")
asyncio.run(_main())
@@ -0,0 +1,4 @@
from librarian_vspace.vutils.parallelism_advisor import recommended_workers
print(recommended_workers(kind="cpu"))
print(recommended_workers(kind="io"))
print(recommended_workers(kind="gpu"))
@@ -0,0 +1,141 @@
#!/usr/bin/env python3
"""
Loads vector data using vecmap.loader, reduces dimensions via t-SNE,
and launches an interactive 3D visualization using vecmap.visualizer (Dash/Plotly).
Configuration is primarily driven by environment variables.
"""
from __future__ import annotations
import logging
import os
import pathlib
import sys
import pandas as pd
# Define application directory relative to this script file
APP_DIR = pathlib.Path(__file__).resolve().parent
# Define the source directory containing vecmap, vutils, etc.
SRC_DIR = APP_DIR.parent / "src"
# Define path to .env file relative to APP_DIR
DOTENV_PATH = APP_DIR / ".env"
# --- Explicitly Manage sys.path ---
app_dir_str = str(APP_DIR)
src_dir_str = str(SRC_DIR)
if app_dir_str in sys.path:
try: sys.path.remove(app_dir_str)
except ValueError: pass
if src_dir_str not in sys.path:
sys.path.insert(0, src_dir_str)
elif sys.path[0] != src_dir_str:
try: sys.path.remove(src_dir_str)
except ValueError: pass
sys.path.insert(0, src_dir_str)
print(f"[DEBUG] sys.path start: {sys.path[:3]}")
# --- .env Loader ---
def _load_env_file(path: pathlib.Path) -> None:
print(f"Attempting to load .env file from: {path}")
if not path.is_file(): print(f".env file not found at {path}, skipping."); return
loaded, skipped = 0, 0
try:
with open(path, 'r', encoding='utf-8') as f:
for line in f:
line = line.strip();
if not line or line.startswith("#") or "=" not in line: continue
key, val = line.split("=", 1); key, val = key.strip(), val.strip()
if key not in os.environ: os.environ[key] = val; loaded += 1
else: skipped += 1
print(f"Loaded {loaded} new vars, skipped {skipped} existing vars from .env")
except Exception as e: print(f"Error reading .env file at {path}: {e}")
_load_env_file(DOTENV_PATH)
# --- Logging Setup ---
log_level_str = os.getenv("VECMAP_DEBUG", "false").lower()
log_level = logging.DEBUG if log_level_str in ("true", "1") else logging.INFO
logging.basicConfig(level=log_level, format='[%(asctime)s] [%(levelname)s] [%(name)s:%(lineno)d] %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
if log_level > logging.DEBUG:
for logger_name in ["urllib3", "httpx", "supabase"]: logging.getLogger(logger_name).setLevel(logging.WARNING)
logger = logging.getLogger(__name__)
# --- Imports ---
try:
from librarian_vspace.vecmap.loader import VectorLoader, VectorLoaderError
from librarian_vspace.vecmap.visualizer import VectorVisualizer # Removed DEFAULT_N_CLUSTERS import
import librarian_vspace.vutils
import librarian_vspace.vecembed
logger.debug("Successfully imported components.")
except ImportError as e:
logger.error(f"Failed to import necessary modules: {e}", exc_info=True)
sys.exit(1)
# --- Main Logic ---
def main() -> None:
logger.info("--- Starting VecMap Visualizer ---")
# --- Configuration ---
db_schema = os.getenv("VECTOR_SCHEMA", "librarian")
db_function = os.getenv("VECTOR_FUNCTION", "pdf_chunking")
model_name = os.getenv("EMBED_MODEL", "snowflake-arctic-embed2")
interface_name = os.getenv("EMBED_INTERFACE", "ollama")
embedding_column = os.getenv("EMBEDDING_COLUMN", "embedding")
try: limit_str = os.getenv("VECMAP_LIMIT"); data_limit = int(limit_str) if limit_str else None
except ValueError: logger.warning(f"Invalid VECMAP_LIMIT. Ignoring."); data_limit = None
try: perplexity_str = os.getenv("VECMAP_PERPLEXITY", "30.0"); tsne_perplexity = float(perplexity_str)
except ValueError: logger.warning(f"Invalid VECMAP_PERPLEXITY. Using 30.0."); tsne_perplexity = 30.0
# n_clusters configuration removed
dash_host = os.getenv("VECMAP_HOST", "127.0.0.1")
try: port_str = os.getenv("VECMAP_PORT", "8050"); dash_port = int(port_str)
except ValueError: logger.warning(f"Invalid VECMAP_PORT. Using 8050."); dash_port = 8050
dash_debug = log_level == logging.DEBUG
logger.info("Effective Configuration:")
logger.info(f" Database: schema={db_schema}, function={db_function}")
logger.info(f" Model/Interface: model={model_name}, interface={interface_name}")
logger.info(f" Data Params: column={embedding_column}, limit={data_limit}")
logger.info(f" Processing: perplexity={tsne_perplexity} (n_clusters is now dynamic)") # Updated log
logger.info(f" Server: host={dash_host}, port={dash_port}, debug={dash_debug}")
# --- 1. Initial Load and Reduce ---
initial_df_reduced = pd.DataFrame()
try:
logger.info("Performing initial data load and processing...")
loader = VectorLoader(schema=db_schema, function=db_function, model=model_name, embedding_column=embedding_column)
tsne_params = {"perplexity": tsne_perplexity}
initial_df_reduced = loader.load_and_reduce(limit=data_limit, tsne_params=tsne_params)
if initial_df_reduced.empty: logger.warning("Initial data load resulted in an empty dataset.")
else: logger.info(f"Successfully loaded and reduced {len(initial_df_reduced)} vectors initially.")
except VectorLoaderError as e: logger.error(f"Initial data load failed: {e}", exc_info=dash_debug)
except Exception as e: logger.error(f"Unexpected error during initial data load: {e}", exc_info=dash_debug)
# --- 2. Initialize and Start Visualization ---
try:
logger.info("Initializing VectorVisualizer...")
visualizer = VectorVisualizer(
initial_data=initial_df_reduced,
db_schema=db_schema,
db_function=db_function,
interface_name=interface_name,
model_name=model_name,
embedding_column=embedding_column,
initial_limit=data_limit,
initial_perplexity=tsne_perplexity
# n_clusters argument removed
)
logger.info("Launching visualizer...")
visualizer.run(host=dash_host, port=dash_port, debug=dash_debug)
except TypeError as te:
logger.error(f"TypeError during VectorVisualizer initialization: {te}", exc_info=True)
sys.exit(1)
except Exception as e:
logger.error(f"Failed to initialize or run visualizer: {e}", exc_info=dash_debug)
sys.exit(1)
logger.info("--- VecMap Visualizer finished ---")
if __name__ == "__main__":
main()