Initialize Monorepo
This commit is contained in:
commit
f80792d739
106 files changed
+17076
No files matched your search
@@ -0,0 +1,47 @@
|
||||
import os
|
||||
|
||||
def chunk_file(input_file, output_dir=None, start_num=1, padding=2):
|
||||
"""
|
||||
Split a file into chunks and save each chunk as a separate file.
|
||||
|
||||
Args:
|
||||
input_file (str): Path to the input file
|
||||
output_dir (str, optional): Directory to save chunk files. Defaults to current directory.
|
||||
start_num (int, optional): Starting number for the chunk files. Defaults to 1.
|
||||
padding (int, optional): Number of digits to pad the incremental numbers. Defaults to 2.
|
||||
"""
|
||||
if output_dir and not os.path.exists(output_dir):
|
||||
os.makedirs(output_dir)
|
||||
|
||||
with open(input_file) as f:
|
||||
content = f.read()
|
||||
chunks = content.split("---")
|
||||
|
||||
chunk_count = start_num
|
||||
for chunk in chunks:
|
||||
chunk = chunk.replace('---', '').strip()
|
||||
if not chunk: # Skip empty chunks
|
||||
continue
|
||||
|
||||
# Define output path with padded incremental number
|
||||
file_name = f'chunk_{chunk_count:0{padding}d}.md'
|
||||
if output_dir:
|
||||
outfile_path = os.path.join(output_dir, file_name)
|
||||
else:
|
||||
outfile_path = file_name
|
||||
|
||||
with open(outfile_path, 'w') as outfile:
|
||||
outfile.write(chunk)
|
||||
|
||||
chunk_count += 1
|
||||
|
||||
return chunk_count - start_num # Return the number of chunks written
|
||||
|
||||
# Example usage
|
||||
if __name__ == "__main__":
|
||||
#input_file = "/home/gra/PycharmProjects/librarian_vspace/examples/chunks/knowledge_chunks_detailed.md"
|
||||
input_file = "/home/gra/PycharmProjects/librarian_vspace/examples/chunks/knowledge_chunks_1500.md"
|
||||
# You can specify an output directory or omit it to use the current directory
|
||||
output_dir = "/examples/chunks/chunk_md_x"
|
||||
chunk_file(input_file, output_dir)
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
|
||||
#!/usr/bin/env python3
|
||||
"""examples/demo_run_cluster_export.py
|
||||
|
||||
Launch ClusterExportWorker via FlowArtifact wrapper, mirroring the embedder demo.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import os
|
||||
|
||||
from librarian_vspace.vquery.cluster_export_worker import ClusterExportWorker, ClusterExportInput
|
||||
from librarian_core.workers.base import FlowArtifact
|
||||
|
||||
COURSE_ID = 15512 # example id
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def _load_env(path: Path) -> None:
|
||||
if not path.is_file():
|
||||
return
|
||||
for line in path.read_text().splitlines():
|
||||
if line.strip() and not line.startswith("#") and "=" in line:
|
||||
k, v = [p.strip() for p in line.split("=", 1)]
|
||||
os.environ.setdefault(k, v)
|
||||
|
||||
async def _main() -> None:
|
||||
payload = ClusterExportInput(course_id=COURSE_ID)
|
||||
|
||||
worker = ClusterExportWorker()
|
||||
art = FlowArtifact.new(run_id="", dir=Path.cwd(), data=payload)
|
||||
result_artifact = await worker.flow()(art) # FlowArtifact
|
||||
|
||||
output = result_artifact.data # ClusterExportOutput
|
||||
logger.info("✅ Worker finished – output directory: %s", output.output_dir)
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
|
||||
APP_DIR = Path(__file__).resolve().parent
|
||||
_load_env(APP_DIR / ".env")
|
||||
asyncio.run(_main())
|
||||
@@ -0,0 +1,86 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
from pathlib import Path
|
||||
from typing import Any, List, Dict
|
||||
import json
|
||||
|
||||
from librarian_vspace.vecembed.embedder_worker import EmbedderWorker, EmbedderInput
|
||||
from librarian_core.workers.base import FlowArtifact
|
||||
from librarian_core.temp_payloads.chunk_data import ChunkData
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Configuration
|
||||
# ------------------------------------------------------------------ #
|
||||
# Folder with the small sample dataset (3 × .md files)
|
||||
DEMO_PATH: Path = Path("/home/gra/PycharmProjects/librarian_vspace/examples/chunks/moodle_chunks/51cd7cf6-e782-4f17-af00-30852cdcd5fc/51cd7cf6-e782-4f17-af00-30852cdcd5fc/data/FS25/Effiziente_Algorithmen").expanduser()
|
||||
#DEMO_PATH: Path = Path("/home/gra/PycharmProjects/librarian_vspace/examples/chunks/chunk_md").expanduser()
|
||||
|
||||
# Where to write the concatenated text file
|
||||
# (one level above the dataset folder keeps things tidy)
|
||||
COURSE_ID_POOL = [16301, 16091, 17505, 18239, 17503, 15512]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
INPUT_MODEL=Path("/home/gra/PycharmProjects/librarian_vspace/examples/chunks/moodle_chunks/51cd7cf6-e782-4f17-af00-30852cdcd5fc/51cd7cf6-e782-4f17-af00-30852cdcd5fc/result.json")
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
def _load_env(path: Path) -> None:
|
||||
"""Load KEY=VALUE pairs from a .env file if present."""
|
||||
if not path.is_file():
|
||||
return
|
||||
for line in path.read_text().splitlines():
|
||||
if line.strip() and not line.startswith("#") and "=" in line:
|
||||
k, v = [p.strip() for p in line.split("=", 1)]
|
||||
os.environ.setdefault(k, v)
|
||||
|
||||
|
||||
def discover_chunks(root: Path) -> List[Path]:
|
||||
"""Return all markdown files in the dataset folder."""
|
||||
return sorted(root.glob("*.md"))
|
||||
|
||||
|
||||
def build_course(root: Path) -> Dict[str, Any]:
|
||||
"""Minimal dict that satisfies EmbedderWorker's `chunk_course`."""
|
||||
files = [
|
||||
{"file_name": p.name, "file_id": str(random.getrandbits(24))}
|
||||
for p in discover_chunks(root)
|
||||
]
|
||||
if not files:
|
||||
raise FileNotFoundError(f"No .md files found in {root}")
|
||||
return {
|
||||
"path": str(root),
|
||||
"files": files,
|
||||
#"course_id": str(random.choice(COURSE_ID_POOL)),
|
||||
"course_id": "18240"
|
||||
}
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
async def _main() -> None:
|
||||
course = build_course(DEMO_PATH)
|
||||
concat_path = DEMO_PATH
|
||||
|
||||
with open(INPUT_MODEL, 'r') as file:
|
||||
json_data = json.load(file)
|
||||
|
||||
#payload = EmbedderInput(chunk_course=course, concat_path=concat_path)
|
||||
payload = ChunkData.model_validate_json(json_data)
|
||||
worker = EmbedderWorker()
|
||||
logger.info("🔨 Launching EmbedderWorker …")
|
||||
art = FlowArtifact.new(run_id="", dir=concat_path, data=payload)
|
||||
result = await worker.flow()(art) # type: ignore[arg-type]
|
||||
|
||||
logger.info("✅ Worker finished: %s", result)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
if __name__ == "__main__":
|
||||
APP_DIR = Path(__file__).resolve().parent
|
||||
_load_env(APP_DIR / ".env")
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
|
||||
asyncio.run(_main())
|
||||
@@ -0,0 +1,66 @@
|
||||
|
||||
#!/usr/bin/env python3
|
||||
"""examples/demo_run_query.py
|
||||
|
||||
Runs QueryWorker via FlowArtifact wrapper (mirrors cluster export demo).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
from librarian_vspace.vquery.query_worker import QueryWorker, QueryInput
|
||||
from librarian_vspace.models.query_model import VectorSearchRequest
|
||||
from librarian_core.workers.base import FlowArtifact
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
# Config
|
||||
# ------------------------------------------------------------------ #
|
||||
SEARCH_STRING = "integration"
|
||||
COURSE_FILTER_GT = 900 # adjust if needed
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def _load_env(path: Path) -> None:
|
||||
if not path.is_file():
|
||||
return
|
||||
for line in path.read_text().splitlines():
|
||||
if line.strip() and not line.startswith("#") and "=" in line:
|
||||
k, v = [p.strip() for p in line.split("=", 1)]
|
||||
os.environ.setdefault(k, v)
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
async def _main() -> None:
|
||||
# Vector search request
|
||||
vs_req = VectorSearchRequest(
|
||||
interface_name=os.getenv("EMBED_INTERFACE", "ollama"),
|
||||
model_name=os.getenv("EMBED_MODEL", "snowflake-arctic-embed2"),
|
||||
search_string=SEARCH_STRING,
|
||||
filters={"file_id": ("gt", COURSE_FILTER_GT)},
|
||||
top_k=10,
|
||||
)
|
||||
|
||||
payload = QueryInput(
|
||||
request=vs_req,
|
||||
db_schema=os.getenv("VECTOR_SCHEMA", "librarian"),
|
||||
rpc_function=os.getenv("VECTOR_FUNCTION", "pdf_chunking"),
|
||||
embed_model=os.getenv("EMBED_MODEL", "snowflake-arctic-embed2"),
|
||||
)
|
||||
|
||||
worker = QueryWorker()
|
||||
art = FlowArtifact.new(run_id="", dir=Path.cwd(), data=payload)
|
||||
result_artifact = await worker.flow()(art) # FlowArtifact
|
||||
|
||||
response = result_artifact.data # VectorSearchResponse
|
||||
logger.info("✅ Worker finished – received %s results", response.total)
|
||||
for idx, ck in enumerate(response.results, 1):
|
||||
logger.info("• %s: %s", idx, ck.chunk[:80] + ("…" if len(ck.chunk or '') > 80 else ""))
|
||||
|
||||
# ------------------------------------------------------------------ #
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
|
||||
APP_DIR = Path(__file__).resolve().parent
|
||||
_load_env(APP_DIR / ".env")
|
||||
asyncio.run(_main())
|
||||
@@ -0,0 +1,43 @@
|
||||
|
||||
#!/usr/bin/env python3
|
||||
"""examples/demo_run_tsne_export.py
|
||||
|
||||
Launch TsneExportWorker via FlowArtifact wrapper.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
import asyncio
|
||||
import logging
|
||||
from pathlib import Path
|
||||
import os
|
||||
|
||||
from librarian_vspace.vecview.tsne_export_worker import TsneExportWorker, TsneExportInput
|
||||
from librarian_core.workers.base import FlowArtifact
|
||||
|
||||
COURSE_ID = 18240 # choose a course with embeddings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
def _load_env(path: Path) -> None:
|
||||
if not path.is_file():
|
||||
return
|
||||
for line in path.read_text().splitlines():
|
||||
if line.strip() and not line.startswith("#") and "=" in line:
|
||||
k, v = [p.strip() for p in line.split("=", 1)]
|
||||
os.environ.setdefault(k, v)
|
||||
|
||||
async def _main() -> None:
|
||||
payload = TsneExportInput(course_id=COURSE_ID)
|
||||
|
||||
worker = TsneExportWorker()
|
||||
art = FlowArtifact.new(run_id="", dir=Path.cwd(), data=payload)
|
||||
result_artifact = await worker.flow()(art) # FlowArtifact
|
||||
|
||||
output = result_artifact.data # TsneExportOutput
|
||||
logger.info("✅ Worker finished – JSON file: %s", output.json_path)
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO, format="[%(levelname)s] %(message)s")
|
||||
APP_DIR = Path(__file__).resolve().parent
|
||||
_load_env(APP_DIR / ".env")
|
||||
asyncio.run(_main())
|
||||
@@ -0,0 +1,4 @@
|
||||
from librarian_vspace.vutils.parallelism_advisor import recommended_workers
|
||||
print(recommended_workers(kind="cpu"))
|
||||
print(recommended_workers(kind="io"))
|
||||
print(recommended_workers(kind="gpu"))
|
||||
@@ -0,0 +1,141 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Loads vector data using vecmap.loader, reduces dimensions via t-SNE,
|
||||
and launches an interactive 3D visualization using vecmap.visualizer (Dash/Plotly).
|
||||
|
||||
Configuration is primarily driven by environment variables.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import sys
|
||||
import pandas as pd
|
||||
|
||||
# Define application directory relative to this script file
|
||||
APP_DIR = pathlib.Path(__file__).resolve().parent
|
||||
# Define the source directory containing vecmap, vutils, etc.
|
||||
SRC_DIR = APP_DIR.parent / "src"
|
||||
# Define path to .env file relative to APP_DIR
|
||||
DOTENV_PATH = APP_DIR / ".env"
|
||||
|
||||
# --- Explicitly Manage sys.path ---
|
||||
app_dir_str = str(APP_DIR)
|
||||
src_dir_str = str(SRC_DIR)
|
||||
if app_dir_str in sys.path:
|
||||
try: sys.path.remove(app_dir_str)
|
||||
except ValueError: pass
|
||||
if src_dir_str not in sys.path:
|
||||
sys.path.insert(0, src_dir_str)
|
||||
elif sys.path[0] != src_dir_str:
|
||||
try: sys.path.remove(src_dir_str)
|
||||
except ValueError: pass
|
||||
sys.path.insert(0, src_dir_str)
|
||||
print(f"[DEBUG] sys.path start: {sys.path[:3]}")
|
||||
|
||||
# --- .env Loader ---
|
||||
def _load_env_file(path: pathlib.Path) -> None:
|
||||
print(f"Attempting to load .env file from: {path}")
|
||||
if not path.is_file(): print(f".env file not found at {path}, skipping."); return
|
||||
loaded, skipped = 0, 0
|
||||
try:
|
||||
with open(path, 'r', encoding='utf-8') as f:
|
||||
for line in f:
|
||||
line = line.strip();
|
||||
if not line or line.startswith("#") or "=" not in line: continue
|
||||
key, val = line.split("=", 1); key, val = key.strip(), val.strip()
|
||||
if key not in os.environ: os.environ[key] = val; loaded += 1
|
||||
else: skipped += 1
|
||||
print(f"Loaded {loaded} new vars, skipped {skipped} existing vars from .env")
|
||||
except Exception as e: print(f"Error reading .env file at {path}: {e}")
|
||||
_load_env_file(DOTENV_PATH)
|
||||
|
||||
# --- Logging Setup ---
|
||||
log_level_str = os.getenv("VECMAP_DEBUG", "false").lower()
|
||||
log_level = logging.DEBUG if log_level_str in ("true", "1") else logging.INFO
|
||||
logging.basicConfig(level=log_level, format='[%(asctime)s] [%(levelname)s] [%(name)s:%(lineno)d] %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
|
||||
if log_level > logging.DEBUG:
|
||||
for logger_name in ["urllib3", "httpx", "supabase"]: logging.getLogger(logger_name).setLevel(logging.WARNING)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# --- Imports ---
|
||||
try:
|
||||
from librarian_vspace.vecmap.loader import VectorLoader, VectorLoaderError
|
||||
from librarian_vspace.vecmap.visualizer import VectorVisualizer # Removed DEFAULT_N_CLUSTERS import
|
||||
import librarian_vspace.vutils
|
||||
import librarian_vspace.vecembed
|
||||
logger.debug("Successfully imported components.")
|
||||
except ImportError as e:
|
||||
logger.error(f"Failed to import necessary modules: {e}", exc_info=True)
|
||||
sys.exit(1)
|
||||
|
||||
# --- Main Logic ---
|
||||
def main() -> None:
|
||||
logger.info("--- Starting VecMap Visualizer ---")
|
||||
|
||||
# --- Configuration ---
|
||||
db_schema = os.getenv("VECTOR_SCHEMA", "librarian")
|
||||
db_function = os.getenv("VECTOR_FUNCTION", "pdf_chunking")
|
||||
model_name = os.getenv("EMBED_MODEL", "snowflake-arctic-embed2")
|
||||
interface_name = os.getenv("EMBED_INTERFACE", "ollama")
|
||||
embedding_column = os.getenv("EMBEDDING_COLUMN", "embedding")
|
||||
try: limit_str = os.getenv("VECMAP_LIMIT"); data_limit = int(limit_str) if limit_str else None
|
||||
except ValueError: logger.warning(f"Invalid VECMAP_LIMIT. Ignoring."); data_limit = None
|
||||
try: perplexity_str = os.getenv("VECMAP_PERPLEXITY", "30.0"); tsne_perplexity = float(perplexity_str)
|
||||
except ValueError: logger.warning(f"Invalid VECMAP_PERPLEXITY. Using 30.0."); tsne_perplexity = 30.0
|
||||
|
||||
# n_clusters configuration removed
|
||||
|
||||
dash_host = os.getenv("VECMAP_HOST", "127.0.0.1")
|
||||
try: port_str = os.getenv("VECMAP_PORT", "8050"); dash_port = int(port_str)
|
||||
except ValueError: logger.warning(f"Invalid VECMAP_PORT. Using 8050."); dash_port = 8050
|
||||
dash_debug = log_level == logging.DEBUG
|
||||
|
||||
logger.info("Effective Configuration:")
|
||||
logger.info(f" Database: schema={db_schema}, function={db_function}")
|
||||
logger.info(f" Model/Interface: model={model_name}, interface={interface_name}")
|
||||
logger.info(f" Data Params: column={embedding_column}, limit={data_limit}")
|
||||
logger.info(f" Processing: perplexity={tsne_perplexity} (n_clusters is now dynamic)") # Updated log
|
||||
logger.info(f" Server: host={dash_host}, port={dash_port}, debug={dash_debug}")
|
||||
|
||||
# --- 1. Initial Load and Reduce ---
|
||||
initial_df_reduced = pd.DataFrame()
|
||||
try:
|
||||
logger.info("Performing initial data load and processing...")
|
||||
loader = VectorLoader(schema=db_schema, function=db_function, model=model_name, embedding_column=embedding_column)
|
||||
tsne_params = {"perplexity": tsne_perplexity}
|
||||
initial_df_reduced = loader.load_and_reduce(limit=data_limit, tsne_params=tsne_params)
|
||||
if initial_df_reduced.empty: logger.warning("Initial data load resulted in an empty dataset.")
|
||||
else: logger.info(f"Successfully loaded and reduced {len(initial_df_reduced)} vectors initially.")
|
||||
except VectorLoaderError as e: logger.error(f"Initial data load failed: {e}", exc_info=dash_debug)
|
||||
except Exception as e: logger.error(f"Unexpected error during initial data load: {e}", exc_info=dash_debug)
|
||||
|
||||
# --- 2. Initialize and Start Visualization ---
|
||||
try:
|
||||
logger.info("Initializing VectorVisualizer...")
|
||||
visualizer = VectorVisualizer(
|
||||
initial_data=initial_df_reduced,
|
||||
db_schema=db_schema,
|
||||
db_function=db_function,
|
||||
interface_name=interface_name,
|
||||
model_name=model_name,
|
||||
embedding_column=embedding_column,
|
||||
initial_limit=data_limit,
|
||||
initial_perplexity=tsne_perplexity
|
||||
# n_clusters argument removed
|
||||
)
|
||||
logger.info("Launching visualizer...")
|
||||
visualizer.run(host=dash_host, port=dash_port, debug=dash_debug)
|
||||
except TypeError as te:
|
||||
logger.error(f"TypeError during VectorVisualizer initialization: {te}", exc_info=True)
|
||||
sys.exit(1)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to initialize or run visualizer: {e}", exc_info=dash_debug)
|
||||
sys.exit(1)
|
||||
|
||||
logger.info("--- VecMap Visualizer finished ---")
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user