Initialize Monorepo

This commit is contained in:
DotNaos committed 2025-05-24 12:15:48 +02:00
commit f80792d739
106 files changed
+17076

No files matched your search

@@ -0,0 +1,9 @@
"""Embedding‑related helpers."""
from __future__ import annotations
from .vector_inserter import VectorInserter
from .embedding_generator import EmbeddingGenerator
from .embedding_workflow import EmbeddingWorkflow
__all__ = ["VectorInserter", "EmbeddingGenerator", "EmbeddingWorkflow"]
@@ -0,0 +1,155 @@
"""Parallel‑aware embedding helpers.
* **embed_single_file()** – embed one file (sync).
* **run_embedder()** – embed all files in a course (async, kept for back‑compat).
* **_create_hnsw_index()** – helper to (re)build PGVector HNSW index.
This file contains no Prefect code; it’s pure embedding logic.
"""
from __future__ import annotations
import asyncio
import logging
from functools import lru_cache
from pathlib import Path
from types import SimpleNamespace
from typing import Any, List, Union
from postgrest import APIResponse
from librarian_core.temp_payloads.chunk_data import ChunkCourse, ChunkFile
from librarian_vspace.vecembed.embedding_generator import EmbeddingGenerator
from librarian_vspace.vecembed.vector_inserter import VectorInserter
from librarian_vspace.vecembed.embedding_workflow import EmbeddingWorkflow
from librarian_vspace.vutils.supabase_singleton import MySupabase
logger = logging.getLogger(__name__)
@lru_cache(maxsize=1)
def _autodiscover_pg_conn():
supa = MySupabase.get_client() # type: ignore
if supa is None:
raise RuntimeError("MySupabase.get_client() returned None – no DB connection.")
return supa
def _create_hnsw_index(
supa,
table_fqn: str,
*,
column_name: str = "embedding",
query_operator: str = "<=>",
m: int = 16,
ef: int = 64,
) -> None:
if "." not in table_fqn:
raise ValueError("table_fqn must be schema.table")
schema, table = table_fqn.split(".", 1)
try:
supa.schema(schema).rpc(
"create_or_reindex_hnsw",
dict(
p_schema=schema,
p_table=table,
p_column=column_name,
p_operator=query_operator,
p_m=m,
p_ef=ef,
),
).execute()
except Exception:
logger.exception("Failed to run create_or_reindex_hnsw")
# --------------------------------------------------------------------------- #
# single file #
# --------------------------------------------------------------------------- #
def embed_single_file(
*,
course_id: str,
file_entry: dict | ChunkFile | SimpleNamespace,
concat_path: Union[str, Path],
db_schema: str = "librarian",
db_function: str = "pdf_chunking",
interface_name: str = "ollama",
model_name: str = "snowflake-arctic-embed2",
file_type: str = "md",
) -> Path | None:
if isinstance(file_entry, (dict, SimpleNamespace)):
file_name = file_entry["file_name"] if isinstance(file_entry, dict) else file_entry.file_name
file_id = file_entry["file_id"] if isinstance(file_entry, dict) else file_entry.file_id
else:
file_name, file_id = file_entry.file_name, file_entry.file_id
chunk_path = Path(concat_path) / file_name
if not chunk_path.exists():
logger.warning("Missing chunk file %s – skipping", chunk_path)
return None
generator = EmbeddingGenerator()
inserter = VectorInserter(schema=db_schema, function=db_function, model=model_name)
wf = EmbeddingWorkflow(
chunk_path=chunk_path,
course_id=course_id,
file_id=file_id,
file_type=file_type,
interface_name=interface_name,
model_name=model_name,
generator=generator,
inserter=inserter,
)
wf.process()
return chunk_path
# --------------------------------------------------------------------------- #
async def run_embedder(
course: ChunkCourse,
concat_path: Union[str, Path],
*,
db_schema: str = "librarian",
db_function: str = "pdf_chunking",
interface_name: str = "ollama",
model_name: str = "snowflake-arctic-embed2",
file_type: str = "md",
vector_column: str = "embedding",
query_operator: str = "<=>",
hnsw_m: int = 16,
hnsw_ef: int = 64,
max_parallel_files: int | None = None,
) -> Path:
supa_client = _autodiscover_pg_conn()
root = Path(concat_path)
sem = asyncio.Semaphore(max_parallel_files or len(course.files) or 1)
async def _wrapper(cf):
async with sem:
return await asyncio.to_thread(
embed_single_file,
course_id=course.course_id,
file_entry=cf,
concat_path=root,
db_schema=db_schema,
db_function=db_function,
interface_name=interface_name,
model_name=model_name,
file_type=file_type,
)
await asyncio.gather(*[asyncio.create_task(_wrapper(cf)) for cf in course.files])
inserter = VectorInserter(schema=db_schema, function=db_function, model=model_name)
_create_hnsw_index(
supa_client,
inserter.table_fqn(),
column_name=vector_column,
query_operator=query_operator,
m=hnsw_m,
ef=hnsw_ef,
)
return root
__all__ = ["embed_single_file", "run_embedder", "_create_hnsw_index", "_autodiscover_pg_conn"]
@@ -0,0 +1,67 @@
"""EmbedderWorker – Prefect‑mapped per file."""
from __future__ import annotations
from pathlib import Path
from types import SimpleNamespace
from typing import Any, List
from prefect import get_run_logger, task, unmapped
from pydantic import BaseModel, Field
from librarian_core.workers.base import Worker
@task(name="embed_file", retries=2, retry_delay_seconds=5, log_prints=True, tags=["embed_file"])
def embed_file_task(course_dict: dict | SimpleNamespace, file_entry: dict, concat_path: Path) -> Path | None:
from librarian_vspace.vecembed.embedder import embed_single_file
cid = course_dict["course_id"] if isinstance(course_dict, dict) else course_dict.course_id
return embed_single_file(course_id=cid, file_entry=file_entry, concat_path=concat_path)
class EmbedderInput(BaseModel):
chunk_courses: List[Any] = Field(default_factory=list, alias="chunk_courses")
concat_path: Path
chunk_course: Any | None = None
def model_post_init(self, _):
if not self.chunk_courses and self.chunk_course is not None:
self.chunk_courses = [self.chunk_course]
model_config = dict(populate_by_name=True, extra="allow")
class EmbedderOutput(BaseModel):
result_paths: List[Path]
class EmbedderWorker(Worker[EmbedderInput, EmbedderOutput]):
input_model = EmbedderInput
output_model = EmbedderOutput
async def __run__(self, payload: EmbedderInput) -> EmbedderOutput:
log = get_run_logger()
total_files = sum(len(c["files"]) if isinstance(c, dict) else len(c.files) for c in payload.chunk_courses)
log.info("Embedding %d files", total_files)
result_paths: List[Path] = []
# constants – could be parameterised later
schema = "librarian"
func = "pdf_chunking"
model_name = "snowflake-arctic-embed2"
for course in payload.chunk_courses:
files = course["files"] if isinstance(course, dict) else course.files
futures = embed_file_task.map(unmapped(course), files, unmapped(payload.concat_path))
for fut in futures:
path = fut.result()
if path:
result_paths.append(path)
# rebuild index once per course
from librarian_vspace.vecembed.embedder import _create_hnsw_index, _autodiscover_pg_conn
from librarian_vspace.vecembed.vector_inserter import VectorInserter
supa = _autodiscover_pg_conn()
inserter = VectorInserter(schema=schema, function=func, model=model_name)
_create_hnsw_index(supa, inserter.table_fqn())
for p in result_paths:
self.stage(p, new_name=p.name)
return EmbedderOutput(result_paths=result_paths)
@@ -0,0 +1,21 @@
"""Factory for embedding back‑ends."""
import logging
from typing import Any, List, Optional, Tuple, Dict, Type
from librarian_vspace.vecembed.embedding_interface import EmbeddingInterface
from librarian_vspace.vecembed.ollama_embedder import OllamaEmbedder
logger = logging.getLogger(__name__)
class EmbeddingGenerator:
_registry: Dict[str, Type[EmbeddingInterface]] = {
"ollama": OllamaEmbedder,
}
def generate_embedding(self, interface_name: str, model_name: str, text_to_embed: str, identifier: Any) -> Tuple[str, Optional[List[float]], Any]:
cls = self._registry.get(interface_name.lower())
if not cls:
raise ValueError(f"Unsupported embedding interface: {interface_name}")
embedder = cls(model_name=model_name)
return embedder.embed(text_to_embed, identifier)
@@ -0,0 +1,14 @@
from __future__ import annotations
from abc import ABC, abstractmethod
from typing import Any, List, Optional, Tuple
class EmbeddingInterface(ABC):
"""Contract for any embedding service implementation."""
def __init__(self, model_name: str, **kwargs: Any) -> None:
self.model_name = model_name
@abstractmethod
def embed(self, text_or_chunk: str, identifier: Any) -> Tuple[str, Optional[List[float]], Any]:
"""Return (original_text, embedding, identifier) — embedding may be None on failure."""
pass
@@ -0,0 +1,92 @@
"""Orchestrates loading, embedding, and storing a text chunk."""
from __future__ import annotations
import logging
from pathlib import Path
from typing import Any, Optional, Dict, Union
# Import the worker classes for type hinting
from librarian_vspace.vecembed.embedding_generator import EmbeddingGenerator
from librarian_vspace.vecembed.vector_inserter import VectorInserter
logger = logging.getLogger(__name__)
class EmbeddingWorkflow:
# Accept generator and inserter instances in __init__
def __init__(self,
chunk_path: Union[str, Path],
course_id: Any,
file_id: Any,
file_type: str,
interface_name: str, # Still needed for generate_embedding method
model_name: str, # Still needed for generate_embedding method
generator: EmbeddingGenerator, # Accept pre-instantiated generator
inserter: VectorInserter, # Accept pre-instantiated inserter
# db_schema and db_function are now implicit via the inserter
# db_schema: str = "librarian",
# db_function: str = "pdf_chunking",
):
self.chunk_path = Path(chunk_path)
self.course_id = course_id
self.file_id = file_id
self.file_type = file_type
# Keep interface_name and model_name as they are passed to the generator's method
self.interface_name = interface_name
self.model_name = model_name
# Assign the passed instances instead of creating new ones
self.generator = generator
self.inserter = inserter
# No need to store db_schema/db_function here if inserter handles it
# ---------------- helpers ----------------
def _load_chunk(self) -> Optional[str]:
try:
text = self.chunk_path.read_text(encoding="utf-8").strip()
if not text:
logger.warning("Chunk %s is empty", self.chunk_path)
return None
return text
except Exception as exc:
logger.error("Failed to read %s: %s", self.chunk_path, exc)
return None
def process(self) -> bool:
chunk_text = self._load_chunk()
if chunk_text is None:
return False
# Use the shared generator instance
original_text, vector, _ = self.generator.generate_embedding(
interface_name=self.interface_name, # Pass parameters to the method
model_name=self.model_name, # Pass parameters to the method
text_to_embed=chunk_text,
identifier=self.file_id,
)
if vector is None:
# Log failure within generator if not already done, or here
logger.error(f"Failed to generate embedding for {self.chunk_path}")
return False
payload: Dict[str, Any] = {
"chunk": original_text,
"course_id": self.course_id,
"file_id": self.file_id,
"file_type": self.file_type,
"embedding": vector,
}
# Use the shared inserter instance
insert_result = self.inserter.insert_vector(payload)
if insert_result is None:
logger.error(f"Failed to insert vector for {self.chunk_path}")
return False
logger.debug(f"Successfully processed and inserted {self.chunk_path}")
return True # Indicate success
# Keep __all__ if needed
# __all__ = ["EmbeddingWorkflow"]
@@ -0,0 +1,44 @@
"""Ollama-based embedding implementation (env handled at application layer)."""
from __future__ import annotations
import logging
import os
from typing import Any, List, Optional, Tuple
import requests
from librarian_vspace.vecembed.embedding_interface import EmbeddingInterface
logger = logging.getLogger(__name__)
class OllamaEmbedder(EmbeddingInterface):
def __init__(self, model_name: str, **kwargs: Any) -> None:
super().__init__(model_name=model_name)
self.base_url = os.getenv("OLLAMA_BASE_URL")
if not self.base_url:
raise ValueError("OLLAMA_BASE_URL not configured – ensure env is set in the examples layer")
self.api_endpoint = f"{self.base_url.rstrip('/')}/api/embeddings"
def embed(self, text_or_chunk: str, identifier: Any) -> Tuple[str, Optional[List[float]], Any]:
payload = {"model": self.model_name, "prompt": text_or_chunk}
vector: Optional[List[float]] = None
try:
logger.debug("Requesting embedding for id=%s", identifier)
resp = requests.post(self.api_endpoint, json=payload, timeout=3600, headers={"Content-Type": "application/json"})
resp.raise_for_status()
data = resp.json()
if isinstance(data.get("embedding"), list):
vector = data["embedding"]
logger.debug("Received embedding dim=%d for id=%s", len(vector), identifier)
else:
logger.error("Invalid response from Ollama: %s", data)
except requests.exceptions.Timeout:
logger.error("Timeout contacting Ollama at %s", self.api_endpoint)
except requests.exceptions.RequestException as exc:
logger.error("HTTP error contacting Ollama: %s", exc)
except Exception:
logger.exception("Unexpected error during embed for id=%s", identifier)
return text_or_chunk, vector, identifier
Whitespace-only changes.
@@ -0,0 +1,23 @@
from __future__ import annotations
import logging
from typing import Any, Dict, List, Optional
from librarian_vspace.vutils.vector_class import BaseVectorOperator
logger = logging.getLogger(__name__)
class VectorInserter(BaseVectorOperator):
"""High-level write helper for embeddings."""
def insert_vector(self, data: Dict[str, Any]) -> Optional[List[Dict[str, Any]]]:
if not self.table:
logger.error("Table resolution failed earlier")
return None
preview = {k: (f"<vector,len={len(v)}>" if k == "embedding" else v) for k, v in data.items()}
logger.debug("Insert → %s.%s :: %s", self.schema, self.table, preview)
try:
resp = self.spc.schema(self.schema).table(self.table).insert(data).execute()
return resp.data if isinstance(resp.data, list) else []
except Exception:
logger.exception("Insert failed for %s", self.table_fqn())
return None