Initialize Monorepo
This commit is contained in:
commit
f80792d739
106 files changed
+17076
No files matched your search
@@ -0,0 +1,9 @@
|
||||
|
||||
"""Embedding‑related helpers."""
|
||||
from __future__ import annotations
|
||||
|
||||
from .vector_inserter import VectorInserter
|
||||
from .embedding_generator import EmbeddingGenerator
|
||||
from .embedding_workflow import EmbeddingWorkflow
|
||||
|
||||
__all__ = ["VectorInserter", "EmbeddingGenerator", "EmbeddingWorkflow"]
|
||||
@@ -0,0 +1,155 @@
|
||||
|
||||
"""Parallel‑aware embedding helpers.
|
||||
|
||||
* **embed_single_file()** – embed one file (sync).
|
||||
* **run_embedder()** – embed all files in a course (async, kept for back‑compat).
|
||||
* **_create_hnsw_index()** – helper to (re)build PGVector HNSW index.
|
||||
|
||||
This file contains no Prefect code; it’s pure embedding logic.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, List, Union
|
||||
|
||||
from postgrest import APIResponse
|
||||
|
||||
from librarian_core.temp_payloads.chunk_data import ChunkCourse, ChunkFile
|
||||
from librarian_vspace.vecembed.embedding_generator import EmbeddingGenerator
|
||||
from librarian_vspace.vecembed.vector_inserter import VectorInserter
|
||||
from librarian_vspace.vecembed.embedding_workflow import EmbeddingWorkflow
|
||||
from librarian_vspace.vutils.supabase_singleton import MySupabase
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
@lru_cache(maxsize=1)
|
||||
def _autodiscover_pg_conn():
|
||||
supa = MySupabase.get_client() # type: ignore
|
||||
if supa is None:
|
||||
raise RuntimeError("MySupabase.get_client() returned None – no DB connection.")
|
||||
return supa
|
||||
|
||||
|
||||
def _create_hnsw_index(
|
||||
supa,
|
||||
table_fqn: str,
|
||||
*,
|
||||
column_name: str = "embedding",
|
||||
query_operator: str = "<=>",
|
||||
m: int = 16,
|
||||
ef: int = 64,
|
||||
) -> None:
|
||||
if "." not in table_fqn:
|
||||
raise ValueError("table_fqn must be schema.table")
|
||||
schema, table = table_fqn.split(".", 1)
|
||||
try:
|
||||
supa.schema(schema).rpc(
|
||||
"create_or_reindex_hnsw",
|
||||
dict(
|
||||
p_schema=schema,
|
||||
p_table=table,
|
||||
p_column=column_name,
|
||||
p_operator=query_operator,
|
||||
p_m=m,
|
||||
p_ef=ef,
|
||||
),
|
||||
).execute()
|
||||
except Exception:
|
||||
logger.exception("Failed to run create_or_reindex_hnsw")
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# single file #
|
||||
# --------------------------------------------------------------------------- #
|
||||
def embed_single_file(
|
||||
*,
|
||||
course_id: str,
|
||||
file_entry: dict | ChunkFile | SimpleNamespace,
|
||||
concat_path: Union[str, Path],
|
||||
db_schema: str = "librarian",
|
||||
db_function: str = "pdf_chunking",
|
||||
interface_name: str = "ollama",
|
||||
model_name: str = "snowflake-arctic-embed2",
|
||||
file_type: str = "md",
|
||||
) -> Path | None:
|
||||
|
||||
if isinstance(file_entry, (dict, SimpleNamespace)):
|
||||
file_name = file_entry["file_name"] if isinstance(file_entry, dict) else file_entry.file_name
|
||||
file_id = file_entry["file_id"] if isinstance(file_entry, dict) else file_entry.file_id
|
||||
else:
|
||||
file_name, file_id = file_entry.file_name, file_entry.file_id
|
||||
|
||||
chunk_path = Path(concat_path) / file_name
|
||||
if not chunk_path.exists():
|
||||
logger.warning("Missing chunk file %s – skipping", chunk_path)
|
||||
return None
|
||||
|
||||
generator = EmbeddingGenerator()
|
||||
inserter = VectorInserter(schema=db_schema, function=db_function, model=model_name)
|
||||
|
||||
wf = EmbeddingWorkflow(
|
||||
chunk_path=chunk_path,
|
||||
course_id=course_id,
|
||||
file_id=file_id,
|
||||
file_type=file_type,
|
||||
interface_name=interface_name,
|
||||
model_name=model_name,
|
||||
generator=generator,
|
||||
inserter=inserter,
|
||||
)
|
||||
wf.process()
|
||||
return chunk_path
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
async def run_embedder(
|
||||
course: ChunkCourse,
|
||||
concat_path: Union[str, Path],
|
||||
*,
|
||||
db_schema: str = "librarian",
|
||||
db_function: str = "pdf_chunking",
|
||||
interface_name: str = "ollama",
|
||||
model_name: str = "snowflake-arctic-embed2",
|
||||
file_type: str = "md",
|
||||
vector_column: str = "embedding",
|
||||
query_operator: str = "<=>",
|
||||
hnsw_m: int = 16,
|
||||
hnsw_ef: int = 64,
|
||||
max_parallel_files: int | None = None,
|
||||
) -> Path:
|
||||
|
||||
supa_client = _autodiscover_pg_conn()
|
||||
root = Path(concat_path)
|
||||
sem = asyncio.Semaphore(max_parallel_files or len(course.files) or 1)
|
||||
|
||||
async def _wrapper(cf):
|
||||
async with sem:
|
||||
return await asyncio.to_thread(
|
||||
embed_single_file,
|
||||
course_id=course.course_id,
|
||||
file_entry=cf,
|
||||
concat_path=root,
|
||||
db_schema=db_schema,
|
||||
db_function=db_function,
|
||||
interface_name=interface_name,
|
||||
model_name=model_name,
|
||||
file_type=file_type,
|
||||
)
|
||||
|
||||
await asyncio.gather(*[asyncio.create_task(_wrapper(cf)) for cf in course.files])
|
||||
|
||||
inserter = VectorInserter(schema=db_schema, function=db_function, model=model_name)
|
||||
_create_hnsw_index(
|
||||
supa_client,
|
||||
inserter.table_fqn(),
|
||||
column_name=vector_column,
|
||||
query_operator=query_operator,
|
||||
m=hnsw_m,
|
||||
ef=hnsw_ef,
|
||||
)
|
||||
return root
|
||||
|
||||
__all__ = ["embed_single_file", "run_embedder", "_create_hnsw_index", "_autodiscover_pg_conn"]
|
||||
@@ -0,0 +1,67 @@
|
||||
|
||||
"""EmbedderWorker – Prefect‑mapped per file."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Any, List
|
||||
|
||||
from prefect import get_run_logger, task, unmapped
|
||||
from pydantic import BaseModel, Field
|
||||
from librarian_core.workers.base import Worker
|
||||
|
||||
@task(name="embed_file", retries=2, retry_delay_seconds=5, log_prints=True, tags=["embed_file"])
|
||||
def embed_file_task(course_dict: dict | SimpleNamespace, file_entry: dict, concat_path: Path) -> Path | None:
|
||||
from librarian_vspace.vecembed.embedder import embed_single_file
|
||||
cid = course_dict["course_id"] if isinstance(course_dict, dict) else course_dict.course_id
|
||||
return embed_single_file(course_id=cid, file_entry=file_entry, concat_path=concat_path)
|
||||
|
||||
class EmbedderInput(BaseModel):
|
||||
chunk_courses: List[Any] = Field(default_factory=list, alias="chunk_courses")
|
||||
concat_path: Path
|
||||
chunk_course: Any | None = None
|
||||
def model_post_init(self, _):
|
||||
if not self.chunk_courses and self.chunk_course is not None:
|
||||
self.chunk_courses = [self.chunk_course]
|
||||
model_config = dict(populate_by_name=True, extra="allow")
|
||||
|
||||
class EmbedderOutput(BaseModel):
|
||||
result_paths: List[Path]
|
||||
|
||||
class EmbedderWorker(Worker[EmbedderInput, EmbedderOutput]):
|
||||
input_model = EmbedderInput
|
||||
output_model = EmbedderOutput
|
||||
|
||||
async def __run__(self, payload: EmbedderInput) -> EmbedderOutput:
|
||||
log = get_run_logger()
|
||||
total_files = sum(len(c["files"]) if isinstance(c, dict) else len(c.files) for c in payload.chunk_courses)
|
||||
log.info("Embedding %d files", total_files)
|
||||
|
||||
result_paths: List[Path] = []
|
||||
|
||||
# constants – could be parameterised later
|
||||
schema = "librarian"
|
||||
func = "pdf_chunking"
|
||||
model_name = "snowflake-arctic-embed2"
|
||||
|
||||
for course in payload.chunk_courses:
|
||||
files = course["files"] if isinstance(course, dict) else course.files
|
||||
futures = embed_file_task.map(unmapped(course), files, unmapped(payload.concat_path))
|
||||
for fut in futures:
|
||||
path = fut.result()
|
||||
if path:
|
||||
result_paths.append(path)
|
||||
|
||||
# rebuild index once per course
|
||||
from librarian_vspace.vecembed.embedder import _create_hnsw_index, _autodiscover_pg_conn
|
||||
from librarian_vspace.vecembed.vector_inserter import VectorInserter
|
||||
|
||||
supa = _autodiscover_pg_conn()
|
||||
inserter = VectorInserter(schema=schema, function=func, model=model_name)
|
||||
_create_hnsw_index(supa, inserter.table_fqn())
|
||||
|
||||
for p in result_paths:
|
||||
self.stage(p, new_name=p.name)
|
||||
|
||||
return EmbedderOutput(result_paths=result_paths)
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
|
||||
"""Factory for embedding back‑ends."""
|
||||
import logging
|
||||
from typing import Any, List, Optional, Tuple, Dict, Type
|
||||
|
||||
from librarian_vspace.vecembed.embedding_interface import EmbeddingInterface
|
||||
from librarian_vspace.vecembed.ollama_embedder import OllamaEmbedder
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class EmbeddingGenerator:
|
||||
_registry: Dict[str, Type[EmbeddingInterface]] = {
|
||||
"ollama": OllamaEmbedder,
|
||||
}
|
||||
|
||||
def generate_embedding(self, interface_name: str, model_name: str, text_to_embed: str, identifier: Any) -> Tuple[str, Optional[List[float]], Any]:
|
||||
cls = self._registry.get(interface_name.lower())
|
||||
if not cls:
|
||||
raise ValueError(f"Unsupported embedding interface: {interface_name}")
|
||||
embedder = cls(model_name=model_name)
|
||||
return embedder.embed(text_to_embed, identifier)
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
from __future__ import annotations
|
||||
from abc import ABC, abstractmethod
|
||||
from typing import Any, List, Optional, Tuple
|
||||
|
||||
class EmbeddingInterface(ABC):
|
||||
"""Contract for any embedding service implementation."""
|
||||
|
||||
def __init__(self, model_name: str, **kwargs: Any) -> None:
|
||||
self.model_name = model_name
|
||||
|
||||
@abstractmethod
|
||||
def embed(self, text_or_chunk: str, identifier: Any) -> Tuple[str, Optional[List[float]], Any]:
|
||||
"""Return (original_text, embedding, identifier) — embedding may be None on failure."""
|
||||
pass
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
"""Orchestrates loading, embedding, and storing a text chunk."""
|
||||
from __future__ import annotations
|
||||
import logging
|
||||
from pathlib import Path
|
||||
from typing import Any, Optional, Dict, Union
|
||||
|
||||
# Import the worker classes for type hinting
|
||||
from librarian_vspace.vecembed.embedding_generator import EmbeddingGenerator
|
||||
from librarian_vspace.vecembed.vector_inserter import VectorInserter
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class EmbeddingWorkflow:
|
||||
# Accept generator and inserter instances in __init__
|
||||
def __init__(self,
|
||||
chunk_path: Union[str, Path],
|
||||
course_id: Any,
|
||||
file_id: Any,
|
||||
file_type: str,
|
||||
interface_name: str, # Still needed for generate_embedding method
|
||||
model_name: str, # Still needed for generate_embedding method
|
||||
generator: EmbeddingGenerator, # Accept pre-instantiated generator
|
||||
inserter: VectorInserter, # Accept pre-instantiated inserter
|
||||
# db_schema and db_function are now implicit via the inserter
|
||||
# db_schema: str = "librarian",
|
||||
# db_function: str = "pdf_chunking",
|
||||
):
|
||||
self.chunk_path = Path(chunk_path)
|
||||
self.course_id = course_id
|
||||
self.file_id = file_id
|
||||
self.file_type = file_type
|
||||
# Keep interface_name and model_name as they are passed to the generator's method
|
||||
self.interface_name = interface_name
|
||||
self.model_name = model_name
|
||||
|
||||
# Assign the passed instances instead of creating new ones
|
||||
self.generator = generator
|
||||
self.inserter = inserter
|
||||
|
||||
# No need to store db_schema/db_function here if inserter handles it
|
||||
|
||||
# ---------------- helpers ----------------
|
||||
def _load_chunk(self) -> Optional[str]:
|
||||
try:
|
||||
text = self.chunk_path.read_text(encoding="utf-8").strip()
|
||||
if not text:
|
||||
logger.warning("Chunk %s is empty", self.chunk_path)
|
||||
return None
|
||||
return text
|
||||
except Exception as exc:
|
||||
logger.error("Failed to read %s: %s", self.chunk_path, exc)
|
||||
return None
|
||||
|
||||
def process(self) -> bool:
|
||||
chunk_text = self._load_chunk()
|
||||
if chunk_text is None:
|
||||
return False
|
||||
|
||||
# Use the shared generator instance
|
||||
original_text, vector, _ = self.generator.generate_embedding(
|
||||
interface_name=self.interface_name, # Pass parameters to the method
|
||||
model_name=self.model_name, # Pass parameters to the method
|
||||
text_to_embed=chunk_text,
|
||||
identifier=self.file_id,
|
||||
)
|
||||
|
||||
if vector is None:
|
||||
# Log failure within generator if not already done, or here
|
||||
logger.error(f"Failed to generate embedding for {self.chunk_path}")
|
||||
return False
|
||||
|
||||
payload: Dict[str, Any] = {
|
||||
"chunk": original_text,
|
||||
"course_id": self.course_id,
|
||||
"file_id": self.file_id,
|
||||
"file_type": self.file_type,
|
||||
"embedding": vector,
|
||||
}
|
||||
|
||||
# Use the shared inserter instance
|
||||
insert_result = self.inserter.insert_vector(payload)
|
||||
|
||||
if insert_result is None:
|
||||
logger.error(f"Failed to insert vector for {self.chunk_path}")
|
||||
return False
|
||||
|
||||
logger.debug(f"Successfully processed and inserted {self.chunk_path}")
|
||||
return True # Indicate success
|
||||
|
||||
|
||||
# Keep __all__ if needed
|
||||
# __all__ = ["EmbeddingWorkflow"]
|
||||
@@ -0,0 +1,44 @@
|
||||
|
||||
"""Ollama-based embedding implementation (env handled at application layer)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, List, Optional, Tuple
|
||||
|
||||
import requests
|
||||
|
||||
from librarian_vspace.vecembed.embedding_interface import EmbeddingInterface
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class OllamaEmbedder(EmbeddingInterface):
|
||||
def __init__(self, model_name: str, **kwargs: Any) -> None:
|
||||
super().__init__(model_name=model_name)
|
||||
self.base_url = os.getenv("OLLAMA_BASE_URL")
|
||||
if not self.base_url:
|
||||
raise ValueError("OLLAMA_BASE_URL not configured – ensure env is set in the examples layer")
|
||||
self.api_endpoint = f"{self.base_url.rstrip('/')}/api/embeddings"
|
||||
|
||||
|
||||
def embed(self, text_or_chunk: str, identifier: Any) -> Tuple[str, Optional[List[float]], Any]:
|
||||
payload = {"model": self.model_name, "prompt": text_or_chunk}
|
||||
vector: Optional[List[float]] = None
|
||||
try:
|
||||
logger.debug("Requesting embedding for id=%s", identifier)
|
||||
resp = requests.post(self.api_endpoint, json=payload, timeout=3600, headers={"Content-Type": "application/json"})
|
||||
resp.raise_for_status()
|
||||
data = resp.json()
|
||||
if isinstance(data.get("embedding"), list):
|
||||
vector = data["embedding"]
|
||||
logger.debug("Received embedding dim=%d for id=%s", len(vector), identifier)
|
||||
else:
|
||||
logger.error("Invalid response from Ollama: %s", data)
|
||||
except requests.exceptions.Timeout:
|
||||
logger.error("Timeout contacting Ollama at %s", self.api_endpoint)
|
||||
except requests.exceptions.RequestException as exc:
|
||||
logger.error("HTTP error contacting Ollama: %s", exc)
|
||||
except Exception:
|
||||
logger.exception("Unexpected error during embed for id=%s", identifier)
|
||||
return text_or_chunk, vector, identifier
|
||||
Whitespace-only changes.
@@ -0,0 +1,23 @@
|
||||
|
||||
from __future__ import annotations
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
from librarian_vspace.vutils.vector_class import BaseVectorOperator
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
class VectorInserter(BaseVectorOperator):
|
||||
"""High-level write helper for embeddings."""
|
||||
|
||||
def insert_vector(self, data: Dict[str, Any]) -> Optional[List[Dict[str, Any]]]:
|
||||
if not self.table:
|
||||
logger.error("Table resolution failed earlier")
|
||||
return None
|
||||
preview = {k: (f"<vector,len={len(v)}>" if k == "embedding" else v) for k, v in data.items()}
|
||||
logger.debug("Insert → %s.%s :: %s", self.schema, self.table, preview)
|
||||
try:
|
||||
resp = self.spc.schema(self.schema).table(self.table).insert(data).execute()
|
||||
return resp.data if isinstance(resp.data, list) else []
|
||||
except Exception:
|
||||
logger.exception("Insert failed for %s", self.table_fqn())
|
||||
return None
|
||||
Reference in new issue
Block a user