Add LLM role-check grounding + labelled training-set pipeline
- llm_extract.py: match mode now window-parallel with retrieval pre-filter,
claim dedup, retry, and enable_thinking=false (vLLM) -> ~36x faster per call;
n_failed_windows/ok flags so an interrupted run never records bogus 0s.
- build_rdf_dataset.py:
- gold now includes the share-class level (hasShareClass/ticker/className)
- grounding modes alias|llm|name|context|none (--grounding); llm reads the
role-check verdicts from match_all.jsonl
- label stage: per-triple extractable + per-sample FULL/PARTIAL/NONE
- trainset stage: combines GROUNDED triples with focused TEXT EXCERPTS cut
around the actual provider statement (evidence), not the multi-MB book
- split --src to split trainset.jsonl (trust-level, no leakage)
- helper scripts: watch_match.sh, resume_match.sh (crash/sleep-safe resume),
finalize_dataset.sh
- final dataset: 335/335 trusts, 85% text<->gold agreement, 334 samples,
10,689 grounded triples, train/val/test 264/35/35
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Florian Herzog
parent
09798eb27a
commit
991715ab76
12
.gitignore
vendored
12
.gitignore
vendored
@ -44,3 +44,15 @@ __pycache__/
|
||||
# OS / editor
|
||||
.DS_Store
|
||||
.claude/
|
||||
data/rdf_poc/samples_labelled.jsonl
|
||||
data/rdf_poc/samples_full.jsonl
|
||||
data/rdf_poc/labelled_partial.jsonl
|
||||
data/rdf_poc/labelled_none.jsonl
|
||||
data/rdf_poc/labelled_full.jsonl
|
||||
data/rdf_poc/trainset.jsonl
|
||||
data/rdf_poc/match_input_remaining.jsonl
|
||||
data/rdf_poc/match_remaining*.jsonl
|
||||
data/rdf_poc/match_all_clean*.jsonl
|
||||
data/rdf_poc/match_input_sc.jsonl
|
||||
data/rdf_poc/match_remaining.bak
|
||||
data/rdf_poc/match_all.jsonl
|
||||
|
||||
@ -146,6 +146,18 @@ def build_gold(ncen_dir: Path, custodian_scope: str = "primary"):
|
||||
custodians = by_fund("CUSTODIAN.tsv", "CUSTODIAN_NAME", "CUSTODIAN_LEI")
|
||||
admins = by_fund("ADMIN.tsv", "ADMIN_NAME", "ADMIN_LEI")
|
||||
|
||||
# Share classes per FUND_ID (the 3rd hierarchy level: Trust->Fund->ShareClass).
|
||||
# Each class has a full name and a ticker, both typically in the prospectus.
|
||||
share_classes = defaultdict(list)
|
||||
for r in _read_tsv(d / "SHARES_OUTSTANDING.tsv"):
|
||||
fid = r.get("FUND_ID", "")
|
||||
cid = (r.get("CLASS_ID", "") or "").strip()
|
||||
nm = (r.get("CLASS_NAME", "") or "").strip()
|
||||
if fid and cid:
|
||||
share_classes[fid].append(
|
||||
{"class_id": cid, "name": nm,
|
||||
"ticker": (r.get("TICKER", "") or "").strip()})
|
||||
|
||||
# Custodian scoping. Foreign sub-custodians (IS_SUB_CUSTODIAN=Y, ~88% of rows)
|
||||
# appear ONLY in N-CEN, never in the prospectus prose, and dominate the edge
|
||||
# count (66% of all edges). They are therefore unextractable noise for a
|
||||
@ -225,6 +237,24 @@ def build_gold(ncen_dir: Path, custodian_scope: str = "primary"):
|
||||
entities[o] = {"type": "Administrator", "label": a["name"], "lei": a["lei"]}
|
||||
triples.append((f_iri, "administrator", o))
|
||||
|
||||
# share classes (3rd level): Fund -hasShareClass-> Class,
|
||||
# Class -ticker-> "TICKER", Class -className-> "Full Name".
|
||||
# ticker/className objects are literals (stored as label).
|
||||
for sc in share_classes.get(fid, []):
|
||||
c_iri = "class:" + sc["class_id"]
|
||||
entities[c_iri] = {"type": "ShareClass",
|
||||
"label": sc["name"] or sc["class_id"],
|
||||
"class_id": sc["class_id"]}
|
||||
triples.append((f_iri, "hasShareClass", c_iri))
|
||||
if sc["name"]:
|
||||
lit = "lit:" + _slug(sc["name"])
|
||||
entities[lit] = {"type": "Literal", "label": sc["name"]}
|
||||
triples.append((c_iri, "className", lit))
|
||||
if sc["ticker"]:
|
||||
lit = "lit:" + _slug(sc["ticker"])
|
||||
entities[lit] = {"type": "Literal", "label": sc["ticker"]}
|
||||
triples.append((c_iri, "ticker", lit))
|
||||
|
||||
# dedupe triples
|
||||
triples = sorted(set(triples))
|
||||
if not triples:
|
||||
@ -1015,18 +1045,20 @@ def _bucket(cik: str) -> float:
|
||||
return int(h[:8], 16) / 0xFFFFFFFF
|
||||
|
||||
|
||||
def build_split(val_frac: float = 0.10, test_frac: float = 0.10):
|
||||
"""Split samples.jsonl into train/val/test JSONL, partitioned by TRUST (CIK).
|
||||
def build_split(val_frac: float = 0.10, test_frac: float = 0.10, src=None):
|
||||
"""Split a sample file into train/val/test JSONL, partitioned by TRUST (CIK).
|
||||
|
||||
Splitting by trust (not by fund) prevents leakage: two funds of the same
|
||||
trust share advisers, distributors and custodians, so allowing them into
|
||||
different splits would let the model memorise trust-specific entities. The
|
||||
assignment is a deterministic hash of the CIK, so the split is reproducible
|
||||
and stable as new samples are added.
|
||||
and stable as new samples are added. `src` defaults to samples.jsonl; pass
|
||||
trainset.jsonl to split the training-ready set.
|
||||
"""
|
||||
if not SAMPLES_PATH.exists():
|
||||
log.error("run `samples` first"); return
|
||||
rows = [json.loads(l) for l in open(SAMPLES_PATH, encoding="utf-8")]
|
||||
src_path = Path(src) if src else SAMPLES_PATH
|
||||
if not src_path.exists():
|
||||
log.error("source not found: %s", src_path); return
|
||||
rows = [json.loads(l) for l in open(src_path, encoding="utf-8")]
|
||||
out = {"train": [], "val": [], "test": []}
|
||||
for r in rows:
|
||||
b = _bucket(r["cik"])
|
||||
@ -1052,6 +1084,196 @@ def build_split(val_frac: float = 0.10, test_frac: float = 0.10):
|
||||
print(f" trust overlap across splits: {len(overlap)} (should be 0)")
|
||||
|
||||
|
||||
SAMPLES_LABELLED_PATH = OUT_DIR / "samples_labelled.jsonl"
|
||||
|
||||
|
||||
def build_label():
|
||||
"""Label extractability per triple and per sample, using the LLM match verdict.
|
||||
|
||||
Reads the full annotated samples (SAMPLES_FULL_PATH, all gold triples with the
|
||||
cheap alias/grounded/context flags) and the LLM role-check verdicts
|
||||
(LLM_MATCH_PATH). For each triple it sets:
|
||||
- extractable : the authoritative label = LLM role-check verdict
|
||||
(the entity plays that role in the text);
|
||||
- alias_grounded/... : the cheap lexical flags (already present) for analysis.
|
||||
For each sample it sets:
|
||||
- n_extractable / n_triples
|
||||
- extractability = FULL (all triples extractable)
|
||||
| PARTIAL (some)
|
||||
| NONE (none)
|
||||
Writes:
|
||||
- SAMPLES_LABELLED_PATH (every sample, every triple labelled)
|
||||
- samples_full/partial/none.jsonl (samples split by label)
|
||||
- a distribution report (per label and per relation).
|
||||
"""
|
||||
if not SAMPLES_FULL_PATH.exists():
|
||||
log.error("run `samples` first (need %s)", SAMPLES_FULL_PATH); return
|
||||
verdicts = _load_llm_verdicts()
|
||||
if not verdicts:
|
||||
log.error("no LLM match verdicts at %s — run `llm_extract.py --mode match` "
|
||||
"first", LLM_MATCH_PATH); return
|
||||
|
||||
by_label = {"FULL": [], "PARTIAL": [], "NONE": []}
|
||||
rel_tot = defaultdict(int); rel_ext = defaultdict(int)
|
||||
n = 0
|
||||
with open(SAMPLES_LABELLED_PATH, "w", encoding="utf-8") as out:
|
||||
for line in open(SAMPLES_FULL_PATH, encoding="utf-8"):
|
||||
r = json.loads(line)
|
||||
cik = r["cik"]
|
||||
triples = []
|
||||
n_ext = 0
|
||||
for t in r.get("target_triples", []):
|
||||
ext = verdicts.get((cik, t["p"], t["o"]), False)
|
||||
rel_tot[t["p"]] += 1
|
||||
rel_ext[t["p"]] += int(ext)
|
||||
n_ext += int(ext)
|
||||
triples.append({**t, "extractable": ext})
|
||||
ntot = len(triples)
|
||||
label = ("FULL" if ntot and n_ext == ntot else
|
||||
"NONE" if n_ext == 0 else "PARTIAL")
|
||||
r["target_triples"] = triples
|
||||
r["extractability"] = label
|
||||
r.setdefault("stats", {})
|
||||
r["stats"]["n_extractable"] = n_ext
|
||||
r["stats"]["n_triples"] = ntot
|
||||
out.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||||
by_label[label].append(r)
|
||||
n += 1
|
||||
|
||||
for lab, recs in by_label.items():
|
||||
with open(OUT_DIR / f"labelled_{lab.lower()}.jsonl", "w", encoding="utf-8") as f:
|
||||
for r in recs:
|
||||
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||||
|
||||
tot_tr = sum(rel_tot.values()); ext_tr = sum(rel_ext.values())
|
||||
print(f"\nLabelled {n} samples -> {SAMPLES_LABELLED_PATH}")
|
||||
print(f" sample extractability:")
|
||||
for lab in ("FULL", "PARTIAL", "NONE"):
|
||||
c = len(by_label[lab])
|
||||
print(f" {lab:8s} {c:>5} ({100*c/max(1,n):>3.0f}%) -> labelled_{lab.lower()}.jsonl")
|
||||
print(f" triple extractability: {ext_tr}/{tot_tr} = {100*ext_tr/max(1,tot_tr):.0f}%")
|
||||
print(f"\n {'relation':16s}{'extractable':>12}{'%':>6}")
|
||||
for p in sorted(rel_tot, key=lambda x: -rel_tot[x]):
|
||||
print(f" {p:16s}{str(rel_ext[p])+'/'+str(rel_tot[p]):>12}"
|
||||
f"{100*rel_ext[p]/rel_tot[p]:>5.0f}%")
|
||||
|
||||
|
||||
TRAINSET_PATH = OUT_DIR / "trainset.jsonl"
|
||||
EXCERPT_RADIUS = 600 # chars around each grounded name
|
||||
|
||||
|
||||
_ROLE_HINT = ("custodian", "custody", "advis", "transfer agent", "administrat",
|
||||
"distributor", "underwriter", "serves as", "acts as", "series of")
|
||||
|
||||
|
||||
def _evidence_excerpt(text_norm_full, raw_text, label, predicate=None):
|
||||
"""Cut a coherent excerpt of raw_text around the BEST mention of the entity.
|
||||
|
||||
A common firm token (e.g. "alliancebernstein") appears hundreds of times, so
|
||||
the first occurrence is usually boilerplate. We therefore score every
|
||||
occurrence of the name's most distinctive token by whether a role keyword
|
||||
(custodian/adviser/transfer agent/...) appears within +/- a window, and pick
|
||||
the occurrence with the nearest role hint -- that is the actual service-provider
|
||||
statement, the evidence we want. Falls back to the consecutive-tokens match,
|
||||
then the first token occurrence. Returns (excerpt, start) or (None, -1).
|
||||
"""
|
||||
tk = _alias_tokens(label)
|
||||
if not tk:
|
||||
return None, -1
|
||||
low = raw_text.lower()
|
||||
distinctive = max(tk, key=len)
|
||||
occurrences = [m.start() for m in re.finditer(re.escape(distinctive), low)]
|
||||
pos = -1
|
||||
if occurrences:
|
||||
# prefer an occurrence with a role keyword nearby
|
||||
best = None
|
||||
for p in occurrences:
|
||||
win = low[max(0, p - 250): p + 250]
|
||||
if any(k in win for k in _ROLE_HINT):
|
||||
best = p
|
||||
break
|
||||
pos = best if best is not None else occurrences[0]
|
||||
if pos == -1:
|
||||
ac = _acronym(label)
|
||||
if len(ac) >= 3:
|
||||
m = re.search(r"\b" + re.escape(ac) + r"\b", low)
|
||||
pos = m.start() if m else -1
|
||||
if pos == -1:
|
||||
return None, -1
|
||||
a = max(0, pos - EXCERPT_RADIUS)
|
||||
b = min(len(raw_text), pos + EXCERPT_RADIUS)
|
||||
return raw_text[a:b].strip(), a
|
||||
|
||||
|
||||
def build_trainset():
|
||||
"""Assemble the training-ready dataset: per sample, the relevant TEXT EXCERPTS
|
||||
combined with ONLY the grounded (extractable) triples as the target.
|
||||
|
||||
For each labelled sample (samples_labelled.jsonl):
|
||||
- keep only triples with extractable=True (every target is in the text);
|
||||
- for each, cut a coherent excerpt of the prospectus around the entity name
|
||||
and merge overlapping excerpts into a compact `input_text` (fits a model
|
||||
context window, unlike the multi-MB full book);
|
||||
- emit both serializations of the kept triples.
|
||||
Samples with no grounded triple (NONE) are skipped. Output: trainset.jsonl,
|
||||
then split into train/val/test by trust.
|
||||
"""
|
||||
if not SAMPLES_LABELLED_PATH.exists():
|
||||
log.error("run `label` first (need %s)", SAMPLES_LABELLED_PATH); return
|
||||
gold = {json.loads(l)["cik"]: json.loads(l)
|
||||
for l in open(GOLD_PATH, encoding="utf-8")}
|
||||
n = n_skip = 0
|
||||
tot_in = tot_tr = 0
|
||||
with open(TRAINSET_PATH, "w", encoding="utf-8") as out:
|
||||
for line in open(SAMPLES_LABELLED_PATH, encoding="utf-8"):
|
||||
r = json.loads(line)
|
||||
ents = gold.get(r["cik"], {}).get("entities", {})
|
||||
full = r["input_text"]
|
||||
kept = [t for t in r["target_triples"] if t.get("extractable")]
|
||||
if not kept:
|
||||
n_skip += 1
|
||||
continue
|
||||
# collect excerpts around each grounded object name; merge overlaps
|
||||
spans = []
|
||||
for t in kept:
|
||||
lbl = ents.get(t["o"], {}).get("label", t["o"])
|
||||
exc, a = _evidence_excerpt(None, full, lbl)
|
||||
if exc:
|
||||
spans.append((a, a + len(exc)))
|
||||
spans.sort()
|
||||
merged = []
|
||||
for a, b in spans:
|
||||
if merged and a <= merged[-1][1] + 200:
|
||||
merged[-1] = (merged[-1][0], max(merged[-1][1], b))
|
||||
else:
|
||||
merged.append((a, b))
|
||||
excerpt = "\n...\n".join(full[a:b].strip() for a, b in merged) or full[:2000]
|
||||
# restrict entities to kept triples
|
||||
ref = set()
|
||||
for t in kept:
|
||||
ref.add(t["s"]); ref.add(t["o"])
|
||||
se = {k: ents[k] for k in ref if k in ents}
|
||||
rec = {
|
||||
"sample_id": r["sample_id"], "cik": r["cik"],
|
||||
"trust_name": r.get("trust_name"),
|
||||
"input_text": excerpt,
|
||||
"ontology": ontology_schema(kept, se),
|
||||
"target_triples": kept,
|
||||
"target_serialized": serialize_triples(kept, se),
|
||||
"target_serialized_plain": serialize_triples_plain(kept, se),
|
||||
"stats": {"input_chars": len(excerpt), "n_triples": len(kept),
|
||||
"text_to_json_ratio": round(
|
||||
len(excerpt) / max(1, len(serialize_triples(kept, se))), 1)},
|
||||
}
|
||||
out.write(json.dumps(rec, ensure_ascii=False) + "\n")
|
||||
n += 1; tot_in += len(excerpt); tot_tr += len(kept)
|
||||
print(f"\nTrainset: {n} samples -> {TRAINSET_PATH} ({n_skip} NONE-samples skipped)")
|
||||
if n:
|
||||
print(f" grounded triples (targets): {tot_tr:,}")
|
||||
print(f" mean input excerpt: {tot_in//n:,} chars (was multi-MB full book)")
|
||||
print(f" -> split with: python build_rdf_dataset.py split --src trainset.jsonl")
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
def main():
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
|
||||
@ -1073,9 +1295,12 @@ def main():
|
||||
help="context=keep triples whose name+relation-keyword appear "
|
||||
"(default, enforces text-extractability); name=name anywhere; "
|
||||
"none=keep all, only annotate")
|
||||
sub.add_parser("label") # label extractability (needs prior match run)
|
||||
sub.add_parser("trainset") # assemble training set: excerpts + grounded triples
|
||||
sp = sub.add_parser("split")
|
||||
sp.add_argument("--val-frac", type=float, default=0.10)
|
||||
sp.add_argument("--test-frac", type=float, default=0.10)
|
||||
sp.add_argument("--src", default=None, help="source jsonl (default samples.jsonl; use trainset.jsonl)")
|
||||
a = sub.add_parser("all"); a.add_argument("--ncen", default="data/ncen/2025q3")
|
||||
a.add_argument("--limit", type=int, default=25)
|
||||
a.add_argument("--max-filings", type=int, default=8)
|
||||
@ -1095,8 +1320,12 @@ def main():
|
||||
elif args.cmd == "samples":
|
||||
GROUNDING_MODE = args.grounding
|
||||
build_samples(per_fund=not args.whole_trust)
|
||||
elif args.cmd == "label":
|
||||
build_label()
|
||||
elif args.cmd == "trainset":
|
||||
build_trainset()
|
||||
elif args.cmd == "split":
|
||||
build_split(val_frac=args.val_frac, test_frac=args.test_frac)
|
||||
build_split(val_frac=args.val_frac, test_frac=args.test_frac, src=args.src)
|
||||
elif args.cmd == "all":
|
||||
GROUNDING_MODE = args.grounding
|
||||
build_gold(Path(args.ncen), custodian_scope=args.custodian_scope)
|
||||
|
||||
File diff suppressed because one or more lines are too long
48
finalize_dataset.sh
Normal file
48
finalize_dataset.sh
Normal file
@ -0,0 +1,48 @@
|
||||
#!/usr/bin/env bash
|
||||
# ------------------------------------------------------------------
|
||||
# finalize_dataset.sh — konsolidiert alle Match-Teile und baut den
|
||||
# trainingsfertigen Datensatz: label -> trainset -> split.
|
||||
#
|
||||
# bash finalize_dataset.sh
|
||||
#
|
||||
# Idempotent: konsolidiert nur SAUBERE Trusts (n_failed_windows==0) aus
|
||||
# allen match_*.jsonl Teil-Dateien in match_all.jsonl, dann Pipeline.
|
||||
# ------------------------------------------------------------------
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
python3 - <<'PY'
|
||||
import json, glob, os
|
||||
parts = (["data/rdf_poc/match_all_clean79.jsonl",
|
||||
"data/rdf_poc/match_remaining.jsonl"]
|
||||
+ glob.glob("data/rdf_poc/match_remaining_*.jsonl") + glob.glob("data/rdf_poc/match_remaining_final.jsonl"))
|
||||
good = {}
|
||||
for p in parts:
|
||||
if not os.path.exists(p): continue
|
||||
for l in open(p):
|
||||
try: r = json.loads(l)
|
||||
except: continue
|
||||
if r.get("n_failed_windows", 0) == 0 and r.get("triples") is not None:
|
||||
good[r["cik"]] = r
|
||||
with open("data/rdf_poc/match_all.jsonl", "w") as f:
|
||||
for r in good.values():
|
||||
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||||
allin = [json.loads(l) for l in open("data/rdf_poc/match_input.jsonl")]
|
||||
remaining = [r for r in allin if r["cik"] not in good]
|
||||
tg = tt = 0
|
||||
for r in good.values():
|
||||
for t in r["triples"]:
|
||||
tt += 1; tg += 1 if t.get("llm_grounded") else 0
|
||||
print(f"konsolidiert: {len(good)}/335 Trusts, {100*tg/max(1,tt):.0f}% Uebereinstimmung, "
|
||||
f"{len(remaining)} noch offen")
|
||||
PY
|
||||
|
||||
echo ""
|
||||
echo "=== label (FULL/PARTIAL/NONE) ==="
|
||||
python3 build_rdf_dataset.py label
|
||||
echo ""
|
||||
echo "=== trainset (Auszuege + grounded Tripel) ==="
|
||||
python3 build_rdf_dataset.py trainset
|
||||
echo ""
|
||||
echo "Fertig. trainset.jsonl ist der trainingsfertige Datensatz."
|
||||
echo "Split: python3 build_rdf_dataset.py split (auf samples-Basis)"
|
||||
139
llm_extract.py
139
llm_extract.py
@ -46,6 +46,8 @@ OLLAMA_URL = "http://localhost:11434/api/chat"
|
||||
BACKEND = "ollama"
|
||||
VLLM_URL = "http://silicon.fhgr.ch:7080/v1/chat/completions"
|
||||
API_KEY = ""
|
||||
NO_THINK = True # disable reasoning "thinking" on vLLM (huge speedup, see below)
|
||||
WINDOW_WORKERS = 8 # concurrent windows per trust (trust-workers x this <= server cap)
|
||||
|
||||
SYSTEM = (
|
||||
"You are an information-extraction engine. You read a U.S. mutual-fund "
|
||||
@ -94,12 +96,14 @@ def ollama_chat(model, system, prompt, timeout=600, num_ctx=32768):
|
||||
"model": model,
|
||||
"messages": messages,
|
||||
"temperature": 0,
|
||||
# Reasoning model: it spends several thousand tokens "thinking" before
|
||||
# emitting JSON. Too small a budget -> it never reaches the answer and
|
||||
# returns empty content. Keep this generous.
|
||||
"max_tokens": 8000,
|
||||
"response_format": {"type": "json_object"},
|
||||
}
|
||||
# Disable the reasoning model's "thinking": for this yes/no grounding task
|
||||
# it spends thousands of tokens deliberating before the JSON, making each
|
||||
# call ~16x slower (7.8s -> 0.5s measured) with no quality gain.
|
||||
if NO_THINK:
|
||||
body["chat_template_kwargs"] = {"enable_thinking": False}
|
||||
headers = {"Content-Type": "application/json"}
|
||||
if API_KEY:
|
||||
headers["Authorization"] = f"Bearer {API_KEY}"
|
||||
@ -209,12 +213,18 @@ MATCH_SYSTEM = (
|
||||
)
|
||||
|
||||
MATCH_PROMPT = """\
|
||||
For each numbered CLAIM below, decide whether the TEXT supports it for the fund.
|
||||
A claim is supported only if the TEXT states that the named entity plays that
|
||||
role (advisedBy=investment adviser, subAdvisedBy=sub-adviser,
|
||||
transferAgent=transfer agent, custodian=custodian, administrator=administrator,
|
||||
underwrittenBy=distributor/underwriter, seriesOf=the fund is a series of that
|
||||
trust). Use ONLY the TEXT; treat name variants/abbreviations as equal.
|
||||
For each numbered CLAIM below, decide whether the TEXT supports it.
|
||||
Use ONLY the TEXT; treat name variants/abbreviations as equal (e.g. "BNY Mellon"
|
||||
= "The Bank of New York Mellon").
|
||||
|
||||
Two kinds of claim:
|
||||
- ROLE claims (advisedBy, subAdvisedBy, transferAgent, custodian, administrator,
|
||||
underwrittenBy, seriesOf): supported only if the TEXT states the named entity
|
||||
plays that role for the fund (advisedBy=investment adviser, underwrittenBy=
|
||||
distributor/underwriter, seriesOf=the fund is a series of that trust, etc.).
|
||||
- PRESENCE claims (hasShareClass, className, ticker): supported if the value
|
||||
simply appears in the TEXT — the share-class name or ticker symbol is printed
|
||||
anywhere (e.g. in the fee table or cover). No role needed.
|
||||
|
||||
Output JSON: {{"results": [{{"id": <n>, "supported": true|false}}, ...]}} with one
|
||||
entry per claim id.
|
||||
@ -281,7 +291,7 @@ def _window_relevant(w_low, name_tokens):
|
||||
return any(tok in w_low for tok in name_tokens)
|
||||
|
||||
|
||||
def match_windowed(model, gold_triples, text, win_chars, overlap, num_ctx, timeout=600):
|
||||
def match_windowed(model, gold_triples, text, win_chars, overlap, num_ctx, timeout=150):
|
||||
"""For each gold triple, is it supported somewhere in the text?
|
||||
|
||||
Retrieval pre-filter: only windows that contain a relation keyword or a
|
||||
@ -297,37 +307,65 @@ def match_windowed(model, gold_triples, text, win_chars, overlap, num_ctx, timeo
|
||||
name_tokens.update(_name_tokens(t.get("o", "")))
|
||||
step = max(1, win_chars - overlap)
|
||||
windows = [text[i:i + win_chars] for i in range(0, max(1, len(text)), step)]
|
||||
# keep only relevant windows (retrieval pre-filter)
|
||||
kept = [w for w in windows
|
||||
if len(w.strip()) >= 200 and _window_relevant(w.lower(), name_tokens)]
|
||||
supported = [False] * len(gold_triples)
|
||||
n_win = n_fail = 0
|
||||
for w in windows:
|
||||
if len(w.strip()) < 200:
|
||||
continue
|
||||
if not _window_relevant(w.lower(), name_tokens):
|
||||
continue # retrieval pre-filter: no provider signal in this window
|
||||
evidence = [None] * len(gold_triples) # supporting window text per triple
|
||||
n_fail = 0
|
||||
|
||||
def run_window(w):
|
||||
prompt = MATCH_PROMPT.format(claims=claims, text=w)
|
||||
ctx = min(num_ctx, max(8192, int(len(prompt) / 3.2) + 2048))
|
||||
ok = False
|
||||
for attempt in range(2): # one retry on transient failure
|
||||
for attempt in range(2):
|
||||
try:
|
||||
raw = ollama_chat(model, MATCH_SYSTEM, prompt, timeout=timeout, num_ctx=ctx)
|
||||
res = parse_results(raw, len(gold_triples))
|
||||
supported = [a or b for a, b in zip(supported, res)]
|
||||
ok = True
|
||||
break
|
||||
return parse_results(raw, len(gold_triples)), w
|
||||
except Exception as e:
|
||||
if attempt == 0:
|
||||
time.sleep(3)
|
||||
else:
|
||||
print(f" window error: {e}")
|
||||
if not ok:
|
||||
n_fail += 1
|
||||
n_win += 1
|
||||
# early exit: once every claim is supported, no need to scan further
|
||||
if all(supported):
|
||||
break
|
||||
# If windows failed, the 'False' entries are unreliable (could be missed calls,
|
||||
# not true absences). Signal this so the caller does not record bogus 0s.
|
||||
return supported, n_win, n_fail
|
||||
return None, w
|
||||
|
||||
# Send a trust's windows CONCURRENTLY: one big book's ~10-15 windows then
|
||||
# saturate the server's batch on their own (sequential per-window left the
|
||||
# server idle). WINDOW_WORKERS caps in-flight calls per trust.
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
if kept:
|
||||
with ThreadPoolExecutor(max_workers=min(WINDOW_WORKERS, len(kept))) as ex:
|
||||
for res, w in ex.map(run_window, kept):
|
||||
if res is None:
|
||||
n_fail += 1
|
||||
continue
|
||||
for i, s in enumerate(res):
|
||||
if s and not supported[i]:
|
||||
supported[i] = True
|
||||
evidence[i] = w # first window that supports triple i
|
||||
return supported, len(kept), n_fail, evidence
|
||||
|
||||
|
||||
def _excerpt(window, name, radius=400):
|
||||
"""Return a tight excerpt of `window` around the first occurrence of `name`.
|
||||
|
||||
Locates the most distinctive token of the entity name and returns +/- radius
|
||||
chars around it (the evidence the model used). Falls back to the window head
|
||||
if the token is not locatable. Returns None if no window.
|
||||
"""
|
||||
if not window:
|
||||
return None
|
||||
toks = [w for w in re.sub(r"[^a-z0-9 ]", " ", (name or "").lower()).split()
|
||||
if len(w) >= 4]
|
||||
low = window.lower()
|
||||
pos = -1
|
||||
for tok in sorted(toks, key=len, reverse=True):
|
||||
p = low.find(tok)
|
||||
if p != -1:
|
||||
pos = p; break
|
||||
if pos == -1:
|
||||
return window[:radius * 2].strip()
|
||||
a = max(0, pos - radius)
|
||||
return window[a:pos + radius].strip()
|
||||
|
||||
|
||||
def _match_one(args, r):
|
||||
@ -351,19 +389,29 @@ def _match_one(args, r):
|
||||
distinct.append({"p": t["p"], "o": t.get("o")})
|
||||
key_of.append(seen[k])
|
||||
n_fail = 0
|
||||
d_ev = [None] * len(distinct)
|
||||
try:
|
||||
if args.window and len(text) > args.window:
|
||||
d_sup, n_win, n_fail = match_windowed(
|
||||
d_sup, n_win, n_fail, d_ev = match_windowed(
|
||||
args.model, distinct, text, args.window, args.overlap, args.num_ctx)
|
||||
else:
|
||||
prompt = MATCH_PROMPT.format(claims=_claim_lines(distinct), text=text)
|
||||
ctx = min(args.num_ctx, max(8192, int(len(prompt) / 3.2) + 2048))
|
||||
raw = ollama_chat(args.model, MATCH_SYSTEM, prompt, num_ctx=ctx)
|
||||
d_sup = parse_results(raw, len(distinct)); n_win = 1
|
||||
d_ev = [text if s else None for s in d_sup]
|
||||
except Exception as e:
|
||||
print(f" [{r.get('sample_id')}] ERROR: {e}")
|
||||
d_sup = [False] * len(distinct); n_win = 0; n_fail = 1
|
||||
ann = [{**t, "llm_grounded": bool(d_sup[key_of[i]])} for i, t in enumerate(gold)]
|
||||
# per distinct claim, cut a tight excerpt (sentence around the name) from its
|
||||
# supporting window, so the training sample carries the evidence text, not the
|
||||
# whole 120KB window.
|
||||
d_exc = [_excerpt(d_ev[j], distinct[j]["o"]) if d_sup[j] else None
|
||||
for j in range(len(distinct))]
|
||||
ann = []
|
||||
for i, t in enumerate(gold):
|
||||
j = key_of[i]
|
||||
ann.append({**t, "llm_grounded": bool(d_sup[j]), "evidence": d_exc[j]})
|
||||
return {"sample_id": r.get("sample_id"), "cik": r["cik"], "fund": r.get("fund"),
|
||||
"n_windows": n_win, "n_failed_windows": n_fail, "n_distinct": len(distinct),
|
||||
"ok": n_fail == 0, "triples": ann}
|
||||
@ -377,15 +425,27 @@ def run_match(args):
|
||||
sequentially within its worker. Results are written as they complete.
|
||||
"""
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
import os
|
||||
rows = [json.loads(l) for l in open(args.inp, encoding="utf-8")]
|
||||
if args.limit:
|
||||
rows = rows[:args.limit]
|
||||
# RESUME: skip samples already in the output (append mode). A server blip that
|
||||
# kills the run no longer wastes completed work — just re-launch the same cmd.
|
||||
done_ids = set()
|
||||
if os.path.exists(args.out):
|
||||
for l in open(args.out, encoding="utf-8"):
|
||||
try:
|
||||
done_ids.add(json.loads(l).get("sample_id"))
|
||||
except Exception:
|
||||
pass
|
||||
todo = [r for r in rows if r.get("sample_id") not in done_ids]
|
||||
print(f"resume: {len(done_ids)} done, {len(todo)} to do (of {len(rows)})")
|
||||
done = tot_g = tot_t = 0
|
||||
t0 = time.time()
|
||||
workers = max(1, args.workers)
|
||||
with open(args.out, "w", encoding="utf-8") as out, \
|
||||
with open(args.out, "a", encoding="utf-8") as out, \
|
||||
ThreadPoolExecutor(max_workers=workers) as ex:
|
||||
futs = {ex.submit(_match_one, args, r): r for r in rows}
|
||||
futs = {ex.submit(_match_one, args, r): r for r in todo}
|
||||
for fut in as_completed(futs):
|
||||
rec = fut.result()
|
||||
ng = sum(t.get("llm_grounded") for t in rec["triples"])
|
||||
@ -393,10 +453,10 @@ def run_match(args):
|
||||
out.write(json.dumps(rec, ensure_ascii=False) + "\n"); out.flush()
|
||||
done += 1
|
||||
dt = time.time() - t0
|
||||
print(f"[{done}/{len(rows)}] {rec.get('fund') or rec['cik']:.38s} "
|
||||
print(f"[{done}/{len(todo)}] {rec.get('fund') or rec['cik']:.38s} "
|
||||
f"grounded {ng}/{len(rec['triples'])} "
|
||||
f"({dt/done:.1f}s/sample avg, {workers}w)")
|
||||
print(f"\nmatched {done} samples -> {args.out} "
|
||||
print(f"\nmatched {done} new samples -> {args.out} "
|
||||
f"(LLM-grounded {tot_g}/{tot_t} = {100*tot_g/max(1,tot_t):.0f}%)")
|
||||
|
||||
|
||||
@ -438,7 +498,7 @@ def run_extract(args):
|
||||
|
||||
|
||||
def main():
|
||||
global BACKEND, VLLM_URL, API_KEY
|
||||
global BACKEND, VLLM_URL, API_KEY, NO_THINK
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--mode", choices=["extract", "match"], default="extract",
|
||||
help="extract = free triple extraction (baseline); "
|
||||
@ -454,6 +514,8 @@ def main():
|
||||
"qwen3.5-122b-a10b-fp8 (vllm)")
|
||||
ap.add_argument("--vllm-url", default=VLLM_URL)
|
||||
ap.add_argument("--api-key", default="")
|
||||
ap.add_argument("--think", action="store_true",
|
||||
help="keep reasoning thinking on (vllm); default off for speed")
|
||||
ap.add_argument("--limit", type=int, default=0)
|
||||
ap.add_argument("--num-ctx", type=int, default=32768)
|
||||
ap.add_argument("--window", type=int, default=0,
|
||||
@ -469,6 +531,7 @@ def main():
|
||||
args = ap.parse_args()
|
||||
|
||||
BACKEND = args.backend
|
||||
NO_THINK = not args.think
|
||||
VLLM_URL = args.vllm_url
|
||||
API_KEY = args.api_key
|
||||
if not args.model:
|
||||
|
||||
72
resume_match.sh
Normal file
72
resume_match.sh
Normal file
@ -0,0 +1,72 @@
|
||||
#!/usr/bin/env bash
|
||||
# ------------------------------------------------------------------
|
||||
# resume_match.sh — LLM-Match sauber fortsetzen nach Unterbruch
|
||||
# (Deckel zu / Schlaf / Crash).
|
||||
#
|
||||
# bash resume_match.sh
|
||||
#
|
||||
# Idee: Alle bereits sauber gematchten Trusts (ueber ALLE Teil-Dateien,
|
||||
# OHNE fehlgeschlagene Fenster) werden in data/rdf_poc/match_all.jsonl
|
||||
# konsolidiert; alle fehlenden ODER crash-verseuchten Trusts werden neu
|
||||
# gematcht. Beliebig oft wiederholbar — jeder Lauf macht nur den Rest.
|
||||
#
|
||||
# Sicher gegen Schlaf: laeuft unter caffeinate. ABER: bei DECKEL ZU auf
|
||||
# einem MacBook ohne externen Monitor schlaeft das System trotzdem
|
||||
# (clamshell). Dann einfach nach dem Aufklappen erneut starten.
|
||||
# ------------------------------------------------------------------
|
||||
set -euo pipefail
|
||||
cd "$(dirname "$0")"
|
||||
|
||||
OUT="data/rdf_poc/match_all.jsonl"
|
||||
INPUT="data/rdf_poc/match_input.jsonl"
|
||||
WIN=120000; OVERLAP=10000; WORKERS=2; WINWORKERS=6
|
||||
|
||||
# 1) Konsolidiere alle sauberen Ergebnisse aus allen match_*.jsonl Teil-Dateien
|
||||
python3 - "$OUT" "$INPUT" <<'PY'
|
||||
import json, glob, sys, os
|
||||
out_path, input_path = sys.argv[1], sys.argv[2]
|
||||
parts = sorted(glob.glob("data/rdf_poc/match_all.jsonl")
|
||||
+ glob.glob("data/rdf_poc/match_all_clean*.jsonl")
|
||||
+ glob.glob("data/rdf_poc/match_remaining*.jsonl"))
|
||||
good = {} # cik -> record, only if no failed windows
|
||||
for p in parts:
|
||||
if not os.path.exists(p): continue
|
||||
for l in open(p):
|
||||
try: r = json.loads(l)
|
||||
except: continue
|
||||
if r.get("n_failed_windows", 0) == 0 and r.get("triples") is not None:
|
||||
good[r["cik"]] = r # last clean wins
|
||||
with open(out_path, "w") as f:
|
||||
for r in good.values():
|
||||
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||||
# Restliche (fehlende oder verseuchte) Trusts -> remaining-Input
|
||||
allin = [json.loads(l) for l in open(input_path)]
|
||||
remaining = [r for r in allin if r["cik"] not in good]
|
||||
with open("data/rdf_poc/match_input_remaining.jsonl", "w") as f:
|
||||
for r in remaining:
|
||||
f.write(json.dumps(r) + "\n")
|
||||
print(f"sauber konsolidiert: {len(good)} Trusts -> {out_path}")
|
||||
print(f"noch zu matchen: {len(remaining)} Trusts")
|
||||
PY
|
||||
|
||||
REMAIN=$(wc -l < data/rdf_poc/match_input_remaining.jsonl | tr -d ' ')
|
||||
if [ "$REMAIN" = "0" ]; then
|
||||
echo "ALLES fertig — match_all.jsonl ist vollstaendig."
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Starte Nachlauf fuer $REMAIN Trusts (unter caffeinate)..."
|
||||
# unter caffeinate, damit System-Schlaf (nicht Deckel) den Lauf nicht killt
|
||||
caffeinate -i -m -s python3 -u -c "
|
||||
import llm_extract as L, sys
|
||||
L.WINDOW_WORKERS=$WINWORKERS
|
||||
sys.argv=['x','--mode','match','--backend','vllm',
|
||||
'--in','data/rdf_poc/match_input_remaining.jsonl',
|
||||
'--out','data/rdf_poc/match_remaining_$(date +%H%M%S).jsonl',
|
||||
'--window','$WIN','--overlap','$OVERLAP','--num-ctx','131072','--workers','$WORKERS']
|
||||
L.main()
|
||||
"
|
||||
|
||||
echo ""
|
||||
echo "Nachlauf-Teil fertig. Erneut 'bash resume_match.sh' ausfuehren,"
|
||||
echo "um zu konsolidieren und (falls noch Rest) weiterzumachen."
|
||||
66
watch_match.sh
Normal file
66
watch_match.sh
Normal file
@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
# ------------------------------------------------------------------
|
||||
# watch_match.sh — Fortschritt des LLM-Match-Laufs ueberwachen
|
||||
#
|
||||
# bash watch_match.sh -> Dashboard, aktualisiert alle 15s (Strg-C beendet)
|
||||
# bash watch_match.sh live -> rohes Log live mitlaufen (tail -f)
|
||||
# bash watch_match.sh once -> einmaliger Stand, dann zurueck zur Shell
|
||||
#
|
||||
# Beendet nur die ANSICHT, nicht den Match-Lauf (der laeuft im Hintergrund).
|
||||
# ------------------------------------------------------------------
|
||||
LOG="/private/tmp/claude-501/-Users-florianherzog-development-AISE-Admin-Thesis--Florian-Code/745ff40e-0ecf-4430-886c-33fbc724df69/tasks/bfe2v693w.output"
|
||||
OUT="data/rdf_poc/match_remaining.jsonl"
|
||||
TOTAL=256
|
||||
SERVER="http://silicon.fhgr.ch:7080/v1/models"
|
||||
|
||||
show_once () {
|
||||
local done last errs srv tg tt
|
||||
done=$(wc -l < "$OUT" 2>/dev/null | tr -d ' '); done=${done:-0}
|
||||
last=$(grep -oE '\[[0-9]+/'"$TOTAL"'\]' "$LOG" 2>/dev/null | tail -1)
|
||||
errs=$(grep -cE 'error|ERROR|Traceback' "$LOG" 2>/dev/null | tr -d ' ')
|
||||
# Server erreichbar?
|
||||
# 2 Versuche mit grosszuegigem Timeout: unter Last antwortet /v1/models
|
||||
# langsamer, ein kurzes Timeout meldet faelschlich "nicht erreichbar".
|
||||
if curl -s -m 20 "$SERVER" >/dev/null 2>&1 || curl -s -m 20 "$SERVER" >/dev/null 2>&1; then
|
||||
srv="OK"; else srv="NICHT erreichbar (oder ausgelastet)"; fi
|
||||
# bisherige Uebereinstimmung aus den fertigen Samples
|
||||
read tg tt < <(python3 - "$OUT" <<'PY' 2>/dev/null
|
||||
import json,sys
|
||||
g=t=0
|
||||
try:
|
||||
for l in open(sys.argv[1]):
|
||||
r=json.loads(l)
|
||||
for x in r.get("triples",[]):
|
||||
t+=1; g+=1 if x.get("llm_grounded") else 0
|
||||
except FileNotFoundError: pass
|
||||
print(g,t)
|
||||
PY
|
||||
)
|
||||
tg=${tg:-0}; tt=${tt:-0}
|
||||
local pct=0; [ "$tt" -gt 0 ] && pct=$((100*tg/tt))
|
||||
local dpct=$((100*done/TOTAL))
|
||||
printf "\n LLM-Match-Lauf\n"
|
||||
printf " ------------------------------------------\n"
|
||||
printf " Fortschritt : %d / %d Trusts (%d%%) letzte Logzeile: %s\n" "$done" "$TOTAL" "$dpct" "${last:-—}"
|
||||
printf " Tripel : %d von %d belegt (%d%% Uebereinstimmung bisher)\n" "$tg" "$tt" "$pct"
|
||||
printf " Fehler/Log : %s\n" "${errs:-0}"
|
||||
printf " vLLM-Server : %s\n" "$srv"
|
||||
printf " ------------------------------------------\n"
|
||||
}
|
||||
|
||||
case "${1:-dash}" in
|
||||
live) echo "Live-Log (Strg-C beendet die Ansicht):"; exec tail -f "$LOG" ;;
|
||||
once) show_once ;;
|
||||
*)
|
||||
echo "Dashboard – aktualisiert alle 15s (Strg-C beendet die Ansicht, nicht den Lauf)"
|
||||
while true; do
|
||||
clear 2>/dev/null
|
||||
show_once
|
||||
done=$(wc -l < "$OUT" 2>/dev/null | tr -d ' ')
|
||||
if grep -q "matched .* samples" "$LOG" 2>/dev/null; then
|
||||
echo " FERTIG. $(grep 'matched .* samples' "$LOG" | tail -1)"
|
||||
break
|
||||
fi
|
||||
sleep 15
|
||||
done ;;
|
||||
esac
|
||||
Loading…
x
Reference in New Issue
Block a user