Portfolio_Risk_Management/tests/test_cleaning.py

import numpy as np
import pandas as pd
import pytest

from quantcore.data.cleaning import (
    CleaningReport,
    clean_fx,
    clean_prices,
    find_spike_blocks,
    fx_scale_exponent,
    longest_missing_run,
)


def test_spike_blocks_and_genuine_moves():
    idx = pd.bdate_range("2024-01-01", periods=40)
    p = pd.Series(100.0, index=idx)
    p.iloc[5] = 200.0  # single wrong tick (2x, reverses next day)
    p.iloc[10:12] = 210.0  # two-day block
    p.iloc[20:] = 40.0  # genuine crash that stays
    p.iloc[30:33] = 45.0  # ordinary small moves
    assert find_spike_blocks(p) == [5, 10, 11]


def test_clean_prices_report_and_gaps():
    idx = pd.bdate_range("2024-01-01", periods=40)
    a = pd.Series(np.linspace(50, 60, 40), index=idx)
    b = a.copy()
    b.iloc[3:8] = -1.0  # broken adjustment stretch
    b.iloc[25] = 300.0  # spike
    from quantcore.data.cleaning import CleaningReport

    rep = CleaningReport()
    out = clean_prices(pd.DataFrame({"A": a, "B": b}), report=rep)
    assert rep.n_nonpositive == 5 and rep.symbols_nonpositive == {"B": 5}
    assert rep.n_spikes == 1 and rep.symbols_spikes == {"B": 1}
    assert out["A"].notna().all()
    assert out["B"].isna().sum() == 6
    runs = longest_missing_run(out)
    assert runs["A"] == 0 and runs["B"] == 5


# ---------------------------------------------------------------------------------------------
# FX quoting-unit changes (the NOK/CHF artefact: 96 observations returned 100x too small)
# ---------------------------------------------------------------------------------------------
def _fx_series(n=500, seed=0):
    rng = np.random.default_rng(seed)
    idx = pd.bdate_range("2016-01-01", periods=n)
    return pd.Series(0.12 * np.exp(np.cumsum(rng.standard_normal(n) * 0.004)), index=idx)


def test_a_quoting_unit_change_is_undone_wherever_it_sits():
    """A stretch quoted 100x too small is repaired, whatever part of the series it covers."""
    good = _fx_series()
    for lo, hi in ((70, 140), (0, 60), (440, 500)):        # middle, start, end
        broken = good.copy()
        broken.iloc[lo:hi] /= 100.0
        fixed = clean_fx(broken.to_frame("NOK"))["NOK"]
        assert np.allclose(fixed.values, good.values), (lo, hi)
        # the artefact is what made the volatility absurd; the repair takes it back
        vol = lambda x: float(np.log(x).diff().std() * np.sqrt(252))
        assert vol(broken) > 2.0 and vol(fixed) == pytest.approx(vol(good), rel=1e-9)


def test_several_unit_changes_and_other_powers_of_ten():
    good = _fx_series()
    broken = good.copy()
    broken.iloc[30:60] /= 100.0
    broken.iloc[200:210] *= 10.0
    broken.iloc[300:301] /= 1000.0
    fixed = clean_fx(broken.to_frame("NOK"))["NOK"]
    assert np.allclose(fixed.values, good.values)
    assert set(fx_scale_exponent(broken).unique()) == {0, -2, 1, -3}


def test_genuine_moves_are_never_rescaled():
    """A devaluation is a price move, not a unit change: nothing short of a factor of ten is touched.

    The 6.1x case is real: ABVX.PA gained 510 % on its trial results (2025-07-23). A loose tolerance
    reads that as a decimal shift and rewrites the year of history that follows it.
    """
    good = _fx_series()
    for factor in (0.75, 1.4, 0.5, 2.0, 3.0, 6.101, 6.421, 1 / 6.101, 8.0):
        moved = good.copy()
        moved.iloc[250:] *= factor
        fixed = clean_fx(moved.to_frame("TRY"))["TRY"]
        assert np.allclose(fixed.values, moved.values), factor


def test_clean_fx_is_idempotent_and_reports_per_currency():
    good = _fx_series()
    frame = pd.DataFrame({"CHF": 1.0, "EUR": good * 8, "NOK": good})
    frame.iloc[100:180, frame.columns.get_loc("NOK")] /= 100.0
    report = CleaningReport()
    fixed = clean_fx(frame, report=report)
    assert report.n_fx_rescaled == 80 and report.fx_rescaled == {"NOK": 80}
    assert np.allclose(fixed["NOK"].values, good.values)
    assert fixed["EUR"].equals(frame["EUR"]) and fixed["CHF"].equals(frame["CHF"])
    again = CleaningReport()
    assert clean_fx(fixed, report=again).equals(fixed) and again.n_fx_rescaled == 0


def test_a_short_series_and_missing_values_are_handled():
    idx = pd.bdate_range("2024-01-01", periods=6)
    assert fx_scale_exponent(pd.Series([0.1], index=idx[:1])).empty is False
    gappy = pd.Series([0.1, np.nan, 0.001, 0.001, np.nan, 0.1], index=idx)
    fixed = clean_fx(gappy.to_frame("NOK"))["NOK"]
    assert fixed.dropna().round(6).tolist() == [0.1, 0.1, 0.1, 0.1]
    assert fixed.isna().tolist() == [False, True, False, False, True, False]


def test_fx_spikes_are_removed_but_a_real_regime_break_is_kept():
    """A reversing tick is a quote error; 2015-01-16 (the SNB floor) is not, and must survive."""
    good = _fx_series(n=400)
    broken = good.copy()
    broken.iloc[120] *= 1.09                         # one-day spike, reverses next day
    broken.iloc[250:] *= 0.81                        # a permanent 19 % repricing: never reversed
    fixed = clean_fx(broken.to_frame("EUR"))["EUR"]
    assert np.isnan(fixed.iloc[120])                 # the tick is gone
    assert fixed.iloc[249] == broken.iloc[249] and fixed.iloc[250] == broken.iloc[250]
    assert fixed.drop(fixed.index[120]).equals(broken.drop(broken.index[120]))


def test_the_unit_repair_runs_before_the_spike_pass():
    """A 100x stretch must be rescaled, not mistaken for two enormous spikes at its edges."""
    good = _fx_series(n=300)
    broken = good.copy()
    broken.iloc[100:160] /= 100.0
    report = CleaningReport()
    fixed = clean_fx(broken.to_frame("NOK"), report=report)["NOK"]
    assert report.n_fx_rescaled == 60 and report.n_fx_spikes == 0
    assert np.allclose(fixed.values, good.values)