From 1e1c8d529904cb793119ec2620b70234bdfcaade Mon Sep 17 00:00:00 2001 From: Julian Pawlowski Date: Mon, 6 Apr 2026 12:18:40 +0000 Subject: [PATCH] feat(periods): handle flat days and absolute low-price scenarios MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three complementary fixes for pathological price days: 1. Adaptive min_periods for flat days (CV ≤ 10%): On days with nearly uniform prices (e.g. solar surplus), enforcing multiple distinct cheap periods is geometrically impossible. _compute_day_effective_min() detects CV ≤ LOW_CV_FLAT_DAY_THRESHOLD and reduces the effective target to 1 for that day (best price only; peak price always runs full relaxation). 2. min_distance scaling on absolute low-price days: When the daily average drops below 0.10 EUR (10 ct), percentage-based min_distance becomes unreliable. The threshold is scaled linearly to zero so the filter neither accepts the entire day nor blocks everything. 3. CV quality gate bypass for absolute low-price periods: Periods with a mean below 0.10 EUR may show high relative CV even though the absolute price differences are fractions of a cent. Both _check_period_quality() and _check_merge_quality_gate() now bypass the CV gate below this threshold. Additionally: span-aware flex warnings now emit INFO/WARNING when base_flex >= 25%/30% and at least one "normal" (non-V-shape) day exists (FLEX_WARNING_VSHAPE_RATIO = 0.5). Previously the constants were defined but never used. Updated 3 test assertions in test_best_price_e2e.py: the flat-day fixture (CV ~5.4%) correctly produces 1 period, not 2. Impact: Best Price periods now appear reliably on V-shape solar days and flat-price days. No more "0 periods" on days where the single cheapest window is a valid and useful result. --- .../period_handlers/level_filtering.py | 25 +++ .../period_handlers/period_overlap.py | 15 ++ .../coordinator/period_handlers/relaxation.py | 168 +++++++++++++++++- tests/test_best_price_e2e.py | 15 +- 4 files changed, 214 insertions(+), 9 deletions(-) diff --git a/custom_components/tibber_prices/coordinator/period_handlers/level_filtering.py b/custom_components/tibber_prices/coordinator/period_handlers/level_filtering.py index 2fe3e36..a57f291 100644 --- a/custom_components/tibber_prices/coordinator/period_handlers/level_filtering.py +++ b/custom_components/tibber_prices/coordinator/period_handlers/level_filtering.py @@ -25,6 +25,14 @@ INDENT_L0 = "" # Entry point / main function FLEX_SCALING_THRESHOLD = 0.20 # 20% - start adjusting min_distance SCALE_FACTOR_WARNING_THRESHOLD = 0.8 # Log when reduction > 20% +# Low absolute price threshold for min_distance scaling (in major currency unit, e.g. EUR/NOK) +# When the daily average price is below this, percentage-based min_distance becomes unreliable: +# even the daily minimum may not fall far enough below average in relative terms. +# Scale min_distance linearly to 0 as avg_price approaches 0. +# Value: 0.10 EUR/NOK = 10 ct/øre. +# At avg ≥ 0.10: full min_distance. At avg = 0.05: 50% min_distance. At avg = 0: 0%. +LOW_PRICE_AVG_THRESHOLD = 0.10 # EUR/NOK major unit (= 10 ct/øre in subunit) + def check_level_with_gap_tolerance( interval_level: int, @@ -203,6 +211,23 @@ def check_interval_criteria( scale_factor, ) + # ============================================================ + # ABSOLUTE LOW-PRICE SCALING: Reduce min_distance when avg is very low + # ============================================================ + # Problem: On days where the entire price level is extremely low (e.g., avg=3 ct), + # even the daily minimum might not fall 5% below the average in relative terms. + # Example: avg=3 ct, min_distance=5% → threshold=2.85 ct. + # If min=2.9 ct, no interval qualifies despite being genuinely cheap. + # + # Solution: Scale min_distance linearly to 0 as avg_price approaches 0. + # At avg=10 ct → full min_distance; at avg=5 ct → 50%; at avg=0 ct → 0%. + # + # This is currency-agnostic: 10 ct EUR and 10 øre NOK are both + # "very low price territory" for their respective markets. + if criteria.avg_price < LOW_PRICE_AVG_THRESHOLD and LOW_PRICE_AVG_THRESHOLD > 0: + low_price_scale = max(0.0, criteria.avg_price / LOW_PRICE_AVG_THRESHOLD) + adjusted_min_distance = adjusted_min_distance * low_price_scale + # Calculate threshold from average (using normalized positive distance) # - Peak price: threshold = avg * (1 + distance/100) → prices must be ABOVE avg+distance # - Best price: threshold = avg * (1 - distance/100) → prices must be BELOW avg-distance diff --git a/custom_components/tibber_prices/coordinator/period_handlers/period_overlap.py b/custom_components/tibber_prices/coordinator/period_handlers/period_overlap.py index cecdc76..ed7a993 100644 --- a/custom_components/tibber_prices/coordinator/period_handlers/period_overlap.py +++ b/custom_components/tibber_prices/coordinator/period_handlers/period_overlap.py @@ -161,12 +161,27 @@ def _check_merge_quality_gate(periods_to_merge: list[tuple[int, dict]], relaxed: Returns True if merge is allowed, False if blocked by Quality Gate. """ + from .relaxation import LOW_PRICE_QUALITY_BYPASS_THRESHOLD # noqa: PLC0415 from .types import PERIOD_MAX_CV # noqa: PLC0415 relaxed_start = relaxed["start"] relaxed_end = relaxed["end"] for _idx, existing in periods_to_merge: + # Low absolute price bypass: if the combined price level is very cheap, + # the quality gate is bypassed (same logic as in _check_period_quality). + # Estimated combined mean = (combined_min + combined_max) / 2 + r_min = relaxed.get("price_min") + r_max = relaxed.get("price_max") + e_min = existing.get("price_min") + e_max = existing.get("price_max") + if None not in (r_min, r_max, e_min, e_max): + combined_min = min(float(r_min), float(e_min)) # type: ignore[arg-type] + combined_max = max(float(r_max), float(e_max)) # type: ignore[arg-type] + combined_mean = (combined_min + combined_max) / 2 + if combined_mean < LOW_PRICE_QUALITY_BYPASS_THRESHOLD: + continue # Very low absolute price → allow merge, check next pair + estimated_cv = _estimate_merged_cv(existing, relaxed) if estimated_cv is not None and estimated_cv > PERIOD_MAX_CV: _LOGGER.debug( diff --git a/custom_components/tibber_prices/coordinator/period_handlers/relaxation.py b/custom_components/tibber_prices/coordinator/period_handlers/relaxation.py index a43fec3..695d8d9 100644 --- a/custom_components/tibber_prices/coordinator/period_handlers/relaxation.py +++ b/custom_components/tibber_prices/coordinator/period_handlers/relaxation.py @@ -40,6 +40,23 @@ FLEX_HIGH_THRESHOLD_RELAXATION = 0.30 # 30% - WARNING: base flex too high for r MIN_DURATION_FALLBACK_MINIMUM = 30 # Minimum period length to try (30 min = 2 intervals) MIN_DURATION_FALLBACK_STEP = 15 # Reduce by 15 min (1 interval) each step +# Low absolute price threshold for quality gate bypass (in major currency unit, e.g. EUR/NOK) +# When the MEAN price of a period is below this level, the CV quality gate is bypassed. +# Relative CV is unreliable at very low absolute prices: a range of 1-4 ct shows CV≈50% +# but is practically homogeneous from a cost perspective. +# Value: LOW_PRICE_AVG_THRESHOLD (subunit) / 100 = 10 ct / 100 = 0.10 EUR/NOK +LOW_PRICE_QUALITY_BYPASS_THRESHOLD = 0.10 # EUR/NOK major unit (= 10 ct/øre) + +# Span-to-ref ratio threshold for suppressing flex warnings on V-shape days. +# When span / ref_price < this on ANY available day, the warning is shown. +# Below 0.5 means span < 50% of ref_price → "normal" day (high flex is genuinely risky). +# Above 0.5 means V-shape day (span-based formula already compensates → warning less relevant). +FLEX_WARNING_VSHAPE_RATIO = 0.5 # span/ref_price ratio below which a day is considered "normal" +# On flat price days (low variation), it is unrealistic to require multiple distinct +# best/peak price periods. Requiring 2+ periods would force relaxation to create +# artificial periods that don't represent genuine price structure. +LOW_CV_FLAT_DAY_THRESHOLD = 10.0 # %: days with CV ≤ this need only 1 period + def _check_period_quality( period: dict, all_prices: list[dict], *, time: TibberPricesTimeService @@ -95,6 +112,16 @@ def _check_period_quality( if cv is None: return True, None + # Low absolute price bypass: when the MEAN period price is very cheap (below + # LOW_PRICE_QUALITY_BYPASS_THRESHOLD), relative CV becomes unreliable. + # Example: period at 1-4 ct (mean 2 ct) shows CV≈50% but is practically + # homogeneous from a cost perspective; the quality gate should not reject it. + # Using mean (not range) to distinguish genuinely cheap days from flat-priced + # normal days: a flat day at 33-36 ct has a small range BUT a high mean → no bypass. + period_mean = sum(period_prices) / len(period_prices) + if period_mean < LOW_PRICE_QUALITY_BYPASS_THRESHOLD: + return True, cv # Passes quality gate: absolute price level is very low + passes = cv <= PERIOD_MAX_CV return passes, cv @@ -418,6 +445,79 @@ def _try_min_duration_fallback( return None, metadata +def _compute_day_effective_min( + prices_by_day: dict, + min_periods: int, + *, + enable_relaxation: bool, + reverse_sort: bool, +) -> tuple[dict, int]: + """ + Compute per-day effective min_periods with flat-day adaptation. + + On days with very low price variation (CV ≤ LOW_CV_FLAT_DAY_THRESHOLD), + requiring multiple distinct cheapest/peak periods is unrealistic. Finding + ONE period is sufficient because there is no meaningful price structure that + would create natural multiple periods. + + This applies ONLY to BEST PRICE periods (reverse_sort=False). For PEAK PRICE + periods, full relaxation should run even on flat days because identifying the + genuinely most expensive window requires the complete filter evaluation. + (Design decision: if the user explicitly disabled relaxation, honour the + configured min_periods exactly regardless.) + + Args: + prices_by_day: Dict of date → list of price dicts + min_periods: Configured minimum periods per day + enable_relaxation: Whether relaxation is enabled + reverse_sort: True for peak price (no adaptation), False for best price + + Returns: + Tuple of (dict of date → effective min_periods for that day, count of flat days detected) + + """ + day_effective_min = {} + flat_day_count = 0 + min_prices_for_cv = 2 # Need at least 2 prices to calculate CV + + for day, day_prices in prices_by_day.items(): + if not enable_relaxation or min_periods <= 1 or reverse_sort: + # Relaxation disabled, already 1, or peak price: no adaptation + day_effective_min[day] = min_periods + continue + + price_values = [float(p["total"]) for p in day_prices if p.get("total") is not None] + + if len(price_values) < min_prices_for_cv: + day_effective_min[day] = min_periods + continue + + day_cv = calculate_coefficient_of_variation(price_values) + + if day_cv is not None and day_cv <= LOW_CV_FLAT_DAY_THRESHOLD: + day_effective_min[day] = 1 + flat_day_count += 1 + _LOGGER_DETAILS.debug( + "%sDay %s: flat price profile (CV=%.1f%% ≤ %.1f%%) → min_periods relaxed to 1", + INDENT_L1, + day, + day_cv, + LOW_CV_FLAT_DAY_THRESHOLD, + ) + else: + day_effective_min[day] = min_periods + + if flat_day_count > 0: + _LOGGER.info( + "Adaptive min_periods: %d flat day(s) (CV ≤ %.0f%%) need only 1 period instead of %d", + flat_day_count, + LOW_CV_FLAT_DAY_THRESHOLD, + min_periods, + ) + + return day_effective_min, flat_day_count + + def calculate_periods_with_relaxation( # noqa: PLR0912, PLR0913, PLR0915 - Per-day relaxation requires many parameters and branches all_prices: list[dict], *, @@ -549,6 +649,55 @@ def calculate_periods_with_relaxation( # noqa: PLR0912, PLR0913, PLR0915 - Per- prices_by_day = group_prices_by_day(all_prices, time=time) total_days = len(prices_by_day) + # ========================================================================= + # SPAN-AWARE FLEX WARNINGS + # ========================================================================= + # With the span-based flex formula (flex_base = max(span, ref_price)), a high + # configured base_flex is less problematic on V-shape days (single low outlier) + # where span dominates. However, on normal days (span ≈ ref_price), high base_flex + # leaves little room for relaxation to escalate effectively. + # + # We only warn when relaxation is enabled AND the flex is high on typical days. + # V-shape days have span >> ref_price; normal days have span ≈ ref_price. + # We use the MEAN span/ref ratio across available days as a proxy: + # - Ratio near 1.0 → span ≈ ref (normal day, warning is relevant) + # - Ratio >> 1.0 → span >> ref (V-shape day, warning less relevant) + # Threshold: if any day has span/ref < 0.5 (i.e., span < 50% of ref_price), + # the spread is not extreme enough to suppress the warning. + if enable_relaxation and prices_by_day: + base_flex = abs(config.flex) + if base_flex >= FLEX_WARNING_THRESHOLD_RELAXATION: + # Check whether any available day is "normal enough" to make the warning relevant + any_normal_day = False + for day_prices in prices_by_day.values(): + prices = [float(p["total"]) for p in day_prices if p.get("total") is not None] + if len(prices) >= 2: # noqa: PLR2004 + day_min = min(prices) + day_avg = sum(prices) / len(prices) + span = abs(day_avg - day_min) + if day_min > 0 and span / day_min < FLEX_WARNING_VSHAPE_RATIO: + any_normal_day = True + break + + if any_normal_day: + if base_flex >= FLEX_HIGH_THRESHOLD_RELAXATION: + _LOGGER.warning( + "Base flex %.0f%% is too high for relaxation mode. " + "Relaxation escalates in 3%% steps from your base - starting at %.0f%% " + "means only ~%.0f steps remain before the 50%% hard limit. " + "Recommendation: Use 15-20%% base flex with relaxation enabled.", + base_flex * 100, + base_flex * 100, + (MAX_FLEX_HARD_LIMIT - base_flex) / 0.03, + ) + else: + _LOGGER.info( + "Base flex %.0f%% is high for relaxation mode (recommended: 15-20%%). " + "Only ~%.0f relaxation steps remain to 50%% hard limit.", + base_flex * 100, + (MAX_FLEX_HARD_LIMIT - base_flex) / 0.03, + ) + _LOGGER.info( "Calculating baseline periods for %d days...", total_days, @@ -567,21 +716,31 @@ def calculate_periods_with_relaxation( # noqa: PLR0912, PLR0913, PLR0915 - Per- # Count periods per day for min_periods check periods_by_day = group_periods_by_day(all_periods) + # Pre-compute per-day effective min_periods (adaptive for flat days) + # On flat price days (low CV), 1 period is sufficient - enforcing min_periods > 1 + # would only produce artificial additional periods of no practical value. + # NOTE: Only applied for best price (reverse_sort=False); peak price always uses + # full relaxation to properly identify the genuinely most expensive window. + day_effective_min, flat_days_count = _compute_day_effective_min( + prices_by_day, min_periods, enable_relaxation=enable_relaxation, reverse_sort=config.reverse_sort + ) + days_meeting_requirement = 0 for day in sorted(prices_by_day.keys()): day_periods = periods_by_day.get(day, []) period_count = len(day_periods) + effective_min = day_effective_min.get(day, min_periods) _LOGGER_DETAILS.debug( "%sDay %s baseline: Found %d periods%s", INDENT_L1, day, period_count, - f" (need {min_periods})" if enable_relaxation else "", + f" (need {effective_min})" if enable_relaxation else "", ) - if period_count >= min_periods: + if period_count >= effective_min: days_meeting_requirement += 1 # Check if relaxation is needed @@ -620,7 +779,7 @@ def calculate_periods_with_relaxation( # noqa: PLR0912, PLR0913, PLR0915 - Per- for day in sorted(prices_by_day.keys()): day_periods = periods_by_day.get(day, []) period_count = len(day_periods) - if period_count >= min_periods: + if period_count >= day_effective_min.get(day, min_periods): days_meeting_requirement += 1 # === MIN DURATION FALLBACK === @@ -650,7 +809,7 @@ def calculate_periods_with_relaxation( # noqa: PLR0912, PLR0913, PLR0915 - Per- for day in sorted(prices_by_day.keys()): day_periods = periods_by_day.get(day, []) period_count = len(day_periods) - if period_count >= min_periods: + if period_count >= day_effective_min.get(day, min_periods): days_meeting_requirement += 1 elif enable_relaxation: @@ -692,6 +851,7 @@ def calculate_periods_with_relaxation( # noqa: PLR0912, PLR0913, PLR0915 - Per- "days_processed": total_days, "days_meeting_requirement": days_meeting_requirement, "relaxation_incomplete": days_meeting_requirement < total_days, + "flat_days_detected": flat_days_count, # Days where adaptive min_periods (CV-based) reduced target to 1 } return final_result diff --git a/tests/test_best_price_e2e.py b/tests/test_best_price_e2e.py index da3608d..d5cfd70 100644 --- a/tests/test_best_price_e2e.py +++ b/tests/test_best_price_e2e.py @@ -148,8 +148,9 @@ class TestBestPriceGenerationWorks: periods = result.get("periods", []) # Validation: periods found - assert len(periods) > 0, "Best periods should generate" - assert 2 <= len(periods) <= 5, f"Expected 2-5 periods, got {len(periods)}" + # Note: test fixture is a flat day (CV≈5.4%). Adaptive min_periods correctly + # returns 1 period instead of forcing a 2nd artificial period. + assert len(periods) >= 1, f"Best periods should generate, got {len(periods)}" def test_positive_flex_produces_periods(self) -> None: """ @@ -188,7 +189,8 @@ class TestBestPriceGenerationWorks: periods_pos = result_pos.get("periods", []) # With positive flex, should find periods - assert len(periods_pos) >= 2, f"Should find periods with positive flex, got {len(periods_pos)}" + # Note: flat day (CV≈5.4%) → adaptive min_periods returns 1 period (correct behavior). + assert len(periods_pos) >= 1, f"Should find periods with positive flex, got {len(periods_pos)}" def test_periods_contain_low_prices(self) -> None: """ @@ -271,8 +273,11 @@ class TestBestPriceGenerationWorks: periods = result.get("periods", []) - # Should find periods via relaxation - assert len(periods) >= 2, "Relaxation should find periods" + # Should find at least 1 period. + # Note: flat day (CV≈5.4%) → adaptive min_periods correctly needs only 1 period, + # which baseline already provides without relaxation. This validates that + # over-relaxation is skipped on truly flat days. + assert len(periods) >= 1, "Should find at least 1 period" # Check if relaxation was used relaxation_meta = result.get("metadata", {}).get("relaxation", {})