Skip to content

fix(infinifi): re-arm liquid reserves alert when cache is stale #331

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
spalen0 wants to merge 1 commit into
base: main
Choose a base branch
from
+19 −0
Draft

fix(infinifi): re-arm liquid reserves alert when cache is stale #331

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 19 additions & 0 deletions protocols/infinifi/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import time
from decimal import Decimal

import requests
Expand All @@ -20,6 +21,7 @@
IUSD_DECIMALS = 18

LIQUID_RESERVES_THRESHOLD = 8_000_000
LIQUID_RESERVES_STALE_AFTER_SECONDS = 12 * 60 * 60 # re-arm crossing alert if cache is older than 12h
BACKING_PER_IUSD_MIN = 0.999
MINT_THRESHOLD_PERCENT = Decimal(Config.get_env("IUSD_LARGE_MINT_THRESHOLD_PERCENT", "0.05"))
REDEMPTION_TO_LIQUID_RATIO_MAX = 0.8
Expand Down Expand Up @@ -216,7 +218,23 @@ def main():
# Alert 1: Low Liquid Reserves
if liquid_reserves > 0:
cache_key_reserves = f"{PROTOCOL}_liquid_reserves"
cache_key_reserves_ts = f"{PROTOCOL}_liquid_reserves_ts"
last_reserves = float(get_last_value_for_key_from_file(cache_filename, cache_key_reserves))
last_reserves_ts_raw = get_last_value_for_key_from_file(cache_filename, cache_key_reserves_ts)
try:
last_reserves_ts = int(last_reserves_ts_raw)
except (TypeError, ValueError):
last_reserves_ts = 0

# If the cached value is older than LIQUID_RESERVES_STALE_AFTER_SECONDS, treat the
# previous reading as "above threshold" so the crossing alert can re-arm after a
# long outage instead of being stuck in the "already alerted" state.
if last_reserves_ts > 0 and (time.time() - last_reserves_ts) > LIQUID_RESERVES_STALE_AFTER_SECONDS:
logger.info(
"Liquid reserves cache is stale (last update > %sh ago); re-arming crossing detection",
LIQUID_RESERVES_STALE_AFTER_SECONDS // 3600,
)
last_reserves = LIQUID_RESERVES_THRESHOLD

if (
last_reserves != 0
Expand All @@ -234,6 +252,7 @@ def main():
send_alert(Alert(AlertSeverity.HIGH, msg, PROTOCOL))

write_last_value_to_file(cache_filename, cache_key_reserves, liquid_reserves)
write_last_value_to_file(cache_filename, cache_key_reserves_ts, int(time.time()))

# Alert 2 and Alert 3 intentionally disabled:
# reserveRatio and illiquidTargetRatio have been persistently violated since inception,
Expand Down