From 0adb525834e045e2e98abe0811f7b2639ebc23ab Mon Sep 17 00:00:00 2001 From: sarangs1621 Date: Sat, 13 Jun 2026 03:52:55 +0530 Subject: [PATCH 1/2] chore: harden production readiness (deps, logging, Sentry, DB pool) - Bump pytest/pytest-asyncio and fastapi/starlette to clear all pip-audit findings (starlette 1.x required two HTTP status constant renames). - Add configurable structured logging (text/json) so existing fail-open warnings are visible in production. - Add optional Sentry error tracking, no-op unless SENTRY_DSN is set. - Make the SQLAlchemy async engine connection pool configurable, with pre-ping and recycle enabled by default for managed Postgres providers. - Document new settings and drop the now-resolved dependency vulnerability note from the README. Co-Authored-By: Claude Sonnet 4.6 --- .env.example | 19 +++++++++++++ README.md | 32 +++++++++++---------- app/core/celery_app.py | 5 ++++ app/core/config.py | 15 ++++++++++ app/core/logging.py | 54 ++++++++++++++++++++++++++++++++++++ app/core/security_headers.py | 2 +- app/core/sentry.py | 23 +++++++++++++++ app/db/session.py | 5 ++++ app/main.py | 7 ++++- requirements-dev.txt | 4 +-- requirements.txt | 3 +- tests/test_logging.py | 54 ++++++++++++++++++++++++++++++++++++ tests/test_sentry.py | 51 ++++++++++++++++++++++++++++++++++ 13 files changed, 254 insertions(+), 20 deletions(-) create mode 100644 app/core/logging.py create mode 100644 app/core/sentry.py create mode 100644 tests/test_logging.py create mode 100644 tests/test_sentry.py diff --git a/.env.example b/.env.example index c10bc05..a39d066 100644 --- a/.env.example +++ b/.env.example @@ -4,6 +4,18 @@ PROJECT_NAME=Sentinel API_V1_PREFIX=/api/v1 BACKEND_CORS_ORIGINS=["http://localhost:3000"] +# --- Logging --- +# LOG_LEVEL: DEBUG | INFO | WARNING | ERROR | CRITICAL +LOG_LEVEL=INFO +# LOG_FORMAT: "text" for human-readable console output, "json" for structured +# single-line logs suited to log aggregators (recommended in production). +LOG_FORMAT=text + +# --- Error tracking (optional) --- +# Leave SENTRY_DSN unset to disable Sentry entirely (default for local dev/tests). +SENTRY_DSN= +SENTRY_TRACES_SAMPLE_RATE=0.0 + # --- Security / JWT --- # Generate with: python -c "import secrets; print(secrets.token_urlsafe(64))" SECRET_KEY=change-me-to-a-long-random-secret @@ -19,6 +31,13 @@ POSTGRES_HOST=localhost POSTGRES_PORT=5432 DATABASE_URL=postgresql+asyncpg://sentinel:sentinel@localhost:5432/sentinel +# SQLAlchemy async engine connection pool tuning. +DB_POOL_SIZE=5 +DB_MAX_OVERFLOW=10 +DB_POOL_TIMEOUT=30 +DB_POOL_RECYCLE_SECONDS=1800 +DB_POOL_PRE_PING=true + # Used by the test suite (separate database) TEST_DATABASE_URL=postgresql+asyncpg://sentinel:sentinel@localhost:5432/sentinel_test diff --git a/README.md b/README.md index cfc4f6c..30fba0d 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,23 @@ Stop it: & "C:\Program Files\PostgreSQL\17\bin\pg_ctl.exe" -D "C:\Users\saran\pgdata\sentinel" stop ``` +## Configuration + +In addition to the database/Redis/JWT settings in `.env.example`, the +following are configurable (see `app/core/config.py` for defaults): + +| Variable | Default | Purpose | +|---|---|---| +| `LOG_LEVEL` | `INFO` | Root logger level (`DEBUG`/`INFO`/`WARNING`/`ERROR`/`CRITICAL`). | +| `LOG_FORMAT` | `text` | `text` for human-readable console logs, `json` for single-line structured logs suited to log aggregators. | +| `SENTRY_DSN` | unset | Enables Sentry error tracking (FastAPI + Celery) when set. No-op when unset. | +| `SENTRY_TRACES_SAMPLE_RATE` | `0.0` | Fraction of requests/tasks to trace when Sentry is enabled. | +| `DB_POOL_SIZE` | `5` | SQLAlchemy async engine connection pool size. | +| `DB_MAX_OVERFLOW` | `10` | Extra connections allowed beyond `DB_POOL_SIZE` under load. | +| `DB_POOL_TIMEOUT` | `30` | Seconds to wait for a pooled connection before raising. | +| `DB_POOL_RECYCLE_SECONDS` | `1800` | Recycle connections older than this, to avoid stale connections on managed Postgres. | +| `DB_POOL_PRE_PING` | `true` | Test connections with a lightweight ping before use. | + ## Database migrations ```bash @@ -143,21 +160,6 @@ fixture credential), regenerate the baseline locally and commit it: detect-secrets scan > .secrets.baseline ``` -### Known issue: pre-existing dependency vulnerabilities - -`pip-audit` currently reports 4 known vulnerabilities, so the `Security -checks` job fails on `main` until these are addressed in a dedicated pass: - -- `pytest==8.3.4` — fix is `9.0.3` (major version bump; needs a compatibility - pass with `pytest-asyncio`/`pytest-cov`/`respx` before upgrading). -- `starlette==0.41.3` (transitive, via `fastapi==0.115.6`) — 3 CVEs, fixes - require `starlette>=0.47`/`1.0.1`, which likely means a `fastapi` major - bump and a full regression pass of the custom middleware stack - (`app/core/security_headers.py`, `app/core/rate_limit.py`, CORS config). - -Tracked as a follow-up; not addressed as part of the CI/CD setup to avoid -bundling a risky framework upgrade with the pipeline rollout. - ## API overview (Phase 1) | Method & Path | Auth | Description | diff --git a/app/core/celery_app.py b/app/core/celery_app.py index 4d84eb7..5e0df4b 100644 --- a/app/core/celery_app.py +++ b/app/core/celery_app.py @@ -1,6 +1,11 @@ from celery import Celery from app.core.config import settings +from app.core.logging import configure_logging +from app.core.sentry import configure_sentry + +configure_logging() +configure_sentry() celery_app = Celery( "sentinel", diff --git a/app/core/config.py b/app/core/config.py index f5e8c0a..a633c7e 100644 --- a/app/core/config.py +++ b/app/core/config.py @@ -1,4 +1,5 @@ from functools import lru_cache +from typing import Literal from pydantic import field_validator, model_validator from pydantic_settings import BaseSettings, SettingsConfigDict @@ -16,6 +17,13 @@ class Settings(BaseSettings): PROJECT_NAME: str = "Sentinel" API_V1_PREFIX: str = "/api/v1" + LOG_LEVEL: Literal["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"] = "INFO" + LOG_FORMAT: Literal["text", "json"] = "text" + + # Error tracking (optional). Leave unset to disable Sentry entirely. + SENTRY_DSN: str | None = None + SENTRY_TRACES_SAMPLE_RATE: float = 0.0 + BACKEND_CORS_ORIGINS: list[str] = [] SECRET_KEY: str @@ -26,6 +34,13 @@ class Settings(BaseSettings): DATABASE_URL: str TEST_DATABASE_URL: str | None = None + # SQLAlchemy async engine connection pool tuning. + DB_POOL_SIZE: int = 5 + DB_MAX_OVERFLOW: int = 10 + DB_POOL_TIMEOUT: int = 30 + DB_POOL_RECYCLE_SECONDS: int = 1800 + DB_POOL_PRE_PING: bool = True + POSTGRES_USER: str = "sentinel" POSTGRES_PASSWORD: str = "sentinel" POSTGRES_DB: str = "sentinel" diff --git a/app/core/logging.py b/app/core/logging.py new file mode 100644 index 0000000..4093ade --- /dev/null +++ b/app/core/logging.py @@ -0,0 +1,54 @@ +import json +import logging +import sys +from datetime import UTC, datetime +from typing import Any + +from app.core.config import settings + +_configured = False + + +class JsonFormatter(logging.Formatter): + """Render log records as single-line JSON for ingestion by log aggregators.""" + + def format(self, record: logging.LogRecord) -> str: + payload: dict[str, Any] = { + "timestamp": datetime.fromtimestamp(record.created, tz=UTC).isoformat(), + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + } + if record.exc_info: + payload["exc_info"] = self.formatException(record.exc_info) + return json.dumps(payload) + + +def configure_logging() -> None: + """Configure the root logger's level and output format. + + Called once at process startup (FastAPI app and Celery workers/beat) so that + `logging.getLogger(__name__)` calls throughout the codebase - e.g. the Redis + fail-open warnings in `app.services.cache` and `app.core.redis` - are emitted + with a consistent, timestamped format at the configured level. + """ + global _configured + if _configured: + return + + handler = logging.StreamHandler(sys.stdout) + if settings.LOG_FORMAT == "json": + handler.setFormatter(JsonFormatter()) + else: + handler.setFormatter( + logging.Formatter( + fmt="%(asctime)s | %(levelname)-8s | %(name)s | %(message)s", + datefmt="%Y-%m-%dT%H:%M:%S%z", + ) + ) + + root_logger = logging.getLogger() + root_logger.setLevel(settings.LOG_LEVEL) + root_logger.addHandler(handler) + + _configured = True diff --git a/app/core/security_headers.py b/app/core/security_headers.py index 7299bfa..75a617d 100644 --- a/app/core/security_headers.py +++ b/app/core/security_headers.py @@ -22,7 +22,7 @@ async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) - too_large = False if too_large: return JSONResponse( - status_code=status.HTTP_413_REQUEST_ENTITY_TOO_LARGE, + status_code=status.HTTP_413_CONTENT_TOO_LARGE, content={"detail": "Request body too large."}, ) diff --git a/app/core/sentry.py b/app/core/sentry.py new file mode 100644 index 0000000..d327cb2 --- /dev/null +++ b/app/core/sentry.py @@ -0,0 +1,23 @@ +import sentry_sdk + +from app.core.config import settings + +_configured = False + + +def configure_sentry() -> None: + """Initialize Sentry error tracking if `SENTRY_DSN` is configured. + + No-op when `SENTRY_DSN` is unset, which is the default for local + development and the test suite. + """ + global _configured + if _configured or not settings.SENTRY_DSN: + return + + sentry_sdk.init( + dsn=settings.SENTRY_DSN, + environment=settings.ENVIRONMENT, + traces_sample_rate=settings.SENTRY_TRACES_SAMPLE_RATE, + ) + _configured = True diff --git a/app/db/session.py b/app/db/session.py index 48f6c56..458175c 100644 --- a/app/db/session.py +++ b/app/db/session.py @@ -8,6 +8,11 @@ settings.DATABASE_URL, echo=settings.ENVIRONMENT == "development", future=True, + pool_size=settings.DB_POOL_SIZE, + max_overflow=settings.DB_MAX_OVERFLOW, + pool_timeout=settings.DB_POOL_TIMEOUT, + pool_recycle=settings.DB_POOL_RECYCLE_SECONDS, + pool_pre_ping=settings.DB_POOL_PRE_PING, ) AsyncSessionLocal = async_sessionmaker( diff --git a/app/main.py b/app/main.py index 9d4d882..ee9cf23 100644 --- a/app/main.py +++ b/app/main.py @@ -12,8 +12,13 @@ PermissionDeniedError, ValidationError, ) +from app.core.logging import configure_logging from app.core.rate_limit import RateLimitMiddleware from app.core.security_headers import SecurityHeadersMiddleware +from app.core.sentry import configure_sentry + +configure_logging() +configure_sentry() def register_exception_handlers(app: FastAPI) -> None: @@ -31,7 +36,7 @@ async def permission_denied_handler(request: Request, exc: PermissionDeniedError @app.exception_handler(ValidationError) async def validation_error_handler(request: Request, exc: ValidationError) -> JSONResponse: - return JSONResponse(status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, content={"detail": str(exc)}) + return JSONResponse(status_code=status.HTTP_422_UNPROCESSABLE_CONTENT, content={"detail": str(exc)}) @app.exception_handler(AuthenticationError) async def authentication_error_handler(request: Request, exc: AuthenticationError) -> JSONResponse: diff --git a/requirements-dev.txt b/requirements-dev.txt index dd0dacf..17283f3 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,7 @@ -r requirements.txt -pytest==8.3.4 -pytest-asyncio==0.25.0 +pytest==9.0.3 +pytest-asyncio==1.4.0 pytest-cov==6.0.0 pytest-timeout==2.4.0 respx==0.22.0 diff --git a/requirements.txt b/requirements.txt index d1a24d4..2f2cb00 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -fastapi==0.115.6 +fastapi==0.136.3 uvicorn[standard]==0.34.0 sqlalchemy==2.0.36 greenlet==3.5.1 @@ -14,3 +14,4 @@ celery==5.4.0 redis==5.2.1 httpx==0.28.1 aiosmtplib==3.0.2 +sentry-sdk[fastapi]==2.62.0 diff --git a/tests/test_logging.py b/tests/test_logging.py new file mode 100644 index 0000000..d93f7d1 --- /dev/null +++ b/tests/test_logging.py @@ -0,0 +1,54 @@ +import json +import logging + +import pytest + +from app.core import logging as app_logging +from app.core.config import settings +from app.core.logging import JsonFormatter, configure_logging + + +def test_json_formatter_renders_expected_fields() -> None: + formatter = JsonFormatter() + record = logging.LogRecord( + name="app.services.cache", + level=logging.WARNING, + pathname=__file__, + lineno=1, + msg="Redis unavailable: %s", + args=("connection refused",), + exc_info=None, + ) + + payload = json.loads(formatter.format(record)) + + assert payload["level"] == "WARNING" + assert payload["logger"] == "app.services.cache" + assert payload["message"] == "Redis unavailable: connection refused" + assert "timestamp" in payload + + +def test_configure_logging_is_idempotent(monkeypatch: pytest.MonkeyPatch) -> None: + root_logger = logging.getLogger() + monkeypatch.setattr(app_logging, "_configured", False) + handlers_before = list(root_logger.handlers) + + try: + configure_logging() + configure_logging() + assert len(root_logger.handlers) == len(handlers_before) + 1 + finally: + root_logger.handlers = handlers_before + + +def test_configure_logging_uses_json_formatter_when_configured(monkeypatch: pytest.MonkeyPatch) -> None: + root_logger = logging.getLogger() + monkeypatch.setattr(app_logging, "_configured", False) + monkeypatch.setattr(settings, "LOG_FORMAT", "json") + handlers_before = list(root_logger.handlers) + + try: + configure_logging() + assert isinstance(root_logger.handlers[-1].formatter, JsonFormatter) + finally: + root_logger.handlers = handlers_before diff --git a/tests/test_sentry.py b/tests/test_sentry.py new file mode 100644 index 0000000..732d8b2 --- /dev/null +++ b/tests/test_sentry.py @@ -0,0 +1,51 @@ +from typing import Any + +import pytest + +from app.core import sentry as app_sentry +from app.core.config import settings + + +def test_configure_sentry_is_noop_without_dsn(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(app_sentry, "_configured", False) + monkeypatch.setattr(settings, "SENTRY_DSN", None) + + calls: list[dict[str, Any]] = [] + monkeypatch.setattr(app_sentry.sentry_sdk, "init", lambda **kwargs: calls.append(kwargs)) + + app_sentry.configure_sentry() + + assert calls == [] + assert app_sentry._configured is False + + +def test_configure_sentry_initializes_when_dsn_set(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(app_sentry, "_configured", False) + monkeypatch.setattr(settings, "SENTRY_DSN", "https://example@sentry.io/1") + monkeypatch.setattr(settings, "SENTRY_TRACES_SAMPLE_RATE", 0.25) + + calls: list[dict[str, Any]] = [] + monkeypatch.setattr(app_sentry.sentry_sdk, "init", lambda **kwargs: calls.append(kwargs)) + + app_sentry.configure_sentry() + + assert calls == [ + { + "dsn": "https://example@sentry.io/1", + "environment": settings.ENVIRONMENT, + "traces_sample_rate": 0.25, + } + ] + assert app_sentry._configured is True + + +def test_configure_sentry_is_idempotent(monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(app_sentry, "_configured", True) + monkeypatch.setattr(settings, "SENTRY_DSN", "https://example@sentry.io/1") + + calls: list[dict[str, Any]] = [] + monkeypatch.setattr(app_sentry.sentry_sdk, "init", lambda **kwargs: calls.append(kwargs)) + + app_sentry.configure_sentry() + + assert calls == [] From 632a47845419ac3b6102cbb5a13b287a0dd9ba5d Mon Sep 17 00:00:00 2001 From: sarangs1621 Date: Sat, 13 Jun 2026 06:48:06 +0530 Subject: [PATCH 2/2] Add Render deployment config and local polling scheduler Adds render.yaml describing the API, worker, beat, and Postgres services for a Render deployment, plus run_local_scheduler.py for running the dispatch tasks without Redis/Celery during local dev. Documents SMTP and production environment variables in .env.example and ignores frontend build artifacts and local celerybeat schedule files. --- .env.example | 20 +++++++++ .gitignore | 8 ++++ render.yaml | 99 ++++++++++++++++++++++++++++++++++++++++++ run_local_scheduler.py | 44 +++++++++++++++++++ 4 files changed, 171 insertions(+) create mode 100644 render.yaml create mode 100644 run_local_scheduler.py diff --git a/.env.example b/.env.example index a39d066..691e996 100644 --- a/.env.example +++ b/.env.example @@ -47,3 +47,23 @@ REDIS_URL=redis://localhost:6379/0 # false when running real worker/beat processes against Redis. CELERY_TASK_ALWAYS_EAGER=true CHECK_DISPATCH_INTERVAL_SECONDS=10 + +# --- SMTP (Phase 5: email notifications) --- +# SMTP_HOST=smtp.example.com +# SMTP_PORT=587 +# SMTP_USERNAME=apikey +# SMTP_PASSWORD=your-smtp-password +# SMTP_USE_TLS=true +# SMTP_FROM_ADDRESS=alerts@yourdomain.com + +# --- Production deployment (Render + Vercel) --- +# For Render: +# DATABASE_URL, REDIS_URL, SECRET_KEY are set automatically by render.yaml +# ENVIRONMENT=production +# LOG_FORMAT=json +# BACKEND_CORS_ORIGINS=["https://your-app.vercel.app"] +# CELERY_TASK_ALWAYS_EAGER=false +# +# For Vercel (frontend): +# NEXT_PUBLIC_API_URL=https://your-api.onrender.com/api/v1 + diff --git a/.gitignore b/.gitignore index 89fa541..e295cff 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,14 @@ htmlcov/ .idea/ .vscode/ +# Frontend (Next.js) +frontend/node_modules/ +frontend/.next/ +frontend/out/ + +# Celery beat schedule (local dev) +celerybeat-schedule* + # OS .DS_Store Thumbs.db diff --git a/render.yaml b/render.yaml new file mode 100644 index 0000000..2fa3928 --- /dev/null +++ b/render.yaml @@ -0,0 +1,99 @@ +services: + - type: web + name: sentinel-api + runtime: python + buildCommand: pip install -r requirements.txt && alembic upgrade head + startCommand: uvicorn app.main:app --host 0.0.0.0 --port $PORT + envVars: + - key: ENVIRONMENT + value: production + - key: LOG_FORMAT + value: json + - key: LOG_LEVEL + value: INFO + - key: SECRET_KEY + generateValue: true + - key: DATABASE_URL + fromDatabase: + name: sentinel-db + property: connectionURI + - key: REDIS_URL + fromService: + type: redis + name: sentinel-redis + property: connectionURI + - key: CELERY_TASK_ALWAYS_EAGER + value: "false" + - key: BACKEND_CORS_ORIGINS + value: '["https://sentinel-dashboard.vercel.app"]' + - key: PYTHON_VERSION + value: "3.12.0" + healthCheckPath: /health + + - type: worker + name: sentinel-worker + runtime: python + buildCommand: pip install -r requirements.txt + startCommand: celery -A app.core.celery_app worker --loglevel=info --concurrency=2 + envVars: + - key: ENVIRONMENT + value: production + - key: LOG_FORMAT + value: json + - key: SECRET_KEY + fromService: + type: web + name: sentinel-api + envVarKey: SECRET_KEY + - key: DATABASE_URL + fromDatabase: + name: sentinel-db + property: connectionURI + - key: REDIS_URL + fromService: + type: redis + name: sentinel-redis + property: connectionURI + - key: CELERY_TASK_ALWAYS_EAGER + value: "false" + - key: PYTHON_VERSION + value: "3.12.0" + + - type: worker + name: sentinel-beat + runtime: python + buildCommand: pip install -r requirements.txt + startCommand: celery -A app.core.celery_app beat --loglevel=info + envVars: + - key: ENVIRONMENT + value: production + - key: LOG_FORMAT + value: json + - key: SECRET_KEY + fromService: + type: web + name: sentinel-api + envVarKey: SECRET_KEY + - key: DATABASE_URL + fromDatabase: + name: sentinel-db + property: connectionURI + - key: REDIS_URL + fromService: + type: redis + name: sentinel-redis + property: connectionURI + - key: CELERY_TASK_ALWAYS_EAGER + value: "false" + - key: PYTHON_VERSION + value: "3.12.0" + +databases: + - name: sentinel-db + plan: free + databaseName: sentinel + user: sentinel + + # Note: Render free Redis is available in some regions. + # If not available, set CELERY_TASK_ALWAYS_EAGER=true on the web service + # and remove the worker/beat services. diff --git a/run_local_scheduler.py b/run_local_scheduler.py new file mode 100644 index 0000000..4242e8b --- /dev/null +++ b/run_local_scheduler.py @@ -0,0 +1,44 @@ +import os +import sys +import time + +sys.path.insert(0, os.path.abspath(os.path.dirname(__file__))) + +from app.core.config import settings +from app.workers.tasks import dispatch_due_checks, dispatch_metric_aggregation, dispatch_pending_notifications + + +def main(): + print("Starting Sentinel Local Scheduler (No Redis required)...") + print("Using eager task execution.") + + last_check = 0 + last_metrics = 0 + + check_interval = settings.CHECK_DISPATCH_INTERVAL_SECONDS + metrics_interval = settings.METRICS_AGGREGATION_INTERVAL_SECONDS + + while True: + now = time.time() + + if now - last_check >= check_interval: + print(f"[{time.strftime('%H:%M:%S')}] Dispatching checks and notifications...") + try: + dispatch_due_checks.delay() + dispatch_pending_notifications.delay() + except Exception as e: + print(f"Error during dispatch: {e}") + last_check = now + + if now - last_metrics >= metrics_interval: + print(f"[{time.strftime('%H:%M:%S')}] Dispatching metrics aggregation...") + try: + dispatch_metric_aggregation.delay() + except Exception as e: + print(f"Error during metrics aggregation: {e}") + last_metrics = now + + time.sleep(1) + +if __name__ == "__main__": + main()