Add JSON logging format support for scan jobs and enforcer

enforcer/enforcer_main.py

@@ -23,6 +23,7 @@
 from fastapi import FastAPI, HTTPException, Response
 from pydantic import BaseModel
 from prometheus_client import generate_latest, CONTENT_TYPE_LATEST
+from pythonjsonlogger.json import JsonFormatter
 
 from enforcer.dal.supabase_dal import SupabaseDal
 from enforcer.patch_manager import patch_container_resources
@@ -35,7 +36,16 @@
 # Configure logging
 logger = logging.getLogger()
 logHandler = logging.StreamHandler(sys.stdout)
-formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
+# When ENABLE_JSON_LOGS_FORMAT is set, emit JSON logs (one object per line) so
+# scrapers like Filebeat can index them; otherwise keep the plain text format.
+if os.environ.get("ENABLE_JSON_LOGS_FORMAT", "false").strip().lower() in ("true", "1", "yes"):
+    formatter: logging.Formatter = JsonFormatter(
+        fmt="%(asctime)s %(levelname)s %(name)s %(filename)s %(lineno)d %(funcName)s %(message)s",
+        datefmt="%Y-%m-%dT%H:%M:%S",
+        rename_fields={"levelname": "severity"},
+    )
+else:
+    formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s")
 logHandler.setFormatter(formatter)
 logger.addHandler(logHandler)
 logger.setLevel(os.environ.get("LOG_LEVEL", "INFO"))

enforcer/requirements.txt

@@ -5,6 +5,7 @@ supabase==2.5
 PyYAML==6.0.1
 cachetools==5.3.3
 prometheus-client==0.20.0
+python-json-logger==3.3.0
 kubernetes==26.1.0
 pyasn1>=0.6.2
 urllib3==2.7.0

helm/krr-enforcer/templates/enforcer.yaml

@@ -118,6 +118,8 @@ spec:
               value: "/etc/webhook/certs/tls.crt"
             - name: LOG_LEVEL
               value: {{ .Values.logLevel | quote }}
+            - name: ENABLE_JSON_LOGS_FORMAT
+              value: {{ .Values.global.enableJsonLogsFormat | default .Values.enableJsonLogsFormat | default false | quote }}
             {{- if .Values.certificate }}
             - name: CERTIFICATE
               value: {{ .Values.certificate | quote }}

helm/krr-enforcer/values.yaml

@@ -1,5 +1,12 @@
 certificate: "" # base64 encoded
 logLevel: INFO
+# When true, logs are emitted as JSON (one object per line) instead of plain
+# text. Useful for log scrapers like Filebeat. Defaults to false.
+enableJsonLogsFormat: false
+
+# Values shared with/from a parent umbrella chart. Declared so templates can
+# safely reference .Values.global.* when the chart is installed standalone.
+global: {}
 
 # Certificate job configuration
 certJob:

pyproject.toml

@@ -42,6 +42,7 @@ urllib3 = "^2.7.0"
 setuptools = "^80.9.0"
 zipp = "^3.19.1"
 tenacity = "^9.0.0"
+python-json-logger = "^3.0.0"
 
 
 [tool.poetry.group.dev.dependencies]

requirements.txt

@@ -33,6 +33,7 @@ pydantic==1.10.15 ; python_version >= "3.10" and python_full_version < "3.13"
 pygments==2.20.0 ; python_version >= "3.10" and python_full_version < "3.13"
 pyparsing==3.1.2 ; python_version >= "3.10" and python_full_version < "3.13"
 python-dateutil==2.9.0.post0 ; python_version >= "3.10" and python_full_version < "3.13"
+python-json-logger==3.3.0 ; python_version >= "3.10" and python_full_version < "3.13"
 pytz==2024.1 ; python_version >= "3.10" and python_full_version < "3.13"
 pyyaml==6.0.1 ; python_version >= "3.10" and python_full_version < "3.13"
 regex==2023.12.25 ; python_version >= "3.10" and python_full_version < "3.13"

robusta_krr/core/models/config.py

@@ -1,12 +1,14 @@
 from __future__ import annotations
 
 import logging
+import os
 import sys
 from typing import Any, Literal, Optional, Union
 
 import pydantic as pd
 from kubernetes import config
 from kubernetes.config.config_exception import ConfigException
+from pythonjsonlogger.json import JsonFormatter
 from rich.console import Console
 from rich.logging import RichHandler
 
@@ -203,12 +205,26 @@ def set_config(config: Config) -> None:
         global _config
 
         _config = config
-        logging.basicConfig(
-            level="NOTSET",
-            format="%(message)s",
-            datefmt="[%X]",
-            handlers=[RichHandler(console=config.logging_console)],
-        )
+        # ENABLE_JSON_LOGS_FORMAT is env-driven (never a CLI flag) so the
+        # interactive CLI keeps its Rich output, while in-cluster scan jobs can
+        # emit JSON logs (one object per line) for scrapers like Filebeat.
+        if os.environ.get("ENABLE_JSON_LOGS_FORMAT", "false").strip().lower() in ("true", "1", "yes"):
+            handler = logging.StreamHandler(config.logging_console.file)
+            handler.setFormatter(
+                JsonFormatter(
+                    fmt="%(asctime)s %(levelname)s %(name)s %(filename)s %(lineno)d %(funcName)s %(message)s",
+                    datefmt="%Y-%m-%dT%H:%M:%S",
+                    rename_fields={"levelname": "severity"},
+                )
+            )
+            logging.basicConfig(level="NOTSET", handlers=[handler], force=True)
+        else:
+            logging.basicConfig(
+                level="NOTSET",
+                format="%(message)s",
+                datefmt="[%X]",
+                handlers=[RichHandler(console=config.logging_console)],
+            )
         logging.getLogger("").setLevel(logging.CRITICAL)
         logger.setLevel(logging.DEBUG if config.verbose else logging.CRITICAL if config.quiet else logging.INFO)