From 267bb6cc5be123b1bed5c92dc3414a58058277b3 Mon Sep 17 00:00:00 2001 From: Sagar Shinde Date: Tue, 7 Jul 2026 12:32:50 +0530 Subject: [PATCH] feat: add FHIR Ingestor CronJob for Sri Lanka production - Create Helm chart for fhir-ingestor - Add CronJob template for scheduled reconciliation - Add Sri Lanka production values (daily 2 AM schedule) - Register fhir-ingestor in Sri Lanka ArgoCD apps.yaml --- .../argocd-apps/apps.yaml | 29 ++++ .../values/fhir-ingestor.yaml | 43 ++++++ k8s/manifests/fhir-ingestor/Chart.yaml | 6 + .../fhir-ingestor/templates/cronjob.yaml | 132 ++++++++++++++++++ k8s/manifests/fhir-ingestor/values.yaml | 68 +++++++++ 5 files changed, 278 insertions(+) create mode 100644 k8s/environments/sri-lanka-production/values/fhir-ingestor.yaml create mode 100644 k8s/manifests/fhir-ingestor/Chart.yaml create mode 100644 k8s/manifests/fhir-ingestor/templates/cronjob.yaml create mode 100644 k8s/manifests/fhir-ingestor/values.yaml diff --git a/k8s/environments/sri-lanka-production/argocd-apps/apps.yaml b/k8s/environments/sri-lanka-production/argocd-apps/apps.yaml index 294f7383..744bb7df 100644 --- a/k8s/environments/sri-lanka-production/argocd-apps/apps.yaml +++ b/k8s/environments/sri-lanka-production/argocd-apps/apps.yaml @@ -400,6 +400,35 @@ spec: - values.yaml - ../../environments/sri-lanka-production/values/patient-scoring.yaml project: default + syncPolicy: + syncOptions: + - PruneLast=true + - CreateNamespace=true + automated: + prune: true + selfHeal: true + +--- + +# FHIR Ingestor - Reconciles FHIR bundle data with Simple database (runs daily) +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: fhir-ingestor +spec: + destination: + name: '' + namespace: simple-v1 + server: 'https://kubernetes.default.svc' + source: + path: k8s/manifests/fhir-ingestor + repoURL: 'https://github.com/simpledotorg/container-deployment.git' + targetRevision: master + helm: + valueFiles: + - values.yaml + - ../../environments/sri-lanka-production/values/fhir-ingestor.yaml + project: default syncPolicy: syncOptions: - PruneLast=true diff --git a/k8s/environments/sri-lanka-production/values/fhir-ingestor.yaml b/k8s/environments/sri-lanka-production/values/fhir-ingestor.yaml new file mode 100644 index 00000000..20cc1b60 --- /dev/null +++ b/k8s/environments/sri-lanka-production/values/fhir-ingestor.yaml @@ -0,0 +1,43 @@ +# FHIR Ingestor configuration for Sri Lanka Production + +image: + repository: simpledotorg/fhir-ingestor + tag: latest + pullPolicy: Always + +# Cron schedule: Daily at 2 AM Sri Lanka time (UTC+5:30) +# 2 AM IST = 8:30 PM UTC previous day +schedule: "30 20 * * *" + +# Operation mode +operation: merger +softDelete: true +dryRun: false + +# FHIR data source - adjust based on how you provide FHIR data +fhir: + localPath: "/data/fhir" + # If using S3: + # s3Bucket: "your-bucket-name" + # s3Key: "fhir-bundles/" + +# Database configuration (matches Simple server database) +database: + secretName: "simple-pguser-simple" + host: "simple-primary" + port: "5432" + name: "simple_db_sri_lanka_production_001" + +# Pushgateway for metrics +pushgateway: + enabled: true + namespace: "monitoring" + +# Resources +resources: + requests: + memory: "512Mi" + cpu: "200m" + limits: + memory: "2Gi" + cpu: "1000m" diff --git a/k8s/manifests/fhir-ingestor/Chart.yaml b/k8s/manifests/fhir-ingestor/Chart.yaml new file mode 100644 index 00000000..27e15b53 --- /dev/null +++ b/k8s/manifests/fhir-ingestor/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: fhir-ingestor +description: FHIR to Simple database reconciliation and merge tool +type: application +version: 0.1.0 +appVersion: "1.0.0" diff --git a/k8s/manifests/fhir-ingestor/templates/cronjob.yaml b/k8s/manifests/fhir-ingestor/templates/cronjob.yaml new file mode 100644 index 00000000..5f7fe210 --- /dev/null +++ b/k8s/manifests/fhir-ingestor/templates/cronjob.yaml @@ -0,0 +1,132 @@ +apiVersion: batch/v1 +kind: CronJob +metadata: + name: fhir-ingestor-cronjob + labels: + app.kubernetes.io/name: fhir-ingestor +spec: + schedule: "{{ .Values.schedule }}" + concurrencyPolicy: Forbid + successfulJobsHistoryLimit: 3 + failedJobsHistoryLimit: 3 + jobTemplate: + spec: + backoffLimit: 2 + template: + metadata: + labels: + app.kubernetes.io/name: fhir-ingestor + app.kubernetes.io/component: cronjob + spec: + {{- with .Values.podSecurityContext }} + securityContext: + {{- toYaml . | nindent 12 }} + {{- end }} + restartPolicy: OnFailure + containers: + - name: fhir-ingestor + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - /bin/bash + - -c + - | + set -e + + echo "============================================" + echo "FHIR Ingestor CronJob Started" + echo "============================================" + echo "Operation: {{ .Values.operation }}" + echo "Database Host: ${DB_HOST}:${DB_PORT}" + echo "Database Name: ${DB_NAME}" + {{- if .Values.softDelete }} + echo "Soft Delete: enabled" + {{- end }} + {{- if .Values.dryRun }} + echo "Dry Run: enabled" + {{- end }} + echo "============================================" + + # Construct DATABASE_URL from environment variables + export SIMPLE_DATABASE_URL="postgres://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:${DB_PORT}/${DB_NAME}" + + cd /app + + {{- if eq .Values.operation "merger" }} + # Run merger + echo "Running FHIR merger..." + python -m app.merger {{ .Values.fhir.localPath }} \ + {{- if .Values.softDelete }} + --soft-delete \ + {{- end }} + {{- if .Values.dryRun }} + --dry-run \ + {{- end }} + + {{- else if eq .Values.operation "gap_analysis" }} + # Run gap analysis + echo "Running FHIR gap analysis..." + python -m app.gap_analysis {{ .Values.fhir.localPath }} --json + {{- end }} + + echo "============================================" + echo "FHIR Ingestor CronJob Completed!" + echo "============================================" + env: + {{- if .Values.pushgateway.enabled }} + - name: PUSHGATEWAY_URL + value: "http://prometheus-pushgateway.{{ .Values.pushgateway.namespace }}.svc.cluster.local:9091" + {{- end }} + - name: DB_USER + valueFrom: + secretKeyRef: + name: {{ .Values.database.secretName }} + key: user + - name: DB_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.database.secretName }} + key: password + - name: DB_HOST + value: "{{ .Values.database.host }}" + - name: DB_PORT + value: "{{ .Values.database.port }}" + - name: DB_NAME + value: "{{ .Values.database.name }}" + - name: LOG_LEVEL + value: "INFO" + {{- if .Values.fhir.s3Bucket }} + - name: S3_BUCKET + value: "{{ .Values.fhir.s3Bucket }}" + - name: S3_KEY + value: "{{ .Values.fhir.s3Key }}" + {{- end }} + {{- with .Values.extraEnv }} + {{- toYaml . | nindent 12 }} + {{- end }} + resources: + {{- toYaml .Values.resources | nindent 14 }} + {{- if .Values.fhir.localPath }} + volumeMounts: + - name: fhir-data + mountPath: {{ .Values.fhir.localPath }} + readOnly: true + {{- end }} + {{- if .Values.fhir.localPath }} + volumes: + - name: fhir-data + persistentVolumeClaim: + claimName: fhir-data-pvc + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 12 }} + {{- end }} diff --git a/k8s/manifests/fhir-ingestor/values.yaml b/k8s/manifests/fhir-ingestor/values.yaml new file mode 100644 index 00000000..f16b585d --- /dev/null +++ b/k8s/manifests/fhir-ingestor/values.yaml @@ -0,0 +1,68 @@ +image: + repository: simpledotorg/fhir-ingestor + pullPolicy: Always + tag: latest + +# Cron schedule: Daily at 2 AM UTC +# Format: minute hour day-of-month month day-of-week +schedule: "0 2 * * *" + +# Operation mode: "merger" or "gap_analysis" +operation: merger + +# Enable soft delete (only for merger operation) +softDelete: true + +# Dry run mode (no actual DB writes) +dryRun: false + +# FHIR data source +fhir: + # S3 bucket containing FHIR bundles + s3Bucket: "" + # S3 key/path to FHIR data (can be zip file or prefix) + s3Key: "" + # Or use a local/mounted path + localPath: "/data/fhir" + +# Database configuration +database: + # Postgres Operator creates this secret automatically + secretName: "simple-pguser-simple" + # Use direct connection to primary (not pgpool) for data ingestion + host: "simple-primary" + port: "5432" + name: "simple" + +# AWS credentials for S3 access (if using S3) +aws: + # Secret containing AWS credentials + secretName: "" + # Or use IAM role (recommended in EKS) + useIAMRole: true + +# Pushgateway configuration for metrics +pushgateway: + enabled: true + namespace: "monitoring" + +# Resource limits +resources: + requests: + memory: "256Mi" + cpu: "100m" + limits: + memory: "1Gi" + cpu: "500m" + +# Node selection +nodeSelector: {} +affinity: {} +tolerations: [] + +# Pod security context +podSecurityContext: + fsGroup: 9999 + +# Additional environment variables +extraEnv: []