Allow setting a custom docker volume directory

src/config.rs

@@ -20,6 +20,8 @@ pub struct Config {
     /// Rusthead Docker image to use (defaults to "samply/rusthead:latest")
     #[serde(default = "default_image")]
     pub image: String,
+    /// Defaults to docker named volumes
+    pub volume_dir: Option<PathBuf>,
     pub git_sync: Option<bool>,
     pub https_proxy_url: Option<Url>,
     pub ccp: Option<CcpConfig>,

src/services/blaze.rs

@@ -3,6 +3,8 @@ use std::{marker::PhantomData, str::FromStr};
 use askama::Template;
 use url::Url;
 
+use crate::utils::filters;
+
 use super::{Service, Traefik};
 
 #[derive(Debug, Template)]

src/services/dnpm_node.rs

@@ -4,6 +4,7 @@ use serde::Deserialize;
 use crate::{
     config::Config,
     services::{Service, Traefik},
+    utils::filters,
 };
 
 #[derive(Debug, Clone, Deserialize)]

src/services/exporter.rs

@@ -9,6 +9,7 @@ use crate::{
         datashield::DataShield,
         postgres::{PgConnectInfo, Postgres},
     },
+    utils::filters,
 };
 
 #[derive(Debug, Template)]

src/services/id_management.rs

@@ -4,7 +4,7 @@ use crate::{
     config::Config,
     modules::CcpDefault,
     services::{OidcClient, OidcProvider, PrivateOidcClient, postgres::PgConnectInfo},
-    utils::capitalize_first_letter,
+    utils::{capitalize_first_letter, filters},
 };
 
 use super::{ForwardProxy, Service, ToCompose, Traefik, postgres::Postgres};

src/services/mod.rs

@@ -183,14 +183,15 @@ service_tuple!(T1, T2, T3);
 service_tuple!(T1, T2, T3, T4);
 
 pub trait ToCompose: Any {
-    fn render(&self) -> anyhow::Result<String>;
+    fn render(&self, config: &'static Config) -> anyhow::Result<String>;
 
     fn service_name(&self) -> String;
 }
 
 impl<T: Template + Service> ToCompose for T {
-    fn render(&self) -> anyhow::Result<String> {
-        Template::render(self)
+    fn render(&self, config: &'static Config) -> anyhow::Result<String> {
+        let values = [("config", config as &dyn Any)];
+        Template::render_with_values(self, &values as &dyn askama::Values)
             .with_context(|| format!("Failed to render {}", std::any::type_name::<T>()))
     }
 
@@ -356,7 +357,7 @@ impl ServiceMap {
             eprintln!("Generating service {service_name}");
             fs::write(
                 services_dir.join(format!("{}.yml", service.service_name())),
-                service.render()?,
+                service.render(self.config)?,
             )?;
         }
         Ok(())

src/services/postgres.rs

@@ -2,6 +2,8 @@ use std::marker::PhantomData;
 
 use askama::Template;
 
+use crate::utils::filters;
+
 use super::Service;
 
 #[derive(Debug, Template)]

src/services/transfair.rs

@@ -4,6 +4,8 @@ use askama::Template;
 use serde::Deserialize;
 use url::Url;
 
+use crate::utils::filters;
+
 use super::{Blaze, BlazeProvider, Service};
 
 #[derive(Debug, Deserialize)]

src/utils.rs

@@ -26,10 +26,12 @@ pub fn secret_from_rng<const N: usize>(rng: &mut impl rand::Rng) -> String {
 }
 
 pub mod filters {
-    use std::path::PathBuf;
+    use std::{fs, path::PathBuf};
 
     use askama::Values;
 
+    use crate::config::Config;
+
     #[askama::filter_fn]
     pub fn path(p: &PathBuf, _: &dyn Values) -> askama::Result<String> {
         Ok(p.canonicalize()
@@ -41,6 +43,36 @@ pub mod filters {
             .display()
             .to_string())
     }
+
+    #[askama::filter_fn]
+    pub fn make_volume(name: &str, ctx: &dyn Values) -> askama::Result<String> {
+        let config = ctx
+            .get_value("config")
+            .unwrap()
+            .downcast_ref::<Config>()
+            .unwrap();
+        if let Some(volume_dir) = &config.volume_dir {
+            let path = config.path.join(&volume_dir);
+            fs::create_dir_all(&path).map_err(|e| {
+                askama::Error::custom(
+                    anyhow::Error::from(e)
+                        .context(format!("Failed to create volume directory {volume_dir:?}")),
+                )
+            })?;
+            let abs_path = path.canonicalize().map_err(|e| {
+                askama::Error::custom(
+                    anyhow::Error::from(e)
+                        .context(format!("Failed to canonicalize volume {volume_dir:?}")),
+                )
+            })?;
+            Ok(format!(
+                "{name}:\n    driver: local\n    driver_opts:\n      o: bind\n      type: none\n      device: {}",
+                abs_path.display()
+            ))
+        } else {
+            Ok(format!("{name}:"))
+        }
+    }
 }
 
 pub mod host {

templates/blaze.yml

@@ -1,12 +1,13 @@
 {% let name = Self::service_name() %}
+{% let data_volume = format!("{}-data", name) %}
 services:
   {{ name }}:
     image: docker.verbis.dkfz.de/cache/samply/blaze:latest
     environment:
       BASE_URL: {{ Self::get_url().to_string().trim_end_matches('/') }}
       ENFORCE_REFERENTIAL_INTEGRITY: "false"
     volumes:
-      - "{{ name }}-data:/app/data"
+      - "{{ data_volume }}:/app/data"
     {% if let Some(traefik_conf) = traefik_conf -%}
     labels:
       - "traefik.enable=true"
@@ -18,4 +19,4 @@ services:
     {%- endif %}
 
 volumes:
-  {{ name }}-data:
+  {{ data_volume|make_volume }}

templates/datashield.yml

@@ -1,4 +1,5 @@
 {% let project = T::network_name() %}
+{% let opal_metadata_volume = format!("{}-opal-metadata-db", project) %}
 
 services:
   {{ opal_host() }}:
@@ -32,7 +33,7 @@ services:
       BEAM_SECRET: {{ tm_beam.secret }}
       BEAM_DATASHIELD_PROXY: request-manager
     volumes:
-      - "{{ project }}-opal-metadata-db:/srv" # Opal metadata
+      - "{{ opal_metadata_volume }}:/srv" # Opal metadata
       - "{{ opal_key_path|path }}:/certs/opal-key.pem"
       - "{{ opal_cert_path|path }}:/certs/opal-cert.pem"
     depends_on:
@@ -44,4 +45,4 @@ services:
       - /srv
 
 volumes:
-  {{ project }}-opal-metadata-db:
+  {{ opal_metadata_volume|make_volume }}

templates/dnpm_node.yml

@@ -1,4 +1,8 @@
 
+{%- let mysql_volume = "dnpm-mysql" -%}
+{%- let authup_volume = "dnpm-authup" -%}
+{%- let backend_data_volume = "dnpm-backend-data" -%}
+
 services:
   dnpm-mysql:
     image: mysql:9
@@ -11,12 +15,12 @@ services:
       MYSQL_ROOT_HOST: "%"
       MYSQL_ROOT_PASSWORD: {{ mysql_root_password }}
     volumes:
-      - dnpm-mysql:/var/lib/mysql
+      - {{ mysql_volume }}:/var/lib/mysql
 
   dnpm-authup:
     image: authup/authup:latest
     volumes:
-      - dnpm-authup:/usr/src/app/writable
+      - {{ authup_volume }}:/usr/src/app/writable
     depends_on:
       dnpm-mysql:
         condition: service_healthy
@@ -64,7 +68,7 @@ services:
       - AUTHUP_URL=robot://system:{{ authup_secret }}@http://dnpm-authup:3000
       - TZ=Europe/Berlin
     volumes:
-      - dnpm-backend-data:/dnpm_data
+      - {{ backend_data_volume }}:/dnpm_data
     configs:
       - source: dnpm-production.conf
         target: /dnpm_config/production.conf
@@ -98,9 +102,9 @@ services:
       - "traefik.http.middlewares.dnpm-backend-peer.ipWhiteList.sourceRange=0.0.0.0/32"
 
 volumes:
-  dnpm-authup:
-  dnpm-mysql:
-  dnpm-backend-data:
+  {{ authup_volume|make_volume }}
+  {{ mysql_volume|make_volume }}
+  {{ backend_data_volume|make_volume }}
 
 configs:
   dnpm-production.conf:

templates/exporter.yml

@@ -1,3 +1,6 @@
+{%- let project = project -%}
+{%- let exporter_files_volume = format!("{}-files", Self::service_name()) -%}
+{%- let reporter_files_volume = format!("{}-reporter-files", project) -%}
 services:
   {{ Self::service_name() }}:
     image: docker.verbis.dkfz.de/{{ project }}/dktk-exporter:latest
@@ -23,7 +26,7 @@ services:
       - "traefik.http.middlewares.exporter_{{ project }}_strip.stripprefix.prefixes=/{{ project }}-exporter"
       - "traefik.http.routers.exporter_{{ project }}.middlewares=exporter_{{ project }}_strip"
     volumes:
-      - "{{ Self::service_name() }}-files:/app/exporter-files/output"
+      - "{{ exporter_files_volume }}:/app/exporter-files/output"
     depends_on:
       - {{ db.host }}
 
@@ -39,7 +42,7 @@ services:
       LOG_FHIR_VALIDATION: "false"
       HTTP_SERVLET_REQUEST_SCHEME: "https"
     volumes:
-      - "{{ project }}-reporter-files:/app/reports"
+      - "{{ reporter_files_volume }}:/app/reports"
     labels:
       - "traefik.enable=true"
       - "traefik.http.routers.reporter_{{ project }}.rule=PathPrefix(`/{{ project }}-reporter`)"
@@ -50,5 +53,5 @@ services:
 
 
 volumes:
-  {{ Self::service_name() }}-files:
-  {{ project }}-reporter-files:
+  {{ exporter_files_volume|make_volume }}
+  {{ reporter_files_volume|make_volume }}

templates/id_management.yml

@@ -1,3 +1,4 @@
+{% let patientlist_data_volume = "patientlist-db-data" %}
 
 services:
   id-manager:
@@ -93,4 +94,4 @@ services:
         condition: service_healthy
 
 volumes:
-  patientlist-db-data:
+  {{ patientlist_data_volume|make_volume }}

templates/postgres.yml

@@ -1,3 +1,4 @@
+{% let data_volume = Self::service_name() %}
 services:
   {{ Self::service_name() }}:
     image: docker.verbis.dkfz.de/cache/postgres:15.6-alpine
@@ -6,7 +7,7 @@ services:
       POSTGRES_DB: {{ realm }}
       POSTGRES_PASSWORD: {{ password }}
     volumes:
-      - "{{ Self::service_name() }}:/var/lib/postgresql/data"
+      - "{{ data_volume }}:/var/lib/postgresql/data"
 
 volumes:
-  {{ Self::service_name() }}:
+  {{ data_volume|make_volume }}

templates/transfair.yml

@@ -1,3 +1,6 @@
+{%- let data_volume = "transfair-data" -%}
+{%- let input_blaze_data_volume = "transfair-input-blaze-data" -%}
+{%- let request_blaze_data_volume = "transfair-request-blaze-data" -%}
 services:
   transfair:
     image: docker.verbis.dkfz.de/cache/samply/transfair:latest
@@ -37,7 +40,7 @@ services:
       - TLS_CA_CERTIFICATES_DIR=/conf/trusted-ca-certs
       - TLS_DISABLE={{ conf.tls_disable }}
     volumes:
-      - transfair-data:/transfair
+      - {{ data_volume }}:/transfair
       - {{ trusted_ca_certs.display() }}:/conf/trusted-ca-certs:ro
 
 {%- if conf.fhir_input.is_none() %}
@@ -51,7 +54,7 @@ services:
       CQL_EXPR_CACHE_SIZE: 8
       ENFORCE_REFERENTIAL_INTEGRITY: "false"
     volumes:
-      - "transfair-input-blaze-data:/app/data"
+      - "{{ input_blaze_data_volume }}:/app/data"
 {%- endif %}
 
 {%- if conf.fhir_requests.is_none() %}
@@ -65,14 +68,10 @@ services:
       CQL_EXPR_CACHE_SIZE: 8
       ENFORCE_REFERENTIAL_INTEGRITY: "false"
     volumes:
-      - "transfair-request-blaze-data:/app/data"
+      - "{{ request_blaze_data_volume }}:/app/data"
 {%- endif %}
 
 volumes:
-{%- if conf.fhir_input.is_none() %}
-  transfair-input-blaze-data:
-{%- endif %}
-{%- if conf.fhir_requests.is_none() %}
-  transfair-request-blaze-data:
-{%- endif %}
-  transfair-data:
+  {% if conf.fhir_input.is_none() %}{{ input_blaze_data_volume|make_volume }}
+  {% endif %}{% if conf.fhir_requests.is_none() %}{{ request_blaze_data_volume|make_volume }}
+  {% endif %}{{ data_volume|make_volume }}

tests/configs/volume-dir.toml

@@ -0,0 +1,6 @@
+site_id = "dummy"
+hostname = "dummy.local"
+volume_dir = "./data"
+
+[ccp]
+exporter = {}

tests/snapshots/example.config/configs@services__dnpm-node.yml.snap

@@ -4,7 +4,6 @@ expression: file
 info: services/dnpm-node.yml
 input_file: tests/configs/example.config.toml
 ---
-
 services:
   dnpm-mysql:
     image: mysql:9

tests/snapshots/volume-dir/configs@.env.snap

@@ -0,0 +1,11 @@
+---
+source: src/config.rs
+expression: file
+info: ".env"
+input_file: tests/configs/volume-dir.toml
+---
+# This file is auto generated please modify config.toml or config.local.toml instead!
+
+CCP_BEAM_PROXY_FOCUS_KEY="LP~Kg3u^X#"
+CCP_EXPORTER_API_KEY="0RV9))l@tt"
+CCP_EXPORTER_DB_PASSWORD="AbkbbWY1mA"

tests/snapshots/volume-dir/configs@.gitignore.snap

@@ -0,0 +1,13 @@
+---
+source: src/config.rs
+expression: file
+info: ".gitignore"
+input_file: tests/configs/volume-dir.toml
+---
+
+.env
+config.local.toml
+/pki
+/trusted-ca-certs
+/traefik-tls
+docker-compose.override.yml