feat(pre-commit): adds pre-commit linting, yaml-compliance checking, …

.github/workflows/_tox.yml

@@ -6,7 +6,6 @@ on:
         description: What to run under tox
         required: true
 
-
 jobs:
   run:
     runs-on: "ubuntu-latest"
@@ -18,5 +17,12 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v7
 
+      - name: Install Argo
+        run: |
+          curl -sLO https://github.com/argoproj/argo-workflows/releases/download/v4.0.6/argo-linux-amd64.gz
+          gunzip argo-linux-amd64.gz
+          chmod +x argo-linux-amd64
+          sudo mv argo-linux-amd64 /usr/local/bin/argo
+
       - name: Run tox
         run: uv run --locked tox -e ${{ inputs.tox }}

.gitignore

@@ -69,3 +69,6 @@ lockfiles/
 
 # ruff cache
 .ruff_cache/
+
+# environment variables file
+*.env

.pre-commit-config.yaml

@@ -17,6 +17,30 @@ repos:
         types: [python]
         require_serial: true
 
+      - id: create-workflow-templates
+        name: creates workflow templates
+        language: system
+        entry: bash scripts/runthefiles.sh
+        types: [python]
+        pass_filenames: true
+        require_serial: true
+
+      - id: confirm-diamond-compatability
+        name: ensure yaml is diamond-compatible
+        language: system
+        entry: python scripts/checkyamlcompliance.py
+        types: [yaml]
+        pass_filenames: true
+        require_serial: true
+
+      - id: lint-yaml-files
+        name: run argo lint on the yaml files
+        language: system
+        entry: bash scripts/lintyaml.sh
+        types: [yaml]
+        pass_filenames: true
+        require_serial: true
+
       - id: ruff-format
         name: format with ruff
         language: system

Dockerfile

@@ -11,6 +11,11 @@ RUN gunzip "argo-linux-amd64.gz"
 RUN chmod +x "argo-linux-amd64"
 RUN mv "./argo-linux-amd64" /usr/local/bin/argo
 
+#Install Kubectl
+RUN curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl"
+RUN chmod +x ./kubectl
+RUN mv ./kubectl /usr/local/bin
+
 # The build stage installs the context into the venv
 FROM developer AS build
 

divisionyaml.yaml

@@ -0,0 +1,66 @@
+apiVersion: argoproj.io/v1alpha1
+kind: ClusterWorkflowTemplate
+metadata:
+  generateName: hera-division-
+  annotations:
+    workflows.argoproj.io/description: |-
+      Takes a numerical input and returns the
+               remainder, output float, and output string to a json file
+    workflows.argoproj.io/title: Division via hera test
+    workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/python-interface-to-workflows
+  labels:
+    workflows.diamond.ac.uk/science-group-examples: 'true'
+spec:
+  entrypoint: divide
+  templates:
+  - name: divide
+    steps:
+    - - name: first
+        template: do-devision
+        arguments:
+          parameters:
+          - name: a
+            value: '2'
+          - name: b
+            value: '5'
+  - name: do-devision
+    inputs:
+      parameters:
+      - name: a
+      - name: b
+    outputs:
+      artifacts:
+      - name: json-output
+        path: /output-dir/output.json
+    script:
+      image: None
+      source: |-
+        import os
+        import sys
+        sys.path.append(os.getcwd())
+        import json
+        try: a = json.loads(r'''{{inputs.parameters.a}}''')
+        except: a = r'''{{inputs.parameters.a}}'''
+        try: b = json.loads(r'''{{inputs.parameters.b}}''')
+        except: b = r'''{{inputs.parameters.b}}'''
+
+        div = a / b
+        intdiv = a // b
+        remain = a % b
+        dictionary_of_results = {'divide': div, 'integer divisor': intdiv, 'remainder': remain}
+        with open('/output-dir/output.json', 'w') as otpt:
+            json.dump(dictionary_of_results, otpt)
+      command:
+      - python
+      volumeMounts:
+      - name: output-dir
+        mountPath: /output-dir/
+  volumeClaimTemplates:
+  - metadata:
+      name: output-dir
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 1Mi

pyproject.toml

@@ -16,6 +16,8 @@ description = " Python alternative to creating and running argo workflows in the
 dependencies = [
     "gql",
     "aiohttp",
+    "hera",
+    "requests",
 ] # Add project dependencies here, e.g. ["click", "numpy"]
 dynamic = ["version"]
 license.file = "LICENSE"

scripts/checkyamlcompliance.py

@@ -0,0 +1,53 @@
+#!/bin/python
+import os
+import re
+
+yamlpath = "src/python_interface_to_workflows/templates/"
+yamllist: list[str] = []
+for x in os.listdir(yamlpath):
+    if x.endswith(".yaml"):
+        yamllist.append(x)
+
+for file in yamllist:
+    z = 0
+    with open(yamlpath + file) as s:
+        filelist = s.readlines()
+        cleanfilenames: list[str] = []
+        for line in filelist:
+            line = line.rstrip()
+            cleanfilenames.append(line)
+    api_ver = any(
+        re.search(r"apiVersion:\sargoproj\.io\/v1alpha1", line)
+        for line in cleanfilenames
+    )
+    # replace regex with r"kind:\sClusterWorkflowTemplate" on graphql
+    clst_tmpt = any(re.search(r"kind:\sWorkflow", line) for line in cleanfilenames)
+    title = any(
+        re.search(r"\s\s\s\sworkflows.argoproj.io/title:\s.+", line)
+        for line in cleanfilenames
+    )
+    repo = any(
+        re.search(r"\s\s\s\sworkflows.diamond.ac.uk/repository:\s.+", line)
+        for line in cleanfilenames
+    )
+    group = any(
+        re.search(r"\s\s\s\sworkflows.diamond.ac.uk/science-group-.+:\s.+", line)
+        for line in cleanfilenames
+    )
+    gen_name = any(re.search(r"\s\sgenerateName:\s.+", line) for line in cleanfilenames)
+    normal_name = any(re.search(r"^\s\sname:\s.+", line) for line in cleanfilenames)
+    if api_ver and clst_tmpt and title and repo and group:
+        if gen_name != normal_name:
+            exit(0)
+        else:
+            print(
+                "generated_name and normal_name error, must have one of these not both"
+            )
+            exit(1)
+    else:
+        print(f"""api_ver present?: {api_ver}
+                cluster template present?: {clst_tmpt}
+                title present?: {title}
+                repository present?: {repo}
+                group present?: {group}""")
+        exit(1)

scripts/lintyaml.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+cd src/python_interface_to_workflows/templates
+for file in *
+do
+argo lint "$file" --offline
+SUCCESSFULLINT=$?
+[ $SUCCESSFULLINT -ne 0 ] && exit 1
+done
+exit 0

scripts/runthefiles.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+cd src/python_interface_to_workflows/workflow_definitions
+for file in *
+do
+uv run "$file"
+done
+mv *.yaml ../templates/
+git add -u ../templates/

src/.env

@@ -0,0 +1,4 @@
+"HOST" = "https://argo-workflows.staging.workflows.diamond.ac.uk/"
+"IMAGE" = "python:3.10"
+"TOKEN" = "Token"
+"NAMESPACE" = "ks10000-3"

src/python_interface_to_workflows/submit_to_graphql.py

@@ -20,10 +20,10 @@ def submit_to_graphql():
       proposalNumber: 10000,
       number: 3
     }
-    parameters: {
-      numinput: "19",
-      numdivisor: "10"
-    }
+     parameters: {
+       numinput: "19",
+       numdivisor: "10"
+     }
     ){
     name
 }

src/python_interface_to_workflows/templates/divisionyaml.yaml

@@ -0,0 +1,60 @@
+apiVersion: argoproj.io/v1alpha1
+kind: Workflow
+metadata:
+  generateName: hera-division-
+  annotations:
+    workflows.argoproj.io/description: |-
+      Takes a numerical input and returns the
+               remainder, output float, and output string to a json file
+    workflows.argoproj.io/title: Division via hera test
+    workflows.diamond.ac.uk/repository: https://github.com/DiamondLightSource/python-interface-to-workflows
+  labels:
+    workflows.diamond.ac.uk/science-group-examples: 'true'
+spec:
+  entrypoint: divide
+  templates:
+  - name: divide
+    steps:
+    - - name: first
+        template: do-division
+        arguments:
+          parameters:
+          - name: a
+            value: '2'
+          - name: b
+            value: '5'
+  - name: do-division
+    inputs:
+      parameters:
+      - name: a
+      - name: b
+    outputs:
+      artifacts:
+      - name: json-output
+        path: /output-dir/output.json
+    script:
+      image: None
+      source: |-
+        import os
+        import sys
+        sys.path.append(os.getcwd())
+        import json
+        try: a = json.loads(r'''{{inputs.parameters.a}}''')
+        except: a = r'''{{inputs.parameters.a}}'''
+        try: b = json.loads(r'''{{inputs.parameters.b}}''')
+        except: b = r'''{{inputs.parameters.b}}'''
+
+        div = a / b
+        intdiv = a // b
+        remain = a % b
+        dictionary_of_results = {'divide': div, 'quotient': intdiv, 'remainder': remain}
+        with open('/output-dir/output.json', 'w') as otpt:
+            json.dump(dictionary_of_results, otpt)
+      command:
+      - python
+      volumeMounts:
+      - name: output-dir
+        mountPath: /output-dir/
+  volumes:
+  - name: output-dir
+    emptyDir: {}

src/python_interface_to_workflows/workflow_definitions/create_division_yaml.py

@@ -0,0 +1,59 @@
+import json
+import os
+
+from hera.shared import global_config
+from hera.workflows import (
+    Artifact,
+    EmptyDirVolume,
+    Steps,
+    Workflow,
+    script,  # pyright: ignore[reportUnknownVariableType]
+)
+from hera.workflows import models as m
+
+global_config.host = os.environ.get("HOST")
+global_config.image = str(os.environ.get("IMAGE"))
+global_config.token = os.environ.get("TOKEN")
+
+
+@script(
+    volume_mounts=[m.VolumeMount(name="output-dir", mount_path="/output-dir/")],
+    outputs=Artifact(name="json-output", path="/output-dir/output.json"),
+)
+def do_division(a: int, b: int):
+    div = a / b
+    intdiv = a // b
+    remain = a % b
+    dictionary_of_results = {
+        "divide": div,
+        "quotient": intdiv,
+        "remainder": remain,
+    }
+    with open("/output-dir/output.json", "w") as otpt:
+        json.dump(dictionary_of_results, otpt)
+
+
+with Workflow(
+    generate_name="hera-division-",  # when running on graphql this should be name
+    entrypoint="divide",
+    namespace=os.environ.get("NAMESPACE"),
+    api_version="argoproj.io/v1alpha1",
+    kind="Workflow",  # ClusterWorkflowTemplate", when on graphql
+    labels={"workflows.diamond.ac.uk/science-group-examples": "true"},
+    annotations={
+        "workflows.argoproj.io/title": "Division via hera test",
+        "workflows.argoproj.io/description": """Takes a numerical input and returns the
+         remainder, output float, and output string to a json file""",
+        "workflows.diamond.ac.uk/repository": "https://github.com/DiamondLightSource/python-interface-to-workflows",
+    },
+    volumes=EmptyDirVolume(name="output-dir", mount_path="/output-dir"),
+) as w:
+    with Steps(name="divide"):
+        do_division(name="first", arguments={"a": 2, "b": 5})
+
+
+with open("divisionyaml.yaml", "w") as div:
+    div.write(w.to_yaml())  # pyright: ignore[reportUnknownMemberType]
+
+
+# w.create()