Adds a testing notebook to run hera files from a notebook

README.md

@@ -31,3 +31,24 @@ Or if it is a commandline tool then you might put some example commands here:
 ```
 python -m python_interface_to_workflows --version
 ```
+
+# Using Copier
+```bash
+mkdir new_directory_path
+cd new_directory_path
+git init 
+git remote add origin {origin ssh}
+git branch -M main
+
+cd ..
+```
+then either:
+```bash
+git clone git@github.com:DiamondLightSource/python-interface-to-workflows.git
+copier copy {this_repo's_path} {new_directory_path}
+```
+or:
+```bash
+copier copy git@github.com:DiamondLightSource/python-copier-template.git new directory path
+```
+rebuild in dev container without cache

scripts/lintyaml.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 cd src/python_interface_to_workflows/templates
-for file in *
+for file in *.yaml;
 do
 argo lint "$file" --offline
 SUCCESSFULLINT=$?

scripts/makecopiercorrect.sh

@@ -7,16 +7,36 @@ cp ../templates/*example*.txt "../../copier_template/src/{{ project_name }}/temp
 for file in "../../copier_template/src/{{ project_name }}/workflow_definitions"/*
 do
     [[ $file == *.jinja ]] && continue
+
     sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file"
     sed -i 's/DiamondLightSource/{{github_org}}/g' "$file"
+
+    sed -i '1i{% raw %}' "$file"
+    echo '{% endraw %}' >> "$file"
+
+    sed -i \
+        -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \
+        -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \
+        "$file"
+
     mv "$file" "$file.jinja"
 done
 
 for file in "../../copier_template/src/{{ project_name }}/templates"/*
 do
     [[ $file == *.jinja ]] && continue
+
     sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file"
     sed -i 's/DiamondLightSource/{{github_org}}/g' "$file"
+
+    sed -i '1i{% raw %}' "$file"
+    echo '{% endraw %}' >> "$file"
+
+    sed -i \
+        -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \
+        -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \
+        "$file"
+
     mv "$file" "$file.jinja"
 done
 git add "../../copier_template/src/{{ project_name }}/workflow_definitions"/*

scripts/runthefiles.sh

@@ -5,4 +5,5 @@ do
 uv run "$file"
 done
 mv *.yaml ../templates/
+mv *.txt ../templates/
 git add -u ../templates/

src/copier_template/pyproject.toml.jinja

@@ -68,7 +68,6 @@ reportMissingImports = false # Ignore missing stubs in imported modules
 reportMissingTypeStubs = "none"
 reportMissingModuleSource="warning"
 reportInvalidTypeForm="warning"
-reportMissingImports="warning"
 reportUndefinedVariable="warning"
 reportAbstractUsage="warning"
 reportArgumentType="warning"

src/copier_template/scripts/lintyaml.sh.jinja

@@ -1,6 +1,6 @@
 #!/bin/bash
 cd src/{{project_name}}/templates
-for file in *
+for file in *.yaml;
 do
 argo lint "$file" --offline
 SUCCESSFULLINT=$?

src/copier_template/scripts/runthefiles.sh.jinja

@@ -1,8 +1,9 @@
 #!/bin/bash
-cd src/{{project_name}}/workflow_definitions
+cd src/python_interface_to_workflows/workflow_definitions
 for file in *
 do
 uv run "$file"
 done
 mv *.yaml ../templates/
+mv *.txt ../templates/
 git add -u ../templates/

src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja

@@ -9,12 +9,12 @@ from {{project_name}}.auth.open_auth_url import open_auth_url
 
 def set_token_env_variable() -> str:
     keycloak_openid = KeycloakOpenID(
-        client_id="workflows-dashboard",
-        server_url="https://identity.diamond.ac.uk/",
-        realm_name="dls",
-        client_secret_key="",
-        pool_maxsize=1,
-    )
+                client_id="workflows-cli",
+                server_url="https://identity.diamond.ac.uk/",
+                realm_name="dls",
+                client_secret_key="",
+                pool_maxsize=1,
+            )
     port = 8000
     code_verifier = generate_code_verifier()
     code_challenge, code_challenge_method = generate_code_challenge(code_verifier)
@@ -25,14 +25,26 @@ def set_token_env_variable() -> str:
         code_challenge=code_challenge,
         code_challenge_method=code_challenge_method,
     )
-    open_auth_url(auth_url)
-    token: dict[str, str] = (  # pyright: ignore[reportUnknownVariableType]
-        keycloak_openid.token(  # pyright: ignore[reportUnknownMemberType]
-            grant_type="authorization_code",
-            code=os.environ["AUTH"],
-            redirect_uri=f"http://localhost:{port}/",
-            code_verifier=code_verifier,
-        )
+    match open_auth_url(auth_url, port):
+        case True:
+            token: dict[str, str] = (  # pyright: ignore[reportUnknownVariableType]
+                keycloak_openid.token(  # pyright: ignore[reportUnknownMemberType]
+                    grant_type="authorization_code",
+                    code=os.environ.get("AUTH"),  # pyright: ignore
+                    redirect_uri=f"http://localhost:{port}/",
+                    code_verifier=code_verifier,
+                )
+            )
+        case False:
+            token: dict[str, str] = keycloak_openid.refresh_token(  # pyright: ignore
+                str(os.environ.get("REFRESHTOKEN"))  # pyright: ignore
+            )
+    token_info: dict[str, int | str | dict[str, list[str]]] = (  # pyright: ignore
+        keycloak_openid.decode_token(token["access_token"])  # pyright: ignore
     )
+    expire_time = int(token_info["exp"]) + 1500  # pyright: ignore
+    dotenv.set_key("src/.env", "EXPIRY", str(expire_time).strip("'"))
     dotenv.set_key("src/.env", "TOKEN", token["access_token"].strip("'"))
+    dotenv.set_key("src/.env", "REFRESHTOKEN", token["refresh_token"].strip("'"))
+    dotenv.load_dotenv(dotenv_path="src/.env", override=True)
     return token["access_token"]  # pyright: ignore[reportUnknownArgumentType]

src/copier_template/src/{{ project_name }}/auth/open_auth_url.py

@@ -1,10 +1,13 @@
 import os
 import socket
+import time
 import urllib.parse
 import webbrowser
 from http.server import BaseHTTPRequestHandler, HTTPServer
 from typing import cast
 
+import dotenv
+
 
 class _ReusingHTTPServer(HTTPServer):
     allow_reuse_address = True
@@ -26,16 +29,23 @@ def do_GET(self):
             self.wfile.write(b"Missing authorization code.")
 
 
-def open_auth_url(auth_url: str) -> None:
-    httpd = _ReusingHTTPServer(("localhost", 8000), CallbackHandler)
-    webbrowser.open(auth_url)
-    try:
-        httpd.handle_request()
-        os.environ["AUTH"] = httpd.auth_code
-    except OSError:
-        os.environ["AUTH"] = ""
-        print("ERROR: Port in use. Please restart your terminal.")
-        exit(1)
-    finally:
-        httpd.socket.shutdown(socket.SHUT_RDWR)
-        httpd.server_close()
+def open_auth_url(auth_url: str, port: int) -> bool:
+    dotenv.load_dotenv(dotenv_path="src/.env", override=True)
+    expiry_str: str = os.environ.get("EXPIRY").strip("'")  # pyright: ignore
+    if (expiry_str == "" or int(expiry_str)) <= float(time.time()):
+        httpd = _ReusingHTTPServer(("localhost", port), CallbackHandler)
+        webbrowser.open(auth_url)
+        try:
+            httpd.handle_request()
+            os.environ["AUTH"] = httpd.auth_code
+            dotenv.set_key("src/.env", "AUTH", httpd.auth_code)
+        except OSError:
+            os.environ["AUTH"] = ""
+            print("ERROR: Port in use. Please restart your terminal.")
+            exit(1)
+        finally:
+            httpd.socket.shutdown(socket.SHUT_RDWR)
+            httpd.server_close()
+        return True
+    else:
+        return False

src/copier_template/src/{{ project_name }}/templates/example.txt.jinja

@@ -1,3 +1,4 @@
+{% raw %}
 apiVersion: argoproj.io/v1alpha1
 kind: Workflow
 metadata:
@@ -7,7 +8,7 @@ metadata:
       Replicates the functionality of
       example.yaml
     workflows.argoproj.io/title: example remade via hera
-    workflows.diamond.ac.uk/repository: https://github.com/{{github_org}}/{{repo_name}}
+    workflows.diamond.ac.uk/repository: https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}
   labels:
     workflows.diamond.ac.uk/science-group-examples: 'true'
 spec:
@@ -212,3 +213,4 @@ spec:
       resources:
         requests:
           storage: 1Gi
+{% endraw %}

src/copier_template/src/{{ project_name }}/workflow_definitions/create_example_template.py.jinja

@@ -1,3 +1,4 @@
+{% raw %}
 from hera.workflows import (
     DAG,
     Artifact,
@@ -153,7 +154,7 @@ with Workflow(
         "workflows.argoproj.io/title": "example remade via hera",
         "workflows.argoproj.io/description": """Replicates the functionality of
 example.yaml""",
-        "workflows.diamond.ac.uk/repository": "https://github.com/{{github_org}}/{{repo_name}}",
+        "workflows.diamond.ac.uk/repository": "https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}",
     },
     volumes=Volume(name="tmpdir", mount_path="/tmp/", size="1Gi"),
 ) as w:
@@ -178,5 +179,6 @@ example.yaml""",
         [install, params] >> makeimages >> makehdf5  # pyright: ignore
 
 
-with open("example.yaml", "w") as div:
+with open("example.txt", "w") as div:
     div.write(w.to_yaml())  # pyright: ignore[reportUnknownMemberType]
+{% endraw %}

src/copier_template/src/{{ project_name }}/workflow_definitions/notebook_division.ipynb.jinja

@@ -0,0 +1,116 @@
+{% raw %}{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "1e3146f1",
+   "metadata": {},
+   "source": [
+    "# Running in VSCode:\n",
+    "\n",
+    "1. Set kernal to python-interface-to-workflows 3.11.x\n",
+    "2. Hit F1, run Jupyter: Import Notebook to Script\n",
+    "3. Click notebook_division.ipynb\n",
+    "4. Run the cells sequentially"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "494174ef",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json\n",
+    "\n",
+    "from hera.workflows import (\n",
+    "    Artifact,\n",
+    "    EmptyDirVolume,\n",
+    "    Steps,\n",
+    "    Workflow,\n",
+    "    script,  # pyright: ignore[reportUnknownVariableType]\n",
+    ")\n",
+    "from hera.workflows import models as m\n",
+    "\n",
+    "\n",
+    "@script(\n",
+    "    volume_mounts=[m.VolumeMount(name=\"output-dir\", mount_path=\"/output-dir/\")],\n",
+    "    outputs=Artifact(name=\"json-output\", path=\"/output-dir/output.json\"),\n",
+    ")\n",
+    "def do_division(a: int, b: int):\n",
+    "    div = a / b\n",
+    "    intdiv = a // b\n",
+    "    remain = a % b\n",
+    "    dictionary_of_results = {\n",
+    "        \"divide\": div,\n",
+    "        \"quotient\": intdiv,\n",
+    "        \"remainder\": remain,\n",
+    "    }\n",
+    "    with open(\"/output-dir/output.json\", \"w\") as otpt:\n",
+    "        json.dump(dictionary_of_results, otpt)\n",
+    "\n",
+    "\n",
+    "with Workflow(\n",
+    "    generate_name=\"hera-division-\",  # when running on graphql this should be name\n",
+    "    entrypoint=\"divide\",\n",
+    "    api_version=\"argoproj.io/v1alpha1\",\n",
+    "    kind=\"Workflow\",  # ClusterWorkflowTemplate\", when on graphql\n",
+    "    labels={\"workflows.diamond.ac.uk/science-group-examples\": \"true\"},\n",
+    "    annotations={\n",
+    "        \"workflows.argoproj.io/title\": \"Division via hera test\",\n",
+    "        \"workflows.argoproj.io/description\": \"\"\"Takes a numerical input and returns\n",
+    "    the remainder, output float, and output string to a json file\"\"\",\n",
+    "        \"workflows.diamond.ac.uk/repository\": \"https://github.com/DiamondLightSource/python-interface-to-workflows\",\n",
+    "    },\n",
+    "    volumes=EmptyDirVolume(name=\"output-dir\", mount_path=\"/output-dir\"),\n",
+    ") as w:\n",
+    "    with Steps(name=\"divide\"):\n",
+    "        do_division(name=\"first\", arguments={\"a\": 2, \"b\": 5})\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "04708f46",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open(\"division_from_jupyter.yaml\", \"w\") as div:\n",
+    "    div.write(w.to_yaml())  # pyright: ignore[reportUnknownMemberType]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7e9f88cb",
+   "metadata": {},
+   "source": [
+    "from {% endraw %}{{project_name}}{% raw %}.submit_to_argo import submit_workflow_to_argo\n",
+    "\n",
+    "submit_workflow_to_argo(w)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "python-interface-to-workflows (3.11.x)",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.15"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
+{% endraw %}