Adds docker image example and instructions on how to create them

README.md

@@ -31,3 +31,24 @@ Or if it is a commandline tool then you might put some example commands here:
 ```
 python -m python_interface_to_workflows --version
 ```
+
+# Using Copier
+```bash
+mkdir new_directory_path
+cd new_directory_path
+git init 
+git remote add origin {origin ssh}
+git branch -M main
+
+cd ..
+```
+then either:
+```bash
+git clone git@github.com:DiamondLightSource/python-interface-to-workflows.git
+copier copy {this_repo's_path} {new_directory_path}
+```
+or:
+```bash
+copier copy git@github.com:DiamondLightSource/python-copier-template.git new directory path
+```
+rebuild in dev container without cache

scripts/checkyamlcompliance.py

@@ -26,6 +26,10 @@
         match yamldata["kind"]:
             case "Workflow":
                 clst_tmpt = True
+            case "WorkflowTemplate":
+                clst_tmpt = True
+            case "ClusterWorkflowTemplate":
+                clst_tmpt = True
             case _:
                 clst_tmpt = False
     else:

scripts/lintyaml.sh

@@ -1,9 +1,10 @@
 #!/bin/bash
 cd src/python_interface_to_workflows/templates
-for file in *
-do
-argo lint "$file" --offline
-SUCCESSFULLINT=$?
-[ $SUCCESSFULLINT -ne 0 ] && exit 1
+for file in *.txt; do
+    yaml_file="${file}.yaml"
+
+    mv "$file" "$yaml_file"
+    argo lint "$yaml_file" --offline
+    mv "$yaml_file" "$file"
 done
 exit 0

scripts/makecopiercorrect.sh

@@ -7,16 +7,36 @@ cp ../templates/*example*.txt "../../copier_template/src/{{ project_name }}/temp
 for file in "../../copier_template/src/{{ project_name }}/workflow_definitions"/*
 do
     [[ $file == *.jinja ]] && continue
+
     sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file"
     sed -i 's/DiamondLightSource/{{github_org}}/g' "$file"
+
+    sed -i '1i{% raw %}' "$file"
+    echo '{% endraw %}' >> "$file"
+
+    sed -i \
+        -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \
+        -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \
+        "$file"
+
     mv "$file" "$file.jinja"
 done
 
 for file in "../../copier_template/src/{{ project_name }}/templates"/*
 do
     [[ $file == *.jinja ]] && continue
+
     sed -i 's/python-interface-to-workflows/{{repo_name}}/g' "$file"
     sed -i 's/DiamondLightSource/{{github_org}}/g' "$file"
+
+    sed -i '1i{% raw %}' "$file"
+    echo '{% endraw %}' >> "$file"
+
+    sed -i \
+        -e 's/{{repo_name}}/{% endraw %}{{repo_name}}{% raw %}/g' \
+        -e 's/{{github_org}}/{% endraw %}{{github_org}}{% raw %}/g' \
+        "$file"
+
     mv "$file" "$file.jinja"
 done
 git add "../../copier_template/src/{{ project_name }}/workflow_definitions"/*

scripts/runthefiles.sh

@@ -5,4 +5,5 @@ do
 uv run "$file"
 done
 mv *.yaml ../templates/
+mv *.txt ../templates/
 git add -u ../templates/

src/copier_template/pyproject.toml.jinja

@@ -68,7 +68,6 @@ reportMissingImports = false # Ignore missing stubs in imported modules
 reportMissingTypeStubs = "none"
 reportMissingModuleSource="warning"
 reportInvalidTypeForm="warning"
-reportMissingImports="warning"
 reportUndefinedVariable="warning"
 reportAbstractUsage="warning"
 reportArgumentType="warning"

src/copier_template/scripts/lintyaml.sh.jinja

@@ -1,9 +1,10 @@
 #!/bin/bash
 cd src/{{project_name}}/templates
-for file in *
-do
-argo lint "$file" --offline
-SUCCESSFULLINT=$?
-[ $SUCCESSFULLINT -ne 0 ] && exit 1
+for file in *.txt; do
+    yaml_file="${file}.yaml"
+
+    mv "$file" "$yaml_file"
+    argo lint "$yaml_file" --offline
+    mv "$yaml_file" "$file"
 done
 exit 0

src/copier_template/scripts/runthefiles.sh.jinja

@@ -5,4 +5,5 @@ do
 uv run "$file"
 done
 mv *.yaml ../templates/
+mv *.txt ../templates/
 git add -u ../templates/

src/copier_template/src/README.md.jinja

@@ -4,7 +4,10 @@
 
 HOST=https://argo-workflows.workflows.diamond.ac.uk/ (to submit to the production cluster)
 DEFAULT_IMAGE= (usually python 3.10)
-NAMESPACE= (the cluster you wish to run the templates on)
+VISIT= (the Visit you wish to run the template on)
+TOKEN=
+EXPIRY=
+AUTH=
 
 3. Build the dev container
 
@@ -22,3 +25,20 @@ submit_workflow_to_argo(w)
 
 NOTE: Be sure to remove this line upon commiting changes, as all workflow definition files are 
 by default, ran on pre-commit, to ensure that any yaml files they create are up to date.
+
+# Building a custom image
+While in src, the same folder as a Dockerfile:
+
+podman build -t ghcr.io/Your-Github-Name/image-name .
+podman login ghcr.io
+podman push ghcr.io/Your-Github-Name/image-name
+
+Then go to your github profile, packages, and set image-name's visibility to public
+After this, you may add 'image' in the script decorator, to run specific scripts within that image
+Alternatively, you can set the default image at the top of the file by adding:
+
+```python
+global_config.set_class_defaults(  # pyright: ignore
+    Script, image=str(os.environ.get("DEFAULT_IMAGE"))
+)
+```

src/copier_template/src/{{ project_name }}/auth/keycloak_checker.py.jinja

@@ -1,20 +1,30 @@
 import os
+from typing import TypedDict, cast
 
 import dotenv
 from keycloak import KeycloakOpenID
 from keycloak.pkce_utils import generate_code_challenge, generate_code_verifier
 
-from {{project_name}}.auth.open_auth_url import open_auth_url
+from {{project_name}}.auth.open_auth_url import token_expired
+
+
+class TokenResponse(TypedDict):
+    access_token: str
+    refresh_token: str
+
+
+class DecodedToken(TypedDict):
+    exp: int
 
 
 def set_token_env_variable() -> str:
     keycloak_openid = KeycloakOpenID(
-        client_id="workflows-dashboard",
-        server_url="https://identity.diamond.ac.uk/",
-        realm_name="dls",
-        client_secret_key="",
-        pool_maxsize=1,
-    )
+                client_id="workflows-cli",
+                server_url="https://identity.diamond.ac.uk/",
+                realm_name="dls",
+                client_secret_key="",
+                pool_maxsize=1,
+            )
     port = 8000
     code_verifier = generate_code_verifier()
     code_challenge, code_challenge_method = generate_code_challenge(code_verifier)
@@ -25,14 +35,38 @@ def set_token_env_variable() -> str:
         code_challenge=code_challenge,
         code_challenge_method=code_challenge_method,
     )
-    open_auth_url(auth_url)
-    token: dict[str, str] = (  # pyright: ignore[reportUnknownVariableType]
-        keycloak_openid.token(  # pyright: ignore[reportUnknownMemberType]
-            grant_type="authorization_code",
-            code=os.environ["AUTH"],
-            redirect_uri=f"http://localhost:{port}/",
-            code_verifier=code_verifier,
+    if token_expired(auth_url, port):
+        token = cast(
+            TokenResponse,
+            keycloak_openid.token(  # pyright: ignore[reportUnknownMemberType]
+                grant_type="authorization_code",
+                code=str(os.environ.get("AUTH")),
+                redirect_uri=f"http://localhost:{port}/",
+                code_verifier=code_verifier,
+            ),
+        )
+    else:
+        token = cast(
+            TokenResponse,
+            keycloak_openid.refresh_token(  # pyright: ignore[reportUnknownMemberType]
+                str(os.environ.get("REFRESHTOKEN"))
+            ),
         )
+    token_info = cast(
+        DecodedToken,
+        keycloak_openid.decode_token(  # pyright: ignore[reportUnknownMemberType]
+            token["access_token"]
+        ),
     )
-    dotenv.set_key("src/.env", "TOKEN", token["access_token"].strip("'"))
-    return token["access_token"]  # pyright: ignore[reportUnknownArgumentType]
+    try:
+        expire_time = int(token_info["exp"]) + 1800
+        dotenv.set_key("src/.env", "EXPIRY", str(expire_time))
+        dotenv.set_key("src/.env", "TOKEN", token["access_token"].strip("'"))
+        dotenv.set_key("src/.env", "REFRESHTOKEN", token["refresh_token"].strip("'"))
+    except AttributeError:
+        print("ERROR:")
+        exit(1)
+    finally:
+        dotenv.load_dotenv(dotenv_path="src/.env", override=True)
+
+    return token["access_token"]

src/copier_template/src/{{ project_name }}/auth/open_auth_url.py

@@ -1,10 +1,13 @@
 import os
 import socket
+import time
 import urllib.parse
 import webbrowser
 from http.server import BaseHTTPRequestHandler, HTTPServer
 from typing import cast
 
+import dotenv
+
 
 class _ReusingHTTPServer(HTTPServer):
     allow_reuse_address = True
@@ -26,12 +29,23 @@ def do_GET(self):
             self.wfile.write(b"Missing authorization code.")
 
 
-def open_auth_url(auth_url: str) -> None:
-    httpd = _ReusingHTTPServer(("localhost", 8000), CallbackHandler)
+def token_expired(auth_url: str, port: int) -> bool:
+    dotenv.load_dotenv(dotenv_path="src/.env", override=True)
+    expiry_str: str = str(os.environ.get("EXPIRY")).strip("'")
+    if (expiry_str == "" or int(expiry_str)) <= float(time.time()):
+        _open_auth_url(auth_url, port)
+        return True
+    else:
+        return False
+
+
+def _open_auth_url(auth_url: str, port: int) -> None:
+    httpd = _ReusingHTTPServer(("localhost", port), CallbackHandler)
     webbrowser.open(auth_url)
     try:
         httpd.handle_request()
         os.environ["AUTH"] = httpd.auth_code
+        dotenv.set_key("src/.env", "AUTH", httpd.auth_code)
     except OSError:
         os.environ["AUTH"] = ""
         print("ERROR: Port in use. Please restart your terminal.")

src/copier_template/src/{{ project_name }}/submit_to_argo.py.jinja

@@ -12,12 +12,11 @@ from {{project_name}}.auth.keycloak_checker import set_token_env_variable
 def submit_workflow_to_argo(w: Workflow):
     set_token_env_variable()
     dotenv.load_dotenv(dotenv_path="src/.env", override=True)
-    w.namespace = os.environ.get("NAMESPACE")
+    w.namespace = os.environ.get("VISIT")
     w.workflows_service = WorkflowsService(
         host=str(os.environ.get("HOST")).strip("'"),
         token=str(os.environ.get("TOKEN")).strip("'"),
     )
-    global_config.image = str(os.environ.get("DEFAULT_IMAGE"))
     submitted_w = cast(m.Workflow, w.create())
     name = submitted_w.metadata.name
     namespace = submitted_w.metadata.namespace

src/copier_template/src/{{ project_name }}/templates/example.txt.jinja

@@ -1,13 +1,14 @@
+{% raw %}
 apiVersion: argoproj.io/v1alpha1
-kind: Workflow
+kind: WorkflowTemplate
 metadata:
   generateName: hera-example-
   annotations:
     workflows.argoproj.io/description: |-
       Replicates the functionality of
       example.yaml
     workflows.argoproj.io/title: example remade via hera
-    workflows.diamond.ac.uk/repository: https://github.com/{{github_org}}/{{repo_name}}
+    workflows.diamond.ac.uk/repository: https://github.com/{% endraw %}{{github_org}}{% raw %}/{% endraw %}{{repo_name}}{% raw %}
   labels:
     workflows.diamond.ac.uk/science-group-examples: 'true'
 spec:
@@ -212,3 +213,4 @@ spec:
       resources:
         requests:
           storage: 1Gi
+{% endraw %}