Skip to content

Instantly share code, notes, and snippets.

@ederign

ederign/gist-pipeline.yaml

Last active July 22, 2026 18:09
Show Gist options

  • Select an option

    Learn more about clone URLs
  • Save ederign/3f86127930c027b454ef5d1a3d27ee80 to your computer and use it in GitHub Desktop.

Select an option

Learn more about clone URLs
Save ederign/3f86127930c027b454ef5d1a3d27ee80 to your computer and use it in GitHub Desktop.
Download ZIP
PR #865 E2E: Kale kubernetes-manifest-format output applied to Kind cluster
Raw
gist-pipeline.yaml
apiVersion: pipelines.kubeflow.org/v2beta1
kind: Pipeline
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"pipelines.kubeflow.org/v2beta1","kind":"Pipeline","metadata":{"annotations":{},"name":"candies-sharing-manifests-eder","namespace":"kubeflow"},"spec":{"description":"Share some candies between three lovely kids","displayName":"candies-sharing-manifests-eder"}}
creationTimestamp: "2026-07-22T18:02:06Z"
generation: 1
name: candies-sharing-manifests-eder
namespace: kubeflow
resourceVersion: "19374"
uid: aec8214c-c3bc-46f4-bd20-f970d56f772c
spec:
description: Share some candies between three lovely kids
displayName: candies-sharing-manifests-eder
Raw
gist-pipelineversion.yaml
apiVersion: pipelines.kubeflow.org/v2beta1
kind: PipelineVersion
metadata:
annotations:
kubectl.kubernetes.io/last-applied-configuration: |
{"apiVersion":"pipelines.kubeflow.org/v2beta1","kind":"PipelineVersion","metadata":{"annotations":{},"name":"candies-v1","namespace":"kubeflow"},"spec":{"description":"Share some candies between three lovely kids","displayName":"candies-v1","pipelineName":"candies-sharing-manifests-eder","pipelineSpec":{"components":{"comp-kid1-step":{"executorLabel":"exec-kid1-step","inputDefinitions":{"parameters":{"candies_param":{"defaultValue":20,"isOptional":true,"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"artifacts":{"kid1_html_report":{"artifactType":{"schemaTitle":"system.HTML","schemaVersion":"0.0.1"}},"kid1_output_artifact":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}},"comp-kid2-step":{"executorLabel":"exec-kid2-step","inputDefinitions":{"artifacts":{"kid1_input_artifact":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}},"parameters":{"candies_param":{"defaultValue":20,"isOptional":true,"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"artifacts":{"kid2_html_report":{"artifactType":{"schemaTitle":"system.HTML","schemaVersion":"0.0.1"}},"kid2_output_artifact":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}}}},"comp-kid3-step":{"executorLabel":"exec-kid3-step","inputDefinitions":{"artifacts":{"kid1_input_artifact":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}},"kid2_input_artifact":{"artifactType":{"schemaTitle":"system.Dataset","schemaVersion":"0.0.1"}}},"parameters":{"candies_param":{"defaultValue":20,"isOptional":true,"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"artifacts":{"kale_metrics_artifact":{"artifactType":{"schemaTitle":"system.Metrics","schemaVersion":"0.0.1"}},"kid3_html_report":{"artifactType":{"schemaTitle":"system.HTML","schemaVersion":"0.0.1"}}}}},"comp-sack-step":{"executorLabel":"exec-sack-step","inputDefinitions":{"parameters":{"candies_param":{"defaultValue":20,"isOptional":true,"parameterType":"NUMBER_INTEGER"}}},"outputDefinitions":{"artifacts":{"sack_html_report":{"artifactType":{"schemaTitle":"system.HTML","schemaVersion":"0.0.1"}}}}}},"deploymentSpec":{"executors":{"exec-kid1-step":{"container":{"args":["--executor_input","{{$}}","--function_to_execute","kid1_step"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location --index-url https://pypi.org/simple 'kfp\u003e=2.0.0' 'kubeflow-kale==2.1.0' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef kid1_step(kid1_html_report: Output[HTML], kid1_output_artifact: Output[Dataset], candies_param: int = 20):\n _kale_pipeline_parameters_block = f'''\n CANDIES = {candies_param}\n '''\n\n _kale_data_loading_block = '''\n # -----------------------DATA LOADING START--------------------------------\n from kale import marshal as _kale_marshal\n _kale_marshal.set_data_dir(\"/tmp/marshal\")\n # -----------------------DATA LOADING END----------------------------------\n '''\n\n _kale_block1 = '''\n import random\n '''\n\n _kale_block2 = '''\n def get_handful(left):\n if left == 0:\n print(\"There are no candies left! I want to cry :(\")\n return 0\n c = random.randint(1, left)\n print(\"I got %s candies!\" % c)\n return c\n '''\n\n _kale_block3 = '''\n # kid1 gets a handful, without looking in the bag!\n kid1 = get_handful(CANDIES)\n '''\n\n _kale_data_saving_block = '''\n # -----------------------DATA SAVING START---------------------------------\n from kale import marshal as _kale_marshal\n _kale_marshal.set_data_dir(\"/tmp/marshal\")\n # Save kid1 to output artifact\n _ = _kale_marshal.save(kid1, \"kid1_artifact\")\n # -----------------------DATA SAVING END-----------------------------------\n '''\n\n # run the code blocks inside a jupyter kernel\n from kale.common.jputils import run_code as _kale_run_code\n\n _kale_blocks = (\n _kale_pipeline_parameters_block,\n _kale_data_loading_block,\n\n _kale_block1,\n _kale_block2,\n _kale_block3,\n _kale_data_saving_block\n )\n\n _kale_html_artifact = _kale_run_code(_kale_blocks)\n with open(kid1_html_report.path, \"w\") as f:\n f.write(_kale_html_artifact)\n # Prepare output artifacts to be retrieved during the pipeline execution\n from kale import marshal as _kale_marshal\n _kale_marshal.set_data_dir(\"/tmp/marshal\")\n import shutil as _shutil\n\n artifact_path = _kale_marshal.get_path(\"kid1_artifact\")\n _shutil.copyfile(artifact_path, kid1_output_artifact.path)\n kid1_output_artifact.metadata[\"marshal_path\"] = artifact_path\n\n"],"env":[{"name":"HOME","value":"/tmp"}],"image":"python:3.12"}},"exec-kid2-step":{"container":{"args":["--executor_input","{{$}}","--function_to_execute","kid2_step"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location --index-url https://pypi.org/simple 'kfp\u003e=2.0.0' 'kubeflow-kale==2.1.0' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef kid2_step(kid2_html_report: Output[HTML], kid1_input_artifact: Input[Dataset], kid2_output_artifact: Output[Dataset], candies_param: int = 20):\n _kale_pipeline_parameters_block = f'''\n CANDIES = {candies_param}\n '''\n # Saves the received artifacts to be retrieved during the nb execution\n from kale import marshal as _kale_marshal\n _kale_marshal.set_data_dir(\"/tmp/marshal\")\n import shutil as _shutil\n artifact_path = kid1_input_artifact.metadata[\"marshal_path\"]\n if artifact_path is not None:\n _shutil.copy(kid1_input_artifact.path, artifact_path)\n\n _kale_data_loading_block = '''\n # -----------------------DATA LOADING START--------------------------------\n from kale import marshal as _kale_marshal\n _kale_marshal.set_data_dir(\"/tmp/marshal\")\n # Load kid1_artifact from input artifact\n kid1 = _kale_marshal.load(\"kid1_artifact\")\n # -----------------------DATA LOADING END----------------------------------\n '''\n\n _kale_block1 = '''\n import random\n '''\n\n _kale_block2 = '''\n def get_handful(left):\n if left == 0:\n print(\"There are no candies left! I want to cry :(\")\n return 0\n c = random.randint(1, left)\n print(\"I got %s candies!\" % c)\n return c\n '''\n\n _kale_block3 = '''\n kid2 = get_handful(CANDIES - kid1)\n '''\n\n _kale_data_saving_block = '''\n # -----------------------DATA SAVING START---------------------------------\n from kale import marshal as _kale_marshal\n _kale_marshal.set_data_dir(\"/tmp/marshal\")\n # Save kid2 to output artifact\n _ = _kale_marshal.save(kid2, \"kid2_artifact\")\n # -----------------------DATA SAVING END-----------------------------------\n '''\n\n # run the code blocks inside a jupyter kernel\n from kale.common.jputils import run_code as _kale_run_code\n\n _kale_blocks = (\n _kale_pipeline_parameters_block,\n _kale_data_loading_block,\n\n _kale_block1,\n _kale_block2,\n _kale_block3,\n _kale_data_saving_block\n )\n\n _kale_html_artifact = _kale_run_code(_kale_blocks)\n with open(kid2_html_report.path, \"w\") as f:\n f.write(_kale_html_artifact)\n # Prepare output artifacts to be retrieved during the pipeline execution\n from kale import marshal as _kale_marshal\n _kale_marshal.set_data_dir(\"/tmp/marshal\")\n import shutil as _shutil\n\n artifact_path = _kale_marshal.get_path(\"kid2_artifact\")\n _shutil.copyfile(artifact_path, kid2_output_artifact.path)\n kid2_output_artifact.metadata[\"marshal_path\"] = artifact_path\n\n"],"env":[{"name":"HOME","value":"/tmp"}],"image":"python:3.12"}},"exec-kid3-step":{"container":{"args":["--executor_input","{{$}}","--function_to_execute","kid3_step"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location --index-url https://pypi.org/simple 'kfp\u003e=2.0.0' 'kubeflow-kale==2.1.0' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef kid3_step(kid3_html_report: Output[HTML], kale_metrics_artifact: Output[Metrics], kid1_input_artifact: Input[Dataset], kid2_input_artifact: Input[Dataset], candies_param: int = 20):\n _kale_pipeline_parameters_block = f'''\n CANDIES = {candies_param}\n '''\n # Saves the received artifacts to be retrieved during the nb execution\n from kale import marshal as _kale_marshal\n _kale_marshal.set_data_dir(\"/tmp/marshal\")\n import shutil as _shutil\n artifact_path = kid1_input_artifact.metadata[\"marshal_path\"]\n if artifact_path is not None:\n _shutil.copy(kid1_input_artifact.path, artifact_path)\n artifact_path = kid2_input_artifact.metadata[\"marshal_path\"]\n if artifact_path is not None:\n _shutil.copy(kid2_input_artifact.path, artifact_path)\n\n _kale_data_loading_block = '''\n # -----------------------DATA LOADING START--------------------------------\n from kale import marshal as _kale_marshal\n _kale_marshal.set_data_dir(\"/tmp/marshal\")\n # Load kid1_artifact from input artifact\n kid1 = _kale_marshal.load(\"kid1_artifact\")\n # Load kid2_artifact from input artifact\n kid2 = _kale_marshal.load(\"kid2_artifact\")\n # -----------------------DATA LOADING END----------------------------------\n '''\n\n _kale_block1 = '''\n import random\n '''\n\n _kale_block2 = '''\n def get_handful(left):\n if left == 0:\n print(\"There are no candies left! I want to cry :(\")\n return 0\n c = random.randint(1, left)\n print(\"I got %s candies!\" % c)\n return c\n '''\n\n _kale_block3 = '''\n kid3 = get_handful(CANDIES - kid1 - kid2)\n '''\n\n _kale_block4 = '''\n from kale.common import kfputils as _kale_kfputils\n _kale_kfp_metrics = {\n \"kid1\": kid1,\n \"kid2\": kid2,\n \"kid3\": kid3\n }\n _kale_kfputils.generate_mlpipeline_metrics(_kale_kfp_metrics)\n '''\n\n _kale_data_saving_block = '''\n # -----------------------DATA SAVING START---------------------------------\n from kale import marshal as _kale_marshal\n _kale_marshal.set_data_dir(\"/tmp/marshal\")\n # -----------------------DATA SAVING END-----------------------------------\n '''\n\n # run the code blocks inside a jupyter kernel\n from kale.common.jputils import run_code as _kale_run_code\n\n _kale_blocks = (\n _kale_pipeline_parameters_block,\n _kale_data_loading_block,\n\n _kale_block1,\n _kale_block2,\n _kale_block3,\n _kale_block4,\n _kale_data_saving_block\n )\n\n _kale_html_artifact = _kale_run_code(_kale_blocks)\n with open(kid3_html_report.path, \"w\") as f:\n f.write(_kale_html_artifact)\n from kale.common.kfputils import load_mlpipeline_metrics\n load_mlpipeline_metrics(kale_metrics_artifact)\n\n"],"env":[{"name":"HOME","value":"/tmp"}],"image":"python:3.12"}},"exec-sack-step":{"container":{"args":["--executor_input","{{$}}","--function_to_execute","sack_step"],"command":["sh","-c","\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location --index-url https://pypi.org/simple 'kfp\u003e=2.0.0' 'kubeflow-kale==2.1.0' \u0026\u0026 \"$0\" \"$@\"\n","sh","-ec","program_path=$(mktemp -d)\n\nprintf \"%s\" \"$0\" \u003e \"$program_path/ephemeral_component.py\"\n_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path \"$program_path/ephemeral_component.py\" \"$@\"\n","\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import *\n\ndef sack_step(sack_html_report: Output[HTML], candies_param: int = 20):\n _kale_pipeline_parameters_block = f'''\n CANDIES = {candies_param}\n '''\n\n _kale_data_loading_block = '''\n # -----------------------DATA LOADING START--------------------------------\n from kale import marshal as _kale_marshal\n _kale_marshal.set_data_dir(\"/tmp/marshal\")\n # -----------------------DATA LOADING END----------------------------------\n '''\n\n _kale_block1 = '''\n import random\n '''\n\n _kale_block2 = '''\n def get_handful(left):\n if left == 0:\n print(\"There are no candies left! I want to cry :(\")\n return 0\n c = random.randint(1, left)\n print(\"I got %s candies!\" % c)\n return c\n '''\n\n _kale_block3 = '''\n print(\"Let's put in a bag %s candies and have three kids get a handful of them each\" % CANDIES)\n '''\n\n _kale_data_saving_block = '''\n # -----------------------DATA SAVING START---------------------------------\n from kale import marshal as _kale_marshal\n _kale_marshal.set_data_dir(\"/tmp/marshal\")\n # -----------------------DATA SAVING END-----------------------------------\n '''\n\n # run the code blocks inside a jupyter kernel\n from kale.common.jputils import run_code as _kale_run_code\n\n _kale_blocks = (\n _kale_pipeline_parameters_block,\n _kale_data_loading_block,\n\n _kale_block1,\n _kale_block2,\n _kale_block3,\n _kale_data_saving_block\n )\n\n _kale_html_artifact = _kale_run_code(_kale_blocks)\n with open(sack_html_report.path, \"w\") as f:\n f.write(_kale_html_artifact)\n\n"],"env":[{"name":"HOME","value":"/tmp"}],"image":"python:3.12"}}}},"pipelineInfo":{"description":"Share some candies between three lovely kids","name":"candies-sharing-manifests-eder"},"root":{"dag":{"tasks":{"kid1-step":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-kid1-step"},"dependentTasks":["sack-step"],"inputs":{"parameters":{"candies_param":{"componentInputParameter":"candies"}}},"taskInfo":{"name":"kid1-step"}},"kid2-step":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-kid2-step"},"dependentTasks":["kid1-step"],"inputs":{"artifacts":{"kid1_input_artifact":{"taskOutputArtifact":{"outputArtifactKey":"kid1_output_artifact","producerTask":"kid1-step"}}},"parameters":{"candies_param":{"componentInputParameter":"candies"}}},"taskInfo":{"name":"kid2-step"}},"kid3-step":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-kid3-step"},"dependentTasks":["kid1-step","kid2-step"],"inputs":{"artifacts":{"kid1_input_artifact":{"taskOutputArtifact":{"outputArtifactKey":"kid1_output_artifact","producerTask":"kid1-step"}},"kid2_input_artifact":{"taskOutputArtifact":{"outputArtifactKey":"kid2_output_artifact","producerTask":"kid2-step"}}},"parameters":{"candies_param":{"componentInputParameter":"candies"}}},"taskInfo":{"name":"kid3-step"}},"sack-step":{"cachingOptions":{"enableCache":true},"componentRef":{"name":"comp-sack-step"},"inputs":{"parameters":{"candies_param":{"componentInputParameter":"candies"}}},"taskInfo":{"name":"sack-step"}}}},"inputDefinitions":{"parameters":{"candies":{"defaultValue":20,"isOptional":true,"parameterType":"NUMBER_INTEGER"}}}},"schemaVersion":"2.1.0","sdkVersion":"kfp-2.16.0"},"platformSpec":{"platforms":{"kubernetes":{"deploymentSpec":{"executors":{"exec-kid1-step":{"securityContext":{"runAsGroup":"0","runAsNonRoot":true,"runAsUser":"65534"}},"exec-kid2-step":{"securityContext":{"runAsGroup":"0","runAsNonRoot":true,"runAsUser":"65534"}},"exec-kid3-step":{"securityContext":{"runAsGroup":"0","runAsNonRoot":true,"runAsUser":"65534"}},"exec-sack-step":{"securityContext":{"runAsGroup":"0","runAsNonRoot":true,"runAsUser":"65534"}}}}}}}}}
creationTimestamp: "2026-07-22T18:02:06Z"
generation: 1
name: candies-v1
namespace: kubeflow
resourceVersion: "19375"
uid: c84ea8f3-a2d0-45f5-89b6-26f958cf642d
spec:
description: Share some candies between three lovely kids
displayName: candies-v1
pipelineName: candies-sharing-manifests-eder
pipelineSpec:
components:
comp-kid1-step:
executorLabel: exec-kid1-step
inputDefinitions:
parameters:
candies_param:
defaultValue: 20
isOptional: true
parameterType: NUMBER_INTEGER
outputDefinitions:
artifacts:
kid1_html_report:
artifactType:
schemaTitle: system.HTML
schemaVersion: 0.0.1
kid1_output_artifact:
artifactType:
schemaTitle: system.Dataset
schemaVersion: 0.0.1
comp-kid2-step:
executorLabel: exec-kid2-step
inputDefinitions:
artifacts:
kid1_input_artifact:
artifactType:
schemaTitle: system.Dataset
schemaVersion: 0.0.1
parameters:
candies_param:
defaultValue: 20
isOptional: true
parameterType: NUMBER_INTEGER
outputDefinitions:
artifacts:
kid2_html_report:
artifactType:
schemaTitle: system.HTML
schemaVersion: 0.0.1
kid2_output_artifact:
artifactType:
schemaTitle: system.Dataset
schemaVersion: 0.0.1
comp-kid3-step:
executorLabel: exec-kid3-step
inputDefinitions:
artifacts:
kid1_input_artifact:
artifactType:
schemaTitle: system.Dataset
schemaVersion: 0.0.1
kid2_input_artifact:
artifactType:
schemaTitle: system.Dataset
schemaVersion: 0.0.1
parameters:
candies_param:
defaultValue: 20
isOptional: true
parameterType: NUMBER_INTEGER
outputDefinitions:
artifacts:
kale_metrics_artifact:
artifactType:
schemaTitle: system.Metrics
schemaVersion: 0.0.1
kid3_html_report:
artifactType:
schemaTitle: system.HTML
schemaVersion: 0.0.1
comp-sack-step:
executorLabel: exec-sack-step
inputDefinitions:
parameters:
candies_param:
defaultValue: 20
isOptional: true
parameterType: NUMBER_INTEGER
outputDefinitions:
artifacts:
sack_html_report:
artifactType:
schemaTitle: system.HTML
schemaVersion: 0.0.1
deploymentSpec:
executors:
exec-kid1-step:
container:
args:
- --executor_input
- '{{$}}'
- --function_to_execute
- kid1_step
command:
- sh
- -c
- |2
if ! [ -x "$(command -v pip)" ]; then
python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
fi
PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location --index-url https://pypi.org/simple 'kfp>=2.0.0' 'kubeflow-kale==2.1.0' && "$0" "$@"
- sh
- -ec
- |
program_path=$(mktemp -d)
printf "%s" "$0" > "$program_path/ephemeral_component.py"
_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@"
- |2+
import kfp
from kfp import dsl
from kfp.dsl import *
from typing import *
def kid1_step(kid1_html_report: Output[HTML], kid1_output_artifact: Output[Dataset], candies_param: int = 20):
_kale_pipeline_parameters_block = f'''
CANDIES = {candies_param}
'''
_kale_data_loading_block = '''
# -----------------------DATA LOADING START--------------------------------
from kale import marshal as _kale_marshal
_kale_marshal.set_data_dir("/tmp/marshal")
# -----------------------DATA LOADING END----------------------------------
'''
_kale_block1 = '''
import random
'''
_kale_block2 = '''
def get_handful(left):
if left == 0:
print("There are no candies left! I want to cry :(")
return 0
c = random.randint(1, left)
print("I got %s candies!" % c)
return c
'''
_kale_block3 = '''
# kid1 gets a handful, without looking in the bag!
kid1 = get_handful(CANDIES)
'''
_kale_data_saving_block = '''
# -----------------------DATA SAVING START---------------------------------
from kale import marshal as _kale_marshal
_kale_marshal.set_data_dir("/tmp/marshal")
# Save kid1 to output artifact
_ = _kale_marshal.save(kid1, "kid1_artifact")
# -----------------------DATA SAVING END-----------------------------------
'''
# run the code blocks inside a jupyter kernel
from kale.common.jputils import run_code as _kale_run_code
_kale_blocks = (
_kale_pipeline_parameters_block,
_kale_data_loading_block,
_kale_block1,
_kale_block2,
_kale_block3,
_kale_data_saving_block
)
_kale_html_artifact = _kale_run_code(_kale_blocks)
with open(kid1_html_report.path, "w") as f:
f.write(_kale_html_artifact)
# Prepare output artifacts to be retrieved during the pipeline execution
from kale import marshal as _kale_marshal
_kale_marshal.set_data_dir("/tmp/marshal")
import shutil as _shutil
artifact_path = _kale_marshal.get_path("kid1_artifact")
_shutil.copyfile(artifact_path, kid1_output_artifact.path)
kid1_output_artifact.metadata["marshal_path"] = artifact_path
env:
- name: HOME
value: /tmp
image: python:3.12
exec-kid2-step:
container:
args:
- --executor_input
- '{{$}}'
- --function_to_execute
- kid2_step
command:
- sh
- -c
- |2
if ! [ -x "$(command -v pip)" ]; then
python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
fi
PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location --index-url https://pypi.org/simple 'kfp>=2.0.0' 'kubeflow-kale==2.1.0' && "$0" "$@"
- sh
- -ec
- |
program_path=$(mktemp -d)
printf "%s" "$0" > "$program_path/ephemeral_component.py"
_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@"
- |2+
import kfp
from kfp import dsl
from kfp.dsl import *
from typing import *
def kid2_step(kid2_html_report: Output[HTML], kid1_input_artifact: Input[Dataset], kid2_output_artifact: Output[Dataset], candies_param: int = 20):
_kale_pipeline_parameters_block = f'''
CANDIES = {candies_param}
'''
# Saves the received artifacts to be retrieved during the nb execution
from kale import marshal as _kale_marshal
_kale_marshal.set_data_dir("/tmp/marshal")
import shutil as _shutil
artifact_path = kid1_input_artifact.metadata["marshal_path"]
if artifact_path is not None:
_shutil.copy(kid1_input_artifact.path, artifact_path)
_kale_data_loading_block = '''
# -----------------------DATA LOADING START--------------------------------
from kale import marshal as _kale_marshal
_kale_marshal.set_data_dir("/tmp/marshal")
# Load kid1_artifact from input artifact
kid1 = _kale_marshal.load("kid1_artifact")
# -----------------------DATA LOADING END----------------------------------
'''
_kale_block1 = '''
import random
'''
_kale_block2 = '''
def get_handful(left):
if left == 0:
print("There are no candies left! I want to cry :(")
return 0
c = random.randint(1, left)
print("I got %s candies!" % c)
return c
'''
_kale_block3 = '''
kid2 = get_handful(CANDIES - kid1)
'''
_kale_data_saving_block = '''
# -----------------------DATA SAVING START---------------------------------
from kale import marshal as _kale_marshal
_kale_marshal.set_data_dir("/tmp/marshal")
# Save kid2 to output artifact
_ = _kale_marshal.save(kid2, "kid2_artifact")
# -----------------------DATA SAVING END-----------------------------------
'''
# run the code blocks inside a jupyter kernel
from kale.common.jputils import run_code as _kale_run_code
_kale_blocks = (
_kale_pipeline_parameters_block,
_kale_data_loading_block,
_kale_block1,
_kale_block2,
_kale_block3,
_kale_data_saving_block
)
_kale_html_artifact = _kale_run_code(_kale_blocks)
with open(kid2_html_report.path, "w") as f:
f.write(_kale_html_artifact)
# Prepare output artifacts to be retrieved during the pipeline execution
from kale import marshal as _kale_marshal
_kale_marshal.set_data_dir("/tmp/marshal")
import shutil as _shutil
artifact_path = _kale_marshal.get_path("kid2_artifact")
_shutil.copyfile(artifact_path, kid2_output_artifact.path)
kid2_output_artifact.metadata["marshal_path"] = artifact_path
env:
- name: HOME
value: /tmp
image: python:3.12
exec-kid3-step:
container:
args:
- --executor_input
- '{{$}}'
- --function_to_execute
- kid3_step
command:
- sh
- -c
- |2
if ! [ -x "$(command -v pip)" ]; then
python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
fi
PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location --index-url https://pypi.org/simple 'kfp>=2.0.0' 'kubeflow-kale==2.1.0' && "$0" "$@"
- sh
- -ec
- |
program_path=$(mktemp -d)
printf "%s" "$0" > "$program_path/ephemeral_component.py"
_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@"
- |2+
import kfp
from kfp import dsl
from kfp.dsl import *
from typing import *
def kid3_step(kid3_html_report: Output[HTML], kale_metrics_artifact: Output[Metrics], kid1_input_artifact: Input[Dataset], kid2_input_artifact: Input[Dataset], candies_param: int = 20):
_kale_pipeline_parameters_block = f'''
CANDIES = {candies_param}
'''
# Saves the received artifacts to be retrieved during the nb execution
from kale import marshal as _kale_marshal
_kale_marshal.set_data_dir("/tmp/marshal")
import shutil as _shutil
artifact_path = kid1_input_artifact.metadata["marshal_path"]
if artifact_path is not None:
_shutil.copy(kid1_input_artifact.path, artifact_path)
artifact_path = kid2_input_artifact.metadata["marshal_path"]
if artifact_path is not None:
_shutil.copy(kid2_input_artifact.path, artifact_path)
_kale_data_loading_block = '''
# -----------------------DATA LOADING START--------------------------------
from kale import marshal as _kale_marshal
_kale_marshal.set_data_dir("/tmp/marshal")
# Load kid1_artifact from input artifact
kid1 = _kale_marshal.load("kid1_artifact")
# Load kid2_artifact from input artifact
kid2 = _kale_marshal.load("kid2_artifact")
# -----------------------DATA LOADING END----------------------------------
'''
_kale_block1 = '''
import random
'''
_kale_block2 = '''
def get_handful(left):
if left == 0:
print("There are no candies left! I want to cry :(")
return 0
c = random.randint(1, left)
print("I got %s candies!" % c)
return c
'''
_kale_block3 = '''
kid3 = get_handful(CANDIES - kid1 - kid2)
'''
_kale_block4 = '''
from kale.common import kfputils as _kale_kfputils
_kale_kfp_metrics = {
"kid1": kid1,
"kid2": kid2,
"kid3": kid3
}
_kale_kfputils.generate_mlpipeline_metrics(_kale_kfp_metrics)
'''
_kale_data_saving_block = '''
# -----------------------DATA SAVING START---------------------------------
from kale import marshal as _kale_marshal
_kale_marshal.set_data_dir("/tmp/marshal")
# -----------------------DATA SAVING END-----------------------------------
'''
# run the code blocks inside a jupyter kernel
from kale.common.jputils import run_code as _kale_run_code
_kale_blocks = (
_kale_pipeline_parameters_block,
_kale_data_loading_block,
_kale_block1,
_kale_block2,
_kale_block3,
_kale_block4,
_kale_data_saving_block
)
_kale_html_artifact = _kale_run_code(_kale_blocks)
with open(kid3_html_report.path, "w") as f:
f.write(_kale_html_artifact)
from kale.common.kfputils import load_mlpipeline_metrics
load_mlpipeline_metrics(kale_metrics_artifact)
env:
- name: HOME
value: /tmp
image: python:3.12
exec-sack-step:
container:
args:
- --executor_input
- '{{$}}'
- --function_to_execute
- sack_step
command:
- sh
- -c
- |2
if ! [ -x "$(command -v pip)" ]; then
python3 -m ensurepip || python3 -m ensurepip --user || apt-get install python3-pip
fi
PIP_DISABLE_PIP_VERSION_CHECK=1 python3 -m pip install --quiet --no-warn-script-location --index-url https://pypi.org/simple 'kfp>=2.0.0' 'kubeflow-kale==2.1.0' && "$0" "$@"
- sh
- -ec
- |
program_path=$(mktemp -d)
printf "%s" "$0" > "$program_path/ephemeral_component.py"
_KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@"
- |2+
import kfp
from kfp import dsl
from kfp.dsl import *
from typing import *
def sack_step(sack_html_report: Output[HTML], candies_param: int = 20):
_kale_pipeline_parameters_block = f'''
CANDIES = {candies_param}
'''
_kale_data_loading_block = '''
# -----------------------DATA LOADING START--------------------------------
from kale import marshal as _kale_marshal
_kale_marshal.set_data_dir("/tmp/marshal")
# -----------------------DATA LOADING END----------------------------------
'''
_kale_block1 = '''
import random
'''
_kale_block2 = '''
def get_handful(left):
if left == 0:
print("There are no candies left! I want to cry :(")
return 0
c = random.randint(1, left)
print("I got %s candies!" % c)
return c
'''
_kale_block3 = '''
print("Let's put in a bag %s candies and have three kids get a handful of them each" % CANDIES)
'''
_kale_data_saving_block = '''
# -----------------------DATA SAVING START---------------------------------
from kale import marshal as _kale_marshal
_kale_marshal.set_data_dir("/tmp/marshal")
# -----------------------DATA SAVING END-----------------------------------
'''
# run the code blocks inside a jupyter kernel
from kale.common.jputils import run_code as _kale_run_code
_kale_blocks = (
_kale_pipeline_parameters_block,
_kale_data_loading_block,
_kale_block1,
_kale_block2,
_kale_block3,
_kale_data_saving_block
)
_kale_html_artifact = _kale_run_code(_kale_blocks)
with open(sack_html_report.path, "w") as f:
f.write(_kale_html_artifact)
env:
- name: HOME
value: /tmp
image: python:3.12
pipelineInfo:
description: Share some candies between three lovely kids
name: candies-sharing-manifests-eder
root:
dag:
tasks:
kid1-step:
cachingOptions:
enableCache: true
componentRef:
name: comp-kid1-step
dependentTasks:
- sack-step
inputs:
parameters:
candies_param:
componentInputParameter: candies
taskInfo:
name: kid1-step
kid2-step:
cachingOptions:
enableCache: true
componentRef:
name: comp-kid2-step
dependentTasks:
- kid1-step
inputs:
artifacts:
kid1_input_artifact:
taskOutputArtifact:
outputArtifactKey: kid1_output_artifact
producerTask: kid1-step
parameters:
candies_param:
componentInputParameter: candies
taskInfo:
name: kid2-step
kid3-step:
cachingOptions:
enableCache: true
componentRef:
name: comp-kid3-step
dependentTasks:
- kid1-step
- kid2-step
inputs:
artifacts:
kid1_input_artifact:
taskOutputArtifact:
outputArtifactKey: kid1_output_artifact
producerTask: kid1-step
kid2_input_artifact:
taskOutputArtifact:
outputArtifactKey: kid2_output_artifact
producerTask: kid2-step
parameters:
candies_param:
componentInputParameter: candies
taskInfo:
name: kid3-step
sack-step:
cachingOptions:
enableCache: true
componentRef:
name: comp-sack-step
inputs:
parameters:
candies_param:
componentInputParameter: candies
taskInfo:
name: sack-step
inputDefinitions:
parameters:
candies:
defaultValue: 20
isOptional: true
parameterType: NUMBER_INTEGER
schemaVersion: 2.1.0
sdkVersion: kfp-2.16.0
platformSpec:
platforms:
kubernetes:
deploymentSpec:
executors:
exec-kid1-step:
securityContext:
runAsGroup: "0"
runAsNonRoot: true
runAsUser: "65534"
exec-kid2-step:
securityContext:
runAsGroup: "0"
runAsNonRoot: true
runAsUser: "65534"
exec-kid3-step:
securityContext:
runAsGroup: "0"
runAsNonRoot: true
runAsUser: "65534"
exec-sack-step:
securityContext:
runAsGroup: "0"
runAsNonRoot: true
runAsUser: "65534"
Raw
gist-pods.txt
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
candies-sharing-manifests-eder-zbpqw-system-container-driver-2189791183 0/2 Completed 0 2m14s 10.244.0.44 kfp-control-plane <none> <none>
candies-sharing-manifests-eder-zbpqw-system-container-driver-3848934528 0/2 Completed 0 82s 10.244.0.46 kfp-control-plane <none> <none>
candies-sharing-manifests-eder-zbpqw-system-container-driver-4003634019 0/2 Completed 0 30s 10.244.0.48 kfp-control-plane <none> <none>
candies-sharing-manifests-eder-zbpqw-system-container-impl-1092004492 0/2 Completed 0 2m4s 10.244.0.45 kfp-control-plane <none> <none>
candies-sharing-manifests-eder-zbpqw-system-container-impl-4170595629 0/2 Completed 0 72s 10.244.0.47 kfp-control-plane <none> <none>
candies-sharing-manifests-eder-zbpqw-system-container-impl-479457176 2/2 Running 0 20s 10.244.0.49 kfp-control-plane <none> <none>
candies-sharing-manifests-eder-zbpqw-system-dag-driver-4017940383 0/2 Completed 0 2m24s 10.244.0.43 kfp-control-plane <none> <none>
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment