Last active
November 28, 2025 16:56
-
-
Save jesuino/5199164d93a8e9112785d7aaf37b4792 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # PIPELINE DEFINITION | |
| # Name: kale-pipeline | |
| # Description: Train a Random Forest classifier on the Iris dataset | |
| # Inputs: | |
| # max_depth: int [Default: 2.0] | |
| # n_estimators: int [Default: 500.0] | |
| components: | |
| comp-evaluate-model-step: | |
| executorLabel: exec-evaluate-model-step | |
| inputDefinitions: | |
| artifacts: | |
| model_artifact: | |
| artifactType: | |
| schemaTitle: system.Model | |
| schemaVersion: 0.0.1 | |
| x_tst_artifact: | |
| artifactType: | |
| schemaTitle: system.Dataset | |
| schemaVersion: 0.0.1 | |
| y_tst_artifact: | |
| artifactType: | |
| schemaTitle: system.Dataset | |
| schemaVersion: 0.0.1 | |
| parameters: | |
| max_depth_param: | |
| defaultValue: 2.0 | |
| isOptional: true | |
| parameterType: NUMBER_INTEGER | |
| n_estimators_param: | |
| defaultValue: 500.0 | |
| isOptional: true | |
| parameterType: NUMBER_INTEGER | |
| outputDefinitions: | |
| artifacts: | |
| accuracy_artifact: | |
| artifactType: | |
| schemaTitle: system.Dataset | |
| schemaVersion: 0.0.1 | |
| evaluate_model_html_report: | |
| artifactType: | |
| schemaTitle: system.HTML | |
| schemaVersion: 0.0.1 | |
| f1_artifact: | |
| artifactType: | |
| schemaTitle: system.Dataset | |
| schemaVersion: 0.0.1 | |
| precision_artifact: | |
| artifactType: | |
| schemaTitle: system.Dataset | |
| schemaVersion: 0.0.1 | |
| recall_artifact: | |
| artifactType: | |
| schemaTitle: system.Dataset | |
| schemaVersion: 0.0.1 | |
| comp-load-transform-data-step: | |
| executorLabel: exec-load-transform-data-step | |
| inputDefinitions: | |
| parameters: | |
| max_depth_param: | |
| defaultValue: 2.0 | |
| isOptional: true | |
| parameterType: NUMBER_INTEGER | |
| n_estimators_param: | |
| defaultValue: 500.0 | |
| isOptional: true | |
| parameterType: NUMBER_INTEGER | |
| outputDefinitions: | |
| artifacts: | |
| load_transform_data_html_report: | |
| artifactType: | |
| schemaTitle: system.HTML | |
| schemaVersion: 0.0.1 | |
| x_trn_artifact: | |
| artifactType: | |
| schemaTitle: system.Dataset | |
| schemaVersion: 0.0.1 | |
| x_tst_artifact: | |
| artifactType: | |
| schemaTitle: system.Dataset | |
| schemaVersion: 0.0.1 | |
| y_trn_artifact: | |
| artifactType: | |
| schemaTitle: system.Dataset | |
| schemaVersion: 0.0.1 | |
| y_tst_artifact: | |
| artifactType: | |
| schemaTitle: system.Dataset | |
| schemaVersion: 0.0.1 | |
| comp-print-model-metrics-step: | |
| executorLabel: exec-print-model-metrics-step | |
| inputDefinitions: | |
| artifacts: | |
| accuracy_artifact: | |
| artifactType: | |
| schemaTitle: system.Dataset | |
| schemaVersion: 0.0.1 | |
| f1_artifact: | |
| artifactType: | |
| schemaTitle: system.Dataset | |
| schemaVersion: 0.0.1 | |
| precision_artifact: | |
| artifactType: | |
| schemaTitle: system.Dataset | |
| schemaVersion: 0.0.1 | |
| recall_artifact: | |
| artifactType: | |
| schemaTitle: system.Dataset | |
| schemaVersion: 0.0.1 | |
| parameters: | |
| max_depth_param: | |
| defaultValue: 2.0 | |
| isOptional: true | |
| parameterType: NUMBER_INTEGER | |
| n_estimators_param: | |
| defaultValue: 500.0 | |
| isOptional: true | |
| parameterType: NUMBER_INTEGER | |
| outputDefinitions: | |
| artifacts: | |
| print_model_metrics_html_report: | |
| artifactType: | |
| schemaTitle: system.HTML | |
| schemaVersion: 0.0.1 | |
| comp-train-model-step: | |
| executorLabel: exec-train-model-step | |
| inputDefinitions: | |
| artifacts: | |
| x_trn_artifact: | |
| artifactType: | |
| schemaTitle: system.Dataset | |
| schemaVersion: 0.0.1 | |
| y_trn_artifact: | |
| artifactType: | |
| schemaTitle: system.Dataset | |
| schemaVersion: 0.0.1 | |
| parameters: | |
| max_depth_param: | |
| defaultValue: 2.0 | |
| isOptional: true | |
| parameterType: NUMBER_INTEGER | |
| n_estimators_param: | |
| defaultValue: 500.0 | |
| isOptional: true | |
| parameterType: NUMBER_INTEGER | |
| outputDefinitions: | |
| artifacts: | |
| model_artifact: | |
| artifactType: | |
| schemaTitle: system.Model | |
| schemaVersion: 0.0.1 | |
| train_model_html_report: | |
| artifactType: | |
| schemaTitle: system.HTML | |
| schemaVersion: 0.0.1 | |
| deploymentSpec: | |
| executors: | |
| exec-evaluate-model-step: | |
| container: | |
| args: | |
| - --executor_input | |
| - '{{$}}' | |
| - --function_to_execute | |
| - evaluate_model_step | |
| command: | |
| - sh | |
| - -c | |
| - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ | |
| \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ | |
| \ python3 -m pip install --quiet --no-warn-script-location --index-url https://pypi.org/simple\ | |
| \ --extra-index-url https://test.pypi.org/simple --trusted-host https://pypi.org/simple\ | |
| \ --trusted-host https://test.pypi.org/simple 'kfp==2.14.6' 'dev-kubeflow-kale'\ | |
| \ 'numpy' 'scikit-learn' && \"$0\" \"$@\"\n" | |
| - sh | |
| - -ec | |
| - 'program_path=$(mktemp -d) | |
| printf "%s" "$0" > "$program_path/ephemeral_component.py" | |
| _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" | |
| ' | |
| - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ | |
| \ *\n\ndef evaluate_model_step(evaluate_model_html_report: Output[HTML],\ | |
| \ model_artifact: Input[Model], x_tst_artifact: Input[Dataset], y_tst_artifact:\ | |
| \ Input[Dataset], accuracy_artifact: Output[Dataset], f1_artifact: Output[Dataset],\ | |
| \ precision_artifact: Output[Dataset], recall_artifact: Output[Dataset],\ | |
| \ n_estimators_param: int = 500, max_depth_param: int = 2):\n _kale_pipeline_parameters_block\ | |
| \ = '''\n N_ESTIMATORS = 500\n MAX_DEPTH = 2\n '''\n \ | |
| \ # Saves the received artifacts to be retrieved during the nb execution\n\ | |
| \ from kale import marshal as _kale_marshal\n _kale_marshal.set_data_dir(\"\ | |
| /marshal\")\n import shutil as _shutil\n artifact_path = model_artifact.metadata[\"\ | |
| marshal_path\"]\n if artifact_path is not None:\n _shutil.copy(model_artifact.path,\ | |
| \ artifact_path)\n artifact_path = x_tst_artifact.metadata[\"marshal_path\"\ | |
| ]\n if artifact_path is not None:\n _shutil.copy(x_tst_artifact.path,\ | |
| \ artifact_path)\n artifact_path = y_tst_artifact.metadata[\"marshal_path\"\ | |
| ]\n if artifact_path is not None:\n _shutil.copy(y_tst_artifact.path,\ | |
| \ artifact_path)\n\n _kale_data_loading_block = '''\n # -----------------------DATA\ | |
| \ LOADING START--------------------------------\n from kale import marshal\ | |
| \ as _kale_marshal\n _kale_marshal.set_data_dir(\"/marshal\")\n #\ | |
| \ Load model_artifact from input artifact\n model = _kale_marshal.load(\"\ | |
| model_artifact\")\n # Load x_tst_artifact from input artifact\n x_tst\ | |
| \ = _kale_marshal.load(\"x_tst_artifact\")\n # Load y_tst_artifact from\ | |
| \ input artifact\n y_tst = _kale_marshal.load(\"y_tst_artifact\")\n \ | |
| \ # -----------------------DATA LOADING END----------------------------------\n\ | |
| \ '''\n\n _kale_block1 = '''\n import sklearn\n import numpy\ | |
| \ as np\n\n from sklearn import datasets\n from sklearn.model_selection\ | |
| \ import train_test_split\n from sklearn.ensemble import RandomForestClassifier\n\ | |
| \ from sklearn.metrics import precision_score, recall_score, f1_score,\ | |
| \ accuracy_score\n '''\n\n _kale_block2 = '''\n preds = model.predict(x_tst)\n\ | |
| \ precision = precision_score(y_tst, preds, average='macro')\n recall\ | |
| \ = recall_score(y_tst, preds, average='macro')\n f1 = f1_score(y_tst,\ | |
| \ preds, average='macro')\n accuracy = accuracy_score(y_tst, preds)\n\ | |
| \ print(accuracy)\n '''\n\n _kale_data_saving_block = '''\n \ | |
| \ # -----------------------DATA SAVING START---------------------------------\n\ | |
| \ from kale import marshal as _kale_marshal\n _kale_marshal.set_data_dir(\"\ | |
| /marshal\")\n # Save accuracy to output artifact\n _kale_marshal.save(accuracy,\ | |
| \ \"accuracy_artifact\")\n # Save f1 to output artifact\n _kale_marshal.save(f1,\ | |
| \ \"f1_artifact\")\n # Save precision to output artifact\n _kale_marshal.save(precision,\ | |
| \ \"precision_artifact\")\n # Save recall to output artifact\n _kale_marshal.save(recall,\ | |
| \ \"recall_artifact\")\n # -----------------------DATA SAVING END-----------------------------------\n\ | |
| \ '''\n\n # run the code blocks inside a jupyter kernel\n from\ | |
| \ kale.common.jputils import run_code as _kale_run_code\n from kale.common.kfputils\ | |
| \ import \\\n update_uimetadata as _kale_update_uimetadata\n\n \ | |
| \ _kale_blocks = (\n _kale_pipeline_parameters_block,\n _kale_data_loading_block,\n\ | |
| \n _kale_block1,\n _kale_block2,\n _kale_data_saving_block\n\ | |
| \ )\n\n _kale_html_artifact = _kale_run_code(_kale_blocks)\n with\ | |
| \ open(evaluate_model_html_report.path, \"w\") as f:\n f.write(_kale_html_artifact)\n\ | |
| \ _kale_update_uimetadata('evaluate_model_html_report')\n # Prepare\ | |
| \ output artifacts to be retrieved during the pipeline execution\n from\ | |
| \ kale import marshal as _kale_marshal\n _kale_marshal.set_data_dir(\"\ | |
| /marshal\")\n import shutil as _shutil\n\n artifact_path = _kale_marshal.get_path(\"\ | |
| accuracy_artifact\")\n _shutil.copyfile(artifact_path, accuracy_artifact.path)\n\ | |
| \ accuracy_artifact.metadata[\"marshal_path\"] = artifact_path\n artifact_path\ | |
| \ = _kale_marshal.get_path(\"f1_artifact\")\n _shutil.copyfile(artifact_path,\ | |
| \ f1_artifact.path)\n f1_artifact.metadata[\"marshal_path\"] = artifact_path\n\ | |
| \ artifact_path = _kale_marshal.get_path(\"precision_artifact\")\n \ | |
| \ _shutil.copyfile(artifact_path, precision_artifact.path)\n precision_artifact.metadata[\"\ | |
| marshal_path\"] = artifact_path\n artifact_path = _kale_marshal.get_path(\"\ | |
| recall_artifact\")\n _shutil.copyfile(artifact_path, recall_artifact.path)\n\ | |
| \ recall_artifact.metadata[\"marshal_path\"] = artifact_path\n\n" | |
| image: python:3.10 | |
| exec-load-transform-data-step: | |
| container: | |
| args: | |
| - --executor_input | |
| - '{{$}}' | |
| - --function_to_execute | |
| - load_transform_data_step | |
| command: | |
| - sh | |
| - -c | |
| - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ | |
| \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ | |
| \ python3 -m pip install --quiet --no-warn-script-location --index-url https://pypi.org/simple\ | |
| \ --extra-index-url https://test.pypi.org/simple --trusted-host https://pypi.org/simple\ | |
| \ --trusted-host https://test.pypi.org/simple 'kfp==2.14.6' 'dev-kubeflow-kale'\ | |
| \ 'numpy' 'scikit-learn' && \"$0\" \"$@\"\n" | |
| - sh | |
| - -ec | |
| - 'program_path=$(mktemp -d) | |
| printf "%s" "$0" > "$program_path/ephemeral_component.py" | |
| _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" | |
| ' | |
| - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ | |
| \ *\n\ndef load_transform_data_step(load_transform_data_html_report: Output[HTML],\ | |
| \ x_trn_artifact: Output[Dataset], x_tst_artifact: Output[Dataset], y_trn_artifact:\ | |
| \ Output[Dataset], y_tst_artifact: Output[Dataset], n_estimators_param:\ | |
| \ int = 500, max_depth_param: int = 2):\n _kale_pipeline_parameters_block\ | |
| \ = '''\n N_ESTIMATORS = 500\n MAX_DEPTH = 2\n '''\n\n\ | |
| \ _kale_data_loading_block = '''\n # -----------------------DATA LOADING\ | |
| \ START--------------------------------\n from kale import marshal as\ | |
| \ _kale_marshal\n _kale_marshal.set_data_dir(\"/marshal\")\n # -----------------------DATA\ | |
| \ LOADING END----------------------------------\n '''\n\n _kale_block1\ | |
| \ = '''\n import sklearn\n import numpy as np\n\n from sklearn\ | |
| \ import datasets\n from sklearn.model_selection import train_test_split\n\ | |
| \ from sklearn.ensemble import RandomForestClassifier\n from sklearn.metrics\ | |
| \ import precision_score, recall_score, f1_score, accuracy_score\n '''\n\ | |
| \n _kale_block2 = '''\n x, y = datasets.load_iris(return_X_y=True)\n\ | |
| \ x_trn, x_tst, y_trn, y_tst = train_test_split(x, y, test_size=.2)\n\ | |
| \ '''\n\n _kale_data_saving_block = '''\n # -----------------------DATA\ | |
| \ SAVING START---------------------------------\n from kale import marshal\ | |
| \ as _kale_marshal\n _kale_marshal.set_data_dir(\"/marshal\")\n #\ | |
| \ Save x_trn to output artifact\n _kale_marshal.save(x_trn, \"x_trn_artifact\"\ | |
| )\n # Save x_tst to output artifact\n _kale_marshal.save(x_tst, \"\ | |
| x_tst_artifact\")\n # Save y_trn to output artifact\n _kale_marshal.save(y_trn,\ | |
| \ \"y_trn_artifact\")\n # Save y_tst to output artifact\n _kale_marshal.save(y_tst,\ | |
| \ \"y_tst_artifact\")\n # -----------------------DATA SAVING END-----------------------------------\n\ | |
| \ '''\n\n # run the code blocks inside a jupyter kernel\n from\ | |
| \ kale.common.jputils import run_code as _kale_run_code\n from kale.common.kfputils\ | |
| \ import \\\n update_uimetadata as _kale_update_uimetadata\n\n \ | |
| \ _kale_blocks = (\n _kale_pipeline_parameters_block,\n _kale_data_loading_block,\n\ | |
| \n _kale_block1,\n _kale_block2,\n _kale_data_saving_block\n\ | |
| \ )\n\n _kale_html_artifact = _kale_run_code(_kale_blocks)\n with\ | |
| \ open(load_transform_data_html_report.path, \"w\") as f:\n f.write(_kale_html_artifact)\n\ | |
| \ _kale_update_uimetadata('load_transform_data_html_report')\n # Prepare\ | |
| \ output artifacts to be retrieved during the pipeline execution\n from\ | |
| \ kale import marshal as _kale_marshal\n _kale_marshal.set_data_dir(\"\ | |
| /marshal\")\n import shutil as _shutil\n\n artifact_path = _kale_marshal.get_path(\"\ | |
| x_trn_artifact\")\n _shutil.copyfile(artifact_path, x_trn_artifact.path)\n\ | |
| \ x_trn_artifact.metadata[\"marshal_path\"] = artifact_path\n artifact_path\ | |
| \ = _kale_marshal.get_path(\"x_tst_artifact\")\n _shutil.copyfile(artifact_path,\ | |
| \ x_tst_artifact.path)\n x_tst_artifact.metadata[\"marshal_path\"] =\ | |
| \ artifact_path\n artifact_path = _kale_marshal.get_path(\"y_trn_artifact\"\ | |
| )\n _shutil.copyfile(artifact_path, y_trn_artifact.path)\n y_trn_artifact.metadata[\"\ | |
| marshal_path\"] = artifact_path\n artifact_path = _kale_marshal.get_path(\"\ | |
| y_tst_artifact\")\n _shutil.copyfile(artifact_path, y_tst_artifact.path)\n\ | |
| \ y_tst_artifact.metadata[\"marshal_path\"] = artifact_path\n\n" | |
| image: python:3.10 | |
| exec-print-model-metrics-step: | |
| container: | |
| args: | |
| - --executor_input | |
| - '{{$}}' | |
| - --function_to_execute | |
| - print_model_metrics_step | |
| command: | |
| - sh | |
| - -c | |
| - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ | |
| \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ | |
| \ python3 -m pip install --quiet --no-warn-script-location --index-url https://pypi.org/simple\ | |
| \ --extra-index-url https://test.pypi.org/simple --trusted-host https://pypi.org/simple\ | |
| \ --trusted-host https://test.pypi.org/simple 'kfp==2.14.6' 'dev-kubeflow-kale'\ | |
| \ 'numpy' 'scikit-learn' && \"$0\" \"$@\"\n" | |
| - sh | |
| - -ec | |
| - 'program_path=$(mktemp -d) | |
| printf "%s" "$0" > "$program_path/ephemeral_component.py" | |
| _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" | |
| ' | |
| - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ | |
| \ *\n\ndef print_model_metrics_step(print_model_metrics_html_report: Output[HTML],\ | |
| \ accuracy_artifact: Input[Dataset], f1_artifact: Input[Dataset], precision_artifact:\ | |
| \ Input[Dataset], recall_artifact: Input[Dataset], n_estimators_param: int\ | |
| \ = 500, max_depth_param: int = 2):\n _kale_pipeline_parameters_block\ | |
| \ = '''\n N_ESTIMATORS = 500\n MAX_DEPTH = 2\n '''\n \ | |
| \ # Saves the received artifacts to be retrieved during the nb execution\n\ | |
| \ from kale import marshal as _kale_marshal\n _kale_marshal.set_data_dir(\"\ | |
| /marshal\")\n import shutil as _shutil\n artifact_path = accuracy_artifact.metadata[\"\ | |
| marshal_path\"]\n if artifact_path is not None:\n _shutil.copy(accuracy_artifact.path,\ | |
| \ artifact_path)\n artifact_path = f1_artifact.metadata[\"marshal_path\"\ | |
| ]\n if artifact_path is not None:\n _shutil.copy(f1_artifact.path,\ | |
| \ artifact_path)\n artifact_path = precision_artifact.metadata[\"marshal_path\"\ | |
| ]\n if artifact_path is not None:\n _shutil.copy(precision_artifact.path,\ | |
| \ artifact_path)\n artifact_path = recall_artifact.metadata[\"marshal_path\"\ | |
| ]\n if artifact_path is not None:\n _shutil.copy(recall_artifact.path,\ | |
| \ artifact_path)\n\n _kale_data_loading_block = '''\n # -----------------------DATA\ | |
| \ LOADING START--------------------------------\n from kale import marshal\ | |
| \ as _kale_marshal\n _kale_marshal.set_data_dir(\"/marshal\")\n #\ | |
| \ Load accuracy_artifact from input artifact\n accuracy = _kale_marshal.load(\"\ | |
| accuracy_artifact\")\n # Load f1_artifact from input artifact\n f1\ | |
| \ = _kale_marshal.load(\"f1_artifact\")\n # Load precision_artifact from\ | |
| \ input artifact\n precision = _kale_marshal.load(\"precision_artifact\"\ | |
| )\n # Load recall_artifact from input artifact\n recall = _kale_marshal.load(\"\ | |
| recall_artifact\")\n # -----------------------DATA LOADING END----------------------------------\n\ | |
| \ '''\n\n _kale_block1 = '''\n import sklearn\n import numpy\ | |
| \ as np\n\n from sklearn import datasets\n from sklearn.model_selection\ | |
| \ import train_test_split\n from sklearn.ensemble import RandomForestClassifier\n\ | |
| \ from sklearn.metrics import precision_score, recall_score, f1_score,\ | |
| \ accuracy_score\n '''\n\n _kale_block2 = '''\n print(precision)\n\ | |
| \ print(recall)\n print(f1)\n print(accuracy)\n '''\n\n _kale_data_saving_block\ | |
| \ = '''\n # -----------------------DATA SAVING START---------------------------------\n\ | |
| \ from kale import marshal as _kale_marshal\n _kale_marshal.set_data_dir(\"\ | |
| /marshal\")\n # -----------------------DATA SAVING END-----------------------------------\n\ | |
| \ '''\n\n # run the code blocks inside a jupyter kernel\n from\ | |
| \ kale.common.jputils import run_code as _kale_run_code\n from kale.common.kfputils\ | |
| \ import \\\n update_uimetadata as _kale_update_uimetadata\n\n \ | |
| \ _kale_blocks = (\n _kale_pipeline_parameters_block,\n _kale_data_loading_block,\n\ | |
| \n _kale_block1,\n _kale_block2,\n _kale_data_saving_block\n\ | |
| \ )\n\n _kale_html_artifact = _kale_run_code(_kale_blocks)\n with\ | |
| \ open(print_model_metrics_html_report.path, \"w\") as f:\n f.write(_kale_html_artifact)\n\ | |
| \ _kale_update_uimetadata('print_model_metrics_html_report')\n\n" | |
| image: python:3.10 | |
| exec-train-model-step: | |
| container: | |
| args: | |
| - --executor_input | |
| - '{{$}}' | |
| - --function_to_execute | |
| - train_model_step | |
| command: | |
| - sh | |
| - -c | |
| - "\nif ! [ -x \"$(command -v pip)\" ]; then\n python3 -m ensurepip ||\ | |
| \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\ | |
| \ python3 -m pip install --quiet --no-warn-script-location --index-url https://pypi.org/simple\ | |
| \ --extra-index-url https://test.pypi.org/simple --trusted-host https://pypi.org/simple\ | |
| \ --trusted-host https://test.pypi.org/simple 'kfp==2.14.6' 'dev-kubeflow-kale'\ | |
| \ 'numpy' 'scikit-learn' && \"$0\" \"$@\"\n" | |
| - sh | |
| - -ec | |
| - 'program_path=$(mktemp -d) | |
| printf "%s" "$0" > "$program_path/ephemeral_component.py" | |
| _KFP_RUNTIME=true python3 -m kfp.dsl.executor_main --component_module_path "$program_path/ephemeral_component.py" "$@" | |
| ' | |
| - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\ | |
| \ *\n\ndef train_model_step(train_model_html_report: Output[HTML], x_trn_artifact:\ | |
| \ Input[Dataset], y_trn_artifact: Input[Dataset], model_artifact: Output[Model],\ | |
| \ n_estimators_param: int = 500, max_depth_param: int = 2):\n _kale_pipeline_parameters_block\ | |
| \ = '''\n N_ESTIMATORS = 500\n MAX_DEPTH = 2\n '''\n \ | |
| \ # Saves the received artifacts to be retrieved during the nb execution\n\ | |
| \ from kale import marshal as _kale_marshal\n _kale_marshal.set_data_dir(\"\ | |
| /marshal\")\n import shutil as _shutil\n artifact_path = x_trn_artifact.metadata[\"\ | |
| marshal_path\"]\n if artifact_path is not None:\n _shutil.copy(x_trn_artifact.path,\ | |
| \ artifact_path)\n artifact_path = y_trn_artifact.metadata[\"marshal_path\"\ | |
| ]\n if artifact_path is not None:\n _shutil.copy(y_trn_artifact.path,\ | |
| \ artifact_path)\n\n _kale_data_loading_block = '''\n # -----------------------DATA\ | |
| \ LOADING START--------------------------------\n from kale import marshal\ | |
| \ as _kale_marshal\n _kale_marshal.set_data_dir(\"/marshal\")\n #\ | |
| \ Load x_trn_artifact from input artifact\n x_trn = _kale_marshal.load(\"\ | |
| x_trn_artifact\")\n # Load y_trn_artifact from input artifact\n y_trn\ | |
| \ = _kale_marshal.load(\"y_trn_artifact\")\n # -----------------------DATA\ | |
| \ LOADING END----------------------------------\n '''\n\n _kale_block1\ | |
| \ = '''\n import sklearn\n import numpy as np\n\n from sklearn\ | |
| \ import datasets\n from sklearn.model_selection import train_test_split\n\ | |
| \ from sklearn.ensemble import RandomForestClassifier\n from sklearn.metrics\ | |
| \ import precision_score, recall_score, f1_score, accuracy_score\n '''\n\ | |
| \n _kale_block2 = '''\n model = RandomForestClassifier(n_estimators=int(N_ESTIMATORS),\n\ | |
| \ max_depth=int(MAX_DEPTH))\n model.fit(x_trn,\ | |
| \ y_trn)\n '''\n\n _kale_data_saving_block = '''\n # -----------------------DATA\ | |
| \ SAVING START---------------------------------\n from kale import marshal\ | |
| \ as _kale_marshal\n _kale_marshal.set_data_dir(\"/marshal\")\n #\ | |
| \ Save model to output artifact\n _kale_marshal.save(model, \"model_artifact\"\ | |
| )\n # -----------------------DATA SAVING END-----------------------------------\n\ | |
| \ '''\n\n # run the code blocks inside a jupyter kernel\n from\ | |
| \ kale.common.jputils import run_code as _kale_run_code\n from kale.common.kfputils\ | |
| \ import \\\n update_uimetadata as _kale_update_uimetadata\n\n \ | |
| \ _kale_blocks = (\n _kale_pipeline_parameters_block,\n _kale_data_loading_block,\n\ | |
| \n _kale_block1,\n _kale_block2,\n _kale_data_saving_block\n\ | |
| \ )\n\n _kale_html_artifact = _kale_run_code(_kale_blocks)\n with\ | |
| \ open(train_model_html_report.path, \"w\") as f:\n f.write(_kale_html_artifact)\n\ | |
| \ _kale_update_uimetadata('train_model_html_report')\n # Prepare output\ | |
| \ artifacts to be retrieved during the pipeline execution\n from kale\ | |
| \ import marshal as _kale_marshal\n _kale_marshal.set_data_dir(\"/marshal\"\ | |
| )\n import shutil as _shutil\n\n artifact_path = _kale_marshal.get_path(\"\ | |
| model_artifact\")\n _shutil.copyfile(artifact_path, model_artifact.path)\n\ | |
| \ model_artifact.metadata[\"marshal_path\"] = artifact_path\n\n" | |
| image: python:3.10 | |
| pipelineInfo: | |
| description: Train a Random Forest classifier on the Iris dataset | |
| name: kale-pipeline | |
| root: | |
| dag: | |
| tasks: | |
| evaluate-model-step: | |
| cachingOptions: | |
| enableCache: true | |
| componentRef: | |
| name: comp-evaluate-model-step | |
| dependentTasks: | |
| - load-transform-data-step | |
| - train-model-step | |
| inputs: | |
| artifacts: | |
| model_artifact: | |
| taskOutputArtifact: | |
| outputArtifactKey: model_artifact | |
| producerTask: train-model-step | |
| x_tst_artifact: | |
| taskOutputArtifact: | |
| outputArtifactKey: x_tst_artifact | |
| producerTask: load-transform-data-step | |
| y_tst_artifact: | |
| taskOutputArtifact: | |
| outputArtifactKey: y_tst_artifact | |
| producerTask: load-transform-data-step | |
| parameters: | |
| max_depth_param: | |
| componentInputParameter: max_depth | |
| n_estimators_param: | |
| componentInputParameter: n_estimators | |
| taskInfo: | |
| name: evaluate-model-step | |
| load-transform-data-step: | |
| cachingOptions: | |
| enableCache: true | |
| componentRef: | |
| name: comp-load-transform-data-step | |
| inputs: | |
| parameters: | |
| max_depth_param: | |
| componentInputParameter: max_depth | |
| n_estimators_param: | |
| componentInputParameter: n_estimators | |
| taskInfo: | |
| name: load-transform-data-step | |
| print-model-metrics-step: | |
| cachingOptions: | |
| enableCache: true | |
| componentRef: | |
| name: comp-print-model-metrics-step | |
| dependentTasks: | |
| - evaluate-model-step | |
| inputs: | |
| artifacts: | |
| accuracy_artifact: | |
| taskOutputArtifact: | |
| outputArtifactKey: accuracy_artifact | |
| producerTask: evaluate-model-step | |
| f1_artifact: | |
| taskOutputArtifact: | |
| outputArtifactKey: f1_artifact | |
| producerTask: evaluate-model-step | |
| precision_artifact: | |
| taskOutputArtifact: | |
| outputArtifactKey: precision_artifact | |
| producerTask: evaluate-model-step | |
| recall_artifact: | |
| taskOutputArtifact: | |
| outputArtifactKey: recall_artifact | |
| producerTask: evaluate-model-step | |
| parameters: | |
| max_depth_param: | |
| componentInputParameter: max_depth | |
| n_estimators_param: | |
| componentInputParameter: n_estimators | |
| taskInfo: | |
| name: print-model-metrics-step | |
| train-model-step: | |
| cachingOptions: | |
| enableCache: true | |
| componentRef: | |
| name: comp-train-model-step | |
| dependentTasks: | |
| - load-transform-data-step | |
| inputs: | |
| artifacts: | |
| x_trn_artifact: | |
| taskOutputArtifact: | |
| outputArtifactKey: x_trn_artifact | |
| producerTask: load-transform-data-step | |
| y_trn_artifact: | |
| taskOutputArtifact: | |
| outputArtifactKey: y_trn_artifact | |
| producerTask: load-transform-data-step | |
| parameters: | |
| max_depth_param: | |
| componentInputParameter: max_depth | |
| n_estimators_param: | |
| componentInputParameter: n_estimators | |
| taskInfo: | |
| name: train-model-step | |
| inputDefinitions: | |
| parameters: | |
| max_depth: | |
| defaultValue: 2.0 | |
| isOptional: true | |
| parameterType: NUMBER_INTEGER | |
| n_estimators: | |
| defaultValue: 500.0 | |
| isOptional: true | |
| parameterType: NUMBER_INTEGER | |
| schemaVersion: 2.1.0 | |
| sdkVersion: kfp-2.14.6 |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment