diff --git a/ml-models/anomaly-detection/.gitignore b/anomaly-detection/.gitignore similarity index 100% rename from ml-models/anomaly-detection/.gitignore rename to anomaly-detection/.gitignore diff --git a/ml-models/anomaly-detection/1-preprocessing.ipynb b/anomaly-detection/1-preprocessing.ipynb similarity index 100% rename from ml-models/anomaly-detection/1-preprocessing.ipynb rename to anomaly-detection/1-preprocessing.ipynb diff --git a/ml-models/anomaly-detection/2-feature_extraction.ipynb b/anomaly-detection/2-feature_extraction.ipynb similarity index 100% rename from ml-models/anomaly-detection/2-feature_extraction.ipynb rename to anomaly-detection/2-feature_extraction.ipynb diff --git a/ml-models/anomaly-detection/3-training.ipynb b/anomaly-detection/3-training.ipynb similarity index 100% rename from ml-models/anomaly-detection/3-training.ipynb rename to anomaly-detection/3-training.ipynb diff --git a/ml-models/anomaly-detection/4-verification.ipynb b/anomaly-detection/4-verification.ipynb similarity index 100% rename from ml-models/anomaly-detection/4-verification.ipynb rename to anomaly-detection/4-verification.ipynb diff --git a/ml-models/anomaly-detection/5-push-model.ipynb b/anomaly-detection/5-push-model.ipynb similarity index 99% rename from ml-models/anomaly-detection/5-push-model.ipynb rename to anomaly-detection/5-push-model.ipynb index 6176f3e..20187dc 100644 --- a/ml-models/anomaly-detection/5-push-model.ipynb +++ b/anomaly-detection/5-push-model.ipynb @@ -54,7 +54,7 @@ "ops_repo_url = (\n", " f'http://{git_user}:{git_password}@{ops_repo_location.lstrip(\"http://\")}'\n", ")\n", - "model_artifact_id = 'model.joblib'" + "model_artifact_id = f'model-{timestamp}.joblib'" ] }, { diff --git a/ml-models/anomaly-detection/6-test-request.ipynb b/anomaly-detection/6-test-request.ipynb similarity index 100% rename from ml-models/anomaly-detection/6-test-request.ipynb rename to anomaly-detection/6-test-request.ipynb diff --git a/ml-models/anomaly-detection/deployment.py b/anomaly-detection/deployment.py similarity index 100% rename from ml-models/anomaly-detection/deployment.py rename to anomaly-detection/deployment.py diff --git a/ml-models/anomaly-detection/pipeline/data_ingestion.py b/anomaly-detection/pipeline/data_ingestion.py similarity index 100% rename from ml-models/anomaly-detection/pipeline/data_ingestion.py rename to anomaly-detection/pipeline/data_ingestion.py diff --git a/ml-models/anomaly-detection/pipeline/deployment.py b/anomaly-detection/pipeline/deployment.py similarity index 100% rename from ml-models/anomaly-detection/pipeline/deployment.py rename to anomaly-detection/pipeline/deployment.py diff --git a/ml-models/anomaly-detection/pipeline/feature_extraction.py b/anomaly-detection/pipeline/feature_extraction.py similarity index 100% rename from ml-models/anomaly-detection/pipeline/feature_extraction.py rename to anomaly-detection/pipeline/feature_extraction.py diff --git a/ml-models/anomaly-detection/pipeline/preprocessing.py b/anomaly-detection/pipeline/preprocessing.py similarity index 100% rename from ml-models/anomaly-detection/pipeline/preprocessing.py rename to anomaly-detection/pipeline/preprocessing.py diff --git a/ml-models/anomaly-detection/pipeline/push-model.py b/anomaly-detection/pipeline/push-model.py similarity index 98% rename from ml-models/anomaly-detection/pipeline/push-model.py rename to anomaly-detection/pipeline/push-model.py index 8dc21de..a20002d 100644 --- a/ml-models/anomaly-detection/pipeline/push-model.py +++ b/anomaly-detection/pipeline/push-model.py @@ -22,7 +22,7 @@ ops_repo_url = ( f'http://{git_user}:{git_password}@{ops_repo_location.lstrip("http://")}' ) -model_artifact_id = 'model.joblib' +model_artifact_id = f'model-{timestamp}.joblib' print(f'Uploading model to bucket {s3_bucket_name}' diff --git a/anomaly-detection/pipeline/training.pipeline b/anomaly-detection/pipeline/training.pipeline new file mode 100644 index 0000000..1d1e0bb --- /dev/null +++ b/anomaly-detection/pipeline/training.pipeline @@ -0,0 +1,473 @@ +{ + "doc_type": "pipeline", + "version": "3.0", + "json_schema": "http://api.dataplatform.ibm.com/schemas/common-pipeline/pipeline-flow/pipeline-flow-v3-schema.json", + "id": "elyra-auto-generated-pipeline", + "primary_pipeline": "primary", + "pipelines": [ + { + "id": "primary", + "nodes": [ + { + "id": "8740898a-a811-4eeb-b7c1-e9ca27d50fd7", + "type": "execution_node", + "op": "execute-python-node", + "app_data": { + "component_parameters": { + "dependencies": [], + "include_subdirectories": false, + "outputs": [ + "sensor-training-data.csv" + ], + "env_vars": [], + "kubernetes_pod_annotations": [], + "kubernetes_pod_labels": [], + "kubernetes_secrets": [], + "kubernetes_shared_mem_size": {}, + "kubernetes_tolerations": [], + "mounted_volumes": [], + "filename": "preprocessing.py" + }, + "label": "", + "ui_data": { + "label": "preprocessing.py", + "image": "/notebook/ml-development/jupyterlab/static/elyra/python.svg", + "x_pos": 61, + "y_pos": 287, + "description": "Run Python script" + } + }, + "inputs": [ + { + "id": "inPort", + "app_data": { + "ui_data": { + "cardinality": { + "min": 0, + "max": -1 + }, + "label": "Input Port" + } + }, + "links": [ + { + "id": "2969e14d-aca1-4833-b43b-66205af16d2c", + "node_id_ref": "fb579808-3c78-4119-b489-33d9af81b647", + "port_id_ref": "outPort" + } + ] + } + ], + "outputs": [ + { + "id": "outPort", + "app_data": { + "ui_data": { + "cardinality": { + "min": 0, + "max": -1 + }, + "label": "Output Port" + } + } + } + ] + }, + { + "id": "b30fc9dc-c7fa-4868-a3ac-ca7e84477cad", + "type": "execution_node", + "op": "execute-python-node", + "app_data": { + "component_parameters": { + "dependencies": [], + "include_subdirectories": false, + "outputs": [ + "features.pickle", + "labels.pickle" + ], + "env_vars": [], + "kubernetes_pod_annotations": [], + "kubernetes_pod_labels": [], + "kubernetes_secrets": [], + "kubernetes_tolerations": [], + "mounted_volumes": [], + "filename": "feature_extraction.py" + }, + "label": "", + "ui_data": { + "label": "feature_extraction.py", + "image": "/notebook/ml-development/jupyterlab/static/elyra/python.svg", + "x_pos": 298, + "y_pos": 286, + "description": "Run Python script" + } + }, + "inputs": [ + { + "id": "inPort", + "app_data": { + "ui_data": { + "cardinality": { + "min": 0, + "max": -1 + }, + "label": "Input Port" + } + }, + "links": [ + { + "id": "d8002555-53d5-464b-ae30-3c273fa44b3c", + "node_id_ref": "8740898a-a811-4eeb-b7c1-e9ca27d50fd7", + "port_id_ref": "outPort" + } + ] + } + ], + "outputs": [ + { + "id": "outPort", + "app_data": { + "ui_data": { + "cardinality": { + "min": 0, + "max": -1 + }, + "label": "Output Port" + } + } + } + ] + }, + { + "id": "384fb260-d2a1-43bc-9ce7-88d174e73ed2", + "type": "execution_node", + "op": "execute-python-node", + "app_data": { + "component_parameters": { + "dependencies": [], + "include_subdirectories": false, + "outputs": [ + "model.joblib" + ], + "env_vars": [], + "kubernetes_pod_annotations": [], + "kubernetes_pod_labels": [], + "kubernetes_secrets": [], + "kubernetes_tolerations": [], + "mounted_volumes": [], + "filename": "training.py" + }, + "label": "", + "ui_data": { + "label": "training.py", + "image": "/notebook/ml-development/jupyterlab/static/elyra/python.svg", + "x_pos": 539, + "y_pos": 287, + "description": "Run Python script" + } + }, + "inputs": [ + { + "id": "inPort", + "app_data": { + "ui_data": { + "cardinality": { + "min": 0, + "max": -1 + }, + "label": "Input Port" + } + }, + "links": [ + { + "id": "99c78748-2d68-46ba-8835-d0ba4ed84eb4", + "node_id_ref": "b30fc9dc-c7fa-4868-a3ac-ca7e84477cad", + "port_id_ref": "outPort" + } + ] + } + ], + "outputs": [ + { + "id": "outPort", + "app_data": { + "ui_data": { + "cardinality": { + "min": 0, + "max": -1 + }, + "label": "Output Port" + } + } + } + ] + }, + { + "id": "91f20909-91d6-445f-b1ac-e3605818209f", + "type": "execution_node", + "op": "execute-python-node", + "app_data": { + "component_parameters": { + "dependencies": [], + "include_subdirectories": false, + "outputs": [], + "env_vars": [], + "kubernetes_pod_annotations": [], + "kubernetes_pod_labels": [], + "kubernetes_secrets": [], + "kubernetes_tolerations": [], + "mounted_volumes": [], + "filename": "verification.py" + }, + "label": "", + "ui_data": { + "label": "verification.py", + "image": "/notebook/ml-development/jupyterlab/static/elyra/python.svg", + "x_pos": 471, + "y_pos": 461, + "description": "Run Python script" + } + }, + "inputs": [ + { + "id": "inPort", + "app_data": { + "ui_data": { + "cardinality": { + "min": 0, + "max": -1 + }, + "label": "Input Port" + } + }, + "links": [ + { + "id": "b88e36d0-9652-49a2-9a11-35b265876526", + "node_id_ref": "384fb260-d2a1-43bc-9ce7-88d174e73ed2", + "port_id_ref": "outPort" + } + ] + } + ], + "outputs": [ + { + "id": "outPort", + "app_data": { + "ui_data": { + "cardinality": { + "min": 0, + "max": -1 + }, + "label": "Output Port" + } + } + } + ] + }, + { + "id": "b3ca4f79-27ea-462f-a481-12418f815673", + "type": "execution_node", + "op": "execute-python-node", + "app_data": { + "component_parameters": { + "dependencies": [ + "deployment.py" + ], + "include_subdirectories": false, + "outputs": [], + "env_vars": [], + "kubernetes_pod_annotations": [], + "kubernetes_pod_labels": [], + "kubernetes_secrets": [ + { + "env_var": "S3_ENDPOINT_URL", + "name": "aws-connection-user-bucket", + "key": "AWS_S3_ENDPOINT" + }, + { + "env_var": "S3_ACCESS_KEY", + "name": "aws-connection-user-bucket", + "key": "AWS_ACCESS_KEY_ID" + }, + { + "env_var": "S3_SECRET_KEY", + "name": "aws-connection-user-bucket", + "key": "AWS_SECRET_ACCESS_KEY" + }, + { + "env_var": "S3_BUCKET_NAME", + "name": "aws-connection-user-bucket", + "key": "AWS_S3_BUCKET" + }, + { + "env_var": "username", + "name": "gitea-admin-secret-and-branch", + "key": "username" + }, + { + "env_var": "password", + "name": "gitea-admin-secret-and-branch", + "key": "password" + }, + { + "env_var": "branch", + "name": "gitea-admin-secret-and-branch", + "key": "branch" + } + ], + "kubernetes_shared_mem_size": {}, + "kubernetes_tolerations": [], + "mounted_volumes": [], + "filename": "push-model.py" + }, + "label": "", + "ui_data": { + "label": "push-model.py", + "image": "/notebook/ml-development/jupyterlab/static/elyra/python.svg", + "x_pos": 879, + "y_pos": 493, + "description": "Run Python script" + } + }, + "inputs": [ + { + "id": "inPort", + "app_data": { + "ui_data": { + "cardinality": { + "min": 0, + "max": -1 + }, + "label": "Input Port" + } + }, + "links": [ + { + "id": "fed63043-04b3-42ab-b4ea-bf416abcf858", + "node_id_ref": "91f20909-91d6-445f-b1ac-e3605818209f", + "port_id_ref": "outPort" + } + ] + } + ], + "outputs": [ + { + "id": "outPort", + "app_data": { + "ui_data": { + "cardinality": { + "min": 0, + "max": -1 + }, + "label": "Output Port" + } + } + } + ] + }, + { + "id": "fb579808-3c78-4119-b489-33d9af81b647", + "type": "execution_node", + "op": "execute-python-node", + "app_data": { + "component_parameters": { + "dependencies": [], + "include_subdirectories": false, + "outputs": [ + "raw-data.csv" + ], + "env_vars": [], + "kubernetes_pod_annotations": [], + "kubernetes_pod_labels": [], + "kubernetes_secrets": [ + { + "env_var": "S3_ENDPOINT_URL", + "name": "aws-connection-user-bucket", + "key": "AWS_S3_ENDPOINT" + }, + { + "env_var": "S3_ACCESS_KEY_ID", + "name": "aws-connection-user-bucket", + "key": "AWS_ACCESS_KEY_ID" + }, + { + "env_var": "S3_SECRET_ACCESS_KEY", + "name": "aws-connection-user-bucket", + "key": "AWS_SECRET_ACCESS_KEY" + }, + { + "env_var": "S3_BUCKET_NAME", + "name": "aws-connection-user-bucket", + "key": "AWS_S3_BUCKET" + } + ], + "kubernetes_shared_mem_size": {}, + "kubernetes_tolerations": [], + "mounted_volumes": [], + "filename": "data_ingestion.py" + }, + "label": "", + "ui_data": { + "label": "data_ingestion.py", + "image": "/notebook/ml-development/jupyterlab/static/elyra/python.svg", + "x_pos": 156, + "y_pos": 176, + "description": "Run Python script" + } + }, + "inputs": [ + { + "id": "inPort", + "app_data": { + "ui_data": { + "cardinality": { + "min": 0, + "max": -1 + }, + "label": "Input Port" + } + } + } + ], + "outputs": [ + { + "id": "outPort", + "app_data": { + "ui_data": { + "cardinality": { + "min": 0, + "max": -1 + }, + "label": "Output Port" + } + } + } + ] + } + ], + "app_data": { + "ui_data": { + "comments": [] + }, + "version": 8, + "runtime_type": "KUBEFLOW_PIPELINES", + "properties": { + "pipeline_defaults": { + "kubernetes_shared_mem_size": {}, + "kubernetes_pod_annotations": [], + "kubernetes_tolerations": [], + "mounted_volumes": [], + "kubernetes_pod_labels": [], + "env_vars": [], + "kubernetes_secrets": [], + "runtime_image": "quay.io/hybridcloudpatterns/manuela-runtime:main" + }, + "name": "training", + "runtime": "Data Science Pipelines" + } + }, + "runtime_ref": "" + } + ], + "schemas": [] +} diff --git a/ml-models/anomaly-detection/pipeline/training.py b/anomaly-detection/pipeline/training.py similarity index 100% rename from ml-models/anomaly-detection/pipeline/training.py rename to anomaly-detection/pipeline/training.py diff --git a/ml-models/anomaly-detection/pipeline/verification.py b/anomaly-detection/pipeline/verification.py similarity index 100% rename from ml-models/anomaly-detection/pipeline/verification.py rename to anomaly-detection/pipeline/verification.py diff --git a/ml-models/anomaly-detection/raw-data.csv b/anomaly-detection/raw-data.csv similarity index 100% rename from ml-models/anomaly-detection/raw-data.csv rename to anomaly-detection/raw-data.csv