Step Functions JSON

{
  "Comment": "Python ML Pipeline — AWS Step Functions workflow demonstrating all 8 state types (Task, Pass, Choice, Wait, Parallel, Map, Succeed, Fail). Uses Python Lambda functions for data validation, feature engineering, model training, and deployment. Built with AWS CDK / Boto3.",
  "StartAt": "InitializePipeline",
  "States": {
    "InitializePipeline": {
      "Type": "Pass",
      "Result": {
        "pipelineVersion": "2.0",
        "framework": "scikit-learn",
        "region": "us-east-1"
      },
      "ResultPath": "$.config",
      "Next": "ValidateDataSource"
    },

"ValidateDataSource": {
      "Type": "Task",
      "Resource": "arn:aws:lambda:us-east-1:123456789012:function:py-validate-datasource",
      "Retry": [{
        "ErrorEquals": ["Lambda.ServiceException"],
        "IntervalSeconds": 2,
        "MaxAttempts": 3,
        "BackoffRate": 2
      }],
      "Catch": [{
        "ErrorEquals": ["States.ALL"],
        "Next": "PipelineError"
      }],
      "Next": "CheckDataFormat"
    },

"CheckDataFormat": {
      "Type": "Choice",
      "Choices": [
        {
          "Variable": "$.dataFormat",
          "StringEquals": "PARQUET",
          "Next": "RunDataQualityChecks"
        },
        {
          "Variable": "$.dataFormat",
          "StringEquals": "CSV",
          "Next": "ConvertToParquet"
        }
      ],
      "Default": "ConvertToParquet"
    },

"ConvertToParquet": {
      "Type": "Task",
      "Resource": "arn:aws:lambda:us-east-1:123456789012:function:py-convert-to-parquet",
      "Catch": [{
        "ErrorEquals": ["States.ALL"],
        "Next": "PipelineError"
      }],
      "Next": "RunDataQualityChecks"
    },

"RunDataQualityChecks": {
      "Type": "Task",
      "Resource": "arn:aws:lambda:us-east-1:123456789012:function:py-data-quality-check",
      "Next": "DataQualityDecision"
    },

"DataQualityDecision": {
      "Type": "Choice",
      "Choices": [
        {
          "Variable": "$.qualityScore",
          "NumericGreaterThanEquals": 80,
          "Next": "ParallelFeatureEngineering"
        }
      ],
      "Default": "DataQualityFailed"
    },

"ParallelFeatureEngineering": {
      "Type": "Parallel",
      "Branches": [
        {
          "StartAt": "NumericalFeatures",
          "States": {
            "NumericalFeatures": {
              "Type": "Task",
              "Resource": "arn:aws:lambda:us-east-1:123456789012:function:py-numerical-features",
              "Next": "ScaleFeatures"
            },
            "ScaleFeatures": {
              "Type": "Task",
              "Resource": "arn:aws:lambda:us-east-1:123456789012:function:py-scale-features",
              "End": true
            }
          }
        },
        {
          "StartAt": "CategoricalFeatures",
          "States": {
            "CategoricalFeatures": {
              "Type": "Task",
              "Resource": "arn:aws:lambda:us-east-1:123456789012:function:py-categorical-features",
              "Next": "EncodeFeatures"
            },
            "EncodeFeatures": {
              "Type": "Task",
              "Resource": "arn:aws:lambda:us-east-1:123456789012:function:py-encode-features",
              "End": true
            }
          }
        }
      ],
      "Catch": [{
        "ErrorEquals": ["States.ALL"],
        "Next": "PipelineError"
      }],
      "Next": "TrainModelsMap"
    },

"TrainModelsMap": {
      "Type": "Map",
      "ItemsPath": "$.modelConfigs",
      "MaxConcurrency": 3,
      "Iterator": {
        "StartAt": "TrainModel",
        "States": {
          "TrainModel": {
            "Type": "Task",
            "Resource": "arn:aws:lambda:us-east-1:123456789012:function:py-train-model",
            "Next": "EvaluateModel"
          },
          "EvaluateModel": {
            "Type": "Task",
            "Resource": "arn:aws:lambda:us-east-1:123456789012:function:py-evaluate-model",
            "End": true
          }
        }
      },
      "Next": "SelectBestModel"
    },

"SelectBestModel": {
      "Type": "Task",
      "Resource": "arn:aws:lambda:us-east-1:123456789012:function:py-select-best-model",
      "Next": "DeploymentApproval"
    },

"DeploymentApproval": {
      "Type": "Choice",
      "Choices": [
        {
          "Variable": "$.accuracy",
          "NumericGreaterThanEquals": 95,
          "Next": "DeployModel"
        }
      ],
      "Default": "WaitForManualApproval"
    },

"WaitForManualApproval": {
      "Type": "Wait",
      "Seconds": 3600,
      "Next": "CheckApproval"
    },

"CheckApproval": {
      "Type": "Choice",
      "Choices": [
        {
          "Variable": "$.approvalStatus",
          "StringEquals": "APPROVED",
          "Next": "DeployModel"
        }
      ],
      "Default": "DeploymentRejected"
    },

"DeployModel": {
      "Type": "Task",
      "Resource": "arn:aws:lambda:us-east-1:123456789012:function:py-deploy-sagemaker-endpoint",
      "Catch": [{
        "ErrorEquals": ["States.ALL"],
        "Next": "PipelineError"
      }],
      "Next": "NotifyStakeholders"
    },

"NotifyStakeholders": {
      "Type": "Task",
      "Resource": "arn:aws:lambda:us-east-1:123456789012:function:py-notify-via-sns",
      "Next": "PipelineSucceeded"
    },

"PipelineSucceeded": {
      "Type": "Succeed"
    },

"DataQualityFailed": {
      "Type": "Fail",
      "Cause": "Data quality score below threshold."
    },

"DeploymentRejected": {
      "Type": "Fail",
      "Cause": "Model deployment rejected during manual review."
    },

"PipelineError": {
      "Type": "Fail",
      "Cause": "Unexpected pipeline error occurred."
    }
  }
}

Mermaid Code

Diagram

Send Feedback

Overwrite Mermaid edits?