From 4fb847e5e30334fcae4b835a15da4eef51d7fe2e Mon Sep 17 00:00:00 2001 From: Aditi Sharma Date: Tue, 9 Jul 2024 18:38:58 +0000 Subject: [PATCH 1/3] Temporarily remove failing input for integ test --- tests/integ/test_processing.py | 76 +++++++++++++++++----------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/tests/integ/test_processing.py b/tests/integ/test_processing.py index 8ceb3f2195..e9c9230ae0 100644 --- a/tests/integ/test_processing.py +++ b/tests/integ/test_processing.py @@ -766,25 +766,25 @@ def _get_processing_inputs_with_all_parameters(bucket): s3_compression_type="None", ), ), - ProcessingInput( - input_name="redshift_dataset_definition", - app_managed=True, - dataset_definition=DatasetDefinition( - local_path="/opt/ml/processing/input/rdd", - data_distribution_type="FullyReplicated", - input_mode="File", - redshift_dataset_definition=RedshiftDatasetDefinition( - cluster_id="integ-test-cluster-prod-us-west-2", - database="dev", - db_user="awsuser", - query_string="SELECT * FROM shoes", - cluster_role_arn="arn:aws:iam::037210630505:role/RedshiftClusterRole-prod-us-west-2", - output_s3_uri=f"s3://{bucket}/rdd", - output_format="CSV", - output_compression="None", - ), - ), - ), + # ProcessingInput( + # input_name="redshift_dataset_definition", + # app_managed=True, + # dataset_definition=DatasetDefinition( + # local_path="/opt/ml/processing/input/rdd", + # data_distribution_type="FullyReplicated", + # input_mode="File", + # redshift_dataset_definition=RedshiftDatasetDefinition( + # cluster_id="integ-test-cluster-prod-us-west-2", + # database="dev", + # db_user="awsuser", + # query_string="SELECT * FROM shoes", + # cluster_role_arn="arn:aws:iam::037210630505:role/RedshiftClusterRole-prod-us-west-2", + # output_s3_uri=f"s3://{bucket}/rdd", + # output_format="CSV", + # output_compression="None", + # ), + # ), + # ), ProcessingInput( input_name="athena_dataset_definition", app_managed=True, @@ -853,25 +853,25 @@ def _get_processing_job_inputs_and_outputs(bucket, output_kms_key): "S3CompressionType": "None", }, }, - { - "InputName": "redshift_dataset_definition", - "AppManaged": True, - "DatasetDefinition": { - "RedshiftDatasetDefinition": { - "ClusterId": "integ-test-cluster-prod-us-west-2", - "Database": "dev", - "DbUser": "awsuser", - "QueryString": "SELECT * FROM shoes", - "ClusterRoleArn": "arn:aws:iam::037210630505:role/RedshiftClusterRole-prod-us-west-2", - "OutputS3Uri": f"s3://{bucket}/rdd", - "OutputFormat": "CSV", - "OutputCompression": "None", - }, - "LocalPath": "/opt/ml/processing/input/rdd", - "DataDistributionType": "FullyReplicated", - "InputMode": "File", - }, - }, + # { + # "InputName": "redshift_dataset_definition", + # "AppManaged": True, + # "DatasetDefinition": { + # "RedshiftDatasetDefinition": { + # "ClusterId": "integ-test-cluster-prod-us-west-2", + # "Database": "dev", + # "DbUser": "awsuser", + # "QueryString": "SELECT * FROM shoes", + # "ClusterRoleArn": "arn:aws:iam::037210630505:role/RedshiftClusterRole-prod-us-west-2", + # "OutputS3Uri": f"s3://{bucket}/rdd", + # "OutputFormat": "CSV", + # "OutputCompression": "None", + # }, + # "LocalPath": "/opt/ml/processing/input/rdd", + # "DataDistributionType": "FullyReplicated", + # "InputMode": "File", + # }, + # }, { "InputName": "athena_dataset_definition", "AppManaged": True, From db9cde939ec56e89d782d31d39d5de716c9cd7a2 Mon Sep 17 00:00:00 2001 From: Aditi Sharma Date: Tue, 9 Jul 2024 19:08:23 +0000 Subject: [PATCH 2/3] Fix Flake8 Check --- tests/integ/test_processing.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integ/test_processing.py b/tests/integ/test_processing.py index e9c9230ae0..111fecfdc6 100644 --- a/tests/integ/test_processing.py +++ b/tests/integ/test_processing.py @@ -20,7 +20,6 @@ from sagemaker import image_uris, Session from sagemaker.dataset_definition.inputs import ( DatasetDefinition, - RedshiftDatasetDefinition, AthenaDatasetDefinition, S3Input, ) From 6c384e416c171bdd4021ef3c18cb1aa4437626f3 Mon Sep 17 00:00:00 2001 From: Aditi Sharma Date: Tue, 9 Jul 2024 19:12:46 +0000 Subject: [PATCH 3/3] Remove faulty integ test input --- tests/integ/test_processing.py | 38 ---------------------------------- 1 file changed, 38 deletions(-) diff --git a/tests/integ/test_processing.py b/tests/integ/test_processing.py index 111fecfdc6..3be778ba84 100644 --- a/tests/integ/test_processing.py +++ b/tests/integ/test_processing.py @@ -765,25 +765,6 @@ def _get_processing_inputs_with_all_parameters(bucket): s3_compression_type="None", ), ), - # ProcessingInput( - # input_name="redshift_dataset_definition", - # app_managed=True, - # dataset_definition=DatasetDefinition( - # local_path="/opt/ml/processing/input/rdd", - # data_distribution_type="FullyReplicated", - # input_mode="File", - # redshift_dataset_definition=RedshiftDatasetDefinition( - # cluster_id="integ-test-cluster-prod-us-west-2", - # database="dev", - # db_user="awsuser", - # query_string="SELECT * FROM shoes", - # cluster_role_arn="arn:aws:iam::037210630505:role/RedshiftClusterRole-prod-us-west-2", - # output_s3_uri=f"s3://{bucket}/rdd", - # output_format="CSV", - # output_compression="None", - # ), - # ), - # ), ProcessingInput( input_name="athena_dataset_definition", app_managed=True, @@ -852,25 +833,6 @@ def _get_processing_job_inputs_and_outputs(bucket, output_kms_key): "S3CompressionType": "None", }, }, - # { - # "InputName": "redshift_dataset_definition", - # "AppManaged": True, - # "DatasetDefinition": { - # "RedshiftDatasetDefinition": { - # "ClusterId": "integ-test-cluster-prod-us-west-2", - # "Database": "dev", - # "DbUser": "awsuser", - # "QueryString": "SELECT * FROM shoes", - # "ClusterRoleArn": "arn:aws:iam::037210630505:role/RedshiftClusterRole-prod-us-west-2", - # "OutputS3Uri": f"s3://{bucket}/rdd", - # "OutputFormat": "CSV", - # "OutputCompression": "None", - # }, - # "LocalPath": "/opt/ml/processing/input/rdd", - # "DataDistributionType": "FullyReplicated", - # "InputMode": "File", - # }, - # }, { "InputName": "athena_dataset_definition", "AppManaged": True,