diff --git a/tests/integ/test_git.py b/tests/integ/test_git.py index 20ae91e204..98a4cdbeb0 100644 --- a/tests/integ/test_git.py +++ b/tests/integ/test_git.py @@ -1,4 +1,4 @@ -# Copyright 2017-2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"). You # may not use this file except in compliance with the License. A copy of @@ -24,7 +24,6 @@ from sagemaker.pytorch.defaults import PYTORCH_VERSION from sagemaker.pytorch.estimator import PyTorch from sagemaker.sklearn.estimator import SKLearn -from sagemaker.mxnet.model import MXNetModel from sagemaker.sklearn.model import SKLearnModel from tests.integ import DATA_DIR, PYTHON_VERSION @@ -33,29 +32,27 @@ GIT_REPO = "https://github.com/aws/sagemaker-python-sdk.git" BRANCH = "test-branch-git-config" COMMIT = "ae15c9d7d5b97ea95ea451e4662ee43da3401d73" + PRIVATE_GIT_REPO = "https://github.com/git-support-test/test-git.git" PRIVATE_BRANCH = "master" PRIVATE_COMMIT = "a46d6f9add3532ca3e4e231e4108b6bad15b7373" + PRIVATE_GIT_REPO_2FA = "https://github.com/git-support-test-2fa/test-git.git" PRIVATE_GIT_REPO_2FA_SSH = "git@github.com:git-support-test-2fa/test-git.git" PRIVATE_BRANCH_2FA = "master" PRIVATE_COMMIT_2FA = "52381dee030eb332a7e42d9992878d7261eb21d4" + CODECOMMIT_REPO = ( "https://git-codecommit.us-west-2.amazonaws.com/v1/repos/sagemaker-python-sdk-git-testing-repo/" ) CODECOMMIT_BRANCH = "master" -# Since personal access tokens will delete themselves if they are committed to GitHub repos, -# we cannot hard code them here, but have to encrypt instead -ENCRYPTED_PRIVATE_REPO_TOKEN = "e-4_1-1dc_71-f0e_f7b54a0f3b7db2757163da7b5e8c3" -PRIVATE_REPO_TOKEN = ENCRYPTED_PRIVATE_REPO_TOKEN.replace("-", "").replace("_", "") - # endpoint tests all use the same port, so we use this lock to prevent concurrent execution LOCK_PATH = os.path.join(tempfile.gettempdir(), "sagemaker_test_git_lock") @pytest.mark.local_mode -def test_git_support_with_pytorch(sagemaker_local_session): +def test_github(sagemaker_local_session): script_path = "mnist.py" data_path = os.path.join(DATA_DIR, "pytorch_mnist") git_config = {"repo": GIT_REPO, "branch": BRANCH, "commit": COMMIT} @@ -85,7 +82,7 @@ def test_git_support_with_pytorch(sagemaker_local_session): @pytest.mark.local_mode @pytest.mark.skip("needs a secure authentication approach") -def test_git_support_with_mxnet(sagemaker_local_session): +def test_private_github(sagemaker_local_session): script_path = "mnist.py" data_path = os.path.join(DATA_DIR, "mxnet_mnist") git_config = { @@ -94,7 +91,7 @@ def test_git_support_with_mxnet(sagemaker_local_session): "commit": PRIVATE_COMMIT, "2FA_enabled": False, "username": "git-support-test", - "password": "", # TODO: find a more secure approach + "password": "", # TODO: find a secure approach } source_dir = "mxnet" dependencies = ["foo/bar.py"] @@ -126,21 +123,8 @@ def test_git_support_with_mxnet(sagemaker_local_session): with lock.lock(LOCK_PATH): try: serving_script_path = "mnist_hosting_with_custom_handlers.py" - client = sagemaker_local_session.sagemaker_client - desc = client.describe_training_job(TrainingJobName=mx.latest_training_job.name) - model_data = desc["ModelArtifacts"]["S3ModelArtifacts"] - model = MXNetModel( - model_data, - "SageMakerRole", - entry_point=serving_script_path, - source_dir=source_dir, - dependencies=dependencies, - py_version=PYTHON_VERSION, - sagemaker_session=sagemaker_local_session, - framework_version=MXNet.LATEST_VERSION, - git_config=git_config, - ) - predictor = model.deploy(initial_instance_count=1, instance_type="local") + predictor = mx.deploy(1, "local", entry_point=serving_script_path) + data = numpy.zeros(shape=(1, 1, 28, 28)) result = predictor.predict(data) assert result is not None @@ -148,9 +132,9 @@ def test_git_support_with_mxnet(sagemaker_local_session): predictor.delete_endpoint() -@pytest.mark.skipif(PYTHON_VERSION != "py3", reason="Scikit-learn image supports only python 3.") @pytest.mark.local_mode -def test_git_support_with_sklearn(sagemaker_local_session, sklearn_full_version): +@pytest.mark.skip("needs a secure authentication approach") +def test_private_github_with_2fa(sagemaker_local_session, sklearn_full_version): script_path = "mnist.py" data_path = os.path.join(DATA_DIR, "sklearn_mnist") git_config = { @@ -158,14 +142,15 @@ def test_git_support_with_sklearn(sagemaker_local_session, sklearn_full_version) "branch": PRIVATE_BRANCH_2FA, "commit": PRIVATE_COMMIT_2FA, "2FA_enabled": True, - "token": PRIVATE_REPO_TOKEN, + "token": "", # TODO: find a secure approach } source_dir = "sklearn" + sklearn = SKLearn( entry_point=script_path, role="SageMakerRole", source_dir=source_dir, - py_version=PYTHON_VERSION, + py_version="py3", # Scikit-learn supports only Python 3 train_instance_count=1, train_instance_type="local", sagemaker_session=sagemaker_local_session, @@ -202,9 +187,7 @@ def test_git_support_with_sklearn(sagemaker_local_session, sklearn_full_version) @pytest.mark.local_mode -def test_git_support_with_sklearn_ssh_passphrase_not_configured( - sagemaker_local_session, sklearn_full_version -): +def test_github_with_ssh_passphrase_not_configured(sagemaker_local_session, sklearn_full_version): script_path = "mnist.py" data_path = os.path.join(DATA_DIR, "sklearn_mnist") git_config = { @@ -213,11 +196,12 @@ def test_git_support_with_sklearn_ssh_passphrase_not_configured( "commit": PRIVATE_COMMIT_2FA, } source_dir = "sklearn" + sklearn = SKLearn( entry_point=script_path, role="SageMakerRole", source_dir=source_dir, - py_version=PYTHON_VERSION, + py_version="py3", # Scikit-learn supports only Python 3 train_instance_count=1, train_instance_type="local", sagemaker_session=sagemaker_local_session, @@ -227,6 +211,7 @@ def test_git_support_with_sklearn_ssh_passphrase_not_configured( ) train_input = "file://" + os.path.join(data_path, "train") test_input = "file://" + os.path.join(data_path, "test") + with pytest.raises(subprocess.CalledProcessError) as error: sklearn.fit({"train": train_input, "test": test_input}) assert "returned non-zero exit status" in str(error) @@ -234,7 +219,7 @@ def test_git_support_with_sklearn_ssh_passphrase_not_configured( @pytest.mark.local_mode @pytest.mark.skip("needs a secure authentication approach") -def test_git_support_codecommit_with_mxnet(sagemaker_local_session): +def test_codecommit(sagemaker_local_session): script_path = "mnist.py" data_path = os.path.join(DATA_DIR, "mxnet_mnist") git_config = { @@ -272,21 +257,7 @@ def test_git_support_codecommit_with_mxnet(sagemaker_local_session): with lock.lock(LOCK_PATH): try: - client = sagemaker_local_session.sagemaker_client - desc = client.describe_training_job(TrainingJobName=mx.latest_training_job.name) - model_data = desc["ModelArtifacts"]["S3ModelArtifacts"] - model = MXNetModel( - model_data, - "SageMakerRole", - entry_point=script_path, - source_dir=source_dir, - dependencies=dependencies, - py_version=PYTHON_VERSION, - sagemaker_session=sagemaker_local_session, - framework_version=MXNet.LATEST_VERSION, - git_config=git_config, - ) - predictor = model.deploy(1, "local") + predictor = mx.deploy(1, "local") data = numpy.zeros(shape=(1, 1, 28, 28)) result = predictor.predict(data)