fix: run_notebook_test.sh skip incompatible and use excplicit python version for pip

Author: benieric

tests/scripts/run-notebook-test.sh

@@ -58,9 +58,22 @@ for env in base /home/ec2-user/anaconda3/envs/*; do
 
     sudo -u ec2-user -E sh -c 'source /home/ec2-user/anaconda3/bin/activate "$env"'
 
+    if [ $env = base ]; then
+        ENV_PYTHON=python
+    else
+        ENV_PYTHON="$env/bin/python"
+    fi
+
+    PYTHON_VERSION=$($ENV_PYTHON -c"import sys; print('{}{}'.format(sys.version_info[0], sys.version_info[1]))")
+    if [ $PYTHON_VERSION -lt 38 ]; then
+        echo "Skipping because $env uses py$PYTHON_VERSION which is incompatible with SageMaker Python SDK"
+        sudo -u ec2-user -E sh -c 'source /home/ec2-user/anaconda3/bin/deactivate'
+        continue
+    fi
+
     echo "Updating SageMaker Python SDK..."
-    pip install --upgrade pip
-    pip install "$TARBALL_DIRECTORY/sagemaker.tar.gz"
+    $ENV_PYTHON -m pip install --upgrade pip
+    $ENV_PYTHON -m pip install "$TARBALL_DIRECTORY/sagemaker.tar.gz"
 
     sudo -u ec2-user -E sh -c 'source /home/ec2-user/anaconda3/bin/deactivate'