Using unique prefix accross notebooks: prefix = cv-sagemaker-immersionday

Author: marcelcastrobr

01_preprocessing/data_preprocessing.ipynb

@@ -92,7 +92,7 @@
     "role = sagemaker.get_execution_role()\n",
     "\n",
     "default_bucket = sagemaker_session.default_bucket() # or use your own custom bucket name\n",
-    "base_job_prefix = \"preprocess\" # or define your own prefix"
+    "base_job_prefix = \"cv-sagemaker-immersionday\" # or define your own prefix"
    ]
   },
   {

02_training/training.ipynb

@@ -31,7 +31,7 @@
     "account = sagemaker_session.account_id()\n",
     "region = sagemaker_session.boto_region_name\n",
     "default_bucket = sagemaker_session.default_bucket() # or use your own custom bucket name\n",
-    "prefix = 'postprocessing'"
+    "prefix = 'cv-sagemaker-immersionday'"
    ]
   },
   {
@@ -40,7 +40,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "processed_data_prefix = f'preprocess/outputs' #prefix generated by data processing module\n",
+    "processed_data_prefix = f'{prefix}/outputs' #prefix generated by data processing module\n",
     "processed_data_s3_uri = f's3://{default_bucket}/{processed_data_prefix}'"
    ]
   },

03_model_evaluation/confusion_matrix.png

@@ -73,7 +73,7 @@
     "account = sess.account_id()\n",
     "region = sess.boto_region_name\n",
     "bucket = sess.default_bucket() # or use your own custom bucket name\n",
-    "prefix = 'postprocessing'"
+    "prefix = 'cv-sagemaker-immersionday'"
    ]
   },
   {
@@ -97,10 +97,10 @@
    "outputs": [],
    "source": [
     "s3_client = boto3.client('s3')\n",
-    "model_dir = s3_client.list_objects_v2(Bucket=bucket, Delimiter='/', Prefix=f'{prefix}/')['CommonPrefixes'][-1]['Prefix']\n",
+    "model_dir = s3_client.list_objects_v2(Bucket=bucket, Delimiter='/', Prefix=f'{prefix}/{prefix}')['CommonPrefixes'][-1]['Prefix']\n",
     "\n",
-    "s3_images = f's3://{bucket}/preprocess/outputs/test/'\n",
-    "s3_manifest = f's3://{bucket}/preprocess/outputs/manifest'\n",
+    "s3_images = f's3://{bucket}/{prefix}/outputs/test/'\n",
+    "s3_manifest = f's3://{bucket}/{prefix}/outputs/manifest'\n",
     "s3_model = f's3://{bucket}/{model_dir}output'\n",
     "\n",
     "print(f's3_images: {s3_images},\\n s3_manifest: {s3_manifest},\\n s3_model: {s3_model}')"
@@ -345,6 +345,13 @@
    "source": [
     "![confusion_matrix.png](./confusion_matrix.png)"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

03_model_evaluation/model-evaluation-processing-job.ipynb

@@ -81,8 +81,8 @@
     "\n",
     "# Feel free to specify different bucket/folders here if you wish.\n",
     "bucket = sess.default_bucket()\n",
-    "prefix = \"preprocess\"\n",
-    "model_prefix = 'postprocessing-model-evaluation'\n",
+    "prefix = 'cv-sagemaker-immersionday'\n",
+    "\n",
     "\n",
     "TF_FRAMEWORK_VERSION = '2.4.1'\n",
     "ENDPOINT_INSTANCE_TYPE = 'ml.c5.4xlarge'"
@@ -113,8 +113,8 @@
    "source": [
     "# Get S3 URI of the model artifact\n",
     "s3_client = boto3.client('s3')\n",
-    "postproc_dir = s3_client.list_objects_v2(Bucket=bucket, Delimiter='/', Prefix=f'{model_prefix}/')['CommonPrefixes'][-1]['Prefix']\n",
-    "bird_model_path = f's3://{bucket}/{postproc_dir}output/model.tar.gz'\n",
+    "model_dir = s3_client.list_objects_v2(Bucket=bucket, Delimiter='/', Prefix=f'{prefix}/{prefix}')['CommonPrefixes'][-1]['Prefix']\n",
+    "bird_model_path = f's3://{bucket}/{model_dir}output/model.tar.gz'\n",
     "print(f'bird_model_path: {bird_model_path}')"
    ]
   },
@@ -275,7 +275,7 @@
     "\n",
     "#Update the below variable with your endpoint name from previous cell output\n",
     "#tf_endpoint_name='<SAGEMAKER DEPLOYED ENDPOINT NAME>'\n",
-    "tf_endpoint_name = 'tensorflow-inference-2022-11-02-13-39-59-198'\n",
+    "tf_endpoint_name = 'tensorflow-inference-2022-11-03-12-22-07-444'\n",
     "\n",
     "serializer = IdentitySerializer(content_type=\"application/x-image\")\n",
     "deserializer = JSONDeserializer(accept='application/json')\n",
@@ -321,10 +321,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "outputs_dir = s3_client.list_objects_v2(Bucket=bucket, Delimiter='/', Prefix=f'{prefix}/outputs/')['CommonPrefixes'][0]['Prefix']\n",
-    "sample_images_uri = f'{outputs_dir}test'\n",
+    "sample_images = cv_utils.get_n_random_images(bucket,prefix=f'{prefix}/outputs/test',n=2)\n",
     "\n",
-    "sample_images = cv_utils.get_n_random_images(bucket,prefix=sample_images_uri,n=2)\n",
     "local_paths = cv_utils.download_images_locally(bucket,sample_images)\n",
     "print(local_paths)"
    ]