Merge pull request #11 from absynthe/fix/connect-02-03

Fix: connect 02 and 03

Author: absynthe

01_preprocessing/data_preprocessing.ipynb

@@ -296,7 +296,7 @@
   "kernelspec": {
    "display_name": "Python 3 (Data Science)",
    "language": "python",
-   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/datascience-1.0"
+   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:eu-west-1:470317259841:image/datascience-1.0"
   },
   "language_info": {
    "codemirror_mode": {

02_training/training.ipynb

@@ -43,8 +43,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "processed_data_prefix = f'{prefix}/outputs' #prefix generated by data processing module\n",
-    "processed_data_s3_uri = f's3://{default_bucket}/{processed_data_prefix}'"
+    "processed_data_s3_uri = f's3://{default_bucket}/{prefix}/outputs'\n",
+    "model_output_path = f's3://{default_bucket}/{prefix}/outputs/model'"
    ]
   },
   {
@@ -147,7 +147,7 @@
     "    hyperparameter_ranges,\n",
     "    metric_definitions=metric_definitions,\n",
     "    objective_type=\"Maximize\",\n",
-    "    max_jobs=6,\n",
+    "    max_jobs=2, # Low because of demo purposes\n",
     "    max_parallel_jobs=2,\n",
     "    base_tuning_job_name=\"cv-hpo\",\n",
     ")\n",
@@ -185,6 +185,22 @@
     "\n",
     "print(f\"\\nBest model artifact file is uploaded here: {best_model_uri}\")"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Copy the best model into the prefix used for this project. Note that's possible to directly reference the tuner's output, as above, but given the modularity of this workshop, the artefacts are copied to a known location in S3."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "!aws s3 cp {best_model_uri} {model_output_path}/model.tar.gz"
+   ]
   }
  ],
  "metadata": {

03_model_evaluation/model-evaluation-processing-job.ipynb

@@ -96,14 +96,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "s3_client = boto3.client('s3')\n",
-    "model_dir = s3_client.list_objects_v2(Bucket=bucket, Delimiter='/', Prefix=f'{prefix}/{prefix}')['CommonPrefixes'][-1]['Prefix']\n",
-    "\n",
     "s3_images = f's3://{bucket}/{prefix}/outputs/test/'\n",
     "s3_manifest = f's3://{bucket}/{prefix}/outputs/manifest'\n",
-    "s3_model = f's3://{bucket}/{model_dir}output'\n",
+    "s3_model = f's3://{bucket}/{prefix}/outputs/model/'\n",
     "\n",
-    "print(f's3_images: {s3_images},\\n s3_manifest: {s3_manifest},\\n s3_model: {s3_model}')"
+    "print(f's3_images: {s3_images},\\ns3_manifest: {s3_manifest},\\ns3_model: {s3_model}')"
    ]
   },
   {
@@ -281,20 +278,20 @@
    "outputs": [],
    "source": [
     "script_processor.run(\n",
-    "                        code='evaluation.py',\n",
-    "                        arguments=[\"--model-file\", \"model.tar.gz\"],\n",
-    "                        inputs=[ProcessingInput(source=s3_images, \n",
-    "                                                destination=\"/opt/ml/processing/input/test\"),\n",
-    "                                ProcessingInput(source=s3_manifest, \n",
-    "                                                destination=\"/opt/ml/processing/input/manifest\"),\n",
-    "                                ProcessingInput(source=s3_model, \n",
-    "                                                destination=\"/opt/ml/processing/model\"),\n",
-    "                               ],\n",
-    "                        outputs=[\n",
-    "                            ProcessingOutput(output_name=\"evaluation\", source=\"/opt/ml/processing/evaluation\", \n",
-    "                                             destination=s3_evaluation_output),\n",
-    "                        ]\n",
-    "                    )"
+    "    code='evaluation.py',\n",
+    "    arguments=[\"--model-file\", \"model.tar.gz\"],\n",
+    "    inputs=[ProcessingInput(source=s3_images, \n",
+    "                            destination=\"/opt/ml/processing/input/test\"),\n",
+    "            ProcessingInput(source=s3_manifest, \n",
+    "                            destination=\"/opt/ml/processing/input/manifest\"),\n",
+    "            ProcessingInput(source=s3_model, \n",
+    "                            destination=\"/opt/ml/processing/model\"),\n",
+    "           ],\n",
+    "    outputs=[\n",
+    "        ProcessingOutput(output_name=\"evaluation\", source=\"/opt/ml/processing/evaluation\", \n",
+    "                         destination=s3_evaluation_output),\n",
+    "    ]\n",
+    ")"
    ]
   },
   {
@@ -359,7 +356,7 @@
   "kernelspec": {
    "display_name": "Python 3 (Data Science)",
    "language": "python",
-   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/datascience-1.0"
+   "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:eu-west-1:470317259841:image/datascience-1.0"
   },
   "language_info": {
    "codemirror_mode": {