diff --git a/01_preprocessing/data_preprocessing.ipynb b/01_preprocessing/data_preprocessing.ipynb index cb09fdf..d9a4b56 100644 --- a/01_preprocessing/data_preprocessing.ipynb +++ b/01_preprocessing/data_preprocessing.ipynb @@ -296,7 +296,7 @@ "kernelspec": { "display_name": "Python 3 (Data Science)", "language": "python", - "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/datascience-1.0" + "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:eu-west-1:470317259841:image/datascience-1.0" }, "language_info": { "codemirror_mode": { diff --git a/02_training/training.ipynb b/02_training/training.ipynb index 959f841..3451764 100644 --- a/02_training/training.ipynb +++ b/02_training/training.ipynb @@ -43,8 +43,8 @@ "metadata": {}, "outputs": [], "source": [ - "processed_data_prefix = f'{prefix}/outputs' #prefix generated by data processing module\n", - "processed_data_s3_uri = f's3://{default_bucket}/{processed_data_prefix}'" + "processed_data_s3_uri = f's3://{default_bucket}/{prefix}/outputs'\n", + "model_output_path = f's3://{default_bucket}/{prefix}/outputs/model'" ] }, { @@ -147,7 +147,7 @@ " hyperparameter_ranges,\n", " metric_definitions=metric_definitions,\n", " objective_type=\"Maximize\",\n", - " max_jobs=6,\n", + " max_jobs=2, # Low because of demo purposes\n", " max_parallel_jobs=2,\n", " base_tuning_job_name=\"cv-hpo\",\n", ")\n", @@ -185,6 +185,22 @@ "\n", "print(f\"\\nBest model artifact file is uploaded here: {best_model_uri}\")" ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copy the best model into the prefix used for this project. Note that's possible to directly reference the tuner's output, as above, but given the modularity of this workshop, the artefacts are copied to a known location in S3." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!aws s3 cp {best_model_uri} {model_output_path}/model.tar.gz" + ] } ], "metadata": { diff --git a/03_model_evaluation/model-evaluation-processing-job.ipynb b/03_model_evaluation/model-evaluation-processing-job.ipynb index b9c33f3..960bc12 100644 --- a/03_model_evaluation/model-evaluation-processing-job.ipynb +++ b/03_model_evaluation/model-evaluation-processing-job.ipynb @@ -96,14 +96,11 @@ "metadata": {}, "outputs": [], "source": [ - "s3_client = boto3.client('s3')\n", - "model_dir = s3_client.list_objects_v2(Bucket=bucket, Delimiter='/', Prefix=f'{prefix}/{prefix}')['CommonPrefixes'][-1]['Prefix']\n", - "\n", "s3_images = f's3://{bucket}/{prefix}/outputs/test/'\n", "s3_manifest = f's3://{bucket}/{prefix}/outputs/manifest'\n", - "s3_model = f's3://{bucket}/{model_dir}output'\n", + "s3_model = f's3://{bucket}/{prefix}/outputs/model/'\n", "\n", - "print(f's3_images: {s3_images},\\n s3_manifest: {s3_manifest},\\n s3_model: {s3_model}')" + "print(f's3_images: {s3_images},\\ns3_manifest: {s3_manifest},\\ns3_model: {s3_model}')" ] }, { @@ -281,20 +278,20 @@ "outputs": [], "source": [ "script_processor.run(\n", - " code='evaluation.py',\n", - " arguments=[\"--model-file\", \"model.tar.gz\"],\n", - " inputs=[ProcessingInput(source=s3_images, \n", - " destination=\"/opt/ml/processing/input/test\"),\n", - " ProcessingInput(source=s3_manifest, \n", - " destination=\"/opt/ml/processing/input/manifest\"),\n", - " ProcessingInput(source=s3_model, \n", - " destination=\"/opt/ml/processing/model\"),\n", - " ],\n", - " outputs=[\n", - " ProcessingOutput(output_name=\"evaluation\", source=\"/opt/ml/processing/evaluation\", \n", - " destination=s3_evaluation_output),\n", - " ]\n", - " )" + " code='evaluation.py',\n", + " arguments=[\"--model-file\", \"model.tar.gz\"],\n", + " inputs=[ProcessingInput(source=s3_images, \n", + " destination=\"/opt/ml/processing/input/test\"),\n", + " ProcessingInput(source=s3_manifest, \n", + " destination=\"/opt/ml/processing/input/manifest\"),\n", + " ProcessingInput(source=s3_model, \n", + " destination=\"/opt/ml/processing/model\"),\n", + " ],\n", + " outputs=[\n", + " ProcessingOutput(output_name=\"evaluation\", source=\"/opt/ml/processing/evaluation\", \n", + " destination=s3_evaluation_output),\n", + " ]\n", + ")" ] }, { @@ -359,7 +356,7 @@ "kernelspec": { "display_name": "Python 3 (Data Science)", "language": "python", - "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:us-east-1:081325390199:image/datascience-1.0" + "name": "python3__SAGEMAKER_INTERNAL__arn:aws:sagemaker:eu-west-1:470317259841:image/datascience-1.0" }, "language_info": { "codemirror_mode": {