diff --git a/notebooks/utils.py b/notebooks/utils.py index c6f9390..10b4615 100644 --- a/notebooks/utils.py +++ b/notebooks/utils.py @@ -81,6 +81,9 @@ def copy_s3_files(in_fileglob, out_folder): in_fileglob: s3 uri of flies (wild card can be used) out_folder: local path where data will be stored ''' + if os.path.isdir(out_folder) == 0: + mode = 0o777 + os.makedirs(out_folder, mode, exist_ok = False) matching_keys = find_matching_s3_keys(in_fileglob) in_bucket_name = _split_s3_uri(in_fileglob)[0] out_scheme = urlparse(out_folder).scheme @@ -120,7 +123,6 @@ def load_data(inpath): for key in keys: file = key.split('/')[-1] if os.path.exists(os.path.join(temp_path, file)) == 0: - print(os.path.join(s3dir, file)) copy_s3_files(os.path.join(s3dir, file), temp_path) else: print(key, ' already exist') diff --git a/notebooks/worksheet1.ipynb b/notebooks/worksheet1.ipynb index 02ac280..b6869eb 100755 --- a/notebooks/worksheet1.ipynb +++ b/notebooks/worksheet1.ipynb @@ -317,9 +317,19 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": false - }, + "metadata": {}, + "outputs": [], + "source": [ + "# download data from S3 bucket in data directory\n", + "from utils import copy_s3_files, flush_data\n", + "\n", + "copy_s3_files('s3://ias-pyprecis/data/sample_data.nc', 'data/')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ "# import the necessary modules\n", @@ -331,7 +341,7 @@ "%matplotlib inline \n", "\n", "# provide the path of your sample data\n", - "sample_data = '/project/ciid/projects/PRECIS/worksheets/data/sample_data.nc'\n", + "sample_data = 'data/sample_data.nc'\n", "\n", "# Constraint the reading to a single variable and load it into an Iris cube\n", "cube = iris.load_cube(sample_data)\n", @@ -454,18 +464,29 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": false - }, + "metadata": {}, + "outputs": [], + "source": [ + "# download data from S3 buket to data directory\n", + "from utils import copy_s3_files\n", + "\n", + "copy_s3_files('s3://ias-pyprecis/data/pp/cahpa/*', 'data/pp/cahpa/')\n", + "copy_s3_files('s3://ias-pyprecis/data/pp/cahpb/*', 'data/pp/cahpb/')" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], "source": [ - "datadir = '/project/ciid/projects/PRECIS/worksheets/data'\n", + "datadir = 'data/'\n", "\n", "rim_width = 8 # width of rim (in number of grid boxes)\n", "\n", "for runid in ['cahpa', 'cahpb']:\n", " ppdir = os.path.join(datadir, 'pp', runid)\n", - " \n", + "\n", " # find all the files from which to remove the rim\n", " file_list = glob.glob(ppdir + '/*pm[ghij]*.pp')\n", " \n", @@ -483,13 +504,28 @@ " # add meta data stating that rim has been removed\n", " rrcube.attributes['rim_removed'] = '{} point rim removed'.format(rim_width)\n", " trimmed_cubes.append(rrcube)\n", + " \n", " rrcubes = iris.cube.CubeList(trimmed_cubes)\n", " # Write out the trimmed data file\n", - " outfile = os.path.join(datadir, 'historical', runid + '.mon.1961_1990.rr.nc')\n", + " #outfile = os.path.join(datadir, 'historical', runid + '.mon.1961_1990.rr.nc')\n", + " outfile = os.path.join(datadir, runid + '.mon.1961_1990.rr.nc')\n", + "\n", " iris.save(rrcubes, outfile)\n", " print('Saved {}'.format(outfile))" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Delete pp data from the disk\n", + "from utils import flush_data\n", + "flush_data('data/pp/cahpa/*')\n", + "flush_data('data/pp/cahpb/*')" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -564,14 +600,14 @@ "\n", "for runid in ['cahpa', 'cahpb']:\n", " # Get data directory\n", - " infile = os.path.join(datadir, 'historical', runid + '.mon.1961_1990.rr.nc')\n", + " infile = os.path.join(datadir, runid + '.mon.1961_1990.rr.nc')\n", " # This will load all the variables in the file into a CubeList\n", " datacubes = iris.load(infile)\n", " for cube in datacubes:\n", " # get the STASH code\n", " cubeSTASH = cube.attributes['STASH']\n", " # Make the output file name\n", - " outfile = os.path.join(datadir, 'historical', runid + '.mon.1961_1990.' + stash_codes[str(cubeSTASH)] + '.rr.nc')\n", + " outfile = os.path.join(datadir, runid + '.mon.1961_1990.' + stash_codes[str(cubeSTASH)] + '.rr.nc')\n", " # Save the file\n", " iris.save(cube, outfile)\n", " print('Saved {}'.format(outfile)) " @@ -653,10 +689,11 @@ } ], "metadata": { + "instance_type": "ml.t3.medium", "kernelspec": { - "display_name": "pyprecis-environment", + "display_name": "Python [conda env:pyprecis-environment] (arn:aws:sagemaker:eu-west-2:198477955030:image-version/abtraining/1)", "language": "python", - "name": "pyprecis-environment" + "name": "conda-env-pyprecis-environment-py__SAGEMAKER_INTERNAL__arn:aws:sagemaker:eu-west-2:198477955030:image-version/abtraining/1" }, "language_info": { "codemirror_mode": { @@ -676,5 +713,5 @@ } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/notebooks/worksheet2.ipynb b/notebooks/worksheet2.ipynb index 8e2ce92..36fe9e5 100755 --- a/notebooks/worksheet2.ipynb +++ b/notebooks/worksheet2.ipynb @@ -75,7 +75,8 @@ "import iris.quickplot as qplt\n", "import cartopy.crs as ccrs\n", "from mpl_toolkits.axes_grid1 import AxesGrid\n", - "from cartopy.mpl.geoaxes import GeoAxes" + "from cartopy.mpl.geoaxes import GeoAxes\n", + "from utils import copy_s3_files, flush_data" ] }, { @@ -104,6 +105,16 @@ "Before running the code, take a look at it line-by-line to understand what steps are being taken. Then click in the box and press ctrl + enter to run the code." ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# we first need to download APHRODITE data\n", + "copy_s3_files('s3://ias-pyprecis/data/APHRODITE/*.nc', 'data/APHRODITE/')" + ] + }, { "cell_type": "code", "execution_count": null, @@ -111,14 +122,14 @@ "outputs": [], "source": [ "# Provide the names of the directories where the netCDF model files are stored\n", - "DATADIR = '/project/ciid/projects/PRECIS/worksheets/data/'\n", + "DATADIR = 'data/'\n", "\n", "# Load and print the HadCM3Q0 (cahpa) model cube data\n", - "infile = os.path.join(DATADIR, 'historical', 'cahpa.mon.1961_1990.pr.rr.nc')\n", + "infile = os.path.join(DATADIR, 'cahpa.mon.1961_1990.pr.rr.nc')\n", "cahpaData = iris.load_cube(infile)\n", "\n", "# Load and print the ECHAM5 (cahpb) model cube data\n", - "infile = os.path.join(DATADIR, 'historical', 'cahpb.mon.1961_1990.pr.rr.nc')\n", + "infile = os.path.join(DATADIR, 'cahpb.mon.1961_1990.pr.rr.nc')\n", "cahpbData = iris.load_cube(infile)\n", "\n", "# Load and print the APHRODITE observation cube data\n", @@ -309,7 +320,7 @@ "cahpaData.remove_coord('forecast_period')\n", "cahpaData.remove_coord('forecast_reference_time')\n", "# Save the new cube as a new netCDF file\n", - "outfile = os.path.join(DATADIR, 'historical', 'cahpa.mon.1961_1990.pr.rr.mmday-1.nc')\n", + "outfile = os.path.join(DATADIR, 'cahpa.mon.1961_1990.pr.rr.mmday-1.nc')\n", "iris.save(cahpaData, outfile)" ] }, @@ -338,7 +349,7 @@ "# Remove extraneous cube metadata. This helps make cube comparisons easier later.\n", "\n", "# Save the new cube as a new netCDF file using the `outfile` filename we've provided below!\n", - "outfile = os.path.join(DATADIR, 'historical', 'cahpb.mon.1961_1990.pr.rr.mmday-1.nc')\n", + "outfile = os.path.join(DATADIR, 'cahpb.mon.1961_1990.pr.rr.mmday-1.nc')\n", "\n" ] }, @@ -373,7 +384,7 @@ "\n", "# Loop through two model runs\n", "for jobid in ['cahpa', 'cahpb']:\n", - " infile = os.path.join(DATADIR, 'historical', jobid + '.mon.1961_1990.pr.rr.mmday-1.nc')\n", + " infile = os.path.join(DATADIR, jobid + '.mon.1961_1990.pr.rr.mmday-1.nc')\n", "\n", " # Load the data\n", " data = iris.load_cube(infile)\n", @@ -437,6 +448,16 @@ "Follow step d) and complete the code yourself. The file name to load is: `aphro.mon.1961_1990.nc`. We've given you the infile and outfile names to make sure you load and save it in the right place for later!" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# we first need to download APHRODITE data\n", + "copy_s3_files('s3://ias-pyprecis/data/climatology/*.nc', 'data/climatology/')" + ] + }, { "cell_type": "code", "execution_count": null, @@ -444,7 +465,7 @@ "outputs": [], "source": [ "# Directory names where data is read from and stored to\n", - "infile = os.path.join(DATADIR, 'APHRODITE', 'aphro.mon.1961_1990.nc')\n", + "infile = os.path.join(DATADIR, 'climatology', 'aphro.mon.1961_1990.nc')\n", "\n", "\n", "# Load the aphrodite data\n", @@ -460,7 +481,7 @@ "\n", "\n", "# save the seasonal mean cube as a NetCDF file\n", - "outfile = os.path.join(DATADIR, 'climatology', 'aphro.OND.mean.1961_1990.pr.mmday-1.nc')\n", + "outfile = os.path.join(DATADIR, 'aphro.OND.mean.1961_1990.pr.mmday-1.nc')\n", "\n", "\n", "# print the APHRODITE seasonal mean cube\n", @@ -550,7 +571,7 @@ "outputs": [], "source": [ "# Directory name where data is read from\n", - "indir = os.path.join(DATADIR, 'climatology')\n", + "indir = DATADIR\n", "\n", "# load cahpa model data\n", "infile = os.path.join(indir, 'cahpa.OND.mean.1961_1990.pr.mmday-1.nc')\n", @@ -663,10 +684,11 @@ } ], "metadata": { + "instance_type": "ml.t3.medium", "kernelspec": { - "display_name": "pyprecis-environment", + "display_name": "Python [conda env:pyprecis-environment] (arn:aws:sagemaker:eu-west-2:198477955030:image-version/abtraining/1)", "language": "python", - "name": "pyprecis-environment" + "name": "conda-env-pyprecis-environment-py__SAGEMAKER_INTERNAL__arn:aws:sagemaker:eu-west-2:198477955030:image-version/abtraining/1" }, "language_info": { "codemirror_mode": { @@ -686,5 +708,5 @@ } }, "nbformat": 4, - "nbformat_minor": 1 + "nbformat_minor": 4 } diff --git a/notebooks/worksheet3.ipynb b/notebooks/worksheet3.ipynb index 058265d..1619ceb 100755 --- a/notebooks/worksheet3.ipynb +++ b/notebooks/worksheet3.ipynb @@ -63,9 +63,11 @@ "import cartopy.crs as ccrs\n", "from mpl_toolkits.axes_grid1 import AxesGrid\n", "from cartopy.mpl.geoaxes import GeoAxes\n", + "from utils import copy_s3_files, flush_data\n", + "\n", "\n", "# Provide the names of the directories where the netCDF model files are stored\n", - "DATADIR = '/project/ciid/projects/PRECIS/worksheets/data/'\n", + "DATADIR = 'data/'\n", "\n", "# Directory name where data is read from\n", "HISTDIR = os.path.join(DATADIR, 'historical')\n", @@ -179,7 +181,7 @@ "outputs": [], "source": [ "# Load the HadCM3Q0 (cahpa) model cube data as need grid information from it\n", - "infile = os.path.join(HISTDIR, 'cahpa.mon.1961_1990.pr.rr.nc')\n", + "infile = os.path.join(DATADIR, 'cahpa.mon.1961_1990.pr.rr.nc')\n", "cahpa_cube = iris.load_cube(infile)\n", "\n", "pole_lat = cahpa_cube.coord_system().grid_north_pole_latitude\n", @@ -226,8 +228,8 @@ "\n", "for jobid in ['cahpa', 'cahpb']:\n", " # Directory name where data are read from and stored to\n", - " infile = os.path.join(DATADIR, 'historical', jobid + '.mon.1961_1990.pr.rr.mmday-1.nc')\n", - " \n", + " infile = os.path.join(DATADIR, jobid + '.mon.1961_1990.pr.rr.mmday-1.nc')\n", + " print(infile)\n", " # Load the baseline precipitation data using the KL_constraint - the command below\n", " # loads the data into a cube constrained by the area chosen\n", " data = iris.load_cube(infile)\n", @@ -236,7 +238,7 @@ " grid_latitude=rotated_lats)\n", "\n", " # save the constrained cube\n", - " outfile = os.path.join(DATADIR, 'historical', jobid + '.mon.1961_1990.pr.rr.mmday-1.KL.nc')\n", + " outfile = os.path.join(DATADIR, jobid + '.mon.1961_1990.pr.rr.mmday-1.KL.nc')\n", " iris.save(data_KL, outfile)\n", " print('Saved: {}'.format(outfile))" ] @@ -302,7 +304,7 @@ "source": [ "for jobid in ['cahpa', 'cahpb']:\n", " # Set up the path to the data\n", - " infile = os.path.join(DATADIR, 'historical', jobid + '.mon.1961_1990.pr.rr.mmday-1.KL.nc')\n", + " infile = os.path.join(DATADIR, jobid + '.mon.1961_1990.pr.rr.mmday-1.KL.nc')\n", " \n", " # Load the data extracted around Kuala Lumpur created in previous step\n", " data = iris.load_cube(infile)\n", @@ -745,6 +747,17 @@ "**j) Plot a series of figures** that shows 1) the monthly cycles of temperature and rainfall comparing the 6 models and the observations; and 2) the monthly differences between the models and observations" ] }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# we first need to download CRU and netcdf data\n", + "copy_s3_files('s3://ias-pyprecis/data/CRU/*.nc', 'data/CRU/')\n", + "copy_s3_files('s3://ias-pyprecis/data/netcdf/*.nc', 'data/netcdf/')" + ] + }, { "cell_type": "code", "execution_count": null, @@ -755,7 +768,7 @@ "Here are some useful varibles you might like to use in your scripts\n", "'''\n", "# Some helpful data locations\n", - "DATADIR = '/project/precis/worksheets/data'\n", + "DATADIR = 'data'\n", "APHRODIR = os.path.join(DATADIR, 'APHRODITE')\n", "CRUDIR = os.path.join(DATADIR, 'CRU')\n", "CLIMDIR = os.path.join(DATADIR, 'climatology')\n", @@ -991,10 +1004,11 @@ } ], "metadata": { + "instance_type": "ml.t3.medium", "kernelspec": { - "display_name": "pyprecis-environment", + "display_name": "Python [conda env:pyprecis-environment] (arn:aws:sagemaker:eu-west-2:198477955030:image-version/abtraining/1)", "language": "python", - "name": "pyprecis-environment" + "name": "conda-env-pyprecis-environment-py__SAGEMAKER_INTERNAL__arn:aws:sagemaker:eu-west-2:198477955030:image-version/abtraining/1" }, "language_info": { "codemirror_mode": { @@ -1014,5 +1028,5 @@ } }, "nbformat": 4, - "nbformat_minor": 1 + "nbformat_minor": 4 } diff --git a/notebooks/worksheet4.ipynb b/notebooks/worksheet4.ipynb index f2ef03d..4cea79d 100755 --- a/notebooks/worksheet4.ipynb +++ b/notebooks/worksheet4.ipynb @@ -68,7 +68,7 @@ "import numpy.ma as ma\n", "\n", "# Some helpful data locations\n", - "DATADIR = '/project/ciid/projects/PRECIS/worksheets/data'\n", + "DATADIR = 'data'\n", "CLIMDIR = os.path.join(DATADIR, 'climatology')\n", "HISTDIR = os.path.join(DATADIR, 'historical')\n", "FUTRDIR = os.path.join(DATADIR, 'future')\n", @@ -442,9 +442,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": false - }, + "metadata": {}, "outputs": [], "source": [ "# Read in the monthly series\n", @@ -588,10 +586,11 @@ } ], "metadata": { + "instance_type": "ml.t3.medium", "kernelspec": { - "display_name": "Python 3", + "display_name": "Python [conda env:pyprecis-environment] (arn:aws:sagemaker:eu-west-2:198477955030:image-version/abtraining/1)", "language": "python", - "name": "python3" + "name": "conda-env-pyprecis-environment-py__SAGEMAKER_INTERNAL__arn:aws:sagemaker:eu-west-2:198477955030:image-version/abtraining/1" }, "language_info": { "codemirror_mode": { @@ -611,5 +610,5 @@ } }, "nbformat": 4, - "nbformat_minor": 1 + "nbformat_minor": 4 } diff --git a/notebooks/worksheet5.ipynb b/notebooks/worksheet5.ipynb index 0285f90..6261df0 100755 --- a/notebooks/worksheet5.ipynb +++ b/notebooks/worksheet5.ipynb @@ -81,7 +81,7 @@ "from iris.analysis import Aggregator\n", "\n", "# Some helpful data locations\n", - "DATADIR = '/project/ciid/projects/PRECIS/worksheets/data'\n", + "DATADIR = 'data'\n", "CLIMDIR = os.path.join(DATADIR, 'climatology')\n", "HISTDIR = os.path.join(DATADIR, 'historical')\n", "FUTRDIR = os.path.join(DATADIR, 'future')\n", @@ -712,9 +712,7 @@ { "cell_type": "code", "execution_count": null, - "metadata": { - "scrolled": false - }, + "metadata": {}, "outputs": [], "source": [ "# HINT: The filenames have the following pattern: runid + '.day.pc95.bias.pr.mmday-1.nc'\n", @@ -793,10 +791,11 @@ } ], "metadata": { + "instance_type": "ml.t3.medium", "kernelspec": { - "display_name": "Python 3", + "display_name": "Python [conda env:pyprecis-environment] (arn:aws:sagemaker:eu-west-2:198477955030:image-version/abtraining/1)", "language": "python", - "name": "python3" + "name": "conda-env-pyprecis-environment-py__SAGEMAKER_INTERNAL__arn:aws:sagemaker:eu-west-2:198477955030:image-version/abtraining/1" }, "language_info": { "codemirror_mode": { @@ -816,5 +815,5 @@ } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 } diff --git a/notebooks/worksheet6.ipynb b/notebooks/worksheet6.ipynb index f4be6f1..1d8f00a 100755 --- a/notebooks/worksheet6.ipynb +++ b/notebooks/worksheet6.ipynb @@ -86,7 +86,7 @@ "from iris.analysis import Aggregator\n", "\n", "# Some helpful data locations\n", - "DATADIR = '/project/ciid/projects/PRECIS/worksheets/data'\n", + "DATADIR = 'data'\n", "PPDIR = os.path.join(DATADIR, 'pp')\n", "CLIMDIR = os.path.join(DATADIR, 'climatology')\n", "HISTDIR = os.path.join(DATADIR, 'historical')\n", @@ -230,10 +230,11 @@ } ], "metadata": { + "instance_type": "ml.t3.medium", "kernelspec": { - "display_name": "Python 3", + "display_name": "Python [conda env:pyprecis-environment] (arn:aws:sagemaker:eu-west-2:198477955030:image-version/abtraining/1)", "language": "python", - "name": "python3" + "name": "conda-env-pyprecis-environment-py__SAGEMAKER_INTERNAL__arn:aws:sagemaker:eu-west-2:198477955030:image-version/abtraining/1" }, "language_info": { "codemirror_mode": { @@ -245,9 +246,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.6" + "version": "3.6.10" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 }