Download script

The following is a Python script that downloads netCDF files provided by the reanalysis intercomparison dataset. It is left to the user to modify the script to download the desired data files.

Requirements: Python 3, wget

Those who wish to use the OPeNDAP functionalities can modify this script to suit their needs.

Usage: Save the following lines of code into a file, such as RIDscript.py, and then execute it in a terminal with Python using the command “python RIDscript.py”.

# This Python script downloads RID files

import os
import urllib.request
import time
from xml.dom import minidom

source = "https://www.jamstec.go.jp/RID/thredds/catalog/testAll/"
destination = "myRID"

# data_types = ["zonal", "single-level"]
data_types = ["single-level"]

# temporal_resolutions = ["monthly",'daily']
temporal_resolutions = ["monthly"]

# reanalyses = ['20CRv2','20CRv2c','20CRv3','CFSR','CFSv2','ERA-20C','ERA-Interim','ERA5','ERA5.1','JRA-25','JRA-3Q','JRA-55','JRA-55C','MERRA','MERRA-2','NCEP-DOE-R2','NCEP-DOE-R1']
reanalyses = ["ERA5", "JRA-55", "MERRA-2"]

if os.path.isdir(destination) == False:
    print("Destination directory does not exist, creating")
    os.makedirs(destination)

for data_type in data_types:
    
    if data_type == "zonal":
        # file_types = ['core','dt','fluxes','moist','mom','temp-pr','tem-qg','tem-thermo']
        file_types = ["core", "fluxes", "tem-qg"]
    if data_type == "single-level":
        # file_types = ["z500hPa",'z10hPa','t850hPa','q850hPa']
        file_types = ["z500hPa"]

    for temporal_resolution in temporal_resolutions:
        for file_type in file_types:
            for reanalysis in reanalyses:
                src = (
                    source
                    + data_type
                    + "/common-grid/"
                    + temporal_resolution
                    + "/"
                    + file_type
                    + "/"
                    + reanalysis
                    + "/"
                )
                print(src)

                response = urllib.request.urlopen(src)
                if response.getcode() == 200:
                    print("Received response from server")
                    dom = minidom.parse(response)

                    files = dom.getElementsByTagName("dataset")

                    for filenb, file in enumerate(files):
                        
                        time.sleep(1)
                        
                        filename = file.attributes["name"].value

                        if ".nc" in filename:
                            urlPath = file.attributes["urlPath"].value
                            downloadpath = (
                                "https://www.jamstec.go.jp/RID/thredds/fileServer/"
                                + urlPath
                            )
                            opendappath = (
                                "https://www.jamstec.go.jp/RID/thredds/dodsC/" + urlPath
                            )
                            print("Download URL:", downloadpath)
                            print("OpenDap path", opendappath)

                            # Download
                            file_input = downloadpath
                            file_output = localtree + filename

                            wget_code = (
                                "wget -c -N -O " + file_output + " " + file_input
                            )
                            print(wget_code)
                            os.system(wget_code)

                        else:
                            # Create folder if it does not exist
                            localtree = (
                                destination + "/" + file.attributes["name"].value
                            )
                            os.makedirs(localtree, exist_ok=True)