CTDAS functionality

cases/icon-art-CTDAS/ICBC/icon_era5_inicond.sh

@@ -0,0 +1,179 @@
+#!/bin/bash
+
+cd {ERA5_folder}
+
+{cfg.cdo_nco_cmd}
+
+set -x
+
+# ---------------------------------
+# -- Pre-processing
+# ---------------------------------
+
+# -- Put all variables in the same file
+cdo -O merge {era5_ml_file} {era5_surf_file} era5_original.nc
+
+# -- Change variable and coordinates names to be consistent with ICON nomenclature
+cdo setpartabn,mypartab,convert era5_original.nc tmp.nc
+
+# -- Order the variables alphabetically
+ncks -O tmp.nc data_in.nc
+rm tmp.nc era5_original.nc
+
+# ---------------------------------
+# -- Re-mapping
+# ---------------------------------
+
+# -- Retrieve the dynamic horizontal grid
+cdo -s selgrid,2 {cfg.input_files_scratch_dynamics_grid_filename} triangular-grid.nc
+
+# -- Create the weights for remapping ERA5 latlon grid onto the triangular grid
+cdo gendis,triangular-grid.nc data_in.nc weights.nc
+
+# -- Extract the land-sea mask variable in input and output files
+cdo selname,LSM data_in.nc LSM_in.nc
+ncrename -h -v LSM,FR_LAND LSM_in.nc
+cdo selname,FR_LAND {cfg.input_files_scratch_extpar_filename} LSM_out_tmp.nc
+
+# -- Add time dimension to LSM_out.nc
+ncecat -O -u time LSM_out_tmp.nc LSM_out_tmp.nc
+ncks -h -A -v time LSM_in.nc LSM_out_tmp.nc
+
+# -- Create two different files for land- and sea-mask
+cdo -L setctomiss,0. -ltc,0.5  LSM_in.nc oceanmask_in.nc
+cdo -L setctomiss,0. -gec,0.5 LSM_in.nc landmask_in.nc
+cdo -L setctomiss,0. -ltc,0.5 LSM_out_tmp.nc oceanmask_out.nc
+cdo -L setctomiss,0. -gec,0.5 LSM_out_tmp.nc landmask_out.nc
+cdo setrtoc2,0.5,1.0,1,0 LSM_out_tmp.nc LSM_out.nc
+rm LSM_in.nc LSM_out_tmp.nc
+
+# -- Select surface sea variables defined only on sea
+ncks -O -h -v SST,CI data_in.nc datasea_in.nc
+
+# -- Select surface variables defined on both that must be remap differently on sea and on land
+ncks -O -h -v SKT,STL1,STL2,STL3,STL4,ALB_SNOW,W_SNOW,T_SNOW data_in.nc dataland_in.nc
+
+# -----------------------------------------------------------------------------
+# -- Remap land and ocean area differently for variables
+# -----------------------------------------------------------------------------
+
+# -- Ocean part
+# -----------------
+
+# -- Apply the ocean mask (by dividing)
+cdo div dataland_in.nc oceanmask_in.nc tmp1_land.nc
+cdo div datasea_in.nc oceanmask_in.nc tmp1_sea.nc
+
+# -- Set missing values to a distance-weighted average
+cdo setmisstodis tmp1_land.nc tmp2_land.nc
+cdo setmisstodis tmp1_sea.nc tmp2_sea.nc
+
+# -- Remap 
+cdo remapdis,triangular-grid.nc tmp2_land.nc tmp3_land.nc
+cdo remapdis,triangular-grid.nc tmp2_sea.nc tmp3_sea.nc
+
+# -- Apply the ocean mask to remapped variables (by dividing)
+cdo div tmp3_land.nc oceanmask_out.nc dataland_ocean_out.nc
+cdo div tmp3_sea.nc oceanmask_out.nc datasea_ocean_out.nc
+
+# -- Clean the repository
+rm tmp*.nc oceanmask*.nc
+
+# # -- Land part
+# # -----------------
+
+cdo div dataland_in.nc landmask_in.nc tmp1.nc
+cdo setmisstodis tmp1.nc tmp2.nc
+cdo remapdis,triangular-grid.nc tmp2.nc tmp3.nc
+cdo div tmp3.nc landmask_out.nc dataland_land_out.nc
+rm tmp*.nc landmask*.nc dataland_in.nc datasea_in.nc
+
+# -- merge remapped land and ocean part
+# --------------------------------------
+
+cdo ifthenelse LSM_out.nc dataland_land_out.nc dataland_ocean_out.nc dataland_out.nc
+rm dataland_ocean_out.nc dataland_land_out.nc
+
+# remap the rest and merge all files
+# --------------------------------------
+
+# -- Select all variables apart from these ones
+ncks -O -h -x -v SKT,STL1,STL2,STL3,STL4,SMIL1,SMIL2,SMIL3,SMIL4,ALB_SNOW,W_SNOW,T_SNOW,SST,CI,LSM data_in.nc datarest_in.nc
+
+# -- Remap
+cdo -s remapdis,triangular-grid.nc datarest_in.nc era5_final.nc
+rm datarest_in.nc
+
+# -- Fill NaN values for SST and CI
+cdo setmisstodis -selname,SST,CI datasea_ocean_out.nc dataland_ocean_out_filled.nc
+rm datasea_ocean_out.nc
+
+# -- Merge remapped files plus land sea mask from EXTPAR
+ncks -h -A dataland_out.nc era5_final.nc
+ncks -h -A dataland_ocean_out_filled.nc era5_final.nc
+ncks -h -A -v FR_LAND LSM_out.nc era5_final.nc
+ncrename -h -v FR_LAND,LSM era5_final.nc
+rm LSM_out.nc dataland_out.nc
+
+# ------------------------------------------------------------------------
+# -- Convert the (former) SWVLi variables to real soil moisture indices
+# ------------------------------------------------------------------------
+
+# -- Properties of IFS soil types (see Table 1 ERA5 Data documentation 
+# -- https://confluence.ecmwf.int/display/CKB/ERA5%3A+data+documentation)
+# Soil type   1     2     3     4     5     6     7
+wiltingp=(0 0.059 0.151 0.133 0.279 0.335 0.267 0.151) # wilting point
+fieldcap=(0 0.244 0.347 0.383 0.448 0.541 0.663 0.347) # field capacity
+
+ncks -O -h -v SMIL1,SMIL2,SMIL3,SMIL4,SLT data_in.nc swvl.nc
+rm data_in.nc
+
+# -- Loop over the soil types and apply the right constants
+smi_equation=""
+for ilev in {{1..4}}; do
+
+    smi_equation="${{smi_equation}}SMIL${{ilev}} = (SMIL${{ilev}} - ${{wiltingp[1]}}) / (${{fieldcap[1]}} - ${{wiltingp[1]}}) * (SLT==1)"
+    for ist in {{2..7}}; do
+        smi_equation="${{smi_equation}} + (SMIL${{ilev}} - ${{wiltingp[$ist]}}) / (${{fieldcap[$ist]}} - ${{wiltingp[$ist]}}) * (SLT==${{ist}})"
+    done
+    smi_equation="${{smi_equation}};"
+
+done
+
+cdo expr,"${{smi_equation}}" swvl.nc smil_in.nc
+rm swvl.nc
+
+# -- Remap SMIL variables
+cdo -s remapdis,triangular-grid.nc smil_in.nc smil_out.nc
+rm smil_in.nc
+
+# -- Overwrite the variables SMIL1,SMIL2,SMIL3,SMIL4
+ncks -A -v SMIL1,SMIL2,SMIL3,SMIL4 smil_out.nc era5_final.nc
+rm smil_out.nc
+
+# --------------------------------------
+# -- Create the LNSP variable
+# --------------------------------------
+
+# -- Apply logarithm to surface pressure
+cdo expr,'LNPS=ln(PS); Q=QV; GEOP_SFC=GEOSP' era5_final.nc tmp.nc
+
+# -- Put the new variable LNSP in the original file
+ncks -A -v LNPS,Q,GEOP_SFC tmp.nc era5_final.nc
+rm tmp.nc
+
+# ---------------------------------
+# -- Post-processing
+# ---------------------------------
+
+# -- Rename dimensions and order alphabetically
+ncrename -h -d cell,ncells era5_final.nc
+ncrename -h -d nv,vertices era5_final.nc
+ncks -O era5_final.nc {inicond_filename} 
+rm era5_final.nc
+
+# -- Clean the repository
+rm weights.nc
+rm triangular-grid.nc
+
+{cfg.cdo_nco_cmd_post}

cases/icon-art-CTDAS/ICBC/icon_era5_nudging.sh

@@ -0,0 +1,63 @@
+#!/bin/bash
+
+cd {ERA5_folder}
+
+{cfg.cdo_nco_cmd}
+
+# ---------------------------------
+# -- Pre-processing
+# ---------------------------------
+
+rm -f {filename}
+
+# -- Put all variables in the same file
+cdo -O merge {era5_ml_file} {era5_surf_file} era5_original.nc
+
+# -- Change variable and coordinates names to be consistent with ICON nomenclature
+cdo setpartabn,mypartab,convert era5_original.nc tmp.nc
+
+# -- Order the variables alphabetically 
+ncks -O tmp.nc data_in.nc
+rm tmp.nc era5_original.nc
+
+# ---------------------------------
+# -- Re-mapping
+# ---------------------------------
+
+# -- Retrieve the dynamic horizontal grid
+cdo -s selgrid,2 {cfg.input_files_scratch_dynamics_grid_filename} triangular-grid.nc
+
+# -- Create the weights for remapping ERA5 latlon grid onto the triangular grid
+cdo gendis,triangular-grid.nc data_in.nc weights.nc
+
+# -- Remap
+cdo -s remapdis,triangular-grid.nc data_in.nc era5_final.nc
+rm data_in.nc
+
+# --------------------------------------
+# -- Create the LNSP variable
+# --------------------------------------
+
+# -- Apply logarithm to surface pressure
+cdo expr,'LNPS=ln(PS); Q=QV; GEOP_SFC=GEOSP' era5_final.nc tmp.nc
+
+# -- Put the new variable LNSP in the original file
+ncks -A -v LNPS,Q,GEOP_SFC tmp.nc era5_final.nc
+rm tmp.nc
+
+# ---------------------------------
+# -- Post-processing
+# ---------------------------------
+
+# -- Rename dimensions and order alphabetically
+ncrename -h -d cell,ncells era5_final.nc
+ncrename -h -d nv,vertices era5_final.nc
+ncks -O era5_final.nc {filename}
+rm era5_final.nc
+
+
+# -- Clean the repository
+rm weights.nc
+rm triangular-grid.nc
+
+{cfg.cdo_nco_cmd_post}

cases/icon-art-CTDAS/ICBC/icon_era5_splitfiles.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+
+cd {ERA5_folder}
+
+{cfg.cdo_nco_cmd}
+
+# Loop over ml and surf files
+for ml_file in {ml_files}; do
+    # Convert GRIB to NetCDF for ml file and then process it
+    cdo -t ecmwf -f nc copy "${{ml_file}}" "${{ml_file%.grib}}.nc"
+
+    # Show timestamp and split for ml file
+    cdo showtimestamp "${{ml_file%.grib}}.nc" > list_ml.txt
+    cdo -splitsel,1 "${{ml_file%.grib}}.nc" split_ml_
+
+    times_ml=($(cat list_ml.txt))
+    x=0
+    for f in $(ls split_ml_*.nc); do
+        mv $f era5_ml_${{times_ml[$x]}}.nc
+        let x=$x+1
+    done
+done
+
+for surf_file in {surf_files}; do
+    # Convert GRIB to NetCDF for surf file and then process it
+    cdo -t ecmwf -f nc copy "${{surf_file}}" "${{surf_file%.grib}}.nc"
+
+    # Show timestamp and split for surf file
+    cdo showtimestamp "${{surf_file%.grib}}.nc" > list_surf.txt
+    cdo -splitsel,1 "${{surf_file%.grib}}.nc" split_surf_
+
+    times_surf=($(cat list_surf.txt))
+    y=0
+    for f in $(ls split_surf_*.nc); do
+        mv $f era5_surf_${{times_surf[$y]}}.nc
+        let y=$y+1
+    done
+done
+
+{cfg.cdo_nco_cmd_post}

cases/icon-art-CTDAS/ICBC/icon_species_inicond.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+
+cd {ERA5_folder}
+
+{cfg.cdo_nco_cmd}
+
+set -x
+
+# 1. Remap
+cdo griddes {inicond_filename} > triangular-grid.txt
+cdo remapbil,triangular-grid.txt {CAMS_file} cams_triangle.nc
+
+# 2. Write out the hybrid levels
+cat >CAMS_levels.txt <<EOL
+#
+# zaxisID 1
+#
+zaxistype = hybrid
+size      = 79
+name      = level
+longname  = "hybrid level at layer midpoints"
+units     = "level"
+levels    =  
+EOL
+ncks -v level cams_triangle.nc | sed -e '1,/data:/d' -e '$d' | sed 's/^[ ]*level = //' | sed 's/;$//'| tr -d '\n' >> CAMS_levels.txt
+echo '' >> CAMS_levels.txt
+echo 'vctsize   = 160' >> CAMS_levels.txt
+echo 'vct       = ' >> CAMS_levels.txt
+ncks -v ap cams_triangle.nc | sed -e '1,/data:/d' -e '$d' | sed 's/^[ ]*ap = //' | sed 's/;$//' | tr -d '\n' >> CAMS_levels.txt
+ncks -v bp cams_triangle.nc | sed -e '1,/data:/d' -e '$d' | sed 's/^[ ]*bp = //' | sed 's/;$//' | tr -d '\n' >> CAMS_levels.txt
+echo '' >> CAMS_levels.txt
+echo 'formula = "hyam hybm (mlev=ap+bp*aps)"' >> CAMS_levels.txt
+cdo setzaxis,CAMS_levels.txt cams_triangle.nc cams_withhybrid.nc
+
+# 3. Add required variables
+# --- CAMS
+ncrename -O -v Psurf,PS -d level,lev -v level,lev cams_withhybrid.nc
+ncap2 -s 'P0=1.0; lnsp=ln(PS); lev[lev]=array(0,1,$lev)' cams_withhybrid.nc -O cams_withhybrid_with_P.nc
+ncks -C -v P0,PS,lnsp,CO2,hyam,hybm,hyai,hybi,lev,clon,clat cams_withhybrid_with_P.nc -O cams_light.nc
+ncatted -a _FillValue,CO2,m,f,1.0e36 -O cams_light.nc
+# --- ERA5
+ncap2 -s 'P0=1.0; PS=PS(0,:)' {inicond_filename} -O data_in_with_P.nc
+ncks -C -v hyam,hybm,hyai,hybi,clon,clat,P0 data_in_with_P.nc -O era5_light.nc
+ncks -A -v PS cams_light.nc era5_light.nc
+
+# 4. Remap
+ncremap --no_stdin --vrt_fl=era5_light.nc -v CO2 cams_light.nc cams_remapped.nc
+ncrename -O -d nhym,lev cams_remapped.nc
+
+# 5. Place in inicond file
+ncks -A -v CO2 cams_remapped.nc {inicond_filename}
+ncap2 -s 'M_Air=28.9647; M_CO2=44.01; CO2_new[time,lev,ncells]=CO2*(M_CO2/M_Air)*(1-QV);' {inicond_filename}
+ncks -C -O -x -v CO2 {inicond_filename} {era5_cams_ini_file} # Remove old CO2 variable
+ncrename -v CO2_new,CO2 {era5_cams_ini_file} # Rename CO2_new to CO2
+ncrename -d .cell,ncells {era5_cams_ini_file}
+ncrename -d .nv,vertices {era5_cams_ini_file}
+
+{cfg.cdo_nco_cmd_post}