adding files to new git repo

3_SummaActors_Run/copernicus/caf-application.conf

+caf {
+  # Parameters selecting a default scheduler.
+  scheduler {
+    max-threads = 8
+  }
+}
\ No newline at end of file

3_SummaActors_Run/copernicus/run_allHRUs.sh

+#!/bin/bash
+#SBATCH --cpus-per-task=8
+#SBATCH --time=24:00:00
+#SBATCH --mem=32G
+#SBATCH --job-name=SummaActorsOutputBugTest
+#SBATCH --mail-user=kyle.klenk@usask.ca
+#SBATCH --mail-type=ALL
+#SBATCH --output=/scratch/gwf/gwf_cmt/kck540/summaActors/slurm/slurm-%A_%a.out
+#SBATCH --array=0-258%4
+#SBATCH --account=hpc_c_giws_clark
+
+# EXAMPLE: sbatch --array=0-258%4 run_allHRUs.sh
+
+# Define the GRU settings
+gruMax=517315 # North America, Merit Hydro basins
+gruCount=2000
+
+# Get the array ID for further use
+offset=$SLURM_ARRAY_TASK_ID 
+
+# Start at 1 for array ID 1, 2022 for array ID 2, etc
+gruStart=$(( 1 + gruCount*offset ))
+check=$(( $gruStart + $gruCount ))
+# Check that we don't specify too many basins
+if [ $check -gt $gruMax ]
+then
+    gruCount=$(( gruMax-gruStart+1 ))
+fi
+
+/globalhome/kck540/HPC/SummaActors/bin/summaMain -g ${gruStart} -c ${gruCount} -m /project/gwf/gwf_cmt/kck540/domain_NorthAmerica/settings/SUMMA/fileManager.txt
\ No newline at end of file

3_SummaActors_Run/copernicus/run_client_server.sh

+#!/bin/bash
+#SBATCH --cpus-per-task=1
+#SBATCH --time=8:00:00
+#SBATCH --mem=2G
+#SBATCH --job-name=SummaActorsOutputBugTest
+#SBATCH --mail-user=kyle.klenk@usask.ca
+#SBATCH --mail-type=ALL
+#SBATCH --output=/scratch/gwf/gwf_cmt/kck540/summaActors/slurm/slurm-%A_%a.out
+#SBATCH --account=hpc_c_giws_clark
+
+# EXAMPLE: sbatch --array=0-50 run_server_client.sh
+
+gruMax=517315 # North America, Merit Hydro basins
+gruCount=50000 # The number of GRUs you want to compute
+startGRU=1   # The starting index of the GRU you would like to compute
+offset=$SLURM_ARRAY_TASK_ID
+
+if [ $offset -eq 0 ]
+then
+    /globalhome/kck540/HPC/SummaActors/bin/summaMain -s -p 4444 -g 1 -c $gruCount -m /project/gwf/gwf_cmt/kck540/domain_NorthAmerica/settings/SUMMA/fileManager.txt
+else
+    /globalhome/kck540/HPC/SummaActors/bin/summaMain -p 4444 -j $offset -m /project/gwf/gwf_cmt/kck540/domain_NorthAmerica/settings/SUMMA/fileManager.txt
+fi

3_SummaActors_Run/copernicus/run_one.sh

+#!/bin/bash
+#SBATCH --cpus-per-task=8
+#SBATCH --time=36:00:00
+#SBATCH --mem=32G
+#SBATCH --job-name=SummaActors-1-250
+#SBATCH --mail-user=kyle.klenk@usask.ca
+#SBATCH --mail-type=ALL
+#SBATCH --output=/scratch/gwf/gwf_cmt/kck540/summaActors/slurm-Mar-21/1-250-8CPU-%A.out
+#SBATCH --account=hpc_c_giws_clark
+
+/globalhome/kck540/HPC/SummaActors/bin/summaMain -g 1 -c 250 -m /project/gwf/gwf_cmt/kck540/domain_NorthAmerica/settings/SUMMA/fileManager.txt -v -8CPU --config-file=/globalhome/kck540/HPC/SummaActors/3_SummaActors_Run/copernicus/caf-application.conf

3_SummaActors_Run/copernicus/run_summa.sh

+#!/bin/bash
+#SBATCH --cpus-per-task=8
+#SBATCH --time=71:00:00
+#SBATCH --mem=24G
+#SBATCH --job-name=SummaActorsNA
+#SBATCH --mail-user=kyle.klenk@usask.ca
+#SBATCH --mail-type=ALL
+#SBATCH --output=/project/gwf/gwf_cmt/kck540/domain_NorthAmerica/simulations/slurm/slurm-%A_%a.out
+#SBATCH --account=hpc_c_giws_clark
+
+# Ensure array size is correct ie. --array=1-[numJobs]
+#SBATCH --array=0-2499
+
+JOBLIST=/globalhome/kck540/HPC/SummaActors/3_MS_model_runs/run_graham_serialSumma_NA_gridEra5_1/copernicus/joblist/summa_joblist.txt
+
+
+# Make the log directory in case 
+mkdir -p logs/
+# ----------------------------------------------------------------------------------------------
+# RUN WITH
+# sbatch run_summa.sh
+# ----------------------------------------------------------------------------------------------
+
+JOBSTRING=$(sed -n "${SLURM_ARRAY_TASK_ID}p" $JOBLIST)
+
+# Seperate the SUMA call and its agruments from the ">" delimiter
+OIFS=$IFS
+IFS=">"
+read -ra COMMAND <<< "$JOBSTRING"
+IFS=$OIFS
+
+#--------------------------------------------------------------------
+# RUN
+#--------------------------------------------------------------------
+echo "Starting run at: `date`"
+
+${COMMAND[0]} > ${COMMAND[1]}
+
+# Echo end-of-run
+echo "Program finished with exit code $? ar: `date`"
+
+${JOBSTRING}
\ No newline at end of file

3_SummaActors_Run/copernicus/run_test.sh

+#!/bin/bash
+#SBATCH --cpus-per-task=1
+#SBATCH --time=0:30:00
+#SBATCH --mem=250M
+#SBATCH --job-name=SummaActors
+#SBATCH --mail-user=kyle.klenk@usask.ca
+#SBATCH --mail-type=ALL
+#SBATCH --output=/scratch/gwf/gwf_cmt/kck540/summaActors/slurm/slurm-%A_%a.out
+#SBATCH --account=hpc_c_giws_clark
+
+
+/globalhome/kck540/HPC/SummaActors/bin/summaTest -g 1 -c 4

3_SummaActors_Run/copernicus/summaActors.sh

+#!/bin/bash
+#SBATCH --cpus-per-task=6
+#SBATCH --time=24:00:00
+#SBATCH --mem=24G
+#SBATCH --job-name=SummaActors
+#SBATCH --mail-user=kyle.klenk@usask.ca
+#SBATCH --mail-type=ALL
+#SBATCH --output=/scratch/gwf/gwf_cmt/kck540/summaActors/slurm/slurm-%A.out
+#SBATCH --account=hpc_c_giws_clark
+
+/globalhome/kck540/HPC/SummaActors/bin/summaMain -g 1000 -c 518

3_SummaActors_Run/graham/run_allHRUs.sh

+#!/bin/bash
+#SBATCH --cpus-per-task=8
+#SBATCH --time=30:00:00
+#SBATCH --mem=32G
+#SBATCH --job-name=SummaActorsStressTest
+#SBATCH --mail-user=kyle.klenk@usask.ca
+#SBATCH --mail-type=ALL
+#SBATCH --output=/home/kklenk/scratch/SummaActorsOutput/slurm/slurm-%A_%a.out
+#SBATCH --account=def-spiteri
+
+# EXAMPLE: sbatch --array=0-258 run_allHRUs.sh
+
+# Define the GRU settings
+gruMax=517315 # North America, Merit Hydro basins
+gruCount=2000
+
+# Get the array ID for further use
+offset=$SLURM_ARRAY_TASK_ID 
+
+# Start at 1 for array ID 1, 2022 for array ID 2, etc
+gruStart=$(( 1 + gruCount*offset ))
+check=$(( $gruStart + $gruCount ))
+# Check that we don't specify too many basins
+if [ $check -gt $gruMax ]
+then
+    gruCount=$(( gruMax-gruStart+1 ))
+fi
+
+/home/kklenk/SummaProjects/SummaActors/bin/summaMain -g ${gruStart} -c ${gruCount} -m /project/6008034/kklenk/settings/SummaActorsSettings/fileManager.txt
+

3_SummaActors_Run/graham/run_client.sh

+#!/bin/bash
+#SBATCH --cpus-per-task=1
+#SBATCH --time=0:30:00
+#SBATCH --mem=2G
+#SBATCH --job-name=Summa-StressTest
+#SBATCH --mail-user=kyle.klenk@usask.ca
+#SBATCH --mail-type=ALL
+#SBATCH --output=/home/kklenk/scratch/SummaActorsOutput/slurm/slurm-client%A_%a.out
+#SBATCH --account=rpp-kshook
+
+# ----------------------------------------------------------------------------------------------
+# RUN WITH:
+# sbatch --array1-[number of jobs] [script name]
+# 
+# EXAMPLE: sbatch --array=0-100 run_all.sh
+# ----------------------------------------------------------------------------------------------
+
+/home/kklenk/SummaProjects/SummaActors/bin/summaTest -g 1 -c 100
\ No newline at end of file

3_SummaActors_Run/graham/run_one.sh

+#!/bin/bash
+#SBATCH --cpus-per-task=4
+#SBATCH --time=02:00:00
+#SBATCH --mem=8G
+#SBATCH --job-name=TEST_summa_na_grid_era5
+#SBATCH --mail-user=kyle.klenk@usask.ca
+#SBATCH --mail-type=ALL
+#SBATCH --output=/home/kklenk/scratch/SummaActorsOutput/slurm/slurm-%A.out
+#SBATCH --account=rpp-kshook
+
+/home/kklenk/SummaProjects/SummaActors/bin/summaMain -g 1 -c 100 -m /project/6008034/kklenk/settings/SummaActorsSettings/fileManager.txt
+1
\ No newline at end of file

3_SummaActors_Run/graham/run_server.sh

+#!/bin/bash
+#SBATCH --cpus-per-task=1
+#SBATCH --time=96:00:00
+#SBATCH --mem=1G
+#SBATCH --job-name=Summa-StressTest
+#SBATCH --mail-user=kyle.klenk@usask.ca
+#SBATCH --mail-type=ALL
+#SBATCH --output=/home/kklenk/scratch/SummaActorsOutput/slurm/slurmServer-%A_%a.out
+#SBATCH --account=rpp-kshook
+
+/home/kklenk/SummaProjects/SummaActors/bin/summaMain -s -p 4444 -g 1 -c 1000 -m /project/6008034/kklenk/settings/SummaActorsSettings/fileManager.txt
\ No newline at end of file

3_SummaActors_Run/graham/run_server_client.sh

+#!/bin/bash
+#SBATCH --cpus-per-task=1
+#SBATCH --time=24:00:00
+#SBATCH --mem=2G
+#SBATCH --job-name=SummaActors-StressTest
+#SBATCH --mail-user=kyle.klenk@usask.ca
+#SBATCH --mail-type=ALL
+#SBATCH --output=/home/kklenk/scratch/SummaActorsOutput/slurm/slurm-%A_%a.out
+#SBATCH --account=def-spiteri
+
+# ----------------------------------------------------------------------------------------------
+# RUN WITH:
+# sbatch --array1-[number of jobs] [script name]
+# 
+# EXAMPLE: sbatch --array=0-50 run_server_client.sh
+# ----------------------------------------------------------------------------------------------
+
+gruMax=517315 # North America, Merit Hydro basins
+gruCount=10000 # The number of GRUs you want to compute
+startGRU=1   # The starting index of the GRU you would like to compute
+offset=$SLURM_ARRAY_TASK_ID
+
+if [ $offset -eq 0 ]
+then
+    /home/kklenk/SummaProjects/SummaActors/bin/summaMain -s -p 4444 -g 1 -c $gruCount -m /project/6008034/kklenk/settings/SummaActorsSettings/fileManager.txt
+else
+    /home/kklenk/SummaProjects/SummaActors/bin/summaMain -p 4444 -j $offset -m /project/6008034/kklenk/settings/SummaActorsSettings/fileManager.txt
+fi

3_SummaActors_Run/graham/run_summa.sh

+#!/bin/bash
+#SBATCH --cpus-per-task=1
+#SBATCH --time=96:00:00
+#SBATCH --mem=1G
+#SBATCH --job-name=TEST_summa_na_grid_era5
+#SBATCH --mail-user=kyle.klenk@usask.ca
+#SBATCH --mail-type=ALL
+#SBATCH --output=/home/kklenk/slurmlog/slurm-%A_%a.out
+#SBATCH --account=rpp-kshook
+
+# Ensure array size is correct ie. --array=1-[numJobs]
+#SBATCH --array=1-10
+
+JOBLIST=/home/kklenk/actors/summa/code/graham/3_MS_model_runs/run_graham_serialSumma_NA_gridEra5_1/joblist/summa_joblist.txt
+
+
+# Make the log directory in case 
+mdkir -p logs/
+# ----------------------------------------------------------------------------------------------
+# RUN WITH
+# sbatch run_summa.sh
+# ----------------------------------------------------------------------------------------------
+
+JOBSTRING=$(sed -n "${SLURM_ARRAY_TASK_ID}p" $JOBLIST)
+
+# Seperate the SUMA call and its agruments from the ">" delimiter
+OIFS=$IFS
+IFS=">"
+read -ra COMMAND <<< "$JOBSTRING"
+IFS=$OIFS
+
+#--------------------------------------------------------------------
+# RUN
+#--------------------------------------------------------------------
+echo "Starting run at: `date`"
+
+${COMMAND[0]} > ${COMMAND[1]}
+
+# Echo end-of-run
+echo "Program finished with exit code $? ar: `date`"
+
+${JOBSTRING}
\ No newline at end of file

3_SummaActors_Run/graham/test.sh

+#!/bin/bash
+gruMax=517315 # North America, Merit Hydro basins
+gruCount=2000 # The number of GRUs you want to compute
+jobID=0
+
+
+for i in {0..258}
+do
+    gruStart=$(( 1 + gruCount*jobID))
+    check=$(( $gruStart + $gruCount ))
+    if [ $check -gt $gruMax ]
+    then
+        echo "HERe"
+        gruCount=$(( gruMax-gruStart+1 ))
+    fi
+    echo "gruStart = $gruStart, gruCount = $gruCount"
+    jobID=$(( 1 + jobID ))
+done
\ No newline at end of file

4_SummaActors_PostProcess/compareOutput.py

+from os import listdir
+from os.path import isfile, join
+from pathlib import Path
+import xarray as xr 
+
+numHRU = 10
+
+time = 'time'
+scalarSWE = 'scalarSWE'
+scalarCanopyWat = 'scalarCanopyWat'
+scalarAquiferStorage = 'scalarAquiferStorage'
+scalarTotalSoilWat = 'scalarTotalSoilWat'
+scalarSenHeatTotal = 'scalarSenHeatTotal'
+scalarLatHeatTotal = 'scalarLatHeatTotal'
+scalarRainPlusMelt = 'scalarRainPlusMelt'
+scalarInfiltration = 'scalarInfiltration'
+scalarSurfaceRunoff = 'scalarSurfaceRunoff'
+scalarSoilBaseflow = 'scalarSoilBaseflow'
+scalarSoilDrainage = 'scalarSoilDrainage'
+scalarAquiferBaseflow = 'scalarAquiferBaseflow'
+scalarTotalET = 'scalarTotalET'
+scalarTotalRunoff = 'scalarTotalRunoff'
+scalarNetRadiation = 'scalarNetRadiation'
+
+varList = [time, scalarSWE, scalarCanopyWat, scalarAquiferStorage, scalarTotalSoilWat, \
+    scalarSenHeatTotal, scalarLatHeatTotal, scalarRainPlusMelt, scalarInfiltration, \
+    scalarSurfaceRunoff, scalarSoilBaseflow, scalarSoilDrainage, scalarAquiferBaseflow, \
+    scalarTotalET, scalarTotalRunoff, scalarNetRadiation]
+
+# varList = [time, scalarSWE]
+time = "day"
+filename = "out_{}.txt".format(time)
+originalPath = Path('/scratch/gwf/gwf_cmt/kck540/summaActors/netcdf/SummaActors_G000001-000010_{}.nc'.format(time))
+actorsPath = Path('/scratch/gwf/gwf_cmt/kck540/summaActors/netcdf/SummaActorsGRU1-10_{}.nc'.format(time))
+
+originalDataset = xr.open_dataset(originalPath)
+actorsDataset = xr.open_dataset(actorsPath)
+
+allHRUsOriginal = []
+allHRUsActors = []
+
+for ihru in range(0, numHRU):
+  allHRUsOriginal.append(originalDataset.isel(hru=ihru).copy())
+
+for ihru in range(0, numHRU):
+  allHRUsActors.append(actorsDataset.isel(hru=ihru).copy())
+
+file = open(filename, "w")
+for i in range(0, numHRU):
+  file.write("________HRU {}________\n".format(i+1))
+  for var in varList:
+    file.write("&&&& VAR = {} &&&&\n".format(var))
+    dataOrig = []
+    dataAct = []
+    for data in allHRUsOriginal[i][var].values:
+      dataOrig.append(data)
+    for data in allHRUsActors[i][var].values:
+      dataAct.append(data)
+
+    marginOfError = 0
+    if var == time:
+      for a in range(0, len(dataAct)):
+        if dataOrig[a] != dataAct[a]:
+          file.write("{} = Actor and {} = original is different\n".format(dataAct[a], dataOrig[a]))
+        # else:
+        #   file.write("{} = Actor and {} = original is the same\n".format(dataAct[a], dataOrig[a]))
+    else:
+      for a in range(0, len(dataAct)):
+        diff = dataOrig[a] - dataAct[a]
+        if diff < -marginOfError or diff > marginOfError:
+          file.write("{}: {} = Actor and {} = original is different:   {}\n".format(a, dataAct[a], dataOrig[a], abs(dataAct[a] - dataOrig[a])))
+        else:
+          file.write("{}: {} = Actor and {} = original is the same\n".format(a, dataAct[a], dataOrig[a]))
+
+        
+
+file.close()
\ No newline at end of file

4_SummaActors_PostProcess/mergeNetcdf.py

+import subprocess
+import os.path
+
+# bashCommand = subprocess.run(["ncrcat"])
+
+listOfFiles = []
+listOfFiles.append("ncrcat")
+for x in range(1, 519):
+    file = "/home/kklenk/scratch/SummaActorsOutput/netcdf/SummaActorsGRU-{}HRU-1_day.nc".format(x)
+    listOfFiles.append(file)
+    print(os.path.isfile(file))
+
+listOfFiles.append("SummaActors_G000001-000518_day.nc")
+
+bashCommand = subprocess.run(listOfFiles)
+print("Exit Code = %d" % bashCommand.returncode)
\ No newline at end of file

Dockerfile

+#FROM ubuntu:20.04
+FROM linuxbrew/brew:latest
+
+
+# This dockerfile creates the environement for compiling and
+# running SUMMA4CHM. Once started cd into /code/build/source/cppwrap.
+# make lib
+# make
+# export LD_LIBRARY_PATH=/code/build/source/cppwrap:D_LIBRARY_PATH
+# ./program
+
+WORKDIR /code
+
+RUN apt-get update && \
+    DEBIAN_FRONTEND="noninteractive" apt-get install -y software-properties-common \
+    libnetcdf-dev \
+    libnetcdff-dev \
+    liblapack-dev
+
+RUN add-apt-repository ppa:ubuntu-toolchain-r/test -y \
+    && apt-get update \
+    && apt-get install -y gfortran-7
+
+RUN apt update -y \
+    && apt upgrade -y \
+    && DEBIAN_FRONTEND="noninteractive" apt install -y \
+         cmake \
+         g++ \
+         git \
+         libssl-dev \
+         make \
+         gfortran \
+         gdb \
+    && apt-get autoclean
+
+RUN brew install caf
+
+ADD . /code
+
+ENV LD_LIBRARY_PATH=/code/build:/home/linuxbrew/.linuxbrew/Cellar/caf/0.18.5/lib/
+
+# RUN cp -r /home/linuxbrew/.linuxbrew/Cellar/caf/0.18.5/lib/* /usr/local/lib/
+# RUN cp -r /home/linuxbrew/.linuxbrew/Cellar/caf/0.18.*/include/caf /usr/local/include/
+# RUN cp -f /usr/local/lib/libcaf_core.so.0.18.* /code/build/source/cppwrap/
+
+
+
+

README.md

+#### Compiling Summa-Actors ####
+SummaActors can be compiled by modifing the following components in the makefile
+ - F_MASTER = directory/above/build
+ - FC = gfortran
+ - CC = g++
+ - INCLUDES = Path/to/netcdf/includes
+ - LIBRARIES = Path/to/netcdf/lib & Path/to/openblas
+        -lnetcdff -lopenblas
+ - ACTORS_INCLUDES = $INCLUDES & Path/to/CAF/includes
+ - ACTORS_LIBRARIES = $LIBRARIES & PATH/to/CAF/lib
+        -lcaf_core -lcaf_io -lsumma -lopenblas -lnetcdff
+
+Once all above variables are set compilation is done with:
+
+make
+
+There is an example bash file included in the /build directory that can be modified for compiling Summa. This is benficial in cluster environments as you can load required modules within the script. The script can then be run with:
+
+source compilation_script
+
+#### Running Summa-Actors #####
+Once the binary is compiled it can be run like the following example command:
+
+./summaMain -g 1 -c 10 -m /path/to/file/manager/ --config-file=/path/to/actors/config/file
+
+ -g = starting index of the first GRU to compute
+ -c = number of grus to run
+ -m = path to the file manager
+ --config-file = /path/to/config/file
+
+#### Config File ####
+the configuraton file is used to specifiy the number of threads that Summa-Actors can spawn. If you would like to spawn as many as your system has then you can omit this argument. This argument is mainly needed for cluster environments to ensure that Summa-Actors threads to Core ratio is 1:1.
+
+The contents of the configuration file look like the following:
+caf {
+  # Parameters selecting a default scheduler.
+  scheduler {
+    max-threads = 8
+  }
+}
+
+

build/example_compile.sh

+#!/bin/bash
+
+#### load modules if using Compute Canada or Copernicus ####
+# module load gcc/9.3.0
+# module load netcdf-fortran
+# module load openblas
+# module load caf
+
+#### Specifiy Master Directory, parent of build directory
+
+# export F_MASTER=/path/to/summaActors
+
+#### Specifiy Compilers ####
+# export FC=gfortran
+# export CC=g++
+
+#### Includes and Libraries ####
+
+# export INCLUDES = NETCDF and OPENBLAS
+# export LIBRARIES = NETCDF and OPENBLAS
+
+# export ACTORS_INCLUDES = C++ Actor Framework 
+# export ACTORS_LIBRARIES = C++ Actor Framework and
+#   The directory in which libsumma.so resides
+
+#### Compile with the Makefile ####
+# make -f ${F_MASTER}/build/makefile lib # libsumma.so part
+
+# mv libsumma.so ${F_MASTER}/bin # optional move of libsumma (just ensure that summaMain knows where to find it)
+
+# make -f ${F_MASTER}/build/makefile main # summaMain part
+
+# mv summaMain ${F_MASTER}/bin # optional move, cleans things up
+
+# export LD_LIBRARY_PATH=Path/to/libsumma.so
\ No newline at end of file