Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • gwu479/Summa-Actors
  • numerical_simulations_lab/actors/Summa-Actors
2 results
Show changes
Commits on Source (495)
395 additional commits have been omitted to prevent performance issues.
Hide whitespace changes
Inline Side-by-side
Showing
with 647 additions and 369 deletions
.gitignore 0 → 100644
View file @ 8e5f43c0
bin/summa_actors
build/cmake/build
build/summa
build/summa-sundials
build/.vscode
.vscode/settings.json
.DS_Store
laugh_tests
build/summa_sundials
build/summa_original
build/source/testing/containers/output_container/main
bin/submission_script.sh
build/summa_kinsol
bin/summa_kinsol
build/flibs-0.9/
build/cmake/build_be/
build/cmake/build_kinsol/
bin/data_be.csv
bin/data_kinsol.csv
bin/submission_copern.sh
bin/summa_be
bin/data_kinsol_state_vec.csv
bin/plot_resuduial.py
bin/state.png
bin/submission_script_array.sh
build/source/testing/containers/output_container/out.txt
build/summa_old
bin/config.json
bin/summa_sundials
build/cmake_build/
build/source/engine/access_write.f90 → .gitlab-ci.yml
View file @ 8e5f43c0
File moved
3_SummaActors_Run/copernicus/caf-application.conf deleted 100644 → 0
View file @ c29d3324
caf {
# Parameters selecting a default scheduler.
scheduler {
max-threads = 8
}
}
\ No newline at end of file
3_SummaActors_Run/copernicus/run_client_server.sh deleted 100644 → 0
View file @ c29d3324
#!/bin/bash
#SBATCH --cpus-per-task=1
#SBATCH --time=8:00:00
#SBATCH --mem=2G
#SBATCH --job-name=SummaActorsOutputBugTest
#SBATCH --mail-user=kyle.klenk@usask.ca
#SBATCH --mail-type=ALL
#SBATCH --output=/scratch/gwf/gwf_cmt/kck540/summaActors/slurm/slurm-%A_%a.out
#SBATCH --account=hpc_c_giws_clark
# EXAMPLE: sbatch --array=0-50 run_server_client.sh
gruMax=517315 # North America, Merit Hydro basins
gruCount=50000 # The number of GRUs you want to compute
startGRU=1 # The starting index of the GRU you would like to compute
offset=$SLURM_ARRAY_TASK_ID
if [ $offset -eq 0 ]
then
/globalhome/kck540/HPC/SummaActors/bin/summaMain -s -p 4444 -g 1 -c $gruCount -m /project/gwf/gwf_cmt/kck540/domain_NorthAmerica/settings/SUMMA/fileManager.txt
else
/globalhome/kck540/HPC/SummaActors/bin/summaMain -p 4444 -j $offset -m /project/gwf/gwf_cmt/kck540/domain_NorthAmerica/settings/SUMMA/fileManager.txt
fi
3_SummaActors_Run/copernicus/run_one.sh deleted 100644 → 0
View file @ c29d3324
#!/bin/bash
#SBATCH --cpus-per-task=8
#SBATCH --time=36:00:00
#SBATCH --mem=32G
#SBATCH --job-name=SummaActors-1-250
#SBATCH --mail-user=kyle.klenk@usask.ca
#SBATCH --mail-type=ALL
#SBATCH --output=/scratch/gwf/gwf_cmt/kck540/summaActors/slurm-Mar-21/1-250-8CPU-%A.out
#SBATCH --account=hpc_c_giws_clark
/globalhome/kck540/HPC/SummaActors/bin/summaMain -g 1 -c 250 -m /project/gwf/gwf_cmt/kck540/domain_NorthAmerica/settings/SUMMA/fileManager.txt -v -8CPU --config-file=/globalhome/kck540/HPC/SummaActors/3_SummaActors_Run/copernicus/caf-application.conf
3_SummaActors_Run/copernicus/run_summa.sh deleted 100644 → 0
View file @ c29d3324
#!/bin/bash
#SBATCH --cpus-per-task=8
#SBATCH --time=71:00:00
#SBATCH --mem=24G
#SBATCH --job-name=SummaActorsNA
#SBATCH --mail-user=kyle.klenk@usask.ca
#SBATCH --mail-type=ALL
#SBATCH --output=/project/gwf/gwf_cmt/kck540/domain_NorthAmerica/simulations/slurm/slurm-%A_%a.out
#SBATCH --account=hpc_c_giws_clark
# Ensure array size is correct ie. --array=1-[numJobs]
#SBATCH --array=0-2499
JOBLIST=/globalhome/kck540/HPC/SummaActors/3_MS_model_runs/run_graham_serialSumma_NA_gridEra5_1/copernicus/joblist/summa_joblist.txt
# Make the log directory in case
mkdir -p logs/
# ----------------------------------------------------------------------------------------------
# RUN WITH
# sbatch run_summa.sh
# ----------------------------------------------------------------------------------------------
JOBSTRING=$(sed -n "${SLURM_ARRAY_TASK_ID}p" $JOBLIST)
# Seperate the SUMA call and its agruments from the ">" delimiter
OIFS=$IFS
IFS=">"
read -ra COMMAND <<< "$JOBSTRING"
IFS=$OIFS
#--------------------------------------------------------------------
# RUN
#--------------------------------------------------------------------
echo "Starting run at: `date`"
${COMMAND[0]} > ${COMMAND[1]}
# Echo end-of-run
echo "Program finished with exit code $? ar: `date`"
${JOBSTRING}
\ No newline at end of file
3_SummaActors_Run/copernicus/run_test.sh deleted 100644 → 0
View file @ c29d3324
#!/bin/bash
#SBATCH --cpus-per-task=1
#SBATCH --time=0:30:00
#SBATCH --mem=250M
#SBATCH --job-name=SummaActors
#SBATCH --mail-user=kyle.klenk@usask.ca
#SBATCH --mail-type=ALL
#SBATCH --output=/scratch/gwf/gwf_cmt/kck540/summaActors/slurm/slurm-%A_%a.out
#SBATCH --account=hpc_c_giws_clark
/globalhome/kck540/HPC/SummaActors/bin/summaTest -g 1 -c 4
3_SummaActors_Run/copernicus/summaActors.sh deleted 100644 → 0
View file @ c29d3324
#!/bin/bash
#SBATCH --cpus-per-task=6
#SBATCH --time=24:00:00
#SBATCH --mem=24G
#SBATCH --job-name=SummaActors
#SBATCH --mail-user=kyle.klenk@usask.ca
#SBATCH --mail-type=ALL
#SBATCH --output=/scratch/gwf/gwf_cmt/kck540/summaActors/slurm/slurm-%A.out
#SBATCH --account=hpc_c_giws_clark
/globalhome/kck540/HPC/SummaActors/bin/summaMain -g 1000 -c 518
3_SummaActors_Run/graham/run_allHRUs.sh deleted 100644 → 0
View file @ c29d3324
#!/bin/bash
#SBATCH --cpus-per-task=8
#SBATCH --time=30:00:00
#SBATCH --mem=32G
#SBATCH --job-name=SummaActorsStressTest
#SBATCH --mail-user=kyle.klenk@usask.ca
#SBATCH --mail-type=ALL
#SBATCH --output=/home/kklenk/scratch/SummaActorsOutput/slurm/slurm-%A_%a.out
#SBATCH --account=def-spiteri
# EXAMPLE: sbatch --array=0-258 run_allHRUs.sh
# Define the GRU settings
gruMax=517315 # North America, Merit Hydro basins
gruCount=2000
# Get the array ID for further use
offset=$SLURM_ARRAY_TASK_ID
# Start at 1 for array ID 1, 2022 for array ID 2, etc
gruStart=$(( 1 + gruCount*offset ))
check=$(( $gruStart + $gruCount ))
# Check that we don't specify too many basins
if [ $check -gt $gruMax ]
then
gruCount=$(( gruMax-gruStart+1 ))
fi
/home/kklenk/SummaProjects/SummaActors/bin/summaMain -g ${gruStart} -c ${gruCount} -m /project/6008034/kklenk/settings/SummaActorsSettings/fileManager.txt
3_SummaActors_Run/graham/run_one.sh deleted 100644 → 0
View file @ c29d3324
#!/bin/bash
#SBATCH --cpus-per-task=4
#SBATCH --time=02:00:00
#SBATCH --mem=8G
#SBATCH --job-name=TEST_summa_na_grid_era5
#SBATCH --mail-user=kyle.klenk@usask.ca
#SBATCH --mail-type=ALL
#SBATCH --output=/home/kklenk/scratch/SummaActorsOutput/slurm/slurm-%A.out
#SBATCH --account=rpp-kshook
/home/kklenk/SummaProjects/SummaActors/bin/summaMain -g 1 -c 100 -m /project/6008034/kklenk/settings/SummaActorsSettings/fileManager.txt
1
\ No newline at end of file
3_SummaActors_Run/graham/run_server.sh deleted 100644 → 0
View file @ c29d3324
#!/bin/bash
#SBATCH --cpus-per-task=1
#SBATCH --time=96:00:00
#SBATCH --mem=1G
#SBATCH --job-name=Summa-StressTest
#SBATCH --mail-user=kyle.klenk@usask.ca
#SBATCH --mail-type=ALL
#SBATCH --output=/home/kklenk/scratch/SummaActorsOutput/slurm/slurmServer-%A_%a.out
#SBATCH --account=rpp-kshook
/home/kklenk/SummaProjects/SummaActors/bin/summaMain -s -p 4444 -g 1 -c 1000 -m /project/6008034/kklenk/settings/SummaActorsSettings/fileManager.txt
\ No newline at end of file
3_SummaActors_Run/graham/run_server_client.sh deleted 100755 → 0
View file @ c29d3324
#!/bin/bash
#SBATCH --cpus-per-task=1
#SBATCH --time=24:00:00
#SBATCH --mem=2G
#SBATCH --job-name=SummaActors-StressTest
#SBATCH --mail-user=kyle.klenk@usask.ca
#SBATCH --mail-type=ALL
#SBATCH --output=/home/kklenk/scratch/SummaActorsOutput/slurm/slurm-%A_%a.out
#SBATCH --account=def-spiteri
# ----------------------------------------------------------------------------------------------
# RUN WITH:
# sbatch --array1-[number of jobs] [script name]
#
# EXAMPLE: sbatch --array=0-50 run_server_client.sh
# ----------------------------------------------------------------------------------------------
gruMax=517315 # North America, Merit Hydro basins
gruCount=10000 # The number of GRUs you want to compute
startGRU=1 # The starting index of the GRU you would like to compute
offset=$SLURM_ARRAY_TASK_ID
if [ $offset -eq 0 ]
then
/home/kklenk/SummaProjects/SummaActors/bin/summaMain -s -p 4444 -g 1 -c $gruCount -m /project/6008034/kklenk/settings/SummaActorsSettings/fileManager.txt
else
/home/kklenk/SummaProjects/SummaActors/bin/summaMain -p 4444 -j $offset -m /project/6008034/kklenk/settings/SummaActorsSettings/fileManager.txt
fi
3_SummaActors_Run/graham/run_summa.sh deleted 100644 → 0
View file @ c29d3324
#!/bin/bash
#SBATCH --cpus-per-task=1
#SBATCH --time=96:00:00
#SBATCH --mem=1G
#SBATCH --job-name=TEST_summa_na_grid_era5
#SBATCH --mail-user=kyle.klenk@usask.ca
#SBATCH --mail-type=ALL
#SBATCH --output=/home/kklenk/slurmlog/slurm-%A_%a.out
#SBATCH --account=rpp-kshook
# Ensure array size is correct ie. --array=1-[numJobs]
#SBATCH --array=1-10
JOBLIST=/home/kklenk/actors/summa/code/graham/3_MS_model_runs/run_graham_serialSumma_NA_gridEra5_1/joblist/summa_joblist.txt
# Make the log directory in case
mdkir -p logs/
# ----------------------------------------------------------------------------------------------
# RUN WITH
# sbatch run_summa.sh
# ----------------------------------------------------------------------------------------------
JOBSTRING=$(sed -n "${SLURM_ARRAY_TASK_ID}p" $JOBLIST)
# Seperate the SUMA call and its agruments from the ">" delimiter
OIFS=$IFS
IFS=">"
read -ra COMMAND <<< "$JOBSTRING"
IFS=$OIFS
#--------------------------------------------------------------------
# RUN
#--------------------------------------------------------------------
echo "Starting run at: `date`"
${COMMAND[0]} > ${COMMAND[1]}
# Echo end-of-run
echo "Program finished with exit code $? ar: `date`"
${JOBSTRING}
\ No newline at end of file
3_SummaActors_Run/graham/test.sh deleted 100755 → 0
View file @ c29d3324
#!/bin/bash
gruMax=517315 # North America, Merit Hydro basins
gruCount=2000 # The number of GRUs you want to compute
jobID=0
for i in {0..258}
do
gruStart=$(( 1 + gruCount*jobID))
check=$(( $gruStart + $gruCount ))
if [ $check -gt $gruMax ]
then
echo "HERe"
gruCount=$(( gruMax-gruStart+1 ))
fi
echo "gruStart = $gruStart, gruCount = $gruCount"
jobID=$(( 1 + jobID ))
done
\ No newline at end of file
Dockerfile deleted 100644 → 0
View file @ c29d3324
#FROM ubuntu:20.04
FROM linuxbrew/brew:latest
# This dockerfile creates the environement for compiling and
# running SUMMA4CHM. Once started cd into /code/build/source/cppwrap.
# make lib
# make
# export LD_LIBRARY_PATH=/code/build/source/cppwrap:D_LIBRARY_PATH
# ./program
WORKDIR /code
RUN apt-get update && \
DEBIAN_FRONTEND="noninteractive" apt-get install -y software-properties-common \
libnetcdf-dev \
libnetcdff-dev \
liblapack-dev
RUN add-apt-repository ppa:ubuntu-toolchain-r/test -y \
&& apt-get update \
&& apt-get install -y gfortran-7
RUN apt update -y \
&& apt upgrade -y \
&& DEBIAN_FRONTEND="noninteractive" apt install -y \
cmake \
g++ \
git \
libssl-dev \
make \
gfortran \
gdb \
&& apt-get autoclean
RUN brew install caf
ADD . /code
ENV LD_LIBRARY_PATH=/code/build:/home/linuxbrew/.linuxbrew/Cellar/caf/0.18.5/lib/
# RUN cp -r /home/linuxbrew/.linuxbrew/Cellar/caf/0.18.5/lib/* /usr/local/lib/
# RUN cp -r /home/linuxbrew/.linuxbrew/Cellar/caf/0.18.*/include/caf /usr/local/include/
# RUN cp -f /usr/local/lib/libcaf_core.so.0.18.* /code/build/source/cppwrap/
README.md
View file @ 8e5f43c0
#### Compiling Summa-Actors ####
SummaActors can be compiled by modifing the following components in the makefile
- F_MASTER = directory/above/build
- FC = gfortran
- CC = g++
- INCLUDES = Path/to/netcdf/includes
- LIBRARIES = Path/to/netcdf/lib & Path/to/openblas
-lnetcdff -lopenblas
- ACTORS_INCLUDES = $INCLUDES & Path/to/CAF/includes
- ACTORS_LIBRARIES = $LIBRARIES & PATH/to/CAF/lib
-lcaf_core -lcaf_io -lsumma -lopenblas -lnetcdff
Once all above variables are set compilation is done with:
make
There is an example bash file included in the /build directory that can be modified for compiling Summa. This is benficial in cluster environments as you can load required modules within the script. The script can then be run with:
source compilation_script
#### Running Summa-Actors #####
Once the binary is compiled it can be run like the following example command:
./summaMain -g 1 -c 10 -m /path/to/file/manager/ --config-file=/path/to/actors/config/file
-g = starting index of the first GRU to compute
-c = number of grus to run
-m = path to the file manager
--config-file = /path/to/config/file
#### Config File ####
the configuraton file is used to specifiy the number of threads that Summa-Actors can spawn. If you would like to spawn as many as your system has then you can omit this argument. This argument is mainly needed for cluster environments to ensure that Summa-Actors threads to Core ratio is 1:1.
The contents of the configuration file look like the following:
caf {
# Parameters selecting a default scheduler.
scheduler {
max-threads = 8
}
}
# **NOTICE**
**This repository has been frozen. SUMMA-Actors project development has moved to: https://github.com/uofs-simlab/Summa-Actors**
# SUMMA-Actors: Structure for Unifying Multiple Modeling Alternatives with Actors
SUMMA-Actors is a modified version of the already existing SUMMA software that can be found [here](https://github.com/CH-Earth/summa#readme). SUMMA-Actors uses the Actor Model to increase scalability and fault-tolerance. It is built using the [C++ Actor Framework](https://github.com/actor-framework/actor-framework).
## Documentation
A more in-depth documentation can be found [here](https://git.cs.usask.ca/numerical_simulations_lab/actors/Summa-Actors/-/wikis/home). SUMMA-Actors depends on many files from the [original SUMMA repo](https://github.com/CH-Earth/summa). Below is a quick start guide for compiling and running SUMMA-Actors, please consult our wiki for more in-depth documentation. Consider creating an issue for any missing documentation, asking questions, or providing suggestions.
## Preliminaries
SUMMA-Actors is meant to wrap around the existing SUMMA implementation. Therefore you will need to clone the SUMMA repo into the SUMMA-Actors build directory. SUMMA-Actors works with the master branch of SUMMA, which can be found [here](https://github.com/CH-Earth/summa). Below the directory structure is explained in more detail and the steps to compiling SUMMA-Actors are explained.
## Directory Structure
SUMMA-Actors is set up with the following sub-directories, we will consider the top level Summa Actors directory the `root_dir`:
- bin
- build
- includes
- source
- summa (https://github.com/KyleKlenk/summa/tree/summa-actors, summa-actors branch)
- CMakeLists.txt
- utils
- README.md
First clone Summa-Actors to your workstation. Then cd into `build/` and clone `summa` into Summa-Actor's build directory as folder summa.
## Compiling SUMMA-Actors
### Dependencies
SUMMA-Actors has only one additional dependency, the [C++ Actor Framework](https://github.com/actor-framework/actor-framework), specifically the [0.18.6
release](https://github.com/actor-framework/actor-framework/archive/refs/tags/0.18.6.tar.gz).
The following steps can be used to install the C++ Actor Framework on a Linux system:
```bash
wget https://github.com/actor-framework/actor-framework/archive/refs/tags/0.18.6.tar.gz
tar -xzf 0.18.6.tar.gz
cd actor-framework-0.18.6/
./configure --prefix=/path/to/install
cd build
make # [-j]
make install # [as root if necessary]
```
Additional dependencies required by both SUMMA and SUMMA actors are:
* g++
* gfortran
* [NetCDF-Fortran](https://github.com/Unidata/netcdf-fortran)
* [OpenBLAS](https://github.com/xianyi/OpenBLAS)
* [SUNDIALS V6.6](https://github.com/LLNL/sundials/releases/tag/v6.6.0) (optional)
### Steps To Compile
```
git clone https://git.cs.usask.ca/numerical_simulations_lab/actors/Summa-Actors.git
cd Summa-Actors/build
git clone -b summa-actors https://github.com/KyleKlenk/summa.git
cmake -B cmake_build -S . -DCMAKE_BUILD_TYPE=Build_Type
cmake --build cmake_build --target all -j
```
Available build types are:
* BE or BE_Debug: Builds for local machine
* BE_Cluster or BE_Cluster_Debug: See below section (Modules for Alliance Machines)
### Modules for Alliance Machines
To compile on Alliance machines the following modules can be loaded:
```bash
module load StdEnv/2020
module load gcc/9.3.0
module load openblas/0.3.17
module load netcdf-fortran/4.5.2
module load caf
```
## Running SUMMA-Actors
SUMMA-Actors can be run in two modes, distributed and non-distributed. The distributed mode used to create ad-hoc clusters. The command line arguments are kept as close to the original SUMMA as possible. However, there are options not yet available in SUMMA-Actors that are in the original SUMMA. Here is a full list of the available and unavailable options:
```
Usage: summa_actors -m master_file [-g startGRU countGRU] [-c config_file] [-b backup_server] [-s server_mode]
Available options:
-m, --master: Define path/name of master file (can be specified in config)
-g, --gru: Run a subset of countGRU GRUs starting from index startGRU
-c, --config: Path name of the Summa-Actors config file (optional but recommended)
--gen-config: Generate a config file
-b, --backup-server: Start backup server, requires a server and config_file
-s, --server-mode: Enable server mode
-h, --help: Print this help message
Unimplemented Options:
-n --newFile Define frequency [noNewFiles,newFileEveryOct1] of new output files
-s --suffix Add fileSuffix to the output files
-h --hru Run a single HRU with index of iHRU
-r --restart Define frequency [y,m,d,e,never] to write restart files
-p --progress Define frequency [m,d,h,never] to print progress
-v --version Display version information of the current build
```
Instructions for using each mode are provided below, if you just want to skip ahead and get started using the default options.
### Non-Distributed Mode (Similar to original SUMMA)
Using SUMMA-Actors in non-distributed mode is like running the normal SUMMA. The difference is that HRUs will run concurrently in SUMMA-Actors and maximize resource use. Usage is very close to SUMMA, and the same input files are required. In depth documentation for configuring a SUMMA run can be found [here](https://summa.readthedocs.io/en/latest/).
Usage is as follows: ./summa_actors -m master_file [-g startGRU countGRU] [-c config_file]
The master_file for SUMMA-Actors can either be defined explicitly on the command line or it can be included in the new to SUMMA-Actors config_file. The config_file is optional but it is highly recommended as it does allow users to fine tune how SUMMA-Actor will perform. We are working to automate these aspects but for now its best to define it for the domain and compute environment in use. Using `./summa_actors --gen-config` will generate a configuration file that can then be filled in. See the section on config_file in this readme for more information.
### Distributed Mode
Using SUMMA-Actors in distributed mode allows SUMMA-Actors to solve batches and dymically assign them to nodes and reassign them in the event of node failures.
To use this feature there are 3 actors that can be spawned to cluster nodes together and add additional redundancy to the system. All settings are to be defined in the config_file for this mode. In the config file the user must set `distributed_mode` to true, define a `port`, and configure the domain to run. The domain is defined by how many total GRUs the user wishes to compute and the number of GRUs that should be assembled into a batch. Finally, the `server_list` should be set to the hostname of the backup_servers that are started.
To start the system one start the server first with `./summa_actors -c -s`
Backup servers can be added with `./summa_actors -c -b`
Clients can simply be added with `./summa_actors -c`
NOTE: Each system will need a copy of the forcing data or input data, or the data should be in a singular location like on a cluster system.
### Config File
The config file is a JSON file that is used to configure SUMMA-Actors. It is highly recommended to use a config file as it allows the user to fine tune how SUMMA-Actors will perform. Using `./summa_actors --gen-config` will generate a configuration file that can then be filled in. Below is a list of the available options and their descriptions.
#### Distributed_Settings
The distributed settings are used to configure the distributed mode of SUMMA-Actors. The following options are available:
- distributed_mode: (true/false) Enables distributed mode
- servers_list: (list of strings) List of hostnames for backup servers
- port: (int) Port to use for communication
- total_hru_count: (int) Total number of HRUs in the entire domain
- num_hru_per_batch: (int) Number of HRUs to assemble into a batches
#### Summa_Actor
The Summa_Actor settings are used to restrict how many HRU actor can run at once. The following options are available:
- max_gru_per_job: (int) Maximum number of HRUs that can be run at once
#### Job_Actor
The Job_Actor settings can specify the file manager path so it does not need to be specified on the command line, and the maximum number of times to attempt an HRU before giving up. The following options are available:
- file_manager_path: (string) Path to the file manager file
- max_run_attempts: (int) Maximum number of times to attempt an HRU before giving up
#### File_Access_Actor
The File_Access_Actor settings are used to configure the file access actor. The following options are available:
- num_partitions_in_output_buffer: (int) Number of partitions in the output buffer
- num_timesteps_in_output_buffer: (int) Number of timesteps in the output buffer
#### HRU_Actor
The HRU_Actor settings are used to configure the HRU actor. The following options are available:
- print_output: (true/false) Print output to the screen
- output_frequency: (int) Frequency to print output to the screen
- dt_init_factor: (int) Factor to multiply dt_init by
- rel_tol: (float) Relative tolerance for the HRU actor (Requires Sundials)
- abs_tol: (float) Absolute tolerance for the HRU actor (Requires Sundials)
## Credits
The initial implementation of SUMMA is credited to the initial publications below. These
publications can be found in [Water Resources Research](http://onlinelibrary.wiley.com/journal/10.1002/(ISSN)1944-7973).
* Clark, M. P., B. Nijssen, J. D. Lundquist, D. Kavetski, D. E. Rupp, R. A. Woods, J. E. Freer, E. D. Gutmann, A. W. Wood, L. D. Brekke, J. R. Arnold, D. J. Gochis, R. M. Rasmussen, 2015a: A unified approach for process-based hydrologic modeling: Part 1. Modeling concept. _Water Resources Research_, [doi:10.1002/2015WR017198](http://dx.doi.org/10.1002/2015WR017198).<a id="clark_2015a"></a>
* Clark, M. P., B. Nijssen, J. D. Lundquist, D. Kavetski, D. E. Rupp, R. A. Woods, J. E. Freer, E. D. Gutmann, A. W. Wood, D. J. Gochis, R. M. Rasmussen, D. G. Tarboton, V. Mahat, G. N. Flerchinger, D. G. Marks, 2015b: A unified approach for process-based hydrologic modeling: Part 2. Model implementation and case studies. _Water Resources Research_, [doi:10.1002/2015WR017200](http://dx.doi.org/10.1002/2015WR017200).<a id="clark_2015b"></a>
We also credit the original creators of the C++ Actor Framework which allowed us to implement the actor model into SUMMA-Actors. Links to their research work can be found
below.
* Charousset, D., Schmidt, T. C., Hiesgen, R., Wählisch, M., 2013: Native actors:
a scalable software platform for distributed, heterogeneous environments. _AGERE!_,
[doi:10.1145/2541329.2541336](http://dx.doi.org/10.1145/2541329.2541336).
* Charousset, D., Schmidt, T. C., Hiesgen, R., 2016: Revisiting actor programming in
C++. _Computer Languages, Systems & Structures_, [doi:10.1016/j.cl.2016.01.002](http://
dx.doi.org/10.1016/j.cl.2016.01.002)
actorsChanges.txt 0 → 100644
View file @ 8e5f43c0
driver
/summaActors_alarms.f90 -- remove
/summaActors_globalData.-- could combine and remove, renamed for now
/summaActors_type.f90 -- could combine and remove, renamed for now, adds a new function, but don't use both
/summaActors_util.f90 -- could combine and remove, renamed for now, adds a new function, but don't use both
/summaversion.inc -- remove
dshare
/csv_file_1d.f90 -- remove?? not seeing it used anywhere?
/csv_file_2d.f90 -- remove?? not seeing it used anywhere?
/csv_file.f90 -- remove?? IS this only in kinsol? do we need?
/data_types.f90 -- has a lot
/globalData.f90 -- remove with ACTORS_ACTIVE adds forcing stuff and tmZoneOffsetFracDay, fracJulDay, yearLength not as global
/varLookup.f90 -- remove with ACTORS_ACTIVE uses a iso_c binding call
engine
can remove sundials folder shouldn't need with SUNDIALS_ACTIVE
/sundials/coupled_em.f90 -- remove with ACTORS_ACTIVE needs fracJulDay, yearLength
/sundials/mDecisions.f90 -- remove with ACTORS_ACTIVE called with iso_c binding
/sundials/t2enthalpy.f90 -- remove
/alloc_fileAccess.f90 -- new
/allocspace[Actors].f90 -- this is a replacement, rename, and see how changes CHECK IF NECESSARY!
/check_icondActors.f90-- could combine and remove, renamed for now, adds a new function, but don't use both
/checkStruc.f90 -- remove
/coupled_em.f90 -- remove shouldn't need the non-sundials version
/derivforce.f90 -- remove with ACTORS_ACTIVE tmZoneOffsetFracDay needs to be variable not global
/ffile_info.f90 -- very different, call this ffile_infoActors?
/mDecisions.f90 -- remove shouldn't need the non-sundials version
/read_dimension.-- remove, part of read_attrb, - some deallocation changes, use this in all (not actually used in actors, ??)
- also Actors assumes don't have aspect variable .. why?
/read_pinit.f90 -- remove
/vegPhenlgy.f90 -- remove with ACTORS_ACTIVE needs fracJulDay, yearLength
SUMMA_FILEACCESS_INTERFACE = \
output_structure.f90 \
cppwrap_fileAccess.f90 \
read_attrb.f90 \
read_force.f90 \
read_param.f90 \
read_icondFromStructure.f90 \
ARE THESE ALL SORT OF REPEATS?? DO WE EVEN USE ALL OF THEM? specifically read_icondFromStructure ... what is that ... do we call its
Can we give them the same names as the regular summa files if they are repeats, so we know to change them??
changed calls in run_oneHRU so can use (modded) currently in actors /derivforce.f90 /coupled_em.f90 /vegPhenlgy.f90 as is
hookup
/ascii_util.f90 -- use Kyle's version everywhere!! (so remove)
/summaActors_FileManager.-- could combine and remove, renamed for now??
lapack -- remove
/Makefile -- remove
/README -- remove
netcdf
/def_output.f90 -- has a lot ... use Kyle's? Hard to tell which is Actors specific and which is just changed
/netcdf_util.f90 -- remove
/read_icondActors.-- has a lot ... could combine and remove, renamed for now,
/writeOutput.f90 -- part of modelwrite.f90 -- renamed to modelwrite so can replace, and then IS THIS VERSION BETTER can we change the other version and put a few compiler directives in?
/writeRestart.f90 -- remove, same in modelwrite, part of it
build/CMakeLists.txt 0 → 100644
View file @ 8e5f43c0
cmake_minimum_required(VERSION 3.10 FATAL_ERROR)
project(summa_actors LANGUAGES CXX Fortran)
enable_language(C)
SET (CMAKE_Fortran_COMPILER gfortran)
include(FortranCInterface)
FortranCInterface_VERIFY(CXX)
get_filename_component(PARENT_DIR "${CMAKE_CURRENT_SOURCE_DIR}" DIRECTORY)
set(EXEC_DIR ${PARENT_DIR}/bin) # set the output directory for executables
SET(F_MASTER ${PARENT_DIR}/build/summa)
# Add options for build type
set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${EXEC_DIR})
set(CMAKE_CONFIGURATION_TYPES BE BE_Debug BE_Cluster BE_Cluster_Debug)
# Set Compiler Options
if(CMAKE_BUILD_TYPE MATCHES Debug)
message("\nSetting Debug Options\n")
add_compile_definitions(DEBUG)
set(FLAGS_NOAH -g -O0 -fbacktrace -fbounds-check -ffree-form -ffree-line-length-none -fmax-errors=0 -fPIC -Wfatal-errors ${FLAGS_OPT})
set(FLAGS_ALL -g -O0 -fbacktrace -fbounds-check -ffree-line-length-none -fmax-errors=0 -fPIC -Wfatal-errors -cpp ${FLAGS_OPT})
set(FLAGS_CXX -g -O0 -fbounds-check -Wfatal-errors -std=c++17 ${FLAGS_OPT})
else()
message("\nSetting Release Options")
set(FLAGS_NOAH -O3 -ffree-form -ffree-line-length-none -fmax-errors=0 -fPIC -Wfatal-errors ${FLAGS_OPT})
set(FLAGS_ALL -O3 -ffree-line-length-none -fmax-errors=0 -fPIC -Wfatal-errors -cpp ${FLAGS_OPT})
set(FLAGS_CXX -O3 -Wfatal-errors -std=c++17 ${FLAGS_OPT})
endif()
find_package(CAF COMPONENTS core io REQUIRED)
set(CAF_INCLUDES ${CAF_INCLUDE_DIRS})
set(CAF_LIBRARIES CAF::core CAF::io)
set(EXEC_NAME summa_be)
if (CMAKE_BUILD_TYPE MATCHES Cluster)
find_package(OpenBLAS REQUIRED)
# Set include directories
set(INCLUDES $ENV{EBROOTNETCDFMINFORTRAN}/include ${netCDF_INCLUDES} ${OpenBLAS_INCLUDES})
set(LIBRARIES SUMMA_NOAHMP ${OpenBLAS_LIBRARIES} -lnetcdff)
set(INC_ACTORS
${CAF_INCLUDES}
${PARENT_DIR}/build/includes/global
${PARENT_DIR}/build/includes/summa_actor
${PARENT_DIR}/build/includes/job_actor
${PARENT_DIR}/build/includes/file_access_actor
${PARENT_DIR}/build/includes/hru_actor)
set(LIB_ACTORS
${CAF_LIBRARIES}
-lnetcdf
-lcaf_core
-lcaf_io)
else()
# Set include and library for Fortran portion of the code
set(INCLUDES "/usr/include" "/usr/local/include")
set(LIBRARIES -lnetcdff -lopenblas SUMMA_NOAHMP)
# Set include and library for C++ portion of the code
set(INC_ACTORS CAF::core CAF::io ${INCLUDES}
"${PARENT_DIR}/build/includes/global"
"${PARENT_DIR}/build/includes/summa_actor"
"${PARENT_DIR}/build/includes/job_actor"
"${PARENT_DIR}/build/includes/file_access_actor"
"${PARENT_DIR}/build/includes/hru_actor")
link_directories("/usr/local/lib")
set(LIB_ACTORS CAF::core CAF::io -lopenblas -lnetcdff -lnetcdf)
endif()
# Define directories that contains source code
set(DRIVER_DIR ${F_MASTER}/build/source/driver)
set(DSHARE_DIR ${F_MASTER}/build/source/dshare)
set(ENGINE_DIR ${F_MASTER}/build/source/engine)
set(HOOKUP_DIR ${F_MASTER}/build/source/hookup)
set(NETCDF_DIR ${F_MASTER}/build/source/netcdf)
set(NOAHMP_DIR ${F_MASTER}/build/source/noah-mp)
# Define Actors specific directories
set(ACTORS_DIR ${PARENT_DIR}/build/source/actors)
set(FILE_ACCESS_DIR ${ACTORS_DIR}/file_access_actor)
set(JOB_ACTOR_DIR ${ACTORS_DIR}/job_actor)
set(HRU_ACTOR_DIR ${ACTORS_DIR}/hru_actor)
# NOAHMP modules
set(NOAHMP
${NOAHMP_DIR}/module_model_constants.F
${NOAHMP_DIR}/module_sf_noahutl.F
${NOAHMP_DIR}/module_sf_noahlsm.F
${NOAHMP_DIR}/module_sf_noahmplsm.F)
# Free versions of numerical recipes utilities for NOAH-MP modules
set(NRUTIL
${ENGINE_DIR}/f2008funcs.f90
${ENGINE_DIR}/nr_utility.f90
${ENGINE_DIR}/nrtype.f90)
# Free versions of numerical recipes procedures for SUMMA modules
set(NRPROC
${ENGINE_DIR}/expIntegral.f90
${ENGINE_DIR}/spline_int.f90)
# Hook-up modules
set(HOOKUP
${HOOKUP_DIR}/ascii_util.f90
${HOOKUP_DIR}/summaFileManager.f90)
# Data modules
set(DATAMS
${DSHARE_DIR}/data_types.f90
${ACTORS_DIR}/global/actor_data_types.f90
${DSHARE_DIR}/flxMapping.f90
${DSHARE_DIR}/get_ixname.f90
${DSHARE_DIR}/globalData.f90
${DSHARE_DIR}/multiconst.f90
${DSHARE_DIR}/outpt_stat.f90
${DSHARE_DIR}/popMetadat.f90
${DSHARE_DIR}/var_lookup.f90)
# Utility modules
set(UTILMS
${ENGINE_DIR}/matrixOper.f90
${ENGINE_DIR}/mDecisions.f90
${ENGINE_DIR}/snow_utils.f90
${ENGINE_DIR}/soil_utils.f90
${ENGINE_DIR}/time_utils.f90
${ENGINE_DIR}/updatState.f90)
# NetCDF routines
set(NETCDF
${NETCDF_DIR}/def_output.f90
${NETCDF_DIR}/netcdf_util.f90
${NETCDF_DIR}/read_icond.f90)
# Preliminary modules
set(PRELIM
${ENGINE_DIR}/allocspace.f90
${ENGINE_DIR}/check_icond.f90
${ENGINE_DIR}/checkStruc.f90
${ENGINE_DIR}/childStruc.f90
${ENGINE_DIR}/conv_funcs.f90
${ENGINE_DIR}/convE2Temp.f90
${FILE_ACCESS_DIR}/ffile_info.f90
${ENGINE_DIR}/read_pinit.f90
${ENGINE_DIR}/read_attrb.f90
${ENGINE_DIR}/paramCheck.f90
${ENGINE_DIR}/pOverwrite.f90
${ENGINE_DIR}/sunGeomtry.f90
${ENGINE_DIR}/read_param.f90)
# Model run support modules
set(MODRUN
${ENGINE_DIR}/canopySnow.f90
${ENGINE_DIR}/derivforce.f90
${ENGINE_DIR}/getVectorz.f90
${ENGINE_DIR}/indexState.f90
${ENGINE_DIR}/layerMerge.f90
${ENGINE_DIR}/layerDivide.f90
${ENGINE_DIR}/qTimeDelay.f90
${ENGINE_DIR}/snowAlbedo.f90
${ENGINE_DIR}/snwCompact.f90
${ENGINE_DIR}/tempAdjust.f90
${ENGINE_DIR}/updateVars.f90
${ENGINE_DIR}/var_derive.f90
${ENGINE_DIR}/volicePack.f90)
# Solver main modules
set(SOLVER
${ENGINE_DIR}/bigAquifer.f90
${ENGINE_DIR}/computFlux.f90
${ENGINE_DIR}/computJacob.f90
${ENGINE_DIR}/computResid.f90
${ENGINE_DIR}/coupled_em.f90
${ENGINE_DIR}/diagn_evar.f90
${ENGINE_DIR}/eval8summa.f90
${ENGINE_DIR}/groundwatr.f90
${ENGINE_DIR}/opSplittin.f90
${ENGINE_DIR}/snowLiqFlx.f90
${ENGINE_DIR}/soilLiqFlx.f90
${ENGINE_DIR}/ssdNrgFlux.f90
${ENGINE_DIR}/stomResist.f90
${ENGINE_DIR}/summaSolve.f90
${ENGINE_DIR}/systemSolv.f90
${ENGINE_DIR}/varSubstep.f90
${ENGINE_DIR}/vegLiqFlux.f90
${ENGINE_DIR}/vegNrgFlux.f90
${ENGINE_DIR}/vegPhenlgy.f90
${ENGINE_DIR}/vegSWavRad.f90)
set(DRIVER
${DRIVER_DIR}/summa_alarms.f90
${DRIVER_DIR}/summa_globalData.f90)
# Actors interface modules
set(INTERFACE
${ACTORS_DIR}/global/cppwrap_auxiliary.f90
${ACTORS_DIR}/global/cppwrap_datatypes.f90
${ACTORS_DIR}/global/cppwrap_metadata.f90)
set(FILE_ACCESS_INTERFACE
${FILE_ACCESS_DIR}/cppwrap_fileAccess.f90
${FILE_ACCESS_DIR}/output_structure.f90
${FILE_ACCESS_DIR}/read_force.f90
${FILE_ACCESS_DIR}/fileAccess_writeOutput.f90)
set(JOB_INTERFACE
${JOB_ACTOR_DIR}/job_actor.f90)
set(HRU_INTERFACE
${HRU_ACTOR_DIR}/hru_init.f90
${HRU_ACTOR_DIR}/hru_read.f90
${HRU_ACTOR_DIR}/hru_modelRun.f90
${HRU_ACTOR_DIR}/hru_writeOutput.f90)
# Actors actual actor modules
set(ACTORS_GLOBAL
${ACTORS_DIR}/global/auxiliary.cpp
${ACTORS_DIR}/global/global.cpp
${ACTORS_DIR}/global/message_atoms.cpp
${ACTORS_DIR}/global/settings_functions.cpp
${ACTORS_DIR}/global/timing_info.cpp)
set(SUMMA_ACTOR
${ACTORS_DIR}/summa_actor/batch.cpp
${ACTORS_DIR}/summa_actor/batch_container.cpp
${ACTORS_DIR}/summa_actor/client.cpp
${ACTORS_DIR}/summa_actor/client_container.cpp
${ACTORS_DIR}/summa_actor/summa_actor.cpp
${ACTORS_DIR}/summa_actor/summa_backup_server.cpp
${ACTORS_DIR}/summa_actor/summa_client.cpp
${ACTORS_DIR}/summa_actor/summa_server.cpp)
set(FILE_ACCESS_ACTOR
${ACTORS_DIR}/file_access_actor/file_access_actor.cpp
${ACTORS_DIR}/file_access_actor/forcing_file_info.cpp
${ACTORS_DIR}/file_access_actor/output_container.cpp)
set(JOB_ACTOR
${ACTORS_DIR}/job_actor/GRU.cpp
${ACTORS_DIR}/job_actor/job_actor.cpp)
set(HRU_ACTOR
${ACTORS_DIR}/hru_actor/hru_actor.cpp)
#=========================================================================================
# COMPILE PART 3: Collect the subroutines into build groups depending on build type
#=========================================================================================
set(COMM_ALL
${NRPROC}
${HOOKUP}
${DATAMS}
${UTILMS}
${INTERFACE})
set(SUMMA_ALL
${NETCDF}
${PRELIM}
${MODRUN}
${SOLVER}
${DRIVER})
set(SUMMA_ALL
${SUMMA_ALL}
${FILE_ACCESS_INTERFACE}
${JOB_INTERFACE}
${HRU_INTERFACE})
set(MAIN_ACTOR ${ACTORS_DIR}/main.cpp)
# Define version number, not working correctly
set(VERSIONFILE ${DRIVER_DIR}/summaversion.inc)
execute_process(COMMAND " ${GIT_EXECUTABLE} tag | tail -n 1" OUTPUT_VARIABLE VERSION)
execute_process(COMMAND "date" OUTPUT_VARIABLE BULTTIM)
execute_process(COMMAND " ${GIT_EXECUTABLE} describe --long --all --always | sed -e's/heads\///'" OUTPUT_VARIABLE GITBRCH)
execute_process(COMMAND " ${GIT_EXECUTABLE} rev-parse HEAD" OUTPUT_VARIABLE GITHASH)
#=========================================================================================
# COMPILE PART 4: Do the compilation
#=========================================================================================
# update version information, not working correctly
file(WRITE ${VERSIONFILE} "character(len=64), parameter :: summaVersion = '${VERSION}'\n")
file(APPEND ${VERSIONFILE} "character(len=64), parameter :: buildTime = ''\n")
file(APPEND ${VERSIONFILE} "character(len=64), parameter :: gitBranch = '${GITBRCH}'\n")
file(APPEND ${VERSIONFILE} "character(len=64), parameter :: gitHash = '${GITHASH}'")
# Build SUMMA_NOAHMP Object
add_library(SUMMA_NOAHMP OBJECT ${NOAHMP} ${NRUTIL})
target_compile_options(SUMMA_NOAHMP PRIVATE ${FLAGS_NOAH})
# Build SUMMA_COMM Object
add_library(SUMMA_COMM OBJECT ${COMM_ALL})
target_compile_options(SUMMA_COMM PRIVATE ${FLAGS_ALL})
target_include_directories(SUMMA_COMM PRIVATE ${INCLUDES})
target_link_libraries(SUMMA_COMM PUBLIC SUMMA_NOAHMP ${FLAGS_ALL}) # added flags to the link step
add_library(summaactors SHARED ${SUMMA_ALL})
target_compile_options(summaactors PRIVATE ${FLAGS_ALL})
target_include_directories(summaactors PUBLIC ${INCLUDES})
target_link_libraries(summaactors PUBLIC ${LIBRARIES} SUMMA_NOAHMP SUMMA_COMM)
add_executable(${EXEC_NAME}
${MAIN_ACTOR}
${ACTORS_GLOBAL}
${FILE_ACCESS_ACTOR}
${JOB_ACTOR}
${HRU_ACTOR}
${SUMMA_ACTOR}
${SUMMA_CLIENT}
${SUMMA_SERVER})
set_property(TARGET ${EXEC_NAME} PROPERTY LINKER_LANGUAGE Fortran)
target_compile_options(${EXEC_NAME} PUBLIC ${FLAGS_CXX})
target_include_directories(${EXEC_NAME} PUBLIC ${INC_ACTORS})
target_link_libraries( ${EXEC_NAME} ${LIB_ACTORS} summaactors)
\ No newline at end of file
build/example_compile.sh deleted 100644 → 0
View file @ c29d3324
#!/bin/bash
#### load modules if using Compute Canada or Copernicus ####
# module load gcc/9.3.0
# module load netcdf-fortran
# module load openblas
# module load caf
#### Specifiy Master Directory, parent of build directory
# export F_MASTER=/path/to/summaActors
#### Specifiy Compilers ####
# export FC=gfortran
# export CC=g++
#### Includes and Libraries ####
# export INCLUDES = NETCDF and OPENBLAS
# export LIBRARIES = NETCDF and OPENBLAS
# export ACTORS_INCLUDES = C++ Actor Framework
# export ACTORS_LIBRARIES = C++ Actor Framework and
# The directory in which libsumma.so resides
#### Compile with the Makefile ####
# make -f ${F_MASTER}/build/makefile lib # libsumma.so part
# mv libsumma.so ${F_MASTER}/bin # optional move of libsumma (just ensure that summaMain knows where to find it)
# make -f ${F_MASTER}/build/makefile main # summaMain part
# mv summaMain ${F_MASTER}/bin # optional move, cleans things up
# export LD_LIBRARY_PATH=Path/to/libsumma.so
\ No newline at end of file
build/includes/file_access_actor/file_access_actor.hpp 0 → 100644
View file @ 8e5f43c0
#pragma once
#include "caf/actor.hpp"
#include "forcing_file_info.hpp"
#include "timing_info.hpp"
#include "output_container.hpp"
#include "settings_functions.hpp"
#include "fortran_data_types.hpp"
#include "auxilary.hpp"
#include "global.hpp"
/*********************************************
* File Access Actor Fortran Functions
*********************************************/
extern "C" {
void fileAccessActor_init_fortran(void* handle_forcing_file_info, int* num_forcing_files, int* num_timesteps,
int* num_timesteps_output_buffer, void* handle_output_ncid, int* startGRU,
int* numGRU, int* numHRU, int* err);
void writeOutput_fortran(void* handle_ncid, int* num_steps, int* start_gru, int* max_gru,
bool* writeParamFlag, int* err);
void read_forcingFile(void* forcFileInfo, int* currentFile, int* stepsInFile,
int* startGRU, int* numGRU, int* err);
void FileAccessActor_DeallocateStructures(void* handle_forcFileInfo, void* handle_ncid);
}
/*********************************************
* File Access Actor state variables
*********************************************/
namespace caf {
struct file_access_state {
// Variables set on Spawn
caf::actor parent;
int start_gru;
int num_gru;
void *handle_forcing_file_info = new_handle_file_info(); // Handle for the forcing file information
void *handle_ncid = new_handle_var_i(); // output file ids
int num_vectors_in_output_manager;
int num_steps;
int stepsInCurrentFile;
int numFiles;
int filesLoaded;
int err;
int num_output_steps;
Output_Container* output_container;
File_Access_Actor_Settings file_access_actor_settings;
std::vector<Forcing_File_Info> forcing_file_list; // list of steps in file
// Timing Variables
TimingInfo file_access_timing;
};
// called to spawn a file_access_actor
behavior file_access_actor(stateful_actor<file_access_state>* self, int startGRU, int numGRU,
File_Access_Actor_Settings file_access_actor_settings, actor parent);
/*********************************************
* Functions for the file access actor
*********************************************/
/* Setup and call the fortran routine that writes the output */
void writeOutput(stateful_actor<file_access_state>* self, Output_Partition* partition);
} // end namespace
\ No newline at end of file