From dd25f2359dad4f81882b784accaad8d14fa30a53 Mon Sep 17 00:00:00 2001
From: KyleKlenk <kyle.c.klenk@gmail.com>
Date: Wed, 23 Mar 2022 11:05:04 -0600
Subject: [PATCH] Directory Reconfiguration
---
.../copernicus/run_client_server.sh | 23 ----------
3_SummaActors_Run/copernicus/run_one.sh | 11 -----
3_SummaActors_Run/copernicus/run_summa.sh | 42 -------------------
3_SummaActors_Run/copernicus/run_test.sh | 12 ------
3_SummaActors_Run/copernicus/summaActors.sh | 11 -----
3_SummaActors_Run/graham/run_allHRUs.sh | 30 -------------
3_SummaActors_Run/graham/run_client.sh | 18 --------
3_SummaActors_Run/graham/run_one.sh | 12 ------
3_SummaActors_Run/graham/run_server.sh | 11 -----
3_SummaActors_Run/graham/run_server_client.sh | 28 -------------
3_SummaActors_Run/graham/run_summa.sh | 42 -------------------
3_SummaActors_Run/graham/test.sh | 18 --------
.../caf-application.conf | 2 +-
Dockerfile => utils/docker/Dockerfile | 0
.../docker/docker-compose.yml | 0
.../netcdf}/compareOutput.py | 0
.../netcdf}/mergeNetcdf.py | 0
.../sbatch}/run_allHRUs.sh | 0
18 files changed, 1 insertion(+), 259 deletions(-)
delete mode 100644 3_SummaActors_Run/copernicus/run_client_server.sh
delete mode 100644 3_SummaActors_Run/copernicus/run_one.sh
delete mode 100644 3_SummaActors_Run/copernicus/run_summa.sh
delete mode 100644 3_SummaActors_Run/copernicus/run_test.sh
delete mode 100644 3_SummaActors_Run/copernicus/summaActors.sh
delete mode 100644 3_SummaActors_Run/graham/run_allHRUs.sh
delete mode 100644 3_SummaActors_Run/graham/run_client.sh
delete mode 100644 3_SummaActors_Run/graham/run_one.sh
delete mode 100644 3_SummaActors_Run/graham/run_server.sh
delete mode 100755 3_SummaActors_Run/graham/run_server_client.sh
delete mode 100644 3_SummaActors_Run/graham/run_summa.sh
delete mode 100755 3_SummaActors_Run/graham/test.sh
rename {3_SummaActors_Run/copernicus => config}/caf-application.conf (78%)
rename Dockerfile => utils/docker/Dockerfile (100%)
rename docker-compose.yml => utils/docker/docker-compose.yml (100%)
rename {4_SummaActors_PostProcess => utils/netcdf}/compareOutput.py (100%)
rename {4_SummaActors_PostProcess => utils/netcdf}/mergeNetcdf.py (100%)
rename {3_SummaActors_Run/copernicus => utils/sbatch}/run_allHRUs.sh (100%)
diff --git a/3_SummaActors_Run/copernicus/run_client_server.sh b/3_SummaActors_Run/copernicus/run_client_server.sh
deleted file mode 100644
index 86ed356..0000000
--- a/3_SummaActors_Run/copernicus/run_client_server.sh
+++ /dev/null
@@ -1,23 +0,0 @@
-#!/bin/bash
-#SBATCH --cpus-per-task=1
-#SBATCH --time=8:00:00
-#SBATCH --mem=2G
-#SBATCH --job-name=SummaActorsOutputBugTest
-#SBATCH --mail-user=kyle.klenk@usask.ca
-#SBATCH --mail-type=ALL
-#SBATCH --output=/scratch/gwf/gwf_cmt/kck540/summaActors/slurm/slurm-%A_%a.out
-#SBATCH --account=hpc_c_giws_clark
-
-# EXAMPLE: sbatch --array=0-50 run_server_client.sh
-
-gruMax=517315 # North America, Merit Hydro basins
-gruCount=50000 # The number of GRUs you want to compute
-startGRU=1 # The starting index of the GRU you would like to compute
-offset=$SLURM_ARRAY_TASK_ID
-
-if [ $offset -eq 0 ]
-then
- /globalhome/kck540/HPC/SummaActors/bin/summaMain -s -p 4444 -g 1 -c $gruCount -m /project/gwf/gwf_cmt/kck540/domain_NorthAmerica/settings/SUMMA/fileManager.txt
-else
- /globalhome/kck540/HPC/SummaActors/bin/summaMain -p 4444 -j $offset -m /project/gwf/gwf_cmt/kck540/domain_NorthAmerica/settings/SUMMA/fileManager.txt
-fi
diff --git a/3_SummaActors_Run/copernicus/run_one.sh b/3_SummaActors_Run/copernicus/run_one.sh
deleted file mode 100644
index aac01f3..0000000
--- a/3_SummaActors_Run/copernicus/run_one.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-#SBATCH --cpus-per-task=8
-#SBATCH --time=36:00:00
-#SBATCH --mem=32G
-#SBATCH --job-name=SummaActors-1-250
-#SBATCH --mail-user=kyle.klenk@usask.ca
-#SBATCH --mail-type=ALL
-#SBATCH --output=/scratch/gwf/gwf_cmt/kck540/summaActors/slurm-Mar-21/1-250-8CPU-%A.out
-#SBATCH --account=hpc_c_giws_clark
-
-/globalhome/kck540/HPC/SummaActors/bin/summaMain -g 1 -c 250 -m /project/gwf/gwf_cmt/kck540/domain_NorthAmerica/settings/SUMMA/fileManager.txt -v -8CPU --config-file=/globalhome/kck540/HPC/SummaActors/3_SummaActors_Run/copernicus/caf-application.conf
diff --git a/3_SummaActors_Run/copernicus/run_summa.sh b/3_SummaActors_Run/copernicus/run_summa.sh
deleted file mode 100644
index 2f67b80..0000000
--- a/3_SummaActors_Run/copernicus/run_summa.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-#SBATCH --cpus-per-task=8
-#SBATCH --time=71:00:00
-#SBATCH --mem=24G
-#SBATCH --job-name=SummaActorsNA
-#SBATCH --mail-user=kyle.klenk@usask.ca
-#SBATCH --mail-type=ALL
-#SBATCH --output=/project/gwf/gwf_cmt/kck540/domain_NorthAmerica/simulations/slurm/slurm-%A_%a.out
-#SBATCH --account=hpc_c_giws_clark
-
-# Ensure array size is correct ie. --array=1-[numJobs]
-#SBATCH --array=0-2499
-
-JOBLIST=/globalhome/kck540/HPC/SummaActors/3_MS_model_runs/run_graham_serialSumma_NA_gridEra5_1/copernicus/joblist/summa_joblist.txt
-
-
-# Make the log directory in case
-mkdir -p logs/
-# ----------------------------------------------------------------------------------------------
-# RUN WITH
-# sbatch run_summa.sh
-# ----------------------------------------------------------------------------------------------
-
-JOBSTRING=$(sed -n "${SLURM_ARRAY_TASK_ID}p" $JOBLIST)
-
-# Seperate the SUMA call and its agruments from the ">" delimiter
-OIFS=$IFS
-IFS=">"
-read -ra COMMAND <<< "$JOBSTRING"
-IFS=$OIFS
-
-#--------------------------------------------------------------------
-# RUN
-#--------------------------------------------------------------------
-echo "Starting run at: `date`"
-
-${COMMAND[0]} > ${COMMAND[1]}
-
-# Echo end-of-run
-echo "Program finished with exit code $? ar: `date`"
-
-${JOBSTRING}
\ No newline at end of file
diff --git a/3_SummaActors_Run/copernicus/run_test.sh b/3_SummaActors_Run/copernicus/run_test.sh
deleted file mode 100644
index c6dced4..0000000
--- a/3_SummaActors_Run/copernicus/run_test.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-#SBATCH --cpus-per-task=1
-#SBATCH --time=0:30:00
-#SBATCH --mem=250M
-#SBATCH --job-name=SummaActors
-#SBATCH --mail-user=kyle.klenk@usask.ca
-#SBATCH --mail-type=ALL
-#SBATCH --output=/scratch/gwf/gwf_cmt/kck540/summaActors/slurm/slurm-%A_%a.out
-#SBATCH --account=hpc_c_giws_clark
-
-
-/globalhome/kck540/HPC/SummaActors/bin/summaTest -g 1 -c 4
diff --git a/3_SummaActors_Run/copernicus/summaActors.sh b/3_SummaActors_Run/copernicus/summaActors.sh
deleted file mode 100644
index f4934cf..0000000
--- a/3_SummaActors_Run/copernicus/summaActors.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-#SBATCH --cpus-per-task=6
-#SBATCH --time=24:00:00
-#SBATCH --mem=24G
-#SBATCH --job-name=SummaActors
-#SBATCH --mail-user=kyle.klenk@usask.ca
-#SBATCH --mail-type=ALL
-#SBATCH --output=/scratch/gwf/gwf_cmt/kck540/summaActors/slurm/slurm-%A.out
-#SBATCH --account=hpc_c_giws_clark
-
-/globalhome/kck540/HPC/SummaActors/bin/summaMain -g 1000 -c 518
diff --git a/3_SummaActors_Run/graham/run_allHRUs.sh b/3_SummaActors_Run/graham/run_allHRUs.sh
deleted file mode 100644
index 67ec5b2..0000000
--- a/3_SummaActors_Run/graham/run_allHRUs.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-#SBATCH --cpus-per-task=8
-#SBATCH --time=30:00:00
-#SBATCH --mem=32G
-#SBATCH --job-name=SummaActorsStressTest
-#SBATCH --mail-user=kyle.klenk@usask.ca
-#SBATCH --mail-type=ALL
-#SBATCH --output=/home/kklenk/scratch/SummaActorsOutput/slurm/slurm-%A_%a.out
-#SBATCH --account=def-spiteri
-
-# EXAMPLE: sbatch --array=0-258 run_allHRUs.sh
-
-# Define the GRU settings
-gruMax=517315 # North America, Merit Hydro basins
-gruCount=2000
-
-# Get the array ID for further use
-offset=$SLURM_ARRAY_TASK_ID
-
-# Start at 1 for array ID 1, 2022 for array ID 2, etc
-gruStart=$(( 1 + gruCount*offset ))
-check=$(( $gruStart + $gruCount ))
-# Check that we don't specify too many basins
-if [ $check -gt $gruMax ]
-then
- gruCount=$(( gruMax-gruStart+1 ))
-fi
-
-/home/kklenk/SummaProjects/SummaActors/bin/summaMain -g ${gruStart} -c ${gruCount} -m /project/6008034/kklenk/settings/SummaActorsSettings/fileManager.txt
-
diff --git a/3_SummaActors_Run/graham/run_client.sh b/3_SummaActors_Run/graham/run_client.sh
deleted file mode 100644
index 6001c94..0000000
--- a/3_SummaActors_Run/graham/run_client.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-#SBATCH --cpus-per-task=1
-#SBATCH --time=0:30:00
-#SBATCH --mem=2G
-#SBATCH --job-name=Summa-StressTest
-#SBATCH --mail-user=kyle.klenk@usask.ca
-#SBATCH --mail-type=ALL
-#SBATCH --output=/home/kklenk/scratch/SummaActorsOutput/slurm/slurm-client%A_%a.out
-#SBATCH --account=rpp-kshook
-
-# ----------------------------------------------------------------------------------------------
-# RUN WITH:
-# sbatch --array1-[number of jobs] [script name]
-#
-# EXAMPLE: sbatch --array=0-100 run_all.sh
-# ----------------------------------------------------------------------------------------------
-
-/home/kklenk/SummaProjects/SummaActors/bin/summaTest -g 1 -c 100
\ No newline at end of file
diff --git a/3_SummaActors_Run/graham/run_one.sh b/3_SummaActors_Run/graham/run_one.sh
deleted file mode 100644
index 05474ec..0000000
--- a/3_SummaActors_Run/graham/run_one.sh
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/bin/bash
-#SBATCH --cpus-per-task=4
-#SBATCH --time=02:00:00
-#SBATCH --mem=8G
-#SBATCH --job-name=TEST_summa_na_grid_era5
-#SBATCH --mail-user=kyle.klenk@usask.ca
-#SBATCH --mail-type=ALL
-#SBATCH --output=/home/kklenk/scratch/SummaActorsOutput/slurm/slurm-%A.out
-#SBATCH --account=rpp-kshook
-
-/home/kklenk/SummaProjects/SummaActors/bin/summaMain -g 1 -c 100 -m /project/6008034/kklenk/settings/SummaActorsSettings/fileManager.txt
-1
\ No newline at end of file
diff --git a/3_SummaActors_Run/graham/run_server.sh b/3_SummaActors_Run/graham/run_server.sh
deleted file mode 100644
index 04e285c..0000000
--- a/3_SummaActors_Run/graham/run_server.sh
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-#SBATCH --cpus-per-task=1
-#SBATCH --time=96:00:00
-#SBATCH --mem=1G
-#SBATCH --job-name=Summa-StressTest
-#SBATCH --mail-user=kyle.klenk@usask.ca
-#SBATCH --mail-type=ALL
-#SBATCH --output=/home/kklenk/scratch/SummaActorsOutput/slurm/slurmServer-%A_%a.out
-#SBATCH --account=rpp-kshook
-
-/home/kklenk/SummaProjects/SummaActors/bin/summaMain -s -p 4444 -g 1 -c 1000 -m /project/6008034/kklenk/settings/SummaActorsSettings/fileManager.txt
\ No newline at end of file
diff --git a/3_SummaActors_Run/graham/run_server_client.sh b/3_SummaActors_Run/graham/run_server_client.sh
deleted file mode 100755
index 2825cc4..0000000
--- a/3_SummaActors_Run/graham/run_server_client.sh
+++ /dev/null
@@ -1,28 +0,0 @@
-#!/bin/bash
-#SBATCH --cpus-per-task=1
-#SBATCH --time=24:00:00
-#SBATCH --mem=2G
-#SBATCH --job-name=SummaActors-StressTest
-#SBATCH --mail-user=kyle.klenk@usask.ca
-#SBATCH --mail-type=ALL
-#SBATCH --output=/home/kklenk/scratch/SummaActorsOutput/slurm/slurm-%A_%a.out
-#SBATCH --account=def-spiteri
-
-# ----------------------------------------------------------------------------------------------
-# RUN WITH:
-# sbatch --array1-[number of jobs] [script name]
-#
-# EXAMPLE: sbatch --array=0-50 run_server_client.sh
-# ----------------------------------------------------------------------------------------------
-
-gruMax=517315 # North America, Merit Hydro basins
-gruCount=10000 # The number of GRUs you want to compute
-startGRU=1 # The starting index of the GRU you would like to compute
-offset=$SLURM_ARRAY_TASK_ID
-
-if [ $offset -eq 0 ]
-then
- /home/kklenk/SummaProjects/SummaActors/bin/summaMain -s -p 4444 -g 1 -c $gruCount -m /project/6008034/kklenk/settings/SummaActorsSettings/fileManager.txt
-else
- /home/kklenk/SummaProjects/SummaActors/bin/summaMain -p 4444 -j $offset -m /project/6008034/kklenk/settings/SummaActorsSettings/fileManager.txt
-fi
diff --git a/3_SummaActors_Run/graham/run_summa.sh b/3_SummaActors_Run/graham/run_summa.sh
deleted file mode 100644
index 48d26a7..0000000
--- a/3_SummaActors_Run/graham/run_summa.sh
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/bin/bash
-#SBATCH --cpus-per-task=1
-#SBATCH --time=96:00:00
-#SBATCH --mem=1G
-#SBATCH --job-name=TEST_summa_na_grid_era5
-#SBATCH --mail-user=kyle.klenk@usask.ca
-#SBATCH --mail-type=ALL
-#SBATCH --output=/home/kklenk/slurmlog/slurm-%A_%a.out
-#SBATCH --account=rpp-kshook
-
-# Ensure array size is correct ie. --array=1-[numJobs]
-#SBATCH --array=1-10
-
-JOBLIST=/home/kklenk/actors/summa/code/graham/3_MS_model_runs/run_graham_serialSumma_NA_gridEra5_1/joblist/summa_joblist.txt
-
-
-# Make the log directory in case
-mdkir -p logs/
-# ----------------------------------------------------------------------------------------------
-# RUN WITH
-# sbatch run_summa.sh
-# ----------------------------------------------------------------------------------------------
-
-JOBSTRING=$(sed -n "${SLURM_ARRAY_TASK_ID}p" $JOBLIST)
-
-# Seperate the SUMA call and its agruments from the ">" delimiter
-OIFS=$IFS
-IFS=">"
-read -ra COMMAND <<< "$JOBSTRING"
-IFS=$OIFS
-
-#--------------------------------------------------------------------
-# RUN
-#--------------------------------------------------------------------
-echo "Starting run at: `date`"
-
-${COMMAND[0]} > ${COMMAND[1]}
-
-# Echo end-of-run
-echo "Program finished with exit code $? ar: `date`"
-
-${JOBSTRING}
\ No newline at end of file
diff --git a/3_SummaActors_Run/graham/test.sh b/3_SummaActors_Run/graham/test.sh
deleted file mode 100755
index b8a975b..0000000
--- a/3_SummaActors_Run/graham/test.sh
+++ /dev/null
@@ -1,18 +0,0 @@
-#!/bin/bash
-gruMax=517315 # North America, Merit Hydro basins
-gruCount=2000 # The number of GRUs you want to compute
-jobID=0
-
-
-for i in {0..258}
-do
- gruStart=$(( 1 + gruCount*jobID))
- check=$(( $gruStart + $gruCount ))
- if [ $check -gt $gruMax ]
- then
- echo "HERe"
- gruCount=$(( gruMax-gruStart+1 ))
- fi
- echo "gruStart = $gruStart, gruCount = $gruCount"
- jobID=$(( 1 + jobID ))
-done
\ No newline at end of file
diff --git a/3_SummaActors_Run/copernicus/caf-application.conf b/config/caf-application.conf
similarity index 78%
rename from 3_SummaActors_Run/copernicus/caf-application.conf
rename to config/caf-application.conf
index 682c8da..529bd51 100644
--- a/3_SummaActors_Run/copernicus/caf-application.conf
+++ b/config/caf-application.conf
@@ -1,6 +1,6 @@
caf {
# Parameters selecting a default scheduler.
scheduler {
- max-threads = 8
+ max-threads = 2
}
}
\ No newline at end of file
diff --git a/Dockerfile b/utils/docker/Dockerfile
similarity index 100%
rename from Dockerfile
rename to utils/docker/Dockerfile
diff --git a/docker-compose.yml b/utils/docker/docker-compose.yml
similarity index 100%
rename from docker-compose.yml
rename to utils/docker/docker-compose.yml
diff --git a/4_SummaActors_PostProcess/compareOutput.py b/utils/netcdf/compareOutput.py
similarity index 100%
rename from 4_SummaActors_PostProcess/compareOutput.py
rename to utils/netcdf/compareOutput.py
diff --git a/4_SummaActors_PostProcess/mergeNetcdf.py b/utils/netcdf/mergeNetcdf.py
similarity index 100%
rename from 4_SummaActors_PostProcess/mergeNetcdf.py
rename to utils/netcdf/mergeNetcdf.py
diff --git a/3_SummaActors_Run/copernicus/run_allHRUs.sh b/utils/sbatch/run_allHRUs.sh
similarity index 100%
rename from 3_SummaActors_Run/copernicus/run_allHRUs.sh
rename to utils/sbatch/run_allHRUs.sh
--
GitLab