diff --git a/bin/submission_script_array.sh b/bin/submission_script_array.sh index 37dc1fc3156d12793cd2eaa822cbf6624d59b7b6..d6becea03f67899d3218ecb3cd2cbf761cac4c5c 100644 --- a/bin/submission_script_array.sh +++ b/bin/submission_script_array.sh @@ -1,20 +1,22 @@ #!/bin/bash -#SBATCH --cpus-per-task=8 +#SBATCH --cpus-per-task=32 +#SBATCH --nodes=1 #SBATCH --time=24:00:00 -#SBATCH --mem=32G +#SBATCH --mem=0 +#SBATCH --exclusive #SBATCH --constraint=broadwell #SBATCH --job-name=Summa-Actors #SBATCH --mail-user=kyle.klenk@usask.ca #SBATCH --mail-type=ALL -#SBATCH --output=/home/kklenk/scratch/OUTPUT/FULL_NA_RUN/slurm/slurm-%A_%a.out +#SBATCH --output=/home/kklenk/scratch/BE_Output/slurm/slurm-%A_%a.out #SBATCH --account=rpp-kshook -#SBATCH --array=0-517 +#SBATCH --array=0-129 gru_max=517315 -gru_count=1000 -max_job=518 -summa_exe=/home/kklenk/Summa-Projects/Summa-Actors/bin/summaMain -config_summa=/home/kklenk/projects/rpp-kshook/kklenk/INPUT/summa_settings/Summa_Actors_Settings.json +gru_count=4000 +max_job=130 +summa_exe=/home/kklenk/Summa-Projects/Summa-Actors/bin/summa_actors +config_summa=/home/kklenk/projects/rpp-kshook/kklenk/Sundials_Settings/Summa_Actors_Settings.json config_caf=/home/kklenk/Summa-Projects/Summa-Actors/bin/caf-application.conf offset=$SLURM_ARRAY_TASK_ID diff --git a/bin/submit_summa_original.sh b/bin/submit_summa_original.sh new file mode 100644 index 0000000000000000000000000000000000000000..788635a2941cd54538c072b8bc616f624a10c0b6 --- /dev/null +++ b/bin/submit_summa_original.sh @@ -0,0 +1,51 @@ +#!/bin/bash +#SBATCH --cpus-per-task=8 +#SBATCH --time=72:00:00 +#SBATCH --mem=32G +#SBATCH --constraint=broadwell +#SBATCH --job-name=Summa-Actors +#SBATCH --mail-user=kyle.klenk@usask.ca +#SBATCH --mail-type=ALL +#SBATCH --output=/home/kklenk/scratch/Sundials_Output/slurm/slurm-%A_%a.out +#SBATCH --account=rpp-kshook +#SBATCH --array=0-517 + +total_gru=517315 +gru_per_job=1000 +max_job=518 +cpu_num=8 +summa_exe=summa.exe +file_manager=fileManager.txt +log_dir=logs + +offset=$SLURM_ARRAY_TASK_ID +gru_start=$(( 1 + gru_per_job*offset )) +job_check=$(( $gru_start + $gru_per_job )) + +# Adust the number of grus for the last job +if [ $job_check -gt $total_gru ] +then + gru_per_job=$(( gruMax-gru_start+1 )) +fi + +gru_per_task=$(( gru_per_job/cpu_num )) + +for ((cpu=0; cpu<cpu_num; cpu++)) +do + task_gru_start=$(( gru_start + cpu * gru_per_task )) + task_check=$(( task_gru_start + gru_per_task )) + + if [ $task_check -gt $total_gru ] || \ + ( [ $task_check -lt $total_gru ] && \ + [ $cpu -eq $(( cpu_num-1 )) ] && \ + [ $offset -eq $(( max_job-1 )) ] ) + then + gru_per_task=$(( total_gru - task_gru_start)) + fi + + $summa_exe \ + -g $task_gru_start $gru_per_task \ + -m $file_manager \ + > $log_dir/summa_log_$task_gru_start.txt & +done +wait diff --git a/build/makefiles/compile_graham.sh b/build/makefiles/compile_graham.sh index e9fff35038ae8f3ae0f50a3814ccdb3cbb67a6f9..13c2881fc3f3ab4ad72d80c3af1fc56d9e9140a8 100644 --- a/build/makefiles/compile_graham.sh +++ b/build/makefiles/compile_graham.sh @@ -8,23 +8,23 @@ module load caf #### parent directory of the 'build' directory #### -export ROOT_DIR=/home/kklenk/Summa-Projects/Summa-Actors +# export ROOT_DIR=/home/kklenk/Summa-Projects/Summa-Actors -export INCLUDES="-I$EBROOTNETCDFMINFORTRAN/include" -export LIBRARIES="-L$EBROOTNETCDFMINFORTRAN/lib64\ - -L$EBROOTOPENBLAS/lib\ - -lnetcdff -lopenblas" +# export INCLUDES="-I$EBROOTNETCDFMINFORTRAN/include" +# export LIBRARIES="-L$EBROOTNETCDFMINFORTRAN/lib64\ +# -L$EBROOTOPENBLAS/lib\ +# -lnetcdff -lopenblas" -# INCLUDES FOR Actors Component -export ACTORS_INCLUDES="-I$EBROOTCAF/include\ - -I$EBROOTNETCDFMINFORTRAN/include" +# # INCLUDES FOR Actors Component +# export ACTORS_INCLUDES="-I$EBROOTCAF/include\ +# -I$EBROOTNETCDFMINFORTRAN/include" -export ACTORS_LIBRARIES="-L$EBROOTCAF/lib\ - -L$EBROOTCAF/lib64\ - -L$EBROOTNETCDFMINFORTRAN/lib64\ - -L$EBROOTOPENBLAS/lib\ - -L$ROOT_DIR/bin\ - -lcaf_core -lcaf_io -lsumma -lopenblas -lnetcdff" +# export ACTORS_LIBRARIES="-L$EBROOTCAF/lib\ +# -L$EBROOTCAF/lib64\ +# -L$EBROOTNETCDFMINFORTRAN/lib64\ +# -L$EBROOTOPENBLAS/lib\ +# -L$ROOT_DIR/bin\ +# -lcaf_core -lcaf_io -lsumma -lopenblas -lnetcdff" -make -f ${ROOT_DIR}/build/makefiles/makefile_cluster cpp -export LD_LIBRARY_PATH=${ROOT_DIR}/bin \ No newline at end of file +# make -f ${ROOT_DIR}/build/makefiles/makefile_cluster cpp +# export LD_LIBRARY_PATH=${ROOT_DIR}/bin \ No newline at end of file diff --git a/build/source/actors/file_access_actor/fortran_code/writeOutputFromOutputStructure.f90 b/build/source/actors/file_access_actor/fortran_code/writeOutputFromOutputStructure.f90 index 1e025c39fd07368700e32d34489c1976cfa6ebf1..42380d20dd25a201eb79f684ef0c64c013594fde 100644 --- a/build/source/actors/file_access_actor/fortran_code/writeOutputFromOutputStructure.f90 +++ b/build/source/actors/file_access_actor/fortran_code/writeOutputFromOutputStructure.f90 @@ -603,9 +603,9 @@ subroutine writeBasin(ncid,outputTimestep,outputTimestepUpdate,nSteps,& end subroutine writeBasin - ! ************************************************************************************** - ! public subroutine writeTime: write current time to all files - ! ************************************************************************************** +! ************************************************************************************** +! public subroutine writeTime: write current time to all files +! ************************************************************************************** subroutine writeTime(ncid,outputTimestep,iStep,meta,dat,err,message) USE data_types,only:var_info ! metadata type USE var_lookup,only:iLookStat ! index into stat structure diff --git a/build/summa b/build/summa new file mode 160000 index 0000000000000000000000000000000000000000..fa9adf808229a45085defdc2bb8ef05836b9b3aa --- /dev/null +++ b/build/summa @@ -0,0 +1 @@ +Subproject commit fa9adf808229a45085defdc2bb8ef05836b9b3aa diff --git a/build/summa-sundials b/build/summa-sundials new file mode 160000 index 0000000000000000000000000000000000000000..c37bac1ddcee9fb00deeeef0df1880587f85c02a --- /dev/null +++ b/build/summa-sundials @@ -0,0 +1 @@ +Subproject commit c37bac1ddcee9fb00deeeef0df1880587f85c02a diff --git a/utils/StatisticsScripts/resourageUsage.py b/utils/StatisticsScripts/resourageUsage.py index 49e9e8ef8a88a98606799dbe2750ed3cd30fd71b..9de967c57c9c001fc5cc2badaaf602bfac775028 100644 --- a/utils/StatisticsScripts/resourageUsage.py +++ b/utils/StatisticsScripts/resourageUsage.py @@ -28,7 +28,7 @@ def seffCommand(jobId, numJobs, gru_per_job): raise Exception("Something went wrong") csvFile = open(output_csv_name, 'w') - header = ["startHRU", "numHRU", "#-CPU", "CPU Efficiency", "Wall-Clock Time", "Memory Used", "Status"] + header = ["startHRU", "numHRU", "#-CPU", "CPU Efficiency", "Wall-Clock Time", "CPU-Utilized", "Memory Used", "Status"] writer = csv.writer(csvFile) @@ -55,6 +55,10 @@ def seffCommand(jobId, numJobs, gru_per_job): wallClock = line.decode().split(" ")[-1] wallClock = wallClock.strip() + if b'CPU Utilized: ' in line: + cpuUtilized = line.decode().split(" ")[-1] + cpuUtilized = cpuUtilized.strip() + if b'Memory Utilized:' in line: memory = line.decode().split(" ")[2] memory = memory.strip() @@ -66,6 +70,7 @@ def seffCommand(jobId, numJobs, gru_per_job): rowData.append(cores) rowData.append(effeciency) rowData.append(wallClock) + rowData.append(cpuUtilized) rowData.append(memory) rowData.append(status) writer.writerow(rowData) diff --git a/utils/netcdf/OutputVerification/checkOutput.py b/utils/netcdf/OutputVerification/checkOutput.py index 5dfec8708e34f380842834dd971d7901c65a4a47..12b31dbdddd03374e143db5953c254be8e88e109 100644 --- a/utils/netcdf/OutputVerification/checkOutput.py +++ b/utils/netcdf/OutputVerification/checkOutput.py @@ -80,15 +80,15 @@ def get_output_vars(model_output_file): -num_hru = 10 +num_hru = 125 print("Checking output for", num_hru, "HRUs") -dataset_1 = "/scratch/kck540/Summa_Sundials/non-actors/SummaOriginal_G000001-000010_timestep.nc" -dataset_2 = "/scratch/kck540/Summa_Sundials/actors/SummaActorsGRU1-10_timestep.nc" +dataset_1 = "/home/kklenk/scratch/Sundials_non_Actors_Output/netcdf/Summa-Sundials-_G112001-112125_timestep.nc" +dataset_2 = "/home/kklenk/scratch/Sundials_Actors_Output/netcdf/Summa-Actors-Sundials-GRU112001-500_timestep.nc" # dataset_1 = "/scratch/kck540/Summa_Sundials/non-actors/SummaOriginal-BE_G000001-000002_timestep.nc" # dataset_2 = "/scratch/kck540/Summa_Sundials/actors/SummaActors-BEGRU1-2_timestep.nc" -model_output_file = "/gladwell/kck540/Summa_Distributed_Settings/Summa_Settings/outputControl.txt" +model_output_file = "/home/kklenk/projects/rpp-kshook/kklenk/Sundials_Settings/outputControl.txt" output_vars = get_output_vars(model_output_file) verify_data(dataset_1, dataset_2, num_hru, output_vars)