From 5e81748b048439a7d495307008402aaca7b9b279 Mon Sep 17 00:00:00 2001 From: KyleKlenk <kyle.c.klenk@gmail.com> Date: Wed, 22 Feb 2023 09:47:59 -0600 Subject: [PATCH] Moved submission scripts to utils modified cmakelists for making kinsol output and be added files for outputing csv --- .gitignore | 8 + bin/caf-application.conf | 2 +- build/cmake/CMakeLists.txt | 341 +--------------- build/cmake/be.cmake | 9 +- build/cmake/kinsol.cmake | 3 + build/makefiles/Makefile_Summa | 385 ++++++++++++++++++ build/source/dshare/csv_file.f90 | 274 +++++++++++++ build/source/dshare/csv_file_1d.f90 | 22 + build/source/dshare/csv_file_2d.f90 | 19 + utils/sbatch/submission_copern.sh | 11 + {bin => utils/sbatch}/submission_one.sh | 0 .../sbatch}/submission_script_array.sh | 0 .../sbatch}/submit_summa_original.sh | 0 13 files changed, 745 insertions(+), 329 deletions(-) create mode 100755 build/makefiles/Makefile_Summa create mode 100644 build/source/dshare/csv_file.f90 create mode 100644 build/source/dshare/csv_file_1d.f90 create mode 100644 build/source/dshare/csv_file_2d.f90 create mode 100755 utils/sbatch/submission_copern.sh rename {bin => utils/sbatch}/submission_one.sh (100%) rename {bin => utils/sbatch}/submission_script_array.sh (100%) rename {bin => utils/sbatch}/submit_summa_original.sh (100%) diff --git a/.gitignore b/.gitignore index d331c58..bc92fcc 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,11 @@ build/summa_original build/source/testing/containers/output_container/main bin/submission_script.sh build/summa_kinsol +bin/summa_kinsol +build/flibs-0.9/ +build/cmake/build_be/ +build/cmake/build_kinsol/ +bin/data_be.csv +bin/data_kinsol.csv +bin/submission_copern.sh +bin/summa_be diff --git a/bin/caf-application.conf b/bin/caf-application.conf index 5774be2..3e1e447 100644 --- a/bin/caf-application.conf +++ b/bin/caf-application.conf @@ -1,3 +1,3 @@ caf { - max-threads = 2 + max-threads = 1 } \ No newline at end of file diff --git a/build/cmake/CMakeLists.txt b/build/cmake/CMakeLists.txt index 327dcec..c79d141 100644 --- a/build/cmake/CMakeLists.txt +++ b/build/cmake/CMakeLists.txt @@ -3,20 +3,31 @@ cmake_minimum_required(VERSION 3.10 FATAL_ERROR) set(PARENT_DIR ../../) set(EXEC_DIR ${PARENT_DIR}../bin) set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${EXEC_DIR}) # set the output directory for executables -set(EXEC_NAME summa_actors) project(summa_actors LANGUAGES CXX Fortran) enable_language(C) SET (CMAKE_Fortran_COMPILER gfortran) include(FortranCInterface) FortranCInterface_VERIFY(CXX) -option(SUNDIALS "Use SUNDIALS" OFF) - +# SUMMA-Actors can be built in three configurations: +# 1. SUMMA-Actors with SUNDIALS (Kinsol) solver +# 2. SUMMA-Actors with SUNDIALS (IDA) solver +# 3. SUMMA-Actors without SUNDIALS solver, using the original SUMMA home coded solver +# -------------------------------------------------------------------------------------------- +# Each configuration of Summa has a corresponding CMakeLists.txt file in the build directory include(kinsol.cmake) include(be.cmake) +include(ida.cmake) +# -------------------------------------------------------------------------------------------- +# When compiling Summa-Actors each configuration can be compiled in a Cluster or Debug mode, +# which is set by the CMAKE_BUILD_TYPE variable. The default is Release, non-cluster mode. + + + ######### SET THE PATHS TO THE SUNDIALS LIBRARIES AND INCLUDE FILES ######### ############################################################################# -set(DIR_SUNDIALS "/home/kklenk/projects/rpp-kshook/CompHydCore/SummaSundials/sundials/sundials/instdir") +set(DIR_SUNDIALS "/globalhome/kck540/HPC/Libraries/sundials/instdir") +# set(DIR_SUNDIALS "/home/kklenk/projects/rpp-kshook/CompHydCore/SummaSundials/sundials/sundials/instdir") #set(DIR_SUNDIALS "/usr/local/sundials") ############################################################################# @@ -55,90 +66,6 @@ if (CMAKE_BUILD_TYPE MATCHES Cluster OR CMAKE_BUILD_TYPE MATCHES Cluster_Debug) message("\nBuilding Cluster\n") compile_summa_original(PARENT_DIR, ${DIR_SUNDIALS}) - # if(SUNDIALS) - # # Compiling Summa-Sundials on a Cluster - # link_directories(${DIR_SUNDIALS}/lib64) - # set(CMAKE_BUILD_RPATH "${DIR_SUNDIALS}/lib64") - # set(SUMMA_INCLUDES - # "$ENV{EBROOTNETCDFMINFORTRAN}/include" - # "${DIR_SUNDIALS}/include" - # "${DIR_SUNDIALS}/fortran" - # ${netCDF_INCLUDES} - # ${LAPACK_INCLUDES}) - - # set(SUMMA_LIBS - # -lsundials_fnvecmanyvector_mod - # -lsundials_fida_mod - # -lsundials_fnvecserial_mod - # -lsundials_fsunlinsoldense_mod - # -lsundials_fsunmatrixdense_mod - # -lnetcdff - # ${netCDF_LIBRARIES} - # ${LAPACK_LIBRARIES} - # SUMMA_NOAHMP) - - # set(SUMMA_ACTORS_INCLUDES - # ${CAF_INCLUDES} - # "$ENV{EBROOTNETCDFMINFORTRAN}/include" - # ${LAPACK_INCLUDES} - # "${DIR_SUNDIALS}/include" - # "${PARENT_DIR}/build/includes/global" - # "${PARENT_DIR}/build/includes/summa_actor" - # "${PARENT_DIR}/build/includes/gru_actor" - # "${PARENT_DIR}/build/includes/job_actor" - # "${PARENT_DIR}/build/includes/file_access_actor" - # "${PARENT_DIR}/build/includes/hru_actor") - - # set(SUMMA_ACTORS_LIBS - # ${CAF_LIBRARIES} - # ${netCDF_LIBRARIES} - # ${LAPACK_LIBRARIES} - # -lcaf_core - # -lcaf_io - # summa - # -lnetcdff - # -lsundials_fnvecmanyvector_mod - # -lsundials_fida_mod - # -lsundials_fnvecserial_mod - # -lsundials_fsunlinsoldense_mod - # -lsundials_fsunmatrixdense_mod) - - - # else() - # # Building Summa without sundials on a cluster - # set(SUMMA_INCLUDES - # "$ENV{EBROOTNETCDFMINFORTRAN}/include" - # ${netCDF_INCLUDES} - # ${LAPACK_INCLUDES}) - - # set(SUMMA_LIBS - # -lnetcdff - # ${netCDF_LIBRARIES} - # ${LAPACK_LIBRARIES} - # SUMMA_NOAHMP) - - # set(SUMMA_ACTORS_INCLUDES - # ${CAF_INCLUDES} - # "$ENV{EBROOTNETCDFMINFORTRAN}/include" - # ${LAPACK_INCLUDES} - # "${PARENT_DIR}/build/includes/global" - # "${PARENT_DIR}/build/includes/summa_actor" - # "${PARENT_DIR}/build/includes/gru_actor" - # "${PARENT_DIR}/build/includes/job_actor" - # "${PARENT_DIR}/build/includes/file_access_actor" - # "${PARENT_DIR}/build/includes/hru_actor") - - # set(SUMMA_ACTORS_LIBS - # ${CAF_LIBRARIES} - # ${netCDF_LIBRARIES} - # ${LAPACK_LIBRARIES} - # -lcaf_core - # -lcaf_io - # summa - # -lnetcdff) - - - # endif() elseif(CMAKE_BUILD_TYPE MATCHES Kinsol OR CMAKE_BUILD_TYPE MATCHES Kinsol_Debug) message("\nBuilding Kinsol\n") @@ -228,240 +155,4 @@ else() -lnetcdff) endif() -endif() - - # set(ACTORS_DIR ${PARENT_DIR}/build/source/actors) - # set(DRIVER_DIR ${PARENT_DIR}/build/source/driver) - # set(DSHARE_DIR ${PARENT_DIR}/build/source/dshare) - # set(ENGINE_DIR ${PARENT_DIR}/build/source/engine) - # set(HOOKUP_DIR ${PARENT_DIR}/build/source/hookup) - # set(NETCDF_DIR ${PARENT_DIR}/build/source/netcdf) - # set(NOAHMP_DIR ${PARENT_DIR}/build/source/noah-mp) - # set(FILE_ACCESS_DIR ${ACTORS_DIR}/file_access_actor) - # set(JOB_ACTOR_DIR ${ACTORS_DIR}/job_actor) - # set(HRU_ACTOR_DIR ${ACTORS_DIR}/hru_actor) - # set(GRU_ACTOR_DIR ${ACTORS_DIR}/gru_actor) - # set(SUMMA_DSHARE_DIR ${PARENT_DIR}/build/summa/build/source/dshare) - # set(SUMMA_ENGINE_DIR ${PARENT_DIR}/build/summa/build/source/engine) - # set(SUMMA_NOAHMP_DIR ${PARENT_DIR}/build/summa/build/source/noah-mp) - - # set(NRUTIL - # ${SUMMA_ENGINE_DIR}/nrtype.f90 - # ${SUMMA_ENGINE_DIR}/f2008funcs.f90 - # ${SUMMA_ENGINE_DIR}/nr_utility.f90) - - # set(NRPROC - # ${SUMMA_ENGINE_DIR}/expIntegral.f90 - # ${SUMMA_ENGINE_DIR}/spline_int.f90) - - # SET(HOOKUP - # ${HOOKUP_DIR}/ascii_util.f90 - # ${HOOKUP_DIR}/summaActors_FileManager.f90) - - # SET(DATAMS - # ${SUMMA_DSHARE_DIR}/multiconst.f90 - # ${DSHARE_DIR}/var_lookup.f90 - # ${DSHARE_DIR}/data_types.f90 - # ${DSHARE_DIR}/globalData.f90 - # ${SUMMA_DSHARE_DIR}/flxMapping.f90) - - # SET(DEPENDS_ON_FILEMANAGER - # ${SUMMA_DSHARE_DIR}/get_ixname.f90 - # ${SUMMA_DSHARE_DIR}/popMetadat.f90 - # ${SUMMA_DSHARE_DIR}/outpt_stat.f90) - - # SET(UTILMS - # ${SUMMA_ENGINE_DIR}/time_utils.f90 - # ${ENGINE_DIR}/mDecisions.f90 - # ${SUMMA_ENGINE_DIR}/snow_utils.f90 - # ${SUMMA_ENGINE_DIR}/soil_utils.f90 - # ${SUMMA_ENGINE_DIR}/updatState.f90 - # ${SUMMA_ENGINE_DIR}/matrixOper.f90) - - # set(SOLVER - # ${ENGINE_DIR}/vegPhenlgy.f90 - # ${SUMMA_ENGINE_DIR}/diagn_evar.f90 - # ${SUMMA_ENGINE_DIR}/stomResist.f90 - # ${SUMMA_ENGINE_DIR}/groundwatr.f90 - # ${SUMMA_ENGINE_DIR}/vegSWavRad.f90 - # ${SUMMA_ENGINE_DIR}/vegNrgFlux.f90 - # ${SUMMA_ENGINE_DIR}/ssdNrgFlux.f90 - # ${SUMMA_ENGINE_DIR}/vegLiqFlux.f90 - # ${SUMMA_ENGINE_DIR}/snowLiqFlx.f90 - # ${SUMMA_ENGINE_DIR}/soilLiqFlx.f90 - # ${SUMMA_ENGINE_DIR}/bigAquifer.f90 - # ${SUMMA_ENGINE_DIR}/computFlux.f90 - # ${SUMMA_ENGINE_DIR}/computResid.f90 - # ${SUMMA_ENGINE_DIR}/computJacob.f90 - # ${SUMMA_ENGINE_DIR}/eval8summa.f90 - # ${SUMMA_ENGINE_DIR}/summaSolve.f90 - # ${SUMMA_ENGINE_DIR}/systemSolv.f90 - # ${SUMMA_ENGINE_DIR}/varSubstep.f90 - # ${SUMMA_ENGINE_DIR}/opSplittin.f90 - # ${ENGINE_DIR}/coupled_em.f90) - - # set(INTERFACE - # ${ACTORS_DIR}/global/cppwrap_datatypes.f90 - # ${ACTORS_DIR}/global/cppwrap_auxiliary.f90 - # ${ACTORS_DIR}/global/cppwrap_metadata.f90) - - # set(FILE_ACCESS_INTERFACE - # ${FILE_ACCESS_DIR}/fortran_code/output_structure.f90 - # ${FILE_ACCESS_DIR}/fortran_code/cppwrap_fileAccess.f90 - # ${FILE_ACCESS_DIR}/fortran_code/read_attribute.f90 - # ${FILE_ACCESS_DIR}/fortran_code/read_forcing.f90 - # ${FILE_ACCESS_DIR}/fortran_code/read_param.f90 - # ${FILE_ACCESS_DIR}/fortran_code/read_initcond.f90 - # ${FILE_ACCESS_DIR}/fortran_code/writeOutputFromOutputStructure.f90 - # ${FILE_ACCESS_DIR}/fortran_code/write_to_netcdf.f90) - - # set(JOB_INTERFACE - # ${JOB_ACTOR_DIR}/job_actor.f90) - - # set(HRU_INTERFACE - # ${HRU_ACTOR_DIR}/fortran_code/model_run.f90 - # ${HRU_ACTOR_DIR}/fortran_code/setup_hru.f90 - # ${HRU_ACTOR_DIR}/fortran_code/restart.f90 - # ${HRU_ACTOR_DIR}/fortran_code/hru_actor.f90 - # ${HRU_ACTOR_DIR}/fortran_code/init_hru_actor.f90 - # ${HRU_ACTOR_DIR}/fortran_code/outputStrucWrite.f90 - # ${HRU_ACTOR_DIR}/fortran_code/hru_writeOutput.f90) - - # set(GRU_INTERFACE - # ${GRU_ACTOR_DIR}/gru_actor.f90) - - # set(PRELIM - # ${SUMMA_ENGINE_DIR}/conv_funcs.f90 - # ${SUMMA_ENGINE_DIR}/sunGeomtry.f90 - # ${SUMMA_ENGINE_DIR}/convE2Temp.f90 - # ${ENGINE_DIR}/allocspaceActors.f90 - # ${ENGINE_DIR}/alloc_fileAccess.f90 - # ${ENGINE_DIR}/checkStruc.f90 - # ${SUMMA_ENGINE_DIR}/childStruc.f90 - # ${ENGINE_DIR}/ffile_info.f90 - # ${ENGINE_DIR}/read_dimension.f90 - # ${ENGINE_DIR}/read_pinit.f90 - # ${SUMMA_ENGINE_DIR}/pOverwrite.f90 - # ${SUMMA_ENGINE_DIR}/paramCheck.f90 - # ${ENGINE_DIR}/check_icondActors.f90) - - # set(NOAHMP - # ${SUMMA_NOAHMP_DIR}/module_model_constants.F - # ${SUMMA_NOAHMP_DIR}/module_sf_noahutl.F - # ${SUMMA_NOAHMP_DIR}/module_sf_noahlsm.F - # ${SUMMA_NOAHMP_DIR}/module_sf_noahmplsm.F) - - # set(MODRUN - # ${SUMMA_ENGINE_DIR}/indexState.f90 - # ${SUMMA_ENGINE_DIR}/getVectorz.f90 - # ${SUMMA_ENGINE_DIR}/updateVars.f90 - # ${SUMMA_ENGINE_DIR}/var_derive.f90 - # ${ENGINE_DIR}/derivforce.f90 - # ${ENGINE_DIR}/sundials/t2enthalpy.f90 - # ${SUMMA_ENGINE_DIR}/snowAlbedo.f90 - # ${SUMMA_ENGINE_DIR}/canopySnow.f90 - # ${SUMMA_ENGINE_DIR}/tempAdjust.f90 - # ${SUMMA_ENGINE_DIR}/snwCompact.f90 - # ${SUMMA_ENGINE_DIR}/layerMerge.f90 - # ${SUMMA_ENGINE_DIR}/layerDivide.f90 - # ${SUMMA_ENGINE_DIR}/volicePack.f90 - # ${SUMMA_ENGINE_DIR}/qTimeDelay.f90) - - # set(NETCDF - # ${NETCDF_DIR}/netcdf_util.f90 - # ${NETCDF_DIR}/def_output.f90 - # ${NETCDF_DIR}/writeOutput.f90 - # ${NETCDF_DIR}/read_icondActors.f90) - - # set(DRIVER - # ${DRIVER_DIR}/summaActors_type.f90 - # ${DRIVER_DIR}/summaActors_util.f90 - # ${DRIVER_DIR}/summaActors_globalData.f90 - # ${DRIVER_DIR}/summaActors_alarms.f90) - - # set(COMM_ALL - # ${NRPROC} - # ${DATAMS} - # ${INTERFACE} - # ${HOOKUP} - # ${DEPENDS_ON_FILEMANAGER} - # ${UTILMS}) - - # set(SUMMA_ALL - # ${NETCDF} - # ${PRELIM} - # ${MODRUN} - # ${SOLVER} - # ${DRIVER} - # ${JOB_INTERFACE} - # ${FILE_ACCESS_INTERFACE} - # ${HRU_INTERFACE} - # ${GRU_INTERFACE}) - - # set(ACTORS_GLOBAL - # ${ACTORS_DIR}/global/global.cpp - # ${ACTORS_DIR}/global/timing_info.cpp - # ${ACTORS_DIR}/global/message_atoms.cpp - # ${ACTORS_DIR}/global/settings_functions.cpp - # ${ACTORS_DIR}/global/auxiliary.cpp) - - # set(SUMMA_ACTOR - # ${ACTORS_DIR}/summa_actor/summa_actor.cpp - # ${ACTORS_DIR}/summa_actor/summa_client.cpp - # ${ACTORS_DIR}/summa_actor/summa_server.cpp - # ${ACTORS_DIR}/summa_actor/summa_backup_server.cpp - # ${ACTORS_DIR}/summa_actor/batch/batch.cpp - # ${ACTORS_DIR}/summa_actor/batch/batch_container.cpp - # ${ACTORS_DIR}/summa_actor/client/client.cpp - # ${ACTORS_DIR}/summa_actor/client/client_container.cpp) - - # set(HRU_ACTOR - # ${ACTORS_DIR}/hru_actor/cpp_code/hru_actor.cpp) - - # set(GRU_ACTOR - # ${ACTORS_DIR}/gru_actor/gru_actor.cpp) - - # set(FILE_ACCESS_ACTOR - # ${ACTORS_DIR}/file_access_actor/cpp_code/file_access_actor.cpp - # ${ACTORS_DIR}/file_access_actor/cpp_code/forcing_file_info.cpp - # ${ACTORS_DIR}/file_access_actor/cpp_code/output_container.cpp) - - # set(JOB_ACTOR - # ${ACTORS_DIR}/job_actor/job_actor.cpp - # ${ACTORS_DIR}/job_actor/GRUinfo.cpp) - - # set(MAIN - # ${ACTORS_DIR}/main.cpp) - - # ####################### START OF COMPILATION ########################################### - # # Build NOAH_MP Object - # add_library(SUMMA_NOAHMP OBJECT - # ${NOAHMP} - # ${NRUTIL}) - # target_compile_options(SUMMA_NOAHMP PRIVATE ${SUMMA_NOAHMP_OPTIONS}) - # # Build SUMMA_COMM Object - # add_library(SUMMA_COMM OBJECT - # ${COMM_ALL}) - # target_compile_options(SUMMA_COMM PRIVATE ${SUMMA_ALL_OPTIONS}) - # target_include_directories(SUMMA_COMM PRIVATE ${SUMMA_INCLUDES}) - # target_link_libraries(SUMMA_COMM PUBLIC ${SUMMA_LIBS}) - # # Build SUMMA Shared Library - # add_library(summa SHARED - # ${SUMMA_ALL}) - # target_compile_options(summa PRIVATE ${SUMMA_ALL_OPTIONS}) - # target_include_directories(summa PUBLIC ${SUMMA_INCLUDES}) - # target_link_libraries(summa PUBLIC ${SUMMA_LIBS} SUMMA_COMM) - # # Build SUMMA Executable - # add_executable(${EXEC_NAME} - # ${ACTORS_GLOBAL} - # ${HRU_ACTOR} - # ${GRU_ACTOR} - # ${FILE_ACCESS_ACTOR} - # ${JOB_ACTOR} - # ${SUMMA_ACTOR} - # ${SUMMA_CLIENT} - # ${SUMMA_SERVER} - # ${MAIN}) - # set_property(TARGET ${EXEC_NAME} PROPERTY LINKER_LANGUAGE Fortran) - # target_include_directories(${EXEC_NAME} PUBLIC ${SUMMA_ACTORS_INCLUDES}) - # target_link_libraries( ${EXEC_NAME} ${SUMMA_ACTORS_LIBS}) \ No newline at end of file +endif() \ No newline at end of file diff --git a/build/cmake/be.cmake b/build/cmake/be.cmake index 456a3be..af65330 100644 --- a/build/cmake/be.cmake +++ b/build/cmake/be.cmake @@ -1,4 +1,6 @@ function (compile_summa_original PARENT_DIR, DIR_SUNDIALS) + set(EXEC_NAME summa_be) + set(SUMMA_INCLUDES "$ENV{EBROOTNETCDFMINFORTRAN}/include" ${netCDF_INCLUDES} @@ -42,9 +44,9 @@ function (compile_summa_original PARENT_DIR, DIR_SUNDIALS) set(JOB_ACTOR_DIR ${ACTORS_DIR}/job_actor) set(HRU_ACTOR_DIR ${ACTORS_DIR}/hru_actor) set(GRU_ACTOR_DIR ${ACTORS_DIR}/gru_actor) - set(SUMMA_DSHARE_DIR ${PARENT_DIR}/build/summa/build/source/dshare) - set(SUMMA_ENGINE_DIR ${PARENT_DIR}/build/summa/build/source/engine) - set(SUMMA_NOAHMP_DIR ${PARENT_DIR}/build/summa/build/source/noah-mp) + set(SUMMA_DSHARE_DIR ${PARENT_DIR}/build/summa_original/build/source/dshare) + set(SUMMA_ENGINE_DIR ${PARENT_DIR}/build/summa_original/build/source/engine) + set(SUMMA_NOAHMP_DIR ${PARENT_DIR}/build/summa_original/build/source/noah-mp) set(NRUTIL ${SUMMA_ENGINE_DIR}/nrtype.f90 @@ -61,6 +63,7 @@ function (compile_summa_original PARENT_DIR, DIR_SUNDIALS) SET(DATAMS ${SUMMA_DSHARE_DIR}/multiconst.f90 + ${DSHARE_DIR}/csv_file.f90 ${DSHARE_DIR}/var_lookup.f90 ${DSHARE_DIR}/data_types.f90 ${DSHARE_DIR}/globalData.f90 diff --git a/build/cmake/kinsol.cmake b/build/cmake/kinsol.cmake index 31659a1..c9b5b72 100644 --- a/build/cmake/kinsol.cmake +++ b/build/cmake/kinsol.cmake @@ -3,6 +3,8 @@ ##################################################################### function(compile_with_kinsol PARENT_DIR, DIR_SUNDIALS) find_package(LAPACK REQUIRED) + set(EXEC_NAME summa_kinsol) + message("DIR_SUNDIALS: ${DIR_SUNDIALS}") # Building Summa with sundials on local machine @@ -122,6 +124,7 @@ function(compile_with_kinsol PARENT_DIR, DIR_SUNDIALS) SET(DATAMS ${SUMMA_DSHARE_DIR}/multiconst.f90 + ${DSHARE_DIR}/csv_file.f90 ${DSHARE_DIR}/var_lookup.f90 ${DSHARE_DIR}/data_types.f90 ${DSHARE_DIR}/globalData.f90 diff --git a/build/makefiles/Makefile_Summa b/build/makefiles/Makefile_Summa new file mode 100755 index 0000000..dbd339f --- /dev/null +++ b/build/makefiles/Makefile_Summa @@ -0,0 +1,385 @@ +#======================================================================== +# Makefile to compile SUMMA +#======================================================================== +# +# Recommended use: Copy this file to Makefile.local, edit it to your +# heart's content, and then run `make -f build/Makefile.local` from +# your top level SUMMA directory. Don't include the Makefile.local in +# any pull requests you make. +# +# Note that Makefile configurations that we commonly use can be found on +# the SUMMA wiki at: +# https://github.com/NCAR/summa/wiki/SUMMA-Makefile-Part-0-configuration +# feel free to add yours to that page. +# +# To troubleshoot your paths and setup, type 'make check' +# +# At a minimum you will need to set the following: +# * F_MASTER - top level summa directory +# * FC - compiler suite +# * FC_EXE - compiler executable +# * INCLUDES - path to include files +# * LIBRARIES - path to and libraries to include +# +# Some further options can be specified for OpenMP, etc. See in Part 0 of +# the Makefile. You don't need to make any changes in PART 1 and +# following unless you are doing SUMMA development and changed what +# needs to be compiled + +#======================================================================== +# PART 0: User-configurable part +#======================================================================== + +# The variables can be specified in one of two ways: +# * delete the '##' in front of the variable, fill out the entry, +# save the file and run make +# * make no changes to this file, but specify the variables in your +# environment before you run make + +# Define core directory below which everything resides. This is the +# parent directory of the 'build' directory +F_MASTER = /globalhome/kck540/HPC/Summa-Projects/Summa-Actors/build/summa + +# Define the Fortran Compiler. If you are using gfortran, then this needs +# to be version 6 or higher. This variable is simply used to select the right +# compiler flags in the ifeq statements in this Makefile. The compiler +# executable is set separately as FC_EXE +# Currently this is either gfortran or ifort +FC = gfortran + +# Define the path for the compiler executable. This is the actual executable +# that is invoked. For example, FC=gfortran and FC_EXE=/usr/bin-gfortran-mp-6 +# FC and FC_EXE have to be consistent +FC_EXE = gfortran + +# Define the NetCDF and LAPACK libraries and path to include files. +# INCLUDES needs to be of the form (no quotes around the string): +# INCLUDES = -I<include path 1> -I<include path 2> -I<...> -I<include path n> +# LIBRARIES needs to be of the form ( no quotes around the string): +# LIBRARIES = '-L<netcdf library path> -lnetcdff -L<blas library path> -lblas -L<etc> -l<etc>' +# If none of this makes sense, please talk to your system +# administrator. +INCLUDES = -I$(EBROOTNETCDFMINFORTRAN)/include +LIBRARIES = -L$(EBROOTNETCDFMINFORTRAN)/lib64 -L$(EBROOTOPENBLAS)/lib -lnetcdff -lopenblas + +# Eventually we plan move to a real configure script, but for now we like +# to keep track of successful compilations of SUMMA on different platforms +# and with different compilers. If you are successful compiling SUMMA, +# please add your configuration (operating system and compiler plus +# part 0 of the Makefile) to the SUMMA wiki on github. + +# Define compiler flags. If you use a different compiler, +# you will need to figure out what the equivalent flags are +# and may need to update this section + +# ------------ define compiler flags ---------------------------------------- + +# define open MP flags +isOpenMP = +FLAGS_OMP = +LIBOPENMP = + +# Define compiler flags. If you use a different compiler, +# you will need to figure out what the equivalent flags are +# and may need to update this section + +# gfortran compiler flags +ifeq "$(FC)" "gfortran" + + ifeq "$(isOpenMP)" "yes" + FLAGS_OMP = -fopenmp + endif + +# Production runs +FLAGS_NOAH = -O3 -ffree-form -ffree-line-length-none -fmax-errors=0 $(FLAGS_OMP) +FLAGS_COMM = -O3 -ffree-line-length-none -fmax-errors=0 $(FLAGS_OMP) +FLAGS_SUMMA = -O3 -ffree-line-length-none -fmax-errors=0 $(FLAGS_OMP) + +# Debug runs +#FLAGS_NOAH = -p -g -ffree-form -ffree-line-length-none -fmax-errors=0 -fbacktrace -Wno-unused -Wno-unused-dummy-argument +#FLAGS_COMM = -p -g -Wall -ffree-line-length-none -fmax-errors=0 -fbacktrace -fcheck=bounds +#FLAGS_SUMMA = -p -g -Wall -ffree-line-length-none -fmax-errors=0 -fbacktrace -fcheck=bounds + +endif + +# ifort compiler flags +ifeq "$(FC)" "ifort" + + ifeq "$(isOpenMP)" "yes" + FLAGS_OMP = -qopenmp + endif + +# Production runs +FLAGS_NOAH = -O3 -noerror_limit -FR -auto -fltconsistency $(FLAGS_OMP) +FLAGS_COMM = -O3 -FR -auto -fltconsistency -fpe0 $(FLAGS_OMP) +FLAGS_SUMMA = -O3 -FR -auto -fltconsistency -fpe0 $(FLAGS_OMP) + +# Debug runs +#FLAGS_NOAH = -O0 -p -g -warn nounused -noerror_limit -FR -auto -WB -traceback -fltconsistency +#FLAGS_COMM = -O0 -p -g -debug -warn all -check all -FR -auto -WB -traceback -fltconsistency -fpe0 +#FLAGS_SUMMA = -O0 -p -g -debug -warn all -check all -FR -auto -WB -traceback -fltconsistency -fpe0 +endif + +#======================================================================== +# PART 1: Define directory paths +#======================================================================== + +# Core directory that contains source code +F_KORE_DIR = $(F_MASTER)/build/source + +# Location of the compiled modules +MOD_PATH = $(F_MASTER)/build + +# Define the directory for the executables +EXE_PATH = $(F_MASTER)/bin + +#======================================================================== +# PART 2: Assemble all of the SUMMA sub-routines +#======================================================================== + +# Define directories +DRIVER_DIR = $(F_KORE_DIR)/driver +HOOKUP_DIR = $(F_KORE_DIR)/hookup +NETCDF_DIR = $(F_KORE_DIR)/netcdf +DSHARE_DIR = $(F_KORE_DIR)/dshare +NUMREC_DIR = $(F_KORE_DIR)/numrec +NOAHMP_DIR = $(F_KORE_DIR)/noah-mp +ENGINE_DIR = $(F_KORE_DIR)/engine + +# utilities +SUMMA_NRUTIL= \ + nrtype.f90 \ + f2008funcs.f90 \ + nr_utility.f90 +NRUTIL = $(patsubst %, $(ENGINE_DIR)/%, $(SUMMA_NRUTIL)) + +# +# Numerical recipes procedures +# NOTE: all numerical recipes procedures are now replaced with free versions +SUMMA_NRPROC= \ + expIntegral.f90 \ + spline_int.f90 +NRPROC = $(patsubst %, $(ENGINE_DIR)/%, $(SUMMA_NRPROC)) + +# Hook-up modules (set files and directory paths) +SUMMA_HOOKUP= \ + ascii_util.f90 \ + summaFileManager.f90 +HOOKUP = $(patsubst %, $(HOOKUP_DIR)/%, $(SUMMA_HOOKUP)) + +# Data modules +SUMMA_DATAMS= \ + multiconst.f90 \ + var_lookup.f90 \ + data_types.f90 \ + globalData.f90 \ + flxMapping.f90 \ + get_ixname.f90 \ + popMetadat.f90 \ + outpt_stat.f90 +DATAMS = $(patsubst %, $(DSHARE_DIR)/%, $(SUMMA_DATAMS)) + +# utility modules +SUMMA_UTILMS= \ + time_utils.f90 \ + mDecisions.f90 \ + snow_utils.f90 \ + soil_utils.f90 \ + updatState.f90 \ + matrixOper.f90 +UTILMS = $(patsubst %, $(ENGINE_DIR)/%, $(SUMMA_UTILMS)) + +# Model guts +SUMMA_MODGUT= \ + MODGUT = $(patsubst %, $(ENGINE_DIR)/%, $(SUMMA_MODGUT)) + +# Solver +SUMMA_SOLVER= \ + vegPhenlgy.f90 \ + diagn_evar.f90 \ + stomResist.f90 \ + groundwatr.f90 \ + vegSWavRad.f90 \ + vegNrgFlux.f90 \ + ssdNrgFlux.f90 \ + vegLiqFlux.f90 \ + snowLiqFlx.f90 \ + soilLiqFlx.f90 \ + bigAquifer.f90 \ + computFlux.f90 \ + computResid.f90 \ + computJacob.f90 \ + eval8summa.f90 \ + summaSolve.f90 \ + systemSolv.f90 \ + varSubstep.f90 \ + opSplittin.f90 \ + coupled_em.f90 \ + run_oneHRU.f90 \ + run_oneGRU.f90 +SOLVER = $(patsubst %, $(ENGINE_DIR)/%, $(SUMMA_SOLVER)) + +# Define routines for SUMMA preliminaries +SUMMA_PRELIM= \ + conv_funcs.f90 \ + sunGeomtry.f90 \ + convE2Temp.f90 \ + allocspace.f90 \ + checkStruc.f90 \ + childStruc.f90 \ + ffile_info.f90 \ + read_attrb.f90 \ + read_pinit.f90 \ + pOverwrite.f90 \ + read_param.f90 \ + paramCheck.f90 \ + check_icond.f90 +PRELIM = $(patsubst %, $(ENGINE_DIR)/%, $(SUMMA_PRELIM)) + +SUMMA_NOAHMP= \ + module_model_constants.F \ + module_sf_noahutl.F \ + module_sf_noahlsm.F \ + module_sf_noahmplsm.F + +NOAHMP = $(patsubst %, $(NOAHMP_DIR)/%, $(SUMMA_NOAHMP)) + +# Define routines for the SUMMA model runs +SUMMA_MODRUN = \ + indexState.f90 \ + getVectorz.f90 \ + updateVars.f90 \ + var_derive.f90 \ + read_force.f90 \ + derivforce.f90 \ + snowAlbedo.f90 \ + canopySnow.f90 \ + tempAdjust.f90 \ + snwCompact.f90 \ + layerMerge.f90 \ + layerDivide.f90 \ + volicePack.f90 \ + qTimeDelay.f90 +MODRUN = $(patsubst %, $(ENGINE_DIR)/%, $(SUMMA_MODRUN)) + +# Define routines for the solver +SUMMA_MSOLVE = \ + +# Define NetCDF routines +SUMMA_NETCDF = \ + netcdf_util.f90 \ + def_output.f90 \ + modelwrite.f90 \ + read_icond.f90 +NETCDF = $(patsubst %, $(NETCDF_DIR)/%, $(SUMMA_NETCDF)) + +# ... stitch together common programs +COMM_ALL = $(NRUTIL) $(NRPROC) $(HOOKUP) $(DATAMS) $(UTILMS) + +# ... stitch together SUMMA programs +SUMMA_ALL = $(NETCDF) $(PRELIM) $(MODRUN) $(SOLVER) + +# Define the driver routine +SUMMA_DRIVER= \ + summa_type.f90 \ + summa_util.f90 \ + summa_alarms.f90 \ + summa_globalData.f90 \ + summa_defineOutput.f90 \ + summa_init.f90 \ + summa_setup.f90 \ + summa_restart.f90 \ + summa_forcing.f90 \ + summa_modelRun.f90 \ + summa_writeOutput.f90 \ + summa_driver.f90 +DRIVER = $(patsubst %, $(DRIVER_DIR)/%, $(SUMMA_DRIVER)) + +# Define the executable +DRIVER__EX = summa.exe + +# Define version number +VERSIONFILE = $(DRIVER_DIR)/summaversion.inc +VERSION = $(shell git tag | tail -n 1) +BULTTIM = $(shell date) +GITBRCH = $(shell git describe --long --all --always | sed -e's/heads\///') +GITHASH = $(shell git rev-parse HEAD) + +#======================================================================== +# PART 3: Checks +#====================================================================== +# make sure that the paths are defined. These are just some high level checks +ifndef F_MASTER + $(error F_MASTER is undefined) +endif +ifndef FC + $(error FC is undefined: Specify your compiler) +endif +ifndef FC_EXE + $(error FC_EXE is undefined: Specify your compiler executable) +endif +ifndef FLAGS_SUMMA + $(error Specify flags for your compiler: $(FC)) +endif +ifndef INCLUDES + $(error INCLUDES is undefined) +endif +ifndef LIBRARIES + $(error LIBRARIES is undefined) +endif + +#======================================================================== +# PART 4: compilation +#====================================================================== + +# Compile +all: compile_noah compile_comm compile_summa link clean install + +check: + $(info) + $(info Displaying make variables:) + $(info F_MASTER : $(F_MASTER)) + $(info EXE_PATH : $(EXE_PATH)) + $(info FC : $(FC)) + $(info INCLUDES : $(INCLUDES)) + $(info LIBRARIES : $(LIBRARIES)) + $(info FLAGS_NOAH : $(FLAGS_NOAH)) + $(info FLAGS_COMM : $(FLAGS_COMM)) + $(info FLAGS_SUMMA: $(FLAGS_SUMMA)) + $(info) + +# update version information +update_version: + echo "character(len=64), parameter :: summaVersion = '${VERSION}'" > $(VERSIONFILE) + echo "character(len=64), parameter :: buildTime = '${BULTTIM}'" >> $(VERSIONFILE) + echo "character(len=64), parameter :: gitBranch = '${GITBRCH}'" >> $(VERSIONFILE) + echo "character(len=64), parameter :: gitHash = '${GITHASH}'" >> $(VERSIONFILE) + +# compile Noah-MP routines +compile_noah: + $(FC_EXE) $(FLAGS_NOAH) -c $(NRUTIL) $(NOAHMP) + +# compile common routines +compile_comm: + $(FC_EXE) $(FLAGS_COMM) -c $(COMM_ALL) $(INCLUDES) + +# compile SUMMA routines +compile_summa: update_version + $(FC_EXE) $(FLAGS_SUMMA) -c $(SUMMA_ALL) $(DRIVER) \ + $(INCLUDES) + +# link routines +link: + $(FC_EXE) -g *.o $(LIBRARIES) -o $(DRIVER__EX) + +# Remove object files +clean: + rm -f *.o + rm -f *.mod + rm -f soil_veg_gen_parm__genmod.f90 + +# Copy the executable to the bin directory +install: + @mkdir -p $(EXE_PATH) + @mv $(DRIVER__EX) $(EXE_PATH) + $(info $(DRIVER__EX) successfully installed in $(EXE_PATH)) diff --git a/build/source/dshare/csv_file.f90 b/build/source/dshare/csv_file.f90 new file mode 100644 index 0000000..165809b --- /dev/null +++ b/build/source/dshare/csv_file.f90 @@ -0,0 +1,274 @@ +! csv_file.f90 -- +! Small module to facilitate writing CSV-files +! +! $Id: csv_file.f90,v 1.2 2006/03/26 19:03:53 arjenmarkus Exp $ +! +! The module contains the following subroutines: +! csv_next_record Advance to the next record +! csv_write_integer Write a single integer to the CSV-file +! csv_write_real Write a single real to the CSV-file +! csv_write_dble Write a single double-precision real to the +! CSV-file +! csv_write_char Write a single character string to the +! CSV-file +! csv_write_integer_1d Write a 1D array of integers to the CSV-file +! csv_write_real_1d Write a 1D array of reals to the CSV-file +! csv_write_dble_1d Write a 1D array of double-precision real to +! the CSV-file +! csv_write_char_1d Write a 1D array of character strings to the +! CSV-file +! csv_write_integer_2d Write a 2D array of integers to the CSV-file +! csv_write_real_2d Write a 2D array of reals to the CSV-file +! csv_write_dble_2d Write a 2D array of double-precision real to +! the CSV-file +! csv_write_char_2d Write a 2D array of character strings to the +! CSV-file +! +! For convenience, the generic name "csv_write" can be used +! instead of the individual routines. +! +! The file to write to must already be opened as a LU-number +! is passed. +! +! Layout of the CSV-file: +! - single items are written to the end of the current record +! - one-dimensional items are also written to the end of the current +! record +! - two-dimensional items are written to separate records, one for +! each row +! - except for the two-dimensional versions, all routines allow +! you to suppress advancing to the next record: +! - for single items you must indicate whether to advance or not +! - for one-dimensional items, the argument is optional. Default +! is to advance. +! +! Note on the format: +! CSV-files apparently come in different guises (Kernighan and Pike, +! The practice of Programming, Addison-Wesley, 1999). This module +! uses the following rules: +! - items are always separated by a single comma (,) +! - string items are delimited by double quotes (") +! - embedded double quotes are treated by doubling the quote +! - trailing blanks are considered irrelevant +! +module csv_file + implicit none + interface csv_write + module procedure csv_write_integer + module procedure csv_write_integer_1d + module procedure csv_write_integer_2d + module procedure csv_write_char + module procedure csv_write_char_1d + module procedure csv_write_char_2d + module procedure csv_write_real + module procedure csv_write_real_1d + module procedure csv_write_real_2d + module procedure csv_write_dble + module procedure csv_write_dble_1d + module procedure csv_write_dble_2d + end interface +contains + +! csv_next_record -- +! Go to the next record (convenience routine) +! Arguments: +! lun LU-number of the CSV-file +! Result: +! The current record is closed, the next write will be to the +! new record +! Note: +! This is a convenience routine, it may result in a superfluous +! comma at the end of the previous record. This does not seem to +! be problematic, though, for MS Excel +! +subroutine csv_next_record( lun ) + integer, intent(in) :: lun + + write(lun,'(a)') '' +end subroutine csv_next_record + +! csv_write_integer/real/dble -- +! Write a single integer/real/double precision real to the CSV-file +! Arguments: +! lun LU-number of the CSV-file +! value Value to write +! advance Advance (.true.) or not, so that more items can be +! written to the same record +! Result: +! The value is written to the current record of the CSV-file +! +subroutine csv_write_integer( lun, value, advance ) + integer, intent(in) :: lun + integer, intent(in) :: value + logical, intent(in) :: advance + + character(len=40) :: buffer + write( buffer, '(I10)' ) value + buffer = adjustl(buffer) + if ( advance ) then + write(lun,'(a)') trim(buffer) + else + ! Most probably: write the comma only when needed + ! - depends on other actions + write(lun,'(a,a)',advance='no') trim(buffer), ',' + endif +end subroutine csv_write_integer + +subroutine csv_write_real( lun, value, advance ) + integer, intent(in) :: lun + real, intent(in) :: value + logical, intent(in) :: advance + + character(len=40) :: buffer + write( buffer, '(G14.6)' ) value + buffer = adjustl(buffer) + if ( advance ) then + write(lun,'(a)') trim(buffer) + else + ! Most probably: write the comma only when needed + ! - depends on other actions + write(lun,'(a,a)',advance='no') trim(buffer), ',' + endif +end subroutine csv_write_real + +subroutine csv_write_dble( lun, value, advance ) + integer, intent(in) :: lun + real(kind=kind(1.0d0)), intent(in) :: value + logical, intent(in) :: advance + + character(len=40) :: buffer + write( buffer, '(G20.12)' ) value + buffer = adjustl(buffer) + if ( advance ) then + write(lun,'(a)') trim(buffer) + else + ! Most probably: write the comma only when needed + ! - depends on other actions + write(lun,'(a,a)',advance='no') trim(buffer), ',' + endif +end subroutine csv_write_dble + +! csv_write_char -- +! Write a single character string to the CSV-file +! Arguments: +! lun LU-number of the CSV-file +! value Value to write +! advance Advance (.true.) or not, so that more items can be +! written to the same record +! Result: +! The value is written to the current record of the CSV-file +! +subroutine csv_write_char( lun, value, advance ) + integer, intent(in) :: lun + character(len=*), intent(in) :: value + logical, intent(in) :: advance + + integer :: k + integer :: pos + integer :: posb + character(len=2*len(value)) :: buffer + + buffer = value + + ! + ! Check for nasty characters (") + ! + k = index( value,'"') + pos = 1 + posb = 1 + do while ( k .ge. 1 ) + buffer(posb:) = value(pos:) + buffer(posb+k:) = '"' // value(pos+k:) + pos = pos + k + 1 + posb = posb + k + 2 + k = index( value(pos:),'"') + enddo + + if ( advance ) then + write(lun,'(3a)') '"',trim(buffer),'"' + else + write(lun,'(3a,a)',advance='no') '"',trim(buffer), '"', ',' + endif +end subroutine csv_write_char + +! csv_write_integer/real/dble_1d -- +! Write a one-dimensional array of items to the CSV-file +! Arguments: +! lun LU-number of the CSV-file +! array Array to write +! advance Advance (.true.) or not, so that more items can be +! written to the same record +! Result: +! The array is written to the current record of the CSV-file +! Note: +! Because the four routines of this type differ only in +! the data type, we use an include file for the body. +! +subroutine csv_write_integer_1d( lun, array, advance ) + integer, dimension(:), intent(in) :: array + + include 'csv_file_1d.f90' + +end subroutine csv_write_integer_1d + +subroutine csv_write_real_1d( lun, array, advance ) + real, dimension(:), intent(in) :: array + + include 'csv_file_1d.f90' + +end subroutine csv_write_real_1d + +subroutine csv_write_dble_1d( lun, array, advance ) + real(kind=kind(1.0d0)), dimension(:), intent(in) :: array + + include 'csv_file_1d.f90' + +end subroutine csv_write_dble_1d + +subroutine csv_write_char_1d( lun, array, advance ) + character(len=*), dimension(:), intent(in) :: array + + include 'csv_file_1d.f90' + +end subroutine csv_write_char_1d + +! csv_write_integer/real/dble_2d -- +! Write a two-dimensional array of items to the CSV-file +! Arguments: +! lun LU-number of the CSV-file +! array Array to write +! Result: +! The array is written to the current record of the CSV-file +! Note: +! Because the four routines of this type differ only in +! the data type, we use an include file for the body. +! +subroutine csv_write_integer_2d( lun, array ) + integer, dimension(:,:), intent(in) :: array + + include 'csv_file_2d.f90' + +end subroutine csv_write_integer_2d + +subroutine csv_write_real_2d( lun, array ) + real, dimension(:,:), intent(in) :: array + + include 'csv_file_2d.f90' + +end subroutine csv_write_real_2d + +subroutine csv_write_dble_2d( lun, array ) + real(kind=kind(1.0d0)), dimension(:,:), intent(in) :: array + + include 'csv_file_2d.f90' + +end subroutine csv_write_dble_2d + +subroutine csv_write_char_2d( lun, array ) + character(len=*), dimension(:,:), intent(in) :: array + + include 'csv_file_2d.f90' + +end subroutine csv_write_char_2d + +end module csv_file diff --git a/build/source/dshare/csv_file_1d.f90 b/build/source/dshare/csv_file_1d.f90 new file mode 100644 index 0000000..485fec1 --- /dev/null +++ b/build/source/dshare/csv_file_1d.f90 @@ -0,0 +1,22 @@ +! cvs_file_1d.f90 -- +! Include file for csv_file.f90: +! contains the body of the one-dimensional version of the +! writing routines. +! +! $Id: csv_file_1d.f90,v 1.2 2006/03/26 19:03:53 arjenmarkus Exp $ +! + integer, intent(in) :: lun + logical, intent(in), optional :: advance + + logical :: adv + integer :: i + + adv = .true. + if ( present(advance) ) adv = advance + + do i = 1,size(array)-1 + call csv_write( lun, array(i), .false. ) + enddo + call csv_write( lun, array(size(array)), adv ) +! +! end of body diff --git a/build/source/dshare/csv_file_2d.f90 b/build/source/dshare/csv_file_2d.f90 new file mode 100644 index 0000000..24d5e4f --- /dev/null +++ b/build/source/dshare/csv_file_2d.f90 @@ -0,0 +1,19 @@ +! cvs_file_2d.f90 -- +! Include file for csv_file.f90: +! contains the body of the two-dimensional version of the +! writing routines. +! +! $Id: csv_file_2d.f90,v 1.2 2006/03/26 19:03:53 arjenmarkus Exp $ +! + integer, intent(in) :: lun + + logical :: adv + integer :: i + + adv = .true. + + do i = 1,size(array,2) + call csv_write( lun, array(:,i), adv ) + enddo +! +! end of body diff --git a/utils/sbatch/submission_copern.sh b/utils/sbatch/submission_copern.sh new file mode 100755 index 0000000..128140b --- /dev/null +++ b/utils/sbatch/submission_copern.sh @@ -0,0 +1,11 @@ +#!/bin/bash +#SBATCH --cpus-per-task=1 +#SBATCH --time=02:00:00 +#SBATCH --mem=32G +#SBATCH --job-name=Summa-Actors +#SBATCH --output=/scratch/gwf/gwf_cmt/kck540/Summa-Actors/slurm/Summa-Actors%A_%a.out +#SBATCH --account=hpc_c_giws_clark + +/globalhome/kck540/HPC/Summa-Projects/Summa-Actors/bin/summa_actors -g 1 -n 50 -c /project/gwf/gwf_cmt/kck540/domain_NorthAmerica/summa_actors_input/Summa_Actors_Settings.json \ + --config-file=/globalhome/kck540/HPC/Summa-Projects/Summa-Actors/bin/caf-application.conf + diff --git a/bin/submission_one.sh b/utils/sbatch/submission_one.sh similarity index 100% rename from bin/submission_one.sh rename to utils/sbatch/submission_one.sh diff --git a/bin/submission_script_array.sh b/utils/sbatch/submission_script_array.sh similarity index 100% rename from bin/submission_script_array.sh rename to utils/sbatch/submission_script_array.sh diff --git a/bin/submit_summa_original.sh b/utils/sbatch/submit_summa_original.sh similarity index 100% rename from bin/submit_summa_original.sh rename to utils/sbatch/submit_summa_original.sh -- GitLab