Skip to content
Snippets Groups Projects
Commit 5adf81e4 authored by Kyle Klenk (kck540)'s avatar Kyle Klenk (kck540)
Browse files

changed file-access actor settings

parent 17987251
No related branches found
No related tags found
No related merge requests found
#### parent directory of the 'build' directory ####
F_MASTER = /home/kklenk/Summa-Actors
# F_MASTER = /home/kklenk/Summa-Actors
#### fortran compiler ####
FC = gfortran
# #### fortran compiler ####
# FC = gfortran
#### C++ compiler ####
CC=g++
# #### C++ compiler ####
# CC=g++
#### Includes AND Libraries ####
INCLUDES = -I/usr/include
LIBRARIES = -L/usr/lib -lnetcdff -lnetcdf -lopenblas
# #### Includes AND Libraries ####
# INCLUDES = -I/usr/include
# LIBRARIES = -L/usr/lib -lnetcdff -lnetcdf -lopenblas
ACTORS_INCLUDES = -I/usr/include -I/usr/local/include
ACTORS_LIBRARIES = -L/home/linuxbrew.linuxbrew/lib -L/home/kklenk/Summa-Actors/build -lcaf_core -lcaf_io -lsumma -lopenblas -lnetcdff
# ACTORS_INCLUDES = -I/usr/include -I/usr/local/include
# ACTORS_LIBRARIES = -L/home/linuxbrew.linuxbrew/lib -L/home/kklenk/Summa-Actors/build -lcaf_core -lcaf_io -lsumma -lopenblas -lnetcdff
# Production runs
......@@ -284,8 +284,6 @@ actors_testLink:
actorsClean:
rm *.o
mv libsumma.so ../bin
mv summaMain ../bin
# Remove object files
clean:
rm -f *.o *.mod soil_veg_gen_parm__genmod.f90
......
......@@ -50,7 +50,7 @@ struct file_access_state {
void *handle_forcFileInfo = new_handle_file_info(); // Handle for the forcing file information
void *handle_ncid = new_handle_var_i(); // output file ids
OutputManager *output_manager;
int num_vectors_in_output_manager = 2;
int num_vectors_in_output_manager = 8;
int num_steps;
int outputStrucSize;
int stepsInCurrentFile;
......
......@@ -82,10 +82,13 @@ def create_output_path(outputPath):
print("Directory Created. Now Creating sub directories for SLURM Data and NetCDF data")
outputNetCDF = outputPath + "netcdf/"
outputSlurm = outputPath + "slurm/"
outputCSV = outputPath + "csv/"
if not exists(outputNetCDF):
os.mkdir(outputNetCDF)
if not exists(outputSlurm):
os.mkdir(outputSlurm)
if not exists(outputCSV):
os.mkdir(outputCSV)
# need to add the file name to outputSlurm
# The job will not be submitted without a file name
......
import subprocess
import csv
from sys import argv
'''
This is a script that gets the resource usage of jobs and output the stats as a csv.
'''
'''
This function uses the seff command and can get the following data:
- Start HRU
- Num HRU
- num CPUs
- CPU-Efficiency
- Wall-Clock Time
- Memory Used
'''
def seffCommand(jobId, numJobs):
input_prompt = "SummaActors: a\nSummaOriginal: o\n"
# Get input from the user
user_response = input(input_prompt)
print(user_response)
if user_response == "a":
output_csv_name = "SummaActors_jobStats_{}.csv".format(jobId)
elif user_response == "o":
output_csv_name = "SummaOriginal_jobStats_{}.csv".format(jobId)
else:
raise Exception("Something went wrong")
csvFile = open(output_csv_name, 'w')
header = ["startHRU", "numHRU", "#-CPU", "CPU Efficiency", "Wall-Clock Time", "Memory Used"]
writer = csv.writer(csvFile)
writer.writerow(header)
startHRU = 1
numHRU = 1000
for i in range(0, int(numJobs)):
print("Job", i)
rowData = []
rowData = [numHRU * i + 1, numHRU]
cmdString = "seff {}_{}".format(jobId, i)
cmd = subprocess.Popen(cmdString, shell=True, stdout=subprocess.PIPE)
for line in cmd.stdout:
if b'Cores per node:' in line:
cores = line.decode().split(" ")[-1]
cores = cores.strip()
if b'CPU Efficiency:' in line:
effeciency = line.decode().split(" ")[2]
effeciency = effeciency.strip()
if b'Job Wall-clock time:' in line:
wallClock = line.decode().split(" ")[-1]
wallClock = wallClock.strip()
if b'Memory Utilized:' in line:
memory = line.decode().split(" ")[2]
memory = memory.strip()
rowData.append(cores)
rowData.append(effeciency)
rowData.append(wallClock)
rowData.append(memory)
writer.writerow(rowData)
csvFile.close()
jobId = argv[1]
print(jobId)
numJobs = argv[2]
print(numJobs)
seffCommand(jobId, numJobs)
import os
import re
import sys
summaryFile = '_log_summaryActors.txt'
ext = ".out"
if len(sys.argv) == 1:
sys.exit('Error: no input folder specified')
else:
folder = sys.argv[1]
def determine_output(folder,file):
outFile = open(folder + file, 'r')
print(outFile)
try:
lines = outFile.readlines()
except UnicodeDecodeError:
outFile.close()
outFile = open(folder + file, encoding = "ISO-8859-1")
lines = outFile.readlines()
counter = 1
for line in reversed(lines):
if counter > 30:
return -1
else:
if "Hours" in line:
hours = re.findall("\d+\.\d+", line)
return hours
counter += 1
try:
os.remove(folder + "/" + summaryFile)
except OSError:
pass
files = []
for file in os.listdir(folder):
if file.endswith(ext):
files.append(file)
files.sort()
total_success = []
computation_time = []
with open(folder + '/' + summaryFile, "w") as sf:
sf.write('Summarizing log files in ' + folder + '\n \n')
sf.write('Log files' + '\n')
for file in files:
value = determine_output(folder, file)
if value == -1:
sf.write("{} - Still Running or Failed\n".format(file))
else:
sf.write("{} - Success after {} hours \n".format(file, value[0]))
......@@ -3,7 +3,7 @@ from os.path import isfile, join
from pathlib import Path
import xarray as xr
numHRU = 1
numHRU = 25
time = 'time'
scalarSWE = 'scalarSWE'
......@@ -28,8 +28,8 @@ varList = [time, scalarSWE, scalarCanopyWat, scalarAquiferStorage, scalarTotalSo
scalarTotalET, scalarTotalRunoff, scalarNetRadiation]
filename = "out.txt"
originalPath = Path('/gladwell/kck540/output/SummaOriginal/failingHRUCheck/SummaOriginal_G079506-079510_day.nc')
actorsPath = Path('/gladwell/kck540/output/SummaActors/FaillingHRUCheck/SummaActorsGRU79500-10_day.nc')
originalPath = Path('/home/kklenk/projects/rpp-kshook/kklenk/SummaOriginalOuput/May-13-2022/netcdf/SummaBE_G000001-000125_day.nc')
actorsPath = Path('/home/kklenk/projects/rpp-kshook/kklenk/SummaActorsOutput/May-26-2022/netcdf/SummaActorsGRU1-500_day.nc')
originalDataset = xr.open_dataset(originalPath)
actorsDataset = xr.open_dataset(actorsPath)
......@@ -37,11 +37,11 @@ actorsDataset = xr.open_dataset(actorsPath)
allHRUsOriginal = []
allHRUsActors = []
# for ihru in range(0, numHRU):
allHRUsOriginal.append(originalDataset.isel(hru=2).copy())
for ihru in range(0, numHRU):
allHRUsOriginal.append(originalDataset.isel(hru=ihru).copy())
# for ihru in range(6, numHRU):
allHRUsActors.append(actorsDataset.isel(hru=7).copy())
for ihru in range(0, numHRU):
allHRUsActors.append(actorsDataset.isel(hru=ihru).copy())
file = open(filename, "w")
for i in range(0, numHRU):
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment