diff --git a/docs/README.rst b/docs/README.rst new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/utils/netcdf/StatisticsScripts/resourageUsage.py b/utils/netcdf/StatisticsScripts/resourageUsage.py index 0e524de141148e3d5bcb4a0cac56eedee636216e..ff27a7b2b51a3b55a5e4825b53483d2190f71536 100644 --- a/utils/netcdf/StatisticsScripts/resourageUsage.py +++ b/utils/netcdf/StatisticsScripts/resourageUsage.py @@ -1,7 +1,53 @@ import subprocess +import csv +from sys import argv + +def seffCommand(jobId, numJobs): + csvFile = open('SummaActors_jobStatistics.csv', 'w') + header = ["startHRU", "numHRU", "CPU", "CPU Efficiency", "Wall-Clock Time", "Memory Used"] + + writer = csv.writer(csvFile) + + writer.writerow(header) + + startHRU = 1 + numHRU = 1000 + for i in range(0, int(numJobs)): + rowData = [] + rowData = [numHRU * i + 1, numHRU] + cmdString = "seff {}_{}".format(jobId, i) + cmd = subprocess.Popen(cmdString, shell=True, stdout=subprocess.PIPE) + for line in cmd.stdout: + if b'Cores per node:' in line: + cores = line.decode().split(" ")[-1] + cores = cores.strip() + + if b'CPU Efficiency:' in line: + effeciency = line.decode().split(" ")[2] + effeciency = effeciency.strip() + + if b'Job Wall-clock time:' in line: + wallClock = line.decode().split(" ")[-1] + wallClock = wallClock.strip() + + if b'Memory Utilized:' in line: + memory = line.decode().split(" ")[2] + memory = memory.strip() + + rowData.append(cores) + rowData.append(effeciency) + rowData.append(wallClock) + rowData.append(memory) + writer.writerow(rowData) + + csvFile.close() + +jobId = argv[1] +print(jobId) + +numJobs = argv[2] +print(numJobs) + +seffCommand(jobId, numJobs) -cmd = subprocess.Popen('seff 59326149_1', shell=True, stdout=subprocess.PIPE) -for line in cmd.stdout: - if b'CPU Utilized:' in line: - print(line) \ No newline at end of file diff --git a/utils/netcdf/StatisticsScripts/summarize_summaActors.py b/utils/netcdf/StatisticsScripts/summarize_summaActors.py new file mode 100644 index 0000000000000000000000000000000000000000..4a629be59f7b60180c1121cd34b2e2777aff508c --- /dev/null +++ b/utils/netcdf/StatisticsScripts/summarize_summaActors.py @@ -0,0 +1,63 @@ +import os +import re +import sys + +summaryFile = '_log_summaryActors.txt' +ext = ".out" + +if len(sys.argv) == 1: + sys.exit('Error: no input folder specified') + +else: + + folder = sys.argv[1] + +def determine_output(folder,file): + outFile = open(folder + file, 'r') + print(outFile) + try: + lines = outFile.readlines() + except UnicodeDecodeError: + outFile.close() + outFile = open(folder + file, encoding = "ISO-8859-1") + lines = outFile.readlines() + counter = 1 + for line in reversed(lines): + if counter > 30: + return -1 + else: + if "Hours" in line: + hours = re.findall("\d+\.\d+", line) + return hours + counter += 1 + + +try: + os.remove(folder + "/" + summaryFile) +except OSError: + pass + +files = [] +for file in os.listdir(folder): + if file.endswith(ext): + files.append(file) + +files.sort() + +total_success = [] + +computation_time = [] + +with open(folder + '/' + summaryFile, "w") as sf: + sf.write('Summarizing log files in ' + folder + '\n \n') + sf.write('Log files' + '\n') + + for file in files: + value = determine_output(folder, file) + if value == -1: + sf.write("{} - Still Running or Failed\n".format(file)) + else: + sf.write("{} - Success after {} hours \n".format(file, value[0])) + + +