From f854dc73587a308f7a8c7c01aa16ca358be5aa34 Mon Sep 17 00:00:00 2001
From: kck540 <kyle.klenk@usask.ca>
Date: Thu, 28 Apr 2022 17:53:48 -0400
Subject: [PATCH] Added some convience scripts for montoring output Also added
file for setting up a read the docs page
---
docs/README.rst | 0
.../StatisticsScripts/resourageUsage.py | 54 ++++++++++++++--
.../summarize_summaActors.py | 63 +++++++++++++++++++
3 files changed, 113 insertions(+), 4 deletions(-)
create mode 100644 docs/README.rst
create mode 100644 utils/netcdf/StatisticsScripts/summarize_summaActors.py
diff --git a/docs/README.rst b/docs/README.rst
new file mode 100644
index 0000000..e69de29
diff --git a/utils/netcdf/StatisticsScripts/resourageUsage.py b/utils/netcdf/StatisticsScripts/resourageUsage.py
index 0e524de..ff27a7b 100644
--- a/utils/netcdf/StatisticsScripts/resourageUsage.py
+++ b/utils/netcdf/StatisticsScripts/resourageUsage.py
@@ -1,7 +1,53 @@
import subprocess
+import csv
+from sys import argv
+
+def seffCommand(jobId, numJobs):
+ csvFile = open('SummaActors_jobStatistics.csv', 'w')
+ header = ["startHRU", "numHRU", "CPU", "CPU Efficiency", "Wall-Clock Time", "Memory Used"]
+
+ writer = csv.writer(csvFile)
+
+ writer.writerow(header)
+
+ startHRU = 1
+ numHRU = 1000
+ for i in range(0, int(numJobs)):
+ rowData = []
+ rowData = [numHRU * i + 1, numHRU]
+ cmdString = "seff {}_{}".format(jobId, i)
+ cmd = subprocess.Popen(cmdString, shell=True, stdout=subprocess.PIPE)
+ for line in cmd.stdout:
+ if b'Cores per node:' in line:
+ cores = line.decode().split(" ")[-1]
+ cores = cores.strip()
+
+ if b'CPU Efficiency:' in line:
+ effeciency = line.decode().split(" ")[2]
+ effeciency = effeciency.strip()
+
+ if b'Job Wall-clock time:' in line:
+ wallClock = line.decode().split(" ")[-1]
+ wallClock = wallClock.strip()
+
+ if b'Memory Utilized:' in line:
+ memory = line.decode().split(" ")[2]
+ memory = memory.strip()
+
+ rowData.append(cores)
+ rowData.append(effeciency)
+ rowData.append(wallClock)
+ rowData.append(memory)
+ writer.writerow(rowData)
+
+ csvFile.close()
+
+jobId = argv[1]
+print(jobId)
+
+numJobs = argv[2]
+print(numJobs)
+
+seffCommand(jobId, numJobs)
-cmd = subprocess.Popen('seff 59326149_1', shell=True, stdout=subprocess.PIPE)
-for line in cmd.stdout:
- if b'CPU Utilized:' in line:
- print(line)
\ No newline at end of file
diff --git a/utils/netcdf/StatisticsScripts/summarize_summaActors.py b/utils/netcdf/StatisticsScripts/summarize_summaActors.py
new file mode 100644
index 0000000..4a629be
--- /dev/null
+++ b/utils/netcdf/StatisticsScripts/summarize_summaActors.py
@@ -0,0 +1,63 @@
+import os
+import re
+import sys
+
+summaryFile = '_log_summaryActors.txt'
+ext = ".out"
+
+if len(sys.argv) == 1:
+ sys.exit('Error: no input folder specified')
+
+else:
+
+ folder = sys.argv[1]
+
+def determine_output(folder,file):
+ outFile = open(folder + file, 'r')
+ print(outFile)
+ try:
+ lines = outFile.readlines()
+ except UnicodeDecodeError:
+ outFile.close()
+ outFile = open(folder + file, encoding = "ISO-8859-1")
+ lines = outFile.readlines()
+ counter = 1
+ for line in reversed(lines):
+ if counter > 30:
+ return -1
+ else:
+ if "Hours" in line:
+ hours = re.findall("\d+\.\d+", line)
+ return hours
+ counter += 1
+
+
+try:
+ os.remove(folder + "/" + summaryFile)
+except OSError:
+ pass
+
+files = []
+for file in os.listdir(folder):
+ if file.endswith(ext):
+ files.append(file)
+
+files.sort()
+
+total_success = []
+
+computation_time = []
+
+with open(folder + '/' + summaryFile, "w") as sf:
+ sf.write('Summarizing log files in ' + folder + '\n \n')
+ sf.write('Log files' + '\n')
+
+ for file in files:
+ value = determine_output(folder, file)
+ if value == -1:
+ sf.write("{} - Still Running or Failed\n".format(file))
+ else:
+ sf.write("{} - Success after {} hours \n".format(file, value[0]))
+
+
+
--
GitLab