Adds utility for summarizing routing performance

runners/all

@@ -2,8 +2,6 @@
 
 set -euo pipefail
 
-shed=$1
-
 epoch=$((7*24*60*60))
 ticks_per_packet=$((60*120))
 time_to_live=$((3*$epoch))
@@ -11,11 +9,16 @@ buffer_size=$((3*7*24*60*60))
 duty=300
 
 run() {
+  local router=$1
+  local community=$2
+  local seed=$3
+  local shed=$4
+
   echo 'start:' $@ >&2
   python main.py \
-    --seed "$3" \
-    --router "$1" \
-    --community "$2" \
+    --router "$router" \
+    --community "$community" \
+    --seed "$seed" \
     --community-args \
       "epoch=${epoch}" \
     --trace "csv" \
@@ -33,28 +36,29 @@ run() {
 
 nproc=$(nproc)
 
-mkdir -p logs/$shed
 
+for shed in $@; do
 for seed in {1..10}; do
 
   for router in direct epidemic; do
     [[ "$(jobs | wc -l)" -ge "$nproc" ]] && wait -n
-    run "$router" "none" "$seed" > "logs/${shed}/${router}_${seed}.log" &
+    mkdir -p "logs/${router}"
+    run "$router" "none" "$seed" "$shed" > "logs/${router}/${shed}_${seed}.log" &
   done
 
   for router in bubble hcbf; do
     for community in kclique louvain; do
       [[ "$(jobs | wc -l)" -ge "$nproc" ]] && wait -n
-      run "$router" "$community" "$seed" > "logs/${shed}/${router}_${community}_${seed}.log" &
+      mkdir -p "logs/${router}_${community}"
+      run "$router" "$community" "$seed" "$shed" > "logs/${router}_${community}/${shed}_${seed}.log" &
     done
   done
 done
+done
 
 wait
 
-for f in direct epidemic {bubble,hcbf}_{kclique,louvain}; do
-  ./util/shed_summary "logs/$shed/$f"*.log > "logs/$shed/${f}_summary.yml"
-done
+./util/runner_all_summary logs/*/*.log
 
 # send slack notification if the script exists
 status=$?

util/runner_all_summary

+#!/usr/bin/env python3
+
+from collections import defaultdict
+import re
+import sys
+
+import ggplot as gg
+import pandas
+import yaml
+
+def record_from_log(log):
+    log_path = log
+    with open(log) as log:
+        # filter out garbage in log
+        log = re.findall(r'\w+:.*', log.read())
+        log = '\n'.join(log)
+        return yaml.load(log)
+
+if __name__ == '__main__':
+    routers = set()
+    traces = defaultdict(list)
+    for log in sys.argv[1:]:
+        record = record_from_log(log)
+        record['trace'] = re.sub(r'.*(shed\d).*', '\\1', record['trace'])
+        if record['community'] != 'none':
+            record['router'] += '_' + record['community']
+        routers.add(record['router'])
+        del record['community']
+        traces[record['trace']].append(record)
+
+    traces = {
+        trace: pandas.DataFrame.from_records(records)
+        for trace, records in traces.items()
+    }
+
+    def get_router(trace, router):
+        trace = traces[trace]
+        return trace[trace['router'] == router]
+
+    summary = []
+
+    for trace in traces:
+        for router in routers:
+            df = get_router(trace, router).mean()
+            df['trace'] = trace
+            df['router'] = router
+            summary.append(df)
+
+    summary = pandas.DataFrame.from_records(summary)
+
+    for router in routers:
+        aes = gg.aes(
+            x='trace',
+            y='delivery_ratio',
+            weight='delivery_ratio',
+        )
+
+        plt = gg.ggplot(aes, summary[summary['router']==router])
+        plt += gg.geom_bar()
+        plt += gg.ggtitle(f'{router}')
+        plt += gg.ylim(low=0, high=1)
+
+        plt.save(f'{router}.png', width=8, height=6, dpi=300)
+
+

util/shed_summarize

@@ -25,6 +25,6 @@ if __name__ == '__main__':
         d = stats.describe(v)
         o[k] = {
             'mean': float(d.mean),
-            'stddev': float(d.variance**(1/2)),
+            'stddev': float(d.variance**(0.5)),
         }
     print(yaml.dump(o))