Add more...

experiments/eval/data/strong-cpu.tsv

@@ -1,10 +1,10 @@
-label	nodes	mean_time	std_time	speedup	speedup_std
+label	nodes	tasks	mean_time	std_time	speedup	speedup_std
-1	1	1132.1093720291667	18.428644622186628	1.0	0.016278148629010596
+1	1	48	1953.3832839708334	14.087268287606769	1.0	0.007211727674340593
-2	2	619.9614289104168	7.890810670766857	1.8260964621925766	0.029725469622481192
+2	2	96	1046.3540047312501	6.0880471399828595	1.866847429396085	0.013463195270347342
-4	4	348.657283046875	3.3739994706945122	3.247054993762918	0.052856043795043856
+4	4	192	566.0709002166666	5.005298021298787	3.4507749527897755	0.024886049224955378
-8	8	200.79244575312498	1.259910347938215	5.63820699420684	0.09177957145282603
+8	8	384	318.1225953708333	9.706195475255436	6.1403475024896865	0.04428251401377301
-16	16	111.37033198671875	0.3943541234366863	10.16526889911915	0.16547175799372044
+16	16	768	178.56471887994792	9.393026972005948	10.939357428631386	0.07889166670756431
-32	32	65.69918741145833	0.43464214827532205	17.231710415823503	0.28050034328084494
+32	32	1536	99.82801128216144	1.4413062581268035	19.56748670921274	0.14111538541812124
-64	64	38.56257490957031	0.12227397957602136	29.35772247273364	0.4778893698204027
+64	64	3072	55.75610567220052	1.0303316198245824	35.034428255357376	0.25265875580386077
-128	128	21.433497051595047	0.14016683538819355	52.81962944749218	0.8598057786755426
+128	128	6144	31.589132888118492	0.32226687791817415	61.83719226764697	0.4459529907801098
-256	256	13.247708324598525	0.030540850807946187	85.45699711149668	1.3910817003698723
+256	256	12288	19.11704500200738	0.18528928846832715	102.18018965617955	0.7368957015128405

experiments/eval/data/strong-gpu.tsv

@@ -1,8 +1,9 @@
 label	nodes	tasks	mean_time	std_time	speedup	speedup_std
-1	1	1	157.5217688	0.5311809049610176	1.0	0.003372111099358209
+1	1	1	316.56674059999995	1.3298279828832311	1.0	0.004200782370133899
-1	1	2	120.15237200000001	0.299200156243067	1.3110167213344732	0.004420894037456185
+1	1	2	239.7297573	1.3705192585818793	1.3205150005797797	0.005547196133932895
-1	1	4	75.69762635	0.5769735213369493	2.0809340582447473	0.007017140834839633
+1	1	4	150.86725470000002	1.463195211854327	2.098313124537819	0.008814556780579045
-2	2	8	52.110670325	0.940887445857671	3.022831366735062	0.010193323203255446
+2	2	8	103.758313	1.3959932139830615	3.051001230137579	0.012816592178818781
-4	4	16	28.215493675000005	0.5598524922936691	5.582811012077745	0.018825858979446598
+4	4	16	56.423368849999996	0.8293394846143929	5.610560784514376	0.02356874483015261
-8	8	32	12.422501156249998	0.31940690526235005	12.680358554102268	0.04275957782413007
+8	8	32	24.168551143749998	0.5503559531334744	13.09829202078025	0.055023074199759194
-16	16	64	7.70770194375	0.04068644910316383	20.43693048194875	0.06891560011499148
+16	16	64	15.05716248125	0.04648043269670786	21.024329185144023	0.08831863138484462
+32	32	128	9.9111531140625	0.08848604506395903	31.940455056721632	0.1341749004963304

experiments/eval/scripts/timing.py

@@ -44,12 +44,13 @@ def get_outdirs(jobid: str):
     return sorted(Path("/p/scratch/cellsinsilico/paul/nastja-out").glob(f"*{jobid}*"))
 
 
-def load_array_mean_timings(jobid: str):
+def load_array_mean_timings(jobid: str, excluded_array_indices):
-    mts = [
+    mts = []
-        load_mean_timings(outdir_path)
+    for outdir_path in get_outdirs(jobid):
-        for outdir_path
+        if any(str(outdir_path).endswith(str(i)) for i in excluded_array_indices):
-        in get_outdirs(jobid)
+            print(f"Not loading timings for {outdir_path} because it was excluded.", file=sys.stderr)
-    ]
+            continue
+        mts.append(load_mean_timings(outdir_path))
 
     return pandas.DataFrame(list(mts), columns=mts[0].index)
 
@@ -81,12 +82,28 @@ if __name__ == "__main__":
     for i, jobid in enumerate(args.jobid, 1):
         print(f"({i:2}/{len(args.jobid):2}) Loading accounting data for {jobid}", file=sys.stderr)
         accounting_data = get_accounting_data(jobid)
-        array_mean_timings = load_array_mean_timings(jobid)
+
+        jobs = []
+        excluded_array_indices = []
+        for array_job in accounting_data["jobs"]:
+            # Get metadata related to array
+            array_main_job = array_job["array"]["job_id"]
+            array_index = array_job["array"]["task_id"]
+            # The last step is the actual job we want the data for
+            # The steps before set up cluster etc.
+            last_step = array_job["steps"][-1]
+            if last_step["state"] != "COMPLETED":
+                print(f"WARNING: {array_main_job}.{array_index} has state {last_step['state']}, excluding it from measurements", file=sys.stderr)
+                excluded_array_indices.append(array_index)
+                continue
+            jobs.append(last_step)
+
+        array_mean_timings = load_array_mean_timings(jobid, excluded_array_indices)
 
         results.append({
             "jobid": jobid,
             "means": array_mean_timings.mean().to_dict(),
             "stds": array_mean_timings.std().to_dict(),
-            "accounting": [array_job["steps"][-1] for array_job in accounting_data["jobs"]]
+            "accounting": jobs
         })
     print(json.dumps(results, indent=2 if args.prettify else None))

experiments/eval/strong-cpu.json

@@ -13,19 +13,19 @@
       "name": "strong-cpu-04-04-03",
       "nodes": 1,
       "tasks": 48,
-      "time": "00:30:00"
+      "time": "01:00:00"
     },
     {
       "name": "strong-cpu-04-04-06",
       "nodes": 2,
       "tasks": 96,
-      "time": "00:30:00"
+      "time": "01:00:00"
     },
     {
       "name": "strong-cpu-04-04-12",
       "nodes": 4,
       "tasks": 192,
-      "time": "00:10:00"
+      "time": "00:20:00"
     },
     {
       "name": "strong-cpu-04-08-12",

experiments/eval/templates/strong-cpu.json

@@ -253,6 +253,6 @@
     "randomseed": 42,
     "statusoutput": 1,
     "timestepguard": 1,
-    "timesteps": 5
+    "timesteps": 10
   }
 }

experiments/eval/templates/strong-gpu.json

@@ -253,7 +253,7 @@
     "randomseed": 42,
     "statusoutput": 1,
     "timestepguard": 1,
-    "timesteps": 5,
+    "timesteps": 10,
     "cuda": {
       "subblocks": {
         "blockDim": [8, 8, 8]