Add stuff for eval

experiments/eval/scripts/generate-strong-cpu-configs.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python
+
+import copy
+import json
+
+from pathlib import Path
+
+SIZE = [384, 384, 384]
+
+with (Path(__file__).parent.parent / "templates" / "strong-cpu.json").open(encoding="utf8") as f:
+    template = json.load(f)
+
+configs = [
+    [  4,  4,  3],
+    [  4,  4,  6],
+    [  4,  4, 12],
+    [  4,  8, 12],
+    [  8,  8, 12],
+    [  8,  8, 24],
+    [  8, 16, 24],
+    [ 16, 16, 24]
+]
+
+out_path = Path(__file__).parent.parent / "generated" / "config"
+
+for c in configs:
+    nc = copy.deepcopy(template)
+    nc["Geometry"]["blockcount"] = c
+    nc["Geometry"]["blocksize"] = [bs // bc for bc, bs in zip(c, SIZE)]
+
+    nc_out_path = out_path / f"strong-cpu-{c[0]:02}-{c[1]:02}-{c[2]:02}.json"
+    print(f"Dumping {(c[0] * c[1] * c[2]) // 48} to {nc_out_path}")
+    with nc_out_path.open("w", encoding="utf8") as f:
+        json.dump(nc, f)

experiments/eval/scripts/substitute.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+
+import jinja2
+import json
+import sys
+
+from pathlib import Path
+
+data = json.load(sys.stdin)
+
+templates_env = jinja2.Environment(
+    loader=jinja2.FileSystemLoader(Path(__file__).parent.parent / "templates"),
+    autoescape=jinja2.select_autoescape()
+)
+
+for possibly_incomplete_batch in data["batches"]:
+    batch = dict(list(data["common"].items()) + list(possibly_incomplete_batch.items()))
+    out_path = Path(__file__).parent.parent / "generated" / "batch" / batch["name"]
+    t = templates_env.get_template(sys.argv[1])
+    print(f"Dumping to {out_path}")
+    t.stream(**batch).dump(str(out_path))