paul/ma
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add weak scaling config generation script

Browse Source
This commit is contained in:
Paul Brinkmeier 2024-03-11 00:14:03 +01:00
parent d41d8b564b
commit 79227e93d9
4 changed files with 389 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
experiments/eval/scripts/build-pgf-datafiles.py
View File

@ -20,21 +20,33 @@ def print_table(data, spec):
if __name__ == "__main__":
p = argparse.ArgumentParser(description="Turn files generated by timing.py into pgf datafiles")
p.add_argument("timing_file")
p.add_argument("--weak", action="store_true")
args = p.parse_args()
with open(args.timing_file, "r", encoding="utf8") as f:
jobs = json.load(f)
scaling_spec = {
"label": lambda job: job["accounting"][0]["nodes"]["count"],
"nodes": lambda job: job["accounting"][0]["nodes"]["count"],
"tasks": lambda job: job["accounting"][0]["tasks"]["count"],
"mean_time": lambda job: job["means"]["TimeStep"],
"std_time": lambda job: job["stds"]["TimeStep"],
"speedup": lambda job: jobs[0]["means"]["TimeStep"] / job["means"]["TimeStep"],
# Standard deviation scaled to speedup
"speedup_std": lambda job: (jobs[0]["means"]["TimeStep"] / job["means"]["TimeStep"]) * (job["stds"]["TimeStep"] / job["means"]["TimeStep"]),
# 95% confidence interval
"speedup_error": lambda job: (jobs[0]["means"]["TimeStep"] / job["means"]["TimeStep"]) * (job["stds"]["TimeStep"] / job["means"]["TimeStep"]) / math.sqrt(len(jobs)) * 1.96,
}
if not args.weak:
scaling_spec = {
"label": lambda job: job["accounting"][0]["nodes"]["count"],
"nodes": lambda job: job["accounting"][0]["nodes"]["count"],
"tasks": lambda job: job["accounting"][0]["tasks"]["count"],
"mean_time": lambda job: job["means"]["TimeStep"],
"std_time": lambda job: job["stds"]["TimeStep"],
"speedup": lambda job: jobs[0]["means"]["TimeStep"] / job["means"]["TimeStep"],
# Standard deviation scaled to speedup
"speedup_std": lambda job: (jobs[0]["means"]["TimeStep"] / job["means"]["TimeStep"]) * (job["stds"]["TimeStep"] / job["means"]["TimeStep"]),
# 95% confidence interval
"speedup_error": lambda job: (jobs[0]["means"]["TimeStep"] / job["means"]["TimeStep"]) * (job["stds"]["TimeStep"] / job["means"]["TimeStep"]) / math.sqrt(len(jobs)) * 1.96,
}
else:
scaling_spec = {
"nodes": lambda job: job["accounting"][0]["nodes"]["count"],
"tasks": lambda job: job["accounting"][0]["tasks"]["count"],
"mean_time": lambda job: job["means"]["TimeStep"],
"std_time": lambda job: job["stds"]["TimeStep"],
"efficiency": lambda job: jobs[0]["means"]["TimeStep"] / job["means"]["TimeStep"],
"efficiency_error": lambda job: (jobs[0]["means"]["TimeStep"] / job["means"]["TimeStep"]) * (job["stds"]["TimeStep"] / job["means"]["TimeStep"]) / math.sqrt(len(jobs)) * 1.96,
}
print_table(jobs, scaling_spec)

152
experiments/eval/scripts/generate-weak-configs.py Executable file
View File

@ -0,0 +1,152 @@
#!/usr/bin/env python
import jinja2
import json
import sys
from dataclasses import dataclass
from pathlib import Path
from typing import Tuple
SIZE = (192, 192, 192)
templates_env = jinja2.Environment(
loader=jinja2.FileSystemLoader(Path(__file__).parent.parent / "templates"),
autoescape=jinja2.select_autoescape()
)
@dataclass
class Experiment:
job_name: str
account: str
partition: str
nastja_binary_path: str
nodes: int
tasks: int
num_blocks: Tuple[int, int, int]
domain_scale: Tuple[int, int, int]
time: str = "00:15:00"
extra_sbatch_line: str = ""
logfile_path: str = "/p/project/cellsinsilico/paulslustigebude/ma/experiments/eval/logs/%x-%A.%a"
config_path: str = "/p/project/cellsinsilico/paulslustigebude/ma/experiments/eval/generated/config/${SLURM_JOB_NAME}.json"
output_dir_path: str = "/p/scratch/cellsinsilico/paul/nastja-out/${SLURM_JOB_NAME}-${SLURM_ARRAY_JOB_ID}.${SLURM_ARRAY_TASK_ID}"
def get_config(self):
with (Path(__file__).parent.parent / "templates" / "weak.json").open(encoding="utf8") as f:
config = json.load(f)
size = (
SIZE[0] * self.domain_scale[0],
SIZE[1] * self.domain_scale[1],
SIZE[2] * self.domain_scale[2],
)
blocksize = (
size[0] // self.num_blocks[0],
size[1] // self.num_blocks[1],
size[2] // self.num_blocks[2],
)
config["Geometry"] = {
"blockcount": list(self.num_blocks),
"blocksize": list(blocksize),
}
cells_filling = [{
"box": [
[0, 0, 0],
list(size)
],
"celltype": 0,
"component": 0,
"pattern": "const",
"seed": 0,
"shape": "cube",
"value": 0,
}]
for z in range(self.domain_scale[2]):
for y in range(self.domain_scale[1]):
for x in range(self.domain_scale[0]):
cx = x * SIZE[0] + SIZE[0] // 2
cy = y * SIZE[1] + SIZE[1] // 2
cz = z * SIZE[2] + SIZE[2] // 2
cells_filling.append({
"shape": "sphere",
"pattern": "voronoi",
"count": 715,
"radius": 38,
"center": [cx, cy, cz],
"box": [
[cx - 38, cy - 38, cz - 38],
[cx + 38, cy + 38, cz + 38]
],
"celltype": 9,
"seed": 758960,
})
config["Filling"]["cells"] = cells_filling
return config
def write_batch_file(self, out_path: Path):
t = templates_env.get_template("strong-batch.j2")
t.stream(
name=self.job_name,
account=self.account,
partition=self.partition,
nodes=self.nodes,
tasks=self.tasks,
extra_sbatch_line=self.extra_sbatch_line,
time=self.time,
logfile_path=self.logfile_path,
nastja_binary_path=self.nastja_binary_path,
config_path=self.config_path,
output_dir_path=self.output_dir_path,
).dump(str(out_path))
def make_cpu_ex(x: int, y: int, z: int) -> Experiment:
num_blocks = x * y * z
assert num_blocks % 48 == 0
num_nodes = num_blocks // 48
assert x % 4 == 0
assert y % 4 == 0
assert z % 3 == 0
return Experiment(
job_name=f"weak-cpu-{x:02}-{y:02}-{z:02}",
account="cellsinsilico",
partition="batch",
nastja_binary_path="/p/project/cellsinsilico/paulslustigebude/nastja/build-nocuda/nastja",
nodes=num_nodes,
tasks=num_blocks,
num_blocks=(x, y, z),
domain_scale=(x // 4, y // 4, z // 3),
)
experiments = [
make_cpu_ex(4, 4, 3),
make_cpu_ex(4, 4, 6),
make_cpu_ex(4, 4, 12),
make_cpu_ex(4, 8, 12),
make_cpu_ex(8, 8, 12),
make_cpu_ex(8, 8, 24),
make_cpu_ex(8, 16, 24),
make_cpu_ex(16, 16, 24),
]
if __name__ == "__main__":
outdir = Path(__file__).parent.parent / "generated"
for e in experiments:
print(f"Generating config for {e.job_name}", file=sys.stderr)
config_path = (outdir / "config" / e.job_name).with_suffix(".json")
with config_path.open("w", encoding="utf8") as f:
json.dump(e.get_config(), f, indent=2)
print(f"Generating batch file for {e.job_name}", file=sys.stderr)
e.write_batch_file(outdir / "batch" / e.job_name)

20
experiments/eval/scripts/make-latex-table.py Executable file
View File

@ -0,0 +1,20 @@
#!/usr/bin/env python
import argparse
import pandas
def show_seconds(s: float) -> str:
return f"{s:.2f}s"
if __name__ == '__main__':
p = argparse.ArgumentParser(description="Make a latex table from a timings tsv")
p.add_argument("timingfile")
args = p.parse_args()
df = pandas.read_csv(args.timingfile, sep="\t")
for i in range(len(df)):
print(f"{df['nodes'][i]} & {df['tasks'][i]} & {show_seconds(df['mean_time'][i])} & {show_seconds(df['std_time'][i])} & {df['speedup'][i]:.02f} & {df['speedup_error'][i]:.02f} \\\\")

193
experiments/eval/templates/weak.json Normal file
View File

@ -0,0 +1,193 @@
{
"Application": "Cells",
"CellsInSilico": {
"2D": false,
"adhesion": {
"matrix": [
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 450.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0],
[0.0, 450.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 50.0]
],
"polarityenabled": false
},
"centerofmass": {
"steps": 1
},
"cleaner": {
"killdistance": 0,
"steps": 100
},
"contactinhibition": {
"enabled": false
},
"division": {
"condition": [
"",
"",
"",
"",
"",
"",
"",
"",
"",
"( volume >= 0.9 * volume0 ) & ( rnd() <= 0.00001 ) & generation < 1"
],
"enabled": true,
"halveSignals": false
},
"dynamicecm": {
"alpha": 2.0,
"beta": 0.5,
"c": 4.0,
"deltat": 0.1,
"ecmCellID": 0,
"enabled": true,
"eta": 0.25,
"k0": 0.1,
"k1": 0.1,
"lambda": 10.0,
"phi": 1.0,
"pushSteps": 10,
"pushWeight": 0.5,
"stepsPerMcs": 100
},
"ecmdegradation": {
"enabled": false
},
"energyfunctions": [
"Volume00",
"Surface01",
"Motility00",
"Adhesion01",
"DynamicECM00"
],
"liquid": 6,
"logcellproperties": {
"enabled": false
},
"orientation": {
"enabled": true,
"motility": "persistentRandomWalk",
"motilityamount": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
],
"numRandomNumbers": 5,
"persistenceMagnitude": 0.0,
"persistentDecay": 0.8,
"recalculationtime": 200
},
"polarity": {
"enabled": false
},
"signaling": {
"constant": false,
"enabled": false
},
"surface": {
"default": {
"storage": "const",
"value": 400.0
},
"lambda": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
5.625,
5.625,
1.0
],
"sizechange": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
-0.05,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
]
},
"temperature": 50.0,
"visitor": {
"checkerboard": "01",
"stepwidth": 10
},
"volume": {
"default": {
"storage": "const",
"value": 500.0
},
"lambda": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
7.5,
7.5,
7.5
],
"sizechange": [
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
-0.05,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0,
0.0
]
}
},
"DefineFunctions": [
"r_angle()=360*rnd()",
"r_size()=400*rnd()"
],
"Filling": {
"initialoutput": false,
"randomseed": 758959
},
"Settings": {
"randomseed": 42,
"statusoutput": 1,
"timesteps": 10
}
}