From fb08f0af77290e98c930dfeece0438e2334ff25b Mon Sep 17 00:00:00 2001 From: Paul Brinkmeier Date: Mon, 5 Feb 2024 13:21:34 +0100 Subject: [PATCH] Add stuff for strong scaling measurements --- experiments/.gitignore | 2 + .../batch/measurements/strong/.gitkeep | 1 + experiments/configs/measurements/.gitkeep | 1 + .../measurements/strong/config_filling_1.json | 44 +++++++ experiments/scripts/gen/strong.py | 77 ++++++++++++ experiments/templates/spheroid.json | 116 ++++++++++++++++++ 6 files changed, 241 insertions(+) create mode 100644 experiments/batch/measurements/strong/.gitkeep create mode 100644 experiments/configs/measurements/.gitkeep create mode 100644 experiments/configs/measurements/strong/config_filling_1.json create mode 100644 experiments/scripts/gen/strong.py create mode 100644 experiments/templates/spheroid.json diff --git a/experiments/.gitignore b/experiments/.gitignore index 244293f..fe3fc10 100644 --- a/experiments/.gitignore +++ b/experiments/.gitignore @@ -3,3 +3,5 @@ venv logs __pycache__ generated/* +batch/measurements/strong/* +configs/measurements/strong/* diff --git a/experiments/batch/measurements/strong/.gitkeep b/experiments/batch/measurements/strong/.gitkeep new file mode 100644 index 0000000..8891e10 --- /dev/null +++ b/experiments/batch/measurements/strong/.gitkeep @@ -0,0 +1 @@ +Generated by scripts/gen/strong.py diff --git a/experiments/configs/measurements/.gitkeep b/experiments/configs/measurements/.gitkeep new file mode 100644 index 0000000..12f3527 --- /dev/null +++ b/experiments/configs/measurements/.gitkeep @@ -0,0 +1 @@ +Autogenerated by scripts/gen/strong.py diff --git a/experiments/configs/measurements/strong/config_filling_1.json b/experiments/configs/measurements/strong/config_filling_1.json new file mode 100644 index 0000000..113aece --- /dev/null +++ b/experiments/configs/measurements/strong/config_filling_1.json @@ -0,0 +1,44 @@ +{ + "DefineFunctions": [ + "r_angle()=360*rnd()", + "r_size()=400*rnd()" + ], + "Filling": { + "cells": [ + { + "_comment": "This is for the dynamic ECM", + "shape": "cube", + "box": [ + [0, 0, 0], + [399, 399, 401] + ], + "value": 0, + "celltype": 0 + }, + { + "shape": "sphere", + "pattern": "voronoi", + "count": 5500, + "radius": 75, + "center": [ + 200, + 200, + 200 + ], + "box": [ + [ + 110, + 110, + 110 + ], + [ + 290, + 290, + 290 + ] + ], + "celltype": 9 + } + ] + } +} diff --git a/experiments/scripts/gen/strong.py b/experiments/scripts/gen/strong.py new file mode 100644 index 0000000..5078a6e --- /dev/null +++ b/experiments/scripts/gen/strong.py @@ -0,0 +1,77 @@ +import copy +import json + +from dataclasses import dataclass +from typing import Tuple + +@dataclass +class Configuration: + nodes: int + tasks: int + gpus_per_node: int + blockcount: Tuple[int, int, int] + blocksize: Tuple[int, int, int] + + def get_domain_size(self) -> int: + return self.blockcount[0] * self.blocksize[0] * self.blockcount[1] * self.blocksize[1] * self.blockcount[2] * self.blocksize[2] + +configurations = [ + Configuration( 1, 12, 1, ( 1, 4, 3), (400, 100, 134)), + Configuration( 1, 24, 2, ( 2, 4, 3), (200, 100, 134)), + Configuration( 1, 48, 4, ( 4, 4, 3), (100, 100, 134)), + Configuration( 2, 96, 4, ( 4, 4, 6), (100, 100, 67)), + Configuration( 4, 192, 4, ( 4, 8, 6), (100, 50, 67)), + Configuration( 8, 384, 4, ( 8, 8, 6), ( 50, 50, 67)), + Configuration(16, 768, 4, ( 8, 16, 6), ( 50, 25, 67)), + Configuration(32, 1536, 4, (16, 16, 6), ( 25, 25, 67)) +] + +with open("templates/spheroid.json") as template_file: + template = json.load(template_file) + +for c in configurations: + print(c) + assert(c.get_domain_size() == configurations[0].get_domain_size()) + + nastja_config = copy.deepcopy(template) + + nastja_config["Geometry"]["blockcount"] = c.blockcount + nastja_config["Geometry"]["blocksize"] = c.blocksize + + label = f"{c.nodes:02}" + + if c.gpus_per_node < 4: + label += f"g{c.gpus_per_node}" + + with open(f"configs/measurements/strong/spheroid_{label}.json", "w") as config_file: + json.dump(nastja_config, config_file, indent=2) + + batch_config = f"""#!/usr/bin/env bash + +#SBATCH --job-name=strong-{label} +#SBATCH --account=hkf6 +#SBATCH --partition=booster +#SBATCH --nodes={c.nodes} +#SBATCH --ntasks={c.tasks} +# Counted per node +#SBATCH --gres=gpu:{c.gpus_per_node} +#SBATCH --time=06:00:00 +#SBATCH --output=logs/strong-{label}-%A_%a.log +#SBATCH --error=logs/strong-{label}-%A_%a.log +#SBATCH --array=1-5 + +SOURCE_DIR=/p/project/cellsinsilico/paulslustigebude +OUTPUT_DIR="/p/scratch/cellsinsilico/paul/nastja-out/strong-{label}-${{SLURM_ARRAY_TASK_ID}}" + +echo "${{OUTPUT_DIR}}" + +mkdir -p "${{OUTPUT_DIR}}" +source "${{SOURCE_DIR}}/activate-nastja-modules" + +srun --unbuffered "${{SOURCE_DIR}}/nastja/build-cuda/nastja" \\ + -c "${{SOURCE_DIR}}/ma/experiments/configs/measurements/strong/spheroid_{c.nodes:02}.json" \\ + -o "${{OUTPUT_DIR}}" +""" + + with open(f"batch/measurements/strong/strong-{label}", "w", encoding="utf8") as batch_config_file: + batch_config_file.write(batch_config) diff --git a/experiments/templates/spheroid.json b/experiments/templates/spheroid.json new file mode 100644 index 0000000..00601b8 --- /dev/null +++ b/experiments/templates/spheroid.json @@ -0,0 +1,116 @@ +{ + "Comments": ["Celltype Usage", "0 Dynamic ECM", "1-5 Not Used", "6 Liquid ", "7 Apoptotic cell Type ", "8 Basic Non Dividing Cell type (surrounding)", "9 Cancer"], + "Application": "Cells", + "CellsInSilico": { + "ecmdegradation": { + "enabled": "false", + "steps": 99999, + "stochastic": "true", + "probability": 0.5 + }, + "energyfunctions": ["Volume00", "Surface01", "Motility00", "Adhesion01", "DynamicECM00"], + "liquid": 6, + "volume": { + "default": { + "storage": "const", + "value": 500 + }, + "lambda": [0, 0, 0, 0, 0, 0, 0, 7.5, 7.5, 7.5], + "sizechange": [0, 0, 0, 0, 0, 0, 0, -0.05, 0, 0, 0, 0, 0, 0, 0] + }, + "surface": { + "default": { + "storage": "const", + "value": 400 + }, + "lambda": [0, 0, 0, 0, 0, 0, 0, 5.625, 5.625, 1], + "sizechange": [0, 0, 0, 0, 0, 0, 0, -0.05, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "adhesion": { + "matrix": [ + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 450], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 0, 0, 0, 0, 0, 0, 0, 0, 0], + [0, 450, 0, 0, 0, 0, 0, 0, 0, 50] + ] + }, + "temperature": 50, + "division": { + "enabled": "true", + "condition": ["", "", "", "", "", "", "", "", "", "( volume >= 0.9 * volume0 ) & ( rnd() <= 0.00001 ) & generation < 1"] + }, + "centerofmass": { + "steps": 1 + }, + "signaling": { + "enabled": false + }, + "orientation": { + "enabled": true, + "motility": "persistentRandomWalk", + "persistenceMagnitude": 0.0, + "recalculationtime": 200, + "motilityamount": [0, 0, 0, 0, 0, 0, 0, 0, 0, 0] + }, + "visitor": { + "stepwidth": 10, + "checkerboard": "01" + }, + "cleaner": { + "killDistance": 20 + }, + "dynamicecm": { + "enabled": true, + "stepsPerMcs": 100, + "pushSteps": 10, + "pushWeight": 2, + "ecmCellID": 0, + "deltat": 0.1, + "eta": 0.25, + "k0": 0.1, + "k1": 0.1, + "c": 4, + "alpha": 2, + "d": 0.3, + "phi": 1 + } + }, + "Geometry": { + "blockcount": [4, 4, 3], + "blocksize": [100, 100, 134] + }, + "Settings": { + "randomseed": 0, + "timesteps": 100 + }, + "WriteActions": ["CellInfo"], + "Writers": { + "CellInfo": { + "field": "", + "groupsize": 0, + "steps": 1, + "writer": "CellInfo" + }, + "ParallelVTK_Cells": { + "field": "cells", + "outputtype": "UInt32", + "printhints": false, + "steps": 100, + "writer": "ParallelVtkImage" + }, + "ParallelVTK_Displacement": { + "writer": "ParallelVtkImage", + "outputtype": "Float32", + "field": "dynamicecm", + "components": [0, 1, 2], + "steps": 100 + } + }, + "Include": "config_filling_1.json" +}