update run and calibration script

src/main/python/calibrate.py

@@ -5,18 +5,15 @@
 import os
 import pandas as pd
 
-try:
-    from matsim import calibration
-except:
-    import calibration
+from matsim.calibration import create_calibration, ASCCalibrator, utils, analysis
 
 # %%
 
 if os.path.exists("mid.csv"):
     srv = pd.read_csv("mid.csv")
     sim = pd.read_csv("sim.csv")
 
-    _, adj = calibration.calc_adjusted_mode_share(sim, srv)
+    _, adj = analysis.calc_adjusted_mode_share(sim, srv)
 
     print(srv.groupby("mode").sum())
 
@@ -30,10 +27,10 @@
 modes = ["walk", "car", "ride", "pt", "bike"]
 fixed_mode = "walk"
 initial = {
-    "bike": -0.141210,
-    "pt": 0,
-    "car": 0,
-    "ride": -2.22873502992
+    "bike": -3.35,
+    "pt": -1.92,
+    "car": -0.79,
+    "ride": -0.76
 }
 
 # Based on MiD 2017, filtered on Lausitz region
@@ -64,16 +61,17 @@ def filter_modes(df):
     return df[df.main_mode.isin(modes)]
 
 
-study, obj = calibration.create_mode_share_study("calib", "matsim-lausitz-1.x-SNAPSHOT-e8458e3.jar",
-                                                 "../input/v1.0/lausitz-v1.0-100pct.config.xml",
-                                                 modes, target,
-                                                 initial_asc=initial,
-                                                 args="--25pct",
-                                                 jvm_args="-Xmx60G -Xmx60G -XX:+AlwaysPreTouch -XX:+UseParallelGC",
-                                                 transform_persons=filter_persons, transform_trips=filter_modes,
-                                                 lr=calibration.linear_lr_scheduler(start=0.3, interval=8),
-                                                 chain_runs=calibration.default_chain_scheduler)
+study, obj = create_calibration(
+    "calib",
+    ASCCalibrator(modes, initial, target, lr=utils.linear_lr_scheduler(start=0.3, interval=12)),
+    "matsim-lausitz-1.x-SNAPSHOT-20c8ab3.jar",
+    "../input/v1.0/lausitz-v1.0-25pct.config.xml",
+    args="--25pct",
+    jvm_args="-Xmx60G -Xmx60G -XX:+AlwaysPreTouch -XX:+UseParallelGC",
+    transform_persons=filter_persons, transform_trips=filter_modes,
+    chain_runs=utils.default_chain_scheduler
+)
 
 # %%
 
-study.optimize(obj, 10)
+study.optimize(obj, 8)

src/main/sh/job.sh

@@ -1,24 +1,27 @@
 #!/bin/bash --login
-#$ -l h_rt=790000
-#$ -j y
-#$ -m a
-#$ -o ./logfile/logfile_$JOB_NAME.log
-#$ -cwd
-#$ -pe mp 12
-#$ -l mem_free=6G
-#$ -N lausitz
+#SBATCH --time=200:00:00
+#SBATCH --partition=smp
+#SBATCH --output=./logfile/logfile_%x-%j.log
+#SBATCH --nodes=1                       # How many computing nodes do you need (for MATSim usually 1)
+#SBATCH --ntasks=1                      # How many tasks should be run (For MATSim usually 1)
+#SBATCH --cpus-per-task=12              # Number of CPUs per task (For MATSim usually 8 - 12)
+#SBATCH --mem=48G                       # RAM for the job
+#SBATCH --job-name=run-scenario         # name of your run, will be displayed in the joblist
+#SBATCH --mail-type=END,FAIL				      # Send email on end, and fail
+#SBATCH --mail-user ...	# Your email address
 
 date
 hostname
 
 jar="matsim-lausitz-*.jar"
 memory="${RUN_MEMORY:-70G}"
-config="${RUN_CONFIG:-lausitz-v1.0-100pct.config.xml}"
+config="${RUN_CONFIG:-lausitz-v1.0-25pct.config.xml}"
 
 arguments=""
 
 # Don't change anything below
 ################
+
 jvm_opts="-Xmx$memory -Xms$memory -XX:+AlwaysPreTouch -XX:+UseParallelGC"
 command="java $jvm_opts $JAVA_OPTS -jar $jar --config $config $RUN_ARGS $arguments run"
 
@@ -27,6 +30,9 @@ if [ -n "$RUN_DIR" ]; then
       command="$command --output $RUN_DIR/$RUN_NAME --runId $RUN_NAME"
 fi
 
+if [ -n "$RUN_NAME" ]; then
+      command="$command --output output/$RUN_NAME --runId $RUN_NAME"
+fi
 
 # Optional parameters
 if [ "$RUN_MONITOR" == "true" ]; then
@@ -46,4 +52,4 @@ echo ""
 module add java/17
 java -version
 
-$command
+$command

src/main/sh/runCalib.sh

@@ -1,11 +1,12 @@
 #!/bin/bash --login
-#$ -l h_rt=790000
-#$ -j y
-#$ -m a
-#$ -cwd
-#$ -pe mp 12
-#$ -l mem_free=6G
-#$ -N calib-lausitz
+#SBATCH --time=200:00:00
+#SBATCH --partition=smp
+#SBATCH --output=logfile_%x-%j.log
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --cpus-per-task=12
+#SBATCH --mem=48G
+#SBATCH --job-name=lausitz-calib
 
 date
 hostname
@@ -21,4 +22,4 @@ source env/bin/activate
 module add java/17
 java -version
 
-$command
+$command

src/main/sh/setup.sh

@@ -1,13 +1,11 @@
 #!/usr/bin/env bash
 
-python3 -m venv env
+python3.9 -m venv env
 source env/bin/activate
 
-module load gcc/9.2.0
-
 pip install --upgrade pip
-pip install optuna geopandas rtree pygeos
 
-# Download latest calibration script
-wget "https://github.com/matsim-vsp/matsim-python-tools/raw/master/matsim/calibration.py"
-wget "https://github.com/matsim-vsp/matsim-python-tools/raw/master/matsim/analysis.py"
+pip install "matsim-tools[calibration]==0.0.16"
+
+# Alternative: Install from a specific branch on github
+#pip install --force-reinstall "matsim-tools[calibration] @ git+https://github.com/matsim-vsp/matsim-python-tools.git@dist-calibration"

src/main/sh/start.sh

@@ -1,13 +1,19 @@
-#!/bin/bash
+#!/usr/bin/env bash
 
 # Helper script to start one or multiple jobs and pass arguments to it
 
-name=$( echo "$*" | sed -e 's/ //g' -e 's/--//g')
+if [ -z "${run_name}" ]; then
+    # Unset or emtpy
+    name=$( echo "$*" | sed -e 's/ //g' -e 's/:/_/g' -e 's/--//g' -e 's/\./_/g' -e 's/\///g')
+else
+    name="${run_name}"
+fi
+
 
 export RUN_ARGS="$*"
 export RUN_NAME="$name"
 
-echo "Starting run kh-$name"
+echo "Starting run $name"
 echo "$*"
 
-qsub -V -N lausitz-"$name" job.sh
+sbatch --export=ALL --job-name matsim-"$name" job.sh