diff --git a/src/main/python/calibrate.py b/src/main/python/calibrate.py index 556d463..bb3f99b 100644 --- a/src/main/python/calibrate.py +++ b/src/main/python/calibrate.py @@ -5,10 +5,7 @@ import os import pandas as pd -try: - from matsim import calibration -except: - import calibration +from matsim.calibration import create_calibration, ASCCalibrator, utils, analysis # %% @@ -16,7 +13,7 @@ srv = pd.read_csv("mid.csv") sim = pd.read_csv("sim.csv") - _, adj = calibration.calc_adjusted_mode_share(sim, srv) + _, adj = analysis.calc_adjusted_mode_share(sim, srv) print(srv.groupby("mode").sum()) @@ -30,10 +27,10 @@ modes = ["walk", "car", "ride", "pt", "bike"] fixed_mode = "walk" initial = { - "bike": -0.141210, - "pt": 0, - "car": 0, - "ride": -2.22873502992 + "bike": -3.35, + "pt": -1.92, + "car": -0.79, + "ride": -0.76 } # Based on MiD 2017, filtered on Lausitz region @@ -64,16 +61,17 @@ def filter_modes(df): return df[df.main_mode.isin(modes)] -study, obj = calibration.create_mode_share_study("calib", "matsim-lausitz-1.x-SNAPSHOT-e8458e3.jar", - "../input/v1.0/lausitz-v1.0-100pct.config.xml", - modes, target, - initial_asc=initial, - args="--25pct", - jvm_args="-Xmx60G -Xmx60G -XX:+AlwaysPreTouch -XX:+UseParallelGC", - transform_persons=filter_persons, transform_trips=filter_modes, - lr=calibration.linear_lr_scheduler(start=0.3, interval=8), - chain_runs=calibration.default_chain_scheduler) +study, obj = create_calibration( + "calib", + ASCCalibrator(modes, initial, target, lr=utils.linear_lr_scheduler(start=0.3, interval=12)), + "matsim-lausitz-1.x-SNAPSHOT-20c8ab3.jar", + "../input/v1.0/lausitz-v1.0-25pct.config.xml", + args="--25pct", + jvm_args="-Xmx60G -Xmx60G -XX:+AlwaysPreTouch -XX:+UseParallelGC", + transform_persons=filter_persons, transform_trips=filter_modes, + chain_runs=utils.default_chain_scheduler +) # %% -study.optimize(obj, 10) +study.optimize(obj, 8) diff --git a/src/main/sh/job.sh b/src/main/sh/job.sh index 1281b25..7fa2dd2 100644 --- a/src/main/sh/job.sh +++ b/src/main/sh/job.sh @@ -1,24 +1,27 @@ #!/bin/bash --login -#$ -l h_rt=790000 -#$ -j y -#$ -m a -#$ -o ./logfile/logfile_$JOB_NAME.log -#$ -cwd -#$ -pe mp 12 -#$ -l mem_free=6G -#$ -N lausitz +#SBATCH --time=200:00:00 +#SBATCH --partition=smp +#SBATCH --output=./logfile/logfile_%x-%j.log +#SBATCH --nodes=1 # How many computing nodes do you need (for MATSim usually 1) +#SBATCH --ntasks=1 # How many tasks should be run (For MATSim usually 1) +#SBATCH --cpus-per-task=12 # Number of CPUs per task (For MATSim usually 8 - 12) +#SBATCH --mem=48G # RAM for the job +#SBATCH --job-name=run-scenario # name of your run, will be displayed in the joblist +#SBATCH --mail-type=END,FAIL # Send email on end, and fail +#SBATCH --mail-user ... # Your email address date hostname jar="matsim-lausitz-*.jar" memory="${RUN_MEMORY:-70G}" -config="${RUN_CONFIG:-lausitz-v1.0-100pct.config.xml}" +config="${RUN_CONFIG:-lausitz-v1.0-25pct.config.xml}" arguments="" # Don't change anything below ################ + jvm_opts="-Xmx$memory -Xms$memory -XX:+AlwaysPreTouch -XX:+UseParallelGC" command="java $jvm_opts $JAVA_OPTS -jar $jar --config $config $RUN_ARGS $arguments run" @@ -27,6 +30,9 @@ if [ -n "$RUN_DIR" ]; then command="$command --output $RUN_DIR/$RUN_NAME --runId $RUN_NAME" fi +if [ -n "$RUN_NAME" ]; then + command="$command --output output/$RUN_NAME --runId $RUN_NAME" +fi # Optional parameters if [ "$RUN_MONITOR" == "true" ]; then @@ -46,4 +52,4 @@ echo "" module add java/17 java -version -$command +$command \ No newline at end of file diff --git a/src/main/sh/runCalib.sh b/src/main/sh/runCalib.sh index 680ce54..a12979e 100644 --- a/src/main/sh/runCalib.sh +++ b/src/main/sh/runCalib.sh @@ -1,11 +1,12 @@ #!/bin/bash --login -#$ -l h_rt=790000 -#$ -j y -#$ -m a -#$ -cwd -#$ -pe mp 12 -#$ -l mem_free=6G -#$ -N calib-lausitz +#SBATCH --time=200:00:00 +#SBATCH --partition=smp +#SBATCH --output=logfile_%x-%j.log +#SBATCH --nodes=1 +#SBATCH --ntasks=1 +#SBATCH --cpus-per-task=12 +#SBATCH --mem=48G +#SBATCH --job-name=lausitz-calib date hostname @@ -21,4 +22,4 @@ source env/bin/activate module add java/17 java -version -$command +$command \ No newline at end of file diff --git a/src/main/sh/setup.sh b/src/main/sh/setup.sh index 668d49b..dd0a975 100644 --- a/src/main/sh/setup.sh +++ b/src/main/sh/setup.sh @@ -1,13 +1,11 @@ #!/usr/bin/env bash -python3 -m venv env +python3.9 -m venv env source env/bin/activate -module load gcc/9.2.0 - pip install --upgrade pip -pip install optuna geopandas rtree pygeos -# Download latest calibration script -wget "https://github.com/matsim-vsp/matsim-python-tools/raw/master/matsim/calibration.py" -wget "https://github.com/matsim-vsp/matsim-python-tools/raw/master/matsim/analysis.py" \ No newline at end of file +pip install "matsim-tools[calibration]==0.0.16" + +# Alternative: Install from a specific branch on github +#pip install --force-reinstall "matsim-tools[calibration] @ git+https://github.com/matsim-vsp/matsim-python-tools.git@dist-calibration" \ No newline at end of file diff --git a/src/main/sh/start.sh b/src/main/sh/start.sh index 484d6fb..addb7b5 100644 --- a/src/main/sh/start.sh +++ b/src/main/sh/start.sh @@ -1,13 +1,19 @@ -#!/bin/bash +#!/usr/bin/env bash # Helper script to start one or multiple jobs and pass arguments to it -name=$( echo "$*" | sed -e 's/ //g' -e 's/--//g') +if [ -z "${run_name}" ]; then + # Unset or emtpy + name=$( echo "$*" | sed -e 's/ //g' -e 's/:/_/g' -e 's/--//g' -e 's/\./_/g' -e 's/\///g') +else + name="${run_name}" +fi + export RUN_ARGS="$*" export RUN_NAME="$name" -echo "Starting run kh-$name" +echo "Starting run $name" echo "$*" -qsub -V -N lausitz-"$name" job.sh +sbatch --export=ALL --job-name matsim-"$name" job.sh \ No newline at end of file