task_config_template.cfg

[metrics_parameters_inference]
patterns = ['Prediction-Time: (\d+\.\d+|\d+) milliseconds']
metrics = ['prediction_time']
compute_method = ['average']

[metrics_parameters_imperative_hybrid_top_1]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'training: accuracy=(\d+\.\d+|\d+)', 'validation: accuracy=(\d+\.\d+|\d+)', 'time cost: (\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'validation_acc', 'total_training_time']
compute_method = ['average', 'last', 'last', 'total']

[metrics_gpu_symbolic_p3_inference]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec']
metrics = ['speed']
compute_method = ['average']

[metrics_parameters_images_top_1]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)' ]
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total', 'last']

[metrics_parameters_images_top_5]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)', 'Train-top_k_accuracy_5=(\d+\.\d+|\d+)','Validation-top_k_accuracy_5=(\d+\.\d+|\d+)' ]
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc','Train-top_k_accuracy_5','Validation-top_k_accuracy_5']
compute_method = ['average', 'last', 'total', 'last','last','last']

[metrics_mkl_symbolic_c5_inference]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Speed: (\d+\.\d+|\d+) samples/sec', 'Speed: (\d+\.\d+|\d+) samples/sec']
metrics = ['speed', 'speed-p90', 'speed-p50']
compute_method = ['average', 'p90', 'p50']

[mkl_resnet18_cifar10_symbolic]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)' ]
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total', 'last']
command_to_execute = python image_classification/image_classification.py --model resnet18_v2 --dataset cifar10 --mode symbolic --gpus 0 --epochs 25 --log-interval 50 --kvstore local
num_gpus = 0

[resnet50_cifar10_symbolic]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)' ]
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total', 'last']
command_to_execute = python image_classification/image_classification.py --model resnet50_v1 --dataset cifar10 --mode symbolic --gpus 8 --epochs 20 --log-interval 50 --kvstore device
num_gpus = 8

[resnet50_cifar10_symbolic_fp16_batch_size64]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python image_classification/image_classification.py --model resnet50_v1 --dataset cifar10 --mode symbolic --gpus 8 --epochs 20 --log-interval 50 --dtype float16 --batch-size 64 --kvstore device
num_gpus = 8

[resnet50_cifar10_symbolic_fp32_batch_size32]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python image_classification/image_classification.py --model resnet50_v1 --dataset cifar10 --mode symbolic --gpus 8 --epochs 20 --log-interval 50 --batch-size 32  --kvstore device
num_gpus = 8

[resnet50_cifar10_symbolic_fp16_batch_size32]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python image_classification/image_classification.py --model resnet50_v1 --dataset cifar10 --mode symbolic --gpus 8 --epochs 20 --log-interval 50 --dtype float16 --batch-size 32 --kvstore device
num_gpus = 8

[resnet50_cifar10_symbolic_fp32_batch_size16]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python image_classification/image_classification.py --model resnet50_v1 --dataset cifar10 --mode symbolic --gpus 8 --epochs 20 --log-interval 50 --batch-size 16 --kvstore device
num_gpus = 8

[resnet50_cifar10_symbolic_fp32_batch_size64]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python image_classification/image_classification.py --model resnet50_v1 --dataset cifar10 --mode symbolic --gpus 8 --epochs 20 --log-interval 50 --batch-size 64 --kvstore device
num_gpus = 8

[resnet50_cifar10_hybrid]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'training: accuracy=(\d+\.\d+|\d+)', 'validation: accuracy=(\d+\.\d+|\d+)', 'time cost: (\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'validation_acc', 'total_training_time']
compute_method = ['average', 'last', 'last', 'total']
command_to_execute = python image_classification/image_classification.py --model resnet50_v1 --dataset cifar10 --mode hybrid --gpus 8 --epochs 20 --log-interval 50 --kvstore device
num_gpus = 8

[resnet50_cifar10_imperative]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'training: accuracy=(\d+\.\d+|\d+)', 'validation: accuracy=(\d+\.\d+|\d+)', 'time cost: (\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'validation_acc', 'total_training_time']
compute_method = ['average', 'last', 'last', 'total']
command_to_execute = python image_classification/image_classification.py --model resnet50_v1 --dataset cifar10 --gpus 8 --epochs 20 --log-interval 50 --kvstore device
num_gpus = 8

[dawnbench_cifar10_symbolic]
patterns = ['Epoch \d+, Batch \d+, Speed=(\d+\.\d+|\d+)', 'Epoch \d+, Training accuracy=(\d+\.\d+|\d+)', 'Epoch \d+, Validation accuracy=(\d+\.\d+|\d+)', 'Epoch \d+, Duration=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'validation_acc', 'total_training_time']
compute_method = ['average', 'last', 'last', 'total']
command_to_execute = python dawnbench/cifar10.py --gpus 4 --early-stopping-acc 0.86 --epochs 400 --lr 0.05 --total-batch-size 256
num_gpus = 4

[lstm_ptb_imperative]
patterns = ['time cost (\d+\.\d+|\d+)', 'valid loss (\d+\.\d+|\d+)', 'valid ppl (\d+\.\d+|\d+)', 'test loss (\d+\.\d+|\d+)', 'test ppl (\d+\.\d+|\d+)']
metrics = ['total_training_time', 'validation_loss', 'validation_perplexity', 'test_loss', 'test_perplexity']
compute_method = ['total', 'last', 'last', 'last', 'last']
command_to_execute = python word_language_model/word_language_model.py --gpus 8 --nhid 650 --emsize 650 --dropout 0.5 --epochs 40 --data word_language_model/data/ptb. --mode imperative --kvstore device
num_gpus = 8

[lstm_ptb_hybrid]
patterns = ['time cost (\d+\.\d+|\d+)', 'valid loss (\d+\.\d+|\d+)', 'valid ppl (\d+\.\d+|\d+)', 'test loss (\d+\.\d+|\d+)', 'test ppl (\d+\.\d+|\d+)']
metrics = ['total_training_time', 'validation_loss', 'validation_perplexity', 'test_loss', 'test_perplexity']
compute_method = ['total', 'last', 'last', 'last', 'last']
command_to_execute = python word_language_model/word_language_model.py --gpus 8 --nhid 650 --emsize 650 --dropout 0.5 --epochs 40 --data word_language_model/data/ptb. --mode hybrid --kvstore device
num_gpus = 8

[lstm_ptb_symbolic]
patterns = ['Time cost=(\d+\.\d+|\d+)', 'Train-perplexity=(\d+\.\d+|\d+)', 'Validation-perplexity=(\d+\.\d+|\d+)', 'Speed: (\d+\.\d+|\d+) samples/sec']
metrics = ['total_training_time', 'train_perplexity', 'validation_perplexity', 'speed']
compute_method = ['total', 'last', 'last', 'average']
command_to_execute = python word_language_model/lstm_bucketing.py --num-hidden 650 --num-embed 650 --gpus 8 --epochs 25 --kv-store device
num_gpus = 8

[bidaf_1gpu_float32]
patterns = ['Time per epoch (\d+\.\d+|\d+)', 'train loss (\d+\.\d+|\d+)', 'exact_match\': (\d+\.\d+)', 'f1\': (\d+\.\d+)']
metrics = ['total_training_time', 'train_loss', 'exact_match', 'F1']
compute_method = ['total', 'last', 'last', 'last']
command_to_execute = bash question_answering/scripts/run_test_1gpu.sh
num_gpus = 1

[bidaf_multigpu_float32]
patterns = ['Time per epoch (\d+\.\d+|\d+)', 'train loss (\d+\.\d+|\d+)', 'exact_match\': (\d+\.\d+)', 'f1\': (\d+\.\d+)']
metrics = ['total_training_time', 'train_loss', 'exact_match', 'F1']
compute_method = ['total', 'last', 'last', 'last']
command_to_execute = bash question_answering/scripts/run_test_multigpu.sh
num_gpus = 4

[mkl_lstm_ptb_symbolic]
patterns = ['Time cost=(\d+\.\d+|\d+)', 'Train-perplexity=(\d+\.\d+|\d+)', 'Validation-perplexity=(\d+\.\d+|\d+)', 'Speed: (\d+\.\d+|\d+) samples/sec']
metrics = ['total_training_time', 'train_perplexity', 'validation_perplexity', 'speed']
compute_method = ['total', 'last', 'last', 'average']
command_to_execute = python word_language_model/lstm_bucketing.py --num-hidden 650 --num-embed 650 --gpus 0 --epochs 30 --kv-store local
num_gpus = 0

[mkl_lstm_ptb_hybrid]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'time cost (\d+\.\d+|\d+)', 'valid loss (\d+\.\d+|\d+)', 'valid ppl (\d+\.\d+|\d+)', 'train loss (\d+\.\d+|\d+)', 'train ppl (\d+\.\d+|\d+)']
metrics = ['speed', 'total_training_time', 'validation_loss', 'validation_perplexity', 'train_loss', 'train_perplexity']
compute_method = ['average', 'total', 'last', 'last', 'last', 'last']
command_to_execute = python word_language_model/word_language_model_train.py --gpus 0 --nhid 650 --emsize 650 --dropout 0.5 --epochs 20 --data word_language_model/data/ptb. --mode hybrid --kvstore local
num_gpus = 0

[mkl_lstm_ptb_imperative]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'time cost (\d+\.\d+|\d+)', 'valid loss (\d+\.\d+|\d+)', 'valid ppl (\d+\.\d+|\d+)', 'train loss (\d+\.\d+|\d+)', 'train ppl (\d+\.\d+|\d+)']
metrics = ['speed', 'total_training_time', 'validation_loss', 'validation_perplexity', 'train_loss', 'train_perplexity']
compute_method = ['average', 'total', 'last', 'last', 'last', 'last']
command_to_execute = python word_language_model/word_language_model_train.py --gpus 0 --nhid 650 --emsize 650 --dropout 0.5 --epochs 20 --data word_language_model/data/ptb. --mode imperative --kvstore local
num_gpus = 0

[mkl_lstm_ptb_inference]
patterns = ['Infer time is (\d+\.\d+\d+)']
metrics = ['infer_time_per_batch']
compute_method = ['average']
command_to_execute = python word_language_model/lstm_bucketing_infer.py --gpus 0
num_gpus = 0

[resnet50_imagenet_symbolic_fp16_batch_size32_p3_16]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python /home/ubuntu/mxnet/example/image-classification/train_imagenet.py --data-train /home/ubuntu/imagenet/imagenet1k-train.rec --data-val /home/ubuntu/imagenet/imagenet1k-val.rec --gpus 1,0,2,3,4,5,6,7 --batch-size 256 --data-nthreads 15 --num-epochs 80 --dtype float16
num_gpus = 1

[resnet50_imagenet_symbolic_fp16_batch_size64_p3_16]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python /home/ubuntu/mxnet/example/image-classification/train_imagenet.py --data-train /home/ubuntu/imagenet/imagenet1k-train.rec --data-val /home/ubuntu/imagenet/imagenet1k-val.rec --gpus 1,0,2,3,4,5,6,7 --batch-size 512 --data-nthreads 15 --num-epochs 80 --dtype float16
num_gpus = 1

[resnet50_imagenet_symbolic_fp16_batch_size128_p3_16]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python /home/ubuntu/mxnet/example/image-classification/train_imagenet.py --data-train /home/ubuntu/imagenet/imagenet1k-train.rec --data-val /home/ubuntu/imagenet/imagenet1k-val.rec --gpus 1,0,2,3,4,5,6,7 --batch-size 1024 --data-nthreads 32 --num-epochs 80 --dtype float16
num_gpus = 1

[resnet50_imagenet_symbolic_fp32_batch_size32_p3_16]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python /home/ubuntu/mxnet/example/image-classification/train_imagenet.py --data-train /home/ubuntu/imagenet/imagenet1k-train.rec --data-val /home/ubuntu/imagenet/imagenet1k-val.rec --gpus 1,0,2,3,4,5,6,7 --batch-size 256 --data-nthreads 15 --num-epochs 80
num_gpus = 1

[resnet50_imagenet_symbolic_fp32_batch_size64_p3_16]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python /home/ubuntu/mxnet/example/image-classification/train_imagenet.py --data-train /home/ubuntu/imagenet/imagenet1k-train.rec --data-val /home/ubuntu/imagenet/imagenet1k-val.rec --gpus 1,0,2,3,4,5,6,7 --batch-size 512 --data-nthreads 15 --num-epochs 80
num_gpus = 1

[resnet50_imagenet_symbolic_fp16_batch_size32_p3_8]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python /home/ubuntu/mxnet/example/image-classification/train_imagenet.py --data-train /home/ubuntu/imagenet/imagenet1k-train.rec --data-val /home/ubuntu/imagenet/imagenet1k-val.rec --gpus 1,0,2,3 --batch-size 256 --data-nthreads 15 --num-epochs 80 --dtype float16
num_gpus = 1

[resnet50_imagenet_symbolic_fp16_batch_size64_p3_8]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python /home/ubuntu/mxnet/example/image-classification/train_imagenet.py --data-train /home/ubuntu/imagenet/imagenet1k-train.rec --data-val /home/ubuntu/imagenet/imagenet1k-val.rec --gpus 1,0,2,3 --batch-size 512 --data-nthreads 15 --num-epochs 80 --dtype float16
num_gpus = 1

[resnet50_imagenet_symbolic_fp32_batch_size32_p3_8]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python /home/ubuntu/mxnet/example/image-classification/train_imagenet.py --data-train /home/ubuntu/imagenet/imagenet1k-train.rec --data-val /home/ubuntu/imagenet/imagenet1k-val.rec --gpus 1,0,2,3 --batch-size 256 --data-nthreads 15 --num-epochs 80
num_gpus = 1

[resnet50_imagenet_symbolic_fp32_batch_size64_p3_8]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total','last']
command_to_execute = python /home/ubuntu/mxnet/example/image-classification/train_imagenet.py --data-train /home/ubuntu/imagenet/imagenet1k-train.rec --data-val /home/ubuntu/imagenet/imagenet1k-val.rec --gpus 1,0,2,3 --batch-size 512 --data-nthreads 15 --num-epochs 80
num_gpus = 1

[metrics_parameters_distributed_top_k]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Train-top_k_accuracy_\d=(\d+\.\d+|\d+)','Validation-top_k_accuracy_\d=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'validation_acc', 'total_training_time','training_acc_top5','validation_acc_top5']
compute_method = ['average_aggregate', 'last', 'last', 'total','last','last']

[metrics_parameters_distributed]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)']
metrics = ['speed', 'training_acc', 'validation_acc', 'total_training_time']
compute_method = ['average_aggregate', 'last', 'last', 'total']

[tensorflow_resnet50_p3_2xlg]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Time']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=imagenet --num_gpus=1 --batch_size=32 --model=resnet50 --variable_update=parameter_server --print_training_accuracy=True --num_batches=100
num_gpus = 1

[tensorflow_resnet50_p3_8xlg]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Time']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=imagenet --num_gpus=4 --batch_size=32 --model=resnet50 --variable_update=parameter_server --print_training_accuracy=True --num_batches=100
num_gpus = 4

[tensorflow_resnet50_p3_16xlg]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Time']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=imagenet --num_gpus=8 --batch_size=32 --model=resnet50 --variable_update=parameter_server --print_training_accuracy=True --num_batches=100
num_gpus = 8

[tensorflow_resnet152_p3_2xlg]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Training']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=imagenet --num_gpus=1 --batch_size=32 --model=resnet152 --variable_update=parameter_server --print_training_accuracy=True --num_batches=100
num_gpus = 1

[tensorflow_resnet152_p3_8xlg]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Training']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=imagenet --num_gpus=4 --batch_size=32 --model=resnet152 --variable_update=parameter_server --print_training_accuracy=True --num_batches=100
num_gpus = 4

[tensorflow_resnet152_p3_16xlg]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Training time']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=imagenet --num_gpus=8 --batch_size=32 --model=resnet152 --variable_update=parameter_server --print_training_accuracy=True --num_batches=100
num_gpus = 8

[tensorflow_resnet56_p3_2xlg_fp16]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Training time']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=cifar10 --data_dir=cifar-10-batches-py --num_gpus=1 --batch_size=32 --model=resnet56 --variable_update=replicated --print_training_accuracy=True --use_fp16=True --use_tf_layers=False --target_accuracy=0.94
num_gpus = 1

[tensorflow_resnet56_p3_8xlg_fp16]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Training time']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=cifar10 --data_dir=cifar-10-batches-py --num_gpus=4 --batch_size=32 --model=resnet56 --variable_update=replicated --print_training_accuracy=True --use_fp16=True --use_tf_layers=False --target_accuracy=0.94
num_gpus = 4

[tensorflow_resnet56_p3_16xlg_fp16]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Training time']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=cifar10 --data_dir=cifar-10-batches-py --num_gpus=8 --batch_size=32 --model=resnet56 --variable_update=replicated --print_training_accuracy=True --use_fp16=True --use_tf_layers=False --target_accuracy=0.94
num_gpus = 8

[tensorflow_resnet50_p3_2xlg_fp16]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Time']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=imagenet --num_gpus=1 --batch_size=32 --model=resnet50 --variable_update=replicated --print_training_accuracy=True --num_batches=100 --use_fp16=True --use_tf_layers=False
num_gpus = 1

[tensorflow_resnet50_p3_8xlg_fp16]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Time']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=imagenet --num_gpus=4 --batch_size=32 --model=resnet50 --variable_update=replicated --print_training_accuracy=True --num_batches=100 --use_fp16=True --use_tf_layers=False
num_gpus = 4

[tensorflow_resnet50_p3_16xlg_fp16]
patterns = ['images/sec: (\d+\.\d+)', 'time: (\d+\.\d+)']
metrics = ['Images per sec', 'Time']
compute_method = ['average', 'last']
command_to_execute = python tensorflow_benchmark/tf_cnn_benchmarks/tf_cnn_benchmarks.py --data_name=imagenet --num_gpus=8 --batch_size=32 --model=resnet50 --variable_update=replicated --print_training_accuracy=True --num_batches=100 --use_fp16=True --use_tf_layers=False
num_gpus = 8

[onnx_mxnet_import_model_inference_test_cpu]
patterns = ['Average_inference_time_bvlc_alexnet_cpu: (\d+\.\d+|\d+)', 'Average_inference_time_bvlc_googlenet_cpu: (\d+\.\d+|\d+)', 'Average_inference_time_bvlc_reference_caffenet_cpu: (\d+\.\d+|\d+)', 'Average_inference_time_bvlc_reference_rcnn_ilsvrc13_cpu: (\d+\.\d+|\d+)', 'Average_inference_time_densenet121_cpu: (\d+\.\d+|\d+)', 'Average_inference_time_resnet50_cpu: (\d+\.\d+|\d+)', 'Average_inference_time_shufflenet_cpu: (\d+\.\d+|\d+)', 'Average_inference_time_squeezenet_cpu: (\d+\.\d+|\d+)', 'Average_inference_time_vgg19_cpu: (\d+\.\d+|\d+)', 'P50_inference_time_bvlc_alexnet_cpu: (\d+\.\d+|\d+)', 'P50_inference_time_bvlc_googlenet_cpu: (\d+\.\d+|\d+)', 'P50_inference_time_bvlc_reference_caffenet_cpu: (\d+\.\d+|\d+)', 'P50_inference_time_bvlc_reference_rcnn_ilsvrc13_cpu: (\d+\.\d+|\d+)', 'P50_inference_time_densenet121_cpu: (\d+\.\d+|\d+)', 'P50_inference_time_resnet50_cpu: (\d+\.\d+|\d+)', 'P50_inference_time_shufflenet_cpu: (\d+\.\d+|\d+)', 'P50_inference_time_squeezenet_cpu: (\d+\.\d+|\d+)', 'P50_inference_time_vgg19_cpu: (\d+\.\d+|\d+)', 'P90_inference_time_bvlc_alexnet_cpu: (\d+\.\d+|\d+)', 'P90_inference_time_bvlc_googlenet_cpu: (\d+\.\d+|\d+)', 'P90_inference_time_bvlc_reference_caffenet_cpu: (\d+\.\d+|\d+)', 'P90_inference_time_bvlc_reference_rcnn_ilsvrc13_cpu: (\d+\.\d+|\d+)', 'P90_inference_time_densenet121_cpu: (\d+\.\d+|\d+)', 'P90_inference_time_resnet50_cpu: (\d+\.\d+|\d+)', 'P90_inference_time_shufflenet_cpu: (\d+\.\d+|\d+)', 'P90_inference_time_squeezenet_cpu: (\d+\.\d+|\d+)', 'P90_inference_time_vgg19_cpu: (\d+\.\d+|\d+)', 'P99_inference_time_bvlc_alexnet_cpu: (\d+\.\d+|\d+)', 'P99_inference_time_bvlc_googlenet_cpu: (\d+\.\d+|\d+)', 'P99_inference_time_bvlc_reference_caffenet_cpu: (\d+\.\d+|\d+)', 'P99_inference_time_bvlc_reference_rcnn_ilsvrc13_cpu: (\d+\.\d+|\d+)', 'P99_inference_time_densenet121_cpu: (\d+\.\d+|\d+)', 'P99_inference_time_resnet50_cpu: (\d+\.\d+|\d+)', 'P99_inference_time_shufflenet_cpu: (\d+\.\d+|\d+)', 'P99_inference_time_squeezenet_cpu: (\d+\.\d+|\d+)', 'P99_inference_time_vgg19_cpu: (\d+\.\d+|\d+)']
metrics = ['Average_inference_time_bvlc_alexnet_cpu','Average_inference_time_bvlc_googlenet_cpu','Average_inference_time_bvlc_reference_caffenet_cpu','Average_inference_time_bvlc_reference_rcnn_ilsvrc13_cpu','Average_inference_time_densenet121_cpu','Average_inference_time_resnet50_cpu','Average_inference_time_shufflenet_cpu','Average_inference_time_squeezenet_cpu','Average_inference_time_vgg19_cpu', 'P50_inference_time_bvlc_alexnet_cpu','P50_inference_time_bvlc_googlenet_cpu','P50_inference_time_bvlc_reference_caffenet_cpu','P50_inference_time_bvlc_reference_rcnn_ilsvrc13_cpu','P50_inference_time_densenet121_cpu','P50_inference_time_resnet50_cpu','P50_inference_time_shufflenet_cpu','P50_inference_time_squeezenet_cpu','P50_inference_time_vgg19_cpu','P90_inference_time_bvlc_alexnet_cpu','P90_inference_time_bvlc_googlenet_cpu','P90_inference_time_bvlc_reference_caffenet_cpu','P90_inference_time_bvlc_reference_rcnn_ilsvrc13_cpu','P90_inference_time_densenet121_cpu','P90_inference_time_resnet50_cpu','P90_inference_time_shufflenet_cpu','P90_inference_time_squeezenet_cpu','P90_inference_time_vgg19_cpu', 'P99_inference_time_bvlc_alexnet_cpu','P99_inference_time_bvlc_googlenet_cpu','P99_inference_time_bvlc_reference_caffenet_cpu','P99_inference_time_bvlc_reference_rcnn_ilsvrc13_cpu','P99_inference_time_densenet121_cpu','P99_inference_time_resnet50_cpu','P99_inference_time_shufflenet_cpu','P99_inference_time_squeezenet_cpu','P99_inference_time_vgg19_cpu']
compute_method = ['last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last']
command_to_execute = sudo bash ./onnx_benchmark/setup.sh "cpu" && python3  ./onnx_benchmark/import_benchmarkscript.py "cpu"
num_gpus = 0

[onnx_mxnet_import_model_inference_test_gpu]
patterns = ['Average_inference_time_bvlc_alexnet_gpu: (\d+\.\d+|\d+)', 'Average_inference_time_bvlc_googlenet_gpu: (\d+\.\d+|\d+)', 'Average_inference_time_bvlc_reference_caffenet_gpu: (\d+\.\d+|\d+)', 'Average_inference_time_bvlc_reference_rcnn_ilsvrc13_gpu: (\d+\.\d+|\d+)', 'Average_inference_time_densenet121_gpu: (\d+\.\d+|\d+)', 'Average_inference_time_resnet50_gpu: (\d+\.\d+|\d+)', 'Average_inference_time_shufflenet_gpu: (\d+\.\d+|\d+)', 'Average_inference_time_squeezenet_gpu: (\d+\.\d+|\d+)', 'Average_inference_time_vgg19_gpu: (\d+\.\d+|\d+)', 'P50_inference_time_bvlc_alexnet_gpu: (\d+\.\d+|\d+)', 'P50_inference_time_bvlc_googlenet_gpu: (\d+\.\d+|\d+)', 'P50_inference_time_bvlc_reference_caffenet_gpu: (\d+\.\d+|\d+)', 'P50_inference_time_bvlc_reference_rcnn_ilsvrc13_gpu: (\d+\.\d+|\d+)', 'P50_inference_time_densenet121_gpu: (\d+\.\d+|\d+)', 'P50_inference_time_resnet50_gpu: (\d+\.\d+|\d+)', 'P50_inference_time_shufflenet_gpu: (\d+\.\d+|\d+)', 'P50_inference_time_squeezenet_gpu: (\d+\.\d+|\d+)', 'P50_inference_time_vgg19_gpu: (\d+\.\d+|\d+)', 'P90_inference_time_bvlc_alexnet_gpu: (\d+\.\d+|\d+)', 'P90_inference_time_bvlc_googlenet_gpu: (\d+\.\d+|\d+)', 'P90_inference_time_bvlc_reference_caffenet_gpu: (\d+\.\d+|\d+)', 'P90_inference_time_bvlc_reference_rcnn_ilsvrc13_gpu: (\d+\.\d+|\d+)', 'P90_inference_time_densenet121_gpu: (\d+\.\d+|\d+)', 'P90_inference_time_resnet50_gpu: (\d+\.\d+|\d+)', 'P90_inference_time_shufflenet_gpu: (\d+\.\d+|\d+)', 'P90_inference_time_squeezenet_gpu: (\d+\.\d+|\d+)', 'P90_inference_time_vgg19_gpu: (\d+\.\d+|\d+)', 'P99_inference_time_bvlc_alexnet_gpu: (\d+\.\d+|\d+)', 'P99_inference_time_bvlc_googlenet_gpu: (\d+\.\d+|\d+)', 'P99_inference_time_bvlc_reference_caffenet_gpu: (\d+\.\d+|\d+)', 'P99_inference_time_bvlc_reference_rcnn_ilsvrc13_gpu: (\d+\.\d+|\d+)', 'P99_inference_time_densenet121_gpu: (\d+\.\d+|\d+)', 'P99_inference_time_resnet50_gpu: (\d+\.\d+|\d+)', 'P99_inference_time_shufflenet_gpu: (\d+\.\d+|\d+)', 'P99_inference_time_squeezenet_gpu: (\d+\.\d+|\d+)', 'P99_inference_time_vgg19_gpu: (\d+\.\d+|\d+)']
metrics = ['Average_inference_time_bvlc_alexnet_gpu','Average_inference_time_bvlc_googlenet_gpu','Average_inference_time_bvlc_reference_caffenet_gpu','Average_inference_time_bvlc_reference_rcnn_ilsvrc13_gpu','Average_inference_time_densenet121_gpu','Average_inference_time_resnet50_gpu','Average_inference_time_shufflenet_gpu','Average_inference_time_squeezenet_gpu','Average_inference_time_vgg19_gpu', 'P50_inference_time_bvlc_alexnet_gpu','P50_inference_time_bvlc_googlenet_gpu','P50_inference_time_bvlc_reference_caffenet_gpu','P50_inference_time_bvlc_reference_rcnn_ilsvrc13_gpu','P50_inference_time_densenet121_gpu','P50_inference_time_resnet50_gpu','P50_inference_time_shufflenet_gpu','P50_inference_time_squeezenet_gpu','P50_inference_time_vgg19_gpu', 'P90_inference_time_bvlc_alexnet_gpu','P90_inference_time_bvlc_googlenet_gpu','P90_inference_time_bvlc_reference_caffenet_gpu','P90_inference_time_bvlc_reference_rcnn_ilsvrc13_gpu','P90_inference_time_densenet121_gpu','P90_inference_time_resnet50_gpu','P90_inference_time_shufflenet_gpu','P90_inference_time_squeezenet_gpu','P90_inference_time_vgg19_gpu', 'P99_inference_time_bvlc_alexnet_gpu','P99_inference_time_bvlc_googlenet_gpu','P99_inference_time_bvlc_reference_caffenet_gpu','P99_inference_time_bvlc_reference_rcnn_ilsvrc13_gpu','P99_inference_time_densenet121_gpu','P99_inference_time_resnet50_gpu','P99_inference_time_shufflenet_gpu','P99_inference_time_squeezenet_gpu','P99_inference_time_vgg19_gpu']
compute_method = ['last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last']
command_to_execute = sudo bash ./onnx_benchmark/setup.sh "gpu"  && python3  ./onnx_benchmark/import_benchmarkscript.py "gpu"
num_gpus = 1

[test_resnet50_imagenet-480px-256px-q95_p3.16x_fp16_docker]
patterns = ['Speed: (\d+\.\d+|\d+) samples/sec', 'Train-accuracy=(\d+\.\d+|\d+)', 'Time cost=(\d+\.\d+|\d+)', 'Validation-accuracy=(\d+\.\d+|\d+)' ]
metrics = ['speed', 'training_acc', 'total_training_time', 'validation_acc']
compute_method = ['average', 'last', 'total', 'last']
command_to_execute = bash image_classification/scripts/test.sh
num_gpus = 8

[mms_resnet_18_cpu]
patterns = ["'latency_resnet-18_Inference_Request_Average': (\d+\.\d+|\d+)", "'latency_resnet-18_Inference_Request_Median': (\d+\.\d+|\d+)", "'latency_resnet-18_Inference_Request_Throughput': (\d+\.\d+|\d+)", "'latency_resnet-18_Inference_Request_aggregate_report_90_line': (\d+\.\d+|\d+)", "'latency_resnet-18_Inference_Request_aggregate_report_99_line': (\d+\.\d+|\d+)", "'latency_resnet-18_Inference_Request_aggregate_report_error': '(\d+\.\d+|\d+)\%'", "'throughput_resnet-18_Inference_Request_Average': (\d+\.\d+|\d+)", "'throughput_resnet-18_Inference_Request_Median': (\d+\.\d+|\d+)", "'throughput_resnet-18_Inference_Request_Throughput': (\d+\.\d+|\d+)", "'throughput_resnet-18_Inference_Request_aggregate_report_90_line': (\d+\.\d+|\d+)", "'throughput_resnet-18_Inference_Request_aggregate_report_99_line': (\d+\.\d+|\d+)", "'throughput_resnet-18_Inference_Request_aggregate_report_error': '(\d+\.\d+|\d+)\%'"]
metrics = ['latency_resnet-18_Inference_Request_Average', 'latency_resnet-18_Inference_Request_Median', 'latency_resnet-18_Inference_Request_Throughput', 'latency_resnet-18_Inference_Request_aggregate_report_90_line', 'latency_resnet-18_Inference_Request_aggregate_report_99_line', 'latency_resnet-18_Inference_Request_aggregate_report_error', 'throughput_resnet-18_Inference_Request_Average', 'throughput_resnet-18_Inference_Request_Median', 'throughput_resnet-18_Inference_Request_Throughput', 'throughput_resnet-18_Inference_Request_aggregate_report_90_line', 'throughput_resnet-18_Inference_Request_aggregate_report_99_line', 'throughput_resnet-18_Inference_Request_aggregate_report_error']
compute_method = ['last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last']
command_to_execute = python3 /mxnet-model-server/benchmarks/benchmark.py -l 100 -s --model resnet-18 -w 16 && sh /mxnet-model-server/benchmarks/upload_results_to_s3.sh False
num_gpus = 0

[mms_resnet_18_gpu]
patterns = ["'latency_resnet-18_Inference_Request_Average': (\d+\.\d+|\d+)", "'latency_resnet-18_Inference_Request_Median': (\d+\.\d+|\d+)", "'latency_resnet-18_Inference_Request_Throughput': (\d+\.\d+|\d+)", "'latency_resnet-18_Inference_Request_aggregate_report_90_line': (\d+\.\d+|\d+)", "'latency_resnet-18_Inference_Request_aggregate_report_99_line': (\d+\.\d+|\d+)", "'latency_resnet-18_Inference_Request_aggregate_report_error': '(\d+\.\d+|\d+)\%'", "'throughput_resnet-18_Inference_Request_Average': (\d+\.\d+|\d+)", "'throughput_resnet-18_Inference_Request_Median': (\d+\.\d+|\d+)", "'throughput_resnet-18_Inference_Request_Throughput': (\d+\.\d+|\d+)", "'throughput_resnet-18_Inference_Request_aggregate_report_90_line': (\d+\.\d+|\d+)", "'throughput_resnet-18_Inference_Request_aggregate_report_99_line': (\d+\.\d+|\d+)", "'throughput_resnet-18_Inference_Request_aggregate_report_error': '(\d+\.\d+|\d+)\%'"]
metrics = ['latency_resnet-18_Inference_Request_Average', 'latency_resnet-18_Inference_Request_Median', 'latency_resnet-18_Inference_Request_Throughput', 'latency_resnet-18_Inference_Request_aggregate_report_90_line', 'latency_resnet-18_Inference_Request_aggregate_report_99_line', 'latency_resnet-18_Inference_Request_aggregate_report_error', 'throughput_resnet-18_Inference_Request_Average', 'throughput_resnet-18_Inference_Request_Median', 'throughput_resnet-18_Inference_Request_Throughput', 'throughput_resnet-18_Inference_Request_aggregate_report_90_line', 'throughput_resnet-18_Inference_Request_aggregate_report_99_line', 'throughput_resnet-18_Inference_Request_aggregate_report_error']
compute_method = ['last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last']
command_to_execute = python3 /mxnet-model-server/benchmarks/benchmark.py -l 100 -g 8 -s --model resnet-18 -w 16 && sh /mxnet-model-server/benchmarks/upload_results_to_s3.sh True
num_gpus = 8

[mms_lstm_ptb_cpu]
patterns = ["'latency_lstm_ptb_Inference_Request_Average': (\d+\.\d+|\d+)", "'latency_lstm_ptb_Inference_Request_Median': (\d+\.\d+|\d+)", "'latency_lstm_ptb_Inference_Request_Throughput': (\d+\.\d+|\d+)", "'latency_lstm_ptb_Inference_Request_aggregate_report_90_line': (\d+\.\d+|\d+)", "'latency_lstm_ptb_Inference_Request_aggregate_report_99_line': (\d+\.\d+|\d+)", "'latency_lstm_ptb_Inference_Request_aggregate_report_error': '(\d+\.\d+|\d+)\%'", "'throughput_lstm_ptb_Inference_Request_Average': (\d+\.\d+|\d+)", "'throughput_lstm_ptb_Inference_Request_Median': (\d+\.\d+|\d+)", "'throughput_lstm_ptb_Inference_Request_Throughput': (\d+\.\d+|\d+)", "'throughput_lstm_ptb_Inference_Request_aggregate_report_90_line': (\d+\.\d+|\d+)", "'throughput_lstm_ptb_Inference_Request_aggregate_report_99_line': (\d+\.\d+|\d+)", "'throughput_lstm_ptb_Inference_Request_aggregate_report_error': '(\d+\.\d+|\d+)\%'"]
metrics = ['latency_lstm_ptb_Inference_Request_Average', 'latency_lstm_ptb_Inference_Request_Median', 'latency_lstm_ptb_Inference_Request_Throughput', 'latency_lstm_ptb_Inference_Request_aggregate_report_90_line', 'latency_lstm_ptb_Inference_Request_aggregate_report_99_line', 'latency_lstm_ptb_Inference_Request_aggregate_report_error', 'throughput_lstm_ptb_Inference_Request_Average', 'throughput_lstm_ptb_Inference_Request_Median', 'throughput_lstm_ptb_Inference_Request_Throughput', 'throughput_lstm_ptb_Inference_Request_aggregate_report_90_line', 'throughput_lstm_ptb_Inference_Request_aggregate_report_99_line', 'throughput_lstm_ptb_Inference_Request_aggregate_report_error']
compute_method = ['last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last']
command_to_execute = python3 /mxnet-model-server/benchmarks/benchmark.py -l 100 -s --model lstm_ptb -w 16 && sh /mxnet-model-server/benchmarks/upload_results_to_s3.sh False
num_gpus = 0

[mms_lstm_ptb_gpu]
patterns = ["'latency_lstm_ptb_Inference_Request_Average': (\d+\.\d+|\d+)", "'latency_lstm_ptb_Inference_Request_Median': (\d+\.\d+|\d+)", "'latency_lstm_ptb_Inference_Request_Throughput': (\d+\.\d+|\d+)", "'latency_lstm_ptb_Inference_Request_aggregate_report_90_line': (\d+\.\d+|\d+)", "'latency_lstm_ptb_Inference_Request_aggregate_report_99_line': (\d+\.\d+|\d+)", "'latency_lstm_ptb_Inference_Request_aggregate_report_error': '(\d+\.\d+|\d+)\%'", "'throughput_lstm_ptb_Inference_Request_Average': (\d+\.\d+|\d+)", "'throughput_lstm_ptb_Inference_Request_Median': (\d+\.\d+|\d+)", "'throughput_lstm_ptb_Inference_Request_Throughput': (\d+\.\d+|\d+)", "'throughput_lstm_ptb_Inference_Request_aggregate_report_90_line': (\d+\.\d+|\d+)", "'throughput_lstm_ptb_Inference_Request_aggregate_report_99_line': (\d+\.\d+|\d+)", "'throughput_lstm_ptb_Inference_Request_aggregate_report_error': '(\d+\.\d+|\d+)\%'"]
metrics = ['latency_lstm_ptb_Inference_Request_Average', 'latency_lstm_ptb_Inference_Request_Median', 'latency_lstm_ptb_Inference_Request_Throughput', 'latency_lstm_ptb_Inference_Request_aggregate_report_90_line', 'latency_lstm_ptb_Inference_Request_aggregate_report_99_line', 'latency_lstm_ptb_Inference_Request_aggregate_report_error', 'throughput_lstm_ptb_Inference_Request_Average', 'throughput_lstm_ptb_Inference_Request_Median', 'throughput_lstm_ptb_Inference_Request_Throughput', 'throughput_lstm_ptb_Inference_Request_aggregate_report_90_line', 'throughput_lstm_ptb_Inference_Request_aggregate_report_99_line', 'throughput_lstm_ptb_Inference_Request_aggregate_report_error']
compute_method = ['last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last']
command_to_execute = python3 /mxnet-model-server/benchmarks/benchmark.py -l 100 -g 8 -s --model lstm_ptb -w 16 && sh /mxnet-model-server/benchmarks/upload_results_to_s3.sh True
num_gpus = 8

[mms_noop_cpu]
patterns = ["'latency_noop-v1.0_Inference_Request_Average': (\d+\.\d+|\d+)", "'latency_noop-v1.0_Inference_Request_Median': (\d+\.\d+|\d+)", "'latency_noop-v1.0_Inference_Request_Throughput': (\d+\.\d+|\d+)", "'latency_noop-v1.0_Inference_Request_aggregate_report_90_line': (\d+\.\d+|\d+)", "'latency_noop-v1.0_Inference_Request_aggregate_report_99_line': (\d+\.\d+|\d+)", "'latency_noop-v1.0_Inference_Request_aggregate_report_error': '(\d+\.\d+|\d+)\%'", "'throughput_noop-v1.0_Inference_Request_Average': (\d+\.\d+|\d+)", "'throughput_noop-v1.0_Inference_Request_Median': (\d+\.\d+|\d+)", "'throughput_noop-v1.0_Inference_Request_Throughput': (\d+\.\d+|\d+)", "'throughput_noop-v1.0_Inference_Request_aggregate_report_90_line': (\d+\.\d+|\d+)", "'throughput_noop-v1.0_Inference_Request_aggregate_report_99_line': (\d+\.\d+|\d+)", "'throughput_noop-v1.0_Inference_Request_aggregate_report_error': '(\d+\.\d+|\d+)\%'"]
metrics = ['latency_noop_Inference_Request_Average', 'latency_noop_Inference_Request_Median', 'latency_noop_Inference_Request_Throughput', 'latency_noop-v1.0_Inference_Request_aggregate_report_90_line', 'latency_noop_Inference_Request_aggregate_report_99_line', 'latency_noop_Inference_Request_aggregate_report_error', 'throughput_noop_Inference_Request_Average', 'throughput_noop_Inference_Request_Median', 'throughput_noop_Inference_Request_Throughput', 'throughput_noop_Inference_Request_aggregate_report_90_line', 'throughput_noop_Inference_Request_aggregate_report_99_line', 'throughput_noop_Inference_Request_aggregate_report_error']
compute_method = ['last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last']
command_to_execute = python3 /mxnet-model-server/benchmarks/benchmark.py -l 100 -s --model noop-v1.0 -w 16 && sh /mxnet-model-server/benchmarks/upload_results_to_s3.sh False
num_gpus = 0

[mms_noop_gpu]
patterns = ["'latency_noop-v1.0_Inference_Request_Average': (\d+\.\d+|\d+)", "'latency_noop-v1.0_Inference_Request_Median': (\d+\.\d+|\d+)", "'latency_noop-v1.0_Inference_Request_Throughput': (\d+\.\d+|\d+)", "'latency_noop-v1.0_Inference_Request_aggregate_report_90_line': (\d+\.\d+|\d+)", "'latency_noop-v1.0_Inference_Request_aggregate_report_99_line': (\d+\.\d+|\d+)", "'latency_noop-v1.0_Inference_Request_aggregate_report_error': '(\d+\.\d+|\d+)\%'", "'throughput_noop-v1.0_Inference_Request_Average': (\d+\.\d+|\d+)", "'throughput_noop-v1.0_Inference_Request_Median': (\d+\.\d+|\d+)", "'throughput_noop-v1.0_Inference_Request_Throughput': (\d+\.\d+|\d+)", "'throughput_noop-v1.0_Inference_Request_aggregate_report_90_line': (\d+\.\d+|\d+)", "'throughput_noop-v1.0_Inference_Request_aggregate_report_99_line': (\d+\.\d+|\d+)", "'throughput_noop-v1.0_Inference_Request_aggregate_report_error': '(\d+\.\d+|\d+)\%'"]
metrics = ['latency_noop_Inference_Request_Average', 'latency_noop_Inference_Request_Median', 'latency_noop_Inference_Request_Throughput', 'latency_noop-v1.0_Inference_Request_aggregate_report_90_line', 'latency_noop_Inference_Request_aggregate_report_99_line', 'latency_noop_Inference_Request_aggregate_report_error', 'throughput_noop_Inference_Request_Average', 'throughput_noop_Inference_Request_Median', 'throughput_noop_Inference_Request_Throughput', 'throughput_noop_Inference_Request_aggregate_report_90_line', 'throughput_noop_Inference_Request_aggregate_report_99_line', 'throughput_noop_Inference_Request_aggregate_report_error']
compute_method = ['last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last']
command_to_execute = python3 /mxnet-model-server/benchmarks/benchmark.py -l 100 -g 8 -s --model noop-v1.0 -w 16 && sh /mxnet-model-server/benchmarks/upload_results_to_s3.sh True
num_gpus = 8

[mms_metric]
patterns = ["Model latency P50: (\d+\.?\d*)", "Model latency P90: (\d+\.?\d*)", "Model latency P99: (\d+\.?\d*)", "MMS throughput: (\d+\.?\d*)", "MMS latency P50: (\d+\.?\d*)", "MMS latency P90: (\d+\.?\d*)", "MMS latency P99: (\d+\.?\d*)", "MMS latency mean: (\d+\.?\d*)", "MMS error rate: (\d+\.?\d*)"]
metrics = ['model_latency_p50', 'model_latency_p90', 'model_latency_p99', 'mms_throughput', 'mms_latency_p50', 'mms_latency_p90', 'mms_latency_p99', 'mms_latency_mean', 'mms_latency_error']
compute_method = ['last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last']

[inference_policy]
patterns = ['single_inference_p99 (\d+\.\d+|\d+)', 'single_inference_p90 (\d+\.\d+|\d+)', 'single_inference_p50 (\d+\.\d+|\d+)', 'single_inference_average (\d+\.\d+|\d+)', 'batch_inference_4x_p99 (\d+\.\d+|\d+)', 'batch_inference_4x_p90 (\d+\.\d+|\d+)', 'batch_inference_4x_p50 (\d+\.\d+|\d+)', 'batch_inference_4x_average (\d+\.\d+|\d+)', 'batch_inference_2x_p99 (\d+\.\d+|\d+)', 'batch_inference_2x_p90 (\d+\.\d+|\d+)', 'batch_inference_2x_p50 (\d+\.\d+|\d+)', 'batch_inference_2x_average (\d+\.\d+|\d+)', 'batch_inference_1x_p99 (\d+\.\d+|\d+)', 'batch_inference_1x_p90 (\d+\.\d+|\d+)', 'batch_inference_1x_p50 (\d+\.\d+|\d+)', 'batch_inference_1x_average (\d+\.\d+|\d+)']
metrics = ['single_inference_p99', 'single_inference_p90', 'single_inference_p50', 'single_inference_average', 'batch_inference_4x_p99', 'batch_inference_4x_p90', 'batch_inference_4x_p50', 'batch_inference_4x_average', 'batch_inference_2x_p99', 'batch_inference_2x_p90', 'batch_inference_2x_p50', 'batch_inference_2x_average', 'batch_inference_1x_p99', 'batch_inference_1x_p90', 'batch_inference_1x_p50', 'batch_inference_1x_average']
compute_method = ['last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last']

[scala_inference_ssd_cpu]
patterns = ['single_inference_p99 (\d+\.\d+|\d+)', 'single_inference_p90 (\d+\.\d+|\d+)', 'single_inference_p50 (\d+\.\d+|\d+)', 'single_inference_average (\d+\.\d+|\d+)', 'batch_inference_4x_p99 (\d+\.\d+|\d+)', 'batch_inference_4x_p90 (\d+\.\d+|\d+)', 'batch_inference_4x_p50 (\d+\.\d+|\d+)', 'batch_inference_4x_average (\d+\.\d+|\d+)', 'batch_inference_2x_p99 (\d+\.\d+|\d+)', 'batch_inference_2x_p90 (\d+\.\d+|\d+)', 'batch_inference_2x_p50 (\d+\.\d+|\d+)', 'batch_inference_2x_average (\d+\.\d+|\d+)', 'batch_inference_1x_p99 (\d+\.\d+|\d+)', 'batch_inference_1x_p90 (\d+\.\d+|\d+)', 'batch_inference_1x_p50 (\d+\.\d+|\d+)', 'batch_inference_1x_average (\d+\.\d+|\d+)']
metrics = ['single_inference_p99', 'single_inference_p90', 'single_inference_p50', 'single_inference_average', 'batch_inference_4x_p99', 'batch_inference_4x_p90', 'batch_inference_4x_p50', 'batch_inference_4x_average', 'batch_inference_2x_p99', 'batch_inference_2x_p90', 'batch_inference_2x_p50', 'batch_inference_2x_average', 'batch_inference_1x_p99', 'batch_inference_1x_p90', 'batch_inference_1x_p50', 'batch_inference_1x_average']
compute_method = ['last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last']
command_to_execute = cd $HOME/benchmarkai/scala-mxnet/scala-bm/ && bash bin/get_resnet50_ssd_data.sh && bash bin/run_ssd.sh cpu /tmp/resnet50_ssd/resnet50_ssd_model /tmp/resnet50_ssd/images/dog.jpg 4 2000
num_gpus = 0

[scala_inference_ssd_gpu]
patterns = ['single_inference_p99 (\d+\.\d+|\d+)', 'single_inference_p90 (\d+\.\d+|\d+)', 'single_inference_p50 (\d+\.\d+|\d+)', 'single_inference_average (\d+\.\d+|\d+)', 'batch_inference_4x_p99 (\d+\.\d+|\d+)', 'batch_inference_4x_p90 (\d+\.\d+|\d+)', 'batch_inference_4x_p50 (\d+\.\d+|\d+)', 'batch_inference_4x_average (\d+\.\d+|\d+)', 'batch_inference_2x_p99 (\d+\.\d+|\d+)', 'batch_inference_2x_p90 (\d+\.\d+|\d+)', 'batch_inference_2x_p50 (\d+\.\d+|\d+)', 'batch_inference_2x_average (\d+\.\d+|\d+)', 'batch_inference_1x_p99 (\d+\.\d+|\d+)', 'batch_inference_1x_p90 (\d+\.\d+|\d+)', 'batch_inference_1x_p50 (\d+\.\d+|\d+)', 'batch_inference_1x_average (\d+\.\d+|\d+)']
metrics = ['single_inference_p99', 'single_inference_p90', 'single_inference_p50', 'single_inference_average', 'batch_inference_4x_p99', 'batch_inference_4x_p90', 'batch_inference_4x_p50', 'batch_inference_4x_average', 'batch_inference_2x_p99', 'batch_inference_2x_p90', 'batch_inference_2x_p50', 'batch_inference_2x_average', 'batch_inference_1x_p99', 'batch_inference_1x_p90', 'batch_inference_1x_p50', 'batch_inference_1x_average']
compute_method = ['last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last']
command_to_execute = cd $HOME/benchmarkai/scala-mxnet/scala-bm/ && bash bin/get_resnet50_ssd_data.sh && export SCALA_TEST_ON_GPU=1 && bash bin/run_ssd.sh gpu /tmp/resnet50_ssd/resnet50_ssd_model /tmp/resnet50_ssd/images/dog.jpg 4 2000
num_gpus = 0

[scala_inference_charrnn_cpu]
patterns = ['single_inference_p99 (\d+\.\d+|\d+)', 'single_inference_p90 (\d+\.\d+|\d+)', 'single_inference_p50 (\d+\.\d+|\d+)', 'single_inference_average (\d+\.\d+|\d+)']
metrics = ['single_inference_p99', 'single_inference_p90', 'single_inference_p50', 'single_inference_average']
compute_method = ['last', 'last', 'last', 'last']
command_to_execute = cd /tmp/ && wget https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci/RNN/obama.zip && unzip obama.zip && cd $HOME/benchmarkai/scala-mxnet/scala-bm/ && bash bin/run_charrnn_example.sh cpu /tmp/obama /tmp/obama.txt The\ Joke 1000
num_gpus = 0

[scala_inference_charrnn_gpu]
patterns = ['single_inference_p99 (\d+\.\d+|\d+)', 'single_inference_p90 (\d+\.\d+|\d+)', 'single_inference_p50 (\d+\.\d+|\d+)', 'single_inference_average (\d+\.\d+|\d+)']
metrics = ['single_inference_p99', 'single_inference_p90', 'single_inference_p50', 'single_inference_average']
compute_method = ['last', 'last', 'last', 'last']
command_to_execute = cd /tmp/ && wget https://s3.us-east-2.amazonaws.com/mxnet-scala/scala-example-ci/RNN/obama.zip && unzip obama.zip && cd $HOME/benchmarkai/scala-mxnet/scala-bm/ && export SCALA_TEST_ON_GPU=1 && bash bin/run_charrnn_example.sh gpu /tmp/obama /tmp/obama.txt The\ Joke 1000
num_gpus = 0

[java_inference_ssd_cpu]
patterns = ['single_inference_p99 (\d+\.\d+|\d+)', 'single_inference_p90 (\d+\.\d+|\d+)', 'single_inference_p50 (\d+\.\d+|\d+)', 'single_inference_average (\d+\.\d+|\d+)', 'batch_inference_4x_p99 (\d+\.\d+|\d+)', 'batch_inference_4x_p90 (\d+\.\d+|\d+)', 'batch_inference_4x_p50 (\d+\.\d+|\d+)', 'batch_inference_4x_average (\d+\.\d+|\d+)', 'batch_inference_2x_p99 (\d+\.\d+|\d+)', 'batch_inference_2x_p90 (\d+\.\d+|\d+)', 'batch_inference_2x_p50 (\d+\.\d+|\d+)', 'batch_inference_2x_average (\d+\.\d+|\d+)', 'batch_inference_1x_p99 (\d+\.\d+|\d+)', 'batch_inference_1x_p90 (\d+\.\d+|\d+)', 'batch_inference_1x_p50 (\d+\.\d+|\d+)', 'batch_inference_1x_average (\d+\.\d+|\d+)']
metrics = ['single_inference_p99', 'single_inference_p90', 'single_inference_p50', 'single_inference_average', 'batch_inference_4x_p99', 'batch_inference_4x_p90', 'batch_inference_4x_p50', 'batch_inference_4x_average', 'batch_inference_2x_p99', 'batch_inference_2x_p90', 'batch_inference_2x_p50', 'batch_inference_2x_average', 'batch_inference_1x_p99', 'batch_inference_1x_p90', 'batch_inference_1x_p50', 'batch_inference_1x_average']
compute_method = ['last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last']
command_to_execute = cd $HOME/benchmarkai/scala-mxnet/java-bm/ && bash bin/get_resnet50_ssd_data.sh && bash bin/run_ssd.sh cpu /tmp/resnet50_ssd/resnet50_ssd_model /tmp/resnet50_ssd/images/dog.jpg 4 2000
num_gpus = 0

[java_inference_ssd_gpu]
patterns = ['single_inference_p99 (\d+\.\d+|\d+)', 'single_inference_p90 (\d+\.\d+|\d+)', 'single_inference_p50 (\d+\.\d+|\d+)', 'single_inference_average (\d+\.\d+|\d+)', 'batch_inference_4x_p99 (\d+\.\d+|\d+)', 'batch_inference_4x_p90 (\d+\.\d+|\d+)', 'batch_inference_4x_p50 (\d+\.\d+|\d+)', 'batch_inference_4x_average (\d+\.\d+|\d+)', 'batch_inference_2x_p99 (\d+\.\d+|\d+)', 'batch_inference_2x_p90 (\d+\.\d+|\d+)', 'batch_inference_2x_p50 (\d+\.\d+|\d+)', 'batch_inference_2x_average (\d+\.\d+|\d+)', 'batch_inference_1x_p99 (\d+\.\d+|\d+)', 'batch_inference_1x_p90 (\d+\.\d+|\d+)', 'batch_inference_1x_p50 (\d+\.\d+|\d+)', 'batch_inference_1x_average (\d+\.\d+|\d+)']
metrics = ['single_inference_p99', 'single_inference_p90', 'single_inference_p50', 'single_inference_average', 'batch_inference_4x_p99', 'batch_inference_4x_p90', 'batch_inference_4x_p50', 'batch_inference_4x_average', 'batch_inference_2x_p99', 'batch_inference_2x_p90', 'batch_inference_2x_p50', 'batch_inference_2x_average', 'batch_inference_1x_p99', 'batch_inference_1x_p90', 'batch_inference_1x_p50', 'batch_inference_1x_average']
compute_method = ['last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last']
command_to_execute = cd $HOME/benchmarkai/scala-mxnet/java-bm/ && bash bin/get_resnet50_ssd_data.sh && export SCALA_TEST_ON_GPU=1 && bash bin/run_ssd.sh gpu /tmp/resnet50_ssd/resnet50_ssd_model /tmp/resnet50_ssd/images/dog.jpg 4 2000
num_gpus = 0

[non_e2e_resnet18_inference_cpu]
patterns = ['Single Inference: Average prediction time per sample: (\d+\.\d+|\d+) ms', 'Batch Inference: Average prediction time per sample: (\d+\.\d+|\d+) ms']
metrics = ['single_inference_avg', 'batch_inference_avg']
compute_method = ['last', 'last']
command_to_execute = python end_to_end_model_benchmark/module_inference.py --model_path https://s3.us-east-2.amazonaws.com/mxnet-public/end_to_end_models --model_name resnet18_v1 --iterations 500 --use_gpus 0 --end_to_end False
num_gpus = 0

[non_e2e_resnet18_inference_gpu]
patterns = ['Single Inference: Average prediction time per sample: (\d+\.\d+|\d+) ms', 'Batch Inference: Average prediction time per sample: (\d+\.\d+|\d+) ms']
metrics = ['single_inference_avg', 'batch_inference_avg']
compute_method = ['last', 'last']
command_to_execute = python end_to_end_model_benchmark/module_inference.py --model_path https://s3.us-east-2.amazonaws.com/mxnet-public/end_to_end_models --model_name resnet18_v1 --iterations 500 --use_gpus 1 --end_to_end False
num_gpus = 1

[e2e_resnet18_inference_cpu]
patterns = ['Single Inference: Average prediction time per sample: (\d+\.\d+|\d+) ms', 'Batch Inference: Average prediction time per sample: (\d+\.\d+|\d+) ms']
metrics = ['single_inference_avg', 'batch_inference_avg']
compute_method = ['last', 'last']
command_to_execute = python end_to_end_model_benchmark/module_inference.py --model_path https://s3.us-east-2.amazonaws.com/mxnet-public/end_to_end_models --model_name resnet18_v1_end_to_end --iterations 500 --use_gpus 0 --end_to_end True
num_gpus = 0

[e2e_resnet18_inference_gpu]
patterns = ['Single Inference: Average prediction time per sample: (\d+\.\d+|\d+) ms', 'Batch Inference: Average prediction time per sample: (\d+\.\d+|\d+) ms']
metrics = ['single_inference_avg', 'batch_inference_avg']
compute_method = ['last', 'last']
command_to_execute = python end_to_end_model_benchmark/module_inference.py --model_path https://s3.us-east-2.amazonaws.com/mxnet-public/end_to_end_models --model_name resnet18_v1_end_to_end --iterations 500 --use_gpus 1 --end_to_end True
num_gpus = 1

[non_e2e_scala_resnet18_inference_cpu]
patterns = ['Non E2E single_inference_average (\d+\.\d+)ms', 'batch_inference_average (\d+\.\d+)ms']
metrics = ['single_inference_avg', 'batch_inference_avg']
compute_method = ['last', 'last']
command_to_execute = cd $HOME/benchmarkai/end_to_end_model_benchmark/ && bash get_model.sh non_e2e && bash run_benchmark.sh scala non_e2e cpu 500
num_gpus = 0

[non_e2e_scala_resnet18_inference_gpu]
patterns = ['Non E2E single_inference_average (\d+\.\d+)ms', 'batch_inference_average (\d+\.\d+)ms']
metrics = ['single_inference_avg', 'batch_inference_avg']
compute_method = ['last', 'last']
command_to_execute = cd $HOME/benchmarkai/end_to_end_model_benchmark/ && bash get_model.sh non_e2e && bash run_benchmark.sh scala non_e2e gpu 500
num_gpus = 1

[e2e_scala_resnet18_inference_cpu]
patterns = ['E2E single_inference_average (\d+\.\d+)ms', 'batch_inference_average (\d+\.\d+)ms']
metrics = ['single_inference_avg', 'batch_inference_avg']
compute_method = ['last', 'last']
command_to_execute = cd $HOME/benchmarkai/end_to_end_model_benchmark/ && bash get_model.sh e2e && bash run_benchmark.sh scala e2e cpu 500
num_gpus = 0

[e2e_scala_resnet18_inference_gpu]
patterns = ['E2E single_inference_average (\d+\.\d+)ms', 'batch_inference_average (\d+\.\d+)ms']
metrics = ['single_inference_avg', 'batch_inference_avg']
compute_method = ['last', 'last']
command_to_execute = cd $HOME/benchmarkai/end_to_end_model_benchmark/ && bash get_model.sh e2e && bash run_benchmark.sh scala e2e gpu 500
num_gpus = 1

[non_e2e_java_resnet18_inference_cpu]
patterns = ['Non E2E single_inference_average (\d+\.\d+)ms', 'batch_inference_average (\d+\.\d+)ms']
metrics = ['single_inference_avg', 'batch_inference_avg']
compute_method = ['last', 'last']
command_to_execute = cd $HOME/benchmarkai/end_to_end_model_benchmark/ && bash get_model.sh non_e2e && bash run_benchmark.sh java non_e2e cpu 500
num_gpus = 0

[non_e2e_java_resnet18_inference_gpu]
patterns = ['Non E2E single_inference_average (\d+\.\d+)ms', 'batch_inference_average (\d+\.\d+)ms']
metrics = ['single_inference_avg', 'batch_inference_avg']
compute_method = ['last', 'last']
command_to_execute = cd $HOME/benchmarkai/end_to_end_model_benchmark/ && bash get_model.sh non_e2e && bash run_benchmark.sh java non_e2e gpu 500
num_gpus = 1

[e2e_java_resnet18_inference_cpu]
patterns = ['E2E single_inference_average (\d+\.\d+)ms', 'batch_inference_average (\d+\.\d+)ms']
metrics = ['single_inference_avg', 'batch_inference_avg']
compute_method = ['last', 'last']
command_to_execute = cd $HOME/benchmarkai/end_to_end_model_benchmark/ && bash get_model.sh e2e && bash run_benchmark.sh java e2e cpu 500
num_gpus = 0

[e2e_java_resnet18_inference_gpu]
patterns = ['E2E single_inference_average (\d+\.\d+)ms', 'batch_inference_average (\d+\.\d+)ms']
metrics = ['single_inference_avg', 'batch_inference_avg']
compute_method = ['last', 'last']
command_to_execute = cd $HOME/benchmarkai/end_to_end_model_benchmark/ && bash get_model.sh e2e && bash run_benchmark.sh java e2e gpu 500
num_gpus = 1

[dependency_update_mlp]
patterns = ['Train_acc (\d+\.\d+)', 'Test_acc (\d+\.\d+)', 'Speed (\d+\.\d+)']
metrics = ['training_acc', 'testing_acc', 'speed']
compute_method = ['last', 'last','average']
command_to_execute = python dependency_update/mlp.py
num_gpus = 1

[djl_training_gpu]
patterns = ['python_res50_cifar10_sym speed P50: (\d+\.\d+)', 'python_res50_cifar10_sym accuracy: (\d+\.\d+)', 'python_res50_cifar10_imp speed P50: (\d+\.\d+)', 'python_res50_cifar10_imp accuracy: (\d+\.\d+)', 'djl_res50_cifar10_imp speed P50: (\d+\.\d+)', 'djl_res50_cifar10_imp accuracy: (\d+\.\d+)', 'djl_res50_cifar10_sym speed P50: (\d+\.\d+)', 'djl_res50_cifar10_sym accuracy: (\d+\.\d+)', 'djl_res50_cifar10_sym_pretrain speed P50: (\d+\.\d+)', 'djl_res50_cifar10_sym_pretrain accuracy: (\d+\.\d+)']
metrics = ['python_res50_cifar10_sym_speed_p50', 'python_res50_cifar10_sym_accuracy', 'python_res50_cifar10_imp_speed_p50', 'python_res50_cifar10_imp_accuracy', 'djl_res50_cifar10_imp_speed_p50', 'djl_res50_cifar10_imp_accuracy', 'djl_res50_cifar10_sym_speed_p50', 'djl_res50_cifar10_sym_accuracy', 'djl_res50_cifar10_sym_pretrain_speed_p50', 'djl_res50_cifar10_sym_pretrain_accuracy']
compute_method = ['last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last']
command_to_execute = cd $HOME/benchmarkai/djl && bash benchmark_training.sh 3
num_gpus = 1

[djl_inference_cpu]
patterns = ['djl_res18 inference P50: (\d+\.\d+)', 'djl_res18 inference P90: (\d+\.\d+)', 'djl_res18 preprocess P50: (\d+\.\d+)', 'djl_res18 preprocess P90: (\d+\.\d+)', 'djl_res18 postprocess P50: (\d+\.\d+)', 'djl_res18 postprocess P90: (\d+\.\d+)', 'djl_res50 inference P50: (\d+\.\d+)', 'djl_res50 inference P90: (\d+\.\d+)', 'djl_res50 preprocess P50: (\d+\.\d+)', 'djl_res50 preprocess P90: (\d+\.\d+)', 'djl_res50 postprocess P50: (\d+\.\d+)', 'djl_res50 postprocess P90: (\d+\.\d+)', 'djl_res152 inference P50: (\d+\.\d+)', 'djl_res152 inference P90: (\d+\.\d+)', 'djl_res152 preprocess P50: (\d+\.\d+)', 'djl_res152 preprocess P90: (\d+\.\d+)', 'djl_res152 postprocess P50: (\d+\.\d+)', 'djl_res152 postprocess P90: (\d+\.\d+)', 'djl_res50_cifar10 inference P50: (\d+\.\d+)', 'djl_res50_cifar10 inference P90: (\d+\.\d+)', 'djl_res50_cifar10 preprocess P50: (\d+\.\d+)', 'djl_res50_cifar10 preprocess P90: (\d+\.\d+)', 'djl_res50_cifar10 postprocess P50: (\d+\.\d+)', 'djl_res50_cifar10 postprocess P90: (\d+\.\d+)', 'djl_res50_cifar10_imp inference P50: (\d+\.\d+)', 'djl_res50_cifar10_imp inference P90: (\d+\.\d+)', 'djl_res50_cifar10_imp preprocess P50: (\d+\.\d+)', 'djl_res50_cifar10_imp preprocess P90: (\d+\.\d+)', 'djl_res50_cifar10_imp postprocess P50: (\d+\.\d+)', 'djl_res50_cifar10_imp postprocess P90: (\d+\.\d+)', 'djl_ssd_resnet50 inference P50: (\d+\.\d+)', 'djl_ssd_resnet50 inference P90: (\d+\.\d+)', 'djl_ssd_resnet50 preprocess P50: (\d+\.\d+)', 'djl_ssd_resnet50 preprocess P90: (\d+\.\d+)', 'djl_ssd_resnet50 postprocess P50: (\d+\.\d+)', 'djl_ssd_resnet50 postprocess P90: (\d+\.\d+)', 'djl_ssd_vgg16 inference P50: (\d+\.\d+)', 'djl_ssd_vgg16 inference P90: (\d+\.\d+)', 'djl_ssd_vgg16 preprocess P50: (\d+\.\d+)', 'djl_ssd_vgg16 preprocess P90: (\d+\.\d+)', 'djl_ssd_vgg16 postprocess P50: (\d+\.\d+)', 'djl_ssd_vgg16 postprocess P90: (\d+\.\d+)', 'djl_naive_res18 inference P50: (\d+\.\d+)', 'djl_naive_res18 inference P90: (\d+\.\d+)', 'djl_naive_res18 preprocess P50: (\d+\.\d+)', 'djl_naive_res18 preprocess P90: (\d+\.\d+)', 'djl_naive_res18 postprocess P50: (\d+\.\d+)', 'djl_naive_res18 postprocess P90: (\d+\.\d+)', 'djl_naive_res18 heap P90: (\d+\.\d+)' ,'djl_naive_res18 nonHeap P90: (\d+\.\d+)', 'djl_naive_res18 cpu P90: (\d+\.\d+)', 'djl_naive_res18 rss P90: (\d+\.\d+)', 'djl_multithread_res18 inference P50: (\d+\.\d+)', 'djl_multithread_res18 inference P90: (\d+\.\d+)', 'djl_multithread_res18 preprocess P50: (\d+\.\d+)', 'djl_multithread_res18 preprocess P90: (\d+\.\d+)', 'djl_multithread_res18 postprocess P50: (\d+\.\d+)', 'djl_multithread_res18 postprocess P90: (\d+\.\d+)', 'djl_multithread_res18 heap P90: (\d+\.\d+)' ,'djl_multithread_res18 nonHeap P90: (\d+\.\d+)', 'djl_multithread_res18 cpu P90: (\d+\.\d+)', 'djl_multithread_res18 rss P90: (\d+\.\d+)', 'djl_multithread_res18_threadsafe inference P50: (\d+\.\d+)', 'djl_multithread_res18_threadsafe inference P90: (\d+\.\d+)', 'djl_multithread_res18_threadsafe preprocess P50: (\d+\.\d+)', 'djl_multithread_res18_threadsafe preprocess P90: (\d+\.\d+)', 'djl_multithread_res18_threadsafe postprocess P50: (\d+\.\d+)', 'djl_multithread_res18_threadsafe postprocess P90: (\d+\.\d+)', 'djl_multithread_res18_threadsafe heap P90: (\d+\.\d+)' ,'djl_multithread_res18_threadsafe nonHeap P90: (\d+\.\d+)', 'djl_multithread_res18_threadsafe cpu P90: (\d+\.\d+)', 'djl_multithread_res18_threadsafe rss P90: (\d+\.\d+)', 'djl_multithread_res18_imp inference P50: (\d+\.\d+)', 'djl_multithread_res18_imp inference P90: (\d+\.\d+)', 'djl_multithread_res18_imp preprocess P50: (\d+\.\d+)', 'djl_multithread_res18_imp preprocess P90: (\d+\.\d+)', 'djl_multithread_res18_imp postprocess P50: (\d+\.\d+)', 'djl_multithread_res18_imp postprocess P90: (\d+\.\d+)', 'djl_multithread_res18_imp heap P90: (\d+\.\d+)' ,'djl_multithread_res18_imp nonHeap P90: (\d+\.\d+)', 'djl_multithread_res18_imp cpu P90: (\d+\.\d+)', 'djl_multithread_res18_imp rss P90: (\d+\.\d+)']
metrics = ['djl_res18_inference_p50', 'djl_res18_inference_p90', 'djl_res18_preprocesse_p50', 'djl_res18_preprocesse_p90', 'djl_res18_postprocess_p50', 'djl_res18_postprocess_p90','djl_res50_inference_p50', 'djl_res50_inference_p90', 'djl_res50_preprocesse_p50', 'djl_res50_preprocesse_p90', 'djl_res50_postprocess_p50', 'djl_res50_postprocess_p90','djl_res152_inference_p50', 'djl_res152_inference_p90', 'djl_res152_preprocesse_p50', 'djl_res152_preprocesse_p90', 'djl_res152_postprocess_p50', 'djl_res152_postprocess_p90','djl_res50_cifar10_inference_p50', 'djl_res50_cifar10_inference_p90', 'djl_res50_cifar10_preprocesse_p50', 'djl_res50_cifar10_preprocesse_p90', 'djl_res50_cifar10_postprocess_p50', 'djl_res50_cifar10_postprocess_p90','djl_res50_cifar10_imp_inference_p50', 'djl_res50_cifar10_imp_inference_p90', 'djl_res50_cifar10_imp_preprocesse_p50', 'djl_res50_cifar10_imp_preprocesse_p90', 'djl_res50_cifar10_imp_postprocess_p50', 'djl_res50_cifar10_imp_postprocess_p90','djl_ssd_resnet50_inference_p50', 'djl_ssd_resnet50_inference_p90', 'djl_ssd_resnet50_preprocesse_p50', 'djl_ssd_resnet50_preprocesse_p90', 'djl_ssd_resnet50_postprocess_p50', 'djl_ssd_resnet50_postprocess_p90', 'djl_ssd_vgg16_inference_p50', 'djl_ssd_vgg16_inference_p90', 'djl_ssd_vgg16_preprocesse_p50', 'djl_ssd_vgg16_preprocesse_p90', 'djl_ssd_vgg16_postprocess_p50', 'djl_ssd_vgg16_postprocess_p90', 'djl_naive_res18_inference_p50', 'djl_naive_res18_inference_p90', 'djl_naive_res18_preprocess_p50', 'djl_naive_res18_preprocess_p90', 'djl_naive_res18_postprocess_p50', 'djl_naive_res18_postprocess_p90', 'djl_naive_res18_heap_p90', 'djl_naive_res18_nonheap_p90', 'djl_naive_res18_cpu_p90', 'djl_naive_res18_rss_p90', 'djl_multithread_res18_inference_p50', 'djl_multithread_res18_inference_p90', 'djl_multithread_res18_preprocess_p50', 'djl_multithread_res18_preprocess_p90', 'djl_multithread_res18_postprocess_p50', 'djl_multithread_res18_postprocess_p90', 'djl_multithread_res18_heap_p90', 'djl_multithread_res18_nonheap_p90', 'djl_multithread_res18_cpu_p90', 'djl_multithread_res18_rss_p90', 'djl_multithread_res18_threadsafe_inference_p50', 'djl_multithread_res18_threadsafe_inference_p90', 'djl_multithread_res18_threadsafe_preprocess_p50', 'djl_multithread_res18_threadsafe_preprocess_p90', 'djl_multithread_res18_threadsafe_postprocess_p50', 'djl_multithread_res18_threadsafe_postprocess_p90', 'djl_multithread_res18_threadsafe_heap_p90', 'djl_multithread_res18_threadsafe_nonheap_p90', 'djl_multithread_res18_threadsafe_cpu_p90', 'djl_multithread_res18_threadsafe_rss_p90', 'djl_multithread_res18_imp_inference_p50', 'djl_multithread_res18_imp_inference_p90', 'djl_multithread_res18_imp_preprocess_p50', 'djl_multithread_res18_imp_preprocess_p90', 'djl_multithread_res18_imp_postprocess_p50', 'djl_multithread_res18_imp_postprocess_p90', 'djl_multithread_res18_imp_heap_p90', 'djl_multithread_res18_imp_nonheap_p90', 'djl_multithread_res18_imp_cpu_p90', 'djl_multithread_res18_imp_rss_p90']
compute_method = ['last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last']
command_to_execute = cd $HOME/benchmarkai/djl && bash benchmark_inference.sh 10
num_gpus = 0

[djl_inference_gpu]
patterns = ['djl_res18 inference P50: (\d+\.\d+)', 'djl_res18 inference P90: (\d+\.\d+)', 'djl_res18 preprocess P50: (\d+\.\d+)', 'djl_res18 preprocess P90: (\d+\.\d+)', 'djl_res18 postprocess P50: (\d+\.\d+)', 'djl_res18 postprocess P90: (\d+\.\d+)', 'djl_res50 inference P50: (\d+\.\d+)', 'djl_res50 inference P90: (\d+\.\d+)', 'djl_res50 preprocess P50: (\d+\.\d+)', 'djl_res50 preprocess P90: (\d+\.\d+)', 'djl_res50 postprocess P50: (\d+\.\d+)', 'djl_res50 postprocess P90: (\d+\.\d+)', 'djl_res152 inference P50: (\d+\.\d+)', 'djl_res152 inference P90: (\d+\.\d+)', 'djl_res152 preprocess P50: (\d+\.\d+)', 'djl_res152 preprocess P90: (\d+\.\d+)', 'djl_res152 postprocess P50: (\d+\.\d+)', 'djl_res152 postprocess P90: (\d+\.\d+)', 'djl_res50_cifar10 inference P50: (\d+\.\d+)', 'djl_res50_cifar10 inference P90: (\d+\.\d+)', 'djl_res50_cifar10 preprocess P50: (\d+\.\d+)', 'djl_res50_cifar10 preprocess P90: (\d+\.\d+)', 'djl_res50_cifar10 postprocess P50: (\d+\.\d+)', 'djl_res50_cifar10 postprocess P90: (\d+\.\d+)', 'djl_res50_cifar10_imp inference P50: (\d+\.\d+)', 'djl_res50_cifar10_imp inference P90: (\d+\.\d+)', 'djl_res50_cifar10_imp preprocess P50: (\d+\.\d+)', 'djl_res50_cifar10_imp preprocess P90: (\d+\.\d+)', 'djl_res50_cifar10_imp postprocess P50: (\d+\.\d+)', 'djl_res50_cifar10_imp postprocess P90: (\d+\.\d+)', 'djl_ssd_resnet50 inference P50: (\d+\.\d+)', 'djl_ssd_resnet50 inference P90: (\d+\.\d+)', 'djl_ssd_resnet50 preprocess P50: (\d+\.\d+)', 'djl_ssd_resnet50 preprocess P90: (\d+\.\d+)', 'djl_ssd_resnet50 postprocess P50: (\d+\.\d+)', 'djl_ssd_resnet50 postprocess P90: (\d+\.\d+)', 'djl_ssd_vgg16 inference P50: (\d+\.\d+)', 'djl_ssd_vgg16 inference P90: (\d+\.\d+)', 'djl_ssd_vgg16 preprocess P50: (\d+\.\d+)', 'djl_ssd_vgg16 preprocess P90: (\d+\.\d+)', 'djl_ssd_vgg16 postprocess P50: (\d+\.\d+)', 'djl_ssd_vgg16 postprocess P90: (\d+\.\d+)', 'djl_naive_res18 inference P50: (\d+\.\d+)', 'djl_naive_res18 inference P90: (\d+\.\d+)', 'djl_naive_res18 preprocess P50: (\d+\.\d+)', 'djl_naive_res18 preprocess P90: (\d+\.\d+)', 'djl_naive_res18 postprocess P50: (\d+\.\d+)', 'djl_naive_res18 postprocess P90: (\d+\.\d+)', 'djl_naive_res18 heap P90: (\d+\.\d+)' ,'djl_naive_res18 nonHeap P90: (\d+\.\d+)', 'djl_naive_res18 cpu P90: (\d+\.\d+)', 'djl_naive_res18 rss P90: (\d+\.\d+)', 'djl_multithread_res18 inference P50: (\d+\.\d+)', 'djl_multithread_res18 inference P90: (\d+\.\d+)', 'djl_multithread_res18 preprocess P50: (\d+\.\d+)', 'djl_multithread_res18 preprocess P90: (\d+\.\d+)', 'djl_multithread_res18 postprocess P50: (\d+\.\d+)', 'djl_multithread_res18 postprocess P90: (\d+\.\d+)', 'djl_multithread_res18 heap P90: (\d+\.\d+)' ,'djl_multithread_res18 nonHeap P90: (\d+\.\d+)', 'djl_multithread_res18 cpu P90: (\d+\.\d+)', 'djl_multithread_res18 rss P90: (\d+\.\d+)', 'djl_multithread_res18_threadsafe inference P50: (\d+\.\d+)', 'djl_multithread_res18_threadsafe inference P90: (\d+\.\d+)', 'djl_multithread_res18_threadsafe preprocess P50: (\d+\.\d+)', 'djl_multithread_res18_threadsafe preprocess P90: (\d+\.\d+)', 'djl_multithread_res18_threadsafe postprocess P50: (\d+\.\d+)', 'djl_multithread_res18_threadsafe postprocess P90: (\d+\.\d+)', 'djl_multithread_res18_threadsafe heap P90: (\d+\.\d+)' ,'djl_multithread_res18_threadsafe nonHeap P90: (\d+\.\d+)', 'djl_multithread_res18_threadsafe cpu P90: (\d+\.\d+)', 'djl_multithread_res18_threadsafe rss P90: (\d+\.\d+)', 'djl_multithread_res18_imp inference P50: (\d+\.\d+)', 'djl_multithread_res18_imp inference P90: (\d+\.\d+)', 'djl_multithread_res18_imp preprocess P50: (\d+\.\d+)', 'djl_multithread_res18_imp preprocess P90: (\d+\.\d+)', 'djl_multithread_res18_imp postprocess P50: (\d+\.\d+)', 'djl_multithread_res18_imp postprocess P90: (\d+\.\d+)', 'djl_multithread_res18_imp heap P90: (\d+\.\d+)' ,'djl_multithread_res18_imp nonHeap P90: (\d+\.\d+)', 'djl_multithread_res18_imp cpu P90: (\d+\.\d+)', 'djl_multithread_res18_imp rss P90: (\d+\.\d+)']
metrics = ['djl_res18_inference_p50', 'djl_res18_inference_p90', 'djl_res18_preprocesse_p50', 'djl_res18_preprocesse_p90', 'djl_res18_postprocess_p50', 'djl_res18_postprocess_p90','djl_res50_inference_p50', 'djl_res50_inference_p90', 'djl_res50_preprocesse_p50', 'djl_res50_preprocesse_p90', 'djl_res50_postprocess_p50', 'djl_res50_postprocess_p90','djl_res152_inference_p50', 'djl_res152_inference_p90', 'djl_res152_preprocesse_p50', 'djl_res152_preprocesse_p90', 'djl_res152_postprocess_p50', 'djl_res152_postprocess_p90','djl_res50_cifar10_inference_p50', 'djl_res50_cifar10_inference_p90', 'djl_res50_cifar10_preprocesse_p50', 'djl_res50_cifar10_preprocesse_p90', 'djl_res50_cifar10_postprocess_p50', 'djl_res50_cifar10_postprocess_p90','djl_res50_cifar10_imp_inference_p50', 'djl_res50_cifar10_imp_inference_p90', 'djl_res50_cifar10_imp_preprocesse_p50', 'djl_res50_cifar10_imp_preprocesse_p90', 'djl_res50_cifar10_imp_postprocess_p50', 'djl_res50_cifar10_imp_postprocess_p90','djl_ssd_resnet50_inference_p50', 'djl_ssd_resnet50_inference_p90', 'djl_ssd_resnet50_preprocesse_p50', 'djl_ssd_resnet50_preprocesse_p90', 'djl_ssd_resnet50_postprocess_p50', 'djl_ssd_resnet50_postprocess_p90', 'djl_ssd_vgg16_inference_p50', 'djl_ssd_vgg16_inference_p90', 'djl_ssd_vgg16_preprocesse_p50', 'djl_ssd_vgg16_preprocesse_p90', 'djl_ssd_vgg16_postprocess_p50', 'djl_ssd_vgg16_postprocess_p90', 'djl_naive_res18_inference_p50', 'djl_naive_res18_inference_p90', 'djl_naive_res18_preprocess_p50', 'djl_naive_res18_preprocess_p90', 'djl_naive_res18_postprocess_p50', 'djl_naive_res18_postprocess_p90', 'djl_naive_res18_heap_p90', 'djl_naive_res18_nonheap_p90', 'djl_naive_res18_cpu_p90', 'djl_naive_res18_rss_p90', 'djl_multithread_res18_inference_p50', 'djl_multithread_res18_inference_p90', 'djl_multithread_res18_preprocess_p50', 'djl_multithread_res18_preprocess_p90', 'djl_multithread_res18_postprocess_p50', 'djl_multithread_res18_postprocess_p90', 'djl_multithread_res18_heap_p90', 'djl_multithread_res18_nonheap_p90', 'djl_multithread_res18_cpu_p90', 'djl_multithread_res18_rss_p90', 'djl_multithread_res18_threadsafe_inference_p50', 'djl_multithread_res18_threadsafe_inference_p90', 'djl_multithread_res18_threadsafe_preprocess_p50', 'djl_multithread_res18_threadsafe_preprocess_p90', 'djl_multithread_res18_threadsafe_postprocess_p50', 'djl_multithread_res18_threadsafe_postprocess_p90', 'djl_multithread_res18_threadsafe_heap_p90', 'djl_multithread_res18_threadsafe_nonheap_p90', 'djl_multithread_res18_threadsafe_cpu_p90', 'djl_multithread_res18_threadsafe_rss_p90', 'djl_multithread_res18_imp_inference_p50', 'djl_multithread_res18_imp_inference_p90', 'djl_multithread_res18_imp_preprocess_p50', 'djl_multithread_res18_imp_preprocess_p90', 'djl_multithread_res18_imp_postprocess_p50', 'djl_multithread_res18_imp_postprocess_p90', 'djl_multithread_res18_imp_heap_p90', 'djl_multithread_res18_imp_nonheap_p90', 'djl_multithread_res18_imp_cpu_p90', 'djl_multithread_res18_imp_rss_p90']
compute_method = ['last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last', 'last']
command_to_execute = cd $HOME/benchmarkai/djl && bash benchmark_inference.sh 10
num_gpus = 1