benchmarks.sh

#! /usr/bin/sh

# Copyright (c) Meta Platforms, Inc. and affiliates.

# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.


set -o xtrace
trap : SIGTERM SIGINT

BENCHMARKS=(
  "alexnet,eval,1"
  "alexnet,eval,32"
  "alexnet,eval,64"
  "alexnet,eval,128"
  "alexnet,eval,256"
  "alexnet,eval,512"
  "alexnet,eval,1024"
  "alexnet,eval,2048"
  "alexnet,eval,4096"
  "alexnet,eval,8192"
  "alexnet,train,1"
  "alexnet,train,32"
  "alexnet,train,64"
  "alexnet,train,128"
  "alexnet,train,256"
  "alexnet,train,512"
  "alexnet,train,1024"
  "alexnet,train,2048"
  "alexnet,train,4096"
  "alexnet,train,8192"
  "bert,eval,1"
  "bert,eval,32"
  "bert,eval,64"
  "bert,eval,128"
  "bert,eval,256"
  "bert,eval,512"
  "bert,eval,1024"
  "bert,eval,2048"
  "bert,eval,4096"
  "bert,eval,8192"
  "bert,train,1"
  "bert,train,32"
  "bert,train,64"
  "bert,train,128"
  "bert,train,256"
  "bert,train,512"
  "bert,train,1024"
  "bert,train,2048"
  "bert,train,4096"
  "bert,train,8192"
  #"conformer,eval,1"
  #"conformer,eval,32"
  #"conformer,eval,64"
  #"conformer,eval,128"
  #"conformer,eval,256"
  #"conformer,eval,512"
  #"conformer,eval,1024"
  #"conformer,eval,2048"
  #"conformer,eval,4096"
  #"conformer,eval,8192"
  #"conformer,train,1"
  #"conformer,train,32"
  #"conformer,train,64"
  #"conformer,train,128"
  #"conformer,train,256"
  #"conformer,train,512"
  #"conformer,train,1024"
  #"conformer,train,2048"
  #"conformer,train,4096"
  #"conformer,train,8192"
  #"deeplab,eval,1"
  #"deeplab,eval,32"
  #"deeplab,eval,64"
  #"deeplab,eval,128"
  #"deeplab,eval,256"
  #"deeplab,eval,512"
  #"deeplab,eval,1024"
  #"deeplab,eval,2048"
  #"deeplab,eval,4096"
  #"deeplab,eval,8192"
  #"deeplab,train,1"
  #"deeplab,train,32"
  #"deeplab,train,64"
  #"deeplab,train,128"
  #"deeplab,train,256"
  #"deeplab,train,512"
  #"deeplab,train,1024"
  #"deeplab,train,2048"
  #"deeplab,train,4096"
  #"deeplab,train,8192"
  "efficientnet,eval,1"
  "efficientnet,eval,32"
  "efficientnet,eval,64"
  "efficientnet,eval,128"
  "efficientnet,eval,256"
  "efficientnet,eval,512"
  "efficientnet,eval,1024"
  "efficientnet,eval,2048"
  "efficientnet,eval,4096"
  "efficientnet,eval,8192"
  "efficientnet,train,1"
  "efficientnet,train,32"
  "efficientnet,train,64"
  "efficientnet,train,128"
  "efficientnet,train,256"
  "efficientnet,train,512"
  "efficientnet,train,1024"
  "efficientnet,train,2048"
  "efficientnet,train,4096"
  "efficientnet,train,8192"
  #"emformer,eval,1"
  #"emformer,eval,32"
  #"emformer,eval,64"
  #"emformer,eval,128"
  #"emformer,eval,256"
  #"emformer,eval,512"
  #"emformer,eval,1024"
  #"emformer,eval,2048"
  #"emformer,eval,4096"
  #"emformer,eval,8192"
  #"emformer,train,1"
  #"emformer,train,32"
  #"emformer,train,64"
  #"emformer,train,128"
  #"emformer,train,256"
  #"emformer,train,512"
  #"emformer,train,1024"
  #"emformer,train,2048"
  #"emformer,train,4096"
  #"emformer,train,8192"
  "googlenet,eval,1"
  "googlenet,eval,32"
  "googlenet,eval,64"
  "googlenet,eval,128"
  "googlenet,eval,256"
  "googlenet,eval,512"
  "googlenet,eval,1024"
  "googlenet,eval,2048"
  "googlenet,eval,4096"
  "googlenet,eval,8192"
  "googlenet,train,1"
  "googlenet,train,32"
  "googlenet,train,64"
  "googlenet,train,128"
  "googlenet,train,256"
  "googlenet,train,512"
  "googlenet,train,1024"
  "googlenet,train,2048"
  "googlenet,train,4096"
  "googlenet,train,8192"
  "inception,eval,1"
  "inception,eval,32"
  "inception,eval,64"
  "inception,eval,128"
  "inception,eval,256"
  "inception,eval,512"
  "inception,eval,1024"
  "inception,eval,2048"
  "inception,eval,4096"
  "inception,eval,8192"
  "inception,train,1"
  "inception,train,32"
  "inception,train,64"
  "inception,train,128"
  "inception,train,256"
  "inception,train,512"
  "inception,train,1024"
  "inception,train,2048"
  "inception,train,4096"
  "inception,train,8192"
  "mnasnet,eval,1"
  "mnasnet,eval,32"
  "mnasnet,eval,64"
  "mnasnet,eval,128"
  "mnasnet,eval,256"
  "mnasnet,eval,512"
  "mnasnet,eval,1024"
  "mnasnet,eval,2048"
  "mnasnet,eval,4096"
  "mnasnet,eval,8192"
  "mnasnet,train,1"
  "mnasnet,train,32"
  "mnasnet,train,64"
  "mnasnet,train,128"
  "mnasnet,train,256"
  "mnasnet,train,512"
  "mnasnet,train,1024"
  "mnasnet,train,2048"
  "mnasnet,train,4096"
  "mnasnet,train,8192"
  "mobilenet,eval,1"
  "mobilenet,eval,32"
  "mobilenet,eval,64"
  "mobilenet,eval,128"
  "mobilenet,eval,256"
  "mobilenet,eval,512"
  "mobilenet,eval,1024"
  "mobilenet,eval,2048"
  "mobilenet,eval,4096"
  "mobilenet,eval,8192"
  "mobilenet,train,1"
  "mobilenet,train,32"
  "mobilenet,train,64"
  "mobilenet,train,128"
  "mobilenet,train,256"
  "mobilenet,train,512"
  "mobilenet,train,1024"
  "mobilenet,train,2048"
  "mobilenet,train,4096"
  "mobilenet,train,8192"
  #"raft,eval,1"
  #"raft,eval,32"
  #"raft,eval,64"
  #"raft,eval,128"
  #"raft,eval,256"
  #"raft,eval,512"
  #"raft,eval,1024"
  #"raft,eval,2048"
  #"raft,eval,4096"
  #"raft,eval,8192"
  #"raft,train,1"
  #"raft,train,32"
  #"raft,train,64"
  #"raft,train,128"
  #"raft,train,256"
  #"raft,train,512"
  #"raft,train,1024"
  #"raft,train,2048"
  #"raft,train,4096"
  #"raft,train,8192"
  "resnet,eval,1"
  "resnet,eval,32"
  "resnet,eval,64"
  "resnet,eval,128"
  "resnet,eval,256"
  "resnet,eval,512"
  "resnet,eval,1024"
  "resnet,eval,2048"
  "resnet,eval,4096"
  "resnet,eval,8192"
  "resnet,train,1"
  "resnet,train,32"
  "resnet,train,64"
  "resnet,train,128"
  "resnet,train,256"
  "resnet,train,512"
  "resnet,train,1024"
  "resnet,train,2048"
  "resnet,train,4096"
  "resnet,train,8192"
  "resnet50,eval,1"
  "resnet50,eval,32"
  "resnet50,eval,64"
  "resnet50,eval,128"
  "resnet50,eval,256"
  "resnet50,eval,512"
  "resnet50,eval,1024"
  "resnet50,eval,2048"
  "resnet50,eval,4096"
  "resnet50,eval,8192"
  "resnet50,train,1"
  "resnet50,train,32"
  "resnet50,train,64"
  "resnet50,train,128"
  "resnet50,train,256"
  "resnet50,train,512"
  "resnet50,train,1024"
  "resnet50,train,2048"
  "resnet50,train,4096"
  "resnet50,train,8192"
  "resnet3d,eval,1"
  "resnet3d,eval,32"
  "resnet3d,eval,64"
  "resnet3d,eval,128"
  "resnet3d,eval,256"
  "resnet3d,eval,512"
  "resnet3d,eval,1024"
  "resnet3d,eval,2048"
  "resnet3d,eval,4096"
  "resnet3d,eval,8192"
  "resnet3d,train,1"
  "resnet3d,train,32"
  "resnet3d,train,64"
  "resnet3d,train,128"
  "resnet3d,train,256"
  "resnet3d,train,512"
  "resnet3d,train,1024"
  "resnet3d,train,2048"
  "resnet3d,train,4096"
  "resnet3d,train,8192"
  "squeezenet,eval,1"
  "squeezenet,eval,32"
  "squeezenet,eval,64"
  "squeezenet,eval,128"
  "squeezenet,eval,256"
  "squeezenet,eval,512"
  "squeezenet,eval,1024"
  "squeezenet,eval,2048"
  "squeezenet,eval,4096"
  "squeezenet,eval,8192"
  "squeezenet,train,1"
  "squeezenet,train,32"
  "squeezenet,train,64"
  "squeezenet,train,128"
  "squeezenet,train,256"
  "squeezenet,train,512"
  "squeezenet,train,1024"
  "squeezenet,train,2048"
  "squeezenet,train,4096"
  "squeezenet,train,8192"
  #"ssd,eval,1"
  #"ssd,eval,32"
  #"ssd,eval,64"
  #"ssd,eval,128"
  #"ssd,eval,256"
  #"ssd,eval,512"
  #"ssd,eval,1024"
  #"ssd,eval,2048"
  #"ssd,eval,4096"
  #"ssd,eval,8192"
  #"ssd,train,1"
  #"ssd,train,32"
  #"ssd,train,64"
  #"ssd,train,256"
  #"ssd,train,512"
  #"ssd,train,1024"
  #"ssd,train,2048"
  #"ssd,train,4096"
  #"ssd,train,8192"
  #"swin,eval,1"
  #"swin,eval,32"
  #"swin,eval,64"
  #"swin,eval,128"
  #"swin,eval,256"
  #"swin,eval,512"
  #"swin,eval,1024"
  #"swin,eval,2048"
  #"swin,eval,4096"
  #"swin,eval,8192"
  #"swin,train,1"
  #"swin,train,32"
  #"swin,train,64"
  #"swin,train,256"
  #"swin,train,512"
  #"swin,train,1024"
  #"swin,train,2048"
  #"swin,train,4096"
  #"swin,train,8192"
  "transformer,eval,1"
  "transformer,eval,32"
  "transformer,eval,64"
  "transformer,eval,128"
  "transformer,eval,256"
  "transformer,eval,512"
  "transformer,eval,1024"
  "transformer,eval,2048"
  "transformer,eval,4096"
  "transformer,eval,8192"
  "transformer,train,1"
  "transformer,train,32"
  "transformer,train,64"
  "transformer,train,128"
  "transformer,train,256"
  "transformer,train,512"
  "transformer,train,1024"
  "transformer,train,4096"
  "transformer,train,2048"
  "transformer,train,8192"
  "transformer_dflt,eval,1"
  "transformer_dflt,eval,32"
  "transformer_dflt,eval,64"
  "transformer_dflt,eval,128"
  "transformer_dflt,eval,256"
  "transformer_dflt,eval,512"
  "transformer_dflt,eval,1024"
  "transformer_dflt,eval,2048"
  "transformer_dflt,eval,4096"
  "transformer_dflt,eval,8192"
  "transformer_dflt,train,1"
  "transformer_dflt,train,32"
  "transformer_dflt,train,64"
  "transformer_dflt,train,128"
  "transformer_dflt,train,256"
  "transformer_dflt,train,512"
  "transformer_dflt,train,1024"
  "transformer_dflt,train,2048"
  "transformer_dflt,train,4096"
  "transformer_dflt,train,8192"
  "vgg,eval,1"
  "vgg,eval,32"
  "vgg,eval,64"
  "vgg,eval,128"
  "vgg,eval,256"
  "vgg,eval,512"
  "vgg,eval,1024"
  "vgg,eval,2048"
  "vgg,eval,4096"
  "vgg,eval,8192"
  "vgg,train,1"
  "vgg,train,32"
  "vgg,train,64"
  "vgg,train,128"
  "vgg,train,256"
  "vgg,train,512"
  "vgg,train,1024"
  "vgg,train,2048"
  "vgg,train,4096"
  "vgg,train,8192"
  "vgg16,eval,1"
  "vgg16,eval,32"
  "vgg16,eval,64"
  "vgg16,eval,128"
  "vgg16,eval,256"
  "vgg16,eval,512"
  "vgg16,eval,1024"
  "vgg16,eval,2048"
  "vgg16,eval,4096"
  "vgg16,eval,8192"
  "vgg16,train,1"
  "vgg16,train,32"
  "vgg16,train,64"
  "vgg16,train,128"
  "vgg16,train,256"
  "vgg16,train,512"
  "vgg16,train,1024"
  "vgg16,train,2048"
  "vgg16,train,4096"
  "vgg16,train,8192"
  "vgg19,eval,1"
  "vgg19,eval,32"
  "vgg19,eval,64"
  "vgg19,eval,128"
  "vgg19,eval,256"
  "vgg19,eval,512"
  "vgg19,eval,1024"
  "vgg19,eval,2048"
  "vgg19,eval,4096"
  "vgg19,eval,8192"
  "vgg19,train,1"
  "vgg19,train,32"
  "vgg19,train,64"
  "vgg19,train,128"
  "vgg19,train,256"
  "vgg19,train,512"
  "vgg19,train,1024"
  "vgg19,train,2048"
  "vgg19,train,4096"
  "vgg19,train,8192"
  "vit,eval,1"
  "vit,eval,32"
  "vit,eval,64"
  "vit,eval,128"
  "vit,eval,256"
  "vit,eval,512"
  "vit,eval,1024"
  "vit,eval,2048"
  "vit,eval,4096"
  "vit,eval,8192"
  "vit,train,1"
  "vit,train,32"
  "vit,train,64"
  "vit,train,128"
  "vit,train,256"
  "vit,train,512"
  "vit,train,1024"
  "vit,train,2048"
  "vit,train,4096"
  "vit,train,8192"
  "xlmr,eval,1"
  "xlmr,eval,32"
  "xlmr,eval,64"
  "xlmr,eval,128"
  "xlmr,eval,256"
  "xlmr,eval,512"
  "xlmr,eval,1024"
  "xlmr,eval,2048"
  "xlmr,eval,4096"
  "xlmr,eval,8192"
  "xlmr,train,1"
  "xlmr,train,32"
  "xlmr,train,64"
  "xlmr,train,128"
  "xlmr,train,256"
  "xlmr,train,512"
  "xlmr,train,1024"
  "xlmr,train,2048"
  "xlmr,train,4096"
  "xlmr,train,8192"
  "opt-125m,train,1"
  "opt-125m,train,32"
  "opt-125m,train,64"
  "opt-125m,train,128"
  "opt-125m,train,256"
  "opt-125m,train,512"
  "opt-125m,train,1024"
  "opt-125m,train,2048"
  "opt-125m,train,4096"
  "opt-125m,train,8192"
  "opt-350m,train,1"
  "opt-350m,train,32"
  "opt-350m,train,64"
  "opt-350m,train,128"
  "opt-350m,train,256"
  "opt-350m,train,512"
  "opt-350m,train,1024"
  "opt-350m,train,2048"
  "opt-350m,train,4096"
  "opt-350m,train,8192"
  "opt-1.3b,train,1"
  "opt-1.3b,train,32"
  "opt-1.3b,train,64"
  "opt-1.3b,train,128"
  "opt-1.3b,train,256"
  "opt-1.3b,train,512"
  "opt-1.3b,train,1024"
  "opt-1.3b,train,2048"
  "opt-1.3b,train,4096"
  "opt-1.3b,train,8192"
  "opt-2.7b,train,1"
  "opt-2.7b,train,32"
  "opt-2.7b,train,64"
  "opt-2.7b,train,128"
  "opt-2.7b,train,256"
  "opt-2.7b,train,512"
  "opt-2.7b,train,1024"
  "opt-2.7b,train,2048"
  "opt-2.7b,train,4096"
  "opt-2.7b,train,8192"
  "opt-6.7b,train,1"
  "opt-6.7b,train,32"
  "opt-6.7b,train,64"
  "opt-6.7b,train,128"
  "opt-6.7b,train,256"
  "opt-6.7b,train,512"
  "opt-6.7b,train,1024"
  "opt-6.7b,train,2048"
  "opt-6.7b,train,4096"
  "opt-6.7b,train,8192"
  "opt-13b,train,1"
  "opt-13b,train,32"
  "opt-13b,train,64"
  "opt-13b,train,128"
  "opt-13b,train,256"
  "opt-13b,train,512"
  "opt-13b,train,1024"
  "opt-13b,train,2048"
  "opt-13b,train,4096"
  "opt-13b,train,8192"
  "opt-30b,train,1"
  "opt-30b,train,32"
  "opt-30b,train,64"
  "opt-30b,train,128"
  "opt-30b,train,256"
  "opt-30b,train,512"
  "opt-30b,train,1024"
  "opt-30b,train,2048"
  "opt-30b,train,4096"
  "opt-30b,train,8192"
  "opt-66b,train,1"
  "opt-66b,train,32"
  "opt-66b,train,64"
  "opt-66b,train,128"
  "opt-66b,train,256"
  "opt-66b,train,512"
  "opt-66b,train,1024"
  "opt-66b,train,2048"
  "opt-66b,train,4096"
  "opt-66b,train,8192"
)

first_time=1
for benchmark in "${BENCHMARKS[@]}"; do
  while IFS=',' read -r model mode batch_size; do
    append_log=$([ "${first_time}" == 1 ] && echo || echo "--append-log")

    python benchmarks.py -b "${batch_size}" --model "${model}" --mode "${mode}" ${append_log} $@ &

    # kill python run if user kills shell script, then terminate script
    FIND_PID=$!
    wait $FIND_PID
    if [[ $? -gt 128 ]]
    then
        kill $FIND_PID
        exit 1
    fi

    first_time=0
  done <<< "${benchmark}"
done