V2 Performance Signal Detected by TorchBench CI on '2.6.0.dev20240925+cu124'

[xuzhao9]

TorchBench CI has detected a performance signal.

Base PyTorch version: 2.4.0.dev20240606+cu121

Base PyTorch commit: ffaea656b5d8ff6518669494cc8f664b94f8e8b1

Affected PyTorch version: 2.6.0.dev20240925+cu124

Affected PyTorch commit: 780f4debdbb51347da5b416416392e5849cbfec5

Affected Tests:

• test_train[Super_SloMo-cuda-eager]: +14.62361%
• test_eval[Super_SloMo-cuda-eager]: +11.08300%
• test_train[Super_SloMo-cuda-jit]: +14.74917%
• test_eval[Super_SloMo-cuda-jit]: +11.21654%
• test_eval[alexnet-cuda-eager]: +10.35625%
• test_eval[alexnet-cuda-jit]: +13.74590%
• test_train[dcgan-cuda-eager]: +63.64632%
• test_train[dcgan-cuda-jit]: +63.56424%
• test_eval[pyhpc_equation_of_state-cpu-jit]: +17.70788%
• test_eval[pytorch_stargan-cpu-jit]: -16.09282%
• test_eval[resnext50_32x4d-cpu-jit]: +22.06176%
• test_eval[shufflenet_v2_x1_0-cpu-eager]: +10.89289%
• test_eval[shufflenet_v2_x1_0-cuda-jit]: -27.55771%
• test_train[timm_nfnet-cpu-jit]: -7.59433%
• test_eval[timm_regnet-cuda-eager]: +170.07321%
• test_eval[timm_regnet-cuda-jit]: +355.24401%
• test_train[tts_angular-cpu-eager]: +13.53244%
• test_train[yolov3-cuda-eager]: -9.84477%

cc @xuzhao9

Result json:

{
  "start": "ffaea656b5d8ff6518669494cc8f664b94f8e8b1",
  "end": "780f4debdbb51347da5b416416392e5849cbfec5",
  "threshold": 7,
  "timeout": 120,
  "torchbench_branch": "v2.0",
  "result": [
    {
      "commit1": "ffaea656b5d",
      "commit1_time": "2024-06-05 22:56:32 +0000",
      "commit1_digest": {
        "test_train[Super_SloMo-cuda-eager]": 1.3383655043318867,
        "test_eval[Super_SloMo-cuda-eager]": 0.7830573028884829,
        "test_train[Super_SloMo-cuda-jit]": 1.3295250311493874,
        "test_eval[Super_SloMo-cuda-jit]": 0.7729050191119313,
        "test_eval[alexnet-cuda-eager]": 0.0037484511607483533,
        "test_eval[alexnet-cuda-jit]": 0.0035929080876086258,
        "test_train[dcgan-cuda-eager]": 0.3330545539967716,
        "test_train[dcgan-cuda-jit]": 0.33326116679236295,
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.11547786500304938,
        "test_eval[pytorch_stargan-cpu-jit]": 0.2467015314847231,
        "test_eval[resnext50_32x4d-cpu-jit]": 0.0428968274500221,
        "test_eval[shufflenet_v2_x1_0-cpu-eager]": 0.15370346680283548,
        "test_eval[shufflenet_v2_x1_0-cuda-jit]": 0.012959176220763952,
        "test_train[timm_nfnet-cpu-jit]": 12.09011394539848,
        "test_eval[timm_regnet-cuda-eager]": 0.18956300942227244,
        "test_eval[timm_regnet-cuda-jit]": 0.21449776273220778,
        "test_train[tts_angular-cpu-eager]": 0.20952929174527526,
        "test_train[yolov3-cuda-eager]": 0.9163887814618648
      },
      "commit2": "e98662bed99",
      "commit2_time": "2024-06-05 23:44:54 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09026414761319757
      }
    },
    {
      "commit1": "e98662bed99",
      "commit1_time": "2024-06-05 23:44:54 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09026414761319757
      },
      "commit2": "2fd75667b4f",
      "commit2_time": "2024-06-05 23:46:31 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07834779173135757
      }
    },
    {
      "commit1": "8bcebc8daee",
      "commit1_time": "2024-06-05 23:59:38 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08243191689252853
      },
      "commit2": "d3ad84c38f5",
      "commit2_time": "2024-06-06 00:45:59 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06961122183129191
      }
    },
    {
      "commit1": "d3ad84c38f5",
      "commit1_time": "2024-06-06 00:45:59 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06961122183129191
      },
      "commit2": "80d34217c6e",
      "commit2_time": "2024-06-06 01:03:25 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08117376081645489
      }
    },
    {
      "commit1": "bf2c05352ea",
      "commit1_time": "2024-06-06 01:09:37 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08106971401721239
      },
      "commit2": "6adcf21b2be",
      "commit2_time": "2024-06-06 01:13:07 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06898112883791327
      }
    },
    {
      "commit1": "9795c4224bc",
      "commit1_time": "2024-06-06 01:50:18 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09424022920429706
      },
      "commit2": "c1a43a69e42",
      "commit2_time": "2024-06-06 01:56:12 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06777647649869323
      }
    },
    {
      "commit1": "c1a43a69e42",
      "commit1_time": "2024-06-06 01:56:12 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06777647649869323
      },
      "commit2": "2f7cfecd860",
      "commit2_time": "2024-06-06 02:29:45 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08126764977350831
      }
    },
    {
      "commit1": "638f543ac23",
      "commit1_time": "2024-06-06 06:25:00 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08845418365672231
      },
      "commit2": "c8ff1cd3872",
      "commit2_time": "2024-06-06 06:34:02 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06929754875600339
      }
    },
    {
      "commit1": "de4f8b99469",
      "commit1_time": "2024-06-06 18:45:22 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.0681347287259996
      },
      "commit2": "e8670f6aeae",
      "commit2_time": "2024-06-06 19:47:11 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09162157354876399
      }
    },
    {
      "commit1": "e8670f6aeae",
      "commit1_time": "2024-06-06 19:47:11 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09162157354876399
      },
      "commit2": "7e059b3c959",
      "commit2_time": "2024-06-06 20:25:39 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08550304342061281
      }
    },
    {
      "commit1": "7e059b3c959",
      "commit1_time": "2024-06-06 20:25:39 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08550304342061281
      },
      "commit2": "2184cdd2912",
      "commit2_time": "2024-06-06 20:32:29 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09508963786065579
      }
    },
    {
      "commit1": "2184cdd2912",
      "commit1_time": "2024-06-06 20:32:29 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09508963786065579
      },
      "commit2": "95543004362",
      "commit2_time": "2024-06-06 20:50:11 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06833135010674596
      }
    },
    {
      "commit1": "95543004362",
      "commit1_time": "2024-06-06 20:50:11 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06833135010674596
      },
      "commit2": "baaa914bf7d",
      "commit2_time": "2024-06-06 21:21:40 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08179153874516487
      }
    },
    {
      "commit1": "0c16800b4a2",
      "commit1_time": "2024-06-07 08:40:54 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.0857343127951026
      },
      "commit2": "7efaeb1494c",
      "commit2_time": "2024-06-07 08:57:02 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07593778949230909
      }
    },
    {
      "commit1": "7efaeb1494c",
      "commit1_time": "2024-06-07 08:57:02 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07593778949230909
      },
      "commit2": "5f812655721",
      "commit2_time": "2024-06-07 09:19:07 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08684818418696523
      }
    },
    {
      "commit1": "543a8709431",
      "commit1_time": "2024-06-07 09:24:16 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08239819565787912
      },
      "commit2": "3f9798a4fd2",
      "commit2_time": "2024-06-07 15:17:22 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07438560985028744
      }
    },
    {
      "commit1": "23c156cd2d6",
      "commit1_time": "2024-06-07 15:58:36 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09426412237808109
      },
      "commit2": "ac51f782fe0",
      "commit2_time": "2024-06-07 16:01:46 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08171502957120538
      }
    },
    {
      "commit1": "ac51f782fe0",
      "commit1_time": "2024-06-07 16:01:46 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08171502957120538
      },
      "commit2": "852b7b4c995",
      "commit2_time": "2024-06-07 16:10:11 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08756975969299674
      }
    },
    {
      "commit1": "852b7b4c995",
      "commit1_time": "2024-06-07 16:10:11 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08756975969299674
      },
      "commit2": "8d16a73f0f6",
      "commit2_time": "2024-06-07 16:10:44 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.10079854894429445
      }
    },
    {
      "commit1": "7b9c5e0e3fb",
      "commit1_time": "2024-06-10 16:49:02 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07469062190502881
      },
      "commit2": "8e482e909bd",
      "commit2_time": "2024-06-10 17:11:26 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.10098830945789813
      }
    },
    {
      "commit1": "8e482e909bd",
      "commit1_time": "2024-06-10 17:11:26 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.10098830945789813
      },
      "commit2": "ab3a0b192aa",
      "commit2_time": "2024-06-10 17:12:57 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08275365736335516
      }
    },
    {
      "commit1": "583a56d5a8e",
      "commit1_time": "2024-06-10 22:17:33 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08005137322470546
      },
      "commit2": "2176ef7dfaf",
      "commit2_time": "2024-06-10 22:20:51 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08611205751076341
      }
    },
    {
      "commit1": "98d6a6eb7da",
      "commit1_time": "2024-08-16 22:12:01 +0000",
      "commit1_digest": {
        "test_train[tts_angular-cpu-eager]": 0.24497129386290908
      },
      "commit2": "19ff9059ebe",
      "commit2_time": "2024-08-16 22:41:05 +0000",
      "commit2_digest": {
        "test_train[tts_angular-cpu-eager]": 0.19678172720596193
      }
    },
    {
      "commit1": "5a7b544e5c3",
      "commit1_time": "2024-08-20 22:38:10 +0000",
      "commit1_digest": {
        "test_train[tts_angular-cpu-eager]": 0.21645390912890433
      },
      "commit2": "5f3d22a6096",
      "commit2_time": "2024-08-20 22:51:33 +0000",
      "commit2_digest": {
        "test_train[tts_angular-cpu-eager]": 0.24029462980106472
      }
    },
    {
      "commit1": "6adcf21b2be",
      "commit1_time": "2024-06-06 01:13:07 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06898112883791327
      },
      "commit2": "b4a01614499",
      "commit2_time": "2024-06-06 01:41:06 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07954343426972628
      }
    },
    {
      "commit1": "b4a01614499",
      "commit1_time": "2024-06-06 01:41:06 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07954343426972628
      },
      "commit2": "9795c4224bc",
      "commit2_time": "2024-06-06 01:50:18 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09424022920429706
      }
    },
    {
      "commit1": "baaa914bf7d",
      "commit1_time": "2024-06-06 21:21:40 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08179153874516487
      },
      "commit2": "04272a0e129",
      "commit2_time": "2024-06-06 21:22:09 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09155028499662876
      }
    },
    {
      "commit1": "04272a0e129",
      "commit1_time": "2024-06-06 21:22:09 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09155028499662876
      },
      "commit2": "54fe2d0e89e",
      "commit2_time": "2024-06-06 21:43:29 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07572662765160203
      }
    },
    {
      "commit1": "3f9798a4fd2",
      "commit1_time": "2024-06-07 15:17:22 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.07438560985028744
      },
      "commit2": "771be55bb08",
      "commit2_time": "2024-06-07 15:20:18 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.09510255083441735
      }
    },
    {
      "commit1": "8d16a73f0f6",
      "commit1_time": "2024-06-07 16:10:44 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.10079854894429445
      },
      "commit2": "c219fa5eb94",
      "commit2_time": "2024-06-07 16:13:16 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08620314672589302
      }
    },
    {
      "commit1": "c219fa5eb94",
      "commit1_time": "2024-06-07 16:13:16 +0000",
      "commit1_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.08620314672589302
      },
      "commit2": "128952625be",
      "commit2_time": "2024-06-07 16:15:03 +0000",
      "commit2_digest": {
        "test_eval[pyhpc_equation_of_state-cpu-jit]": 0.06885770494118333
      }
    }
  ]
}

Bisection workflow link: https://github.com/pytorch/benchmark/actions/runs/11037886346