From 76d4920aa8b92bda5d8f0a4e14bd9d7e3759832e Mon Sep 17 00:00:00 2001
From: Huy Do <huydo@meta.com>
Date: Mon, 3 Feb 2025 11:37:01 -0800
Subject: [PATCH] Update TorchBench commit to main (#145455)

Summary:
I'm adding sam2 to TorchBench https://github.com/pytorch/benchmark/issues/2566, so, as part of that, I'm updating PyTorch CI to use latest TorchBench commit.

The corresponding change from TorchBench is https://github.com/pytorch/benchmark/pull/2584

The main thing to call out that the newer transformers added by https://github.com/pytorch/benchmark/pull/2488 is regressing several models. This needs to be investigated further, and I pin the version to unblock this change.

* `hf_Roberta_base` a new model added by https://github.com/pytorch/benchmark/pull/2279, not sure why it fails accuracy on A10G, but it works fine on A100
* `speech_transformer` failures are pre-existing trunk failures, i.e. https://github.com/pytorch/pytorch/actions/runs/13040114684/job/36380989702#step:22:2408

X-link: https://github.com/pytorch/pytorch/pull/145455
Approved by: https://github.com/kit1980

Reviewed By: ZainRizvi

Differential Revision: D69056903

fbshipit-source-id: 6b98c246b9dc8811257ec46d4821ba6ee3363dfe
---
 userbenchmark/dynamo/dynamobench/common.py       | 2 +-
 userbenchmark/dynamo/dynamobench/torchbench.yaml | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/userbenchmark/dynamo/dynamobench/common.py b/userbenchmark/dynamo/dynamobench/common.py
index b3a17f4f66..b88d6ca8f5 100644
--- a/userbenchmark/dynamo/dynamobench/common.py
+++ b/userbenchmark/dynamo/dynamobench/common.py
@@ -189,7 +189,7 @@ class CI(NamedTuple):
     "stable_diffusion_text_encoder",
     "timm_efficientdet",
     "timm_nfnet",
-    "timm_regnet",
+    "timm_resnest",
     "timm_vision_transformer",
     "timm_vovnet",
     "vgg16",
diff --git a/userbenchmark/dynamo/dynamobench/torchbench.yaml b/userbenchmark/dynamo/dynamobench/torchbench.yaml
index 5647ab4900..3911b2c2a2 100644
--- a/userbenchmark/dynamo/dynamobench/torchbench.yaml
+++ b/userbenchmark/dynamo/dynamobench/torchbench.yaml
@@ -34,7 +34,6 @@ tolerance:
     - vgg16
     - mobilenet_v3_large
     - nvidia_deeprecommender
-    - timm_efficientdet
 
   # These models need >1e-3 tolerance
   even_higher:
@@ -42,6 +41,7 @@ tolerance:
     - tacotron2
     - yolov3
     - timm_efficientdet
+    - timm_efficientnet
     - squeezenet1_1
 
   higher_fp16:
@@ -66,6 +66,7 @@ tolerance:
 
 require_larger_multiplier_for_smaller_tensor:
   - yolov3
+  - timm_efficientnet
 
 # These benchmarks took >600s on an i9-11900K CPU
 very_slow: &VERY_SLOW_MODELS