diff --git a/forge/forge/tvm_unique_op_generation.py b/forge/forge/tvm_unique_op_generation.py index c3be9f172..3ec5f0d6c 100644 --- a/forge/forge/tvm_unique_op_generation.py +++ b/forge/forge/tvm_unique_op_generation.py @@ -569,7 +569,7 @@ def extract_and_generate_unique_ops_tests( named_buffers_file_name = os.path.join(module_directory, str(current_module_name) + "_named_buffers.pt") torch.save(named_buffers, named_buffers_file_name) else: - if os.path.exists(param_file_name): + if param_file_name is not None and os.path.exists(param_file_name): os.remove(param_file_name) param_file_name = None diff --git a/forge/test/models/pytorch/text/mistral/test_mistral.py b/forge/test/models/pytorch/text/mistral/test_mistral.py index 219001a1f..122da6268 100644 --- a/forge/test/models/pytorch/text/mistral/test_mistral.py +++ b/forge/test/models/pytorch/text/mistral/test_mistral.py @@ -8,7 +8,7 @@ import forge from forge.verify.verify import verify -from test.models.utils import Framework, build_module_name +from test.models.utils import Framework, Source, Task, build_module_name variants = ["mistralai/Mistral-7B-v0.1"] diff --git a/forge/test/models/pytorch/text/t5/test_t5.py b/forge/test/models/pytorch/text/t5/test_t5.py index 3fbbcf11a..8aae68045 100644 --- a/forge/test/models/pytorch/text/t5/test_t5.py +++ b/forge/test/models/pytorch/text/t5/test_t5.py @@ -3,7 +3,7 @@ # SPDX-License-Identifier: Apache-2.0 import pytest import torch -from transformers import T5Config, T5ForConditionalGeneration +from transformers import AutoTokenizer, T5Config, T5ForConditionalGeneration import forge from forge.verify.verify import verify diff --git a/forge/test/models/pytorch/vision/densenet/test_densenet.py b/forge/test/models/pytorch/vision/densenet/test_densenet.py index 941f5170b..941351f2d 100644 --- a/forge/test/models/pytorch/vision/densenet/test_densenet.py +++ b/forge/test/models/pytorch/vision/densenet/test_densenet.py @@ -120,7 +120,13 @@ def test_densenet_201_pytorch(record_forge_property, variant): pytest.skip("Insufficient host DRAM to run this model (requires a more than 32 GB during compile time)") # Build Module Name - module_name = build_module_name(framework=Framework.PYTORCH, model="densenet", variant=variant) + module_name = build_module_name( + framework=Framework.PYTORCH, + model="densenet", + variant=variant, + task=Task.IMAGE_CLASSIFICATION, + source=Source.TORCHVISION, + ) # Record Forge Property record_forge_property("model_name", module_name) diff --git a/forge/test/models/pytorch/vision/mlp_mixer/test_mlp_mixer.py b/forge/test/models/pytorch/vision/mlp_mixer/test_mlp_mixer.py index 6df1b6c9a..ffcb86e74 100644 --- a/forge/test/models/pytorch/vision/mlp_mixer/test_mlp_mixer.py +++ b/forge/test/models/pytorch/vision/mlp_mixer/test_mlp_mixer.py @@ -49,7 +49,7 @@ def test_mlp_mixer_timm_pytorch(record_forge_property, variant): record_forge_property("model_name", module_name) framework_model = download_model(timm.create_model, variant, pretrained=True) - config = resolve_data_config({}, model=model) + config = resolve_data_config({}, model=framework_model) transform = create_transform(**config) try: diff --git a/forge/test/models/pytorch/vision/monodepth2/utils/depth_decoder.py b/forge/test/models/pytorch/vision/monodepth2/utils/depth_decoder.py index b21612480..dda3008a9 100644 --- a/forge/test/models/pytorch/vision/monodepth2/utils/depth_decoder.py +++ b/forge/test/models/pytorch/vision/monodepth2/utils/depth_decoder.py @@ -15,6 +15,12 @@ import torch import torch.nn as nn +from test.models.pytorch.vision.monodepth2.utils.layers import ( + Conv3x3, + ConvBlock, + upsample, +) + class DepthDecoder(nn.Module): def __init__(self, num_ch_enc, scales=range(4), num_output_channels=1, use_skips=True): diff --git a/forge/test/models/pytorch/vision/resnext/test_resnext.py b/forge/test/models/pytorch/vision/resnext/test_resnext.py index 092471104..290c5f1d3 100644 --- a/forge/test/models/pytorch/vision/resnext/test_resnext.py +++ b/forge/test/models/pytorch/vision/resnext/test_resnext.py @@ -10,7 +10,8 @@ from forge.verify.verify import verify from test.models.pytorch.vision.resnext.utils.utils import ( - load_resnext_model, + get_image_tensor, + get_resnext_model_and_input, post_processing, ) from test.models.utils import Framework, Source, Task, build_module_name @@ -34,7 +35,7 @@ def test_resnext_50_torchhub_pytorch(record_forge_property, variant): record_forge_property("model_name", module_name) # Load the model and prepare input data - framework_model, inputs = load_resnext_model(variant) + framework_model, inputs = get_resnext_model_and_input("pytorch/vision:v0.10.0", variant) # Forge compile framework model compiled_model = forge.compile(framework_model, sample_inputs=inputs, module_name=module_name) @@ -65,12 +66,8 @@ def test_resnext_101_torchhub_pytorch(record_forge_property, variant): # Record Forge Property record_forge_property("model_name", module_name) - # STEP 2: Create Forge module from PyTorch model - framework_model = download_model(torch.hub.load, "pytorch/vision:v0.10.0", variant, pretrained=True) - framework_model.eval() - # Load the model and prepare input data - framework_model, inputs = load_resnext_model(variant) + framework_model, inputs = get_resnext_model_and_input("pytorch/vision:v0.10.0", variant) # Forge compile framework model compiled_model = forge.compile(framework_model, sample_inputs=inputs, module_name=module_name) @@ -102,8 +99,7 @@ def test_resnext_101_32x8d_fb_wsl_pytorch(record_forge_property, variant): # Record Forge Property record_forge_property("model_name", module_name) - # STEP 2: Create Forge module from PyTorch model - # 4 variants + # Load the model and prepare input data framework_model = download_model(torch.hub.load, "facebookresearch/WSL-Images", variant) framework_model.eval() @@ -134,7 +130,7 @@ def test_resnext_14_osmr_pytorch(record_forge_property, variant): # Record Forge Property record_forge_property("model_name", module_name) - # STEP 2: Create Forge module from PyTorch model + # Load the model and prepare input data framework_model = download_model(ptcv_get_model, variant, pretrained=True) framework_model.eval() diff --git a/forge/test/models/pytorch/vision/resnext/utils/utils.py b/forge/test/models/pytorch/vision/resnext/utils/utils.py index 76b954c87..9e511c59f 100644 --- a/forge/test/models/pytorch/vision/resnext/utils/utils.py +++ b/forge/test/models/pytorch/vision/resnext/utils/utils.py @@ -11,12 +11,7 @@ from test.utils import download_model -def load_resnext_model(model_name): - - # Create model - model = download_model(torch.hub.load, "pytorch/vision:v0.10.0", model_name, pretrained=True) - model.eval() - +def get_image_tensor(): # Load data sample url, filename = ("https://github.com/pytorch/hub/raw/master/images/dog.jpg", "dog.jpg") urllib.request.urlretrieve(url, filename) @@ -33,6 +28,16 @@ def load_resnext_model(model_name): ) input_tensor = preprocess(input_image) input_batch = input_tensor.unsqueeze(0) + return input_batch + + +def get_resnext_model_and_input(repo_or_dir, model_name): + + # Create model + model = download_model(torch.hub.load, repo_or_dir, model_name, pretrained=True) + model.eval() + + input_batch = get_image_tensor() return model, [input_batch] diff --git a/forge/test/models/pytorch/vision/swin/test_swin.py b/forge/test/models/pytorch/vision/swin/test_swin.py index f3c995d36..a2a125397 100644 --- a/forge/test/models/pytorch/vision/swin/test_swin.py +++ b/forge/test/models/pytorch/vision/swin/test_swin.py @@ -3,8 +3,8 @@ # SPDX-License-Identifier: Apache-2.0 # STEP 0: import Forge library import pytest -import timm from transformers import ( + SwinForImageClassification, Swinv2ForImageClassification, Swinv2ForMaskedImageModeling, Swinv2Model, @@ -16,7 +16,6 @@ from test.models.pytorch.vision.swin.utils.image_utils import load_image from test.models.utils import Framework, Source, Task, build_module_name -from test.utils import download_model @pytest.mark.nightly @@ -36,8 +35,7 @@ def test_swin_v1_tiny_4_224_hf_pytorch(record_forge_property, variant): # STEP 1: Create Forge module from PyTorch model feature_extractor = ViTImageProcessor.from_pretrained(variant) - # model = SwinForImageClassification.from_pretrained("microsoft/swin-tiny-patch4-window7-224", torchscript=True) - framework_model = download_model(timm.create_model, variant, pretrained=True) + framework_model = SwinForImageClassification.from_pretrained(variant) framework_model.eval() # STEP 2: Prepare input samples diff --git a/forge/test/models/pytorch/vision/vovnet/test_vovnet.py b/forge/test/models/pytorch/vision/vovnet/test_vovnet.py index e5cc0f4b9..491724bcc 100644 --- a/forge/test/models/pytorch/vision/vovnet/test_vovnet.py +++ b/forge/test/models/pytorch/vision/vovnet/test_vovnet.py @@ -84,7 +84,7 @@ def test_vovnet_v1_39_stigma_pytorch(record_forge_property): verify(inputs, framework_model, compiled_model) -def generate_model_vovnet57_imgcls_stigma_pytorch(variant): +def generate_model_vovnet57_imgcls_stigma_pytorch(): # STEP 2: Create Forge module from PyTorch model model, image_tensor = download_model(preprocess_steps, vovnet57) diff --git a/forge/test/models/pytorch/vision/vovnet/utils/src_vovnet_stigma.py b/forge/test/models/pytorch/vision/vovnet/utils/src_vovnet_stigma.py index efcfea704..c97f7f93a 100644 --- a/forge/test/models/pytorch/vision/vovnet/utils/src_vovnet_stigma.py +++ b/forge/test/models/pytorch/vision/vovnet/utils/src_vovnet_stigma.py @@ -102,7 +102,7 @@ def __init__(self, in_ch, stage_ch, concat_ch, block_per_stage, layer_per_block, self.add_module("Pooling", nn.MaxPool2d(kernel_size=3, stride=2, ceil_mode=True)) module_name = f"OSA{stage_num}_1" - self.add_module(module_name, _OSA_module(in_ch, stage_ch, concat_ch, layer_per_blockmodule_name=module_name)) + self.add_module(module_name, _OSA_module(in_ch, stage_ch, concat_ch, layer_per_block, module_name)) for i in range(block_per_stage - 1): module_name = f"OSA{stage_num}_{i+2}" self.add_module( diff --git a/forge/test/models/pytorch/vision/wideresnet/test_wideresnet.py b/forge/test/models/pytorch/vision/wideresnet/test_wideresnet.py index 5a3499e40..65e85f06f 100644 --- a/forge/test/models/pytorch/vision/wideresnet/test_wideresnet.py +++ b/forge/test/models/pytorch/vision/wideresnet/test_wideresnet.py @@ -5,6 +5,7 @@ import pytest import timm +from PIL import Image from timm.data import resolve_data_config from timm.data.transforms_factory import create_transform diff --git a/scripts/model_analysis/exception_rules.py b/scripts/model_analysis/exception_rules.py index 62f0209bc..aef2aaa4a 100644 --- a/scripts/model_analysis/exception_rules.py +++ b/scripts/model_analysis/exception_rules.py @@ -122,6 +122,33 @@ ", compiled_model", ], ), + MatchingExceptionRule( + "tt-metal multi-paged buffer", + [ + "RuntimeError", + "tt_metal/impl/buffers/dispatch.cpp", + "num_pages == 1", + "TODO: add support for multi-paged buffer with page size > 64KB", + ], + ), + MatchingExceptionRule( + "tt-metal buffer allocation", + [ + "RuntimeError", + "tt_metal/impl/allocator/bank_manager.cpp", + "Out of Memory: Not enough space to allocate", + "DRAM buffer", + ], + ), + MatchingExceptionRule( + "tt-metal kernel", + [ + "RuntimeError", + "tt-metal/tt_metal/impl/kernels/kernel.cpp", + "unique+common runtime args targeting kernel", + "are too large", + ], + ), MatchingExceptionRule( "ttnn.tilize validation", [ @@ -327,6 +354,14 @@ "input_shape[3] == 16", ], ), + MatchingExceptionRule( + "ttnn conv2d", + [ + "RuntimeError", + "tt-metal/ttnn/cpp/ttnn/operations/conv/conv2d/device/conv2d_op_sharded_program_factory.cpp", + "act_block_w_datums == round_up(conv_act_size_c * filter_w, TILE_WIDTH)", + ], + ), ], ), ]