diff --git a/device/api/umd/device/blackhole_coordinate_manager.h b/device/api/umd/device/blackhole_coordinate_manager.h index 1e4dae63..a85c027e 100644 --- a/device/api/umd/device/blackhole_coordinate_manager.h +++ b/device/api/umd/device/blackhole_coordinate_manager.h @@ -20,14 +20,18 @@ class BlackholeCoordinateManager : public CoordinateManager { const size_t dram_harvesting_mask, const tt_xy_pair& eth_grid_size, const std::vector& eth_cores, + const size_t eth_harvesting_mask, const tt_xy_pair& arc_grid_size, const std::vector& arc_cores, const tt_xy_pair& pcie_grid_size, const std::vector& pcie_cores); protected: + void assert_coordinate_manager_constructor() override; + void translate_dram_coords() override; void translate_tensix_coords() override; + void translate_eth_coords() override; void fill_tensix_physical_translated_mapping() override; void fill_eth_physical_translated_mapping() override; @@ -38,10 +42,14 @@ class BlackholeCoordinateManager : public CoordinateManager { std::vector get_harvested_tensix_cores() const override; std::vector get_dram_cores() const override; std::vector get_harvested_dram_cores() const override; + std::vector get_eth_cores() const override; + std::vector get_harvested_eth_cores() const override; tt_xy_pair get_tensix_grid_size() const override; tt_xy_pair get_dram_grid_size() const override; + tt_xy_pair get_eth_grid_size() const override; tt_xy_pair get_harvested_tensix_grid_size() const override; tt_xy_pair get_harvested_dram_grid_size() const override; + tt_xy_pair get_harvested_eth_grid_size() const override; private: void map_column_of_dram_banks(const size_t start_bank, const size_t end_bank, const size_t x_coord); diff --git a/device/api/umd/device/coordinate_manager.h b/device/api/umd/device/coordinate_manager.h index 56a453d8..f580fbd6 100644 --- a/device/api/umd/device/coordinate_manager.h +++ b/device/api/umd/device/coordinate_manager.h @@ -26,13 +26,17 @@ class CoordinateManager { const size_t dram_harvesting_mask, const tt_xy_pair& eth_grid_size, const std::vector& eth_cores, + const size_t eth_harvesting_mask, const tt_xy_pair& arc_grid_size, const std::vector& arc_cores, const tt_xy_pair& pcie_grid_size, const std::vector& pcie_cores); static std::shared_ptr create_coordinate_manager( - tt::ARCH arch, const size_t tensix_harvesting_mask = 0, const size_t dram_harvesting_mask = 0); + tt::ARCH arch, + const size_t tensix_harvesting_mask = 0, + const size_t dram_harvesting_mask = 0, + const size_t eth_harvesting_mask = 0); static size_t get_num_harvested(const size_t harvesting_mask); @@ -54,10 +58,9 @@ class CoordinateManager { size_t get_dram_harvesting_mask() const; -private: - static void assert_create_coordinate_manager( - const tt::ARCH arch, const size_t tensix_harvesting_mask, const size_t dram_harvesting_mask); + size_t get_eth_harvesting_mask() const; +private: const std::vector& get_physical_pairs(const CoreType core_type) const; std::vector get_all_physical_cores(const CoreType core_type) const; @@ -78,6 +81,7 @@ class CoordinateManager { const size_t dram_harvesting_mask, const tt_xy_pair& eth_grid_size, const std::vector& eth_cores, + const size_t eth_harvesting_mask, const tt_xy_pair& arc_grid_size, const std::vector& arc_cores, const tt_xy_pair& pcie_grid_size, @@ -85,6 +89,8 @@ class CoordinateManager { void initialize(); + virtual void assert_coordinate_manager_constructor(); + virtual void shuffle_tensix_harvesting_mask(const std::vector& harvesting_locations); virtual void translate_tensix_coords(); @@ -100,10 +106,14 @@ class CoordinateManager { virtual std::vector get_harvested_tensix_cores() const; virtual std::vector get_dram_cores() const; virtual std::vector get_harvested_dram_cores() const; + virtual std::vector get_eth_cores() const; + virtual std::vector get_harvested_eth_cores() const; virtual tt_xy_pair get_tensix_grid_size() const; virtual tt_xy_pair get_dram_grid_size() const; + virtual tt_xy_pair get_eth_grid_size() const; virtual tt_xy_pair get_harvested_tensix_grid_size() const; virtual tt_xy_pair get_harvested_dram_grid_size() const; + virtual tt_xy_pair get_harvested_eth_grid_size() const; /* * Fills the logical to translated mapping for the tensix cores. @@ -163,6 +173,7 @@ class CoordinateManager { tt_xy_pair eth_grid_size; const std::vector eth_cores; + const size_t eth_harvesting_mask; tt_xy_pair arc_grid_size; const std::vector arc_cores; diff --git a/device/api/umd/device/grayskull_coordinate_manager.h b/device/api/umd/device/grayskull_coordinate_manager.h index 20660bd7..5a35dc0f 100644 --- a/device/api/umd/device/grayskull_coordinate_manager.h +++ b/device/api/umd/device/grayskull_coordinate_manager.h @@ -20,6 +20,7 @@ class GrayskullCoordinateManager : public CoordinateManager { const size_t dram_harvesting_mask, const tt_xy_pair& eth_grid_size, const std::vector& eth_cores, + const size_t eth_harvesting_mask, const tt_xy_pair& arc_grid_size, const std::vector& arc_cores, const tt_xy_pair& pcie_grid_size, diff --git a/device/api/umd/device/tt_soc_descriptor.h b/device/api/umd/device/tt_soc_descriptor.h index c5471f2d..c87669ca 100644 --- a/device/api/umd/device/tt_soc_descriptor.h +++ b/device/api/umd/device/tt_soc_descriptor.h @@ -51,8 +51,9 @@ class tt_SocDescriptor { // Constructor used to build object from device descriptor file. tt_SocDescriptor( std::string device_descriptor_path, - const std::size_t tensix_harvesting_mask = 0, - const std::size_t dram_harvesting_mask = 0); + const size_t tensix_harvesting_mask = 0, + const size_t dram_harvesting_mask = 0, + const size_t eth_harvesting_mask = 0); // Copy constructor tt_SocDescriptor(const tt_SocDescriptor &other) : @@ -143,7 +144,8 @@ class tt_SocDescriptor { uint64_t dram_bank_size; private: - void create_coordinate_manager(const std::size_t tensix_harvesting_mask, const std::size_t dram_harvesting_mask); + void create_coordinate_manager( + const size_t tensix_harvesting_mask, const size_t dram_harvesting_mask, const size_t eth_harvesting_mask); void load_core_descriptors_from_device_descriptor(YAML::Node &device_descriptor_yaml); void load_soc_features_from_device_descriptor(YAML::Node &device_descriptor_yaml); void get_cores_and_grid_size_from_coordinate_manager(); diff --git a/device/api/umd/device/wormhole_coordinate_manager.h b/device/api/umd/device/wormhole_coordinate_manager.h index 488f9f23..5246f22b 100644 --- a/device/api/umd/device/wormhole_coordinate_manager.h +++ b/device/api/umd/device/wormhole_coordinate_manager.h @@ -20,6 +20,7 @@ class WormholeCoordinateManager : public CoordinateManager { const size_t dram_harvesting_mask, const tt_xy_pair& eth_grid_size, const std::vector& eth_cores, + const size_t eth_harvesting_mask, const tt_xy_pair& arc_grid_size, const std::vector& arc_cores, const tt_xy_pair& pcie_grid_size, diff --git a/device/blackhole/blackhole_coordinate_manager.cpp b/device/blackhole/blackhole_coordinate_manager.cpp index ac8ad3f3..dfed7ba1 100644 --- a/device/blackhole/blackhole_coordinate_manager.cpp +++ b/device/blackhole/blackhole_coordinate_manager.cpp @@ -5,6 +5,8 @@ */ #include "umd/device/blackhole_coordinate_manager.h" +#include "logger.hpp" + using namespace tt::umd; BlackholeCoordinateManager::BlackholeCoordinateManager( @@ -16,6 +18,7 @@ BlackholeCoordinateManager::BlackholeCoordinateManager( const size_t dram_harvesting_mask, const tt_xy_pair& eth_grid_size, const std::vector& eth_cores, + const size_t eth_harvesting_mask, const tt_xy_pair& arc_grid_size, const std::vector& arc_cores, const tt_xy_pair& pcie_grid_size, @@ -29,6 +32,7 @@ BlackholeCoordinateManager::BlackholeCoordinateManager( dram_harvesting_mask, eth_grid_size, eth_cores, + eth_harvesting_mask, arc_grid_size, arc_cores, pcie_grid_size, @@ -37,6 +41,17 @@ BlackholeCoordinateManager::BlackholeCoordinateManager( initialize(); } +void BlackholeCoordinateManager::assert_coordinate_manager_constructor() { + log_assert(get_num_harvested(dram_harvesting_mask) <= 1, "Only one DRAM bank can be harvested on Blackhole"); + + // TODO: assert that exactly 2 or all 14 (P100) ETH cores are harvested for Blackhole. This is + // going to be true both for all Blackhole products. + // const size_t num_harvested_eth_cores = get_num_harvested(eth_harvesting_mask); + // log_assert( + // num_harvested_eth_cores == 2 || num_harvested_eth_cores == blackhole::ETH_GRID_SIZE.x, "Exactly 2 or 14 ETH + // cores should be harvested on Blackhole"); +} + void BlackholeCoordinateManager::translate_tensix_coords() { if (CoordinateManager::get_num_harvested(tensix_harvesting_mask) > tensix_grid_size.x) { tensix_harvesting_mask = 0; @@ -135,8 +150,47 @@ void BlackholeCoordinateManager::translate_dram_coords() { fill_dram_physical_translated_mapping(); } +void BlackholeCoordinateManager::translate_eth_coords() { + size_t num_harvested_x = CoordinateManager::get_num_harvested(eth_harvesting_mask); + size_t grid_size_x = eth_grid_size.x; + size_t grid_size_y = eth_grid_size.y; + + size_t logical_x = 0; + size_t x_index = grid_size_x - num_harvested_x; + for (size_t x = 0; x < grid_size_x; x++) { + if (eth_harvesting_mask & (1 << x)) { + for (size_t y = 0; y < grid_size_y; y++) { + const tt_xy_pair& physical_core = eth_cores[x + y * grid_size_x]; + const tt_xy_pair& virtual_core = eth_cores[x_index + y * grid_size_x]; + + CoreCoord virtual_coord = + CoreCoord(virtual_core.x, virtual_core.y, CoreType::ETH, CoordSystem::VIRTUAL); + + add_core_translation(virtual_coord, physical_core); + } + x_index++; + } else { + for (size_t y = 0; y < grid_size_y; y++) { + const tt_xy_pair& tensix_core = eth_cores[x + y * grid_size_x]; + const tt_xy_pair& virtual_core = eth_cores[logical_x + y * grid_size_x]; + + CoreCoord logical_coord = CoreCoord(logical_x, y, CoreType::ETH, CoordSystem::LOGICAL); + add_core_translation(logical_coord, tensix_core); + + CoreCoord virtual_coord = + CoreCoord(virtual_core.x, virtual_core.y, CoreType::ETH, CoordSystem::VIRTUAL); + add_core_translation(virtual_coord, tensix_core); + } + logical_x++; + } + } + + fill_eth_physical_translated_mapping(); +} + void BlackholeCoordinateManager::fill_eth_physical_translated_mapping() { - for (size_t x = 0; x < eth_grid_size.x; x++) { + const size_t num_harvested_x = CoordinateManager::get_num_harvested(eth_harvesting_mask); + for (size_t x = 0; x < eth_grid_size.x - num_harvested_x; x++) { for (size_t y = 0; y < eth_grid_size.y; y++) { const size_t translated_x = x + blackhole::eth_translated_coordinate_start_x; const size_t translated_y = y + blackhole::eth_translated_coordinate_start_y; @@ -149,6 +203,19 @@ void BlackholeCoordinateManager::fill_eth_physical_translated_mapping() { add_core_translation(translated_coord, physical_pair); } } + + // Harvested ETH cores should be mapped to the same physical coordinates. + for (size_t y = 0; y < eth_grid_size.y; y++) { + for (size_t x = 0; x < eth_grid_size.x; x++) { + if (eth_harvesting_mask & (1 << x)) { + const tt_xy_pair& physical_pair = eth_cores[x + y * eth_grid_size.x]; + const CoreCoord translated_coord = + CoreCoord(physical_pair.x, physical_pair.y, CoreType::ETH, CoordSystem::TRANSLATED); + + add_core_translation(translated_coord, physical_pair); + } + } + } } void BlackholeCoordinateManager::fill_pcie_physical_translated_mapping() { @@ -317,6 +384,36 @@ std::vector BlackholeCoordinateManager::get_harvested_dram_cores() co return harvested_dram_cores; } +std::vector BlackholeCoordinateManager::get_eth_cores() const { + std::vector harvested_x_coords = get_harvested_indices(eth_harvesting_mask); + std::vector unharvested_eth_cores; + for (size_t y = 0; y < eth_grid_size.y; y++) { + for (size_t x = 0; x < eth_grid_size.x; x++) { + const tt_xy_pair core = eth_cores[y * eth_grid_size.x + x]; + CoreCoord core_coord(core.x, core.y, CoreType::ETH, CoordSystem::PHYSICAL); + if (std::find(harvested_x_coords.begin(), harvested_x_coords.end(), x) == harvested_x_coords.end()) { + unharvested_eth_cores.push_back(core_coord); + } + } + } + return unharvested_eth_cores; +} + +std::vector BlackholeCoordinateManager::get_harvested_eth_cores() const { + std::vector harvested_x_coords = get_harvested_indices(eth_harvesting_mask); + std::vector harvested_eth_cores; + for (size_t y = 0; y < eth_grid_size.y; y++) { + for (size_t x = 0; x < eth_grid_size.x; x++) { + const tt_xy_pair core = eth_cores[y * eth_grid_size.x + x]; + CoreCoord core_coord(core.x, core.y, CoreType::ETH, CoordSystem::PHYSICAL); + if (std::find(harvested_x_coords.begin(), harvested_x_coords.end(), x) != harvested_x_coords.end()) { + harvested_eth_cores.push_back(core_coord); + } + } + } + return harvested_eth_cores; +} + tt_xy_pair BlackholeCoordinateManager::get_harvested_tensix_grid_size() const { return {CoordinateManager::get_num_harvested(tensix_harvesting_mask), tensix_grid_size.y}; } @@ -325,6 +422,10 @@ tt_xy_pair BlackholeCoordinateManager::get_harvested_dram_grid_size() const { return {CoordinateManager::get_num_harvested(dram_harvesting_mask), dram_grid_size.y}; } +tt_xy_pair BlackholeCoordinateManager::get_harvested_eth_grid_size() const { + return {CoordinateManager::get_num_harvested(eth_harvesting_mask), eth_grid_size.y}; +} + tt_xy_pair BlackholeCoordinateManager::get_tensix_grid_size() const { return {tensix_grid_size.x - CoordinateManager::get_num_harvested(tensix_harvesting_mask), tensix_grid_size.y}; } @@ -332,3 +433,7 @@ tt_xy_pair BlackholeCoordinateManager::get_tensix_grid_size() const { tt_xy_pair BlackholeCoordinateManager::get_dram_grid_size() const { return {dram_grid_size.x - CoordinateManager::get_num_harvested(dram_harvesting_mask), dram_grid_size.y}; } + +tt_xy_pair BlackholeCoordinateManager::get_eth_grid_size() const { + return {eth_grid_size.x - CoordinateManager::get_num_harvested(eth_harvesting_mask), eth_grid_size.y}; +} diff --git a/device/coordinate_manager.cpp b/device/coordinate_manager.cpp index 99b052fe..15ee5d04 100644 --- a/device/coordinate_manager.cpp +++ b/device/coordinate_manager.cpp @@ -5,7 +5,6 @@ */ #include "umd/device/coordinate_manager.h" -#include "api/umd/device/tt_core_coordinates.h" #include "logger.hpp" #include "umd/device/blackhole_coordinate_manager.h" #include "umd/device/grayskull_coordinate_manager.h" @@ -22,6 +21,7 @@ CoordinateManager::CoordinateManager( const size_t dram_harvesting_mask, const tt_xy_pair& eth_grid_size, const std::vector& eth_cores, + const size_t eth_harvesting_mask, const tt_xy_pair& arc_grid_size, const std::vector& arc_cores, const tt_xy_pair& pcie_grid_size, @@ -35,12 +35,14 @@ CoordinateManager::CoordinateManager( dram_harvesting_mask(dram_harvesting_mask), eth_grid_size(eth_grid_size), eth_cores(eth_cores), + eth_harvesting_mask(eth_harvesting_mask), arc_grid_size(arc_grid_size), arc_cores(arc_cores), pcie_grid_size(pcie_grid_size), pcie_cores(pcie_cores) {} void CoordinateManager::initialize() { + this->assert_coordinate_manager_constructor(); this->identity_map_physical_cores(); this->translate_tensix_coords(); this->translate_dram_coords(); @@ -49,6 +51,12 @@ void CoordinateManager::initialize() { this->translate_pcie_coords(); } +void CoordinateManager::assert_coordinate_manager_constructor() { + log_assert(dram_harvesting_mask == 0, "DRAM harvesting is supported only for Blackhole"); + + log_assert(eth_harvesting_mask == 0, "ETH harvesting is supported only for Blackhole"); +} + void CoordinateManager::add_core_translation(const CoreCoord& core_coord, const tt_xy_pair& physical_pair) { to_physical_map.insert({core_coord, physical_pair}); from_physical_map.insert({{{physical_pair.x, physical_pair.y}, core_coord.coord_system}, core_coord}); @@ -286,16 +294,6 @@ size_t CoordinateManager::get_tensix_harvesting_mask() const { return physical_l size_t CoordinateManager::get_dram_harvesting_mask() const { return dram_harvesting_mask; } -void CoordinateManager::assert_create_coordinate_manager( - const tt::ARCH arch, const size_t tensix_harvesting_mask, const size_t dram_harvesting_mask) { - log_assert( - !(arch != tt::ARCH::BLACKHOLE && dram_harvesting_mask != 0), "DRAM harvesting is supported only for Blackhole"); - - if (arch == tt::ARCH::BLACKHOLE) { - log_assert(get_num_harvested(dram_harvesting_mask) <= 1, "Only one DRAM bank can be harvested on Blackhole"); - } -} - void CoordinateManager::shuffle_tensix_harvesting_mask(const std::vector& harvesting_locations) { std::vector sorted_harvesting_locations = harvesting_locations; std::sort(sorted_harvesting_locations.begin(), sorted_harvesting_locations.end()); @@ -379,6 +377,10 @@ std::vector CoordinateManager::get_dram_cores() const { return get_al std::vector CoordinateManager::get_harvested_dram_cores() const { return {}; } +std::vector CoordinateManager::get_eth_cores() const { return get_all_physical_cores(CoreType::ETH); } + +std::vector CoordinateManager::get_harvested_eth_cores() const { return {}; } + std::vector CoordinateManager::get_cores(const CoreType core_type) const { switch (core_type) { case CoreType::TENSIX: @@ -386,7 +388,7 @@ std::vector CoordinateManager::get_cores(const CoreType core case CoreType::DRAM: return get_dram_cores(); case CoreType::ETH: - return get_all_physical_cores(CoreType::ETH); + return get_eth_cores(); case CoreType::ARC: return get_all_physical_cores(CoreType::ARC); case CoreType::PCIE: @@ -402,6 +404,8 @@ tt_xy_pair CoordinateManager::get_tensix_grid_size() const { tt_xy_pair CoordinateManager::get_dram_grid_size() const { return dram_grid_size; } +tt_xy_pair CoordinateManager::get_eth_grid_size() const { return eth_grid_size; } + tt_xy_pair CoordinateManager::get_grid_size(const CoreType core_type) const { switch (core_type) { case CoreType::TENSIX: @@ -409,7 +413,7 @@ tt_xy_pair CoordinateManager::get_grid_size(const CoreType core_type) const { case CoreType::DRAM: return get_dram_grid_size(); case CoreType::ETH: - return eth_grid_size; + return get_eth_grid_size(); case CoreType::ARC: return arc_grid_size; case CoreType::PCIE: @@ -425,6 +429,8 @@ std::vector CoordinateManager::get_harvested_cores(const Cor return get_harvested_tensix_cores(); case CoreType::DRAM: return get_harvested_dram_cores(); + case CoreType::ETH: + return get_harvested_eth_cores(); default: throw std::runtime_error("Core type is not supported for getting harvested cores"); } @@ -436,21 +442,26 @@ tt_xy_pair CoordinateManager::get_harvested_tensix_grid_size() const { tt_xy_pair CoordinateManager::get_harvested_dram_grid_size() const { return {0, 0}; } +tt_xy_pair CoordinateManager::get_harvested_eth_grid_size() const { return {0, 0}; } + tt_xy_pair CoordinateManager::get_harvested_grid_size(const CoreType core_type) const { switch (core_type) { case CoreType::TENSIX: return get_harvested_tensix_grid_size(); case CoreType::DRAM: return get_harvested_dram_grid_size(); + case CoreType::ETH: + return get_harvested_eth_grid_size(); default: throw std::runtime_error("Core type is not supported for getting harvested grid size"); } } std::shared_ptr CoordinateManager::create_coordinate_manager( - tt::ARCH arch, const size_t tensix_harvesting_mask, const size_t dram_harvesting_mask) { - assert_create_coordinate_manager(arch, tensix_harvesting_mask, dram_harvesting_mask); - + tt::ARCH arch, + const size_t tensix_harvesting_mask, + const size_t dram_harvesting_mask, + const size_t eth_harvesting_mask) { switch (arch) { case tt::ARCH::GRAYSKULL: return create_coordinate_manager( @@ -463,6 +474,7 @@ std::shared_ptr CoordinateManager::create_coordinate_manager( dram_harvesting_mask, tt::umd::grayskull::ETH_GRID_SIZE, tt::umd::grayskull::ETH_CORES, + eth_harvesting_mask, tt::umd::grayskull::ARC_GRID_SIZE, tt::umd::grayskull::ARC_CORES, tt::umd::grayskull::PCIE_GRID_SIZE, @@ -478,6 +490,7 @@ std::shared_ptr CoordinateManager::create_coordinate_manager( dram_harvesting_mask, tt::umd::wormhole::ETH_GRID_SIZE, tt::umd::wormhole::ETH_CORES, + eth_harvesting_mask, tt::umd::wormhole::ARC_GRID_SIZE, tt::umd::wormhole::ARC_CORES, tt::umd::wormhole::PCIE_GRID_SIZE, @@ -493,6 +506,7 @@ std::shared_ptr CoordinateManager::create_coordinate_manager( dram_harvesting_mask, tt::umd::blackhole::ETH_GRID_SIZE, tt::umd::blackhole::ETH_CORES, + eth_harvesting_mask, tt::umd::blackhole::ARC_GRID_SIZE, tt::umd::blackhole::ARC_CORES, tt::umd::blackhole::PCIE_GRID_SIZE, @@ -514,12 +528,11 @@ std::shared_ptr CoordinateManager::create_coordinate_manager( const size_t dram_harvesting_mask, const tt_xy_pair& eth_grid_size, const std::vector& eth_cores, + const size_t eth_harvesting_mask, const tt_xy_pair& arc_grid_size, const std::vector& arc_cores, const tt_xy_pair& pcie_grid_size, const std::vector& pcie_cores) { - assert_create_coordinate_manager(arch, tensix_harvesting_mask, dram_harvesting_mask); - switch (arch) { case tt::ARCH::GRAYSKULL: return std::make_shared( @@ -531,6 +544,7 @@ std::shared_ptr CoordinateManager::create_coordinate_manager( dram_harvesting_mask, eth_grid_size, eth_cores, + eth_harvesting_mask, arc_grid_size, arc_cores, pcie_grid_size, @@ -545,6 +559,7 @@ std::shared_ptr CoordinateManager::create_coordinate_manager( dram_harvesting_mask, eth_grid_size, eth_cores, + eth_harvesting_mask, arc_grid_size, arc_cores, pcie_grid_size, @@ -559,6 +574,7 @@ std::shared_ptr CoordinateManager::create_coordinate_manager( dram_harvesting_mask, eth_grid_size, eth_cores, + eth_harvesting_mask, arc_grid_size, arc_cores, pcie_grid_size, diff --git a/device/grayskull/grayskull_coordinate_manager.cpp b/device/grayskull/grayskull_coordinate_manager.cpp index d474601f..a97e0ffb 100644 --- a/device/grayskull/grayskull_coordinate_manager.cpp +++ b/device/grayskull/grayskull_coordinate_manager.cpp @@ -16,6 +16,7 @@ GrayskullCoordinateManager::GrayskullCoordinateManager( const size_t dram_harvesting_mask, const tt_xy_pair& eth_grid_size, const std::vector& eth_cores, + const size_t eth_harvesting_mask, const tt_xy_pair& arc_grid_size, const std::vector& arc_cores, const tt_xy_pair& pcie_grid_size, @@ -29,6 +30,7 @@ GrayskullCoordinateManager::GrayskullCoordinateManager( dram_harvesting_mask, eth_grid_size, eth_cores, + eth_harvesting_mask, arc_grid_size, arc_cores, pcie_grid_size, diff --git a/device/tt_soc_descriptor.cpp b/device/tt_soc_descriptor.cpp index 0254bdb8..70d8ddda 100644 --- a/device/tt_soc_descriptor.cpp +++ b/device/tt_soc_descriptor.cpp @@ -182,7 +182,7 @@ tt_xy_pair tt_SocDescriptor::calculate_grid_size(const std::vector & } void tt_SocDescriptor::create_coordinate_manager( - const std::size_t tensix_harvesting_mask, const std::size_t dram_harvesting_mask) { + const size_t tensix_harvesting_mask, const size_t dram_harvesting_mask, const size_t eth_harvesting_mask) { const tt_xy_pair dram_grid_size = tt_xy_pair(dram_cores.size(), dram_cores.empty() ? 0 : dram_cores[0].size()); const tt_xy_pair arc_grid_size = tt_SocDescriptor::calculate_grid_size(arc_cores); const tt_xy_pair pcie_grid_size = tt_SocDescriptor::calculate_grid_size(pcie_cores); @@ -194,6 +194,7 @@ void tt_SocDescriptor::create_coordinate_manager( dram_cores_unpacked.push_back(core); } } + coordinate_manager = CoordinateManager::create_coordinate_manager( arch, worker_grid_size, @@ -204,6 +205,7 @@ void tt_SocDescriptor::create_coordinate_manager( dram_harvesting_mask, eth_grid_size, ethernet_cores, + eth_harvesting_mask, arc_grid_size, arc_cores, pcie_grid_size, @@ -218,8 +220,9 @@ tt::umd::CoreCoord tt_SocDescriptor::translate_coord_to( tt_SocDescriptor::tt_SocDescriptor( std::string device_descriptor_path, - const std::size_t tensix_harvesting_mask, - const std::size_t dram_harvesting_mask) { + const size_t tensix_harvesting_mask, + const size_t dram_harvesting_mask, + const size_t eth_harvesting_mask) { std::ifstream fdesc(device_descriptor_path); if (fdesc.fail()) { throw std::runtime_error( @@ -245,7 +248,7 @@ tt_SocDescriptor::tt_SocDescriptor( arch_name_value = trim(arch_name_value); arch = tt::arch_from_str(arch_name_value); load_soc_features_from_device_descriptor(device_descriptor_yaml); - create_coordinate_manager(tensix_harvesting_mask, dram_harvesting_mask); + create_coordinate_manager(tensix_harvesting_mask, dram_harvesting_mask, eth_harvesting_mask); } int tt_SocDescriptor::get_num_dram_channels() const { @@ -315,6 +318,9 @@ void tt_SocDescriptor::get_cores_and_grid_size_from_coordinate_manager() { harvested_cores_map.insert({CoreType::DRAM, coordinate_manager->get_harvested_cores(CoreType::DRAM)}); harvested_grid_size_map.insert({CoreType::DRAM, coordinate_manager->get_harvested_grid_size(CoreType::DRAM)}); + harvested_cores_map.insert({CoreType::ETH, coordinate_manager->get_harvested_cores(CoreType::ETH)}); + harvested_grid_size_map.insert({CoreType::ETH, coordinate_manager->get_harvested_grid_size(CoreType::ETH)}); + const std::vector dram_cores = cores_map.at(CoreType::DRAM); const tt_xy_pair dram_grid_size = grid_size_map.at(CoreType::DRAM); diff --git a/device/wormhole/wormhole_coordinate_manager.cpp b/device/wormhole/wormhole_coordinate_manager.cpp index 7873c692..872532c0 100644 --- a/device/wormhole/wormhole_coordinate_manager.cpp +++ b/device/wormhole/wormhole_coordinate_manager.cpp @@ -16,6 +16,7 @@ WormholeCoordinateManager::WormholeCoordinateManager( const size_t dram_harvesting_mask, const tt_xy_pair& eth_grid_size, const std::vector& eth_cores, + const size_t eth_harvesting_mask, const tt_xy_pair& arc_grid_size, const std::vector& arc_cores, const tt_xy_pair& pcie_grid_size, @@ -29,6 +30,7 @@ WormholeCoordinateManager::WormholeCoordinateManager( dram_harvesting_mask, eth_grid_size, eth_cores, + eth_harvesting_mask, arc_grid_size, arc_cores, pcie_grid_size, diff --git a/tests/api/test_core_coord_translation_bh.cpp b/tests/api/test_core_coord_translation_bh.cpp index b1b92991..9704b8d5 100644 --- a/tests/api/test_core_coord_translation_bh.cpp +++ b/tests/api/test_core_coord_translation_bh.cpp @@ -562,6 +562,54 @@ TEST(CoordinateManager, CoordinateManagerBlackholeETHTranslation) { } } +// Test ETH harvesting and coordinate translation for Blackhole. +TEST(CoordinateManager, CoordinateManagerBlackholeETHHarvesting) { + const size_t num_harvested_cores = 2; + const std::vector eth_cores = tt::umd::blackhole::ETH_CORES; + const tt_xy_pair eth_grid_size = tt::umd::blackhole::ETH_GRID_SIZE; + for (size_t harvesting_mask = 0; harvesting_mask < 1 << tt::umd::blackhole::DRAM_GRID_SIZE.x; harvesting_mask++) { + // We should have exactly 2 harvested ETH cores. + if (CoordinateManager::get_num_harvested(harvesting_mask) != num_harvested_cores) { + continue; + } + + std::shared_ptr coordinate_manager = + CoordinateManager::create_coordinate_manager(tt::ARCH::BLACKHOLE, 0, 0, harvesting_mask); + + size_t index = 0; + + for (size_t x = 0; x < eth_grid_size.x - num_harvested_cores; x++) { + for (size_t y = 0; y < eth_grid_size.y; y++) { + const CoreCoord eth_logical = CoreCoord(x, y, CoreType::ETH, CoordSystem::LOGICAL); + const CoreCoord eth_virtual = coordinate_manager->translate_coord_to(eth_logical, CoordSystem::VIRTUAL); + const CoreCoord eth_translated = + coordinate_manager->translate_coord_to(eth_logical, CoordSystem::TRANSLATED); + + EXPECT_EQ(eth_virtual.x, eth_cores[index].x); + EXPECT_EQ(eth_virtual.y, eth_cores[index].y); + + EXPECT_EQ(eth_translated.x, tt::umd::blackhole::eth_translated_coordinate_start_x + x); + EXPECT_EQ(eth_translated.y, tt::umd::blackhole::eth_translated_coordinate_start_y); + + index++; + } + } + + // Verify that translated coordinates for harvested cores are same as physical coordinates. + for (size_t x = 0; x < eth_grid_size.x; x++) { + if (harvesting_mask & (1 << x)) { + size_t index = x; + const CoreCoord physical_core = + CoreCoord(eth_cores[index].x, eth_cores[index].y, CoreType::ETH, CoordSystem::PHYSICAL); + const CoreCoord translated_core = + coordinate_manager->translate_coord_to(physical_core, CoordSystem::TRANSLATED); + EXPECT_EQ(translated_core.x, physical_core.x); + EXPECT_EQ(translated_core.y, physical_core.y); + } + } + } +} + // Test that we properly get harvesting mask that is based on the physical layout of the chip. TEST(CoordinateManager, CoordinateManagerBlackholePhysicalLayoutTensixHarvestingMask) { const size_t max_num_harvested_x = 14; diff --git a/tests/api/test_core_coord_translation_gs.cpp b/tests/api/test_core_coord_translation_gs.cpp index 1294c4f0..161d0ae9 100644 --- a/tests/api/test_core_coord_translation_gs.cpp +++ b/tests/api/test_core_coord_translation_gs.cpp @@ -309,6 +309,11 @@ TEST(CoordinateManager, CoordinateManagerGrayskullDRAMHarvestingAssert) { EXPECT_THROW(CoordinateManager::create_coordinate_manager(tt::ARCH::GRAYSKULL, 0, 1), std::runtime_error); } +// Test that we assert properly if ETH harvesting mask is non-zero for Grayskull. +TEST(CoordinateManager, CoordinateManagerGrayskullETHHarvestingAssert) { + EXPECT_THROW(CoordinateManager::create_coordinate_manager(tt::ARCH::GRAYSKULL, 0, 0, 1), std::runtime_error); +} + // Test that we properly get harvesting mask that is based on the physical layout of the chip. TEST(CoordinateManager, CoordinateManagerGrayskullPhysicalLayoutTensixHarvestingMask) { const size_t max_num_harvested_y = 10; diff --git a/tests/api/test_core_coord_translation_wh.cpp b/tests/api/test_core_coord_translation_wh.cpp index b9b4718a..248c5f28 100644 --- a/tests/api/test_core_coord_translation_wh.cpp +++ b/tests/api/test_core_coord_translation_wh.cpp @@ -364,6 +364,11 @@ TEST(CoordinateManager, CoordinateManagerWormholeDRAMHarvestingAssert) { EXPECT_THROW(CoordinateManager::create_coordinate_manager(tt::ARCH::WORMHOLE_B0, 0, 1), std::runtime_error); } +// Test that we assert properly if ETH harvesting mask is non-zero for Wormhole. +TEST(CoordinateManager, CoordinateManagerWormholeETHHarvestingAssert) { + EXPECT_THROW(CoordinateManager::create_coordinate_manager(tt::ARCH::WORMHOLE_B0, 0, 0, 1), std::runtime_error); +} + // Test that we properly get harvesting mask that is based on the physical layout of the chip. TEST(CoordinateManager, CoordinateManagerWormholePhysicalLayoutTensixHarvestingMask) { const size_t max_num_harvested_y = 10; diff --git a/tests/api/test_soc_descriptor.cpp b/tests/api/test_soc_descriptor.cpp index bb2ac3c3..aae40ee3 100644 --- a/tests/api/test_soc_descriptor.cpp +++ b/tests/api/test_soc_descriptor.cpp @@ -153,6 +153,44 @@ TEST(SocDescriptor, SocDescriptorWormholeETHLogicalToPhysical) { } } +// Test ETH translation from logical to physical coordinates. +TEST(SocDescriptor, SocDescriptorBlackholeETHHarvesting) { + const size_t num_eth_cores = tt::umd::blackhole::ETH_CORES.size(); + const size_t num_harvested_eth_cores = 2; + const tt_xy_pair eth_grid_size = tt::umd::blackhole::ETH_GRID_SIZE; + const std::vector blackhole_eth_cores = tt::umd::blackhole::ETH_CORES; + for (size_t eth_harvesting_mask = 0; eth_harvesting_mask < (1 << num_eth_cores); eth_harvesting_mask++) { + if (CoordinateManager::get_num_harvested(eth_harvesting_mask) != num_harvested_eth_cores) { + continue; + } + + tt_SocDescriptor soc_desc( + test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch.yaml"), 0, 0, eth_harvesting_mask); + + const std::vector eth_cores = soc_desc.get_cores(CoreType::ETH); + + EXPECT_EQ(eth_cores.size(), (eth_grid_size.x - num_harvested_eth_cores) * eth_grid_size.y); + + const std::vector harvested_eth_cores = soc_desc.get_harvested_cores(CoreType::ETH); + + EXPECT_EQ(harvested_eth_cores.size(), num_harvested_eth_cores); + + size_t index_harvested = 0; + size_t index_unharvested = 0; + for (size_t x = 0; x < eth_grid_size.x; x++) { + if (eth_harvesting_mask & (1 << x)) { + EXPECT_EQ(harvested_eth_cores[index_harvested].x, blackhole_eth_cores[x].x); + EXPECT_EQ(harvested_eth_cores[index_harvested].y, blackhole_eth_cores[x].y); + index_harvested++; + } else { + EXPECT_EQ(eth_cores[index_unharvested].x, blackhole_eth_cores[x].x); + EXPECT_EQ(eth_cores[index_unharvested].y, blackhole_eth_cores[x].y); + index_unharvested++; + } + } + } +} + // Test soc descriptor API for Blackhole when there is no harvesting. TEST(SocDescriptor, SocDescriptorBlackholeNoHarvesting) { tt_SocDescriptor soc_desc(test_utils::GetAbsPath("tests/soc_descs/blackhole_140_arch_no_eth.yaml"));