Skip to content

Commit

Permalink
Catch hugepage fails earlier, treat them as fatal
Browse files Browse the repository at this point in the history
  • Loading branch information
@joelsmithTT
joelsmithTT committed Dec 6, 2024
1 parent c03bac4 commit 5ea9963
Show file tree
Hide file tree
Showing 3 changed files with 35 additions and 30 deletions.
5 changes: 2 additions & 3 deletions device/api/umd/device/pci_device.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -197,16 +197,15 @@ class PCIDevice {
void detect_hang_read(uint32_t data_read = c_hang_read_value);

// TODO: this also probably has more sense to live in the future TTDevice class.
bool init_hugepage(uint32_t num_host_mem_channels);
void init_hugepage(uint32_t num_host_mem_channels);

/**
* Allocate sysmem without hugepages and map it through IOMMU.
* This is used when the system is protected by an IOMMU. The mappings will
* still appear as hugepages to the caller.
* @param size sysmem size in bytes; size % (1UL << 30) == 0
* @return whether allocation/mapping succeeded.
*/
bool init_iommu(size_t size);
void init_iommu(size_t size);

int get_num_host_mem_channels() const;
hugepage_mapping get_hugepage_mapping(int channel) const;
Expand Down
15 changes: 2 additions & 13 deletions device/cluster.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -267,24 +267,13 @@ void Cluster::create_device(
log_assert(
!(arch_name == tt::ARCH::BLACKHOLE && num_host_mem_channels > 1),
"More channels are not yet supported for Blackhole");
// Same number of host channels per device for now
bool hugepages_initialized = dev->init_hugepage(num_host_mem_channels);

if (!hugepages_initialized) {
log_warning(
LogSiliconDriver,
"Hugepages not initialized for device {} (logical_device_id: {} pci_interface_id: {})",
dev->get_device_num(),
logical_device_id,
pci_interface_id);
}
dev->init_hugepage(num_host_mem_channels);

// Large writes to remote chips require at least one hugepage.
bool no_hugepages = (dev->get_hugepage_mapping(0).mapping == nullptr);
if (target_remote_chips.size() && no_hugepages) {
log_assert(
hugepages_initialized,
"Hugepages must be successfully initialized if workload contains remote chips!");
log_assert(false, "Hugepages must be successfully initialized if workload contains remote chips!");
}
}
// translation layer for harvested coords. Default is identity map
Expand Down
45 changes: 31 additions & 14 deletions device/pcie/pci_device.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,22 @@ static const uint32_t GS_WH_ARC_SCRATCH_6_OFFSET = 0x1FF30078;
// Hugepages must be 1GB in size
const uint32_t HUGEPAGE_REGION_SIZE = 1 << 30; // 1GB

static const char *HUGEPAGE_FAIL_MSG =
"Failed to allocate or map an adequate quantity of 1GB hugepages. This is a critical error and will prevent the "
"application from functioning correctly. Please ensure the system has enough 1GB hugepages available and that the "
"application is requesting an appropriate number of hugepages per device. This per-device quantity is also "
"referred to as the number of host memory channels. Helpful files to examine for debugging are as follows\n"
" /proc/cmdline\n"
" /proc/meminfo\n"
" /proc/buddyinfo\n"
" /sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages\n"
" /sys/kernel/mm/hugepages/hugepages-1048576kB/free_hugepages\n"
" /sys/devices/system/node/node0/hugepages/hugepages-1048576kB/nr_hugepages\n"
" /sys/devices/system/node/node1/hugepages/hugepages-1048576kB/nr_hugepages\n"
" /sys/devices/system/node/node0/hugepages/hugepages-1048576kB/free_hugepages\n"
" /sys/devices/system/node/node1/hugepages/hugepages-1048576kB/free_hugepages\n"
"The application will now terminate.";

using namespace tt;
using namespace tt::umd;

Expand Down Expand Up @@ -705,7 +721,7 @@ tt::umd::architecture_implementation *PCIDevice::get_architecture_implementation
return architecture_implementation.get();
}

bool PCIDevice::init_hugepage(uint32_t num_host_mem_channels) {
void PCIDevice::init_hugepage(uint32_t num_host_mem_channels) {
const size_t hugepage_size = HUGEPAGE_REGION_SIZE;

if (numa_node > numa_max_node()) {
Expand All @@ -717,14 +733,14 @@ bool PCIDevice::init_hugepage(uint32_t num_host_mem_channels) {

if (is_iommu_enabled()) {
size_t size = hugepage_size * num_host_mem_channels;
return init_iommu(size);
init_iommu(size);
return;
}

auto physical_device_id = get_device_num();
std::string hugepage_dir = find_hugepage_dir(hugepage_size);
if (hugepage_dir.empty()) {
log_warning(LogSiliconDriver, "init_hugepage: no huge page mount found for hugepage_size: {}.", hugepage_size);
return false;
log_fatal("init_hugepage: no huge page mount found for hugepage_size: {}.", hugepage_size);
}

bool success = true;
Expand All @@ -742,7 +758,7 @@ bool PCIDevice::init_hugepage(uint32_t num_host_mem_channels) {
physical_device_id,
ch);
success = false;
continue;
break;
}

// Verify opened file size.
Expand All @@ -759,10 +775,10 @@ bool PCIDevice::init_hugepage(uint32_t num_host_mem_channels) {
if (mapping == MAP_FAILED) {
log_warning(
LogSiliconDriver,
"Mapping a hugepage failed. (device: {}, channel {}/{} errno: {}).",
"Mapping a hugepage failed. (device: {}, channel {}/{}, errno: {}).",
physical_device_id,
ch,
num_host_mem_channels - 1,
ch + 1,
num_host_mem_channels,
strerror(errno));
if (hugepage_st.st_size == 0) {
log_warning(
Expand All @@ -774,7 +790,7 @@ bool PCIDevice::init_hugepage(uint32_t num_host_mem_channels) {
print_file_contents(
"/sys/kernel/mm/hugepages/hugepages-1048576kB/nr_hugepages"); // Hardcoded for 1GB hugepage.
success = false;
continue;
break;
}

tenstorrent_pin_pages pin_pages;
Expand All @@ -793,7 +809,7 @@ bool PCIDevice::init_hugepage(uint32_t num_host_mem_channels) {
print_file_contents("/proc/meminfo");
print_file_contents("/proc/buddyinfo");
success = false;
continue;
break;
}

hugepage_mapping_per_channel[ch] = {mapping, hugepage_size, pin_pages.out.physical_address};
Expand All @@ -807,10 +823,13 @@ bool PCIDevice::init_hugepage(uint32_t num_host_mem_channels) {
(unsigned long long)hugepage_mappings.at(device_id).at(ch).physical_address);
}

return success;
if (!success) {
log_error(HUGEPAGE_FAIL_MSG);
std::terminate();
}
}

bool PCIDevice::init_iommu(size_t size) {
void PCIDevice::init_iommu(size_t size) {
const size_t num_fake_mem_channels = size / HUGEPAGE_REGION_SIZE;

if (!is_iommu_enabled()) {
Expand All @@ -837,8 +856,6 @@ bool PCIDevice::init_iommu(size_t size) {
uint8_t *base = static_cast<uint8_t *>(mapping) + ch * HUGEPAGE_REGION_SIZE;
hugepage_mapping_per_channel[ch] = {base, HUGEPAGE_REGION_SIZE, iova + ch * HUGEPAGE_REGION_SIZE};
}

return true;
}

int PCIDevice::get_num_host_mem_channels() const { return hugepage_mapping_per_channel.size(); }
Expand Down

0 comments on commit 5ea9963

Please sign in to comment.