From 9b83729f7c45d6ce26e50587bb672f3e089a5d9b Mon Sep 17 00:00:00 2001 From: NicoAvanzDev <35104310+NicoAvanzDev@users.noreply.github.com> Date: Wed, 2 Oct 2024 13:39:34 +0200 Subject: [PATCH 1/4] RandomAccess.Write throws System.IO.IOException Fix (#108380) --- .../src/System/IO/RandomAccess.Windows.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs index e5c8e17da23e9e..6439cf4b531e59 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs @@ -437,7 +437,7 @@ internal static long ReadScatterAtOffset(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset) { // WriteFileGather does not support sync handles, so we just call WriteFile in a loop - int bytesWritten = 0; + long bytesWritten = 0; int buffersCount = buffers.Count; for (int i = 0; i < buffersCount; i++) { From e14c2f355c5b051326dc34b8cfe3572d5c866d24 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 8 Nov 2024 16:48:13 +0100 Subject: [PATCH 2/4] Handle IOV_MAX limit in RandomAccess, add missing test (#109340) * add test for Int32 overflow for WriteGather in RandomAccess * add failing test fore more than IOV_MAX buffers * fix both the native and managed parts --------- Co-authored-by: Adeel Mujahid <3840695+am11@users.noreply.github.com> Co-authored-by: Stephen Toub --- .../tests/RandomAccess/WriteGatherAsync.cs | 126 ++++++++++++++++++ .../src/System/IO/RandomAccess.Unix.cs | 7 +- src/native/libs/System.Native/pal_io.c | 24 +++- 3 files changed, 153 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs index abf945b2d46bcc..843cf942f42c7b 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs @@ -6,6 +6,7 @@ using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; +using Microsoft.DotNet.XUnitExtensions; using Microsoft.Win32.SafeHandles; using Xunit; @@ -133,5 +134,130 @@ public async Task DuplicatedBufferDuplicatesContentAsync(FileOptions options) Assert.Equal(repeatCount, actualContent.Length); Assert.All(actualContent, actual => Assert.Equal(value, actual)); } + + [OuterLoop("It consumes a lot of resources (disk space and memory).")] + [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.Is64BitProcess), nameof(PlatformDetection.IsReleaseRuntime))] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, true)] + [InlineData(true, false)] + public async Task NoInt32OverflowForLargeInputs(bool asyncFile, bool asyncMethod) + { + // We need to write more than Int32.MaxValue bytes to the disk to reproduce the problem. + // To reduce the number of used memory, we allocate only one write buffer and simply repeat it multiple times. + // For reading, we need unique buffers to ensure that all of them are getting populated with the right data. + + const int BufferCount = 1002; + const int BufferSize = int.MaxValue / 1000; + const long FileSize = (long)BufferCount * BufferSize; + string filePath = GetTestFilePath(); + ReadOnlyMemory writeBuffer = RandomNumberGenerator.GetBytes(BufferSize); + List> writeBuffers = Enumerable.Repeat(writeBuffer, BufferCount).ToList(); + List> readBuffers = new List>(BufferCount); + + try + { + for (int i = 0; i < BufferCount; i++) + { + readBuffers.Add(new byte[BufferSize]); + } + } + catch (OutOfMemoryException) + { + throw new SkipTestException("Not enough memory."); + } + + FileOptions options = asyncFile ? FileOptions.Asynchronous : FileOptions.None; // we need to test both code paths + options |= FileOptions.DeleteOnClose; + + SafeFileHandle? sfh; + try + { + sfh = File.OpenHandle(filePath, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, options, preallocationSize: FileSize); + } + catch (IOException) + { + throw new SkipTestException("Not enough disk space."); + } + + long fileOffset = 0, bytesRead = 0; + try + { + if (asyncMethod) + { + await RandomAccess.WriteAsync(sfh, writeBuffers, fileOffset); + bytesRead = await RandomAccess.ReadAsync(sfh, readBuffers, fileOffset); + } + else + { + RandomAccess.Write(sfh, writeBuffers, fileOffset); + bytesRead = RandomAccess.Read(sfh, readBuffers, fileOffset); + } + } + finally + { + sfh.Dispose(); // delete the file ASAP to avoid running out of resources in CI + } + + Assert.Equal(FileSize, bytesRead); + for (int i = 0; i < BufferCount; i++) + { + Assert.Equal(writeBuffer, readBuffers[i]); + } + } + + [Theory] + [InlineData(false, false)] + [InlineData(false, true)] + [InlineData(true, true)] + [InlineData(true, false)] + public async Task IovLimitsAreRespected(bool asyncFile, bool asyncMethod) + { + // We need to write and read more than IOV_MAX buffers at a time. + // IOV_MAX typical value is 1024. + const int BufferCount = 1026; + const int BufferSize = 1; // the less resources we use, the better + const int FileSize = BufferCount * BufferSize; + + ReadOnlyMemory writeBuffer = RandomNumberGenerator.GetBytes(BufferSize); + ReadOnlyMemory[] writeBuffers = Enumerable.Repeat(writeBuffer, BufferCount).ToArray(); + Memory[] readBuffers = Enumerable.Range(0, BufferCount).Select(_ => new byte[BufferSize].AsMemory()).ToArray(); + + FileOptions options = asyncFile ? FileOptions.Asynchronous : FileOptions.None; // we need to test both code paths + options |= FileOptions.DeleteOnClose; + + using SafeFileHandle sfh = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.ReadWrite, FileShare.None, options); + + if (asyncMethod) + { + await RandomAccess.WriteAsync(sfh, writeBuffers, 0); + } + else + { + RandomAccess.Write(sfh, writeBuffers, 0); + } + + Assert.Equal(FileSize, RandomAccess.GetLength(sfh)); + + long fileOffset = 0; + int bufferOffset = 0; + while (fileOffset < FileSize) + { + ArraySegment> left = new ArraySegment>(readBuffers, bufferOffset, readBuffers.Length - bufferOffset); + + long bytesRead = asyncMethod + ? await RandomAccess.ReadAsync(sfh, left, fileOffset) + : RandomAccess.Read(sfh, left, fileOffset); + + fileOffset += bytesRead; + // The following operation is correct only because the BufferSize is 1. + bufferOffset += (int)bytesRead; + } + + for (int i = 0; i < BufferCount; ++i) + { + Assert.Equal(writeBuffers[i], readBuffers[i]); + } + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs index e06187b5a3de54..bcddd3f4654fc1 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs @@ -196,9 +196,10 @@ internal static unsafe void WriteGatherAtOffset(SafeFileHandle handle, IReadOnly } long bytesWritten; - fixed (Interop.Sys.IOVector* pinnedVectors = &MemoryMarshal.GetReference(vectors)) + Span left = vectors.Slice(buffersOffset); + fixed (Interop.Sys.IOVector* pinnedVectors = &MemoryMarshal.GetReference(left)) { - bytesWritten = Interop.Sys.PWriteV(handle, pinnedVectors, buffersCount, fileOffset); + bytesWritten = Interop.Sys.PWriteV(handle, pinnedVectors, buffersCount - buffersOffset, fileOffset); } FileStreamHelpers.CheckFileCall(bytesWritten, handle.Path); @@ -208,6 +209,8 @@ internal static unsafe void WriteGatherAtOffset(SafeFileHandle handle, IReadOnly } // The write completed successfully but for fewer bytes than requested. + // We need to perform next write where the previous one has finished. + fileOffset += bytesWritten; // We need to try again for the remainder. for (int i = 0; i < buffersCount; i++) { diff --git a/src/native/libs/System.Native/pal_io.c b/src/native/libs/System.Native/pal_io.c index d6d4a84e1faeb2..36e1510dcab8fa 100644 --- a/src/native/libs/System.Native/pal_io.c +++ b/src/native/libs/System.Native/pal_io.c @@ -1855,6 +1855,24 @@ int32_t SystemNative_PWrite(intptr_t fd, void* buffer, int32_t bufferSize, int64 return (int32_t)count; } +#if (HAVE_PREADV || HAVE_PWRITEV) && !defined(TARGET_WASM) +static int GetAllowedVectorCount(int32_t vectorCount) +{ + int allowedCount = (int)vectorCount; + +#if defined(IOV_MAX) + if (IOV_MAX < allowedCount) + { + // We need to respect the limit of items that can be passed in iov. + // In case of writes, the managed code is reponsible for handling incomplete writes. + // In case of reads, we simply returns the number of bytes read and it's up to the users. + allowedCount = IOV_MAX; + } +#endif + return allowedCount; +} +#endif // (HAVE_PREADV || HAVE_PWRITEV) && !defined(TARGET_WASM) + int64_t SystemNative_PReadV(intptr_t fd, IOVector* vectors, int32_t vectorCount, int64_t fileOffset) { assert(vectors != NULL); @@ -1863,7 +1881,8 @@ int64_t SystemNative_PReadV(intptr_t fd, IOVector* vectors, int32_t vectorCount, int64_t count = 0; int fileDescriptor = ToFileDescriptor(fd); #if HAVE_PREADV && !defined(TARGET_WASM) // preadv is buggy on WASM - while ((count = preadv(fileDescriptor, (struct iovec*)vectors, (int)vectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); + int allowedVectorCount = GetAllowedVectorCount(vectorCount); + while ((count = preadv(fileDescriptor, (struct iovec*)vectors, allowedVectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); #else int64_t current; for (int i = 0; i < vectorCount; i++) @@ -1903,7 +1922,8 @@ int64_t SystemNative_PWriteV(intptr_t fd, IOVector* vectors, int32_t vectorCount int64_t count = 0; int fileDescriptor = ToFileDescriptor(fd); #if HAVE_PWRITEV && !defined(TARGET_WASM) // pwritev is buggy on WASM - while ((count = pwritev(fileDescriptor, (struct iovec*)vectors, (int)vectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); + int allowedVectorCount = GetAllowedVectorCount(vectorCount); + while ((count = pwritev(fileDescriptor, (struct iovec*)vectors, allowedVectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); #else int64_t current; for (int i = 0; i < vectorCount; i++) From 5b060b9d370ccb065485114021cff4eb02e34581 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 13 Nov 2024 13:34:46 +0100 Subject: [PATCH 3/4] fix the tests: use AssertExtensions.SequenceEqual to compare the contents of ROM --- .../tests/RandomAccess/WriteGatherAsync.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs index 843cf942f42c7b..1beb28088fbfa6 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs @@ -202,7 +202,7 @@ public async Task NoInt32OverflowForLargeInputs(bool asyncFile, bool asyncMethod Assert.Equal(FileSize, bytesRead); for (int i = 0; i < BufferCount; i++) { - Assert.Equal(writeBuffer, readBuffers[i]); + AssertExtensions.SequenceEqual(writeBuffer.Span, readBuffers[i].Span); } } @@ -256,7 +256,7 @@ public async Task IovLimitsAreRespected(bool asyncFile, bool asyncMethod) for (int i = 0; i < BufferCount; ++i) { - Assert.Equal(writeBuffers[i], readBuffers[i]); + AssertExtensions.SequenceEqual(writeBuffers[i].Span, readBuffers[i].Span); } } } From d6843d9c42872af05831dcea9911a7bf0ffdfa38 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 12 Dec 2024 20:22:36 +0100 Subject: [PATCH 4/4] More WriteGather fixes (#109826) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * don't run these tests in parallel, as each test cases uses more than 4 GB ram and disk! * fix the test: handle incomplete reads that should happen when we hit the max buffer limit * incomplete write fix: - pin the buffers only once - when re-trying, do that only for the actual reminder * Use native memory to get OOM a soon as we run out of memory (hoping to avoid the process getting killed on Linux when OOM happens) * For macOS preadv and pwritev can fail with EINVAL when the total length of all vectors overflows a 32-bit integer. * add an assert that is going to warn us if vector.Count is ever more than Int32.MaxValue --------- Co-authored-by: Michał Petryka <35800402+MichalPetryka@users.noreply.github.com> # Conflicts: # src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs --- .../tests/RandomAccess/WriteGatherAsync.cs | 104 +++++++++++++----- .../src/System/IO/RandomAccess.Unix.cs | 57 +++++----- src/native/libs/System.Native/pal_io.c | 47 ++++++-- 3 files changed, 140 insertions(+), 68 deletions(-) diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs index 1beb28088fbfa6..50752f52b5708e 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Buffers; using System.Collections.Generic; using System.Linq; using System.Security.Cryptography; @@ -13,6 +14,7 @@ namespace System.IO.Tests { [SkipOnPlatform(TestPlatforms.Browser, "async file IO is not supported on browser")] + [Collection(nameof(DisableParallelization))] // don't run in parallel, as some of these tests use a LOT of resources public class RandomAccess_WriteGatherAsync : RandomAccess_Base { protected override ValueTask MethodUnderTest(SafeFileHandle handle, byte[] bytes, long fileOffset) @@ -151,21 +153,6 @@ public async Task NoInt32OverflowForLargeInputs(bool asyncFile, bool asyncMethod const int BufferSize = int.MaxValue / 1000; const long FileSize = (long)BufferCount * BufferSize; string filePath = GetTestFilePath(); - ReadOnlyMemory writeBuffer = RandomNumberGenerator.GetBytes(BufferSize); - List> writeBuffers = Enumerable.Repeat(writeBuffer, BufferCount).ToList(); - List> readBuffers = new List>(BufferCount); - - try - { - for (int i = 0; i < BufferCount; i++) - { - readBuffers.Add(new byte[BufferSize]); - } - } - catch (OutOfMemoryException) - { - throw new SkipTestException("Not enough memory."); - } FileOptions options = asyncFile ? FileOptions.Asynchronous : FileOptions.None; // we need to test both code paths options |= FileOptions.DeleteOnClose; @@ -180,29 +167,86 @@ public async Task NoInt32OverflowForLargeInputs(bool asyncFile, bool asyncMethod throw new SkipTestException("Not enough disk space."); } - long fileOffset = 0, bytesRead = 0; - try + using (sfh) { - if (asyncMethod) + ReadOnlyMemory writeBuffer = RandomNumberGenerator.GetBytes(BufferSize); + List> writeBuffers = Enumerable.Repeat(writeBuffer, BufferCount).ToList(); + + List memoryManagers = new List(BufferCount); + List> readBuffers = new List>(BufferCount); + + try { - await RandomAccess.WriteAsync(sfh, writeBuffers, fileOffset); - bytesRead = await RandomAccess.ReadAsync(sfh, readBuffers, fileOffset); + try + { + for (int i = 0; i < BufferCount; i++) + { + // We are using native memory here to get OOM as soon as possible. + NativeMemoryManager nativeMemoryManager = new(BufferSize); + memoryManagers.Add(nativeMemoryManager); + readBuffers.Add(nativeMemoryManager.Memory); + } + } + catch (OutOfMemoryException) + { + throw new SkipTestException("Not enough memory."); + } + + await Verify(asyncMethod, FileSize, sfh, writeBuffer, writeBuffers, readBuffers); } - else + finally { - RandomAccess.Write(sfh, writeBuffers, fileOffset); - bytesRead = RandomAccess.Read(sfh, readBuffers, fileOffset); + foreach (IDisposable memoryManager in memoryManagers) + { + memoryManager.Dispose(); + } } } - finally - { - sfh.Dispose(); // delete the file ASAP to avoid running out of resources in CI - } - Assert.Equal(FileSize, bytesRead); - for (int i = 0; i < BufferCount; i++) + static async Task Verify(bool asyncMethod, long FileSize, SafeFileHandle sfh, ReadOnlyMemory writeBuffer, List> writeBuffers, List> readBuffers) { - AssertExtensions.SequenceEqual(writeBuffer.Span, readBuffers[i].Span); + if (asyncMethod) + { + await RandomAccess.WriteAsync(sfh, writeBuffers, 0); + } + else + { + RandomAccess.Write(sfh, writeBuffers, 0); + } + + Assert.Equal(FileSize, RandomAccess.GetLength(sfh)); + + long fileOffset = 0; + while (fileOffset < FileSize) + { + long bytesRead = asyncMethod + ? await RandomAccess.ReadAsync(sfh, readBuffers, fileOffset) + : RandomAccess.Read(sfh, readBuffers, fileOffset); + + Assert.InRange(bytesRead, 0, FileSize); + + while (bytesRead > 0) + { + Memory readBuffer = readBuffers[0]; + if (bytesRead >= readBuffer.Length) + { + AssertExtensions.SequenceEqual(writeBuffer.Span, readBuffer.Span); + + bytesRead -= readBuffer.Length; + fileOffset += readBuffer.Length; + + readBuffers.RemoveAt(0); + } + else + { + // A read has finished somewhere in the middle of one of the read buffers. + // Example: buffer had 30 bytes and only 10 were read. + // We don't read the missing part, but try to read the whole buffer again. + // It's not optimal from performance perspective, but it keeps the test logic simple. + break; + } + } + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs index bcddd3f4654fc1..147e3815812d0c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs @@ -168,38 +168,30 @@ internal static unsafe void WriteGatherAtOffset(SafeFileHandle handle, IReadOnly var handles = new MemoryHandle[buffersCount]; Span vectors = buffersCount <= IovStackThreshold ? - stackalloc Interop.Sys.IOVector[IovStackThreshold] : + stackalloc Interop.Sys.IOVector[IovStackThreshold].Slice(0, buffersCount) : new Interop.Sys.IOVector[buffersCount]; try { - int buffersOffset = 0, firstBufferOffset = 0; - while (true) + long totalBytesToWrite = 0; + for (int i = 0; i < buffersCount; i++) { - long totalBytesToWrite = 0; - - for (int i = buffersOffset; i < buffersCount; i++) - { - ReadOnlyMemory buffer = buffers[i]; - totalBytesToWrite += buffer.Length; - - MemoryHandle memoryHandle = buffer.Pin(); - vectors[i] = new Interop.Sys.IOVector { Base = firstBufferOffset + (byte*)memoryHandle.Pointer, Count = (UIntPtr)(buffer.Length - firstBufferOffset) }; - handles[i] = memoryHandle; - - firstBufferOffset = 0; - } + ReadOnlyMemory buffer = buffers[i]; + totalBytesToWrite += buffer.Length; - if (totalBytesToWrite == 0) - { - break; - } + MemoryHandle memoryHandle = buffer.Pin(); + vectors[i] = new Interop.Sys.IOVector { Base = (byte*)memoryHandle.Pointer, Count = (UIntPtr)buffer.Length }; + handles[i] = memoryHandle; + } + int buffersOffset = 0; + while (totalBytesToWrite > 0) + { long bytesWritten; Span left = vectors.Slice(buffersOffset); fixed (Interop.Sys.IOVector* pinnedVectors = &MemoryMarshal.GetReference(left)) { - bytesWritten = Interop.Sys.PWriteV(handle, pinnedVectors, buffersCount - buffersOffset, fileOffset); + bytesWritten = Interop.Sys.PWriteV(handle, pinnedVectors, left.Length, fileOffset); } FileStreamHelpers.CheckFileCall(bytesWritten, handle.Path); @@ -211,22 +203,29 @@ internal static unsafe void WriteGatherAtOffset(SafeFileHandle handle, IReadOnly // The write completed successfully but for fewer bytes than requested. // We need to perform next write where the previous one has finished. fileOffset += bytesWritten; + totalBytesToWrite -= bytesWritten; // We need to try again for the remainder. - for (int i = 0; i < buffersCount; i++) + while (buffersOffset < buffersCount && bytesWritten > 0) { - int n = buffers[i].Length; + int n = (int)vectors[buffersOffset].Count; if (n <= bytesWritten) { - buffersOffset++; bytesWritten -= n; - if (bytesWritten == 0) - { - break; - } + buffersOffset++; } else { - firstBufferOffset = (int)(bytesWritten - n); + // A partial read: the vector needs to point to the new offset. + // But that offset needs to be relative to the previous attempt. + // Example: we have a single buffer with 30 bytes and the first read returned 10. + // The next read should try to read the remaining 20 bytes, but in case it also reads just 10, + // the third attempt should read last 10 bytes (not 20 again). + Interop.Sys.IOVector current = vectors[buffersOffset]; + vectors[buffersOffset] = new Interop.Sys.IOVector + { + Base = current.Base + (int)(bytesWritten), + Count = current.Count - (UIntPtr)(bytesWritten) + }; break; } } diff --git a/src/native/libs/System.Native/pal_io.c b/src/native/libs/System.Native/pal_io.c index 36e1510dcab8fa..35858794dff1af 100644 --- a/src/native/libs/System.Native/pal_io.c +++ b/src/native/libs/System.Native/pal_io.c @@ -1856,19 +1856,48 @@ int32_t SystemNative_PWrite(intptr_t fd, void* buffer, int32_t bufferSize, int64 } #if (HAVE_PREADV || HAVE_PWRITEV) && !defined(TARGET_WASM) -static int GetAllowedVectorCount(int32_t vectorCount) +static int GetAllowedVectorCount(IOVector* vectors, int32_t vectorCount) { +#if defined(IOV_MAX) + const int IovMax = IOV_MAX; +#else + // In theory all the platforms that we support define IOV_MAX, + // but we want to be extra safe and provde a fallback + // in case it turns out to not be true. + // 16 is low, but supported on every platform. + const int IovMax = 16; +#endif + int allowedCount = (int)vectorCount; -#if defined(IOV_MAX) - if (IOV_MAX < allowedCount) + // We need to respect the limit of items that can be passed in iov. + // In case of writes, the managed code is responsible for handling incomplete writes. + // In case of reads, we simply returns the number of bytes read and it's up to the users. + if (IovMax < allowedCount) { - // We need to respect the limit of items that can be passed in iov. - // In case of writes, the managed code is reponsible for handling incomplete writes. - // In case of reads, we simply returns the number of bytes read and it's up to the users. - allowedCount = IOV_MAX; + allowedCount = IovMax; } + +#if defined(TARGET_APPLE) + // For macOS preadv and pwritev can fail with EINVAL when the total length + // of all vectors overflows a 32-bit integer. + size_t totalLength = 0; + for (int i = 0; i < allowedCount; i++) + { + assert(INT_MAX >= vectors[i].Count); + + totalLength += vectors[i].Count; + + if (totalLength > INT_MAX) + { + allowedCount = i; + break; + } + } +#else + (void)vectors; #endif + return allowedCount; } #endif // (HAVE_PREADV || HAVE_PWRITEV) && !defined(TARGET_WASM) @@ -1881,7 +1910,7 @@ int64_t SystemNative_PReadV(intptr_t fd, IOVector* vectors, int32_t vectorCount, int64_t count = 0; int fileDescriptor = ToFileDescriptor(fd); #if HAVE_PREADV && !defined(TARGET_WASM) // preadv is buggy on WASM - int allowedVectorCount = GetAllowedVectorCount(vectorCount); + int allowedVectorCount = GetAllowedVectorCount(vectors, vectorCount); while ((count = preadv(fileDescriptor, (struct iovec*)vectors, allowedVectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); #else int64_t current; @@ -1922,7 +1951,7 @@ int64_t SystemNative_PWriteV(intptr_t fd, IOVector* vectors, int32_t vectorCount int64_t count = 0; int fileDescriptor = ToFileDescriptor(fd); #if HAVE_PWRITEV && !defined(TARGET_WASM) // pwritev is buggy on WASM - int allowedVectorCount = GetAllowedVectorCount(vectorCount); + int allowedVectorCount = GetAllowedVectorCount(vectors, vectorCount); while ((count = pwritev(fileDescriptor, (struct iovec*)vectors, allowedVectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); #else int64_t current;