Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove some allocations under HttpFileSystemBasedFindPackageByIdResource.ConsumeFlatContainerIndexAsync #6265

Closed
wants to merge 7 commits into from
Closed
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 40 additions & 0 deletions build/Shared/Utf8JsonReaderExtensions.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.

using System;
using System.Text.Json;

namespace NuGet
{
internal static class Utf8JsonReaderExtensions
{
internal static string ReadTokenAsString(this ref Utf8JsonReader reader)
{
switch (reader.TokenType)
{
case JsonTokenType.True:
return bool.TrueString;
case JsonTokenType.False:
return bool.FalseString;
case JsonTokenType.Number:
return reader.ReadNumberAsString();
case JsonTokenType.String:
return reader.GetString();
case JsonTokenType.None:
case JsonTokenType.Null:
return null;
default:
throw new InvalidCastException();
}
}

private static string ReadNumberAsString(this ref Utf8JsonReader reader)
{
if (reader.TryGetInt64(out long value))
{
return value.ToString();
}
return reader.GetDouble().ToString();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,12 @@
using System.Linq;
using System.Text;
using System.Text.Json;
using NuGet.Common;

namespace NuGet.ProjectModel
namespace NuGet
{
/// <summary>
/// This struct is used to read over a memeory stream in parts, in order to avoid reading the entire stream into memory.
/// This struct is used to read over a memory stream in parts, in order to avoid reading the entire stream into memory.
/// It functions as a wrapper around <see cref="Utf8JsonStreamReader"/>, while maintaining a stream and a buffer to read from.
/// </summary>
internal ref struct Utf8JsonStreamReader
Expand Down Expand Up @@ -109,6 +110,17 @@ internal void Skip()
}
}

internal void ProcessStringArray<TArg>(Action<string, TArg> callback, TArg arg)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is only used in one place, and where it was used, I couldn't guess how the callback worked without looking at this code. So, I'm not sure it's a good candidate for a shared method.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The calling code was already nested fairly deeply, and this seemed to be somewhat useful, but I don't mind removing it.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Note that as part of this, I added ReadTokenAsString as this class doesn't expose _reader and the existing ReadNextTokenAsString doesn't quite fix the caller's needs.

{
if (TokenType == JsonTokenType.StartArray)
{
while (Read() && TokenType != JsonTokenType.EndArray)
{
callback(_reader.ReadTokenAsString(), arg);
}
}
}

internal IList<T> ReadObjectAsList<T>(IUtf8JsonStreamReaderConverter<T> streamReaderConverter)
{
if (TokenType == JsonTokenType.Null)
Expand Down Expand Up @@ -234,7 +246,7 @@ internal bool ReadNextTokenAsBoolOrThrowAnException(byte[] propertyName)
{
throw new ArgumentException(
string.Format(CultureInfo.CurrentCulture,
Strings.Invalid_AttributeValue,
PublicStrings.Invalid_AttributeValue,
Encoding.UTF8.GetString(propertyName),
_reader.ReadTokenAsString(),
"false"));
Expand Down
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
// Copyright (c) .NET Foundation. All rights reserved.
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
namespace NuGet.ProjectModel

namespace NuGet
{
/// <summary>
/// An abstract class that defines a function for reading a <see cref="Utf8JsonStreamReader"/> into a <typeparamref name="T"/>
Expand Down
6 changes: 0 additions & 6 deletions src/NuGet.Core/NuGet.Commands/Strings.resx
Original file line number Diff line number Diff line change
Expand Up @@ -1117,12 +1117,6 @@ NuGet requires HTTPS sources. Refer to https://aka.ms/nuget-https-everywhere for
NuGet requires HTTPS sources. Refer to https://aka.ms/nuget-https-everywhere for more information.</value>
<comment>{0} - list of server URIs</comment>
</data>
<data name="Invalid_AttributeValue" xml:space="preserve">
<value>The {0} value: `{1}` is invalid. Valid example: `{2}`</value>
<comment>{0} - is the attribute
{1} is the invalid value of the attribute.
{2} a valid attriibute value example</comment>
</data>
<data name="Warning_AuditSourceWithoutVulnerabilityData" xml:space="preserve">
<value>Audit source '{0}' did not provide any vulnerability data.</value>
<comment>{0} is the source name</comment>
Expand Down
12 changes: 11 additions & 1 deletion src/NuGet.Core/NuGet.Common/NuGet.Common.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
<TargetFramework>$(NETFXTargetFramework)</TargetFramework>
</PropertyGroup>

<ItemGroup Condition=" '$(TargetFramework)' == '$(NETFXTargetFramework)' " >
<ItemGroup Condition=" '$(TargetFramework)' == '$(NETFXTargetFramework)' ">
<Reference Include="System.Core" />
<Reference Include="System.IO.Compression" />
<Reference Include="mscorlib" />
Expand All @@ -31,6 +31,11 @@
</ItemGroup>

<ItemGroup>
<Compile Update="PublicStrings.Designer.cs">
<DesignTime>True</DesignTime>
<AutoGen>True</AutoGen>
<DependentUpon>PublicStrings.resx</DependentUpon>
</Compile>
<Compile Update="Strings.Designer.cs">
<DesignTime>True</DesignTime>
<AutoGen>True</AutoGen>
Expand All @@ -48,6 +53,11 @@
</ItemGroup>

<ItemGroup>
<EmbeddedResource Update="PublicStrings.resx">
<Generator>PublicResXFileCodeGenerator</Generator>
<PublicClass>true</PublicClass>
<LastGenOutput>PublicStrings.Designer.cs</LastGenOutput>
</EmbeddedResource>
<EmbeddedResource Update="Strings.resx">
<Generator>ResXFileCodeGenerator</Generator>
<LastGenOutput>Strings.Designer.cs</LastGenOutput>
Expand Down
5 changes: 5 additions & 0 deletions src/NuGet.Core/NuGet.Common/PublicAPI.Unshipped.txt
Original file line number Diff line number Diff line change
@@ -1 +1,6 @@
#nullable enable
NuGet.Common.PublicStrings
~static NuGet.Common.PublicStrings.Culture.get -> System.Globalization.CultureInfo
~static NuGet.Common.PublicStrings.Culture.set -> void
~static NuGet.Common.PublicStrings.Invalid_AttributeValue.get -> string
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is just my personal opinion, but I don't like resource strings as public APIs. There's too much risk of runtime failures if new {x} parameters are added to the string, and callers of the API don't update their string.Format argument list. We've even had this type of failure within the repo, so it'll be much worse if anyone using the NuGet Client SDK packages in their own app use this string for some reason.

I don't think that duplicating the strings, and keeping them internal, in multiple assemblies is bad enough to prefer public APIs.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

See https://github.com/NuGet/NuGet.Client/pull/6265/files#r1970768798 for one complexity of not having these as public.

I totally understand its unusual here, but these are the only options I could come up with:

  1. Have a new .resx in any assembly that needs shared code under a common namespace so that the shared code can compile against it.
  2. Hardcode this string and not have it localized at all
  3. Have a set of public strings in NuGet.Common.dll that we are confident won't change and are pretty generic (this one for example is just telling a user that an attribute value is invalid).
  4. InternalsVisibleTo/friend assembly. I wish you could grant friend access to individual classes...
  5. Make Utf8JsonStreamReader public so that our assemblies can share it
  6. Refactor entire assemblies and combine them
  7. Other?

In this case, it seemed like the least worst option.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I want 6, but let's wait on that until we have a major version.

I'd prefer 1

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Okay I've pushed an update to use a shared resource that gets compiled into each assembly that needs it. At least the string is only defined in one place.

~static NuGet.Common.PublicStrings.ResourceManager.get -> System.Resources.ResourceManager
72 changes: 72 additions & 0 deletions src/NuGet.Core/NuGet.Common/PublicStrings.Designer.cs

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

126 changes: 126 additions & 0 deletions src/NuGet.Core/NuGet.Common/PublicStrings.resx
Original file line number Diff line number Diff line change
@@ -0,0 +1,126 @@
<?xml version="1.0" encoding="utf-8"?>
<root>
<!--
Microsoft ResX Schema

Version 2.0

The primary goals of this format is to allow a simple XML format
that is mostly human readable. The generation and parsing of the
various data types are done through the TypeConverter classes
associated with the data types.

Example:

... ado.net/XML headers & schema ...
<resheader name="resmimetype">text/microsoft-resx</resheader>
<resheader name="version">2.0</resheader>
<resheader name="reader">System.Resources.ResXResourceReader, System.Windows.Forms, ...</resheader>
<resheader name="writer">System.Resources.ResXResourceWriter, System.Windows.Forms, ...</resheader>
<data name="Name1"><value>this is my long string</value><comment>this is a comment</comment></data>
<data name="Color1" type="System.Drawing.Color, System.Drawing">Blue</data>
<data name="Bitmap1" mimetype="application/x-microsoft.net.object.binary.base64">
<value>[base64 mime encoded serialized .NET Framework object]</value>
</data>
<data name="Icon1" type="System.Drawing.Icon, System.Drawing" mimetype="application/x-microsoft.net.object.bytearray.base64">
<value>[base64 mime encoded string representing a byte array form of the .NET Framework object]</value>
<comment>This is a comment</comment>
</data>

There are any number of "resheader" rows that contain simple
name/value pairs.

Each data row contains a name, and value. The row also contains a
type or mimetype. Type corresponds to a .NET class that support
text/value conversion through the TypeConverter architecture.
Classes that don't support this are serialized and stored with the
mimetype set.

The mimetype is used for serialized objects, and tells the
ResXResourceReader how to depersist the object. This is currently not
extensible. For a given mimetype the value must be set accordingly:

Note - application/x-microsoft.net.object.binary.base64 is the format
that the ResXResourceWriter will generate, however the reader can
read any of the formats listed below.

mimetype: application/x-microsoft.net.object.binary.base64
value : The object must be serialized with
: System.Runtime.Serialization.Formatters.Binary.BinaryFormatter
: and then encoded with base64 encoding.

mimetype: application/x-microsoft.net.object.soap.base64
value : The object must be serialized with
: System.Runtime.Serialization.Formatters.Soap.SoapFormatter
: and then encoded with base64 encoding.

mimetype: application/x-microsoft.net.object.bytearray.base64
value : The object must be serialized into a byte array
: using a System.ComponentModel.TypeConverter
: and then encoded with base64 encoding.
-->
<xsd:schema id="root" xmlns="" xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:msdata="urn:schemas-microsoft-com:xml-msdata">
<xsd:import namespace="http://www.w3.org/XML/1998/namespace" />
<xsd:element name="root" msdata:IsDataSet="true">
<xsd:complexType>
<xsd:choice maxOccurs="unbounded">
<xsd:element name="metadata">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="value" type="xsd:string" minOccurs="0" />
</xsd:sequence>
<xsd:attribute name="name" use="required" type="xsd:string" />
<xsd:attribute name="type" type="xsd:string" />
<xsd:attribute name="mimetype" type="xsd:string" />
<xsd:attribute ref="xml:space" />
</xsd:complexType>
</xsd:element>
<xsd:element name="assembly">
<xsd:complexType>
<xsd:attribute name="alias" type="xsd:string" />
<xsd:attribute name="name" type="xsd:string" />
</xsd:complexType>
</xsd:element>
<xsd:element name="data">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
<xsd:element name="comment" type="xsd:string" minOccurs="0" msdata:Ordinal="2" />
</xsd:sequence>
<xsd:attribute name="name" type="xsd:string" use="required" msdata:Ordinal="1" />
<xsd:attribute name="type" type="xsd:string" msdata:Ordinal="3" />
<xsd:attribute name="mimetype" type="xsd:string" msdata:Ordinal="4" />
<xsd:attribute ref="xml:space" />
</xsd:complexType>
</xsd:element>
<xsd:element name="resheader">
<xsd:complexType>
<xsd:sequence>
<xsd:element name="value" type="xsd:string" minOccurs="0" msdata:Ordinal="1" />
</xsd:sequence>
<xsd:attribute name="name" type="xsd:string" use="required" />
</xsd:complexType>
</xsd:element>
</xsd:choice>
</xsd:complexType>
</xsd:element>
</xsd:schema>
<resheader name="resmimetype">
<value>text/microsoft-resx</value>
</resheader>
<resheader name="version">
<value>2.0</value>
</resheader>
<resheader name="reader">
<value>System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
<resheader name="writer">
<value>System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089</value>
</resheader>
<data name="Invalid_AttributeValue" xml:space="preserve">
<value>The {0} value: `{1}` is invalid. Valid example: `{2}`</value>
<comment>{0} - is the attribute
{1} is the invalid value of the attribute.
{2} a valid attriibute value example</comment>
</data>
</root>
6 changes: 3 additions & 3 deletions src/NuGet.Core/NuGet.ProjectModel/GlobalSuppressions.cs
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,8 @@
[assembly: SuppressMessage("Usage", "CA2227:Collection properties should be read only", Justification = "<Pending>", Scope = "member", Target = "~P:NuGet.ProjectModel.LockFile.CentralTransitiveDependencyGroups")]
[assembly: SuppressMessage("Usage", "CA2227:Collection properties should be read only", Justification = "IReadOnlySet<T> is only available in .NET 5+, so we need to use a HashSet<T> here.", Scope = "member", Target = "~P:NuGet.ProjectModel.RestoreAuditProperties.SuppressedAdvisories")]
[assembly: SuppressMessage("Design", "CA1031:Do not catch general exception types", Justification = "<Pending>", Scope = "member", Target = "~M:NuGet.ProjectModel.LockFileFormat.Utf8JsonRead(System.IO.Stream,NuGet.Common.ILogger,System.String,NuGet.ProjectModel.LockFileReadFlags)~NuGet.ProjectModel.LockFile")]
[assembly: SuppressMessage("Performance", "CA1822:Mark members as static", Justification = "<Pending>", Scope = "member", Target = "~M:NuGet.ProjectModel.Utf8JsonStreamLockFileConverter.Read(NuGet.ProjectModel.Utf8JsonStreamReader@,NuGet.ProjectModel.LockFileReadFlags)~NuGet.ProjectModel.LockFile")]
[assembly: SuppressMessage("Performance", "CA1822:Mark members as static", Justification = "<Pending>", Scope = "member", Target = "~M:NuGet.ProjectModel.Utf8JsonStreamLockFileConverter.Read(NuGet.Utf8JsonStreamReader@,NuGet.ProjectModel.LockFileReadFlags)~NuGet.ProjectModel.LockFile")]
[assembly: SuppressMessage("Performance", "CA1822:Mark members as static", Justification = "<Pending>", Scope = "member", Target = "~M:NuGet.ProjectModel.PackageSpecReferenceDependencyProvider.AddLibraryProperties(NuGet.LibraryModel.Library,NuGet.ProjectModel.PackageSpec,NuGet.Frameworks.NuGetFramework)")]
[assembly: SuppressMessage("Performance", "CA1822:Mark members as static", Justification = "<Pending>", Scope = "member", Target = "~M:NuGet.ProjectModel.ProjectRestoreMetadata.GetSources(System.Collections.Generic.IList{NuGet.Configuration.PackageSource})~System.Collections.Generic.HashSet{System.String}")]
[assembly: SuppressMessage("Performance", "CA1822:Mark members as static", Justification = "<Pending>", Scope = "member", Target = "~M:NuGet.ProjectModel.Utf8JsonStreamLockFileTargetLibraryConverter.ReadPackageDependencyList(NuGet.ProjectModel.Utf8JsonStreamReader@)~System.Collections.Generic.IList{NuGet.Packaging.Core.PackageDependency}")]
[assembly: SuppressMessage("Globalization", "CA1305:Specify IFormatProvider", Justification = "<Pending>", Scope = "member", Target = "~M:NuGet.ProjectModel.Utf8JsonReaderExtensions.ReadNumberAsString(System.Text.Json.Utf8JsonReader@)~System.String")]
[assembly: SuppressMessage("Performance", "CA1822:Mark members as static", Justification = "<Pending>", Scope = "member", Target = "~M:NuGet.ProjectModel.Utf8JsonStreamLockFileTargetLibraryConverter.ReadPackageDependencyList(NuGet.Utf8JsonStreamReader@)~System.Collections.Generic.IList{NuGet.Packaging.Core.PackageDependency}")]
[assembly: SuppressMessage("Globalization", "CA1305:Specify IFormatProvider", Justification = "<Pending>", Scope = "member", Target = "~M:NuGet.Utf8JsonReaderExtensions.ReadNumberAsString(System.Text.Json.Utf8JsonReader@)~System.String")]
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ private static LockFile Utf8JsonRead(Stream stream, ILogger log, string path, Lo
{
try
{
var lockFile = JsonUtility.LoadJson(stream, Utf8JsonReaderExtensions.LockFileConverter, flags);
var lockFile = JsonUtility.LoadJson(stream, Utf8JsonStreamLockFileConverter.LockFileConverter, flags);
lockFile.Path = path;
return lockFile;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ internal class Utf8JsonStreamIAssetsLogMessageConverter : IUtf8JsonStreamReaderC
private static readonly byte[] LibraryIdPropertyName = Encoding.UTF8.GetBytes(LogMessageProperties.LIBRARY_ID);
private static readonly byte[] TargetGraphsPropertyName = Encoding.UTF8.GetBytes(LogMessageProperties.TARGET_GRAPHS);

internal static readonly Utf8JsonStreamIAssetsLogMessageConverter IAssetsLogMessageConverter = new Utf8JsonStreamIAssetsLogMessageConverter();

public IAssetsLogMessage Read(ref Utf8JsonStreamReader reader)
{
if (reader.TokenType != JsonTokenType.StartObject)
Expand Down
Loading
Loading