From 2319bfe1549e28883570d899a7f5a727dbbaf13a Mon Sep 17 00:00:00 2001 From: Jeremie Desautels Date: Tue, 28 May 2019 11:42:19 -0400 Subject: [PATCH 1/7] Downgrade GitVersion to version 4.0 because 5.0 beta does not correctly calculate version on release branches without commits. --- build.cake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.cake b/build.cake index c488278..6c5b6cd 100644 --- a/build.cake +++ b/build.cake @@ -2,10 +2,10 @@ #addin nuget:?package=Cake.Coveralls&version=0.9.0 // Install tools. -#tool nuget:?package=GitVersion.CommandLine&version=5.0.0-beta2-97 +#tool nuget:?package=GitVersion.CommandLine&version=4.0.0 #tool nuget:?package=GitReleaseManager&version=0.8.0 #tool nuget:?package=OpenCover&version=4.7.922 -#tool nuget:?package=ReportGenerator&version=4.1.5 +#tool nuget:?package=ReportGenerator&version=4.1.8 #tool nuget:?package=coveralls.io&version=1.4.2 #tool nuget:?package=xunit.runner.console&version=2.4.1 From 9cc3518a3b36a5f2356a7661dfd1b93069ac893c Mon Sep 17 00:00:00 2001 From: Jeremie Desautels Date: Sun, 7 Jul 2019 12:39:41 -0400 Subject: [PATCH 2/7] Refresh resources --- .gitignore | 9 +++++++-- appveyor.yml | 2 +- build.cake | 3 +-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index d6ffc67..f0ea8e7 100644 --- a/.gitignore +++ b/.gitignore @@ -46,9 +46,10 @@ Generated\ Files/ [Tt]est[Rr]esult*/ [Bb]uild[Ll]og.* -# NUNIT +# NUnit *.VisualState.xml TestResult.xml +nunit-*.xml # Build Results of an ATL Project [Dd]ebugPS/ @@ -189,6 +190,8 @@ PublishScripts/ # NuGet Packages *.nupkg +# NuGet Symbol Packages +*.snupkg # The packages folder can be ignored because of Package Restore **/[Pp]ackages/* # except build/, which is used as an MSBuild target. @@ -264,7 +267,9 @@ ServiceFabricBackup/ *.bim.layout *.bim_*.settings *.rptproj.rsuser -*- Backup*.rdl +*- [Bb]ackup.rdl +*- [Bb]ackup ([0-9]).rdl +*- [Bb]ackup ([0-9][0-9]).rdl # Microsoft Fakes FakesAssemblies/ diff --git a/appveyor.yml b/appveyor.yml index 82c4365..f04be9a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,4 +23,4 @@ cache: - tools -> build.cake, tools\packages.config # Environment configuration -image: Visual Studio 2017 +image: Visual Studio 2019 diff --git a/build.cake b/build.cake index 6c5b6cd..5b5c076 100644 --- a/build.cake +++ b/build.cake @@ -5,7 +5,7 @@ #tool nuget:?package=GitVersion.CommandLine&version=4.0.0 #tool nuget:?package=GitReleaseManager&version=0.8.0 #tool nuget:?package=OpenCover&version=4.7.922 -#tool nuget:?package=ReportGenerator&version=4.1.8 +#tool nuget:?package=ReportGenerator&version=4.2.5 #tool nuget:?package=coveralls.io&version=1.4.2 #tool nuget:?package=xunit.runner.console&version=2.4.1 @@ -149,7 +149,6 @@ Task("Restore-NuGet-Packages") DotNetCoreRestore("./Source/", new DotNetCoreRestoreSettings { Sources = new [] { - "https://www.myget.org/F/xunit/api/v3/index.json", "https://api.nuget.org/v3/index.json", } }); From 44e0a9569231c3ed72b88e7b65e25fbb3a03c07d Mon Sep 17 00:00:00 2001 From: Jericho Date: Wed, 15 Jan 2020 11:31:24 -0500 Subject: [PATCH 3/7] Refresh resources (such as build script) --- .gitignore | 1 - appveyor.yml | 1 + build.cake | 167 +++++++++++++++++++++++++++++++++--------- build.ps1 | 10 +-- tools/packages.config | 2 +- 5 files changed, 138 insertions(+), 43 deletions(-) diff --git a/.gitignore b/.gitignore index f0ea8e7..4e35409 100644 --- a/.gitignore +++ b/.gitignore @@ -72,7 +72,6 @@ StyleCopReport.xml *_p.c *_h.h *.ilk -*.meta *.obj *.iobj *.pch diff --git a/appveyor.yml b/appveyor.yml index f04be9a..3950b47 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,6 +4,7 @@ init: # Build script build_script: + - ps: .\build.ps1 -bootstrap - ps: .\build.ps1 -Target AppVeyor # Tests diff --git a/build.cake b/build.cake index 5b5c076..e235d75 100644 --- a/build.cake +++ b/build.cake @@ -1,14 +1,20 @@ -// Install addins. -#addin nuget:?package=Cake.Coveralls&version=0.9.0 +// Install modules +#module nuget:?package=Cake.DotNetTool.Module&version=0.4.0 + +// Install .NET tools +#tool dotnet:?package=BenchmarkDotNet.Tool&version=0.12.0 // Install tools. -#tool nuget:?package=GitVersion.CommandLine&version=4.0.0 -#tool nuget:?package=GitReleaseManager&version=0.8.0 +#tool nuget:?package=GitVersion.CommandLine&version=5.1.3 +#tool nuget:?package=GitReleaseManager&version=0.9.0 #tool nuget:?package=OpenCover&version=4.7.922 -#tool nuget:?package=ReportGenerator&version=4.2.5 +#tool nuget:?package=ReportGenerator&version=4.4.0 #tool nuget:?package=coveralls.io&version=1.4.2 #tool nuget:?package=xunit.runner.console&version=2.4.1 +// Install addins. +#addin nuget:?package=Cake.Coveralls&version=0.10.1 + /////////////////////////////////////////////////////////////////////////////// // ARGUMENTS @@ -29,32 +35,37 @@ var testCoverageFilter = "+[Picton.Messaging]* -[Picton.Messaging]Picton.Messagi var testCoverageExcludeByAttribute = "*.ExcludeFromCodeCoverage*"; var testCoverageExcludeByFile = "*/*Designer.cs;*/*AssemblyInfo.cs"; -var nuGetApiUrl = EnvironmentVariable("NUGET_API_URL"); -var nuGetApiKey = EnvironmentVariable("NUGET_API_KEY"); +var nuGetApiUrl = Argument("NUGET_API_URL", EnvironmentVariable("NUGET_API_URL")); +var nuGetApiKey = Argument("NUGET_API_KEY", EnvironmentVariable("NUGET_API_KEY")); -var myGetApiUrl = EnvironmentVariable("MYGET_API_URL"); -var myGetApiKey = EnvironmentVariable("MYGET_API_KEY"); +var myGetApiUrl = Argument("MYGET_API_URL", EnvironmentVariable("MYGET_API_URL")); +var myGetApiKey = Argument("MYGET_API_KEY", EnvironmentVariable("MYGET_API_KEY")); -var gitHubUserName = EnvironmentVariable("GITHUB_USERNAME"); -var gitHubPassword = EnvironmentVariable("GITHUB_PASSWORD"); +var gitHubToken = Argument("GITHUB_TOKEN", EnvironmentVariable("GITHUB_TOKEN")); +var gitHubUserName = Argument("GITHUB_USERNAME", EnvironmentVariable("GITHUB_USERNAME")); +var gitHubPassword = Argument("GITHUB_PASSWORD", EnvironmentVariable("GITHUB_PASSWORD")); +var gitHubRepoOwner = Argument("GITHUB_REPOOWNER", EnvironmentVariable("GITHUB_REPOOWNER") ?? gitHubUserName); var sourceFolder = "./Source/"; - var outputDir = "./artifacts/"; -var codeCoverageDir = outputDir + "CodeCoverage/"; -var unitTestsProject = sourceFolder + libraryName + ".UnitTests/" + libraryName + ".UnitTests.csproj"; +var codeCoverageDir = $"{outputDir}CodeCoverage/"; +var benchmarkDir = $"{outputDir}Benchmark/"; + +var unitTestsProject = $"{sourceFolder}{libraryName}.UnitTests/{libraryName}.UnitTests.csproj"; +var benchmarkProject = $"{sourceFolder}{libraryName}.Benchmark/{libraryName}.Benchmark.csproj"; var versionInfo = GitVersion(new GitVersionSettings() { OutputType = GitVersionOutput.Json }); -var milestone = string.Concat("v", versionInfo.MajorMinorPatch); +var milestone = versionInfo.MajorMinorPatch; var cakeVersion = typeof(ICakeContext).Assembly.GetName().Version.ToString(); var isLocalBuild = BuildSystem.IsLocalBuild; var isMainBranch = StringComparer.OrdinalIgnoreCase.Equals("master", BuildSystem.AppVeyor.Environment.Repository.Branch); -var isMainRepo = StringComparer.OrdinalIgnoreCase.Equals(gitHubUserName + "/" + gitHubRepo, BuildSystem.AppVeyor.Environment.Repository.Name); +var isMainRepo = StringComparer.OrdinalIgnoreCase.Equals($"{gitHubRepoOwner}/{gitHubRepo}", BuildSystem.AppVeyor.Environment.Repository.Name); var isPullRequest = BuildSystem.AppVeyor.Environment.PullRequest.IsPullRequest; var isTagged = ( BuildSystem.AppVeyor.Environment.Repository.Tag.IsTag && !string.IsNullOrWhiteSpace(BuildSystem.AppVeyor.Environment.Repository.Tag.Name) ); +var isBenchmarkPresent = FileExists(benchmarkProject); /////////////////////////////////////////////////////////////////////////////// @@ -95,11 +106,22 @@ Setup(context => string.IsNullOrEmpty(nuGetApiKey) ? "[NULL]" : new string('*', nuGetApiKey.Length) ); - Information("GitHub Info:\r\n\tRepo: {0}\r\n\tUserName: {1}\r\n\tPassword: {2}", - gitHubRepo, - gitHubUserName, - string.IsNullOrEmpty(gitHubPassword) ? "[NULL]" : new string('*', gitHubPassword.Length) - ); + if (!string.IsNullOrEmpty(gitHubToken)) + { + Information("GitHub Info:\r\n\tRepo: {0}\r\n\tUserName: {1}\r\n\tToken: {2}", + $"{gitHubRepoOwner}/{gitHubRepo}", + gitHubUserName, + new string('*', gitHubToken.Length) + ); + } + else + { + Information("GitHub Info:\r\n\tRepo: {0}\r\n\tUserName: {1}\r\n\tPassword: {2}", + $"{gitHubRepoOwner}/{gitHubRepo}", + gitHubUserName, + string.IsNullOrEmpty(gitHubPassword) ? "[NULL]" : new string('*', gitHubPassword.Length) + ); + } }); Teardown(context => @@ -130,8 +152,8 @@ Task("Clean") { // Clean solution directories. Information("Cleaning {0}", sourceFolder); - CleanDirectories(sourceFolder + "*/bin/" + configuration); - CleanDirectories(sourceFolder + "*/obj/" + configuration); + CleanDirectories($"{sourceFolder}*/bin/{configuration}"); + CleanDirectories($"{sourceFolder}*/obj/{configuration}"); // Clean previous artifacts Information("Cleaning {0}", outputDir); @@ -173,6 +195,7 @@ Task("Run-Unit-Tests") DotNetCoreTest(unitTestsProject, new DotNetCoreTestSettings { NoBuild = true, + NoRestore = true, Configuration = configuration }); }); @@ -184,11 +207,12 @@ Task("Run-Code-Coverage") Action testAction = ctx => ctx.DotNetCoreTest(unitTestsProject, new DotNetCoreTestSettings { NoBuild = true, + NoRestore = true, Configuration = configuration }); OpenCover(testAction, - codeCoverageDir + "coverage.xml", + $"{codeCoverageDir}coverage.xml", new OpenCoverSettings { OldStyle = true, @@ -204,7 +228,7 @@ Task("Run-Code-Coverage") Task("Upload-Coverage-Result") .Does(() => { - CoverallsIo(codeCoverageDir + "coverage.xml"); + CoverallsIo($"{codeCoverageDir}coverage.xml"); }); Task("Generate-Code-Coverage-Report") @@ -212,7 +236,7 @@ Task("Generate-Code-Coverage-Report") .Does(() => { ReportGenerator( - codeCoverageDir + "coverage.xml", + $"{codeCoverageDir}coverage.xml", codeCoverageDir, new ReportGeneratorSettings() { ClassFilters = new[] { "*.UnitTests*" } @@ -230,6 +254,7 @@ Task("Create-NuGet-Package") IncludeSource = false, IncludeSymbols = true, NoBuild = true, + NoRestore = true, NoDependencies = true, OutputDirectory = outputDir, ArgumentCustomization = (args) => @@ -250,7 +275,11 @@ Task("Upload-AppVeyor-Artifacts") .WithCriteria(() => AppVeyor.IsRunningOnAppVeyor) .Does(() => { - foreach (var file in GetFiles(outputDir + "*.*")) + foreach (var file in GetFiles($"{outputDir}*.*")) + { + AppVeyor.UploadArtifact(file.FullPath); + } + foreach (var file in GetFiles($"{benchmarkDir}results/*.*")) { AppVeyor.UploadArtifact(file.FullPath); } @@ -301,15 +330,25 @@ Task("Publish-MyGet") Task("Create-Release-Notes") .Does(() => { - if(string.IsNullOrEmpty(gitHubUserName)) throw new InvalidOperationException("Could not resolve GitHub user name."); - if(string.IsNullOrEmpty(gitHubPassword)) throw new InvalidOperationException("Could not resolve GitHub password."); - - GitReleaseManagerCreate(gitHubUserName, gitHubPassword, gitHubUserName, gitHubRepo, new GitReleaseManagerCreateSettings { + var settings = new GitReleaseManagerCreateSettings + { Name = milestone, Milestone = milestone, Prerelease = false, TargetCommitish = "master" - }); + }; + + if (!string.IsNullOrEmpty(gitHubToken)) + { + GitReleaseManagerCreate(gitHubToken, gitHubRepoOwner, gitHubRepo, settings); + } + else + { + if(string.IsNullOrEmpty(gitHubUserName)) throw new InvalidOperationException("Could not resolve GitHub user name."); + if(string.IsNullOrEmpty(gitHubPassword)) throw new InvalidOperationException("Could not resolve GitHub password."); + + GitReleaseManagerCreate(gitHubUserName, gitHubPassword, gitHubRepoOwner, gitHubRepo, settings); + } }); Task("Publish-GitHub-Release") @@ -320,10 +359,56 @@ Task("Publish-GitHub-Release") .WithCriteria(() => isTagged) .Does(() => { - if(string.IsNullOrEmpty(gitHubUserName)) throw new InvalidOperationException("Could not resolve GitHub user name."); - if(string.IsNullOrEmpty(gitHubPassword)) throw new InvalidOperationException("Could not resolve GitHub password."); + var settings = new GitReleaseManagerCreateSettings + { + Name = milestone, + Milestone = milestone, + Prerelease = false, + TargetCommitish = "master" + }; + + if (!string.IsNullOrEmpty(gitHubToken)) + { + GitReleaseManagerClose(gitHubToken, gitHubRepoOwner, gitHubRepo, milestone); + } + else + { + if(string.IsNullOrEmpty(gitHubUserName)) throw new InvalidOperationException("Could not resolve GitHub user name."); + if(string.IsNullOrEmpty(gitHubPassword)) throw new InvalidOperationException("Could not resolve GitHub password."); + + GitReleaseManagerClose(gitHubUserName, gitHubPassword, gitHubRepoOwner, gitHubRepo, milestone); + } +}); + +Task("Generate-Benchmark-Report") + .IsDependentOn("Build") + .WithCriteria(isBenchmarkPresent) + .Does(() => +{ + var publishDirectory = $"{benchmarkDir}Publish/"; + + DotNetCorePublish(benchmarkProject, new DotNetCorePublishSettings + { + Configuration = configuration, + NoRestore = true, + NoBuild = true, + OutputDirectory = publishDirectory + }); + + var assemblyLocation = MakeAbsolute(File($"{publishDirectory}{libraryName}.Benchmark.dll")).FullPath; + var artifactsLocation = MakeAbsolute(File(benchmarkDir)).FullPath; + var benchmarkToolLocation = Context.Tools.Resolve("dotnet-benchmark.exe"); - GitReleaseManagerClose(gitHubUserName, gitHubPassword, gitHubUserName, gitHubRepo, milestone); + var processResult = StartProcess( + benchmarkToolLocation, + new ProcessSettings() + { + Arguments = $"{assemblyLocation} -f * --artifacts={artifactsLocation}" + }); + if (processResult != 0) + { + throw new Exception($"dotnet-benchmark.exe did not complete successfully. Result code: {processResult}"); + } }); @@ -335,7 +420,16 @@ Task("Coverage") .IsDependentOn("Generate-Code-Coverage-Report") .Does(() => { - StartProcess("cmd", "/c start " + codeCoverageDir + "index.htm"); + StartProcess("cmd", $"/c start {codeCoverageDir}index.htm"); +}); + +Task("Benchmark") + .IsDependentOn("Generate-Benchmark-Report") + .WithCriteria(isBenchmarkPresent) + .Does(() => +{ + var htmlReport = GetFiles($"{benchmarkDir}results/*-report.html", new GlobberSettings { IsCaseSensitive = false }).FirstOrDefault(); + StartProcess("cmd", $"/c start {htmlReport}"); }); Task("ReleaseNotes") @@ -344,6 +438,7 @@ Task("ReleaseNotes") Task("AppVeyor") .IsDependentOn("Run-Code-Coverage") .IsDependentOn("Upload-Coverage-Result") + .IsDependentOn("Generate-Benchmark-Report") .IsDependentOn("Create-NuGet-Package") .IsDependentOn("Upload-AppVeyor-Artifacts") .IsDependentOn("Publish-MyGet") diff --git a/build.ps1 b/build.ps1 index 7f1f813..a336e29 100644 --- a/build.ps1 +++ b/build.ps1 @@ -176,7 +176,7 @@ if(-Not $SkipToolPackageRestore.IsPresent) { ($md5Hash -ne (Get-Content $PACKAGES_CONFIG_MD5 ))) { Write-Verbose -Message "Missing or changed package.config hash..." Get-ChildItem -Exclude packages.config,nuget.exe,Cake.Bakery | - Remove-Item -Recurse + Remove-Item -Recurse -Force } Write-Verbose -Message "Restoring tools from NuGet..." @@ -240,10 +240,10 @@ $CAKE_EXE_INVOCATION = if ($IsLinux -or $IsMacOS) { "`"$CAKE_EXE`"" } - -# Build Cake arguments -$cakeArguments = @("$Script"); -if ($Target) { $cakeArguments += "-target=$Target" } + # Build an array (not a string) of Cake arguments to be joined later +$cakeArguments = @() +if ($Script) { $cakeArguments += "`"$Script`"" } +if ($Target) { $cakeArguments += "-target=`"$Target`"" } if ($Configuration) { $cakeArguments += "-configuration=$Configuration" } if ($Verbosity) { $cakeArguments += "-verbosity=$Verbosity" } if ($ShowDescription) { $cakeArguments += "-showdescription" } diff --git a/tools/packages.config b/tools/packages.config index 997c0e1..99635e9 100644 --- a/tools/packages.config +++ b/tools/packages.config @@ -1,4 +1,4 @@ - + From 70b14aee09ca3c1a7deb91d1914fdfbc35ae66a8 Mon Sep 17 00:00:00 2001 From: Jericho Date: Wed, 15 Jan 2020 11:32:11 -0500 Subject: [PATCH 4/7] Upgrade nuget packages --- .../Picton.Messaging.IntegrationTests.csproj | 6 +++--- .../Picton.Messaging.UnitTests.csproj | 7 +++---- Source/Picton.Messaging/Picton.Messaging.csproj | 8 ++++---- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/Source/Picton.Messaging.IntegrationTests/Picton.Messaging.IntegrationTests.csproj b/Source/Picton.Messaging.IntegrationTests/Picton.Messaging.IntegrationTests.csproj index 67ed383..4f36fde 100644 --- a/Source/Picton.Messaging.IntegrationTests/Picton.Messaging.IntegrationTests.csproj +++ b/Source/Picton.Messaging.IntegrationTests/Picton.Messaging.IntegrationTests.csproj @@ -1,8 +1,8 @@ - + Exe - netcoreapp2.0 + netcoreapp3.1 Picton.Messaging.IntegrationTests Picton.Messaging.IntegrationTests @@ -12,7 +12,7 @@ - + diff --git a/Source/Picton.Messaging.UnitTests/Picton.Messaging.UnitTests.csproj b/Source/Picton.Messaging.UnitTests/Picton.Messaging.UnitTests.csproj index 45cf163..a2020a8 100644 --- a/Source/Picton.Messaging.UnitTests/Picton.Messaging.UnitTests.csproj +++ b/Source/Picton.Messaging.UnitTests/Picton.Messaging.UnitTests.csproj @@ -1,21 +1,20 @@ - net461;net472;netcoreapp2.0 + net461;net472;netcoreapp3.1 Picton.Messaging.UnitTests Picton.Messaging.UnitTests - - + + all runtime; build; native; contentfiles; analyzers - diff --git a/Source/Picton.Messaging/Picton.Messaging.csproj b/Source/Picton.Messaging/Picton.Messaging.csproj index f16a8fb..65e5e97 100644 --- a/Source/Picton.Messaging/Picton.Messaging.csproj +++ b/Source/Picton.Messaging/Picton.Messaging.csproj @@ -32,13 +32,13 @@ - - + + all runtime; build; native; contentfiles; analyzers - + @@ -49,7 +49,7 @@ - + From dc0b0d5307a703eb9d99253587e60962df7a34c6 Mon Sep 17 00:00:00 2001 From: Jericho Date: Thu, 16 Jan 2020 10:46:20 -0500 Subject: [PATCH 5/7] Update build script --- build.cake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.cake b/build.cake index e235d75..b63c834 100644 --- a/build.cake +++ b/build.cake @@ -6,9 +6,9 @@ // Install tools. #tool nuget:?package=GitVersion.CommandLine&version=5.1.3 -#tool nuget:?package=GitReleaseManager&version=0.9.0 +#tool nuget:?package=GitReleaseManager&version=0.10.2 #tool nuget:?package=OpenCover&version=4.7.922 -#tool nuget:?package=ReportGenerator&version=4.4.0 +#tool nuget:?package=ReportGenerator&version=4.4.2 #tool nuget:?package=coveralls.io&version=1.4.2 #tool nuget:?package=xunit.runner.console&version=2.4.1 From 225e91740cee1a235660bbd230d78a494f813041 Mon Sep 17 00:00:00 2001 From: Jericho Date: Wed, 18 Mar 2020 16:01:38 -0400 Subject: [PATCH 6/7] Upgrade to new Azure nuget packages --- GitReleaseManager.yaml | 38 +- GitVersion.yml | 3 + .../Program.cs | 44 +- .../AsyncMessagePumpTests.cs | 408 +++++++++--------- .../MockAzureResponse.cs | 89 ++++ .../Picton.Messaging.UnitTests/MockUtils.cs | 59 +++ Source/Picton.Messaging/AsyncMessagePump.cs | 82 ++-- .../AsyncMessagePumpWithHandlers.cs | 27 +- Source/Picton.Messaging/Extensions.cs | 6 +- Source/Picton.Messaging/Metrics.cs | 18 +- .../Picton.Messaging/Picton.Messaging.csproj | 2 +- .../Utilities/ExcludeFromCodeCoverage.cs | 8 + build.cake | 6 +- tools/packages.config | 2 +- 14 files changed, 488 insertions(+), 304 deletions(-) create mode 100644 Source/Picton.Messaging.UnitTests/MockAzureResponse.cs create mode 100644 Source/Picton.Messaging.UnitTests/MockUtils.cs create mode 100644 Source/Picton.Messaging/Utilities/ExcludeFromCodeCoverage.cs diff --git a/GitReleaseManager.yaml b/GitReleaseManager.yaml index 725bcfa..689d616 100644 --- a/GitReleaseManager.yaml +++ b/GitReleaseManager.yaml @@ -4,6 +4,17 @@ create: footer-content: You can download this release from [nuget.org](https://www.nuget.org/packages/Picton.Messaging/{milestone}) footer-includes-milestone: true milestone-replace-text: '{milestone}' +close: + use-issue-comments: true + issue-comment: |- + :tada: This issue has been resolved in version {milestone} :tada: + + The release is available on: + + - [GitHub Release](https://github.com/{owner}/{repository}/releases/tag/{milestone}) + - [NuGet Package](https://www.nuget.org/packages/Picton.Messaging/{milestone}) + + Your **[GitReleaseManager](https://github.com/GitTools/GitReleaseManager)** bot :package::rocket: export: include-created-date-in-title: true created-date-string-format: MMMM dd, yyyy @@ -11,17 +22,20 @@ export: regex-text: '### Where to get it(\r\n)*You can .*\)' multiline-regex: true issue-labels-include: -- Breaking Change -- Bug -- New Feature -- Improvement -- Documentation + - Breaking Change + - Bug + - New Feature + - Improvement + - Documentation + - Security issue-labels-exclude: -- Question -- Duplicate -- Invalid -- Wontfix + - Question + - Duplicate + - Invalid + - Wontfix + - Build + - Internal Refactoring issue-labels-alias: - - name: Documentation - header: Documentation - plural: Documentation + - name: Documentation + header: Documentation + plural: Documentation diff --git a/GitVersion.yml b/GitVersion.yml index dede53e..366d8f2 100644 --- a/GitVersion.yml +++ b/GitVersion.yml @@ -1,5 +1,8 @@ mode: ContinuousDelivery branches: + feature: + regex: feature(s)?[/-] + mode: ContinuousDeployment develop: regex: dev(elop)?(ment)?$ mode: ContinuousDeployment diff --git a/Source/Picton.Messaging.IntegrationTests/Program.cs b/Source/Picton.Messaging.IntegrationTests/Program.cs index 3b3be8d..271ac0d 100644 --- a/Source/Picton.Messaging.IntegrationTests/Program.cs +++ b/Source/Picton.Messaging.IntegrationTests/Program.cs @@ -1,7 +1,5 @@ using App.Metrics; using App.Metrics.Scheduling; -using Microsoft.Azure.Storage; -using Microsoft.Azure.Storage.Queue; using Picton.Managers; using Picton.Messaging.IntegrationTests.Datadog; using Picton.Messaging.Logging; @@ -57,23 +55,23 @@ static void Main() } // Setup the message queue in Azure storage emulator - var storageAccount = CloudStorageAccount.DevelopmentStorageAccount; + var connectionString = "UseDevelopmentStorage=true"; var queueName = "myqueue"; var numberOfMessages = 25; logger(Logging.LogLevel.Info, () => "Begin integration tests..."); var stringMessagesLogger = logProvider.GetLogger("StringMessages"); - AddStringMessagesToQueue(numberOfMessages, queueName, storageAccount, stringMessagesLogger).Wait(); - ProcessSimpleMessages(queueName, storageAccount, stringMessagesLogger, metrics); + AddStringMessagesToQueue(numberOfMessages, queueName, connectionString, stringMessagesLogger).Wait(); + ProcessSimpleMessages(queueName, connectionString, stringMessagesLogger, metrics); var simpleMessagesLogger = logProvider.GetLogger("SimpleMessages"); - AddSimpleMessagesToQueue(numberOfMessages, queueName, storageAccount, simpleMessagesLogger).Wait(); - ProcessSimpleMessages(queueName, storageAccount, simpleMessagesLogger, metrics); + AddSimpleMessagesToQueue(numberOfMessages, queueName, connectionString, simpleMessagesLogger).Wait(); + ProcessSimpleMessages(queueName, connectionString, simpleMessagesLogger, metrics); var messagesWithHandlerLogger = logProvider.GetLogger("MessagesWithHandler"); - AddMessagesWithHandlerToQueue(numberOfMessages, queueName, storageAccount, messagesWithHandlerLogger).Wait(); - ProcessMessagesWithHandlers(queueName, storageAccount, messagesWithHandlerLogger, metrics); + AddMessagesWithHandlerToQueue(numberOfMessages, queueName, connectionString, messagesWithHandlerLogger).Wait(); + ProcessMessagesWithHandlers(queueName, connectionString, messagesWithHandlerLogger, metrics); // Flush the console key buffer while (Console.KeyAvailable) Console.ReadKey(true); @@ -83,26 +81,23 @@ static void Main() Console.ReadKey(); } - public static async Task AddStringMessagesToQueue(int numberOfMessages, string queueName, CloudStorageAccount storageAccount, Logger logger) + public static async Task AddStringMessagesToQueue(int numberOfMessages, string queueName, string connectionString, Logger logger) { logger(Logging.LogLevel.Info, () => $"Adding {numberOfMessages} string messages to the queue..."); - var cloudQueueClient = storageAccount.CreateCloudQueueClient(); - var cloudQueue = cloudQueueClient.GetQueueReference(queueName); - await cloudQueue.CreateIfNotExistsAsync().ConfigureAwait(false); - await cloudQueue.ClearAsync().ConfigureAwait(false); + var queueManager = new QueueManager(connectionString, queueName); + await queueManager.ClearAsync().ConfigureAwait(false); for (var i = 0; i < numberOfMessages; i++) { - await cloudQueue.AddMessageAsync(new CloudQueueMessage($"Hello world {i}")).ConfigureAwait(false); + await queueManager.AddMessageAsync($"Hello world {i}").ConfigureAwait(false); } } - public static async Task AddSimpleMessagesToQueue(int numberOfMessages, string queueName, CloudStorageAccount storageAccount, Logger logger) + public static async Task AddSimpleMessagesToQueue(int numberOfMessages, string queueName, string connectionString, Logger logger) { logger(Logging.LogLevel.Info, () => $"Adding {numberOfMessages} simple messages to the queue..."); - var queueManager = new QueueManager(queueName, storageAccount); - await queueManager.CreateIfNotExistsAsync().ConfigureAwait(false); + var queueManager = new QueueManager(connectionString, queueName); await queueManager.ClearAsync().ConfigureAwait(false); for (var i = 0; i < numberOfMessages; i++) { @@ -110,12 +105,12 @@ public static async Task AddSimpleMessagesToQueue(int numberOfMessages, string q } } - public static void ProcessSimpleMessages(string queueName, CloudStorageAccount storageAccount, Logger logger, IMetrics metrics) + public static void ProcessSimpleMessages(string queueName, string connectionString, Logger logger, IMetrics metrics) { Stopwatch sw = null; // Configure the message pump - var messagePump = new AsyncMessagePump(queueName, storageAccount, 10, null, TimeSpan.FromMinutes(1), 3, metrics) + var messagePump = new AsyncMessagePump(connectionString, queueName, 10, null, TimeSpan.FromMinutes(1), 3, metrics) { OnMessage = (message, cancellationToken) => { @@ -144,12 +139,11 @@ public static void ProcessSimpleMessages(string queueName, CloudStorageAccount s logger(Logging.LogLevel.Info, () => $"\tDone in {sw.Elapsed.ToDurationString()}"); } - public static async Task AddMessagesWithHandlerToQueue(int numberOfMessages, string queueName, CloudStorageAccount storageAccount, Logger logger) + public static async Task AddMessagesWithHandlerToQueue(int numberOfMessages, string queueName, string connectionString, Logger logger) { logger(Logging.LogLevel.Info, () => $"Adding {numberOfMessages} messages with handlers to the queue..."); - var queueManager = new QueueManager(queueName, storageAccount); - await queueManager.CreateIfNotExistsAsync().ConfigureAwait(false); + var queueManager = new QueueManager(connectionString, queueName); await queueManager.ClearAsync().ConfigureAwait(false); for (var i = 0; i < numberOfMessages; i++) { @@ -157,12 +151,12 @@ public static async Task AddMessagesWithHandlerToQueue(int numberOfMessages, str } } - public static void ProcessMessagesWithHandlers(string queueName, CloudStorageAccount storageAccount, Logger logger, IMetrics metrics) + public static void ProcessMessagesWithHandlers(string queueName, string connectionString, Logger logger, IMetrics metrics) { Stopwatch sw = null; // Configure the message pump - var messagePump = new AsyncMessagePumpWithHandlers(queueName, storageAccount, 10, null, TimeSpan.FromMinutes(1), 3, metrics); + var messagePump = new AsyncMessagePumpWithHandlers(connectionString, queueName, 10, null, TimeSpan.FromMinutes(1), 3, metrics); messagePump.OnQueueEmpty = cancellationToken => { // Stop the timer diff --git a/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs b/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs index 21ce1e4..4a6da36 100644 --- a/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs +++ b/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs @@ -1,30 +1,23 @@ -using Microsoft.Azure.Storage; -using Microsoft.Azure.Storage.Auth; -using Microsoft.Azure.Storage.Blob; -using Microsoft.Azure.Storage.Queue; +using Azure; +using Azure.Storage.Queues.Models; using Moq; +using Picton.Managers; using Shouldly; using System; -using System.Collections.Generic; using System.Linq; -using System.Text; using System.Threading; -using System.Threading.Tasks; using Xunit; namespace Picton.Messaging.UnitTests { public class AsyncMessagePumpTests { - private static readonly string QUEUE_STORAGE_URL = "http://bogus:10001/devstoreaccount1/"; - private static readonly string BLOB_STORAGE_URL = "http://bogus:10002/devstoreaccount1/"; - [Fact] public void Null_cloudQueue_throws() { Should.Throw(() => { - var messagePump = new AsyncMessagePump("myqueue", (CloudStorageAccount)null, 1, null, TimeSpan.FromMinutes(1), 3); + var messagePump = new AsyncMessagePump((QueueManager)null, null, 1, TimeSpan.FromMinutes(1), 1, null); }); } @@ -33,13 +26,11 @@ public void Number_of_concurrent_tasks_too_small_throws() { Should.Throw(() => { - var queueName = "myqueue"; - var mockQueue = GetMockQueue(queueName); - var mockQueueClient = GetMockQueueClient(mockQueue); - var mockBlobContainer = GetMockBlobContainer(); - var mockBlobClient = GetMockBlobClient(mockBlobContainer); + var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); + var mockQueueClient = MockUtils.GetMockQueueClient(); + var queueManager = new QueueManager(mockBlobContainerClient.Object, mockQueueClient.Object); - var messagePump = new AsyncMessagePump("myqueue", mockQueueClient.Object, mockBlobClient.Object, 0, null, TimeSpan.FromMinutes(1), 3); + var messagePump = new AsyncMessagePump(queueManager, null, 0, TimeSpan.FromMinutes(1), 1, null); }); } @@ -48,12 +39,11 @@ public void DequeueCount_too_small_throws() { Should.Throw(() => { - var queueName = "myqueue"; - var mockQueue = GetMockQueue(queueName); - var mockQueueClient = GetMockQueueClient(mockQueue); - var mockBlobContainer = GetMockBlobContainer(); - var mockBlobClient = GetMockBlobClient(mockBlobContainer); - var messagePump = new AsyncMessagePump("myqueue", mockQueueClient.Object, mockBlobClient.Object, 1, null, TimeSpan.FromMinutes(1), 0); + var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); + var mockQueueClient = MockUtils.GetMockQueueClient(); + var queueManager = new QueueManager(mockBlobContainerClient.Object, mockQueueClient.Object); + + var messagePump = new AsyncMessagePump(queueManager, null, 1, TimeSpan.FromMinutes(1), 0, null); }); } @@ -61,13 +51,11 @@ public void DequeueCount_too_small_throws() public void Start_without_OnMessage_throws() { // Arrange - var queueName = "myqueue"; - var mockQueue = GetMockQueue(queueName); - var mockQueueClient = GetMockQueueClient(mockQueue); - var mockBlobContainer = GetMockBlobContainer(); - var mockBlobClient = GetMockBlobClient(mockBlobContainer); + var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); + var mockQueueClient = MockUtils.GetMockQueueClient(); + var queueManager = new QueueManager(mockBlobContainerClient.Object, mockQueueClient.Object); - var messagePump = new AsyncMessagePump(queueName, mockQueueClient.Object, mockBlobClient.Object, 1, null, TimeSpan.FromMinutes(1), 3); + var messagePump = new AsyncMessagePump(queueManager, null, 1, TimeSpan.FromMinutes(1), 3, null); // Act Should.Throw(() => messagePump.Start()); @@ -77,13 +65,11 @@ public void Start_without_OnMessage_throws() public void Stopping_without_starting() { // Arrange - var queueName = "myqueue"; - var mockQueue = GetMockQueue(queueName); - var mockQueueClient = GetMockQueueClient(mockQueue); - var mockBlobContainer = GetMockBlobContainer(); - var mockBlobClient = GetMockBlobClient(mockBlobContainer); + var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); + var mockQueueClient = MockUtils.GetMockQueueClient(); + var queueManager = new QueueManager(mockBlobContainerClient.Object, mockQueueClient.Object); - var messagePump = new AsyncMessagePump(queueName, mockQueueClient.Object, mockBlobClient.Object, 1, null, TimeSpan.FromMinutes(1), 3); + var messagePump = new AsyncMessagePump(queueManager, null, 1, TimeSpan.FromMinutes(1), 3, null); // Act messagePump.Stop(); @@ -100,15 +86,27 @@ public void No_message_processed_when_queue_is_empty() var onQueueEmptyInvokeCount = 0; var onErrorInvokeCount = 0; - var queueName = "myqueue"; - var mockQueue = GetMockQueue(queueName); - var mockQueueClient = GetMockQueueClient(mockQueue); - var mockBlobContainer = GetMockBlobContainer(); - var mockBlobClient = GetMockBlobClient(mockBlobContainer); + var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); + var mockQueueClient = MockUtils.GetMockQueueClient(); + + mockQueueClient + .SetupGet(q => q.MaxPeekableMessages) + .Returns(10); + mockQueueClient + .Setup(q => q.GetPropertiesAsync(It.IsAny())) + .ReturnsAsync((CancellationToken cancellationToken) => + { + var queueProperties = QueuesModelFactory.QueueProperties(null, 0); + return Response.FromValue(queueProperties, new MockAzureResponse(200, "ok")); + }); + mockQueueClient + .Setup(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(Response.FromValue(Enumerable.Empty().ToArray(), new MockAzureResponse(200, "ok"))) + .Verifiable(); - mockQueue.Setup(q => q.GetMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(Enumerable.Empty()); + var queueManager = new QueueManager(mockBlobContainerClient.Object, mockQueueClient.Object); - var messagePump = new AsyncMessagePump(queueName, mockQueueClient.Object, mockBlobClient.Object, 1, null, TimeSpan.FromMinutes(1), 3) + var messagePump = new AsyncMessagePump(queueManager, null, 1, TimeSpan.FromMinutes(1), 3, null) { OnMessage = (message, cancellationToken) => { @@ -140,7 +138,7 @@ public void No_message_processed_when_queue_is_empty() // // What this means is that there is no way to precisely assert the number of times the method has been invoked. // The only thing we know for sure, is that 'GetMessagesAsync' has been invoked at least once - mockQueue.Verify(q => q.GetMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.AtLeast(1)); + mockQueueClient.Verify(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.AtLeast(1)); } [Fact] @@ -152,42 +150,56 @@ public void Message_processed() var onErrorInvokeCount = 0; var lockObject = new Object(); - var cloudMessage = new CloudQueueMessage("Message"); + var cloudMessage = QueuesModelFactory.QueueMessage("abc123", "xyz", "Hello World!", 0, null, DateTimeOffset.UtcNow, null); - var queueName = "myqueue"; - var mockQueue = GetMockQueue(queueName); - var mockQueueClient = GetMockQueueClient(mockQueue); - var mockBlobContainer = GetMockBlobContainer(); - var mockBlobClient = GetMockBlobClient(mockBlobContainer); + var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); + var mockQueueClient = MockUtils.GetMockQueueClient(); - mockQueue.Setup(q => q.GetMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync((int messageCount, TimeSpan? visibilityTimeout, QueueRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken) => - { - if (cloudMessage != null) + mockQueueClient + .SetupGet(q => q.MaxPeekableMessages) + .Returns(10); + mockQueueClient + .Setup(q => q.GetPropertiesAsync(It.IsAny())) + .ReturnsAsync((CancellationToken cancellationToken) => { - lock (lockObject) + var messageCount = cloudMessage == null ? 0 : 1; + var queueProperties = QueuesModelFactory.QueueProperties(null, messageCount); + return Response.FromValue(queueProperties, new MockAzureResponse(200, "ok")); + }); + mockQueueClient + .Setup(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((int? maxMessages, TimeSpan? visibilityTimeout, CancellationToken cancellationToken) => + { + if (cloudMessage != null) { - if (cloudMessage != null) + lock (lockObject) { - // DequeueCount is a private property. Therefore we must use reflection to change its value - var dequeueCountProperty = cloudMessage.GetType().GetProperty("DequeueCount"); - dequeueCountProperty.SetValue(cloudMessage, cloudMessage.DequeueCount + 1); - - return new[] { cloudMessage }; + if (cloudMessage != null) + { + // DequeueCount is a private property. Therefore we must use reflection to change its value + var dequeueCountProperty = cloudMessage.GetType().GetProperty("DequeueCount"); + dequeueCountProperty.SetValue(cloudMessage, cloudMessage.DequeueCount + 1); + + return Response.FromValue(new[] { cloudMessage }, new MockAzureResponse(200, "ok")); + } } } - } - return Enumerable.Empty(); - }); - mockQueue.Setup(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Returns((string messageId, string popReceipt, QueueRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken) => - { - lock (lockObject) + return Response.FromValue(Enumerable.Empty().ToArray(), new MockAzureResponse(200, "ok")); + }); + mockQueueClient + .Setup(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((string messageId, string popReceipt, CancellationToken cancellationToken) => { - cloudMessage = null; - } - return Task.FromResult(true); - }); + lock (lockObject) + { + cloudMessage = null; + } + return new MockAzureResponse(200, "ok"); + }); - var messagePump = new AsyncMessagePump(queueName, mockQueueClient.Object, mockBlobClient.Object, 1, null, TimeSpan.FromMinutes(1), 3) + var queueManager = new QueueManager(mockBlobContainerClient.Object, mockQueueClient.Object); + + var messagePump = new AsyncMessagePump(queueManager, null, 1, TimeSpan.FromMinutes(1), 3, null) { OnMessage = (message, cancellationToken) => { @@ -218,8 +230,8 @@ public void Message_processed() onMessageInvokeCount.ShouldBe(1); onQueueEmptyInvokeCount.ShouldBe(1); onErrorInvokeCount.ShouldBe(0); - mockQueue.Verify(q => q.GetMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.AtLeast(2)); - mockQueue.Verify(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(1)); + mockQueueClient.Verify(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.AtLeast(2)); + mockQueueClient.Verify(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(1)); } [Fact] @@ -232,43 +244,58 @@ public void Poison_message_is_rejected() var isRejected = false; var retries = 3; + var lockObject = new Object(); - var cloudMessage = new CloudQueueMessage("Message"); + var cloudMessage = QueuesModelFactory.QueueMessage("abc123", "xyz", "Hello World!", 0, null, DateTimeOffset.UtcNow, null); - var queueName = "myqueue"; - var mockQueue = GetMockQueue(queueName); - var mockQueueClient = GetMockQueueClient(mockQueue); - var mockBlobContainer = GetMockBlobContainer(); - var mockBlobClient = GetMockBlobClient(mockBlobContainer); + var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); + var mockQueueClient = MockUtils.GetMockQueueClient(); - mockQueue.Setup(q => q.GetMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync((int messageCount, TimeSpan? visibilityTimeout, QueueRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken) => - { - if (cloudMessage != null) + mockQueueClient + .SetupGet(q => q.MaxPeekableMessages) + .Returns(10); + mockQueueClient + .Setup(q => q.GetPropertiesAsync(It.IsAny())) + .ReturnsAsync((CancellationToken cancellationToken) => { - lock (lockObject) + var messageCount = cloudMessage == null ? 0 : 1; + var queueProperties = QueuesModelFactory.QueueProperties(null, messageCount); + return Response.FromValue(queueProperties, new MockAzureResponse(200, "ok")); + }); + mockQueueClient + .Setup(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((int? maxMessages, TimeSpan? visibilityTimeout, CancellationToken cancellationToken) => + { + if (cloudMessage != null) { - if (cloudMessage != null) + lock (lockObject) { - // DequeueCount is a private property. Therefore we must use reflection to change its value - var dequeueCountProperty = cloudMessage.GetType().GetProperty("DequeueCount"); - dequeueCountProperty.SetValue(cloudMessage, retries + 1); // intentionally set 'DequeueCount' to a value exceeding maxRetries to simulate a poison message - - return new[] { cloudMessage }; + if (cloudMessage != null) + { + // DequeueCount is a private property. Therefore we must use reflection to change its value + var dequeueCountProperty = cloudMessage.GetType().GetProperty("DequeueCount"); + dequeueCountProperty.SetValue(cloudMessage, retries + 1); // intentionally set 'DequeueCount' to a value exceeding maxRetries to simulate a poison message + + return Response.FromValue(new[] { cloudMessage }, new MockAzureResponse(200, "ok")); + } } } - } - return Enumerable.Empty(); - }); - mockQueue.Setup(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Returns((string messageId, string popReceipt, QueueRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken) => - { - lock (lockObject) + return Response.FromValue(Enumerable.Empty().ToArray(), new MockAzureResponse(200, "ok")); + }); + mockQueueClient + .Setup(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((string messageId, string popReceipt, CancellationToken cancellationToken) => { - cloudMessage = null; - } - return Task.FromResult(true); - }); + lock (lockObject) + { + cloudMessage = null; + } + return new MockAzureResponse(200, "ok"); + }); + + var queueManager = new QueueManager(mockBlobContainerClient.Object, mockQueueClient.Object); - var messagePump = new AsyncMessagePump(queueName, mockQueueClient.Object, mockBlobClient.Object, 1, null, TimeSpan.FromMinutes(1), retries) + var messagePump = new AsyncMessagePump(queueManager, null, 1, TimeSpan.FromMinutes(1), 3, null) { OnMessage = (message, cancellationToken) => { @@ -302,8 +329,8 @@ public void Poison_message_is_rejected() onQueueEmptyInvokeCount.ShouldBe(1); onErrorInvokeCount.ShouldBe(1); isRejected.ShouldBeTrue(); - mockQueue.Verify(q => q.GetMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.AtLeast(2)); - mockQueue.Verify(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(1)); + mockQueueClient.Verify(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.AtLeast(2)); + mockQueueClient.Verify(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(1)); } [Fact] @@ -316,51 +343,66 @@ public void Poison_message_is_moved() var isRejected = false; var retries = 3; - var lockObject = new Object(); - var cloudMessage = new CloudQueueMessage("Message"); - var queueName = "myqueue"; - var poisonQueueName = $"{queueName}-poison"; + var lockObject = new Object(); + var cloudMessage = QueuesModelFactory.QueueMessage("abc123", "xyz", "Hello World!", 0, null, DateTimeOffset.UtcNow, null); - var mockQueue = GetMockQueue(queueName); - var mockPoisonQueue = GetMockQueue(poisonQueueName); - var mockQueueClient = GetMockQueueClient(new[] { mockQueue, mockPoisonQueue }); - var mockBlobContainer = GetMockBlobContainer(); - var mockBlobClient = GetMockBlobClient(mockBlobContainer); + var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); + var mockQueueClient = MockUtils.GetMockQueueClient(); + var mockPoisonQueueClient = MockUtils.GetMockQueueClient("my_poison_queue"); - mockQueue.Setup(q => q.GetMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync((int messageCount, TimeSpan? visibilityTimeout, QueueRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken) => - { - if (cloudMessage != null) + mockQueueClient + .Setup(q => q.GetPropertiesAsync(It.IsAny())) + .ReturnsAsync((CancellationToken cancellationToken) => { - lock (lockObject) + var messageCount = cloudMessage == null ? 0 : 1; + var queueProperties = QueuesModelFactory.QueueProperties(null, messageCount); + return Response.FromValue(queueProperties, new MockAzureResponse(200, "ok")); + }); + mockQueueClient + .Setup(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((int? maxMessages, TimeSpan? visibilityTimeout, CancellationToken cancellationToken) => + { + if (cloudMessage != null) { - if (cloudMessage != null) + lock (lockObject) { - // DequeueCount is a private property. Therefore we must use reflection to change its value - var dequeueCountProperty = cloudMessage.GetType().GetProperty("DequeueCount"); - dequeueCountProperty.SetValue(cloudMessage, retries + 1); // intentionally set 'DequeueCount' to a value exceeding maxRetries to simulate a poison message - - return new[] { cloudMessage }; + if (cloudMessage != null) + { + // DequeueCount is a private property. Therefore we must use reflection to change its value + var dequeueCountProperty = cloudMessage.GetType().GetProperty("DequeueCount"); + dequeueCountProperty.SetValue(cloudMessage, retries + 1); // intentionally set 'DequeueCount' to a value exceeding maxRetries to simulate a poison message + + return Response.FromValue(new[] { cloudMessage }, new MockAzureResponse(200, "ok")); + } } } - } - return Enumerable.Empty(); - }); - mockQueue.Setup(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Returns((string messageId, string popReceipt, QueueRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken) => - { - lock (lockObject) + return Response.FromValue(Enumerable.Empty().ToArray(), new MockAzureResponse(200, "ok")); + }); + mockQueueClient + .Setup(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((string messageId, string popReceipt, CancellationToken cancellationToken) => { - cloudMessage = null; - } - return Task.FromResult(true); - }); - mockPoisonQueue.Setup(q => q.AddMessageAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Returns((CloudQueueMessage message, TimeSpan? timeToLive, TimeSpan? visibilityTimeout, QueueRequestOptions options, OperationContext operationContext, CancellationToken cancellationToken) => - { - // Nothing to do. We just want to ensure this method is invoked. - return Task.FromResult(true); - }); + lock (lockObject) + { + cloudMessage = null; + } + return new MockAzureResponse(200, "ok"); + }); + + mockPoisonQueueClient + .Setup(q => q.SendMessageAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync((string messageText, TimeSpan? visibilityTimeout, TimeSpan? timeToLive, CancellationToken cancellationToken) => + { + // Nothing to do. We just want to ensure this method is invoked. + var sendReceipt = QueuesModelFactory.SendReceipt("abc123", DateTimeOffset.UtcNow, DateTimeOffset.UtcNow.AddDays(7), "xyz", DateTimeOffset.UtcNow); + return Response.FromValue(sendReceipt, new MockAzureResponse(200, "ok")); + }); + + var queueManager = new QueueManager(mockBlobContainerClient.Object, mockQueueClient.Object); + var poisonQueueManager = new QueueManager(mockBlobContainerClient.Object, mockPoisonQueueClient.Object); - var messagePump = new AsyncMessagePump(queueName, mockQueueClient.Object, mockBlobClient.Object, 1, poisonQueueName, TimeSpan.FromMinutes(1), retries) + var messagePump = new AsyncMessagePump(queueManager, poisonQueueManager, 1, TimeSpan.FromMinutes(1), 3, null) { OnMessage = (message, cancellationToken) => { @@ -394,9 +436,9 @@ public void Poison_message_is_moved() onQueueEmptyInvokeCount.ShouldBe(1); onErrorInvokeCount.ShouldBe(1); isRejected.ShouldBeTrue(); - mockQueue.Verify(q => q.GetMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.AtLeast(2)); - mockQueue.Verify(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(1)); - mockPoisonQueue.Verify(q => q.AddMessageAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(1)); + mockQueueClient.Verify(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.AtLeast(2)); + mockQueueClient.Verify(q => q.DeleteMessageAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(1)); + mockPoisonQueueClient.Verify(q => q.SendMessageAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(1)); } [Fact] @@ -410,15 +452,26 @@ public void Exceptions_in_OnQueueEmpty_are_ignored() var exceptionSimulated = false; var lockObject = new Object(); - var queueName = "myqueue"; - var mockQueue = GetMockQueue(queueName); - var mockQueueClient = GetMockQueueClient(mockQueue); - var mockBlobContainer = GetMockBlobContainer(); - var mockBlobClient = GetMockBlobClient(mockBlobContainer); + var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); + var mockQueueClient = MockUtils.GetMockQueueClient(); - mockQueue.Setup(q => q.GetMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).ReturnsAsync(Enumerable.Empty()); + mockQueueClient + .SetupGet(q => q.MaxPeekableMessages) + .Returns(10); + mockQueueClient + .Setup(q => q.GetPropertiesAsync(It.IsAny())) + .ReturnsAsync((CancellationToken cancellationToken) => + { + var queueProperties = QueuesModelFactory.QueueProperties(null, 0); + return Response.FromValue(queueProperties, new MockAzureResponse(200, "ok")); + }); + mockQueueClient + .Setup(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .ReturnsAsync(Response.FromValue(Enumerable.Empty().ToArray(), new MockAzureResponse(200, "ok"))); + + var queueManager = new QueueManager(mockBlobContainerClient.Object, mockQueueClient.Object); - var messagePump = new AsyncMessagePump(queueName, mockQueueClient.Object, mockBlobClient.Object, 1, null, TimeSpan.FromMinutes(1), 3) + var messagePump = new AsyncMessagePump(queueManager, null, 1, TimeSpan.FromMinutes(1), 3, null) { OnMessage = (message, cancellationToken) => { @@ -454,68 +507,7 @@ public void Exceptions_in_OnQueueEmpty_are_ignored() onMessageInvokeCount.ShouldBe(0); onQueueEmptyInvokeCount.ShouldBeGreaterThan(0); onErrorInvokeCount.ShouldBe(0); - mockQueue.Verify(q => q.GetMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.AtLeast(1)); - } - - private static Mock GetMockBlobContainer(string containerName = "mycontainer") - { - var mockContainerUri = new Uri(BLOB_STORAGE_URL + containerName); - var mockBlobContainer = new Mock(MockBehavior.Strict, mockContainerUri); - mockBlobContainer - .Setup(c => c.CreateIfNotExistsAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(true) - .Verifiable(); - return mockBlobContainer; - } - - private static Mock GetMockBlobClient(Mock mockBlobContainer) - { - var mockBlobStorageUri = new Uri(BLOB_STORAGE_URL); - var mockBlobClient = new Mock(MockBehavior.Strict, mockBlobStorageUri, null); - mockBlobClient - .Setup(c => c.GetContainerReference(It.IsAny())) - .Returns(mockBlobContainer.Object) - .Verifiable(); - return mockBlobClient; - } - - private static Mock GetMockQueue(string queueName) - { - var queueAddres = new Uri(QUEUE_STORAGE_URL + queueName); - var mockQueue = new Mock(MockBehavior.Strict, queueAddres); - mockQueue - .Setup(c => c.CreateIfNotExistsAsync(It.IsAny(), It.IsAny(), It.IsAny())) - .ReturnsAsync(false) - .Verifiable(); - return mockQueue; - } - - private static Mock GetMockQueueClient(Mock mockQueue) - { - return GetMockQueueClient(new[] { mockQueue }); - } - - private static Mock GetMockQueueClient(IEnumerable> mockQueues) - { - var mockQueueStorageUri = new Uri(QUEUE_STORAGE_URL); - var storageCredentials = GetStorageCredentials(); - var mockQueueClient = new Mock(MockBehavior.Strict, mockQueueStorageUri, storageCredentials, null); - foreach (var mockQueue in mockQueues.ToArray()) - { - mockQueueClient - .Setup(c => c.GetQueueReference(mockQueue.Object.Name)) - .Returns(mockQueue.Object) - .Verifiable(); - - } - return mockQueueClient; - } - - private static StorageCredentials GetStorageCredentials() - { - var accountAccessKey = Convert.ToBase64String(Encoding.UTF8.GetBytes("this_is_a_bogus_account_access_key")); - var storageCredentials = new StorageCredentials("account_name", accountAccessKey); - return storageCredentials; + mockQueueClient.Verify(q => q.ReceiveMessagesAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.AtLeast(1)); } } } diff --git a/Source/Picton.Messaging.UnitTests/MockAzureResponse.cs b/Source/Picton.Messaging.UnitTests/MockAzureResponse.cs new file mode 100644 index 0000000..2c15493 --- /dev/null +++ b/Source/Picton.Messaging.UnitTests/MockAzureResponse.cs @@ -0,0 +1,89 @@ +// From: https://github.com/Azure/azure-sdk-for-net/blob/master/sdk/core/Azure.Core/tests/TestFramework/MockResponse.cs + +using Azure; +using Azure.Core; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace Picton.Messaging.UnitTests +{ + public class MockAzureResponse : Response + { + private readonly Dictionary> _headers = new Dictionary>(StringComparer.OrdinalIgnoreCase); + + public MockAzureResponse(int status, string reasonPhrase = null) + { + Status = status; + ReasonPhrase = reasonPhrase; + } + + public override int Status { get; } + + public override string ReasonPhrase { get; } + + public override Stream ContentStream { get; set; } + + public override string ClientRequestId { get; set; } + + public bool IsDisposed { get; private set; } + + public void SetContent(byte[] content) + { + ContentStream = new MemoryStream(content); + } + + public void SetContent(string content) + { + SetContent(Encoding.UTF8.GetBytes(content)); + } + + public void AddHeader(HttpHeader header) + { + if (!_headers.TryGetValue(header.Name, out List values)) + { + _headers[header.Name] = values = new List(); + } + + values.Add(header.Value); + } + + protected override bool TryGetHeader(string name, out string value) + { + if (_headers.TryGetValue(name, out List values)) + { + value = JoinHeaderValue(values); + return true; + } + + value = null; + return false; + } + + protected override bool TryGetHeaderValues(string name, out IEnumerable values) + { + var result = _headers.TryGetValue(name, out List valuesList); + values = valuesList; + return result; + } + + protected override bool ContainsHeader(string name) + { + return TryGetHeaderValues(name, out _); + } + + protected override IEnumerable EnumerateHeaders() => _headers.Select(h => new HttpHeader(h.Key, JoinHeaderValue(h.Value))); + + private static string JoinHeaderValue(IEnumerable values) + { + return string.Join(",", values); + } + + public override void Dispose() + { + IsDisposed = true; + } + } +} diff --git a/Source/Picton.Messaging.UnitTests/MockUtils.cs b/Source/Picton.Messaging.UnitTests/MockUtils.cs new file mode 100644 index 0000000..de2b98b --- /dev/null +++ b/Source/Picton.Messaging.UnitTests/MockUtils.cs @@ -0,0 +1,59 @@ +using Azure; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Models; +using Azure.Storage.Queues; +using Moq; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; + +namespace Picton.Messaging.UnitTests +{ + internal static class MockUtils + { + private static readonly string QUEUE_STORAGE_URL = "http://bogus:10001/devstoreaccount1/"; + private static readonly string BLOB_STORAGE_URL = "http://bogus:10002/devstoreaccount1/"; + + internal static Mock GetMockBlobContainerClient(string containerName = "mycontainer", IEnumerable> mockBlobClients = null) + { + var mockContainerUri = new Uri(BLOB_STORAGE_URL + containerName); + var blobContainerInfo = BlobsModelFactory.BlobContainerInfo(ETag.All, DateTimeOffset.UtcNow); + var mockBlobContainer = new Mock(MockBehavior.Strict, mockContainerUri, (BlobClientOptions)null); + + mockBlobContainer + .Setup(c => c.CreateIfNotExists(It.IsAny(), It.IsAny>(), It.IsAny(), It.IsAny())) + .Returns(Response.FromValue(blobContainerInfo, new MockAzureResponse(200, "ok"))) + .Verifiable(); + + foreach (var blobClient in mockBlobClients?.Select(m => m.Object) ?? Enumerable.Empty()) + { + mockBlobContainer + .Setup(c => c.GetBlobClient(blobClient.Name)) + .Returns(blobClient) + .Verifiable(); + } + + return mockBlobContainer; + } + + internal static Mock GetMockQueueClient(string queueName = "myqueue") + { + var mockQueueStorageUri = new Uri(QUEUE_STORAGE_URL + queueName); + var mockQueueClient = new Mock(MockBehavior.Strict, mockQueueStorageUri, (QueueClientOptions)null); + + mockQueueClient + .SetupGet(q => q.MessageMaxBytes) + .Returns(int.MaxValue); + mockQueueClient + .SetupGet(q => q.MaxPeekableMessages) + .Returns(10); + mockQueueClient + .Setup(c => c.CreateIfNotExists(It.IsAny>(), It.IsAny())) + .Returns(new MockAzureResponse(200, "ok")) + .Verifiable(); + + return mockQueueClient; + } + } +} diff --git a/Source/Picton.Messaging/AsyncMessagePump.cs b/Source/Picton.Messaging/AsyncMessagePump.cs index 378aac0..5c2f485 100644 --- a/Source/Picton.Messaging/AsyncMessagePump.cs +++ b/Source/Picton.Messaging/AsyncMessagePump.cs @@ -1,7 +1,4 @@ using App.Metrics; -using Microsoft.Azure.Storage; -using Microsoft.Azure.Storage.Blob; -using Microsoft.Azure.Storage.Queue; using Picton.Interfaces; using Picton.Managers; using Picton.Messaging.Logging; @@ -80,63 +77,42 @@ public class AsyncMessagePump /// /// Initializes a new instance of the class. /// + /// + /// A connection string includes the authentication information required for your application to access data in an Azure Storage account at runtime. + /// For more information, https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string. + /// /// Name of the queue. - /// The cloud storage account. /// The number of concurrent tasks. /// Name of the queue where messages are automatically moved to when they fail to be processed after 'maxDequeueCount' attempts. You can indicate that you do not want messages to be automatically moved by leaving this value empty. In such a scenario, you are responsible for handling so called 'poison' messages. /// The visibility timeout. /// The maximum dequeue count. /// The system where metrics are published. - public AsyncMessagePump(string queueName, CloudStorageAccount storageAccount, int concurrentTasks = 25, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, IMetrics metrics = null) + [ExcludeFromCodeCoverage] + public AsyncMessagePump(string connectionString, string queueName, int concurrentTasks = 25, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, IMetrics metrics = null) + : this(new QueueManager(connectionString, queueName), string.IsNullOrEmpty(poisonQueueName) ? null : new QueueManager(connectionString, poisonQueueName), concurrentTasks, visibilityTimeout, maxDequeueCount, metrics) { - if (storageAccount == null) throw new ArgumentNullException(nameof(storageAccount)); - - if (concurrentTasks < 1) throw new ArgumentException("Number of concurrent tasks must be greather than zero", nameof(concurrentTasks)); - if (maxDequeueCount < 1) throw new ArgumentException("Number of retries must be greather than zero", nameof(maxDequeueCount)); - - var queueClient = storageAccount.CreateCloudQueueClient(); - var blobClient = storageAccount.CreateCloudBlobClient(); - - _queueManager = new QueueManager(queueName, queueClient, blobClient); - _concurrentTasks = concurrentTasks; - _visibilityTimeout = visibilityTimeout; - _maxDequeueCount = maxDequeueCount; - _metrics = metrics ?? TurnOffMetrics(); - - if (!string.IsNullOrEmpty(poisonQueueName)) - { - _poisonQueueManager = new QueueManager(poisonQueueName, queueClient, blobClient); - } - - InitMessagePump(); } /// /// Initializes a new instance of the class. /// - /// Name of the queue. - /// The queue client. - /// The blob client. + /// The queue manager. + /// The poison queue manager. /// The number of concurrent tasks. - /// Name of the queue where messages are automatically moved to when they fail to be processed after 'maxDequeueCount' attempts. You can indicate that you do not want messages to be automatically moved by leaving this value empty. In such a scenario, you are responsible for handling so called 'poison' messages. /// The visibility timeout. /// The maximum dequeue count. /// The system where metrics are published. - public AsyncMessagePump(string queueName, CloudQueueClient queueClient, CloudBlobClient blobClient, int concurrentTasks = 25, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, IMetrics metrics = null) + public AsyncMessagePump(QueueManager queueManager, QueueManager poisonQueueManager, int concurrentTasks = 25, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, IMetrics metrics = null) { if (concurrentTasks < 1) throw new ArgumentException("Number of concurrent tasks must be greather than zero", nameof(concurrentTasks)); if (maxDequeueCount < 1) throw new ArgumentException("Number of retries must be greather than zero", nameof(maxDequeueCount)); - _queueManager = new QueueManager(queueName, queueClient, blobClient); + _queueManager = queueManager ?? throw new ArgumentNullException(nameof(queueManager)); _concurrentTasks = concurrentTasks; _visibilityTimeout = visibilityTimeout; _maxDequeueCount = maxDequeueCount; _metrics = metrics ?? TurnOffMetrics(); - - if (!string.IsNullOrEmpty(poisonQueueName)) - { - _poisonQueueManager = new QueueManager(poisonQueueName, queueClient, blobClient); - } + _poisonQueueManager = poisonQueueManager; InitMessagePump(); } @@ -220,7 +196,7 @@ private async Task ProcessMessages(TimeSpan? visibilityTimeout = null, Cancellat { try { - messages = await _queueManager.GetMessagesAsync(_concurrentTasks, visibilityTimeout, null, null, cancellationToken).ConfigureAwait(false); + messages = await _queueManager.GetMessagesAsync(_concurrentTasks, visibilityTimeout, cancellationToken).ConfigureAwait(false); } catch (TaskCanceledException) { @@ -264,15 +240,14 @@ private async Task ProcessMessages(TimeSpan? visibilityTimeout = null, Cancellat cancellationToken, TaskCreationOptions.LongRunning); - // Define the task that checks how many messages are queued + // Define the task that checks how many messages are queued in the Azure queue RecurrentCancellableTask.StartNew( async () => { try { var count = await _queueManager.GetApproximateMessageCountAsync(cancellationToken).ConfigureAwait(false); - count += queuedMessages.Count; - _metrics.Measure.Gauge.SetValue(Metrics.QueuedMessagesGauge, count); + _metrics.Measure.Gauge.SetValue(Metrics.QueuedCloudMessagesGauge, count); } catch (TaskCanceledException) { @@ -281,13 +256,32 @@ private async Task ProcessMessages(TimeSpan? visibilityTimeout = null, Cancellat } catch (Exception e) { - _logger.InfoException("An error occured while checking how many message are waiting in the queue. The error was caught and ignored.", e.GetBaseException()); + _logger.InfoException("An error occured while checking how many message are waiting in the Azure queue. The error was caught and ignored.", e.GetBaseException()); } }, TimeSpan.FromMilliseconds(5000), cancellationToken, TaskCreationOptions.LongRunning); + // Define the task that checks how many messages are queued in memory + RecurrentCancellableTask.StartNew( + () => + { + try + { + _metrics.Measure.Gauge.SetValue(Metrics.QueuedMemoryMessagesGauge, queuedMessages.Count); + } + catch (Exception e) + { + _logger.InfoException("An error occured while checking how many message are waiting in the memory queue. The error was caught and ignored.", e.GetBaseException()); + } + + return Task.CompletedTask; + }, + TimeSpan.FromMilliseconds(5000), + cancellationToken, + TaskCreationOptions.LongRunning); + // Define the task pump var pumpTask = Task.Run(async () => { @@ -316,7 +310,7 @@ private async Task ProcessMessages(TimeSpan? visibilityTimeout = null, Cancellat // Delete the processed message from the queue // PLEASE NOTE: we use "CancellationToken.None" to ensure a processed message is deleted from the queue even when the message pump is shutting down - await _queueManager.DeleteMessageAsync(message, null, null, CancellationToken.None).ConfigureAwait(false); + await _queueManager.DeleteMessageAsync(message, CancellationToken.None).ConfigureAwait(false); } catch (Exception ex) { @@ -330,10 +324,10 @@ private async Task ProcessMessages(TimeSpan? visibilityTimeout = null, Cancellat message.Metadata["PoisonExceptionMessage"] = ex.GetBaseException().Message; message.Metadata["PoisonExceptionDetails"] = ex.GetBaseException().ToString(); - await _poisonQueueManager.AddMessageAsync(message.Content, message.Metadata, null, null, null, null, CancellationToken.None).ConfigureAwait(false); + await _poisonQueueManager.AddMessageAsync(message.Content, message.Metadata, null, null, CancellationToken.None).ConfigureAwait(false); } - await _queueManager.DeleteMessageAsync(message, null, null, CancellationToken.None).ConfigureAwait(false); + await _queueManager.DeleteMessageAsync(message, CancellationToken.None).ConfigureAwait(false); } } diff --git a/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs b/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs index 8b7e7e1..12bfa52 100644 --- a/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs +++ b/Source/Picton.Messaging/AsyncMessagePumpWithHandlers.cs @@ -1,8 +1,9 @@ using App.Metrics; -using Microsoft.Azure.Storage; using Microsoft.Extensions.DependencyModel; +using Picton.Managers; using Picton.Messaging.Logging; using Picton.Messaging.Messages; +using Picton.Messaging.Utilities; using System; using System.Collections.Generic; using System.Linq; @@ -69,16 +70,34 @@ public Action OnQueueEmpty /// /// Initializes a new instance of the class. /// + /// + /// A connection string includes the authentication information required for your application to access data in an Azure Storage account at runtime. + /// For more information, https://docs.microsoft.com/en-us/azure/storage/common/storage-configure-connection-string. + /// /// Name of the queue. - /// The cloud storage account. /// The number of concurrent tasks. /// Name of the queue where messages are automatically moved to when they fail to be processed after 'maxDequeueCount' attempts. You can indicate that you do not want messages to be automatically moved by leaving this value empty. In such a scenario, you are responsible for handling so called 'poinson' messages. /// The visibility timeout. /// The maximum dequeue count. /// The system where metrics are published. - public AsyncMessagePumpWithHandlers(string queueName, CloudStorageAccount storageAccount, int concurrentTasks = 25, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, IMetrics metrics = null) + [ExcludeFromCodeCoverage] + public AsyncMessagePumpWithHandlers(string connectionString, string queueName, int concurrentTasks = 25, string poisonQueueName = null, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, IMetrics metrics = null) + : this(new QueueManager(connectionString, queueName), string.IsNullOrEmpty(poisonQueueName) ? null : new QueueManager(connectionString, poisonQueueName), concurrentTasks, visibilityTimeout, maxDequeueCount, metrics) { - _messagePump = new AsyncMessagePump(queueName, storageAccount, concurrentTasks, poisonQueueName, visibilityTimeout, maxDequeueCount, metrics) + } + + /// + /// Initializes a new instance of the class. + /// + /// The queue manager. + /// The poison queue manager. + /// The number of concurrent tasks. + /// The visibility timeout. + /// The maximum dequeue count. + /// The system where metrics are published. + public AsyncMessagePumpWithHandlers(QueueManager queueManager, QueueManager poisonQueueManager, int concurrentTasks = 25, TimeSpan? visibilityTimeout = null, int maxDequeueCount = 3, IMetrics metrics = null) + { + _messagePump = new AsyncMessagePump(queueManager, poisonQueueManager, concurrentTasks, visibilityTimeout, maxDequeueCount, metrics) { OnMessage = (message, cancellationToken) => { diff --git a/Source/Picton.Messaging/Extensions.cs b/Source/Picton.Messaging/Extensions.cs index f8633d0..6b7c5cf 100644 --- a/Source/Picton.Messaging/Extensions.cs +++ b/Source/Picton.Messaging/Extensions.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Text; using System.Threading.Tasks; @@ -11,12 +11,14 @@ public static class Extensions { #region PUBLIC EXTENSION METHODS -#pragma warning disable RECS0154 // Parameter is never used /// /// The purpose of this extension method is to avoid a Visual Studio warning about async calls that are not awaited. /// /// The task. +#pragma warning disable RECS0154 // Parameter is never used +#pragma warning disable IDE0060 // Remove unused parameter public static void IgnoreAwait(this Task task) +#pragma warning restore IDE0060 // Remove unused parameter #pragma warning restore RECS0154 // Parameter is never used { // Intentionaly left blank. diff --git a/Source/Picton.Messaging/Metrics.cs b/Source/Picton.Messaging/Metrics.cs index ca7f1b2..97c70d3 100644 --- a/Source/Picton.Messaging/Metrics.cs +++ b/Source/Picton.Messaging/Metrics.cs @@ -1,4 +1,4 @@ -using App.Metrics; +using App.Metrics; using App.Metrics.Counter; using App.Metrics.Gauge; using App.Metrics.Timer; @@ -45,12 +45,22 @@ internal static class Metrics }; /// - /// Gets the gauge indicating the number of messages waiting in the queue over time. + /// Gets the gauge indicating the number of messages waiting in the Azure queue over time. /// - public static GaugeOptions QueuedMessagesGauge => new GaugeOptions + public static GaugeOptions QueuedCloudMessagesGauge => new GaugeOptions { Context = "Picton", - Name = "QueuedMessages", + Name = "QueuedCloudMessages", + MeasurementUnit = Unit.Items + }; + + /// + /// Gets the gauge indicating the number of messages waiting in the memory queue over time. + /// + public static GaugeOptions QueuedMemoryMessagesGauge => new GaugeOptions + { + Context = "Picton", + Name = "QueuedMemoryMessages", MeasurementUnit = Unit.Items }; } diff --git a/Source/Picton.Messaging/Picton.Messaging.csproj b/Source/Picton.Messaging/Picton.Messaging.csproj index 65e5e97..fd60be9 100644 --- a/Source/Picton.Messaging/Picton.Messaging.csproj +++ b/Source/Picton.Messaging/Picton.Messaging.csproj @@ -39,7 +39,7 @@ - + diff --git a/Source/Picton.Messaging/Utilities/ExcludeFromCodeCoverage.cs b/Source/Picton.Messaging/Utilities/ExcludeFromCodeCoverage.cs new file mode 100644 index 0000000..151bfd7 --- /dev/null +++ b/Source/Picton.Messaging/Utilities/ExcludeFromCodeCoverage.cs @@ -0,0 +1,8 @@ +using System; + +namespace Picton.Messaging.Utilities +{ + internal class ExcludeFromCodeCoverage : Attribute + { + } +} diff --git a/build.cake b/build.cake index b63c834..5df61a3 100644 --- a/build.cake +++ b/build.cake @@ -5,10 +5,10 @@ #tool dotnet:?package=BenchmarkDotNet.Tool&version=0.12.0 // Install tools. -#tool nuget:?package=GitVersion.CommandLine&version=5.1.3 -#tool nuget:?package=GitReleaseManager&version=0.10.2 +#tool nuget:?package=GitVersion.CommandLine&version=5.2.4 +#tool nuget:?package=GitReleaseManager&version=0.11.0 #tool nuget:?package=OpenCover&version=4.7.922 -#tool nuget:?package=ReportGenerator&version=4.4.2 +#tool nuget:?package=ReportGenerator&version=4.5.2 #tool nuget:?package=coveralls.io&version=1.4.2 #tool nuget:?package=xunit.runner.console&version=2.4.1 diff --git a/tools/packages.config b/tools/packages.config index 99635e9..0247344 100644 --- a/tools/packages.config +++ b/tools/packages.config @@ -1,4 +1,4 @@ - + From 9cbeaee125c40c695edf3011ebab68ce8ec6e71c Mon Sep 17 00:00:00 2001 From: Jericho Date: Fri, 20 Mar 2020 22:19:41 -0400 Subject: [PATCH 7/7] Remove duplicate moq setup code in unit tests --- .../AsyncMessagePumpTests.cs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs b/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs index 4a6da36..a945df9 100644 --- a/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs +++ b/Source/Picton.Messaging.UnitTests/AsyncMessagePumpTests.cs @@ -89,9 +89,6 @@ public void No_message_processed_when_queue_is_empty() var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); var mockQueueClient = MockUtils.GetMockQueueClient(); - mockQueueClient - .SetupGet(q => q.MaxPeekableMessages) - .Returns(10); mockQueueClient .Setup(q => q.GetPropertiesAsync(It.IsAny())) .ReturnsAsync((CancellationToken cancellationToken) => @@ -155,9 +152,6 @@ public void Message_processed() var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); var mockQueueClient = MockUtils.GetMockQueueClient(); - mockQueueClient - .SetupGet(q => q.MaxPeekableMessages) - .Returns(10); mockQueueClient .Setup(q => q.GetPropertiesAsync(It.IsAny())) .ReturnsAsync((CancellationToken cancellationToken) => @@ -251,9 +245,6 @@ public void Poison_message_is_rejected() var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); var mockQueueClient = MockUtils.GetMockQueueClient(); - mockQueueClient - .SetupGet(q => q.MaxPeekableMessages) - .Returns(10); mockQueueClient .Setup(q => q.GetPropertiesAsync(It.IsAny())) .ReturnsAsync((CancellationToken cancellationToken) => @@ -455,9 +446,6 @@ public void Exceptions_in_OnQueueEmpty_are_ignored() var mockBlobContainerClient = MockUtils.GetMockBlobContainerClient(); var mockQueueClient = MockUtils.GetMockQueueClient(); - mockQueueClient - .SetupGet(q => q.MaxPeekableMessages) - .Returns(10); mockQueueClient .Setup(q => q.GetPropertiesAsync(It.IsAny())) .ReturnsAsync((CancellationToken cancellationToken) =>