Merge branch 'master' into master

README.md

@@ -21,15 +21,15 @@ Core code and libraries are under the Apache open source license; the vendor lib
 
 | OS      | Status |
 | ------- | ------ |
-| Windows | ![Windows status](https://img.shields.io/teamcity/http/teamcity.labkey.org/s/bt83.svg?label=VS%202022) |
-| Native Linux | ![Linux status](https://img.shields.io/teamcity/http/teamcity.labkey.org/s/bt17.svg?label=GCC%204.9) |
-| Wine Linux | ![Docker-Wine status](https://img.shields.io/teamcity/http/teamcity.labkey.org/s/ProteoWizardAndSkylineDockerContainerWineX8664.svg?label=Docker-Wine) |
+| Windows | ![Windows status](https://img.shields.io/teamcity/https/teamcity.labkey.org/s/bt83.svg?label=VS%202022) |
+| Native Linux | ![Linux status](https://img.shields.io/teamcity/https/teamcity.labkey.org/s/bt17.svg?label=GCC%204.9) |
+| Wine Linux | ![Docker-Wine status](https://img.shields.io/teamcity/https/teamcity.labkey.org/s/ProteoWizardAndSkylineDockerContainerWineX8664.svg?label=Docker-Wine) |
 
 Click [here](https://proteowizard.sourceforge.io/download.html) to visit the official download page.
 
 ### Unofficial toolsets
 ![Unofficial toolset build status](https://github.com/ProteoWizard/pwiz/actions/workflows/build_and_test.yml/badge.svg)
 | OS      | Toolset    |
 | ------- | -------    |
-| Linux   | GCC 9.3    |
-| OS X    | Clang 12   |
+| Linux   | GCC 13    |
+| ~~OS X~~    | ~~Clang 12~~   |

pwiz_tools/BiblioSpec/src/BuildParser.cpp

@@ -195,8 +195,7 @@ void BuildParser::setSpecFileName
 {
     curSpecFileName_.clear();
     if( checkFile ){
-        ifstream file(specfile.c_str());
-        if(!file.good()) {
+        if(!boost::filesystem::exists(specfile)) {
             throw BlibException(true, "Could not open spectrum file '%s' for search results file '%s'.", 
                                 specfile.c_str(), fullFilename_.c_str());
         }

pwiz_tools/BiblioSpec/src/DiaNNSpecLibReader.cpp

@@ -691,7 +691,7 @@ bool DiaNNSpecLibReader::parseFile()
         reportReader.read_header(io::ignore_extra_column, "Run", "File.Name", "Protein.Group", "Precursor.Id", "Q.Value", "RT", "RT.Start", "RT.Stop", "IM");
         char *run, *fileName, *proteinGrp, *precursorId;
         float qValue, rt, rtStart, rtStop, im;
-        string currentRun;
+        string currentRun, currentFilename;
         hasSkippedRuns = false;
         while (reportReader.read_row(run, fileName, proteinGrp, precursorId, qValue, rt, rtStart, rtStop, im))
         {
@@ -701,6 +701,7 @@ bool DiaNNSpecLibReader::parseFile()
                 if (processedRuns.count(run) > 0)
                     continue;
                 currentRun = run;
+                currentFilename = fileName;
                 setSpecFileName(run, false);  // make status output report the current spec file
             }
             else if (!bal::equals(currentRun.c_str(), run))
@@ -735,8 +736,14 @@ bool DiaNNSpecLibReader::parseFile()
             psms_.emplace_back(psm);
         }
 
+        // parse spectrum filename from File.Name column (really filepath)
+        bfs::path currentRunFilepath = bal::replace_all_copy(currentFilename, "\\", "/"); // backslash to slash should work on both Linux and Windows
+        string currentRunFilename = currentRunFilepath.filename().string();
+        if (bal::iends_with(currentRunFilename, ".dia")) // trim DIA extension if present
+            currentRunFilename = currentRunFilename.substr(0, currentRunFilename.length() - 4);
+
         // insert PSMs for the current run
-        buildTables(GENERIC_QVALUE, currentRun);
+        buildTables(GENERIC_QVALUE, currentRunFilename);
         processedRuns.insert(currentRun);
         currentRun.clear();
     }

pwiz_tools/BiblioSpec/src/PwizReader.cpp

@@ -437,6 +437,8 @@ void PwizReader::transferSpec(BiblioSpec::SpecData& returnData,
     returnData.retentionTime = specInfo->retentionTime/60;  // seconds to minutes
     if (specInfo->msLevel > 1)
     {
+        if (specInfo->precursors.empty())
+            BiblioSpec::Verbosity::error("MS2 spectrum %s does not have a precursor", specInfo->scanNumber);
         returnData.mz = specInfo->precursors[0].mz;
     }
     returnData.numPeaks = specInfo->data.size();

pwiz_tools/BiblioSpec/src/SslReader.h

@@ -64,6 +64,11 @@ class sslPSM : public PSM {
             } catch (bad_lexical_cast) {
                 if (bal::istarts_with(value, "index="))
                     psm.specIndex = boost::lexical_cast<int>(value.substr(6));
+                else if (bal::istarts_with(value, "scan="))
+                {
+                    psm.specKey = boost::lexical_cast<int>(trimLeadingZeros(value.substr(5)));
+                    psm.specIndex = -1;
+                }
                 else
                     psm.specName = value;
             }

pwiz_tools/BiblioSpec/tests/Jamfile.jam

@@ -269,7 +269,7 @@ blib-test-build tinyPlink-proxl : -o : output/tinyPlink-proxl.blib : tinyPlink-p
 blib-test-build tinyPeptideProphet-proxl : -o : output/tinyPeptideProphet-proxl.blib : tinyPeptideProphet-proxl.check zbuild.skip-lines : $(TEST_INPUTS_PATH)/tinyPeptideProphet.proxl.xml ;
 blib-test-build tinyMerox-proxl : -o : output/tinyMerox-proxl.blib : tinyMerox-proxl.check zbuild.skip-lines : $(TEST_INPUTS_PATH)/tinyMerox.proxl.xml ;
 blib-test-build tiny-msf : --unicode -o : output/tiny-msf.blib : tiny-msf.check zbuild.skip-lines : $(TEST_INPUTS_PATH)/tiny.msf ;
-blib-test-build tiny-msf-keep : --unicode -o -K : output/tiny-msf-keep.blib : tiny-msf-keep.check zbuild.skip-lines : $(TEST_INPUTS_PATH)/tiny.msf ;
+blib-test-build tiny-msf-keep : --unicode -o -K : output/tiny-msf-keep.blib : tiny-msf-keep.check zbuild.skip-lines : $(TEST_INPUTS_PATH)/tiny.msf : <dependency>tiny-msf ;
 blib-test-build tiny-v2-msf : -o : output/tiny-v2-msf.blib : tiny-v2-msf.check zbuild.skip-lines : $(TEST_INPUTS_PATH)/tiny-v2.msf ;
 blib-test-build tiny-v2-filtered-pdResult : -o : output/tiny-v2-filtered-pdResult.blib : tiny-v2-filtered-pdResult.check zbuild.skip-lines : $(TEST_INPUTS_PATH)/tiny-v2-filtered.pdResult ;
 blib-test-build md_special_filtered-pdResult : --unicode -o : output/md_special_filtered-pdResult.blib : md_special_filtered-pdResult.check zbuild.skip-lines : $(TEST_INPUTS_PATH)/md_special_filtered.pdResult ;

pwiz_tools/BiblioSpec/tests/reference/diann-hela-diapasef.check

@@ -2143,3 +2143,4 @@ id	scoreType	probabilityType
 17	BYONIC SCORE	PROBABILITY_THAT_IDENTIFICATION_IS_INCORRECT
 18	PEPTIDE SHAKER CONFIDENCE	PROBABILITY_THAT_IDENTIFICATION_IS_CORRECT
 19	GENERIC Q-VALUE	PROBABILITY_THAT_IDENTIFICATION_IS_INCORRECT
+20	HARDKLOR IDOTP	PROBABILITY_THAT_IDENTIFICATION_IS_CORRECT

pwiz_tools/BiblioSpec/tests/reference/diann-mass-mod-test.check

@@ -792,7 +792,7 @@ id	peptideSeq	precursorMZ	precursorCharge	peptideModSeq	prevAA	nextAA	copies	num
 444	KLVDATQGGPNITFSSIR	1364.92602539	3	KLVDATQGGPN[+2188.7]ITFSSIR	-	-	1	7	0.0	0.0	0.0	0	24.96589851	24.85079765	25.08440017	3.23755607	N/A	N/A	N/A	N/A	N/A	1	505	0.00100401	19
 445	KLVDATQGGPNITFSSIR	1418.94360351	3	KLVDATQGGPN[+2350.8]ITFSSIR	-	-	1	14	0.0	0.0	0.0	0	24.91099929	24.79570007	25.08699989	4.65936931	N/A	N/A	N/A	N/A	N/A	1	506	0.00027532	19
 id	fileName	idFileName	cutoffScore
-1	fissionyeast_NGP_DIA_1	/BiblioSpec/tests/inputs/diann-mass-mods.tsv.speclib	-1.0
+1	fissionyeast_NGP_DIA_1.mzML	/BiblioSpec/tests/inputs/diann-mass-mods.tsv.speclib	-1.0
 id	scoreType	probabilityType
 0	UNKNOWN	NOT_A_PROBABILITY_VALUE
 1	PERCOLATOR QVALUE	PROBABILITY_THAT_IDENTIFICATION_IS_INCORRECT

pwiz_tools/BiblioSpec/tests/reference/diann-mod-test.check

@@ -369,3 +369,4 @@ id	scoreType	probabilityType
 17	BYONIC SCORE	PROBABILITY_THAT_IDENTIFICATION_IS_INCORRECT
 18	PEPTIDE SHAKER CONFIDENCE	PROBABILITY_THAT_IDENTIFICATION_IS_CORRECT
 19	GENERIC Q-VALUE	PROBABILITY_THAT_IDENTIFICATION_IS_INCORRECT
+20	HARDKLOR IDOTP	PROBABILITY_THAT_IDENTIFICATION_IS_CORRECT

pwiz_tools/BiblioSpec/tests/reference/diann-speclib.check

@@ -2338,8 +2338,8 @@ id	peptideSeq	precursorMZ	precursorCharge	peptideModSeq	prevAA	nextAA	copies	num
 2248	YTEQITNEK	563.27557373	2	YTEQITNEK	-	-	1	12	0.0	0.0	0.0	0	12.63070011	12.36610031	12.99969959	3.59048477	N/A	N/A	N/A	N/A	N/A	2	1124	0.00022728	19
 2249	YYDDTYPSVK	625.78320312	2	YYDDTYPSVK	-	-	1	12	0.0	0.0	0.0	0	22.07699966	21.81990051	22.38899993	2.53049253	N/A	N/A	N/A	N/A	N/A	2	1125	0.00019683	19
 id	fileName	idFileName	cutoffScore
-1	collinsb_X1803_171-A	/BiblioSpec/tests/inputs/diann-swath.speclib	-1.0
-2	collinsb_X1803_172-B	/BiblioSpec/tests/inputs/diann-swath.speclib	-1.0
+1	collinsb_X1803_171-A.mzML	/BiblioSpec/tests/inputs/diann-swath.speclib	-1.0
+2	collinsb_X1803_172-B.mzML	/BiblioSpec/tests/inputs/diann-swath.speclib	-1.0
 id	scoreType	probabilityType
 0	UNKNOWN	NOT_A_PROBABILITY_VALUE
 1	PERCOLATOR QVALUE	PROBABILITY_THAT_IDENTIFICATION_IS_INCORRECT
@@ -2361,3 +2361,4 @@ id	scoreType	probabilityType
 17	BYONIC SCORE	PROBABILITY_THAT_IDENTIFICATION_IS_INCORRECT
 18	PEPTIDE SHAKER CONFIDENCE	PROBABILITY_THAT_IDENTIFICATION_IS_CORRECT
 19	GENERIC Q-VALUE	PROBABILITY_THAT_IDENTIFICATION_IS_INCORRECT
+20	HARDKLOR IDOTP	PROBABILITY_THAT_IDENTIFICATION_IS_CORRECT

pwiz_tools/BiblioSpec/tests/reference/msfragger-diann-predicted.check

@@ -27,8 +27,8 @@ id	peptideSeq	precursorMZ	precursorCharge	peptideModSeq	prevAA	nextAA	copies	num
 23	AAIISAEGDSK	531.27813720	2	AAIISAEGDSK	-	-	1	12	0.0	0.0	0.0	0	13.88990020	13.62680053	14.15470027	2.68091187	N/A	N/A	N/A	N/A	N/A	2	10	0.00019543	19
 24	AALSASEGEEVPQDK	765.86859130	2	AALSASEGEEVPQDK	-	-	1	12	0.0	0.0	0.0	0	18.45459938	18.19440078	18.71549987	2.28969038	N/A	N/A	N/A	N/A	N/A	2	11	0.00019605	19
 id	fileName	idFileName	cutoffScore
-1	collinsb_X1803_171-A	/BiblioSpec/tests/inputs/diann-predicted/lib.predicted.speclib	-1.0
-2	collinsb_X1803_172-B	/BiblioSpec/tests/inputs/diann-predicted/lib.predicted.speclib	-1.0
+1	collinsb_X1803_171-A.mzML	/BiblioSpec/tests/inputs/diann-predicted/lib.predicted.speclib	-1.0
+2	collinsb_X1803_172-B.mzML	/BiblioSpec/tests/inputs/diann-predicted/lib.predicted.speclib	-1.0
 id	scoreType	probabilityType
 0	UNKNOWN	NOT_A_PROBABILITY_VALUE
 1	PERCOLATOR QVALUE	PROBABILITY_THAT_IDENTIFICATION_IS_INCORRECT
@@ -50,3 +50,4 @@ id	scoreType	probabilityType
 17	BYONIC SCORE	PROBABILITY_THAT_IDENTIFICATION_IS_INCORRECT
 18	PEPTIDE SHAKER CONFIDENCE	PROBABILITY_THAT_IDENTIFICATION_IS_CORRECT
 19	GENERIC Q-VALUE	PROBABILITY_THAT_IDENTIFICATION_IS_INCORRECT
+20	HARDKLOR IDOTP	PROBABILITY_THAT_IDENTIFICATION_IS_CORRECT

pwiz_tools/BiblioSpec/tests/reference/msfragger-diann.check

@@ -27,8 +27,8 @@ id	peptideSeq	precursorMZ	precursorCharge	peptideModSeq	prevAA	nextAA	copies	num
 23	AAIISAEGDSK	531.27813720	2	AAIISAEGDSK	-	-	1	12	0.0	0.0	0.0	0	13.88990020	13.62680053	14.15470027	2.68091187	N/A	N/A	N/A	N/A	N/A	2	10	0.00019543	19
 24	AALSASEGEEVPQDK	765.86859130	2	AALSASEGEEVPQDK	-	-	1	12	0.0	0.0	0.0	0	18.45459938	18.19440078	18.71549987	2.28969038	N/A	N/A	N/A	N/A	N/A	2	11	0.00019605	19
 id	fileName	idFileName	cutoffScore
-1	collinsb_X1803_171-A	/BiblioSpec/tests/inputs/library.tsv.speclib	-1.0
-2	collinsb_X1803_172-B	/BiblioSpec/tests/inputs/library.tsv.speclib	-1.0
+1	collinsb_X1803_171-A.mzML	/BiblioSpec/tests/inputs/library.tsv.speclib	-1.0
+2	collinsb_X1803_172-B.mzML	/BiblioSpec/tests/inputs/library.tsv.speclib	-1.0
 id	scoreType	probabilityType
 0	UNKNOWN	NOT_A_PROBABILITY_VALUE
 1	PERCOLATOR QVALUE	PROBABILITY_THAT_IDENTIFICATION_IS_INCORRECT
@@ -50,3 +50,4 @@ id	scoreType	probabilityType
 17	BYONIC SCORE	PROBABILITY_THAT_IDENTIFICATION_IS_INCORRECT
 18	PEPTIDE SHAKER CONFIDENCE	PROBABILITY_THAT_IDENTIFICATION_IS_CORRECT
 19	GENERIC Q-VALUE	PROBABILITY_THAT_IDENTIFICATION_IS_INCORRECT
+20	HARDKLOR IDOTP	PROBABILITY_THAT_IDENTIFICATION_IS_CORRECT

pwiz_tools/Shared/PanoramaClient/PanoramaClient.cs

@@ -773,7 +773,7 @@ public override PanoramaServer EnsureLogin(PanoramaServer pServer)
         public override void DownloadFile(string fileUrl, string fileName, long fileSize, string realName, IProgressMonitor pm, IProgressStatus progressStatus)
         {
             // TODO: Change this to use IRequestHelper
-            using var wc = new WebClientWithCredentials(ServerUri, Username, Password);
+            using var wc = new LabkeySessionWebClient(new PanoramaServer(ServerUri, Username, Password));
             wc.DownloadProgressChanged += (s, e) =>
             {
                 var progressPercent = e.ProgressPercentage > 0 ? e.ProgressPercentage : -1;
@@ -818,9 +818,10 @@ public override void DownloadFile(string fileUrl, string fileName, long fileSize
 
         public override IRequestHelper GetRequestHelper(bool forPublish = false)
         {
+            var panoramaServer = new PanoramaServer(ServerUri, Username, Password);
             var webClient = forPublish
-                ? new NonStreamBufferingWebClient(ServerUri, Username, Password)
-                : new WebClientWithCredentials(ServerUri, Username, Password);
+                ? new NonStreamBufferingWebClient(panoramaServer)
+                : new LabkeySessionWebClient(panoramaServer);
             return new PanoramaRequestHelper(webClient);
         }
 
@@ -839,7 +840,7 @@ public string DownloadStringAsync(Uri queryUri, CancellationToken cancelToken)
             string data = null;
             Exception error = null;
 
-            using (var webClient = GetWebClientForServer(new PanoramaServer(ServerUri, Username, Password)))
+            using (var webClient = new LabkeySessionWebClient(new PanoramaServer(ServerUri, Username, Password)))
             {
                 bool finishedDownloading = false;
                 webClient.DownloadStringAsync(queryUri);
@@ -861,15 +862,6 @@ public string DownloadStringAsync(Uri queryUri, CancellationToken cancelToken)
                 throw error;
             return data;
         }
-
-        private static WebClient GetWebClientForServer(PanoramaServer server)
-        {
-            return server.HasUserAccount()
-                ? new WebClientWithCredentials(server.URI, server.Username, server.Password)
-                : new UTF8WebClient();// Use anonymous client if we were not given a username and password. Otherwise, request will fail.
-                                      // Prior to LK 24.11 a call that failed authentication would proceed as an unauthenticated user
-
-        }
     }
 
     /// <summary>

pwiz_tools/Shared/PanoramaClient/PanoramaFilePicker.cs

@@ -123,8 +123,7 @@ private static Uri BuildQuery(string server, string folderPath, string queryName
         /// </summary>
         private JToken GetJson(Uri queryUri)
         {
-            using (var requestHelper = new PanoramaRequestHelper(new WebClientWithCredentials(queryUri, FolderBrowser.GetActiveServer().Username,
-                       FolderBrowser.GetActiveServer().Password)))
+            using (var requestHelper = new PanoramaRequestHelper(new LabkeySessionWebClient(FolderBrowser.GetActiveServer())))
             {
                 JToken json = requestHelper.Get(queryUri);
                 return json;

pwiz_tools/Shared/PanoramaClient/PanoramaFolderBrowser.cs

@@ -541,7 +541,7 @@ private void AddWebDavFolders(TreeNode node)
             try
             {
                 query = new Uri(string.Concat(folderInfo.Server.URI, PanoramaUtil.WEBDAV, folderInfo.FolderPath, "?method=json"));
-                using var requestHelper = new PanoramaRequestHelper(new WebClientWithCredentials(query, folderInfo.Server.Username, folderInfo.Server.Password));
+                using var requestHelper = new PanoramaRequestHelper(new LabkeySessionWebClient(folderInfo.Server));
                 JToken json = requestHelper.Get(query);
                 if ((int)json[@"fileCount"] != 0)
                 {

pwiz_tools/Shared/PanoramaClient/PanoramaUtil.cs

@@ -642,37 +642,43 @@ public UTF8WebClient()
         }
     }
 
-    public class WebClientWithCredentials : UTF8WebClient
+    public class LabkeySessionWebClient : UTF8WebClient
     {
         private CookieContainer _cookies = new CookieContainer();
         private string _csrfToken;
         private readonly Uri _serverUri;
 
-        private static string LABKEY_CSRF = @"X-LABKEY-CSRF";
+        private const string LABKEY_CSRF = @"X-LABKEY-CSRF";
 
-        public WebClientWithCredentials(Uri serverUri, string username, string password)
+        public LabkeySessionWebClient(PanoramaServer server)
         {
-            // Add the Authorization header
-            Headers.Add(HttpRequestHeader.Authorization, PanoramaServer.GetBasicAuthHeader(username, password));
-            _serverUri = serverUri;
+            if (server == null)
+            {
+                throw new ArgumentNullException(nameof(server));
+            }
+            // Add the Authorization header only if a username and password is provided. Otherwise, the request will fail.
+            // Prior to LK 24.11 a call that failed authentication would proceed as an unauthenticated user
+            if (server.HasUserAccount())
+            {
+                Headers.Add(HttpRequestHeader.Authorization, server.AuthHeader);
+            }
+
+            _serverUri = server.URI;
         }
 
         protected override WebRequest GetWebRequest(Uri address)
         {
             var request = base.GetWebRequest(address);
-
-            var httpWebRequest = request as HttpWebRequest;
-            if (httpWebRequest != null)
+
+            if (request is HttpWebRequest httpWebRequest)
             {
                 httpWebRequest.CookieContainer = _cookies;
 
-                if (request.Method == PanoramaUtil.FORM_POST)
+                if (request.Method == PanoramaUtil.FORM_POST && !string.IsNullOrEmpty(_csrfToken))
                 {
-                    if (!string.IsNullOrEmpty(_csrfToken))
-                    {
-                        // All POST requests to LabKey Server will be checked for a CSRF token
-                        request.Headers.Add(LABKEY_CSRF, _csrfToken);
-                    }
+                    // All POST requests to LabKey Server will be checked for a CSRF token
+                    request.Headers.Add(LABKEY_CSRF, _csrfToken);
+
                 }
             }
             return request;
@@ -681,8 +687,7 @@ protected override WebRequest GetWebRequest(Uri address)
         protected override WebResponse GetWebResponse(WebRequest request)
         {
             var response = base.GetWebResponse(request);
-            var httpResponse = response as HttpWebResponse;
-            if (httpResponse != null)
+            if (response is HttpWebResponse httpResponse)
             {
                 GetCsrfToken(httpResponse, request.RequestUri);
             }
@@ -742,10 +747,10 @@ public void ClearCsrfToken()
         }
     }
 
-    public class NonStreamBufferingWebClient : WebClientWithCredentials
+    public class NonStreamBufferingWebClient : LabkeySessionWebClient
     {
-        public NonStreamBufferingWebClient(Uri serverUri, string username, string password)
-            : base(serverUri, username, password)
+        public NonStreamBufferingWebClient(PanoramaServer server)
+            : base(server)
         {
         }
 

pwiz_tools/Shared/PanoramaClient/RequestHelper.cs

@@ -195,9 +195,9 @@ private JObject ParseResponse(string response, Uri uri, string messageOnError)
 
     public class PanoramaRequestHelper : AbstractRequestHelper
     {
-        private readonly WebClientWithCredentials _client;
+        private readonly LabkeySessionWebClient _client;
 
-        public PanoramaRequestHelper(WebClientWithCredentials webClient)
+        public PanoramaRequestHelper(LabkeySessionWebClient webClient)
         {
             _client = webClient;
         }