diff --git a/AppDynamics.Dexter.Core.csproj b/AppDynamics.Dexter.Core.csproj index 13ae644..7a8ac5c 100644 --- a/AppDynamics.Dexter.Core.csproj +++ b/AppDynamics.Dexter.Core.csproj @@ -42,12 +42,12 @@ - - - - - - + + + + + + diff --git a/AppDynamics.Dexter.csproj b/AppDynamics.Dexter.csproj index 85b8328..5f077ee 100644 --- a/AppDynamics.Dexter.csproj +++ b/AppDynamics.Dexter.csproj @@ -66,20 +66,21 @@ packages\CommandLineParser.1.9.71\lib\net45\CommandLine.dll - - packages\CsvHelper.6.0.0\lib\net45\CsvHelper.dll + + packages\CsvHelper.7.1.1\lib\net45\CsvHelper.dll - - packages\EPPlus.4.5.0.1-beta\lib\net40\EPPlus.dll + + packages\EPPlus.4.5.2.1\lib\net40\EPPlus.dll - - packages\Newtonsoft.Json.10.0.3\lib\net45\Newtonsoft.Json.dll + + packages\Newtonsoft.Json.11.0.2\lib\net45\Newtonsoft.Json.dll - packages\NLog.4.5.0-beta07\lib\net45\NLog.dll + packages\NLog.4.5.6\lib\net45\NLog.dll + @@ -278,7 +279,7 @@ - Always + PreserveNewest Designer diff --git a/EPPlus/EPPlus-master.zip b/EPPlus/EPPlus-master.zip index de1b10a..c494887 100644 Binary files a/EPPlus/EPPlus-master.zip and b/EPPlus/EPPlus-master.zip differ diff --git a/Helpers/FileIOHelper.cs b/Helpers/FileIOHelper.cs index 7206a8f..39ce577 100644 --- a/Helpers/FileIOHelper.cs +++ b/Helpers/FileIOHelper.cs @@ -370,6 +370,8 @@ public static bool WriteListToCSVFile(List listToWrite, ClassMap classM public static bool WriteListToCSVFile(List listToWrite, ClassMap classMap, string csvFilePath, bool appendToExistingFile) { + if (listToWrite == null) return true; + string folderPath = Path.GetDirectoryName(csvFilePath); if (CreateFolder(folderPath) == true) @@ -419,6 +421,8 @@ public static MemoryStream WriteListToMemoryStream(List listToWrite, Class { try { + if (listToWrite == null) return null; + logger.Trace("Writing list with {0} elements containing type {1} to memory stream", listToWrite.Count, typeof(T)); MemoryStream ms = new MemoryStream(1024 * listToWrite.Count); diff --git a/NLog.xsd b/NLog.xsd index 676e629..1bd61d8 100644 --- a/NLog.xsd +++ b/NLog.xsd @@ -331,16 +331,11 @@ - + - - - Name of the target. - - Delay the flush until the LogEvent has been confirmed as written @@ -351,6 +346,11 @@ Condition expression. Log events who meet this condition will cause a flush on the wrapped target. + + + Name of the target. + + Target supports reuse of internal buffers, and doesn't have to constantly allocate new buffers Required for legacy NLog-targets, that expects buffers to remain stable after Write-method exit @@ -366,8 +366,8 @@ - + @@ -385,14 +385,14 @@ Timeout (in milliseconds) after which the contents of buffer will be flushed if there's no write in the specified period of time. Use -1 to disable timed flushes. - + - Indicates whether to use sliding timeout. + Action to take if the buffer overflows. - + - Action to take if the buffer overflows. + Indicates whether to use sliding timeout. @@ -420,23 +420,25 @@ - + - + + + - + - + @@ -474,9 +476,9 @@ Action that should be taken if the will be more connections than . - + - Action that should be taken if the message is larger than maxMessageSize. + Maximum queue size. @@ -499,9 +501,14 @@ Network address. - + - Maximum queue size. + Action that should be taken if the message is larger than maxMessageSize. + + + + + NDLC item separator. @@ -509,14 +516,14 @@ NDC item separator. - + - Indicates whether to include source info (file name and line number) in the information sent over the network. + Indicates whether to include NLog-specific extensions to log4j schema. - + - Indicates whether to include dictionary contents. + Indicates whether to include source info (file name and line number) in the information sent over the network. @@ -529,6 +536,11 @@ Indicates whether to include stack contents. + + + Indicates whether to include dictionary contents. + + Indicates whether to include dictionary contents. @@ -549,9 +561,9 @@ AppInfo field. By default it's the friendly name of the current AppDomain. - + - Indicates whether to include NLog-specific extensions to log4j schema. + Renderer for log4j:event logger-xml-attribute (Default ${logger}) @@ -600,12 +612,12 @@ - - - + + + @@ -628,11 +640,6 @@ Footer. - - - Indicates whether to use default row highlighting rules. - - Indicates whether to auto-check if the console is available. - Disables console writing if Environment.UserInteractive = False (Windows Service) - Disables console writing if Console Standard Input is not available (Non-Console-App) @@ -648,6 +655,11 @@ Indicates whether the error stream (stderr) should be used instead of the output stream (stdout). + + + Indicates whether to use default row highlighting rules. + + Target supports reuse of internal buffers, and doesn't have to constantly allocate new buffers Required for legacy NLog-targets, that expects buffers to remain stable after Write-method exit @@ -701,14 +713,19 @@ + - + + + Compile the ? This can improve the performance, but at the costs of more memory usage. If false, the Regex Cache is used. + + Indicates whether to ignore case when comparing texts. @@ -729,11 +746,6 @@ Indicates whether to match whole words only. - - - Compile the ? This can improve the performance, but at the costs of more memory usage. If false, the Regex Cache is used. - - Background color. @@ -753,9 +765,9 @@ - + @@ -778,11 +790,6 @@ Footer. - - - Indicates whether to send the log messages to the standard error instead of the standard output. - - Indicates whether to auto-check if the console is available - Disables console writing if Environment.UserInteractive = False (Windows Service) - Disables console writing if Console Standard Input is not available (Non-Console-App) @@ -793,6 +800,11 @@ The encoding for writing messages to the . + + + Indicates whether to send the log messages to the standard error instead of the standard output. + + Target supports reuse of internal buffers, and doesn't have to constantly allocate new buffers Required for legacy NLog-targets, that expects buffers to remain stable after Write-method exit @@ -1037,14 +1049,14 @@ + - - - + + @@ -1062,6 +1074,11 @@ Layout that renders event Category. + + + Optional entrytype. When not set, or when not convertable to then determined by + + Layout that renders event ID. @@ -1077,29 +1094,24 @@ Name of the machine on which Event Log service is running. - - - Value to be used as the event Source. - - - + - Action to take if the message is larger than the option. + Maximum Event log size in kilobytes. If null, the value won't be set. Default is 512 Kilobytes as specified by Eventlog API - + - Optional entrytype. When not set, or when not convertable to then determined by + Message length limit to write to the Event Log. - + - Maximum Event log size in kilobytes. If null, the value won't be set. Default is 512 Kilobytes as specified by Eventlog API + Value to be used as the event Source. - + - Message length limit to write to the Event Log. + Action to take if the message is larger than the option. @@ -1153,38 +1165,39 @@ + + - - - - - - - - + + - - + + - + + + + - + + - + + @@ -1216,39 +1229,19 @@ Line ending mode. - - - Way file archives are numbered. - - - - - Name of the file to be used for an archive. - - - - - Indicates whether to automatically archive log files every time the specified time passes. - - - - - Size in bytes above which log files will be automatically archived. Warning: combining this with isn't supported. We cannot create multiple archive files, if they should have the same name. Choose: - - Indicates whether to compress archive files into the zip archive format. - + - Maximum number of archive files that should be kept. + Way file archives are numbered. - + - Gets or set a value indicating whether a managed file stream is forced, instead of using the native implementation. + Name of the file to be used for an archive. @@ -1256,24 +1249,24 @@ Is the an absolute or relative path? - + - Cleanup invalid values in a filename, e.g. slashes in a filename. If set to true, this can impact the performance of massive writes. If set to false, nothing gets written when the filename is wrong. + Indicates whether to automatically archive log files every time the specified time passes. - + - Whether or not this target should just discard all data that its asked to write. Mostly used for when testing NLog Stack except final write + Size in bytes above which log files will be automatically archived. Warning: combining this with isn't supported. We cannot create multiple archive files, if they should have the same name. Choose: - + - Is the an absolute or relative path? + Maximum number of archive files that should be kept. - + - Value indicationg whether file creation calls should be synchronized by a system global mutex. + Indicates whether the footer should be written only when the file is archived. @@ -1281,11 +1274,6 @@ Maximum number of log filenames that should be stored as existing. - - - Indicates whether the footer should be written only when the file is archived. - - Name of the file to write to. @@ -1301,19 +1289,34 @@ Indicates whether to archive old log file on startup. + + + Cleanup invalid values in a filename, e.g. slashes in a filename. If set to true, this can impact the performance of massive writes. If set to false, nothing gets written when the filename is wrong. + + Indicates whether to create directories if they do not exist. + + + Indicates whether to delete old log file on startup. + + File attributes (Windows only). - + - Indicates whether to delete old log file on startup. + Indicates whether to write BOM (byte order mark) in created files + + + + + Indicates whether to enable log file(s) to be deleted. @@ -1321,14 +1324,19 @@ Indicates whether to replace file contents on each write instead of appending log message at the end. - + - Indicates whether to enable log file(s) to be deleted. + Value indicationg whether file creation calls should be synchronized by a system global mutex. - + - Number of times the write is appended on the file before NLog discards the log message. + Gets or set a value indicating whether a managed file stream is forced, instead of using the native implementation. + + + + + Is the an absolute or relative path? @@ -1336,9 +1344,14 @@ Indicates whether concurrent writes to the log file by multiple processes on the same host. - + - Indicates whether to keep log file open instead of opening and closing it on each logging event. + Whether or not this target should just discard all data that its asked to write. Mostly used for when testing NLog Stack except final write + + + + + Delay in milliseconds to wait before attempting to write to the file again. @@ -1371,9 +1384,14 @@ Indicates whether to automatically flush the file buffers after each log message. - + - Delay in milliseconds to wait before attempting to write to the file again. + Number of times the write is appended on the file before NLog discards the log message. + + + + + Indicates whether to keep log file open instead of opening and closing it on each logging event. @@ -1387,6 +1405,13 @@ + + + + + + + @@ -1404,13 +1429,6 @@ - - - - - - - @@ -1546,16 +1564,11 @@ - - - - - - - Name of the target. - - + + + + Interval in which messages will be written up to the number of messages. @@ -1566,6 +1579,11 @@ Maximum allowed number of messages written per . + + + Name of the target. + + Target supports reuse of internal buffers, and doesn't have to constantly allocate new buffers Required for legacy NLog-targets, that expects buffers to remain stable after Write-method exit @@ -1637,7 +1655,6 @@ - @@ -1659,11 +1676,6 @@ Type of the parameter. Obsolete alias for - - - Parameter can combine multiple LogEvents into a single parameter value - - @@ -1673,19 +1685,19 @@ - + + - - - + + + + - - - - + + @@ -1716,9 +1728,14 @@ Footer. - + - Indicates whether to send message as HTML instead of plain text. + Indicates whether NewLine characters in the body should be replaced with tags. + + + + + Priority used for sending mails. @@ -1726,9 +1743,9 @@ Encoding to be used for sending e-mail. - + - Indicates whether to add new lines between log entries. + BCC email addresses separated by semicolons (e.g. john@domain.com;jane@domain.com). @@ -1736,24 +1753,14 @@ CC email addresses separated by semicolons (e.g. john@domain.com;jane@domain.com). - - - Recipients' email addresses separated by semicolons (e.g. john@domain.com;jane@domain.com). - - - - - BCC email addresses separated by semicolons (e.g. john@domain.com;jane@domain.com). - - - + - Mail message body (repeated for each log message send in one mail). + Indicates whether to add new lines between log entries. - + - Mail subject. + Indicates whether to send message as HTML instead of plain text. @@ -1761,19 +1768,19 @@ Sender's email address (e.g. joe@domain.com). - + - Indicates the SMTP client timeout. + Mail message body (repeated for each log message send in one mail). - + - Priority used for sending mails. + Mail subject. - + - Indicates whether NewLine characters in the body should be replaced with tags. + Recipients' email addresses separated by semicolons (e.g. john@domain.com;jane@domain.com). @@ -1781,6 +1788,11 @@ Target supports reuse of internal buffers, and doesn't have to constantly allocate new buffers Required for legacy NLog-targets, that expects buffers to remain stable after Write-method exit + + + Indicates the SMTP client timeout. + + SMTP Server to be used for sending. @@ -1912,13 +1924,13 @@ - - + + @@ -1951,16 +1963,6 @@ Indicates whether to append newline at the end of log message. - - - Action that should be taken if the will be more connections than . - - - - - Action that should be taken if the message is larger than maxMessageSize. - - Network address. @@ -1986,6 +1988,16 @@ Maximum queue size. + + + Action that should be taken if the will be more connections than . + + + + + Action that should be taken if the message is larger than maxMessageSize. + + Target supports reuse of internal buffers, and doesn't have to constantly allocate new buffers Required for legacy NLog-targets, that expects buffers to remain stable after Write-method exit @@ -2005,23 +2017,25 @@ - + - + + + - + - + @@ -2059,9 +2073,9 @@ Action that should be taken if the will be more connections than . - + - Action that should be taken if the message is larger than maxMessageSize. + Maximum queue size. @@ -2084,9 +2098,14 @@ Network address. - + - Maximum queue size. + Action that should be taken if the message is larger than maxMessageSize. + + + + + NDLC item separator. @@ -2094,14 +2113,14 @@ NDC item separator. - + - Indicates whether to include source info (file name and line number) in the information sent over the network. + Indicates whether to include NLog-specific extensions to log4j schema. - + - Indicates whether to include dictionary contents. + Indicates whether to include source info (file name and line number) in the information sent over the network. @@ -2114,6 +2133,11 @@ Indicates whether to include stack contents. + + + Indicates whether to include dictionary contents. + + Indicates whether to include dictionary contents. @@ -2134,9 +2158,9 @@ AppInfo field. By default it's the friendly name of the current AppDomain. - + - Indicates whether to include NLog-specific extensions to log4j schema. + Renderer for log4j:event logger-xml-attribute (Default ${logger}) @@ -2497,74 +2521,81 @@ - - - - - + - + + + + + + + Name of the target. + + + Target supports reuse of internal buffers, and doesn't have to constantly allocate new buffers Required for legacy NLog-targets, that expects buffers to remain stable after Write-method exit + + Should we include the BOM (Byte-order-mark) for UTF? Influences the property. This will only work for UTF-8. - + - Target supports reuse of internal buffers, and doesn't have to constantly allocate new buffers Required for legacy NLog-targets, that expects buffers to remain stable after Write-method exit + Web service method name. Only used with Soap. - + - Encoding. + Web service namespace. Only used with Soap. - + - Value whether escaping be done according to the old NLog style (Very non-standard) + Protocol to be used when calling web service. - + - Value whether escaping be done according to Rfc3986 (Supports Internationalized Resource Identifiers - IRIs) + Custom proxy address, include port separated by a colon - + - Web service method name. Only used with Soap. + Encoding. - + - Web service namespace. Only used with Soap. + Web service URL. - + - Indicates whether to pre-authenticate the HttpWebRequest (Requires 'Authorization' in parameters) + Value whether escaping be done according to the old NLog style (Very non-standard) - + - Protocol to be used when calling web service. + Value whether escaping be done according to Rfc3986 (Supports Internationalized Resource Identifiers - IRIs) - + - Web service URL. + Indicates whether to pre-authenticate the HttpWebRequest (Requires 'Authorization' in parameters) @@ -2577,6 +2608,11 @@ (optional) root namespace of the XML document, if POST of XML document chosen. (see and ). + + + Proxy configuration when calling web service + + @@ -2590,6 +2626,14 @@ + + + + + + + + @@ -2704,6 +2748,7 @@ + @@ -2712,7 +2757,7 @@ - Option to include all properties from the log events + Option to include all properties from the log event (as JSON) @@ -2735,16 +2780,32 @@ Option to suppress the extra spaces in the output json + + + How far should the JSON serializer follow object references before backing off + + - - + + + + + + Layout that will be rendered as the attribute's value. + + + + + Name of the attribute. + + Determines wether or not this attribute will be Json encoded. @@ -2755,14 +2816,9 @@ Indicates whether to escape non-ascii characters - - - Layout that will be rendered as the attribute's value. - - - + - Name of the attribute. + Whether an attribute with empty value should be included in the output @@ -3006,45 +3062,35 @@ - - - + + - + + Action to be taken when filter matches. - - - Layout to be used to filter log messages. - - Default number of unique filter values to expect, will automatically increase if needed - - - Append FilterCount to the when an event is no longer filtered - - - + - Insert FilterCount value into when an event is no longer filtered + Applies the configured action to the initial logevent that starts the timeout period. Used to configure that it should ignore all events until timeout. - + - Applies the configured action to the initial logevent that starts the timeout period. Used to configure that it should ignore all events until timeout. + Layout to be used to filter log messages. @@ -3057,6 +3103,11 @@ Max length of filter values, will truncate if above limit + + + How long before a filter expires, and logging is accepted again + + Default buffer size for the internal buffers @@ -3067,9 +3118,14 @@ Reuse internal buffers, and doesn't have to constantly allocate new buffers - + - How long before a filter expires, and logging is accepted again + Append FilterCount to the when an event is no longer filtered + + + + + Insert FilterCount value into when an event is no longer filtered diff --git a/ProcessingSteps/Extract/ExtractApplicationAndEntityFlowmaps.cs b/ProcessingSteps/Extract/ExtractApplicationAndEntityFlowmaps.cs index a9f512a..6411f8c 100644 --- a/ProcessingSteps/Extract/ExtractApplicationAndEntityFlowmaps.cs +++ b/ProcessingSteps/Extract/ExtractApplicationAndEntityFlowmaps.cs @@ -327,6 +327,8 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job { logger.Warn(ex); loggerConsole.Warn(ex); + + return false; } finally { diff --git a/ProcessingSteps/Extract/ExtractApplicationAndEntityMetrics.cs b/ProcessingSteps/Extract/ExtractApplicationAndEntityMetrics.cs index 20153e8..e1fe86e 100644 --- a/ProcessingSteps/Extract/ExtractApplicationAndEntityMetrics.cs +++ b/ProcessingSteps/Extract/ExtractApplicationAndEntityMetrics.cs @@ -147,6 +147,8 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job { logger.Warn(ex); loggerConsole.Warn(ex); + + return false; } finally { diff --git a/ProcessingSteps/Extract/ExtractControllerAndApplicationConfiguration.cs b/ProcessingSteps/Extract/ExtractControllerAndApplicationConfiguration.cs index c94b40a..fc65957 100644 --- a/ProcessingSteps/Extract/ExtractControllerAndApplicationConfiguration.cs +++ b/ProcessingSteps/Extract/ExtractControllerAndApplicationConfiguration.cs @@ -98,6 +98,8 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job { logger.Warn(ex); loggerConsole.Warn(ex); + + return false; } finally { diff --git a/ProcessingSteps/Extract/ExtractControllerApplicationsAndEntities.cs b/ProcessingSteps/Extract/ExtractControllerApplicationsAndEntities.cs index 413247d..1cb7705 100644 --- a/ProcessingSteps/Extract/ExtractControllerApplicationsAndEntities.cs +++ b/ProcessingSteps/Extract/ExtractControllerApplicationsAndEntities.cs @@ -228,30 +228,6 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job } ); - //Parallel.ForEach( - // nodesList, - // new ParallelOptions { MaxDegreeOfParallelism = NODE_PROPERTIES_EXTRACT_NUMBER_OF_THREADS }, - // () => 0, - // (node, loop, subtotal) => - // { - // ControllerApi controllerApiLocal = new ControllerApi(jobTarget.Controller, jobTarget.UserName, AESEncryptionHelper.Decrypt(jobTarget.UserPassword)); - // controllerApiLocal.PrivateApiLogin(); - - // string nodePropertiesJSON = controllerApi.GetNodeProperties(node.id); - // if (nodePropertiesJSON != String.Empty) FileIOHelper.SaveFileToPath(nodePropertiesJSON, FilePathMap.NodeRuntimePropertiesDataFilePath(jobTarget, jobConfiguration.Input.TimeRange, node)); - - // return 1; - // }, - // (finalResult) => - // { - // Interlocked.Add(ref j, finalResult); - // if (j % 10 == 0) - // { - // Console.Write("[{0}].", j); - // } - // } - //); - loggerConsole.Info("Completed {0} Nodes", nodesList.Count); } @@ -261,6 +237,8 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job { logger.Warn(ex); loggerConsole.Warn(ex); + + return false; } finally { diff --git a/ProcessingSteps/Extract/ExtractEventsAndHealthRuleViolations.cs b/ProcessingSteps/Extract/ExtractEventsAndHealthRuleViolations.cs index c69a55b..d5383a6 100644 --- a/ProcessingSteps/Extract/ExtractEventsAndHealthRuleViolations.cs +++ b/ProcessingSteps/Extract/ExtractEventsAndHealthRuleViolations.cs @@ -113,6 +113,8 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job { logger.Warn(ex); loggerConsole.Warn(ex); + + return false; } finally { diff --git a/ProcessingSteps/Extract/ExtractSnapshots.cs b/ProcessingSteps/Extract/ExtractSnapshots.cs index 92d80b1..b64226d 100644 --- a/ProcessingSteps/Extract/ExtractSnapshots.cs +++ b/ProcessingSteps/Extract/ExtractSnapshots.cs @@ -465,6 +465,7 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job { logger.Warn(ex); loggerConsole.Warn(ex); + return false; } finally { diff --git a/ProcessingSteps/Index/IndexApplicationAndEntityFlowmaps.cs b/ProcessingSteps/Index/IndexApplicationAndEntityFlowmaps.cs index c664bff..0e521ac 100644 --- a/ProcessingSteps/Index/IndexApplicationAndEntityFlowmaps.cs +++ b/ProcessingSteps/Index/IndexApplicationAndEntityFlowmaps.cs @@ -263,6 +263,8 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job { logger.Warn(ex); loggerConsole.Warn(ex); + + return false; } finally { diff --git a/ProcessingSteps/Index/IndexApplicationAndEntityMetrics.cs b/ProcessingSteps/Index/IndexApplicationAndEntityMetrics.cs index 86097dd..69d8280 100644 --- a/ProcessingSteps/Index/IndexApplicationAndEntityMetrics.cs +++ b/ProcessingSteps/Index/IndexApplicationAndEntityMetrics.cs @@ -984,6 +984,8 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job { logger.Warn(ex); loggerConsole.Warn(ex); + + return false; } finally { diff --git a/ProcessingSteps/Index/IndexApplicationConfigurationDifferences.cs b/ProcessingSteps/Index/IndexApplicationConfigurationDifferences.cs index f80f086..75bc246 100644 --- a/ProcessingSteps/Index/IndexApplicationConfigurationDifferences.cs +++ b/ProcessingSteps/Index/IndexApplicationConfigurationDifferences.cs @@ -326,6 +326,8 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job { logger.Warn(ex); loggerConsole.Warn(ex); + + return false; } finally { diff --git a/ProcessingSteps/Index/IndexControllerAndApplicationConfiguration.cs b/ProcessingSteps/Index/IndexControllerAndApplicationConfiguration.cs index 2c1716b..94557b1 100644 --- a/ProcessingSteps/Index/IndexControllerAndApplicationConfiguration.cs +++ b/ProcessingSteps/Index/IndexControllerAndApplicationConfiguration.cs @@ -918,6 +918,8 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job { logger.Warn(ex); loggerConsole.Warn(ex); + + return false; } finally { diff --git a/ProcessingSteps/Index/IndexControllersApplicationsAndEntities.cs b/ProcessingSteps/Index/IndexControllersApplicationsAndEntities.cs index 57cb37f..99e1903 100644 --- a/ProcessingSteps/Index/IndexControllersApplicationsAndEntities.cs +++ b/ProcessingSteps/Index/IndexControllersApplicationsAndEntities.cs @@ -941,6 +941,8 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job { logger.Warn(ex); loggerConsole.Warn(ex); + + return false; } finally { diff --git a/ProcessingSteps/Index/IndexEventsAndHealthRuleViolations.cs b/ProcessingSteps/Index/IndexEventsAndHealthRuleViolations.cs index 52044fe..15b6c36 100644 --- a/ProcessingSteps/Index/IndexEventsAndHealthRuleViolations.cs +++ b/ProcessingSteps/Index/IndexEventsAndHealthRuleViolations.cs @@ -358,6 +358,8 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job { logger.Warn(ex); loggerConsole.Warn(ex); + + return false; } finally { diff --git a/ProcessingSteps/Index/IndexSnapshots.cs b/ProcessingSteps/Index/IndexSnapshots.cs index f8e4f39..7773c55 100644 --- a/ProcessingSteps/Index/IndexSnapshots.cs +++ b/ProcessingSteps/Index/IndexSnapshots.cs @@ -636,6 +636,8 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job { logger.Warn(ex); loggerConsole.Warn(ex); + + return false; } finally { diff --git a/ProcessingSteps/JobStepBase.cs b/ProcessingSteps/JobStepBase.cs index 7a5cd60..f7d8acc 100644 --- a/ProcessingSteps/JobStepBase.cs +++ b/ProcessingSteps/JobStepBase.cs @@ -211,7 +211,7 @@ internal List getMetricsExtractMappingList(JobConfiguratio internal Dictionary> populateMethodCallMappingDictionary(string methodCallLinesToFrameworkTypeMappingFilePath) { - List methodCallLineClassToFrameworkTypeMappingList = FileIOHelper.ReadListFromCSVFile(methodCallLinesToFrameworkTypeMappingFilePath, new MethodCallLineClassTypeMappingReportMap()); + List methodCallLineClassToFrameworkTypeMappingList = FileIOHelper.ReadListFromCSVFile(methodCallLinesToFrameworkTypeMappingFilePath, new MethodCallLineClassTypeMappingReportMap()); methodCallLineClassToFrameworkTypeMappingList = methodCallLineClassToFrameworkTypeMappingList.OrderByDescending(m => m.ClassPrefix).ToList(); Dictionary> methodCallLineClassToFrameworkTypeMappingDictionary = new Dictionary>(26); methodCallLineClassToFrameworkTypeMappingDictionary.Add("a", methodCallLineClassToFrameworkTypeMappingList.Where(m => m.ClassPrefix.Substring(0, 1).ToLower() == "a").ToList()); diff --git a/ProcessingSteps/Report/ReportApplicationAndEntityMetricGraphs.cs b/ProcessingSteps/Report/ReportApplicationAndEntityMetricGraphs.cs index 0639378..52bc596 100644 --- a/ProcessingSteps/Report/ReportApplicationAndEntityMetricGraphs.cs +++ b/ProcessingSteps/Report/ReportApplicationAndEntityMetricGraphs.cs @@ -155,8 +155,8 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job List entityMetricExtractMappingListFiltered = entityMetricExtractMappingList.Where(m => m.EntityType == EntityApplication.ENTITY_TYPE && m.Graph.Length > 0).ToList(); fillMetricValueTablesForEntityType(excelReport, REPORT_METRICS_GRAPHS_SHEET_ENTITIES_METRICS, entityMetricExtractMappingListFiltered, jobTarget, EntityApplication.ENTITY_FOLDER); - fillMetricGraphsForEntityType(excelReport, entityMetricExtractMappingList, applicationsList.OfType().ToList(), jobConfiguration, jobTarget, EntityApplication.ENTITY_FOLDER, EntityApplication.ENTITY_TYPE); - fillTransactionalScatterPlotsForEntityType(excelReport, entityMetricExtractMappingList, applicationsList.OfType().ToList(), jobConfiguration, jobTarget, EntityApplication.ENTITY_FOLDER, EntityApplication.ENTITY_TYPE); + fillMetricGraphsForEntityType(excelReport, entityMetricExtractMappingListFiltered, applicationsList.OfType().ToList(), jobConfiguration, jobTarget, EntityApplication.ENTITY_FOLDER, EntityApplication.ENTITY_TYPE); + fillTransactionalScatterPlotsForEntityType(excelReport, entityMetricExtractMappingListFiltered, applicationsList.OfType().ToList(), jobConfiguration, jobTarget, EntityApplication.ENTITY_FOLDER, EntityApplication.ENTITY_TYPE); finalizeAndSaveIndividualEntityMetricReport(excelReport, FilePathMap.EntityTypeMetricGraphsExcelReportFilePath(applicationsList[0], jobTarget, jobConfiguration.Input.TimeRange, true)); @@ -178,8 +178,8 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job List entityMetricExtractMappingListFiltered = entityMetricExtractMappingList.Where(m => m.EntityType == EntityTier.ENTITY_TYPE && m.Graph.Length > 0).ToList(); fillMetricValueTablesForEntityType(excelReport, REPORT_METRICS_GRAPHS_SHEET_ENTITIES_METRICS, entityMetricExtractMappingListFiltered, jobTarget, EntityTier.ENTITY_FOLDER); - fillMetricGraphsForEntityType(excelReport, entityMetricExtractMappingList, tiersList.OfType().ToList(), jobConfiguration, jobTarget, EntityTier.ENTITY_FOLDER, EntityTier.ENTITY_TYPE); - fillTransactionalScatterPlotsForEntityType(excelReport, entityMetricExtractMappingList, tiersList.OfType().ToList(), jobConfiguration, jobTarget, EntityTier.ENTITY_FOLDER, EntityTier.ENTITY_TYPE); + fillMetricGraphsForEntityType(excelReport, entityMetricExtractMappingListFiltered, tiersList.OfType().ToList(), jobConfiguration, jobTarget, EntityTier.ENTITY_FOLDER, EntityTier.ENTITY_TYPE); + fillTransactionalScatterPlotsForEntityType(excelReport, entityMetricExtractMappingListFiltered, tiersList.OfType().ToList(), jobConfiguration, jobTarget, EntityTier.ENTITY_FOLDER, EntityTier.ENTITY_TYPE); finalizeAndSaveIndividualEntityMetricReport(excelReport, FilePathMap.EntityTypeMetricGraphsExcelReportFilePath(tiersList[0], jobTarget, jobConfiguration.Input.TimeRange, true)); @@ -199,10 +199,11 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job ExcelPackage excelReport = createIndividualEntityMetricGraphsReportTemplate(programOptions, jobConfiguration); List entityMetricExtractMappingListFiltered = entityMetricExtractMappingList.Where(m => m.EntityType == EntityNode.ENTITY_TYPE && m.Graph.Length > 0).ToList(); + fillMetricValueTablesForEntityType(excelReport, REPORT_METRICS_GRAPHS_SHEET_ENTITIES_METRICS, entityMetricExtractMappingListFiltered, jobTarget, EntityNode.ENTITY_FOLDER); - fillMetricGraphsForEntityType(excelReport, entityMetricExtractMappingList, nodesList.OfType().ToList(), jobConfiguration, jobTarget, EntityNode.ENTITY_FOLDER, EntityNode.ENTITY_TYPE); - fillTransactionalScatterPlotsForEntityType(excelReport, entityMetricExtractMappingList, nodesList.OfType().ToList(), jobConfiguration, jobTarget, EntityNode.ENTITY_FOLDER, EntityNode.ENTITY_TYPE); + fillMetricGraphsForEntityType(excelReport, entityMetricExtractMappingListFiltered, nodesList.OfType().ToList(), jobConfiguration, jobTarget, EntityNode.ENTITY_FOLDER, EntityNode.ENTITY_TYPE); + fillTransactionalScatterPlotsForEntityType(excelReport, entityMetricExtractMappingListFiltered, nodesList.OfType().ToList(), jobConfiguration, jobTarget, EntityNode.ENTITY_FOLDER, EntityNode.ENTITY_TYPE); finalizeAndSaveIndividualEntityMetricReport(excelReport, FilePathMap.EntityTypeMetricGraphsExcelReportFilePath(nodesList[0], jobTarget, jobConfiguration.Input.TimeRange, true)); @@ -224,8 +225,8 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job List entityMetricExtractMappingListFiltered = entityMetricExtractMappingList.Where(m => m.EntityType == EntityBackend.ENTITY_TYPE && m.Graph.Length > 0).ToList(); fillMetricValueTablesForEntityType(excelReport, REPORT_METRICS_GRAPHS_SHEET_ENTITIES_METRICS, entityMetricExtractMappingListFiltered, jobTarget, EntityBackend.ENTITY_FOLDER); - fillMetricGraphsForEntityType(excelReport, entityMetricExtractMappingList, backendsList.OfType().ToList(), jobConfiguration, jobTarget, EntityBackend.ENTITY_FOLDER, EntityBackend.ENTITY_TYPE); - fillTransactionalScatterPlotsForEntityType(excelReport, entityMetricExtractMappingList, backendsList.OfType().ToList(), jobConfiguration, jobTarget, EntityBackend.ENTITY_FOLDER, EntityBackend.ENTITY_TYPE); + fillMetricGraphsForEntityType(excelReport, entityMetricExtractMappingListFiltered, backendsList.OfType().ToList(), jobConfiguration, jobTarget, EntityBackend.ENTITY_FOLDER, EntityBackend.ENTITY_TYPE); + fillTransactionalScatterPlotsForEntityType(excelReport, entityMetricExtractMappingListFiltered, backendsList.OfType().ToList(), jobConfiguration, jobTarget, EntityBackend.ENTITY_FOLDER, EntityBackend.ENTITY_TYPE); finalizeAndSaveIndividualEntityMetricReport(excelReport, FilePathMap.EntityTypeMetricGraphsExcelReportFilePath(backendsList[0], jobTarget, jobConfiguration.Input.TimeRange, true)); @@ -247,8 +248,8 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job List entityMetricExtractMappingListFiltered = entityMetricExtractMappingList.Where(m => m.EntityType == EntityBusinessTransaction.ENTITY_TYPE && m.Graph.Length > 0).ToList(); fillMetricValueTablesForEntityType(excelReport, REPORT_METRICS_GRAPHS_SHEET_ENTITIES_METRICS, entityMetricExtractMappingListFiltered, jobTarget, EntityBusinessTransaction.ENTITY_FOLDER); - fillMetricGraphsForEntityType(excelReport, entityMetricExtractMappingList, businessTransactionsList.OfType().ToList(), jobConfiguration, jobTarget, EntityBusinessTransaction.ENTITY_FOLDER, EntityBusinessTransaction.ENTITY_TYPE); - fillTransactionalScatterPlotsForEntityType(excelReport, entityMetricExtractMappingList, businessTransactionsList.OfType().ToList(), jobConfiguration, jobTarget, EntityBusinessTransaction.ENTITY_FOLDER, EntityBusinessTransaction.ENTITY_TYPE); + fillMetricGraphsForEntityType(excelReport, entityMetricExtractMappingListFiltered, businessTransactionsList.OfType().ToList(), jobConfiguration, jobTarget, EntityBusinessTransaction.ENTITY_FOLDER, EntityBusinessTransaction.ENTITY_TYPE); + fillTransactionalScatterPlotsForEntityType(excelReport, entityMetricExtractMappingListFiltered, businessTransactionsList.OfType().ToList(), jobConfiguration, jobTarget, EntityBusinessTransaction.ENTITY_FOLDER, EntityBusinessTransaction.ENTITY_TYPE); finalizeAndSaveIndividualEntityMetricReport(excelReport, FilePathMap.EntityTypeMetricGraphsExcelReportFilePath(businessTransactionsList[0], jobTarget, jobConfiguration.Input.TimeRange, true)); @@ -270,8 +271,8 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job List entityMetricExtractMappingListFiltered = entityMetricExtractMappingList.Where(m => m.EntityType == EntityServiceEndpoint.ENTITY_TYPE && m.Graph.Length > 0).ToList(); fillMetricValueTablesForEntityType(excelReport, REPORT_METRICS_GRAPHS_SHEET_ENTITIES_METRICS, entityMetricExtractMappingListFiltered, jobTarget, EntityServiceEndpoint.ENTITY_FOLDER); - fillMetricGraphsForEntityType(excelReport, entityMetricExtractMappingList, serviceEndpointsList.OfType().ToList(), jobConfiguration, jobTarget, EntityServiceEndpoint.ENTITY_FOLDER, EntityServiceEndpoint.ENTITY_TYPE); - fillTransactionalScatterPlotsForEntityType(excelReport, entityMetricExtractMappingList, serviceEndpointsList.OfType().ToList(), jobConfiguration, jobTarget, EntityServiceEndpoint.ENTITY_FOLDER, EntityServiceEndpoint.ENTITY_TYPE); + fillMetricGraphsForEntityType(excelReport, entityMetricExtractMappingListFiltered, serviceEndpointsList.OfType().ToList(), jobConfiguration, jobTarget, EntityServiceEndpoint.ENTITY_FOLDER, EntityServiceEndpoint.ENTITY_TYPE); + fillTransactionalScatterPlotsForEntityType(excelReport, entityMetricExtractMappingListFiltered, serviceEndpointsList.OfType().ToList(), jobConfiguration, jobTarget, EntityServiceEndpoint.ENTITY_FOLDER, EntityServiceEndpoint.ENTITY_TYPE); finalizeAndSaveIndividualEntityMetricReport(excelReport, FilePathMap.EntityTypeMetricGraphsExcelReportFilePath(serviceEndpointsList[0], jobTarget, jobConfiguration.Input.TimeRange, true)); @@ -293,7 +294,7 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job List entityMetricExtractMappingListFiltered = entityMetricExtractMappingList.Where(m => m.EntityType == EntityError.ENTITY_TYPE && m.Graph.Length > 0).ToList(); fillMetricValueTablesForEntityType(excelReport, REPORT_METRICS_GRAPHS_SHEET_ENTITIES_METRICS, entityMetricExtractMappingListFiltered, jobTarget, EntityError.ENTITY_FOLDER); - fillMetricGraphsForEntityType(excelReport, entityMetricExtractMappingList, errorsList.OfType().ToList(), jobConfiguration, jobTarget, EntityError.ENTITY_FOLDER, EntityError.ENTITY_TYPE); + fillMetricGraphsForEntityType(excelReport, entityMetricExtractMappingListFiltered, errorsList.OfType().ToList(), jobConfiguration, jobTarget, EntityError.ENTITY_FOLDER, EntityError.ENTITY_TYPE); finalizeAndSaveIndividualEntityMetricReport(excelReport, FilePathMap.EntityTypeMetricGraphsExcelReportFilePath(errorsList[0], jobTarget, jobConfiguration.Input.TimeRange, true)); @@ -315,8 +316,8 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job List entityMetricExtractMappingListFiltered = entityMetricExtractMappingList.Where(m => m.EntityType == EntityInformationPoint.ENTITY_TYPE && m.Graph.Length > 0).ToList(); fillMetricValueTablesForEntityType(excelReport, REPORT_METRICS_GRAPHS_SHEET_ENTITIES_METRICS, entityMetricExtractMappingListFiltered, jobTarget, EntityInformationPoint.ENTITY_FOLDER); - fillMetricGraphsForEntityType(excelReport, entityMetricExtractMappingList, informationPointsList.OfType().ToList(), jobConfiguration, jobTarget, EntityInformationPoint.ENTITY_FOLDER, EntityInformationPoint.ENTITY_TYPE); - fillTransactionalScatterPlotsForEntityType(excelReport, entityMetricExtractMappingList, informationPointsList.OfType().ToList(), jobConfiguration, jobTarget, EntityInformationPoint.ENTITY_FOLDER, EntityInformationPoint.ENTITY_TYPE); + fillMetricGraphsForEntityType(excelReport, entityMetricExtractMappingListFiltered, informationPointsList.OfType().ToList(), jobConfiguration, jobTarget, EntityInformationPoint.ENTITY_FOLDER, EntityInformationPoint.ENTITY_TYPE); + fillTransactionalScatterPlotsForEntityType(excelReport, entityMetricExtractMappingListFiltered, informationPointsList.OfType().ToList(), jobConfiguration, jobTarget, EntityInformationPoint.ENTITY_FOLDER, EntityInformationPoint.ENTITY_TYPE); finalizeAndSaveIndividualEntityMetricReport(excelReport, FilePathMap.EntityTypeMetricGraphsExcelReportFilePath(informationPointsList[0], jobTarget, jobConfiguration.Input.TimeRange, true)); @@ -333,6 +334,7 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job { logger.Warn(ex); loggerConsole.Warn(ex); + return false; } finally { @@ -600,19 +602,16 @@ private void fillMetricGraphsForEntityType( { ExcelWorksheet sheetMetrics = excelReportMetrics.Workbook.Worksheets[REPORT_METRICS_GRAPHS_SHEET_ENTITIES_METRICS]; - // Load metric index locations file for later use - List entityMetricValuesLocations = FileIOHelper.ReadListFromCSVFile(FilePathMap.MetricsLocationIndexFilePath(jobTarget, entityFolderName), new EntityHourlyMetricValueLocationReportMap()); - - // Process metrics in groups as indicated by their Graph column. All metrics with same Graph value go into same sheet, with as many sheets as necessary - // Empty graph name means do not output - List entityMetricExtractMappingListFiltered = entityMetricExtractMappingList.Where(m => m.EntityType == entityType && m.Graph.Length > 0).ToList(); + Dictionary> entityMetricValuesLocationsDictionary = indexMetricValueLocationsForFasterAccess(jobConfiguration, jobTarget, entityFolderName, entityType); // Output all graphs - var metricMappingsGrouped = entityMetricExtractMappingListFiltered.GroupBy(m => new { m.Graph }); + var metricMappingsGrouped = entityMetricExtractMappingList.GroupBy(m => new { m.Graph }); foreach (var metricMappingGroup in metricMappingsGrouped) { List entityMetricExtractMappingForThisGraph = metricMappingGroup.ToList(); + Console.Write("{0} {1} starting for {2} entities ", entityFolderName, metricMappingGroup.Key.Graph, entityList.Count); + // Measure the width of the time range to be at least 16 cells int numCellsPerHourRange = entityMetricExtractMappingForThisGraph.Count * 2; if (numCellsPerHourRange < 16) numCellsPerHourRange = 16; @@ -776,8 +775,6 @@ private void fillMetricGraphsForEntityType( // Output entity one at a time for (int indexOfEntity = 0; indexOfEntity < entityList.Count; indexOfEntity++) { - Console.Write("."); - EntityBase entity = entityList[indexOfEntity]; string entityNameForExcelTable = getShortenedEntityNameForExcelTable(entity.EntityName, entity.EntityID); @@ -815,26 +812,26 @@ private void fillMetricGraphsForEntityType( #endregion - // Get indexed table location mapping for this specific metric/entity/timerange combo - List entityMetricValuesLocationsForThisMetric = entityMetricValuesLocations.Where(m => - m.MetricName == metricExtractMapping.MetricName && - m.EntityID == entity.EntityID && - m.FromUtc >= jobTimeRange.From && - m.ToUtc < jobTimeRange.To).ToList(); - if (entityMetricValuesLocationsForThisMetric != null && entityMetricValuesLocationsForThisMetric.Count > 0) + EntityHourlyMetricValueLocation entityMetricValuesLocationForThisMetric = null; + if (entityMetricValuesLocationsDictionary.ContainsKey(entity.EntityID) == true && + entityMetricValuesLocationsDictionary[entity.EntityID].ContainsKey(metricExtractMapping.MetricName) == true) + { + entityMetricValuesLocationForThisMetric = entityMetricValuesLocationsDictionary[entity.EntityID][metricExtractMapping.MetricName][indexOfTimeRange]; + } + + if (entityMetricValuesLocationForThisMetric != null) { // We must have some activity here entityHasActivity = true; // Output the numeric values ExcelTable tableMetrics = sheetMetrics.Tables[String.Format(REPORT_METRICS_GRAPHS_METRIC_TABLE_METRIC_VALUES, entityFolderName, metricExtractMapping.FolderName)]; + if (tableMetrics != null) { - // Should be only metric index for this value 1 in here - EntityHourlyMetricValueLocation entityMetricValuesLocationForThisMetric = entityMetricValuesLocationsForThisMetric[0]; - // Get the range that contains the Sum column of the metric ExcelRangeBase rangeMetricValuesForSum = getSingleColumnRangeFromTable(tableMetrics, "Sum", entityMetricValuesLocationForThisMetric.RowStart, entityMetricValuesLocationForThisMetric.RowEnd); + if (rangeMetricValuesForSum != null) { sheetGraphs.Cells[currentMaxRow, columnIndexOfValueOfCurrentMetric].Formula = String.Format(@"=SUM({0})", rangeMetricValuesForSum.FullAddress); @@ -996,6 +993,11 @@ private void fillMetricGraphsForEntityType( currentMaxRow++; #endregion + + if (indexOfEntity % 50 == 0) + { + Console.Write("[{0}].", indexOfEntity); + } } #region Create tables for entities @@ -1016,6 +1018,8 @@ private void fillMetricGraphsForEntityType( } #endregion + + Console.WriteLine("{0} {1} complete", entityFolderName, metricMappingGroup.Key.Graph); } return; @@ -1038,6 +1042,8 @@ private void fillTransactionalScatterPlotsForEntityType( // Process metrics in the CPM, ART and EPM group List entityMetricExtractMappingListFiltered = entityMetricExtractMappingList.Where(m => m.EntityType == entityType && m.Graph == TRANSACTIONS_METRICS_SET).ToList(); + Dictionary> entityMetricValuesLocationsDictionary = indexMetricValueLocationsForFasterAccess(jobConfiguration, jobTarget, entityFolderName, entityType); + if (entityMetricExtractMappingListFiltered.Count > 0) { #region Set variables for the sheet output based on Entity Type @@ -1201,6 +1207,8 @@ private void fillTransactionalScatterPlotsForEntityType( #endregion + Console.Write("{0} {1} starting for {2} entities ", entityFolderName, sheetName, entityList.Count); + // Output entity one at a time for (int indexOfEntity = 0; indexOfEntity < entityList.Count; indexOfEntity++) { @@ -1221,24 +1229,26 @@ private void fillTransactionalScatterPlotsForEntityType( #endregion - // Get indexed table location mapping for this specific metric/entity/timerange combo - EntityHourlyMetricValueLocation entityMetricValuesLocationsForART = entityMetricValuesLocations.Where(m => - m.MetricName == memART.MetricName && - m.EntityID == entity.EntityID && - m.FromUtc >= jobTimeRange.From && - m.ToUtc < jobTimeRange.To).FirstOrDefault(); - - EntityHourlyMetricValueLocation entityMetricValuesLocationsForCPM = entityMetricValuesLocations.Where(m => - m.MetricName == memCPM.MetricName && - m.EntityID == entity.EntityID && - m.FromUtc >= jobTimeRange.From && - m.ToUtc < jobTimeRange.To).FirstOrDefault(); - - EntityHourlyMetricValueLocation entityMetricValuesLocationsForEPM = entityMetricValuesLocations.Where(m => - m.MetricName == memEPM.MetricName && - m.EntityID == entity.EntityID && - m.FromUtc >= jobTimeRange.From && - m.ToUtc < jobTimeRange.To).FirstOrDefault(); + EntityHourlyMetricValueLocation entityMetricValuesLocationsForART = null; + if (entityMetricValuesLocationsDictionary.ContainsKey(entity.EntityID) == true && + entityMetricValuesLocationsDictionary[entity.EntityID].ContainsKey(memART.MetricName) == true) + { + entityMetricValuesLocationsForART = entityMetricValuesLocationsDictionary[entity.EntityID][memART.MetricName][indexOfTimeRange]; + } + + EntityHourlyMetricValueLocation entityMetricValuesLocationsForCPM = null; + if (entityMetricValuesLocationsDictionary.ContainsKey(entity.EntityID) == true && + entityMetricValuesLocationsDictionary[entity.EntityID].ContainsKey(memCPM.MetricName) == true) + { + entityMetricValuesLocationsForCPM = entityMetricValuesLocationsDictionary[entity.EntityID][memCPM.MetricName][indexOfTimeRange]; + } + + EntityHourlyMetricValueLocation entityMetricValuesLocationsForEPM = null; + if (entityMetricValuesLocationsDictionary.ContainsKey(entity.EntityID) == true && + entityMetricValuesLocationsDictionary[entity.EntityID].ContainsKey(memEPM.MetricName) == true) + { + entityMetricValuesLocationsForEPM = entityMetricValuesLocationsDictionary[entity.EntityID][memEPM.MetricName][indexOfTimeRange]; + } // Get ranges with the metrics ExcelRangeBase rangeARTValues = null; @@ -1400,6 +1410,11 @@ private void fillTransactionalScatterPlotsForEntityType( currentMaxRow++; #endregion + + if (indexOfEntity % 50 == 0) + { + Console.Write("[{0}].", indexOfEntity); + } } #region Create tables for entities @@ -1420,6 +1435,8 @@ private void fillTransactionalScatterPlotsForEntityType( } #endregion + + Console.WriteLine("{0} {1} complete", entityFolderName, sheetName); } } @@ -1562,5 +1579,74 @@ private static void addScatterChartToEntityMetricSheet(ExcelWorksheet sheet, Exc #endregion } + + private Dictionary> indexMetricValueLocationsForFasterAccess( + JobConfiguration jobConfiguration, + JobTarget jobTarget, + string entityFolderName, + string entityType) + { + // Load metric index locations file for later use + List entityMetricValuesLocations = FileIOHelper.ReadListFromCSVFile(FilePathMap.MetricsLocationIndexFilePath(jobTarget, entityFolderName), new EntityHourlyMetricValueLocationReportMap()); + + // Index the metric value locations into this funky dictionary of dictionaries with array in following hierarchy + // Entity ID (123) + // Metric Name (ART, CPM) + // Time range 0 + // ... + // Time range N + Dictionary> entityMetricValuesLocationsDictionary = null; + + // Group them by Entity ID first + var entityMetricValuesLocationsGroupedByEntityID = entityMetricValuesLocations.GroupBy(m => new { m.EntityID }); + if (entityMetricValuesLocationsGroupedByEntityID != null) + { + entityMetricValuesLocationsDictionary = new Dictionary>(entityMetricValuesLocationsGroupedByEntityID.Count()); + foreach (var entityMetricMappingsGroupForEntity in entityMetricValuesLocationsGroupedByEntityID) + { + List entityHourlyMetricValueLocationsForThisEntity = entityMetricMappingsGroupForEntity.ToList(); + + // Group them by Metric Name second + var entityMetricValuesLocationsGroupedByMetricName = entityHourlyMetricValueLocationsForThisEntity.GroupBy(m => new { m.MetricName }); + + long entityID = entityMetricMappingsGroupForEntity.Key.EntityID; + entityMetricValuesLocationsDictionary.Add(entityID, new Dictionary(5)); + foreach (var entityMetricMappingsGroupForEntityMetric in entityMetricValuesLocationsGroupedByMetricName) + { + List entityHourlyMetricValueLocationsForThisEntityAndMetric = entityMetricMappingsGroupForEntityMetric.ToList(); + + // Preallocate the array for each of the timeranges in anticipation of metric data to be inserted there + EntityHourlyMetricValueLocation[] entityHourlyMetricValueLocationsForThisEntityAndMetricArray = new EntityHourlyMetricValueLocation[jobConfiguration.Input.HourlyTimeRanges.Count]; + entityMetricValuesLocationsDictionary[entityID].Add(entityMetricMappingsGroupForEntityMetric.Key.MetricName, entityHourlyMetricValueLocationsForThisEntityAndMetricArray); + + if (entityHourlyMetricValueLocationsForThisEntityAndMetric != null && entityHourlyMetricValueLocationsForThisEntityAndMetric.Count > 0) + { + // Roll through the array of all the time ranges populating the array slots for each hour that has data + int indexOfMetricValueLocation = 0; + for (int indexOfTimeRange = 0; indexOfTimeRange < jobConfiguration.Input.HourlyTimeRanges.Count; indexOfTimeRange++) + { + JobTimeRange jobTimeRange = jobConfiguration.Input.HourlyTimeRanges[indexOfTimeRange]; + EntityHourlyMetricValueLocation entityHourlyMetricValueLocation = entityHourlyMetricValueLocationsForThisEntityAndMetric[indexOfMetricValueLocation]; + + if (entityHourlyMetricValueLocation.FromUtc >= jobTimeRange.From && + entityHourlyMetricValueLocation.ToUtc < jobTimeRange.To) + { + // Found metric values for this entity for this metric in this time range + entityHourlyMetricValueLocationsForThisEntityAndMetricArray[indexOfTimeRange] = entityHourlyMetricValueLocation; + + indexOfMetricValueLocation++; + } + if (indexOfMetricValueLocation >= entityHourlyMetricValueLocationsForThisEntityAndMetric.Count) + { + break; + } + } + } + } + } + } + + return entityMetricValuesLocationsDictionary; + } } } diff --git a/ProcessingSteps/Report/ReportFlameGraphs.cs b/ProcessingSteps/Report/ReportFlameGraphs.cs index e6abd69..362e9c4 100644 --- a/ProcessingSteps/Report/ReportFlameGraphs.cs +++ b/ProcessingSteps/Report/ReportFlameGraphs.cs @@ -300,6 +300,8 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job { logger.Warn(ex); loggerConsole.Warn(ex); + + return false; } finally { diff --git a/ProcessingSteps/Report/ReportIndividualApplicationAndEntityDetails.cs b/ProcessingSteps/Report/ReportIndividualApplicationAndEntityDetails.cs index 7495a8c..bb3d4c1 100644 --- a/ProcessingSteps/Report/ReportIndividualApplicationAndEntityDetails.cs +++ b/ProcessingSteps/Report/ReportIndividualApplicationAndEntityDetails.cs @@ -155,7 +155,7 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job #region Preload all the reports that will be filtered by the subsequent entities - loggerConsole.Info("Completed Entity Details Data Preloading"); + loggerConsole.Info("Entity Details Data Preloading"); List eventsAllList = FileIOHelper.ReadListFromCSVFile(FilePathMap.EventsIndexFilePath(jobTarget), new EventReportMap()); List healthRuleViolationEventsAllList = FileIOHelper.ReadListFromCSVFile(FilePathMap.HealthRuleViolationsIndexFilePath(jobTarget), new HealthRuleViolationEventReportMap()); @@ -166,6 +166,8 @@ public override bool Execute(ProgramOptions programOptions, JobConfiguration job List detectedErrorsAllList = FileIOHelper.ReadListFromCSVFile(FilePathMap.SnapshotsDetectedErrorsIndexFilePath(jobTarget), new DetectedErrorReportMap()); List businessDataAllList = FileIOHelper.ReadListFromCSVFile(FilePathMap.SnapshotsBusinessDataIndexFilePath(jobTarget), new BusinessDataReportMap()); + loggerConsole.Info("Completed Entity Details Data Preloading"); + string relativePathToReportsToRemoveFromLinks = Path.Combine( FilePathMap.getFileSystemSafeString(new Uri(jobTarget.Controller).Host), FilePathMap.getShortenedEntityNameForFileSystem(jobTarget.Application, jobTarget.ApplicationID)); diff --git a/Properties/AssemblyInfo.cs b/Properties/AssemblyInfo.cs index 78c40e3..a9f9942 100644 --- a/Properties/AssemblyInfo.cs +++ b/Properties/AssemblyInfo.cs @@ -31,5 +31,5 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.0.0.29")] -[assembly: AssemblyFileVersion("1.0.0.29")] +[assembly: AssemblyVersion("1.0.0.30")] +[assembly: AssemblyFileVersion("1.0.0.30")] diff --git a/ReportObjects/Configuration/HTTPDataCollector.cs b/ReportObjects/Configuration/HTTPDataCollector.cs index badc473..346a831 100644 --- a/ReportObjects/Configuration/HTTPDataCollector.cs +++ b/ReportObjects/Configuration/HTTPDataCollector.cs @@ -65,7 +65,7 @@ public override string RuleSubType public override String ToString() { return String.Format( - "MethodInvocationDataCollector: {0}/{1}/{2}", + "HTTPDataCollector: {0}/{1}/{2}", this.Controller, this.ApplicationName, this.CollectorName); diff --git a/VisualAnalytics/PowerBI/Design/DetectedEntitiesDesign.pbix b/VisualAnalytics/PowerBI/Design/DetectedEntitiesDesign.pbix index 060ed68..e084d3a 100644 Binary files a/VisualAnalytics/PowerBI/Design/DetectedEntitiesDesign.pbix and b/VisualAnalytics/PowerBI/Design/DetectedEntitiesDesign.pbix differ diff --git a/VisualAnalytics/PowerBI/Design/EntityFlowmapsDesign.pbix b/VisualAnalytics/PowerBI/Design/EntityFlowmapsDesign.pbix index 18f39a7..a861785 100644 Binary files a/VisualAnalytics/PowerBI/Design/EntityFlowmapsDesign.pbix and b/VisualAnalytics/PowerBI/Design/EntityFlowmapsDesign.pbix differ diff --git a/VisualAnalytics/PowerBI/Design/EntityMetricsDesign.pbix b/VisualAnalytics/PowerBI/Design/EntityMetricsDesign.pbix index 2b2ae1f..d402fa0 100644 Binary files a/VisualAnalytics/PowerBI/Design/EntityMetricsDesign.pbix and b/VisualAnalytics/PowerBI/Design/EntityMetricsDesign.pbix differ diff --git a/VisualAnalytics/PowerBI/Design/EventsAndHealthRuleViolationsDesign.pbix b/VisualAnalytics/PowerBI/Design/EventsAndHealthRuleViolationsDesign.pbix index 68dd684..e5c810a 100644 Binary files a/VisualAnalytics/PowerBI/Design/EventsAndHealthRuleViolationsDesign.pbix and b/VisualAnalytics/PowerBI/Design/EventsAndHealthRuleViolationsDesign.pbix differ diff --git a/VisualAnalytics/PowerBI/Design/SnapshotMethodCallsDesign.pbix b/VisualAnalytics/PowerBI/Design/SnapshotMethodCallsDesign.pbix index 9edea0e..9797276 100644 Binary files a/VisualAnalytics/PowerBI/Design/SnapshotMethodCallsDesign.pbix and b/VisualAnalytics/PowerBI/Design/SnapshotMethodCallsDesign.pbix differ diff --git a/VisualAnalytics/PowerBI/Design/SnapshotsDesign.pbix b/VisualAnalytics/PowerBI/Design/SnapshotsDesign.pbix index 602025a..8b788fa 100644 Binary files a/VisualAnalytics/PowerBI/Design/SnapshotsDesign.pbix and b/VisualAnalytics/PowerBI/Design/SnapshotsDesign.pbix differ diff --git a/VisualAnalytics/PowerBI/Templates/DetectedEntities.pbit b/VisualAnalytics/PowerBI/Templates/DetectedEntities.pbit index 87b20d3..c6d9e4e 100644 Binary files a/VisualAnalytics/PowerBI/Templates/DetectedEntities.pbit and b/VisualAnalytics/PowerBI/Templates/DetectedEntities.pbit differ diff --git a/VisualAnalytics/PowerBI/Templates/EntityFlowmaps.pbit b/VisualAnalytics/PowerBI/Templates/EntityFlowmaps.pbit index f67a1f7..92fdd45 100644 Binary files a/VisualAnalytics/PowerBI/Templates/EntityFlowmaps.pbit and b/VisualAnalytics/PowerBI/Templates/EntityFlowmaps.pbit differ diff --git a/VisualAnalytics/PowerBI/Templates/EntityMetrics.pbit b/VisualAnalytics/PowerBI/Templates/EntityMetrics.pbit index 8968396..3e91cf2 100644 Binary files a/VisualAnalytics/PowerBI/Templates/EntityMetrics.pbit and b/VisualAnalytics/PowerBI/Templates/EntityMetrics.pbit differ diff --git a/VisualAnalytics/PowerBI/Templates/EventsAndHealthRuleViolations.pbit b/VisualAnalytics/PowerBI/Templates/EventsAndHealthRuleViolations.pbit index a58cb65..2304df8 100644 Binary files a/VisualAnalytics/PowerBI/Templates/EventsAndHealthRuleViolations.pbit and b/VisualAnalytics/PowerBI/Templates/EventsAndHealthRuleViolations.pbit differ diff --git a/VisualAnalytics/PowerBI/Templates/SnapshotMethodCalls.pbit b/VisualAnalytics/PowerBI/Templates/SnapshotMethodCalls.pbit index 4c16c18..bb41d18 100644 Binary files a/VisualAnalytics/PowerBI/Templates/SnapshotMethodCalls.pbit and b/VisualAnalytics/PowerBI/Templates/SnapshotMethodCalls.pbit differ diff --git a/VisualAnalytics/PowerBI/Templates/Snapshots.pbit b/VisualAnalytics/PowerBI/Templates/Snapshots.pbit index 6ca1b9c..ad0d27c 100644 Binary files a/VisualAnalytics/PowerBI/Templates/Snapshots.pbit and b/VisualAnalytics/PowerBI/Templates/Snapshots.pbit differ diff --git a/VisualAnalytics/Tableau/Design/DetectedEntities.twb b/VisualAnalytics/Tableau/Design/DetectedEntities.twb deleted file mode 100644 index a41a5ff..0000000 --- a/VisualAnalytics/Tableau/Design/DetectedEntities.twb +++ /dev/null @@ -1,1019 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 0 - [applications.csv] - - Count - true - - "UTF-8" - "en_US" - "," - "true" - "en_US" - "" - - - - Controller - 129 - [Controller] - [applications.csv] - Controller - 0 - string - Count - 1 - 1073741823 - true - - - - ApplicationName - 129 - [ApplicationName] - [applications.csv] - ApplicationName - 1 - string - Count - 1 - 1073741823 - true - - - - Description - 129 - [Description] - [applications.csv] - Description - 2 - string - Count - 1 - 1073741823 - true - - - - NumTiers - 20 - [NumTiers] - [applications.csv] - NumTiers - 3 - integer - Sum - true - - - NumNodes - 20 - [NumNodes] - [applications.csv] - NumNodes - 4 - integer - Sum - true - - - NumBackends - 20 - [NumBackends] - [applications.csv] - NumBackends - 5 - integer - Sum - true - - - NumBTs - 20 - [NumBTs] - [applications.csv] - NumBTs - 6 - integer - Sum - true - - - NumSEPs - 20 - [NumSEPs] - [applications.csv] - NumSEPs - 7 - integer - Sum - true - - - NumErrors - 20 - [NumErrors] - [applications.csv] - NumErrors - 8 - integer - Sum - true - - - NumIPs - 20 - [NumIPs] - [applications.csv] - NumIPs - 9 - integer - Sum - true - - - ApplicationID - 20 - [ApplicationID] - [applications.csv] - ApplicationID - 10 - integer - Sum - true - - - DetailLink - 129 - [DetailLink] - [applications.csv] - DetailLink - 11 - string - Count - 1 - 1073741823 - true - - - - MetricGraphLink - 129 - [MetricGraphLink] - [applications.csv] - MetricGraphLink - 12 - string - Count - 1 - 1073741823 - true - - - - FlameGraphLink - 129 - [FlameGraphLink] - [applications.csv] - FlameGraphLink - 13 - string - Count - 1 - 1073741823 - true - - - - ControllerLink - 129 - [ControllerLink] - [applications.csv] - ControllerLink - 14 - string - Count - 1 - 1073741823 - true - - - - ApplicationLink - 129 - [ApplicationLink] - [applications.csv] - ApplicationLink - 15 - string - Count - 1 - 1073741823 - true - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -