From 8177ec3ccca34d0231246fb86fdd66cffd1c2d8d Mon Sep 17 00:00:00 2001 From: Fritz Freiheit Date: Thu, 21 Mar 2024 11:33:10 -0400 Subject: [PATCH] DEEPBLUE-327 - Add email to admin when aptrust bags mismatch checksum - 2 --- app/helpers/deepblue/job_task_helper.rb | 8 +++++- app/services/aptrust/aptrust_report_status.rb | 2 +- app/services/aptrust/aptrust_uploader.rb | 3 ++- .../aptrust/aptrust_uploader_for_work.rb | 6 ++--- .../aptrust/aptrust_uploader_status.rb | 3 +++ app/tasks/deepblue/yaml_populate.rb | 26 +++++++------------ 6 files changed, 26 insertions(+), 22 deletions(-) diff --git a/app/helpers/deepblue/job_task_helper.rb b/app/helpers/deepblue/job_task_helper.rb index 2d953549..24fa9cc0 100644 --- a/app/helpers/deepblue/job_task_helper.rb +++ b/app/helpers/deepblue/job_task_helper.rb @@ -179,6 +179,12 @@ def self.email_failure( targets:, targets.delete_if { |x| x.blank? } return if targets.blank? timestamp_end = DateTime.now if timestamp_end.blank? + backtrace = exception&.backtrace + if backtrace.is_a? Array + backtrace = backtrace[0..30] if bactrace.size > 30 + else + backtrace = [] + end body =<<-END_BODY #{task_name} on #{hostname} failed.
#{timestamp_begin.blank? ? "" : "Began: #{timestamp_begin}
"} @@ -189,7 +195,7 @@ def self.email_failure( targets:, Exception raised:
#{exception.class} #{exception.message}

-#{email_lines_to_html( lines: exception.backtrace[0..30], tag: 'code', join: "\n" )} +#{email_lines_to_html( lines: backtrace, tag: 'code', join: "\n" )}
#{email_lines_to_html( lines: messages, tag: 'code', join: "\n" )} END_BODY diff --git a/app/services/aptrust/aptrust_report_status.rb b/app/services/aptrust/aptrust_report_status.rb index 21e1f4c6..3e598863 100644 --- a/app/services/aptrust/aptrust_report_status.rb +++ b/app/services/aptrust/aptrust_report_status.rb @@ -138,7 +138,7 @@ def process( status: ) row << noid elsif "http_status" == col row << http_status - elsif "aws_bucket_status" + elsif "aws_bucket_status" == col row << aws_bucket_status( noid: noid ) else row << key_values[col] diff --git a/app/services/aptrust/aptrust_uploader.rb b/app/services/aptrust/aptrust_uploader.rb index a33c63a0..5c51cd0b 100644 --- a/app/services/aptrust/aptrust_uploader.rb +++ b/app/services/aptrust/aptrust_uploader.rb @@ -701,9 +701,10 @@ def export_data def export_data_resolve_error( error ) # if export_errors may contain an instance of the ::Deepblue::ExportFilesChecksumMismatch msg_handler.bold_debug [ msg_handler.here, msg_handler.called_from, + "error&.message=#{error&.message}", "error=#{error.pretty_inspect}", "" ] if debug_verbose - note = "#{error}" + note = "#{error&.message}" track( status: ::Aptrust::EVENT_ERROR, note: note ) end diff --git a/app/services/aptrust/aptrust_uploader_for_work.rb b/app/services/aptrust/aptrust_uploader_for_work.rb index 81eef1ca..6c5afd99 100644 --- a/app/services/aptrust/aptrust_uploader_for_work.rb +++ b/app/services/aptrust/aptrust_uploader_for_work.rb @@ -146,7 +146,7 @@ def email_error( error ) task_name = self.class.name task_args = nil exception = error - message = error&.message + event_note = error&.message event = error.class.name.demodulize timestamp_begin = DateTime.now timestamp_end = timestamp_begin @@ -169,8 +169,8 @@ def email_error( error ) end def export_data_resolve_error( error ) - email_error( error ) if error is_a? ::Deepblue::ExportFilesChecksumMismatch - super.export_data_resolve_error( error ) + super + email_error( error ) if error.is_a? ::Deepblue::ExportFilesChecksumMismatch end def export_do_copy?( target_dir, target_file_name ) # TODO: check file size? diff --git a/app/services/aptrust/aptrust_uploader_status.rb b/app/services/aptrust/aptrust_uploader_status.rb index 64876293..a5226d37 100644 --- a/app/services/aptrust/aptrust_uploader_status.rb +++ b/app/services/aptrust/aptrust_uploader_status.rb @@ -4,6 +4,8 @@ class Aptrust::AptrustUploaderStatus + NOTE_MAX_SIZE = 120 unless const_defined? :NOTE_MAX_SIZE + mattr_accessor :aptrust_uplaoder_status_debug_verbose, default: false def self.clean_database! @@ -37,6 +39,7 @@ def self.track_db( noid:, status_event:, note: nil, timestamp: DateTime.now ) "" ] if aptrust_uplaoder_status_debug_verbose begin timestamp ||= DateTime.now + note = note[0..(NOTE_MAX_SIZE-1)] if !note.nil? && note.size > NOTE_MAX_SIZE status = ::Aptrust::Status.for_id( noid: noid ) if status.blank? status = ::Aptrust::Status.new( timestamp: timestamp, event: status_event, event_note: note, noid: noid ) diff --git a/app/tasks/deepblue/yaml_populate.rb b/app/tasks/deepblue/yaml_populate.rb index 850e2e42..39e6c332 100644 --- a/app/tasks/deepblue/yaml_populate.rb +++ b/app/tasks/deepblue/yaml_populate.rb @@ -167,29 +167,23 @@ def run_one_curation_concern( curation_concern: ) end def yaml_bag_work( id:, work: nil ) + work ||= PersistHelper.find id + sz = DeepblueHelper.human_readable_size( work.total_file_size ) + msg = "Bagging work #{id} (#{sz}) to '#{@target_dir}'" + msg = "#{msg} with export files flag set to #{@export_files}" unless @export_files + report_puts msg + log_filename = "#{id}.export.log" @mode = ::Deepblue::MetadataHelper::MODE_BAG - report_puts "Bagging work #{id} to '#{@target_dir}' with export files flag set to #{@export_files}" service = YamlPopulateService.new( mode: @mode, collect_exported_file_set_files: @collect_exported_file_set_files, create_zero_length_files: @create_zero_length_files, overwrite_export_files: @overwrite_export_files, validate_file_checksums: @validate_file_checksums, debug_verbose: @debug_verbose ) - # puts "yaml_bag_work( id: #{id}, work: #{work} )" if DEBUG_VERBOSE - # puts "@target_dir=#{@target_dir}" if DEBUG_VERBOSE - if work.nil? - log_filename = "#{id}.export.log" - service.yaml_populate_work( curation_concern: id, - dir: @target_dir, - export_files: @export_files, - log_filename: log_filename ) - else - log_filename = "#{work.id}.export.log" - service.yaml_populate_work( curation_concern: work, - dir: @target_dir, - export_files: @export_files, - log_filename: log_filename ) - end + service.yaml_populate_work( curation_concern: work, + dir: @target_dir, + export_files: @export_files, + log_filename: log_filename ) @populate_ids << id @populate_stats << service.yaml_populate_stats return service