From fd22c2c97b0270ec35fb7815d5fc735d150dc36e Mon Sep 17 00:00:00 2001 From: Justin Donn Date: Wed, 5 Mar 2025 22:49:36 -0800 Subject: [PATCH] feat(logs): add detailed logs to relationship backfill --- .../linkedin/metadata/dao/EbeanLocalDAO.java | 49 ++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java index 659c518e3..e6597b98c 100644 --- a/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java +++ b/dao-impl/ebean-dao/src/main/java/com/linkedin/metadata/dao/EbeanLocalDAO.java @@ -99,6 +99,8 @@ public enum SchemaConfig { // TODO: clean up once AIM is no longer using existing local relationships - they should make new relationship tables with the aspect column private boolean _useAspectColumnForRelationshipRemoval = false; + private boolean _noisyLogsEnabled = false; + // Which approach to be used for record retrieval when inserting a new record // See GCN-38382 private FindMethodology _findMethodology = FindMethodology.UNIQUE_ID; @@ -136,6 +138,14 @@ public void setUseAspectColumnForRelationshipRemoval(boolean useAspectColumnForR _localRelationshipWriterDAO.setUseAspectColumnForRelationshipRemoval(useAspectColumnForRelationshipRemoval); } + /** + * Set a flag to indicate whether noisy info logs are enabled. Should only be used for debugging. + * @param noisyLogsEnabled whether the logs are enabled + */ + public void setNoisyLogsEnabled(boolean noisyLogsEnabled) { + _noisyLogsEnabled = noisyLogsEnabled; + } + public void setOverwriteLatestVersionEnabled(boolean overwriteLatestVersionEnabled) { if (_schemaConfig == SchemaConfig.NEW_SCHEMA_ONLY) { if (isChangeLogEnabled()) { @@ -681,14 +691,30 @@ public List backfillLo @Nonnull URN urn, @Nonnull Class aspectClass) { AspectKey key = new AspectKey<>(aspectClass, urn, LATEST_VERSION); return runInTransactionWithRetry(() -> { + if (_noisyLogsEnabled) { + log.info("Backfilling local relationships for urn: {}, aspectClass: {}", urn, aspectClass); + } List results = batchGet(Collections.singleton(key), 1); - if (results.size() == 0) { + if (results.isEmpty()) { + if (_noisyLogsEnabled) { + log.info("Not backfilling any relationships because no aspect data was found for urn: {}, aspectClass: {}", urn, aspectClass); + } return new ArrayList<>(); } + if (_noisyLogsEnabled) { + log.info("Trying to convert aspect data from the entity table to a RecordTemplate for urn: {}, aspectClass: {}", urn, aspectClass); + } Optional aspect = toRecordTemplate(aspectClass, results.get(0)); if (aspect.isPresent()) { + if (_noisyLogsEnabled) { + log.info("Successfully converted aspect data to a RecordTemplate for urn: {}, aspectClass: {}", urn, aspectClass); + } return handleRelationshipIngestion(urn, aspect.get(), null, aspectClass, false); } + if (_noisyLogsEnabled) { + log.info("Not backfilling any relationships because aspect data was unable to be converted to a RecordTemplate " + + "for urn: {}, aspectClass: {}", urn, aspectClass); + } return Collections.emptyList(); }, 1); } @@ -906,6 +932,10 @@ public List boolean isSoftDeletion = false; if (newValue == null) { if (oldValue == null) { + if (_noisyLogsEnabled) { + log.info("No relationships will be ingested since oldValue and newValue are both null in handleRelationshipIngestion " + + "for urn: {}, aspectClass: {}.", urn, aspectClass); + } return Collections.emptyList(); } isSoftDeletion = true; @@ -923,6 +953,10 @@ public List localRelationshipUpdates = _localRelationshipBuilderRegistry.getLocalRelationshipBuilder(aspect).buildRelationships(urn, aspect); // default all relationship updates to use REMOVE_ALL_EDGES_FROM_SOURCE localRelationshipUpdates.forEach(update -> update.setRemovalOption(BaseGraphWriterDAO.RemovalOption.REMOVE_ALL_EDGES_FROM_SOURCE)); + if (_noisyLogsEnabled) { + log.info("Was able to use relationship builders to extract relationships in handleRelationshipIngestion for urn: {}, aspectClass: {}. " + + "LocalRelationshipUpdates: {}", urn, aspectClass, localRelationshipUpdates); + } } // If no relationship updates were found using relationship builders, try to get them via the aspect. if (localRelationshipUpdates.isEmpty()) { @@ -932,16 +966,29 @@ public List .map(entry -> new LocalRelationshipUpdates( Arrays.asList(entry.getValue().toArray()), entry.getKey(), BaseGraphWriterDAO.RemovalOption.REMOVE_ALL_EDGES_FROM_SOURCE)) .collect(Collectors.toList()); + if (_noisyLogsEnabled) { + log.info("No relationship builder found or no relationships were extracted from the builder. Trying to get relationships " + + "directly from aspect metadata in handleRelationshipIngestion for urn: {}, aspectClass: {}. " + + "LocalRelationshipUpdates: {}", urn, aspectClass, localRelationshipUpdates); + } } // process relationship soft-deletion if applicable if (isSoftDeletion) { List relationships = new ArrayList<>(); localRelationshipUpdates.forEach(localRelationshipUpdate -> relationships.addAll(localRelationshipUpdate.getRelationships())); + if (_noisyLogsEnabled) { + log.info("Removing relationships because aspect is soft-deleted in handleRelationshipIngestion for urn: {}, aspectClass: {}. " + + "LocalRelationshipUpdates: {}", urn, aspectClass, localRelationshipUpdates); + } _localRelationshipWriterDAO.removeRelationships(urn, aspectClass, relationships); return Collections.emptyList(); } // process relationship ingestion _localRelationshipWriterDAO.processLocalRelationshipUpdates(urn, aspectClass, localRelationshipUpdates, isTestMode); + if (_noisyLogsEnabled && !localRelationshipUpdates.isEmpty()) { + log.info("Relationships successfully ingested in handleRelationshipIngestion for urn: {}, aspectClass: {}. " + + "LocalRelationshipUpdates: {}", urn, aspectClass, localRelationshipUpdates); + } return localRelationshipUpdates; }