Skip to content

Commit

Permalink
Merge pull request #8439 from mandy-chessell/oak2024
Browse files Browse the repository at this point in the history 
Remove duplicate relationships
  • Loading branch information
@mandy-chessell
mandy-chessell authored Oct 13, 2024
2 parents ed73042 + fa8c5b0 commit 09653ab
Show file tree
Hide file tree
Showing 26 changed files with 894 additions and 342 deletions.
6 changes: 3 additions & 3 deletions ...rs/data-manager-connectors/unity-catalog-connectors/UnityCatalog-demo-cataloguing-uc.http
View file
Original file line number Diff line number Diff line change
Expand Up @@ -194,10 +194,10 @@ Content-Type: application/json
"placeholderPropertyValues" : {
"hostURL" : "http://localhost",
"portNumber" : "8080",
"serverName" : "Unity Catalog 1",
"serverName" : "Unity Catalog 3",
"versionIdentifier" : "V1.0",
"description" : "First instance of the Unity Catalog (UC) Server.",
"serverUserId" : "uc1"
"description" : "Third instance of the Unity Catalog (UC) Server.",
"serverUserId" : "uc3"
}
}

Expand Down
31 changes: 9 additions & 22 deletions ...nmetadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncBase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,22 +14,24 @@
import org.odpi.openmetadata.frameworks.connectors.ffdc.InvalidParameterException;
import org.odpi.openmetadata.frameworks.connectors.ffdc.PropertyServerException;
import org.odpi.openmetadata.frameworks.connectors.ffdc.UserNotAuthorizedException;
import org.odpi.openmetadata.frameworks.openmetadata.enums.ElementStatus;
import org.odpi.openmetadata.frameworks.openmetadata.controls.PlaceholderProperty;
import org.odpi.openmetadata.frameworks.governanceaction.properties.ExternalIdentifierProperties;
import org.odpi.openmetadata.frameworks.governanceaction.properties.MetadataCorrelationHeader;
import org.odpi.openmetadata.frameworks.governanceaction.search.ElementProperties;
import org.odpi.openmetadata.frameworks.governanceaction.search.PropertyHelper;
import org.odpi.openmetadata.frameworks.integration.context.OpenMetadataAccess;
import org.odpi.openmetadata.frameworks.integration.iterator.IntegrationIterator;
import org.odpi.openmetadata.frameworks.integration.iterator.MemberElement;
import org.odpi.openmetadata.frameworks.openmetadata.controls.PlaceholderProperty;
import org.odpi.openmetadata.frameworks.openmetadata.enums.ElementStatus;
import org.odpi.openmetadata.frameworks.openmetadata.enums.PermittedSynchronization;
import org.odpi.openmetadata.frameworks.openmetadata.mapper.PropertyFacetValidValues;
import org.odpi.openmetadata.frameworks.openmetadata.types.OpenMetadataProperty;
import org.odpi.openmetadata.frameworks.openmetadata.types.OpenMetadataType;
import org.odpi.openmetadata.integrationservices.catalog.connector.CatalogIntegratorContext;

import java.util.*;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

/**
* Common functions for the synchronizing between Egeria and Unity Catalog.
Expand Down Expand Up @@ -330,33 +332,18 @@ protected boolean noMismatchInExternalIdentifier(String thirdPartyExterna
{
final String methodName = "noMismatchInExternalIdentifier";

if ((thirdPartyExternalIdentifier == null) || (memberElement == null) || (memberElement.getExternalIdentifiers() == null))
if ((thirdPartyExternalIdentifier == null) || (memberElement == null) || (memberElement.getExternalIdentifier() == null))
{
return true;
}

List<String> externalIdentifiers = new ArrayList<>();

for (MetadataCorrelationHeader correlationHeader : memberElement.getExternalIdentifiers())
{
if ((correlationHeader != null) && (correlationHeader.getExternalIdentifier() != null))
{
externalIdentifiers.add(correlationHeader.getExternalIdentifier());

if (thirdPartyExternalIdentifier.equals(correlationHeader.getExternalIdentifier()))
{
return true;
}
}
}

if (externalIdentifiers.isEmpty())
if (thirdPartyExternalIdentifier.equals(memberElement.getExternalIdentifier().getExternalIdentifier()))
{
return true;
}

auditLog.logMessage(methodName, UCAuditCode.IDENTITY_MISMATCH.getMessageDefinition(connectorName,
externalIdentifiers.toString(),
memberElement.getExternalIdentifier().getExternalIdentifier(),
thirdPartyExternalIdentifier,
ucServerEndpoint));
return false;
Expand Down
14 changes: 1 addition & 13 deletions ...data/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncConnector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -225,19 +225,7 @@ public void processEvent(AssetManagerOutTopicEvent event)

RequestedCatalogTarget requestedCatalogTarget = new RequestedCatalogTarget(catalogTarget, null);

Map<String, Object> configurationProperties = connectionProperties.getConfigurationProperties();

if (catalogTarget.getConfigurationProperties() != null)
{
if (configurationProperties == null)
{
configurationProperties = new HashMap<>();
}

configurationProperties.putAll(catalogTarget.getConfigurationProperties());
}

requestedCatalogTarget.setConfigurationProperties(configurationProperties);
requestedCatalogTarget.setConfigurationProperties(super.combineConfigurationProperties(catalogTarget.getConfigurationProperties()));

if (propertyHelper.isTypeOf(catalogTarget.getCatalogTargetElement(), OpenMetadataType.ASSET.typeName))
{
Expand Down
43 changes: 33 additions & 10 deletions ...data/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncFunctions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,8 @@ protected IntegrationIterator refreshEgeria() throws InvalidParameterException,
final String methodName = "refreshEgeriaFunctions";

MetadataCollectionIterator functionIterator = new MetadataCollectionIterator(catalogGUID,
catalogQualifiedName,
catalogGUID,
catalogQualifiedName,
catalogName,
connectorName,
Expand Down Expand Up @@ -364,7 +366,11 @@ private void updateElementInEgeria(FunctionInfo functionInfo,

this.updateSchemaAttributesForUCFunction(memberElement, functionInfo);

context.confirmSynchronization(egeriaFunctionGUID, entityTypeName, functionInfo.getFunction_id());
context.confirmSynchronization(catalogGUID,
catalogQualifiedName,
egeriaFunctionGUID,
entityTypeName,
functionInfo.getFunction_id());
}


Expand All @@ -390,15 +396,26 @@ private void createElementInThirdParty(String schemaName,
// todo - complete parameter mapping
FunctionInfo functionInfo = ucConnector.createFunction(functionProperties);

context.addExternalIdentifier(catalogGUID,
catalogName,
memberElement.getElement().getElementGUID(),
deployedImplementationType.getAssociatedTypeName(),
this.getExternalIdentifierProperties(functionInfo,
functionInfo.getSchema_name(),
UnityCatalogPlaceholderProperty.FUNCTION_NAME.getName(),
functionInfo.getFunction_id(),
PermittedSynchronization.TO_THIRD_PARTY));
if (memberElement.getExternalIdentifier() == null)
{
context.addExternalIdentifier(catalogGUID,
catalogName,
memberElement.getElement().getElementGUID(),
deployedImplementationType.getAssociatedTypeName(),
this.getExternalIdentifierProperties(functionInfo,
functionInfo.getSchema_name(),
UnityCatalogPlaceholderProperty.FUNCTION_NAME.getName(),
functionInfo.getFunction_id(),
PermittedSynchronization.TO_THIRD_PARTY));
}
else
{
context.confirmSynchronization(catalogGUID,
catalogQualifiedName,
memberElement.getElement().getElementGUID(),
deployedImplementationType.getAssociatedTypeName(),
functionInfo.getFunction_id());
}
}


Expand All @@ -425,6 +442,12 @@ private void updateElementInThirdParty(FunctionInfo functionInfo,
memberElement.getElement().getElementGUID(),
functionInfo.getCatalog_name() + "." + functionInfo.getSchema_name() + "." + functionInfo.getName(),
ucServerEndpoint));

context.confirmSynchronization(catalogGUID,
catalogQualifiedName,
memberElement.getElement().getElementGUID(),
deployedImplementationType.getAssociatedTypeName(),
functionInfo.getFunction_id());
}


Expand Down
49 changes: 37 additions & 12 deletions ...etadata/adapters/connectors/unitycatalog/sync/OSSUnityCatalogInsideCatalogSyncSchema.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@ protected IntegrationIterator refreshEgeria() throws InvalidParameterException,
final String methodName = "refreshEgeria";

MetadataCollectionIterator iterator = new MetadataCollectionIterator(catalogGUID,
catalogQualifiedName,
catalogGUID,
catalogQualifiedName,
catalogName,
connectorName,
Expand Down Expand Up @@ -333,7 +335,9 @@ protected void updateElementInEgeria(SchemaInfo schemaInfo,
false,
getElementProperties(schemaInfo));

context.confirmSynchronization(egeriaSchemaGUID,
context.confirmSynchronization(catalogGUID,
catalogQualifiedName,
egeriaSchemaGUID,
deployedImplementationType.getAssociatedTypeName(),
schemaInfo.getSchema_id());
}
Expand All @@ -356,15 +360,26 @@ private void createElementInThirdParty(MemberElement memberElement) throws Prope
super.getUCCommentFomMember(memberElement),
super.getUCPropertiesFomMember(memberElement));

context.addExternalIdentifier(catalogGUID,
catalogQualifiedName,
memberElement.getElement().getElementGUID(),
deployedImplementationType.getAssociatedTypeName(),
this.getExternalIdentifierProperties(schemaInfo,
schemaInfo.getName(),
PlaceholderProperty.SCHEMA_NAME.getName(),
schemaInfo.getSchema_id(),
PermittedSynchronization.TO_THIRD_PARTY));
if (memberElement.getExternalIdentifier() == null)
{
context.addExternalIdentifier(catalogGUID,
catalogQualifiedName,
memberElement.getElement().getElementGUID(),
deployedImplementationType.getAssociatedTypeName(),
this.getExternalIdentifierProperties(schemaInfo,
schemaInfo.getName(),
PlaceholderProperty.SCHEMA_NAME.getName(),
schemaInfo.getSchema_id(),
PermittedSynchronization.TO_THIRD_PARTY));
}
else
{
context.confirmSynchronization(catalogGUID,
catalogQualifiedName,
memberElement.getElement().getElementGUID(),
deployedImplementationType.getAssociatedTypeName(),
schemaInfo.getSchema_id());
}
}


Expand All @@ -374,10 +389,14 @@ private void createElementInThirdParty(MemberElement memberElement) throws Prope
* @param schemaInfo existing schema in UC
* @param memberElement elements from Egeria
*
* @throws PropertyServerException problem communicating with UC
* @throws InvalidParameterException bad call to Egeria
* @throws UserNotAuthorizedException security problem
* @throws PropertyServerException problem communicating with UC or egeria
*/
private void updateElementInThirdParty(SchemaInfo schemaInfo,
MemberElement memberElement) throws PropertyServerException
MemberElement memberElement) throws PropertyServerException,
InvalidParameterException,
UserNotAuthorizedException
{
final String methodName = "updateElementInThirdParty";

Expand All @@ -386,6 +405,12 @@ private void updateElementInThirdParty(SchemaInfo schemaInfo,
memberElement.getElement().getElementGUID(),
schemaInfo.getCatalog_name() + "." + schemaInfo.getName(),
ucServerEndpoint));

context.confirmSynchronization(catalogGUID,
catalogQualifiedName,
memberElement.getElement().getElementGUID(),
deployedImplementationType.getAssociatedTypeName(),
schemaInfo.getSchema_id());
}


Expand Down
Loading

0 comments on commit 09653ab

Please sign in to comment.