From 80dfd6426e3bdb0660e75ca4f35c60a37cbe5867 Mon Sep 17 00:00:00 2001 From: Ho Peng Foong Date: Tue, 11 Feb 2025 17:52:30 +0800 Subject: [PATCH 1/6] ww --- CluedIn.Connector.AzureDataLake.sln | 13 +- Packages.props | 1 + .../Connector/DataLakeClient.cs | 71 ++++++--- .../Connector/DataLakeConnector.cs | 83 ++++++++-- .../DataLakeExportEntitiesJobBase.cs | 130 ++++++++++++++-- .../Connector/IDataLakeClient.cs | 3 + .../SqlDataWriter/ParquetSqlDataWriter.cs | 43 ++++-- .../SqlDataWriter/SqlDataWriterBase.cs | 19 +++ .../DataLakeConstants.cs | 2 + .../DataLakeJobData.cs | 8 +- .../IDataLakeJobData.cs | 2 + .../Properties/AssemblyInfo.cs | 1 + .../Connector.FabricMirroring.csproj | 20 +++ .../Connector/FabricMirroringClient.cs | 39 +++++ .../Connector/FabricMirroringConnector.cs | 109 ++++++++++++++ .../FabricMirroringExportEntitiesJob.cs | 142 ++++++++++++++++++ .../FabricMirroringConnectorComponent.cs | 30 ++++ .../FabricMirroringConnectorJobData.cs | 64 ++++++++ .../FabricMirroringConnectorProvider.cs | 25 +++ .../FabricMirroringConstants.cs | 98 ++++++++++++ .../FabricMirroringJobDataFactory.cs | 18 +++ .../IFabricMirroringConstants.cs | 9 ++ .../InstallComponents.cs | 15 ++ .../Resources/fabricMirroring.svg | 42 ++++++ 24 files changed, 920 insertions(+), 67 deletions(-) create mode 100644 src/Connector.FabricMirroring/Connector.FabricMirroring.csproj create mode 100644 src/Connector.FabricMirroring/Connector/FabricMirroringClient.cs create mode 100644 src/Connector.FabricMirroring/Connector/FabricMirroringConnector.cs create mode 100644 src/Connector.FabricMirroring/Connector/FabricMirroringExportEntitiesJob.cs create mode 100644 src/Connector.FabricMirroring/FabricMirroringConnectorComponent.cs create mode 100644 src/Connector.FabricMirroring/FabricMirroringConnectorJobData.cs create mode 100644 src/Connector.FabricMirroring/FabricMirroringConnectorProvider.cs create mode 100644 src/Connector.FabricMirroring/FabricMirroringConstants.cs create mode 100644 src/Connector.FabricMirroring/FabricMirroringJobDataFactory.cs create mode 100644 src/Connector.FabricMirroring/IFabricMirroringConstants.cs create mode 100644 src/Connector.FabricMirroring/InstallComponents.cs create mode 100644 src/Connector.FabricMirroring/Resources/fabricMirroring.svg diff --git a/CluedIn.Connector.AzureDataLake.sln b/CluedIn.Connector.AzureDataLake.sln index bc92c6c..63c6b67 100644 --- a/CluedIn.Connector.AzureDataLake.sln +++ b/CluedIn.Connector.AzureDataLake.sln @@ -53,11 +53,13 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connector.DataLake.Common", EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connector.OneLake", "src\Connector.OneLake\Connector.OneLake.csproj", "{1AA8B845-9762-47DD-B0E4-3B2C20C7486A}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connector.AzureDatabricks", "src\Connector.AzureDatabricks\Connector.AzureDatabricks.csproj", "{B0A8FAB9-8809-492F-A2B1-C141CE53A724}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connector.AzureDatabricks", "src\Connector.AzureDatabricks\Connector.AzureDatabricks.csproj", "{B0A8FAB9-8809-492F-A2B1-C141CE53A724}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connector.SynapseDataEngineering", "src\Connector.SynapseDataEngineering\Connector.SynapseDataEngineering.csproj", "{61607CFA-7A18-4DD6-9512-83B620B288EB}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connector.SynapseDataEngineering", "src\Connector.SynapseDataEngineering\Connector.SynapseDataEngineering.csproj", "{61607CFA-7A18-4DD6-9512-83B620B288EB}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Connector.AzureAIStudio", "src\Connector.AzureAIStudio\Connector.AzureAIStudio.csproj", "{5F8EDA0E-5F95-4A7A-A7F3-217210674FDE}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connector.AzureAIStudio", "src\Connector.AzureAIStudio\Connector.AzureAIStudio.csproj", "{5F8EDA0E-5F95-4A7A-A7F3-217210674FDE}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Connector.FabricMirroring", "src\Connector.FabricMirroring\Connector.FabricMirroring.csproj", "{B6B64EFA-397F-467F-984E-B7FE73ED92A4}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -97,6 +99,10 @@ Global {5F8EDA0E-5F95-4A7A-A7F3-217210674FDE}.Debug|Any CPU.Build.0 = Debug|Any CPU {5F8EDA0E-5F95-4A7A-A7F3-217210674FDE}.Release|Any CPU.ActiveCfg = Release|Any CPU {5F8EDA0E-5F95-4A7A-A7F3-217210674FDE}.Release|Any CPU.Build.0 = Release|Any CPU + {B6B64EFA-397F-467F-984E-B7FE73ED92A4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B6B64EFA-397F-467F-984E-B7FE73ED92A4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B6B64EFA-397F-467F-984E-B7FE73ED92A4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B6B64EFA-397F-467F-984E-B7FE73ED92A4}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -116,6 +122,7 @@ Global {B0A8FAB9-8809-492F-A2B1-C141CE53A724} = {5256D9B9-8A1D-480D-A8F0-1A69AFA59B31} {61607CFA-7A18-4DD6-9512-83B620B288EB} = {5256D9B9-8A1D-480D-A8F0-1A69AFA59B31} {5F8EDA0E-5F95-4A7A-A7F3-217210674FDE} = {5256D9B9-8A1D-480D-A8F0-1A69AFA59B31} + {B6B64EFA-397F-467F-984E-B7FE73ED92A4} = {5256D9B9-8A1D-480D-A8F0-1A69AFA59B31} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {E6A866CB-824C-4271-8EA6-053B7FC4B134} diff --git a/Packages.props b/Packages.props index 59d1e2b..4b41679 100644 --- a/Packages.props +++ b/Packages.props @@ -25,6 +25,7 @@ + diff --git a/src/Connector.DataLake.Common/Connector/DataLakeClient.cs b/src/Connector.DataLake.Common/Connector/DataLakeClient.cs index 891302f..81b7e8b 100644 --- a/src/Connector.DataLake.Common/Connector/DataLakeClient.cs +++ b/src/Connector.DataLake.Common/Connector/DataLakeClient.cs @@ -9,15 +9,15 @@ namespace CluedIn.Connector.DataLake.Common.Connector { public abstract class DataLakeClient : IDataLakeClient { - public async Task EnsureDataLakeDirectoryExist(IDataLakeJobData configuration) + public Task EnsureDataLakeDirectoryExist(IDataLakeJobData configuration) { - var fileSystemClient = await GetFileSystemClientAsync(configuration); - var directory = GetDirectory(configuration); - var directoryClient = fileSystemClient.GetDirectoryClient(directory); - if (!await directoryClient.ExistsAsync()) - { - directoryClient = await fileSystemClient.CreateDirectoryAsync(directory); - } + return EnsureDataLakeDirectoryExist(configuration, string.Empty); + } + + public async Task EnsureDataLakeDirectoryExist(IDataLakeJobData configuration, string subDirectory) + { + var fileSystemClient = await GetFileSystemClientAsync(configuration, ensureExists: true); + var directoryClient = await GetDirectoryClientAsync(configuration, fileSystemClient, subDirectory, ensureExists: true); return directoryClient; } @@ -68,19 +68,20 @@ protected static TJobData CastJobData(IDataLakeJobData jobData) where return castedJobData; } - public async Task FileInPathExists(IDataLakeJobData configuration, string fileName) + public Task FileInPathExists(IDataLakeJobData configuration, string fileName) { - var serviceClient = GetDataLakeServiceClient(configuration); - var fileSystemName = GetFileSystemName(configuration); - var fileSystemClient = serviceClient.GetFileSystemClient(fileSystemName); + return FileInPathExists(configuration, fileName, string.Empty); + } + public async Task FileInPathExists(IDataLakeJobData configuration, string fileName, string subDirectory) + { + var fileSystemClient = await GetFileSystemClientAsync(configuration, ensureExists: false); if (!await fileSystemClient.ExistsAsync()) { return false; } - var directory = GetDirectory(configuration); - var directoryClient = fileSystemClient.GetDirectoryClient(directory); + var directoryClient = await GetDirectoryClientAsync(configuration, fileSystemClient, subDirectory, ensureExists: false); if (!await directoryClient.ExistsAsync()) { return false; @@ -90,19 +91,21 @@ public async Task FileInPathExists(IDataLakeJobData configuration, string return await dataLakeFileClient.ExistsAsync(); } - public async Task GetFilePathProperties(IDataLakeJobData configuration, string fileName) + public Task GetFilePathProperties(IDataLakeJobData configuration, string fileName) { - var serviceClient = GetDataLakeServiceClient(configuration); - var fileSystemName = GetFileSystemName(configuration); - var fileSystemClient = serviceClient.GetFileSystemClient(fileSystemName); + return GetFilePathProperties(configuration, fileName, string.Empty); + } + + public async Task GetFilePathProperties(IDataLakeJobData configuration, string fileName, string subDirectory) + { + var fileSystemClient = await GetFileSystemClientAsync(configuration, ensureExists: false); if (!await fileSystemClient.ExistsAsync()) { return null; } - var directory = GetDirectory(configuration); - var directoryClient = fileSystemClient.GetDirectoryClient(directory); + var directoryClient = await GetDirectoryClientAsync(configuration, fileSystemClient, subDirectory, ensureExists: false); if (!await directoryClient.ExistsAsync()) { return null; @@ -117,13 +120,37 @@ public async Task GetFilePathProperties(IDataLakeJobData configu return await dataLakeFileClient.GetPropertiesAsync(); } + private async Task GetDirectoryClientAsync( + IDataLakeJobData configuration, + DataLakeFileSystemClient fileSystemClient, + string subDirectory, + bool ensureExists) + { + var directory = GetDirectory(configuration); + var directoryClient = fileSystemClient.GetDirectoryClient(directory); + if (string.IsNullOrWhiteSpace(subDirectory)) + { + return directoryClient; + } + + directoryClient = directoryClient.GetSubDirectoryClient(subDirectory); + + if (ensureExists && !await directoryClient.ExistsAsync()) + { + directoryClient = await fileSystemClient.CreateDirectoryAsync(directoryClient.Path); + } + + return directoryClient; + } + private async Task GetFileSystemClientAsync( - IDataLakeJobData configuration) + IDataLakeJobData configuration, + bool ensureExists) { var dataLakeServiceClient = GetDataLakeServiceClient(configuration); var fileSystemName = GetFileSystemName(configuration); var dataLakeFileSystemClient = dataLakeServiceClient.GetFileSystemClient(fileSystemName); - if (!await dataLakeFileSystemClient.ExistsAsync()) + if (ensureExists && !await dataLakeFileSystemClient.ExistsAsync()) { dataLakeFileSystemClient = await dataLakeServiceClient.CreateFileSystemAsync(fileSystemName); } diff --git a/src/Connector.DataLake.Common/Connector/DataLakeConnector.cs b/src/Connector.DataLake.Common/Connector/DataLakeConnector.cs index 2bec2b6..f4b3887 100644 --- a/src/Connector.DataLake.Common/Connector/DataLakeConnector.cs +++ b/src/Connector.DataLake.Common/Connector/DataLakeConnector.cs @@ -52,6 +52,8 @@ public abstract class DataLakeConnector : ConnectorBaseV2 [typeof(string)] = "NVARCHAR(MAX)" }; + protected IDataLakeJobDataFactory DataLakeJobDataFactory => _dataLakeJobDataFactory; + protected DataLakeConnector( ILogger logger, IDataLakeClient client, @@ -117,14 +119,28 @@ void AddToData(string key, T value) AddToData("ProviderDefinitionId", providerDefinitionId); AddToData("ContainerName", containerName); + var now = _dateTimeOffsetProvider.GetCurrentUtcTime(); + if (!data.ContainsKey("Timestamp")) { - AddToData("Timestamp", _dateTimeOffsetProvider.GetCurrentUtcTime().ToString("O")); + AddToData("Timestamp", now.ToString("O")); } if (!data.ContainsKey("Epoch")) { - AddToData("Epoch", _dateTimeOffsetProvider.GetCurrentUtcTime().ToUnixTimeMilliseconds()); + AddToData("Epoch", now.ToUnixTimeMilliseconds()); + } + + if (jobData.IsDeltaMode) + { + if (!data.ContainsKey("__ChangeType__")) + { + AddToData("__ChangeType__", connectorEntityData.ChangeType.ToString()); + } + //if (!data.ContainsKey("__ModifiedAt__")) + //{ + // AddToData("__ModifiedAt__", now); + //} } // end match previous version of the connector @@ -218,7 +234,7 @@ private async Task WriteToCacheTable( using var transactionScope = new TransactionScope(TransactionScopeAsyncFlowOption.Enabled); await using var connection = new SqlConnection(configurations.StreamCacheConnectionString); await connection.OpenAsync(); - await WriteToCacheTable(connection, syncItem, tableName); + await WriteToCacheTable(connection, syncItem, tableName, useSoftDelete: configurations.IsDeltaMode); transactionScope.Complete(); } catch (SqlException writeDataException) when (writeDataException.IsTableNotFoundException()) @@ -239,7 +255,7 @@ private async Task WriteToCacheTable( { await EnsureCacheTableExists(connection, tableName, syncItem); } - await WriteToCacheTable(connection, syncItem, tableName); + await WriteToCacheTable(connection, syncItem, tableName, useSoftDelete: configurations.IsDeltaMode); transactionScope.Complete(); } catch (Exception ex2) @@ -269,21 +285,19 @@ private Task TryAcquireTableCreationLock(SqlConnection connection, string private async Task WriteToCacheTable( SqlConnection connection, SyncItem syncItem, - string tableName) + string tableName, + bool useSoftDelete) { var propertyKeys = GetPropertyKeysWithoutId(syncItem); if (syncItem.ChangeType == VersionChangeType.Removed) { - var deleteCommandText = $"DELETE FROM [{tableName}] WHERE {DataLakeConstants.IdKey} = @{DataLakeConstants.IdKey}"; - var command = new SqlCommand(deleteCommandText, connection) + if (useSoftDelete) { - CommandType = CommandType.Text - }; - command.Parameters.Add(new SqlParameter($"@{DataLakeConstants.IdKey}", syncItem.EntityId)); - var rowsAffected = await command.ExecuteNonQueryAsync(); - if (rowsAffected != 1) + await SoftDeleteEntity(connection, syncItem, tableName); + } + else { - throw new ApplicationException($"Rows affected for deletion is not 1, it is {rowsAffected}."); + await HardDeleteEntity(connection, syncItem, tableName); } } else @@ -322,6 +336,47 @@ UPDATE [{tableName}] throw new ApplicationException($"Rows affected for insertion of is not 1, it is {rowsAffected}."); } } + + static async Task HardDeleteEntity(SqlConnection connection, SyncItem syncItem, string tableName) + { + var deleteCommandText = $"DELETE FROM [{tableName}] WHERE {DataLakeConstants.IdKey} = @{DataLakeConstants.IdKey}"; + var command = new SqlCommand(deleteCommandText, connection) + { + CommandType = CommandType.Text + }; + command.Parameters.Add(new SqlParameter($"@{DataLakeConstants.IdKey}", syncItem.EntityId)); + var rowsAffected = await command.ExecuteNonQueryAsync(); + if (rowsAffected != 1) + { + throw new ApplicationException($"Rows affected for hard deletion is not 1, it is {rowsAffected}."); + } + } + + static async Task SoftDeleteEntity(SqlConnection connection, SyncItem syncItem, string tableName) + { + var updateSql = $""" + IF EXISTS ( + SELECT 1 FROM [{tableName}] WITH (XLOCK, ROWLOCK) + WHERE {DataLakeConstants.IdKey} = @{DataLakeConstants.IdKey}) + BEGIN + UPDATE [{tableName}] + SET + [__ChangeType__] = {syncItem.ChangeType.ToString()} + WHERE {DataLakeConstants.IdKey} = @{DataLakeConstants.IdKey}; + END + """; + var command = new SqlCommand(updateSql, connection) + { + CommandType = CommandType.Text + }; + command.Parameters.Add(new SqlParameter($"@{DataLakeConstants.IdKey}", syncItem.EntityId)); + + var rowsAffected = await command.ExecuteNonQueryAsync(); + if (rowsAffected != 1) + { + throw new ApplicationException($"Rows affected for soft deletion of is not 1, it is {rowsAffected}."); + } + } } private static List GetPropertyKeysWithoutId(SyncItem syncItem) @@ -523,7 +578,7 @@ private async Task VerifyOperation(SqlConnection connection, string tableName, S { try { - await WriteToCacheTable(connection, syncItem, tableName); + await WriteToCacheTable(connection, syncItem, tableName, useSoftDelete: false); } catch (Exception ex) diff --git a/src/Connector.DataLake.Common/Connector/DataLakeExportEntitiesJobBase.cs b/src/Connector.DataLake.Common/Connector/DataLakeExportEntitiesJobBase.cs index 65a3c66..b0fd0c6 100644 --- a/src/Connector.DataLake.Common/Connector/DataLakeExportEntitiesJobBase.cs +++ b/src/Connector.DataLake.Common/Connector/DataLakeExportEntitiesJobBase.cs @@ -7,6 +7,10 @@ using System.Threading.Tasks; using System.Transactions; +using Apache.Arrow; + +using Azure.Storage.Files.DataLake; + using CluedIn.Connector.DataLake.Common.Connector.SqlDataWriter; using CluedIn.Core; using CluedIn.Core.Data.Relational; @@ -16,12 +20,14 @@ using Microsoft.Data.SqlClient; using Microsoft.Extensions.Logging; +using Parquet; + namespace CluedIn.Connector.DataLake.Common.Connector; internal abstract class DataLakeExportEntitiesJobBase : DataLakeJobBase { private readonly IStreamRepository _streamRepository; - private readonly IDataLakeClient _dataLakeClient; + protected readonly IDataLakeClient _dataLakeClient; private readonly IDataLakeConstants _dataLakeConstants; private readonly IDataLakeJobDataFactory _dataLakeJobDataFactory; private readonly IDateTimeOffsetProvider _dateTimeOffsetProvider; @@ -91,7 +97,7 @@ public override async Task DoRunAsync(ExecutionContext context, IDataLakeJobArgs outputFileName, asOfTime); asOfTime = _dateTimeOffsetProvider.GetCurrentUtcTime(); - outputFileName = GetOutputFileName(configuration, streamId, streamModel.ContainerName, asOfTime, outputFormat); + outputFileName = await GetOutputFileNameAsync(context, configuration, streamId, streamModel.ContainerName, asOfTime, outputFormat); } else if (HasExportedFileBefore(streamId, asOfTime, filePathProperties?.Metadata)) { @@ -109,6 +115,9 @@ public override async Task DoRunAsync(ExecutionContext context, IDataLakeJobArgs } } + var subDirectory = await GetSubDirectory(configuration, streamId); + var directoryClient = await _dataLakeClient.EnsureDataLakeDirectoryExist(configuration, subDirectory); + await InitializeDirectoryAsync(configuration, streamId, directoryClient); var startExportTime = _dateTimeOffsetProvider.GetCurrentUtcTime(); var exportHistory = new ExportHistory( streamId, @@ -125,11 +134,18 @@ public override async Task DoRunAsync(ExecutionContext context, IDataLakeJobArgs await InsertHistory(context, connection, exportHistory); - var getDataSql = $"SELECT * FROM [{tableName}] FOR SYSTEM_TIME AS OF '{asOfTime:o}'"; + var shouldProduceDelta = configuration.IsDeltaMode && LastExport != null; + var getDataSql = shouldProduceDelta + ? $"SELECT * FROM [{tableName}] FOR SYSTEM_TIME AS OF '{asOfTime:o}' WHERE ValidFrom > @ValidFrom" + : $"SELECT * FROM [{tableName}] FOR SYSTEM_TIME AS OF '{asOfTime:o}'"; var command = new SqlCommand(getDataSql, connection) { CommandType = CommandType.Text }; + if (shouldProduceDelta) + { + command.Parameters.Add(new SqlParameter("@ValidFrom", LastExport.DataTime)); + } await using var reader = await command.ExecuteReaderAsync(); var fieldNames = Enumerable.Range(0, reader.VisibleFieldCount) @@ -152,7 +168,6 @@ public override async Task DoRunAsync(ExecutionContext context, IDataLakeJobArgs outputFileName, asOfTime, temporaryOutputFileName); - var directoryClient = await _dataLakeClient.EnsureDataLakeDirectoryExist(configuration); var temporaryFileClient = directoryClient.GetFileClient(temporaryOutputFileName); var totalRows = await writeFileContentsAsync(); await setFilePropertiesAsync(); @@ -183,7 +198,7 @@ public override async Task DoRunAsync(ExecutionContext context, IDataLakeJobArgs async Task writeFileContentsAsync() { var sqlDataWriter = GetSqlDataWriter(outputFormat); - await using var outputStream = await temporaryFileClient.OpenWriteAsync(true); + await using var outputStream = await temporaryFileClient.OpenWriteAsync(configuration.IsOverwriteEnabled); using var bufferedStream = new DataLakeBufferedWriteStream(outputStream); return await sqlDataWriter?.WriteAsync(context, configuration, bufferedStream, fieldNames, reader); } @@ -231,6 +246,17 @@ async Task deleteTargetFileIfExistsAsync() await targetFileClient.DeleteIfExistsAsync(); } } + private protected ExportHistory LastExport { get; private set; } + + protected virtual Task GetSubDirectory(IDataLakeJobData configuration, Guid streamId) + { + return Task.FromResult(string.Empty); + } + + protected virtual Task InitializeDirectoryAsync(IDataLakeJobData configuration, Guid streamId, DataLakeDirectoryClient client) + { + return Task.CompletedTask; + } private static string GetTriggerSource(IDataLakeJobArgs args) { @@ -294,9 +320,14 @@ private async Task GetJobDataAsync(ExecutionContext context, IDat return null; } + await using var connection = new SqlConnection(configuration.StreamCacheConnectionString); + await connection.OpenAsync(); + var lastExport = await GetLastExport(context, connection, streamId, configuration); + LastExport = lastExport; + var asOfTime = GetAsOfTime(context, args, configuration); var outputFormat = configuration.OutputFormat.ToLowerInvariant(); - var outputFileName = GetOutputFileName(configuration, streamId, containerName, asOfTime, outputFormat); + var outputFileName = await GetOutputFileNameAsync(context, configuration, streamId, containerName, asOfTime, outputFormat); return new ExportJobData( streamId, @@ -385,14 +416,14 @@ private static bool HasExportedFileBefore(Guid streamId, DateTimeOffset asOfTime && fileMetadata.DataTime == asOfTime; } - protected virtual string GetOutputFileName(IDataLakeJobData configuration, Guid streamId, string containerName, DateTimeOffset asOfTime, string outputFormat) + protected virtual Task GetOutputFileNameAsync(ExecutionContext context, IDataLakeJobData configuration, Guid streamId, string containerName, DateTimeOffset asOfTime, string outputFormat) { if (HasCustomFileNamePattern(configuration)) { - return GetOutputFileNameUsingPattern(configuration.FileNamePattern, streamId, containerName, asOfTime, outputFormat); + return GetOutputFileNameUsingPatternAsync(context, configuration.FileNamePattern, streamId, containerName, asOfTime, outputFormat); } - return GetDefaultOutputFileName(configuration, streamId, containerName, asOfTime, outputFormat); + return GetDefaultOutputFileNameAsync(context, configuration, streamId, containerName, asOfTime, outputFormat); } private static bool HasCustomFileNamePattern(IDataLakeJobData configuration) @@ -400,15 +431,15 @@ private static bool HasCustomFileNamePattern(IDataLakeJobData configuration) return !string.IsNullOrWhiteSpace(configuration.FileNamePattern); } - protected virtual string GetDefaultOutputFileName(IDataLakeJobData configuration, Guid streamId, string containerName, DateTimeOffset asOfTime, string outputFormat) + protected virtual Task GetDefaultOutputFileNameAsync(ExecutionContext context, IDataLakeJobData configuration, Guid streamId, string containerName, DateTimeOffset asOfTime, string outputFormat) { var fileExtension = GetFileExtension(outputFormat); var streamIdFormatted = streamId.ToString(StreamIdDefaultStringFormat); - return $"{streamIdFormatted}_{asOfTime:yyyyMMddHHmmss}.{fileExtension}"; + return Task.FromResult($"{streamIdFormatted}_{asOfTime:yyyyMMddHHmmss}.{fileExtension}"); } - private static string GetOutputFileNameUsingPattern(string outputFileNamePattern, Guid streamId, string containerName, DateTimeOffset asOfTime, string outputFormat) + private static Task GetOutputFileNameUsingPatternAsync(ExecutionContext context, string outputFileNamePattern, Guid streamId, string containerName, DateTimeOffset asOfTime, string outputFormat) { var timeRegexPattern = @"\{(DataTime)(\:[a-zA-Z0-9\-\._]+)?\}"; var streamIdRegexPattern = @"\{(StreamId)(\:[a-zA-Z0-9\-\._]+)?\}"; @@ -431,7 +462,7 @@ private static string GetOutputFileNameUsingPattern(string outputFileNamePattern }; }); - return outputFormatReplaced; + return Task.FromResult(outputFormatReplaced); } private static string Replace(string pattern, string input, Func formatter) @@ -471,7 +502,7 @@ private DateTimeOffset GetAsOfTime(ExecutionContext context, IDataLakeJobArgs ar return args.InstanceTime; } - private static ISqlDataWriter GetSqlDataWriter(string outputFormat) + protected virtual ISqlDataWriter GetSqlDataWriter(string outputFormat) { var format = outputFormat.Trim(); if (format.Equals(DataLakeConstants.OutputFormats.Csv, StringComparison.OrdinalIgnoreCase)) @@ -484,7 +515,7 @@ private static ISqlDataWriter GetSqlDataWriter(string outputFormat) } else if (format.Equals(DataLakeConstants.OutputFormats.Parquet, StringComparison.OrdinalIgnoreCase)) { - return new ParquetSqlDataWriter(); + return new ParquetSqlDataWriter(new DefaultDataTransformer()); } throw new NotSupportedException($"Format '{outputFormat}' is not supported."); @@ -558,6 +589,73 @@ static async Task insert(SqlConnection connection, ExportHistory exportHistory) } } } + private protected virtual async Task GetLastExport(ExecutionContext context, SqlConnection connection, Guid streamId, IDataLakeJobData configuration) + { + var tableName = GetExportHistoryTableName(streamId); + + try + { + var getSql = $""" + SELECT TOP 1 + StreamId, + DataTime, + TriggerSource, + CronSchedule, + FilePath, + FileFormat, + StartTime, + EndTime, + TotalRows, + Status, + ExporterHostName + FROM + [{tableName}] + WHERE StreamId = @StreamId + """; + var command = new SqlCommand(getSql, connection) + { + CommandType = CommandType.Text + }; + + command.Parameters.Add(new SqlParameter($"@StreamId", streamId)); + + await using var reader = await command.ExecuteReaderAsync(); + var exportHistoryList = new List(); + while (await reader.ReadAsync()) + { + var dataTime = (DateTimeOffset)GetValue("DataTime", reader); + var triggerSource = (string)GetValue("TriggerSource", reader); + var cronSchedule = (string)GetValue("CronSchedule", reader); + var filePath = (string)GetValue("FilePath", reader); + var fileFormat = (string)GetValue("FileFormat", reader); + var startTime = (DateTimeOffset)GetValue("StartTime", reader); + var endTime = (DateTimeOffset?)GetValue("EndTime", reader); + var totalRows = (int?)GetValue("TotalRows", reader); + var status = (string)GetValue("status", reader); + var exporterHostName = (string)GetValue("exporterHostName", reader); + var history = new ExportHistory(streamId, dataTime, triggerSource, cronSchedule, filePath, fileFormat, startTime, endTime, totalRows, status, exporterHostName); + exportHistoryList.Add(history); + } + + return exportHistoryList.Single(); + } + catch (SqlException writeDataException) when (writeDataException.IsTableNotFoundException()) + { + return null; + } + + object GetValue(string key, SqlDataReader reader) + { + var value = reader.GetValue(key); + + if (value == DBNull.Value) + { + return null; + } + + return value; + } + } private static async Task UpdateHistory(ExecutionContext context, SqlConnection connection, ExportHistory exportHistory) { @@ -682,7 +780,7 @@ private record ExportJobData( string OutputFormat, string OutputFileName); - private record ExportHistory( + private protected record ExportHistory( Guid StreamId, DateTimeOffset DataTime, string TriggerSource, diff --git a/src/Connector.DataLake.Common/Connector/IDataLakeClient.cs b/src/Connector.DataLake.Common/Connector/IDataLakeClient.cs index 14cf746..f4d7091 100644 --- a/src/Connector.DataLake.Common/Connector/IDataLakeClient.cs +++ b/src/Connector.DataLake.Common/Connector/IDataLakeClient.cs @@ -8,8 +8,11 @@ namespace CluedIn.Connector.DataLake.Common.Connector; public interface IDataLakeClient { Task EnsureDataLakeDirectoryExist(IDataLakeJobData configuration); + Task EnsureDataLakeDirectoryExist(IDataLakeJobData configuration, string subDirectory); Task SaveData(IDataLakeJobData configuration, string content, string fileName, string contentType); Task DeleteFile(IDataLakeJobData configuration, string fileName); Task FileInPathExists(IDataLakeJobData configuration, string fileName); + Task FileInPathExists(IDataLakeJobData configuration, string fileName, string subDirectory); Task GetFilePathProperties(IDataLakeJobData configuration, string fileName); + Task GetFilePathProperties(IDataLakeJobData configuration, string fileName, string subDirectory); } diff --git a/src/Connector.DataLake.Common/Connector/SqlDataWriter/ParquetSqlDataWriter.cs b/src/Connector.DataLake.Common/Connector/SqlDataWriter/ParquetSqlDataWriter.cs index b689523..69afb4f 100644 --- a/src/Connector.DataLake.Common/Connector/SqlDataWriter/ParquetSqlDataWriter.cs +++ b/src/Connector.DataLake.Common/Connector/SqlDataWriter/ParquetSqlDataWriter.cs @@ -21,10 +21,16 @@ namespace CluedIn.Connector.DataLake.Common.Connector.SqlDataWriter; internal class ParquetSqlDataWriter : SqlDataWriterBase { + public ParquetSqlDataWriter (IDataTransformer dataTransformer) + { + _dataTransformer = dataTransformer; + } // From documentation, it's ambiguous whether we need at least 5k or 50k rows to be optimal - // TODO: Find recommendations or method to calculate row group threshold + // TODO: Find recommendations or method to calculate row group threshold public const int RowGroupThreshold = 10000; private static readonly Regex NonAlphaNumericRegex = new ("[^a-zA-Z0-9_]"); + private readonly IDataTransformer _dataTransformer; + public override async Task WriteOutputAsync( ExecutionContext context, IDataLakeJobData configuration, @@ -33,19 +39,21 @@ public override async Task WriteOutputAsync( SqlDataReader reader) { var fields = new List(); + //var fieldNamesToUse = configuration.IsDeltaMode ? fieldNames.Where(fieldName => fieldName != "__ChangeType__") : fieldNames; foreach (var fieldName in fieldNames) { var type = reader.GetFieldType(fieldName); - var parquetFieldName = configuration.ShouldEscapeVocabularyKeys ? EscapeVocabularyKey(fieldName) : fieldName; - if (fieldName == "Codes" || fieldName == "OutgoingEdges" || fieldName == "IncomingEdges") - { - fields.Add(new DataField(parquetFieldName, typeof(IEnumerable))); - } - else - { - fields.Add(new DataField(parquetFieldName, GetParquetDataType(type, configuration))); - } + var isArrayField = fieldName == "Codes" || fieldName == "OutgoingEdges" || fieldName == "IncomingEdges"; + + var parquetType = isArrayField ? typeof(IEnumerable) : GetParquetDataType(type, configuration); + var transformed = _dataTransformer.GetType(new KeyValuePair(fieldName, parquetType)); + + var transformedFieldName = transformed.Key; + var transformedFieldType = transformed.Value; + + var parquetFieldName = configuration.ShouldEscapeVocabularyKeys ? EscapeVocabularyKey(transformedFieldName) : transformedFieldName; + fields.Add(new DataField(parquetFieldName, transformedFieldType)); } var schema = new ParquetSchema(fields); @@ -56,7 +64,20 @@ public override async Task WriteOutputAsync( while (await reader.ReadAsync()) { - var fieldValues = fieldNames.Select(key => GetValue(key, reader, configuration)); + var fieldValues = fieldNames.Select(key => { + var value = GetValue(key, reader, configuration); + var transformed = _dataTransformer.Transform(new KeyValuePair(key, value)); + var transformedFieldName = transformed.Key; + var transformedFieldValue = transformed.Value; + return transformedFieldValue; + }); + + if (configuration.IsDeltaMode) + { + //var validFrom = GetValue("ValidFrom", reader, configuration); + //var validTo = GetValue("ValidTo", reader, configuration); + //var changeType = GetValue("ChangeType", reader, configuration); + } parquetTable.Add(new ParquetRow(fieldValues)); totalProcessed++; diff --git a/src/Connector.DataLake.Common/Connector/SqlDataWriter/SqlDataWriterBase.cs b/src/Connector.DataLake.Common/Connector/SqlDataWriter/SqlDataWriterBase.cs index 116a168..2e36644 100644 --- a/src/Connector.DataLake.Common/Connector/SqlDataWriter/SqlDataWriterBase.cs +++ b/src/Connector.DataLake.Common/Connector/SqlDataWriter/SqlDataWriterBase.cs @@ -45,3 +45,22 @@ public abstract Task WriteOutputAsync( ICollection fieldNames, SqlDataReader reader); } + +internal interface IDataTransformer +{ + KeyValuePair GetType(KeyValuePair pair); + KeyValuePair Transform(KeyValuePair pair); +} + +internal class DefaultDataTransformer : IDataTransformer +{ + public KeyValuePair GetType(KeyValuePair pair) + { + return pair; + } + + public KeyValuePair Transform(KeyValuePair pair) + { + return pair; + } +} diff --git a/src/Connector.DataLake.Common/DataLakeConstants.cs b/src/Connector.DataLake.Common/DataLakeConstants.cs index 8d5ad4b..600e65f 100644 --- a/src/Connector.DataLake.Common/DataLakeConstants.cs +++ b/src/Connector.DataLake.Common/DataLakeConstants.cs @@ -18,6 +18,8 @@ public abstract class DataLakeConstants : ConfigurationConstantsBase, IDataLakeC public const string ShouldWriteGuidAsString = nameof(ShouldWriteGuidAsString); public const string ShouldEscapeVocabularyKeys = nameof(ShouldEscapeVocabularyKeys); public const string CustomCron = nameof(CustomCron); + public const string IsDeltaMode = nameof(IsDeltaMode); + public const string IsOverwriteEnabled = nameof(IsOverwriteEnabled); public const string IdKey = "Id"; public const string StreamCacheConnectionStringKey = "StreamCache"; diff --git a/src/Connector.DataLake.Common/DataLakeJobData.cs b/src/Connector.DataLake.Common/DataLakeJobData.cs index c963899..71b8aa7 100644 --- a/src/Connector.DataLake.Common/DataLakeJobData.cs +++ b/src/Connector.DataLake.Common/DataLakeJobData.cs @@ -20,6 +20,8 @@ protected DataLakeJobData(IDictionary configurations, string con public virtual bool ShouldWriteGuidAsString => GetConfigurationValue(DataLakeConstants.ShouldWriteGuidAsString) as bool? ?? false; public virtual bool ShouldEscapeVocabularyKeys => GetConfigurationValue(DataLakeConstants.ShouldEscapeVocabularyKeys) as bool? ?? false; public string CustomCron => GetConfigurationValue(DataLakeConstants.CustomCron) as string; + public virtual bool IsDeltaMode => GetConfigurationValue(DataLakeConstants.IsDeltaMode) as bool? ?? false; + public virtual bool IsOverwriteEnabled => GetConfigurationValue(DataLakeConstants.IsOverwriteEnabled) as bool? ?? true; public override int GetHashCode() { @@ -40,6 +42,8 @@ protected virtual void AddToHashCode(HashCode hash) hash.Add(ShouldWriteGuidAsString); hash.Add(ShouldEscapeVocabularyKeys); hash.Add(CustomCron); + hash.Add(IsDeltaMode); + hash.Add(IsOverwriteEnabled); } public override bool Equals(object obj) @@ -59,7 +63,9 @@ public bool Equals(DataLakeJobData other) FileNamePattern == other.FileNamePattern && ShouldWriteGuidAsString == other.ShouldWriteGuidAsString && ShouldEscapeVocabularyKeys == other.ShouldEscapeVocabularyKeys && - CustomCron == other.CustomCron; + CustomCron == other.CustomCron && + IsDeltaMode == other.IsDeltaMode && + IsOverwriteEnabled == other.IsOverwriteEnabled; } protected object GetConfigurationValue(string key) diff --git a/src/Connector.DataLake.Common/IDataLakeJobData.cs b/src/Connector.DataLake.Common/IDataLakeJobData.cs index 7809bab..4dbda10 100644 --- a/src/Connector.DataLake.Common/IDataLakeJobData.cs +++ b/src/Connector.DataLake.Common/IDataLakeJobData.cs @@ -12,4 +12,6 @@ public interface IDataLakeJobData bool ShouldWriteGuidAsString { get; } bool ShouldEscapeVocabularyKeys { get; } string CustomCron { get; } + bool IsDeltaMode { get; } + bool IsOverwriteEnabled { get; } } diff --git a/src/Connector.DataLake.Common/Properties/AssemblyInfo.cs b/src/Connector.DataLake.Common/Properties/AssemblyInfo.cs index 71ab2f4..486b1af 100644 --- a/src/Connector.DataLake.Common/Properties/AssemblyInfo.cs +++ b/src/Connector.DataLake.Common/Properties/AssemblyInfo.cs @@ -3,6 +3,7 @@ [assembly: InternalsVisibleTo("CluedIn.Connector.AzureAIStudio")] [assembly: InternalsVisibleTo("CluedIn.Connector.AzureDataLake")] [assembly: InternalsVisibleTo("CluedIn.Connector.AzureDatabricks")] +[assembly: InternalsVisibleTo("CluedIn.Connector.FabricMirroring")] [assembly: InternalsVisibleTo("CluedIn.Connector.OneLake")] [assembly: InternalsVisibleTo("CluedIn.Connector.SynapseDataEngineering")] [assembly: InternalsVisibleTo("CluedIn.Connector.AzureDataLake.Tests.Unit")] diff --git a/src/Connector.FabricMirroring/Connector.FabricMirroring.csproj b/src/Connector.FabricMirroring/Connector.FabricMirroring.csproj new file mode 100644 index 0000000..8782223 --- /dev/null +++ b/src/Connector.FabricMirroring/Connector.FabricMirroring.csproj @@ -0,0 +1,20 @@ + + + CluedIn.Connector.FabricMirroring provides the ability to post data to FabricMirroring + CluedIn.Connector.FabricMirroring + CluedIn.Connector.FabricMirroring + true + + + + + + + + + + + + + + diff --git a/src/Connector.FabricMirroring/Connector/FabricMirroringClient.cs b/src/Connector.FabricMirroring/Connector/FabricMirroringClient.cs new file mode 100644 index 0000000..d138e2d --- /dev/null +++ b/src/Connector.FabricMirroring/Connector/FabricMirroringClient.cs @@ -0,0 +1,39 @@ +using System; + +using Azure.Identity; +using Azure.Storage.Files.DataLake; + +using CluedIn.Connector.DataLake.Common; +using CluedIn.Connector.DataLake.Common.Connector; + +namespace CluedIn.Connector.FabricMirroring.Connector; + +public class FabricMirroringClient : DataLakeClient +{ + protected override DataLakeServiceClient GetDataLakeServiceClient(IDataLakeJobData configuration) + { + var casted = CastJobData(configuration); + var accountName = "onelake"; + + var sharedKeyCredential = new ClientSecretCredential(casted.TenantId, casted.ClientId, casted.ClientSecret); + + var dfsUri = $"https://{accountName}.dfs.fabric.microsoft.com"; + + var dataLakeServiceClient = new DataLakeServiceClient( + new Uri(dfsUri), + sharedKeyCredential); + return dataLakeServiceClient; + } + + protected override string GetDirectory(IDataLakeJobData configuration) + { + var casted = CastJobData(configuration); + return $"{casted.ItemName}.MountedRelationalDatabase/Files/LandingZone"; + } + + protected override string GetFileSystemName(IDataLakeJobData configuration) + { + var casted = CastJobData(configuration); + return casted.WorkspaceName; + } +} diff --git a/src/Connector.FabricMirroring/Connector/FabricMirroringConnector.cs b/src/Connector.FabricMirroring/Connector/FabricMirroringConnector.cs new file mode 100644 index 0000000..dfda593 --- /dev/null +++ b/src/Connector.FabricMirroring/Connector/FabricMirroringConnector.cs @@ -0,0 +1,109 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +using Azure.Core; +using Azure.Identity; +using Azure.Storage.Files.DataLake; + +using CluedIn.Connector.DataLake.Common.Connector; +using CluedIn.Core; +using CluedIn.Core.Connectors; + +using Microsoft.Extensions.Logging; +using Microsoft.Fabric.Api; + + +namespace CluedIn.Connector.FabricMirroring.Connector; + +public class FabricMirroringConnector : DataLakeConnector +{ + public FabricMirroringConnector( + ILogger logger, + FabricMirroringClient client, + IFabricMirroringConstants constants, + FabricMirroringJobDataFactory dataLakeJobDataFactory, + IDateTimeOffsetProvider dateTimeOffsetProvider) + : base(logger, client, constants, dataLakeJobDataFactory, dateTimeOffsetProvider) + { + } + + public override async Task VerifyConnection(ExecutionContext executionContext, IReadOnlyDictionary config) + { + //var jobData = await DataLakeJobDataFactory.GetConfiguration(executionContext, new Dictionary(config)) as FabricMirroringConnectorJobData; + //var sharedKeyCredential = new ClientSecretCredential(jobData.TenantId, jobData.ClientId, jobData.ClientSecret); + + ////var fabricAdminClient = new FabricAdminClient(sharedKeyCredential); + //var fabricNonAdminClient = new FabricClient(sharedKeyCredential); + + //var fabricClient = fabricNonAdminClient; + //// Get the list of workspaces using the client + //var workspaces = fabricClient.Core.Workspaces.ListWorkspaces().ToList(); + //Console.WriteLine("Number of workspaces: " + workspaces.Count); + //foreach (var workspace in workspaces) + //{ + // Console.WriteLine($"Workspace: {workspace.DisplayName}, Capacity ID: {workspace.CapacityId}"); + //} + //var accountName = "onelake"; + + //var dfsUri = $"https://onelake.dfs.fabric.microsoft.com/"; + + //var dataLakeServiceClient = new DataLakeServiceClient( + // new Uri(dfsUri), + // sharedKeyCredential); + + //{ + // await foreach (var item in dataLakeServiceClient.GetFileSystemsAsync()) + // { + // Console.WriteLine("FIleSystems: " + item.Name); + // var fileSystemClient = dataLakeServiceClient.GetFileSystemClient(item.Name); + // await foreach (var dir in fileSystemClient.GetPathsAsync()) + // { + // Console.WriteLine("dir: " + dir.Name); + // } + + // } + //} + + //{ + // var fileSystemClient = dataLakeServiceClient.GetFileSystemClient(jobData.WorkspaceName); + // var directoryClient = fileSystemClient.GetDirectoryClient("CluedIn_Mirroring.MountedRelationalDatabase/Files/LandingZone/peng"); + + // await foreach (var item in directoryClient.GetPathsAsync()) + // { + // Console.WriteLine("File1: " + item.Name); + // } + + // var fileClient = directoryClient.GetFileClient("_manifest.json"); + // var fileClient2 = directoryClient.GetFileClient("test.json"); + // //fileClient2.CreateIfNotExists(); + // var stream = fileClient2.OpenWrite(true); + // stream.Write(Encoding.UTF8.GetBytes("sssss.sdfd")); + // await stream.FlushAsync(); + // var fileProperties = await fileClient.GetPropertiesAsync(); + + // Console.WriteLine("Fileprop1: " + fileProperties.Value.ContentLength); + // Console.WriteLine("Fileprop1: " + fileProperties.Value.ContentDisposition); + //} + + //{ + // var fileSystemClient = dataLakeServiceClient.GetFileSystemClient(jobData.WorkspaceName); + // var directoryClient = fileSystemClient.GetDirectoryClient("b0af7e2a-62ab-4800-8c17-2e3e50621a05/bd87a442-0e1e-471f-b1dd-ae667f000f4c/Files/LandingZone/peng"); + + // await foreach (var item in directoryClient.GetPathsAsync()) + // { + // Console.WriteLine("File2: " + item.Name); + // } + + // var fileClient = directoryClient.GetFileClient("_manifest.json"); + // var fileProperties = await fileClient.GetPropertiesAsync(); + + // Console.WriteLine("Fileprop2: " + fileProperties.Value.ContentLength); + // Console.WriteLine("Fileprop2: " + fileProperties.Value.ContentDisposition); + //} + + return await base.VerifyConnection(executionContext, config); + } +} diff --git a/src/Connector.FabricMirroring/Connector/FabricMirroringExportEntitiesJob.cs b/src/Connector.FabricMirroring/Connector/FabricMirroringExportEntitiesJob.cs new file mode 100644 index 0000000..f5e74eb --- /dev/null +++ b/src/Connector.FabricMirroring/Connector/FabricMirroringExportEntitiesJob.cs @@ -0,0 +1,142 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using System.Threading.Tasks; + +using Azure.Storage.Files.DataLake; + +using CluedIn.Connector.DataLake.Common; +using CluedIn.Connector.DataLake.Common.Connector; +using CluedIn.Connector.DataLake.Common.Connector.SqlDataWriter; +using CluedIn.Core; +using CluedIn.Core.Data.Parts; +using CluedIn.Core.Streams; + +using Microsoft.Data.SqlClient; +using Microsoft.Data.SqlClient.Server; + +using Newtonsoft.Json; + +using static CluedIn.Connector.DataLake.Common.DataLakeConstants; + +namespace CluedIn.Connector.FabricMirroring.Connector; + +internal class FabricMirroringExportEntitiesJob : DataLakeExportEntitiesJobBase +{ + public FabricMirroringExportEntitiesJob( + ApplicationContext appContext, + IStreamRepository streamRepository, + FabricMirroringClient dataLakeClient, + IFabricMirroringConstants dataLakeConstants, + FabricMirroringJobDataFactory dataLakeJobDataFactory, + IDateTimeOffsetProvider dateTimeOffsetProvider) + : base(appContext, streamRepository, dataLakeClient, dataLakeConstants, dataLakeJobDataFactory, dateTimeOffsetProvider) + { + } + + protected override async Task GetDefaultOutputFileNameAsync(ExecutionContext context, IDataLakeJobData configuration, Guid streamId, string containerName, DateTimeOffset asOfTime, string outputFormat) + { + if (LastExport == null) + { + return $"{1:D20}.{outputFormat.ToLowerInvariant()}"; + } + + + var lastName = Path.GetFileNameWithoutExtension(LastExport.FilePath); + var lastCount = int.Parse(lastName); + + var newCount = lastCount + 1; + return $"{newCount:D20}.{outputFormat.ToLowerInvariant()}"; + + //return await base.GetDefaultOutputFileNameAsync(context, configuration, streamId, containerName, asOfTime, outputFormat); + } + + protected override async Task InitializeDirectoryAsync(IDataLakeJobData configuration, Guid streamId, DataLakeDirectoryClient directoryClient) + { + var fileClient = directoryClient.GetFileClient("_metadata.json"); + + if (!await fileClient.ExistsAsync()) + { + await using var outputStream = await fileClient.OpenWriteAsync(true); + await outputStream.WriteAsync(Encoding.UTF8.GetBytes( + $$""" + { + "keyColumns": ["Id"] + } + """)); + await outputStream.FlushAsync(); + } + } + + private protected override async Task GetLastExport(ExecutionContext context, SqlConnection connection, Guid streamId, IDataLakeJobData configuration) + { + var subDirectory = await GetSubDirectory(configuration, streamId); + if (!await _dataLakeClient.FileInPathExists(configuration, "_metadata.json", subDirectory)) + { + return null; + } + + return await base.GetLastExport(context, connection, streamId, configuration); + } + + protected override Task GetSubDirectory(IDataLakeJobData configuration, Guid streamId) + { + return Task.FromResult(streamId.ToString("N")); + } + + protected override ISqlDataWriter GetSqlDataWriter(string outputFormat) + { + var format = outputFormat.Trim(); + if (format.Equals(DataLakeConstants.OutputFormats.Parquet, StringComparison.OrdinalIgnoreCase)) + { + return new ParquetSqlDataWriter(new FabricDataTransformer()); + } + return base.GetSqlDataWriter(outputFormat); + } +} + +internal class FabricDataTransformer : IDataTransformer +{ + public KeyValuePair GetType(KeyValuePair pair) + { + if (pair.Key == "__ChangeType__") + { + return new KeyValuePair("__rowMarker__", typeof(string)); + } + + if (pair.Value == typeof(IEnumerable)) + { + return new KeyValuePair(pair.Key, typeof(string)); + } + + return pair; + } + + public KeyValuePair Transform(KeyValuePair pair) + { + /* + 0 = INSERT + 1 = UPDATE + 2 = DELETE + 3 = UPSERT + */ + if (pair.Key == "__ChangeType__") + { + var changeType = Enum.Parse(pair.Value as string); + var value = changeType switch + { + VersionChangeType.Removed => 2, + _ => 3, + }; + return new KeyValuePair("__rowMarker__", value.ToString()); + } + + if (pair.Value is IEnumerable) + { + return new KeyValuePair(pair.Key, JsonConvert.SerializeObject(pair.Value)); + } + + return pair; + } +} diff --git a/src/Connector.FabricMirroring/FabricMirroringConnectorComponent.cs b/src/Connector.FabricMirroring/FabricMirroringConnectorComponent.cs new file mode 100644 index 0000000..bafb06c --- /dev/null +++ b/src/Connector.FabricMirroring/FabricMirroringConnectorComponent.cs @@ -0,0 +1,30 @@ +using CluedIn.Connector.DataLake.Common; +using CluedIn.Connector.FabricMirroring.Connector; +using CluedIn.Core; + +using ComponentHost; + +namespace CluedIn.Connector.FabricMirroring; + +[Component(nameof(FabricMirroringConnectorComponent), "Providers", ComponentType.Service, + ServerComponents.ProviderWebApi, + Components.Server, Components.DataStores, Isolation = ComponentIsolation.NotIsolated)] +public sealed class FabricMirroringConnectorComponent : DataLakeConnectorComponentBase +{ + public FabricMirroringConnectorComponent(ComponentInfo componentInfo) : base(componentInfo) + { + Container.Install(new InstallComponents()); + } + + /// Starts this instance. + public override void Start() + { + DefaultStartInternal(); + } + + public const string ComponentName = "FabricMirroring"; + + protected override string ConnectorComponentName => ComponentName; + + protected override string ShortConnectorComponentName => ComponentName; +} diff --git a/src/Connector.FabricMirroring/FabricMirroringConnectorJobData.cs b/src/Connector.FabricMirroring/FabricMirroringConnectorJobData.cs new file mode 100644 index 0000000..6b977c2 --- /dev/null +++ b/src/Connector.FabricMirroring/FabricMirroringConnectorJobData.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; + +using CluedIn.Connector.DataLake.Common; + +namespace CluedIn.Connector.FabricMirroring; + +internal class FabricMirroringConnectorJobData : DataLakeJobData +{ + public FabricMirroringConnectorJobData( + IDictionary configurations, + string containerName = null) + : base(configurations, containerName) + { + } + + public string WorkspaceName => Configurations[FabricMirroringConstants.WorkspaceName] as string; + public string ItemName => Configurations[FabricMirroringConstants.ItemName] as string; + public string ItemType => Configurations[FabricMirroringConstants.ItemType] as string; + public string ItemFolder => Configurations[FabricMirroringConstants.ItemFolder] as string; + public string ClientId => Configurations[FabricMirroringConstants.ClientId] as string; + public string ClientSecret => Configurations[FabricMirroringConstants.ClientSecret] as string; + public string TenantId => Configurations[FabricMirroringConstants.TenantId] as string; + public override bool ShouldWriteGuidAsString => true; + public override bool ShouldEscapeVocabularyKeys => true; + public override bool IsDeltaMode => true; + public override bool IsOverwriteEnabled => false; + + protected override void AddToHashCode(HashCode hash) + { + hash.Add(WorkspaceName); + hash.Add(ItemName); + hash.Add(ItemType); + hash.Add(ItemFolder); + hash.Add(ClientId); + hash.Add(ClientSecret); + hash.Add(TenantId); + + base.AddToHashCode(hash); + } + + public override bool Equals(object obj) + { + return Equals(obj as FabricMirroringConnectorJobData); + } + + public bool Equals(FabricMirroringConnectorJobData other) + { + return other != null && + WorkspaceName == other.WorkspaceName && + ItemName == other.ItemName && + ItemType == other.ItemType && + ItemFolder == other.ItemFolder && + ClientId == other.ClientId && + ClientSecret == other.ClientSecret && + TenantId == other.TenantId && + base.Equals(other); + } + + public override int GetHashCode() + { + return base.GetHashCode(); + } +} diff --git a/src/Connector.FabricMirroring/FabricMirroringConnectorProvider.cs b/src/Connector.FabricMirroring/FabricMirroringConnectorProvider.cs new file mode 100644 index 0000000..8d6be50 --- /dev/null +++ b/src/Connector.FabricMirroring/FabricMirroringConnectorProvider.cs @@ -0,0 +1,25 @@ +using CluedIn.Connector.DataLake.Common; +using CluedIn.Core; +using Microsoft.Extensions.Logging; +using System.Collections.Generic; + +namespace CluedIn.Connector.FabricMirroring; + +public class FabricMirroringConnectorProvider : ConnectorProviderBase +{ + public FabricMirroringConnectorProvider([NotNull] ApplicationContext appContext, + IFabricMirroringConstants configuration, ILogger logger) + : base(appContext, configuration, logger) + { + } + + protected override IEnumerable ProviderNameParts => new[] + { + FabricMirroringConstants.WorkspaceName, + FabricMirroringConstants.ItemFolder, + FabricMirroringConstants.ItemType, + FabricMirroringConstants.ItemName, + FabricMirroringConstants.ClientId, + FabricMirroringConstants.TenantId, + }; +} diff --git a/src/Connector.FabricMirroring/FabricMirroringConstants.cs b/src/Connector.FabricMirroring/FabricMirroringConstants.cs new file mode 100644 index 0000000..dac0aaa --- /dev/null +++ b/src/Connector.FabricMirroring/FabricMirroringConstants.cs @@ -0,0 +1,98 @@ +using CluedIn.Connector.DataLake.Common; +using CluedIn.Core; +using CluedIn.Core.Providers; + +using System; +using System.Collections.Generic; +// ReSharper disable ArgumentsStyleStringLiteral + +namespace CluedIn.Connector.FabricMirroring; +public class FabricMirroringConstants : DataLakeConstants, IFabricMirroringConstants +{ + internal static readonly Guid DataLakeProviderId = Guid.Parse("75CD5880-0537-4C6D-AE14-511C273ACD68"); + + public const string WorkspaceName = nameof(WorkspaceName); + public const string ItemName = nameof(ItemName); + public const string ItemType = nameof(ItemType); + public const string ItemFolder = nameof(ItemFolder); + public const string ClientId = nameof(ClientId); + public const string ClientSecret = nameof(ClientSecret); + public const string TenantId = nameof(TenantId); + + public FabricMirroringConstants(ApplicationContext applicationContext) : base(DataLakeProviderId, + providerName: "FabricMirroring Connector", + componentName: "FabricMirroringConnector", + icon: "Resources.fabricMirroring.svg", + domain: "https://azure.microsoft.com/en-us/services/data-lake-analytics/", + about: "Supports publishing of data to FabricMirroring.", + authMethods: GetFabricMirroringAuthMethods(applicationContext), + guideDetails: "Supports publishing of data to FabricMirroring.", + guideInstructions: "Provide authentication instructions here, if applicable") // TODO: ROK: + { + } + + protected override string CacheKeyword => "FabricMirroringConnector"; + + private static AuthMethods GetFabricMirroringAuthMethods(ApplicationContext applicationContext) + { + var controls = new List + { + new () + { + Name = WorkspaceName, + DisplayName = WorkspaceName, + Type = "input", + IsRequired = true + }, + new () + { + Name = ItemName, + DisplayName = ItemName, + Type = "input", + IsRequired = true + }, + new () + { + Name = ItemType, + DisplayName = ItemType, + Type = "input", + IsRequired = true + }, + new () + { + Name = ItemFolder, + DisplayName = ItemFolder, + Type = "input", + IsRequired = true + }, + new () + { + Name = ClientId, + DisplayName = ClientId, + Type = "input", + IsRequired = true + }, + new () + { + Name = ClientSecret, + DisplayName = ClientSecret, + Type = "password", + IsRequired = true + }, + new () + { + Name = TenantId, + DisplayName = TenantId, + Type = "input", + IsRequired = true + }, + }; + + controls.AddRange(GetAuthMethods(applicationContext)); + + return new AuthMethods + { + Token = controls + }; + } +} diff --git a/src/Connector.FabricMirroring/FabricMirroringJobDataFactory.cs b/src/Connector.FabricMirroring/FabricMirroringJobDataFactory.cs new file mode 100644 index 0000000..00ae6eb --- /dev/null +++ b/src/Connector.FabricMirroring/FabricMirroringJobDataFactory.cs @@ -0,0 +1,18 @@ +using System.Collections.Generic; +using System.Threading.Tasks; + +using CluedIn.Connector.DataLake.Common; +using CluedIn.Core; + +namespace CluedIn.Connector.FabricMirroring; + +public class FabricMirroringJobDataFactory : DataLakeJobDataFactoryBase, IDataLakeJobDataFactory +{ + protected override Task CreateJobData( + ExecutionContext executionContext, + IDictionary authenticationDetails, + string containerName) + { + return Task.FromResult(new FabricMirroringConnectorJobData(authenticationDetails, containerName)); + } +} diff --git a/src/Connector.FabricMirroring/IFabricMirroringConstants.cs b/src/Connector.FabricMirroring/IFabricMirroringConstants.cs new file mode 100644 index 0000000..5dd74d1 --- /dev/null +++ b/src/Connector.FabricMirroring/IFabricMirroringConstants.cs @@ -0,0 +1,9 @@ +using CluedIn.Connector.DataLake.Common; +// ReSharper disable ArgumentsStyleStringLiteral + +namespace CluedIn.Connector.FabricMirroring; + +public interface IFabricMirroringConstants : IDataLakeConstants +{ + +} diff --git a/src/Connector.FabricMirroring/InstallComponents.cs b/src/Connector.FabricMirroring/InstallComponents.cs new file mode 100644 index 0000000..4f228f6 --- /dev/null +++ b/src/Connector.FabricMirroring/InstallComponents.cs @@ -0,0 +1,15 @@ +using Castle.MicroKernel.SubSystems.Configuration; +using Castle.Windsor; + +using CluedIn.Connector.DataLake.Common; +using CluedIn.Connector.FabricMirroring.Connector; + +namespace CluedIn.Connector.FabricMirroring; + +internal class InstallComponents : InstallComponentsBase +{ + public override void Install(IWindsorContainer container, IConfigurationStore store) + { + DefaultInstall(container, store); + } +} diff --git a/src/Connector.FabricMirroring/Resources/fabricMirroring.svg b/src/Connector.FabricMirroring/Resources/fabricMirroring.svg new file mode 100644 index 0000000..5c92217 --- /dev/null +++ b/src/Connector.FabricMirroring/Resources/fabricMirroring.svg @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 5a3d019a7874370647b3aa0d3f9f711768d2947d Mon Sep 17 00:00:00 2001 From: Ho Peng Foong Date: Wed, 12 Feb 2025 13:07:41 +0800 Subject: [PATCH 2/6] s --- .../Connector/DataLakeClient.cs | 12 + .../Connector/DataLakeConnector.cs | 9 +- .../Connector/IDataLakeClient.cs | 1 + .../DataLakeConnectorComponentBase.cs | 2 +- .../EventHandlers/RemoveStreamEventHandler.cs | 2 +- .../UpdateExportTargetEventHandler.cs | 2 +- .../EventHandlers/UpdateStreamScheduleBase.cs | 75 +++--- .../Connector/FabricMirroringConnector.cs | 93 +------ .../RegisterExportTargetEventHandler.cs | 253 ++++++++++++++++++ .../UpdateExportTargetEventHandler.cs | 67 +++++ .../FabricMirroringConnectorJobData.cs | 9 +- .../FabricMirroringConnectorProvider.cs | 2 - .../FabricMirroringConstants.cs | 27 +- .../FabricMirroringJobDataFactory.cs | 19 +- 14 files changed, 417 insertions(+), 156 deletions(-) create mode 100644 src/Connector.FabricMirroring/EventHandlers/RegisterExportTargetEventHandler.cs create mode 100644 src/Connector.FabricMirroring/EventHandlers/UpdateExportTargetEventHandler.cs diff --git a/src/Connector.DataLake.Common/Connector/DataLakeClient.cs b/src/Connector.DataLake.Common/Connector/DataLakeClient.cs index 81b7e8b..b21b0f7 100644 --- a/src/Connector.DataLake.Common/Connector/DataLakeClient.cs +++ b/src/Connector.DataLake.Common/Connector/DataLakeClient.cs @@ -91,6 +91,18 @@ public async Task FileInPathExists(IDataLakeJobData configuration, string return await dataLakeFileClient.ExistsAsync(); } + public async Task DirectoryExists(IDataLakeJobData configuration, string subDirectory) + { + var fileSystemClient = await GetFileSystemClientAsync(configuration, ensureExists: false); + if (!await fileSystemClient.ExistsAsync()) + { + return false; + } + + var directoryClient = await GetDirectoryClientAsync(configuration, fileSystemClient, subDirectory, ensureExists: false); + return await directoryClient.ExistsAsync(); + } + public Task GetFilePathProperties(IDataLakeJobData configuration, string fileName) { return GetFilePathProperties(configuration, fileName, string.Empty); diff --git a/src/Connector.DataLake.Common/Connector/DataLakeConnector.cs b/src/Connector.DataLake.Common/Connector/DataLakeConnector.cs index f4b3887..448d8ee 100644 --- a/src/Connector.DataLake.Common/Connector/DataLakeConnector.cs +++ b/src/Connector.DataLake.Common/Connector/DataLakeConnector.cs @@ -479,7 +479,7 @@ public override async Task VerifyConnection(Execut try { var jobData = await _dataLakeJobDataFactory.GetConfiguration(executionContext, config.ToDictionary(config => config.Key, config => config.Value)); - await _client.EnsureDataLakeDirectoryExist(jobData); + await VerifyDataLakeConnection(jobData); if (jobData.IsStreamCacheEnabled) { @@ -507,7 +507,7 @@ public override async Task VerifyConnection(Execut if (!string.IsNullOrWhiteSpace(jobData.FileNamePattern)) { var trimmed = jobData.FileNamePattern.Trim(); - var invalidCharacters = new[] { '/', '\\', '?', '%' }; + var invalidCharacters = new[] { '/', '\\', '?', '%' }; if (trimmed.StartsWith(".")) { return new ConnectionVerificationResult(false, "File name pattern cannot start with a period."); @@ -528,6 +528,11 @@ public override async Task VerifyConnection(Execut } } + protected virtual async Task VerifyDataLakeConnection(IDataLakeJobData jobData) + { + await _client.EnsureDataLakeDirectoryExist(jobData); + } + private async Task VerifyTableOperations(string connectionString) { var testStreamId = Guid.NewGuid(); diff --git a/src/Connector.DataLake.Common/Connector/IDataLakeClient.cs b/src/Connector.DataLake.Common/Connector/IDataLakeClient.cs index f4d7091..a5f0a1d 100644 --- a/src/Connector.DataLake.Common/Connector/IDataLakeClient.cs +++ b/src/Connector.DataLake.Common/Connector/IDataLakeClient.cs @@ -15,4 +15,5 @@ public interface IDataLakeClient Task FileInPathExists(IDataLakeJobData configuration, string fileName, string subDirectory); Task GetFilePathProperties(IDataLakeJobData configuration, string fileName); Task GetFilePathProperties(IDataLakeJobData configuration, string fileName, string subDirectory); + Task DirectoryExists(IDataLakeJobData configuration, string subDirectory); } diff --git a/src/Connector.DataLake.Common/DataLakeConnectorComponentBase.cs b/src/Connector.DataLake.Common/DataLakeConnectorComponentBase.cs index 5219e43..20c5673 100644 --- a/src/Connector.DataLake.Common/DataLakeConnectorComponentBase.cs +++ b/src/Connector.DataLake.Common/DataLakeConnectorComponentBase.cs @@ -62,7 +62,7 @@ public override void Stop() State = ServiceState.Stopped; } - private protected void SubscribeToEvents(IDataLakeConstants constants, IDataLakeJobDataFactory jobDataFactory, IScheduledJobQueue jobQueue) + private protected virtual void SubscribeToEvents(IDataLakeConstants constants, IDataLakeJobDataFactory jobDataFactory, IScheduledJobQueue jobQueue) { var dateTimeProvider = Container.Resolve(); _updateExportTargetEventHandler = new(ApplicationContext, constants, jobDataFactory, dateTimeProvider, ExportEntitiesJobType, jobQueue); diff --git a/src/Connector.DataLake.Common/EventHandlers/RemoveStreamEventHandler.cs b/src/Connector.DataLake.Common/EventHandlers/RemoveStreamEventHandler.cs index f1d2ced..7b8003f 100644 --- a/src/Connector.DataLake.Common/EventHandlers/RemoveStreamEventHandler.cs +++ b/src/Connector.DataLake.Common/EventHandlers/RemoveStreamEventHandler.cs @@ -47,7 +47,7 @@ protected virtual void Dispose(bool disposing) private void ProcessEvent(RemoveStreamEvent eventData) { - if (!TryGetResourceInfo(eventData, "AccountId", "StreamId", out var organizationId, out var streamId)) + if (!eventData.TryGetResourceInfo("AccountId", "StreamId", out var organizationId, out var streamId)) { return; } diff --git a/src/Connector.DataLake.Common/EventHandlers/UpdateExportTargetEventHandler.cs b/src/Connector.DataLake.Common/EventHandlers/UpdateExportTargetEventHandler.cs index c506482..52ecca2 100644 --- a/src/Connector.DataLake.Common/EventHandlers/UpdateExportTargetEventHandler.cs +++ b/src/Connector.DataLake.Common/EventHandlers/UpdateExportTargetEventHandler.cs @@ -42,7 +42,7 @@ private void ProcessEvent(UpdateExportTargetEvent eventData) private async Task ProcessEventAsync(UpdateExportTargetEvent eventData) { - if (!TryGetResourceInfo(eventData, "AccountId", "Id", out var organizationId, out var providerDefinitionId)) + if (!eventData.TryGetResourceInfo("AccountId", "Id", out var organizationId, out var providerDefinitionId)) { return; } diff --git a/src/Connector.DataLake.Common/EventHandlers/UpdateStreamScheduleBase.cs b/src/Connector.DataLake.Common/EventHandlers/UpdateStreamScheduleBase.cs index 2f088d9..5785ee2 100644 --- a/src/Connector.DataLake.Common/EventHandlers/UpdateStreamScheduleBase.cs +++ b/src/Connector.DataLake.Common/EventHandlers/UpdateStreamScheduleBase.cs @@ -42,44 +42,9 @@ protected UpdateStreamScheduleBase( JobQueue = jobQueue ?? throw new ArgumentNullException(nameof(jobQueue)); } - protected static bool TryGetResourceInfo(RemoteEvent remoteEvent, string organizationIdKey, string resourceIdKey, out Guid organizationId, out Guid resourceId) - { - if (!(remoteEvent.EventData?.StartsWith("{") ?? false)) - { - setResultToDefault(out organizationId, out resourceId); - return false; - } - - var eventObject = JsonUtility.Deserialize(remoteEvent.EventData); - if (!eventObject.TryGetValue(organizationIdKey, out var accountIdToken)) - { - setResultToDefault(out organizationId, out resourceId); - return false; - } - - if (!eventObject.TryGetValue(resourceIdKey, out var resourceIdToken)) - { - setResultToDefault(out organizationId, out resourceId); - return false; - } - - var accountId = accountIdToken.Value(); - var resourceIdString = resourceIdToken.Value(); - resourceId = new Guid(resourceIdString); - organizationId = new Guid(accountId); - - return true; - - static void setResultToDefault(out Guid organizationId, out Guid resourceId) - { - organizationId = Guid.Empty; - resourceId = Guid.Empty; - } - } - protected async Task UpdateStreamScheduleFromStreamEvent(RemoteEvent remoteEvent) { - if (!TryGetResourceInfo(remoteEvent, "AccountId", "StreamId", out var organizationId, out var streamId)) + if (!remoteEvent.TryGetResourceInfo("AccountId", "StreamId", out var organizationId, out var streamId)) { return; } @@ -144,3 +109,41 @@ private async Task IsDataLakeProvider(ExecutionContext executionContext, S return provider.ProviderId == Constants.ProviderId; } } + +internal static class RemoteEventExtensions +{ + internal static bool TryGetResourceInfo(this RemoteEvent remoteEvent, string organizationIdKey, string resourceIdKey, out Guid organizationId, out Guid resourceId) + { + if (!(remoteEvent.EventData?.StartsWith("{") ?? false)) + { + setResultToDefault(out organizationId, out resourceId); + return false; + } + + var eventObject = JsonUtility.Deserialize(remoteEvent.EventData); + if (!eventObject.TryGetValue(organizationIdKey, out var accountIdToken)) + { + setResultToDefault(out organizationId, out resourceId); + return false; + } + + if (!eventObject.TryGetValue(resourceIdKey, out var resourceIdToken)) + { + setResultToDefault(out organizationId, out resourceId); + return false; + } + + var accountId = accountIdToken.Value(); + var resourceIdString = resourceIdToken.Value(); + resourceId = new Guid(resourceIdString); + organizationId = new Guid(accountId); + + return true; + + static void setResultToDefault(out Guid organizationId, out Guid resourceId) + { + organizationId = Guid.Empty; + resourceId = Guid.Empty; + } + } +} diff --git a/src/Connector.FabricMirroring/Connector/FabricMirroringConnector.cs b/src/Connector.FabricMirroring/Connector/FabricMirroringConnector.cs index dfda593..7e5cee5 100644 --- a/src/Connector.FabricMirroring/Connector/FabricMirroringConnector.cs +++ b/src/Connector.FabricMirroring/Connector/FabricMirroringConnector.cs @@ -1,19 +1,10 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -using Azure.Core; -using Azure.Identity; -using Azure.Storage.Files.DataLake; +using System.Threading.Tasks; +using CluedIn.Connector.DataLake.Common; using CluedIn.Connector.DataLake.Common.Connector; using CluedIn.Core; -using CluedIn.Core.Connectors; using Microsoft.Extensions.Logging; -using Microsoft.Fabric.Api; namespace CluedIn.Connector.FabricMirroring.Connector; @@ -30,80 +21,12 @@ public FabricMirroringConnector( { } - public override async Task VerifyConnection(ExecutionContext executionContext, IReadOnlyDictionary config) + protected override Task VerifyDataLakeConnection(IDataLakeJobData jobData) { - //var jobData = await DataLakeJobDataFactory.GetConfiguration(executionContext, new Dictionary(config)) as FabricMirroringConnectorJobData; - //var sharedKeyCredential = new ClientSecretCredential(jobData.TenantId, jobData.ClientId, jobData.ClientSecret); - - ////var fabricAdminClient = new FabricAdminClient(sharedKeyCredential); - //var fabricNonAdminClient = new FabricClient(sharedKeyCredential); - - //var fabricClient = fabricNonAdminClient; - //// Get the list of workspaces using the client - //var workspaces = fabricClient.Core.Workspaces.ListWorkspaces().ToList(); - //Console.WriteLine("Number of workspaces: " + workspaces.Count); - //foreach (var workspace in workspaces) - //{ - // Console.WriteLine($"Workspace: {workspace.DisplayName}, Capacity ID: {workspace.CapacityId}"); - //} - //var accountName = "onelake"; - - //var dfsUri = $"https://onelake.dfs.fabric.microsoft.com/"; - - //var dataLakeServiceClient = new DataLakeServiceClient( - // new Uri(dfsUri), - // sharedKeyCredential); - - //{ - // await foreach (var item in dataLakeServiceClient.GetFileSystemsAsync()) - // { - // Console.WriteLine("FIleSystems: " + item.Name); - // var fileSystemClient = dataLakeServiceClient.GetFileSystemClient(item.Name); - // await foreach (var dir in fileSystemClient.GetPathsAsync()) - // { - // Console.WriteLine("dir: " + dir.Name); - // } - - // } - //} - - //{ - // var fileSystemClient = dataLakeServiceClient.GetFileSystemClient(jobData.WorkspaceName); - // var directoryClient = fileSystemClient.GetDirectoryClient("CluedIn_Mirroring.MountedRelationalDatabase/Files/LandingZone/peng"); - - // await foreach (var item in directoryClient.GetPathsAsync()) - // { - // Console.WriteLine("File1: " + item.Name); - // } - - // var fileClient = directoryClient.GetFileClient("_manifest.json"); - // var fileClient2 = directoryClient.GetFileClient("test.json"); - // //fileClient2.CreateIfNotExists(); - // var stream = fileClient2.OpenWrite(true); - // stream.Write(Encoding.UTF8.GetBytes("sssss.sdfd")); - // await stream.FlushAsync(); - // var fileProperties = await fileClient.GetPropertiesAsync(); - - // Console.WriteLine("Fileprop1: " + fileProperties.Value.ContentLength); - // Console.WriteLine("Fileprop1: " + fileProperties.Value.ContentDisposition); - //} - - //{ - // var fileSystemClient = dataLakeServiceClient.GetFileSystemClient(jobData.WorkspaceName); - // var directoryClient = fileSystemClient.GetDirectoryClient("b0af7e2a-62ab-4800-8c17-2e3e50621a05/bd87a442-0e1e-471f-b1dd-ae667f000f4c/Files/LandingZone/peng"); - - // await foreach (var item in directoryClient.GetPathsAsync()) - // { - // Console.WriteLine("File2: " + item.Name); - // } - - // var fileClient = directoryClient.GetFileClient("_manifest.json"); - // var fileProperties = await fileClient.GetPropertiesAsync(); - - // Console.WriteLine("Fileprop2: " + fileProperties.Value.ContentLength); - // Console.WriteLine("Fileprop2: " + fileProperties.Value.ContentDisposition); - //} - - return await base.VerifyConnection(executionContext, config); + //TODO: verify path exists only if mirroring created + // find out a way to see if it's called by test connection or healthcheck (might have to do ugly reflection to look at stack) + // probably could do it by verifying if jobdata from db = jobdata created from passed config + // but we need to be able to get provider definition id, which is not passed (wtf!) + return base.VerifyDataLakeConnection(jobData); } } diff --git a/src/Connector.FabricMirroring/EventHandlers/RegisterExportTargetEventHandler.cs b/src/Connector.FabricMirroring/EventHandlers/RegisterExportTargetEventHandler.cs new file mode 100644 index 0000000..325b900 --- /dev/null +++ b/src/Connector.FabricMirroring/EventHandlers/RegisterExportTargetEventHandler.cs @@ -0,0 +1,253 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; + +using Azure.Identity; + +using CluedIn.Connector.DataLake.Common; +using CluedIn.Connector.DataLake.Common.EventHandlers; +using CluedIn.Connector.FabricMirroring.Connector; +using CluedIn.Core; +using CluedIn.Core.Data.Relational; +using CluedIn.Core.Events; +using CluedIn.Core.Events.Types; + +using Microsoft.Fabric.Api; +using Microsoft.Fabric.Api.Core.Models; +using Microsoft.Fabric.Api.MirroredDatabase.Models; + +namespace CluedIn.Connector.FabricMirroring.EventHandlers; + +internal abstract class UpdateMirroredDatabaseBase +{ + private readonly ApplicationContext _applicationContext; + private readonly IDataLakeConstants _constants; + private readonly IDataLakeJobDataFactory _jobDataFactory; + private readonly IDateTimeOffsetProvider _dateTimeOffsetProvider; + + protected UpdateMirroredDatabaseBase( + ApplicationContext applicationContext, + IDataLakeConstants constants, + IDataLakeJobDataFactory jobDataFactory, + IDateTimeOffsetProvider dateTimeOffsetProvider) + { + _constants = constants; + _jobDataFactory = jobDataFactory; + _dateTimeOffsetProvider = dateTimeOffsetProvider ?? throw new ArgumentNullException(nameof(dateTimeOffsetProvider)); + } + + private FabricClient FabricClient { get; set; } + + + protected virtual async Task UpdateOrCreateMirroredDatabaseAsync(RemoteEvent eventData) + { + if (!eventData.TryGetResourceInfo("AccountId", "Id", out var organizationId, out var providerDefinitionId)) + { + return null; + } + + var executionContext = _applicationContext.CreateExecutionContext(organizationId); + + var jobData = await _jobDataFactory.GetConfiguration(executionContext, providerDefinitionId, string.Empty) as FabricMirroringConnectorJobData; + if (jobData == null) + { + throw new ApplicationException($"Failed to get job data for ProviderDefinitionId {providerDefinitionId}."); + } + var sharedKeyCredential = new ClientSecretCredential(jobData.TenantId, jobData.ClientId, jobData.ClientSecret); + + FabricClient = new FabricClient(sharedKeyCredential); + + var workspace = await GetWorkspaceAsync(jobData.WorkspaceName); + if (workspace == null) + { + throw new ApplicationException($"Failed to find workspace using {jobData.WorkspaceName}."); + } + + var mirrorDatabaseName = jobData.ItemName; + var mirroredDatabase = await GetMirroredDatabaseAsync(workspace.Id, jobData.ItemName); + if (mirroredDatabase == null) + { + if (!jobData.ShouldCreateMirroredDatabase) + { + throw new ApplicationException($"Mirrored database is not found using workspace {jobData.WorkspaceName} and mirrored database name {jobData.ItemName}."); + } + var result = await FabricClient.MirroredDatabase.Items.CreateMirroredDatabaseAsync( + workspace.Id, + new CreateMirroredDatabaseRequest( + mirrorDatabaseName, + new MirroredDatabaseDefinition( + new List + { + new MirroredDatabaseDefinitionPart + { + Path = "mirroring.json", + Payload = Convert.ToBase64String( + Encoding.UTF8.GetBytes($$""" + { + "properties": { + "source": { + "type": "GenericMirror", + "typeProperties": {} + }, + "target": { + "type": "MountedRelationalDatabase", + "typeProperties": { + "format": "Delta" + } + } + } + } + """) + ), + PayloadType = PayloadType.InlineBase64, + } + } + ) + ) + ); + + var status = await PollForCompletion(workspace.Id, result.Value.Id.Value); + if (status != SqlEndpointProvisioningStatus.Success) + { + throw new ApplicationException($"Failed to provision sql endpoint using workspace {jobData.WorkspaceName} and mirrored database name {jobData.ItemName}."); + } + + } + + mirroredDatabase = await GetMirroredDatabaseAsync(workspace.Id, jobData.ItemName); + + var store = executionContext.Organization.DataStores.GetDataStore(); + var definition = await store.GetByIdAsync(executionContext, providerDefinitionId); + if (definition.IsEnabled) + { + await StartMirroringAsync(mirroredDatabase); + } + else + { + await StopMirroringAsync(mirroredDatabase); + } + + return mirroredDatabase; + } + + private async Task StartMirroringAsync(MirroredDatabase mirroredDatabase) + { + await FabricClient.MirroredDatabase.Mirroring.StartMirroringAsync(mirroredDatabase.WorkspaceId.Value, mirroredDatabase.Id.Value); + } + private async Task StopMirroringAsync(MirroredDatabase mirroredDatabase) + { + await FabricClient.MirroredDatabase.Mirroring.StopMirroringAsync(mirroredDatabase.WorkspaceId.Value, mirroredDatabase.Id.Value); + } + + protected async Task GetJobData(RemoteEvent eventData) + { + if (!eventData.TryGetResourceInfo("AccountId", "Id", out var organizationId, out var providerDefinitionId)) + { + return null; + } + var executionContext = _applicationContext.CreateExecutionContext(organizationId); + + return await _jobDataFactory.GetConfiguration(executionContext, providerDefinitionId, string.Empty) as FabricMirroringConnectorJobData; + } + + protected async Task PollForCompletion(Guid workspaceId, Guid mirroredDatabaseId) + { + var start = _dateTimeOffsetProvider.GetCurrentUtcTime(); + while (true) + { + var result = await FabricClient.MirroredDatabase.Items.GetMirroredDatabaseAsync(workspaceId, mirroredDatabaseId); + var status = result.Value?.Properties?.SqlEndpointProperties?.ProvisioningStatus; + + if (status != null && status != SqlEndpointProvisioningStatus.InProgress) + { + return status.Value; + } + var now = _dateTimeOffsetProvider.GetCurrentUtcTime(); + if (now - start > TimeSpan.FromMinutes(10)) + { + return null; + } + + await Task.Delay(5000); + } + + } + + protected async Task GetMirroredDatabaseAsync(Guid workspaceId, string mirroredDatabaseName) + { + await foreach (var mirroredDatabase in FabricClient.MirroredDatabase.Items.ListMirroredDatabasesAsync(workspaceId)) + { + if (mirroredDatabase.DisplayName == mirroredDatabaseName) + { + return mirroredDatabase; + } + } + + return null; + } + + protected async Task StartMirroringAsync(Guid workspaceId, Guid mirroredDatabaseId) + { + await FabricClient.MirroredDatabase.Mirroring.StartMirroringAsync(workspaceId, mirroredDatabaseId); + } + + protected async Task GetWorkspaceAsync(string workspaceNamke) + { + await foreach (var workspace in FabricClient.Core.Workspaces.ListWorkspacesAsync()) + { + if (workspace.DisplayName == workspaceNamke) + { + return workspace; + } + } + + return null; + } +} +internal class RegisterExportTargetEventHandler : UpdateMirroredDatabaseBase, IDisposable +{ + private readonly IDisposable _subscription; + private bool _disposedValue; + + public RegisterExportTargetEventHandler( + ApplicationContext applicationContext, + FabricMirroringClient client, + IDataLakeConstants constants, + IDataLakeJobDataFactory jobDataFactory, + IDateTimeOffsetProvider dateTimeOffsetProvider) + : base(applicationContext, constants, jobDataFactory, dateTimeOffsetProvider) + { + _subscription = applicationContext.System.Events.Local.Subscribe(ProcessEvent); + } + + private void ProcessEvent(RegisterExportTargetEvent eventData) + { + ProcessEventAsync(eventData).GetAwaiter().GetResult(); + } + + private async Task ProcessEventAsync(RegisterExportTargetEvent eventData) + { + await UpdateOrCreateMirroredDatabaseAsync(eventData); + } + + protected virtual void Dispose(bool disposing) + { + if (!_disposedValue) + { + if (disposing) + { + _subscription.Dispose(); + } + + _disposedValue = true; + } + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } +} diff --git a/src/Connector.FabricMirroring/EventHandlers/UpdateExportTargetEventHandler.cs b/src/Connector.FabricMirroring/EventHandlers/UpdateExportTargetEventHandler.cs new file mode 100644 index 0000000..209384b --- /dev/null +++ b/src/Connector.FabricMirroring/EventHandlers/UpdateExportTargetEventHandler.cs @@ -0,0 +1,67 @@ +using System; +using System.Collections.Generic; +using System.Threading.Tasks; + +using Azure.Identity; + +using CluedIn.Connector.DataLake.Common; +using CluedIn.Connector.FabricMirroring.Connector; +using CluedIn.Core; +using CluedIn.Core.Data.Relational; +using CluedIn.Core.Events; +using CluedIn.Core.Events.Types; + +using Microsoft.Fabric.Api; +using Microsoft.Fabric.Api.Core.Models; +using Microsoft.Fabric.Api.MirroredDatabase.Models; + +using Newtonsoft.Json.Linq; + +namespace CluedIn.Connector.FabricMirroring.EventHandlers; + +internal class UpdateExportTargetEventHandler : UpdateMirroredDatabaseBase, IDisposable +{ + private readonly IDisposable _subscription; + private bool _disposedValue; + + public UpdateExportTargetEventHandler( + ApplicationContext applicationContext, + FabricMirroringClient client, + IDataLakeConstants constants, + IDataLakeJobDataFactory jobDataFactory, + IDateTimeOffsetProvider dateTimeOffsetProvider) + : base(applicationContext, constants, jobDataFactory, dateTimeOffsetProvider) + { + _subscription = applicationContext.System.Events.Local.Subscribe(ProcessEvent); + } + + private void ProcessEvent(UpdateExportTargetEvent eventData) + { + ProcessEventAsync(eventData).GetAwaiter().GetResult(); + } + + private async Task ProcessEventAsync(UpdateExportTargetEvent eventData) + { + await UpdateOrCreateMirroredDatabaseAsync(eventData); + } + + protected virtual void Dispose(bool disposing) + { + if (!_disposedValue) + { + if (disposing) + { + _subscription.Dispose(); + } + + _disposedValue = true; + } + } + + public void Dispose() + { + // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method + Dispose(disposing: true); + GC.SuppressFinalize(this); + } +} diff --git a/src/Connector.FabricMirroring/FabricMirroringConnectorJobData.cs b/src/Connector.FabricMirroring/FabricMirroringConnectorJobData.cs index 6b977c2..50e6b8d 100644 --- a/src/Connector.FabricMirroring/FabricMirroringConnectorJobData.cs +++ b/src/Connector.FabricMirroring/FabricMirroringConnectorJobData.cs @@ -16,8 +16,6 @@ public FabricMirroringConnectorJobData( public string WorkspaceName => Configurations[FabricMirroringConstants.WorkspaceName] as string; public string ItemName => Configurations[FabricMirroringConstants.ItemName] as string; - public string ItemType => Configurations[FabricMirroringConstants.ItemType] as string; - public string ItemFolder => Configurations[FabricMirroringConstants.ItemFolder] as string; public string ClientId => Configurations[FabricMirroringConstants.ClientId] as string; public string ClientSecret => Configurations[FabricMirroringConstants.ClientSecret] as string; public string TenantId => Configurations[FabricMirroringConstants.TenantId] as string; @@ -25,16 +23,16 @@ public FabricMirroringConnectorJobData( public override bool ShouldEscapeVocabularyKeys => true; public override bool IsDeltaMode => true; public override bool IsOverwriteEnabled => false; + public virtual bool ShouldCreateMirroredDatabase => false; protected override void AddToHashCode(HashCode hash) { hash.Add(WorkspaceName); hash.Add(ItemName); - hash.Add(ItemType); - hash.Add(ItemFolder); hash.Add(ClientId); hash.Add(ClientSecret); hash.Add(TenantId); + hash.Add(ShouldCreateMirroredDatabase); base.AddToHashCode(hash); } @@ -49,11 +47,10 @@ public bool Equals(FabricMirroringConnectorJobData other) return other != null && WorkspaceName == other.WorkspaceName && ItemName == other.ItemName && - ItemType == other.ItemType && - ItemFolder == other.ItemFolder && ClientId == other.ClientId && ClientSecret == other.ClientSecret && TenantId == other.TenantId && + ShouldCreateMirroredDatabase == other.ShouldCreateMirroredDatabase && base.Equals(other); } diff --git a/src/Connector.FabricMirroring/FabricMirroringConnectorProvider.cs b/src/Connector.FabricMirroring/FabricMirroringConnectorProvider.cs index 8d6be50..f97cf0d 100644 --- a/src/Connector.FabricMirroring/FabricMirroringConnectorProvider.cs +++ b/src/Connector.FabricMirroring/FabricMirroringConnectorProvider.cs @@ -16,8 +16,6 @@ public FabricMirroringConnectorProvider([NotNull] ApplicationContext appContext, protected override IEnumerable ProviderNameParts => new[] { FabricMirroringConstants.WorkspaceName, - FabricMirroringConstants.ItemFolder, - FabricMirroringConstants.ItemType, FabricMirroringConstants.ItemName, FabricMirroringConstants.ClientId, FabricMirroringConstants.TenantId, diff --git a/src/Connector.FabricMirroring/FabricMirroringConstants.cs b/src/Connector.FabricMirroring/FabricMirroringConstants.cs index dac0aaa..8fcced4 100644 --- a/src/Connector.FabricMirroring/FabricMirroringConstants.cs +++ b/src/Connector.FabricMirroring/FabricMirroringConstants.cs @@ -13,11 +13,10 @@ public class FabricMirroringConstants : DataLakeConstants, IFabricMirroringConst public const string WorkspaceName = nameof(WorkspaceName); public const string ItemName = nameof(ItemName); - public const string ItemType = nameof(ItemType); - public const string ItemFolder = nameof(ItemFolder); public const string ClientId = nameof(ClientId); public const string ClientSecret = nameof(ClientSecret); public const string TenantId = nameof(TenantId); + public const string ShouldCreateMirroredDatabase = nameof(ShouldCreateMirroredDatabase); public FabricMirroringConstants(ApplicationContext applicationContext) : base(DataLakeProviderId, providerName: "FabricMirroring Connector", @@ -42,49 +41,35 @@ private static AuthMethods GetFabricMirroringAuthMethods(ApplicationContext appl Name = WorkspaceName, DisplayName = WorkspaceName, Type = "input", - IsRequired = true + IsRequired = true, }, new () { Name = ItemName, DisplayName = ItemName, Type = "input", - IsRequired = true - }, - new () - { - Name = ItemType, - DisplayName = ItemType, - Type = "input", - IsRequired = true - }, - new () - { - Name = ItemFolder, - DisplayName = ItemFolder, - Type = "input", - IsRequired = true + IsRequired = false, }, new () { Name = ClientId, DisplayName = ClientId, Type = "input", - IsRequired = true + IsRequired = true, }, new () { Name = ClientSecret, DisplayName = ClientSecret, Type = "password", - IsRequired = true + IsRequired = true, }, new () { Name = TenantId, DisplayName = TenantId, Type = "input", - IsRequired = true + IsRequired = true, }, }; diff --git a/src/Connector.FabricMirroring/FabricMirroringJobDataFactory.cs b/src/Connector.FabricMirroring/FabricMirroringJobDataFactory.cs index 00ae6eb..491b58b 100644 --- a/src/Connector.FabricMirroring/FabricMirroringJobDataFactory.cs +++ b/src/Connector.FabricMirroring/FabricMirroringJobDataFactory.cs @@ -1,4 +1,6 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; +using System.Linq; using System.Threading.Tasks; using CluedIn.Connector.DataLake.Common; @@ -15,4 +17,19 @@ protected override Task CreateJobData( { return Task.FromResult(new FabricMirroringConnectorJobData(authenticationDetails, containerName)); } + + public override async Task GetConfiguration(ExecutionContext executionContext, Guid providerDefinitionId, string containerName) + { + var authenticationDetails = await GetAuthenticationDetails(executionContext, providerDefinitionId); + var dictionary = authenticationDetails.Authentication.ToDictionary(detail => detail.Key, detail => detail.Value); + + if (!dictionary.TryGetValue(FabricMirroringConstants.ItemName, out var value) + || string.IsNullOrWhiteSpace(value?.ToString())) + { + dictionary[FabricMirroringConstants.ItemName] = $"CluedIn_ExportTarget_{providerDefinitionId:N}"; + dictionary[FabricMirroringConstants.ShouldCreateMirroredDatabase] = true; + } + + return await GetConfiguration(executionContext, dictionary, containerName); + } } From 60e668f59a8d00bf39d66ac6b8616efd09841db1 Mon Sep 17 00:00:00 2001 From: Ho Peng Foong Date: Thu, 13 Feb 2025 01:28:52 +0800 Subject: [PATCH 3/6] wip --- .../Connector/DataLakeClient.cs | 5 ++ .../Connector/DataLakeConnector.cs | 18 +++-- .../Connector/IDataLakeClient.cs | 1 + .../ConnectorProviderBase.cs | 5 ++ .../CrawlJobDataWrapper.cs | 3 +- .../DataLakeConstants.cs | 32 +++++---- .../DataLakeJobDataFactoryBase.cs | 3 +- .../Connector/FabricMirroringClient.cs | 2 +- .../Connector/FabricMirroringConnector.cs | 14 +++- ...Handler.cs => ExportTargetEventHandler.cs} | 33 +++++---- .../UpdateExportTargetEventHandler.cs | 67 ------------------- .../FabricMirroringConnectorComponent.cs | 10 +++ .../FabricMirroringConnectorJobData.cs | 16 ++--- .../FabricMirroringConnectorProvider.cs | 2 +- .../FabricMirroringConstants.cs | 9 +-- .../FabricMirroringJobDataFactory.cs | 34 +++++++--- 16 files changed, 126 insertions(+), 128 deletions(-) rename src/Connector.FabricMirroring/EventHandlers/{RegisterExportTargetEventHandler.cs => ExportTargetEventHandler.cs} (87%) delete mode 100644 src/Connector.FabricMirroring/EventHandlers/UpdateExportTargetEventHandler.cs diff --git a/src/Connector.DataLake.Common/Connector/DataLakeClient.cs b/src/Connector.DataLake.Common/Connector/DataLakeClient.cs index b21b0f7..fd36e83 100644 --- a/src/Connector.DataLake.Common/Connector/DataLakeClient.cs +++ b/src/Connector.DataLake.Common/Connector/DataLakeClient.cs @@ -91,6 +91,11 @@ public async Task FileInPathExists(IDataLakeJobData configuration, string return await dataLakeFileClient.ExistsAsync(); } + public Task DirectoryExists(IDataLakeJobData configuration) + { + return DirectoryExists(configuration, string.Empty); + } + public async Task DirectoryExists(IDataLakeJobData configuration, string subDirectory) { var fileSystemClient = await GetFileSystemClientAsync(configuration, ensureExists: false); diff --git a/src/Connector.DataLake.Common/Connector/DataLakeConnector.cs b/src/Connector.DataLake.Common/Connector/DataLakeConnector.cs index 448d8ee..4f9a79f 100644 --- a/src/Connector.DataLake.Common/Connector/DataLakeConnector.cs +++ b/src/Connector.DataLake.Common/Connector/DataLakeConnector.cs @@ -54,6 +54,8 @@ public abstract class DataLakeConnector : ConnectorBaseV2 protected IDataLakeJobDataFactory DataLakeJobDataFactory => _dataLakeJobDataFactory; + protected IDataLakeClient Client => _client; + protected DataLakeConnector( ILogger logger, IDataLakeClient client, @@ -432,13 +434,13 @@ private async Task WriteToOutputImmediately( if (connectorEntityData.ChangeType == VersionChangeType.Removed) { - await _client.DeleteFile(configurations, filePathAndName); + await Client.DeleteFile(configurations, filePathAndName); } else { var json = JsonConvert.SerializeObject(data, _immediateOutputSerializerSettings); - await _client.SaveData(configurations, json, filePathAndName, JsonMimeType); + await Client.SaveData(configurations, json, filePathAndName, JsonMimeType); } } @@ -479,7 +481,10 @@ public override async Task VerifyConnection(Execut try { var jobData = await _dataLakeJobDataFactory.GetConfiguration(executionContext, config.ToDictionary(config => config.Key, config => config.Value)); - await VerifyDataLakeConnection(jobData); + if (!await VerifyDataLakeConnection(jobData)) + { + return new ConnectionVerificationResult(false, "Data Lake connection cannot be established."); + } if (jobData.IsStreamCacheEnabled) { @@ -528,9 +533,10 @@ public override async Task VerifyConnection(Execut } } - protected virtual async Task VerifyDataLakeConnection(IDataLakeJobData jobData) + protected virtual async Task VerifyDataLakeConnection(IDataLakeJobData jobData) { - await _client.EnsureDataLakeDirectoryExist(jobData); + await Client.EnsureDataLakeDirectoryExist(jobData); + return true; } private async Task VerifyTableOperations(string connectionString) @@ -612,7 +618,7 @@ private void Flush(IDataLakeJobData configuration, string[] entityData) var timestamp = DateTime.UtcNow.ToString("yyyy-MM-dd HH-mm-ss.fffffff"); var fileName = $"{configuration.ContainerName}.{timestamp}.json"; - _client.SaveData(configuration, content, fileName, JsonMimeType).GetAwaiter().GetResult(); + Client.SaveData(configuration, content, fileName, JsonMimeType).GetAwaiter().GetResult(); } public override async Task CreateContainer(ExecutionContext executionContext, Guid connectorProviderDefinitionId, IReadOnlyCreateContainerModelV2 model) diff --git a/src/Connector.DataLake.Common/Connector/IDataLakeClient.cs b/src/Connector.DataLake.Common/Connector/IDataLakeClient.cs index a5f0a1d..8b413c7 100644 --- a/src/Connector.DataLake.Common/Connector/IDataLakeClient.cs +++ b/src/Connector.DataLake.Common/Connector/IDataLakeClient.cs @@ -15,5 +15,6 @@ public interface IDataLakeClient Task FileInPathExists(IDataLakeJobData configuration, string fileName, string subDirectory); Task GetFilePathProperties(IDataLakeJobData configuration, string fileName); Task GetFilePathProperties(IDataLakeJobData configuration, string fileName, string subDirectory); + Task DirectoryExists(IDataLakeJobData configuration); Task DirectoryExists(IDataLakeJobData configuration, string subDirectory); } diff --git a/src/Connector.DataLake.Common/ConnectorProviderBase.cs b/src/Connector.DataLake.Common/ConnectorProviderBase.cs index 24f0b2c..ad0daa6 100644 --- a/src/Connector.DataLake.Common/ConnectorProviderBase.cs +++ b/src/Connector.DataLake.Common/ConnectorProviderBase.cs @@ -55,6 +55,11 @@ public override Task GetCrawlJobData( { // WARNING: The log output can contain sensitive information _logger.LogDebug($"GetCrawlJobData config input: {JsonConvert.SerializeObject(configuration)}"); + + if (!configuration.TryGetValue("__ProviderDefinitionId__", out var _)) + { + configuration.Add("__ProviderDefinitionId__", providerDefinitionId); + } return Task.FromResult(new CrawlJobDataWrapper(configuration)); } diff --git a/src/Connector.DataLake.Common/CrawlJobDataWrapper.cs b/src/Connector.DataLake.Common/CrawlJobDataWrapper.cs index 9e7dbf0..200b920 100644 --- a/src/Connector.DataLake.Common/CrawlJobDataWrapper.cs +++ b/src/Connector.DataLake.Common/CrawlJobDataWrapper.cs @@ -1,4 +1,5 @@ -using System.Collections.Generic; +using System; +using System.Collections.Generic; using CluedIn.Core.Crawling; diff --git a/src/Connector.DataLake.Common/DataLakeConstants.cs b/src/Connector.DataLake.Common/DataLakeConstants.cs index 600e65f..354d0b2 100644 --- a/src/Connector.DataLake.Common/DataLakeConstants.cs +++ b/src/Connector.DataLake.Common/DataLakeConstants.cs @@ -89,7 +89,8 @@ protected DataLakeConstants( protected abstract string CacheKeyword { get; } - protected static IEnumerable GetAuthMethods(ApplicationContext applicationContext) + protected static IEnumerable GetAuthMethods(ApplicationContext applicationContext, + bool isCustomFileNamePatternSupported = true) { string connectionString = null; if (applicationContext.System.ConnectionStrings.ConnectionStringExists(StreamCacheConnectionStringKey)) @@ -191,29 +192,32 @@ protected static IEnumerable GetAuthMethods(ApplicationContext applicat }, }, }); - controls.Add( - new() - { - Name = FileNamePattern, - DisplayName = "File Name Pattern", - Type = "input", - Help = """ + + if (isCustomFileNamePatternSupported) + { + controls.Add( + new() + { + Name = FileNamePattern, + DisplayName = "File Name Pattern", + Type = "input", + Help = """ Specify a file name pattern for the export file, e.g. {StreamId}_{DataTime}.{OutputFormat}. Available variables are {StreamId}, {DataTime}, {OutputFormat} and {ContainerName}. Variables can also be formatted using formatString modifier. For more information, please refer to the documentation. """, - IsRequired = false, - DisplayDependencies = new[] - { + IsRequired = false, + DisplayDependencies = new[] + { new ControlDisplayDependency { Name = IsStreamCacheEnabled, Operator = ControlDependencyOperator.Exists, UnfulfilledAction = ControlDependencyUnfulfilledAction.Hidden, }, - }, - }); - + }, + }); + } return controls; } } diff --git a/src/Connector.DataLake.Common/DataLakeJobDataFactoryBase.cs b/src/Connector.DataLake.Common/DataLakeJobDataFactoryBase.cs index 9d071e0..ac4ec83 100644 --- a/src/Connector.DataLake.Common/DataLakeJobDataFactoryBase.cs +++ b/src/Connector.DataLake.Common/DataLakeJobDataFactoryBase.cs @@ -35,7 +35,8 @@ protected static void UpdateStreamCacheConnectionString(ExecutionContext executi public virtual async Task GetConfiguration(ExecutionContext executionContext, Guid providerDefinitionId, string containerName) { var authenticationDetails = await GetAuthenticationDetails(executionContext, providerDefinitionId); - return await GetConfiguration(executionContext, authenticationDetails.Authentication.ToDictionary(detail => detail.Key, detail => detail.Value), containerName); + var values = authenticationDetails.Authentication.ToDictionary(detail => detail.Key, detail => detail.Value); + return await GetConfiguration(executionContext, values, containerName); } public virtual async Task GetConfiguration(ExecutionContext executionContext, IDictionary authenticationDetails, string containerName = null) diff --git a/src/Connector.FabricMirroring/Connector/FabricMirroringClient.cs b/src/Connector.FabricMirroring/Connector/FabricMirroringClient.cs index d138e2d..eabf2f6 100644 --- a/src/Connector.FabricMirroring/Connector/FabricMirroringClient.cs +++ b/src/Connector.FabricMirroring/Connector/FabricMirroringClient.cs @@ -28,7 +28,7 @@ protected override DataLakeServiceClient GetDataLakeServiceClient(IDataLakeJobDa protected override string GetDirectory(IDataLakeJobData configuration) { var casted = CastJobData(configuration); - return $"{casted.ItemName}.MountedRelationalDatabase/Files/LandingZone"; + return $"{casted.MirroredDatabaseName}.MountedRelationalDatabase/Files/LandingZone"; } protected override string GetFileSystemName(IDataLakeJobData configuration) diff --git a/src/Connector.FabricMirroring/Connector/FabricMirroringConnector.cs b/src/Connector.FabricMirroring/Connector/FabricMirroringConnector.cs index 7e5cee5..6d27000 100644 --- a/src/Connector.FabricMirroring/Connector/FabricMirroringConnector.cs +++ b/src/Connector.FabricMirroring/Connector/FabricMirroringConnector.cs @@ -1,4 +1,5 @@ -using System.Threading.Tasks; +using System; +using System.Threading.Tasks; using CluedIn.Connector.DataLake.Common; using CluedIn.Connector.DataLake.Common.Connector; @@ -21,12 +22,19 @@ public FabricMirroringConnector( { } - protected override Task VerifyDataLakeConnection(IDataLakeJobData jobData) + protected override async Task VerifyDataLakeConnection(IDataLakeJobData jobData) { //TODO: verify path exists only if mirroring created // find out a way to see if it's called by test connection or healthcheck (might have to do ugly reflection to look at stack) // probably could do it by verifying if jobdata from db = jobdata created from passed config // but we need to be able to get provider definition id, which is not passed (wtf!) - return base.VerifyDataLakeConnection(jobData); + try + { + return await Client.DirectoryExists(jobData); + } + catch (Exception ex) + { + return false; + } } } diff --git a/src/Connector.FabricMirroring/EventHandlers/RegisterExportTargetEventHandler.cs b/src/Connector.FabricMirroring/EventHandlers/ExportTargetEventHandler.cs similarity index 87% rename from src/Connector.FabricMirroring/EventHandlers/RegisterExportTargetEventHandler.cs rename to src/Connector.FabricMirroring/EventHandlers/ExportTargetEventHandler.cs index 325b900..62ebd4a 100644 --- a/src/Connector.FabricMirroring/EventHandlers/RegisterExportTargetEventHandler.cs +++ b/src/Connector.FabricMirroring/EventHandlers/ExportTargetEventHandler.cs @@ -32,8 +32,9 @@ protected UpdateMirroredDatabaseBase( IDataLakeJobDataFactory jobDataFactory, IDateTimeOffsetProvider dateTimeOffsetProvider) { - _constants = constants; - _jobDataFactory = jobDataFactory; + _applicationContext = applicationContext ?? throw new ArgumentNullException(nameof(applicationContext)); + _constants = constants ?? throw new ArgumentNullException(nameof(constants)); + _jobDataFactory = jobDataFactory ?? throw new ArgumentNullException(nameof(jobDataFactory)); _dateTimeOffsetProvider = dateTimeOffsetProvider ?? throw new ArgumentNullException(nameof(dateTimeOffsetProvider)); } @@ -64,13 +65,13 @@ protected virtual async Task UpdateOrCreateMirroredDatabaseAsy throw new ApplicationException($"Failed to find workspace using {jobData.WorkspaceName}."); } - var mirrorDatabaseName = jobData.ItemName; - var mirroredDatabase = await GetMirroredDatabaseAsync(workspace.Id, jobData.ItemName); + var mirrorDatabaseName = jobData.MirroredDatabaseName; + var mirroredDatabase = await GetMirroredDatabaseAsync(workspace.Id, jobData.MirroredDatabaseName); if (mirroredDatabase == null) { if (!jobData.ShouldCreateMirroredDatabase) { - throw new ApplicationException($"Mirrored database is not found using workspace {jobData.WorkspaceName} and mirrored database name {jobData.ItemName}."); + throw new ApplicationException($"Mirrored database is not found using workspace {jobData.WorkspaceName} and mirrored database name {jobData.MirroredDatabaseName}."); } var result = await FabricClient.MirroredDatabase.Items.CreateMirroredDatabaseAsync( workspace.Id, @@ -110,12 +111,12 @@ protected virtual async Task UpdateOrCreateMirroredDatabaseAsy var status = await PollForCompletion(workspace.Id, result.Value.Id.Value); if (status != SqlEndpointProvisioningStatus.Success) { - throw new ApplicationException($"Failed to provision sql endpoint using workspace {jobData.WorkspaceName} and mirrored database name {jobData.ItemName}."); + throw new ApplicationException($"Failed to provision sql endpoint using workspace {jobData.WorkspaceName} and mirrored database name {jobData.MirroredDatabaseName}."); } } - mirroredDatabase = await GetMirroredDatabaseAsync(workspace.Id, jobData.ItemName); + mirroredDatabase = await GetMirroredDatabaseAsync(workspace.Id, jobData.MirroredDatabaseName); var store = executionContext.Organization.DataStores.GetDataStore(); var definition = await store.GetByIdAsync(executionContext, providerDefinitionId); @@ -205,12 +206,13 @@ protected async Task StartMirroringAsync(Guid workspaceId, Guid mirroredDatabase return null; } } -internal class RegisterExportTargetEventHandler : UpdateMirroredDatabaseBase, IDisposable +internal class ExportTargetEventHandler : UpdateMirroredDatabaseBase, IDisposable { - private readonly IDisposable _subscription; + private readonly IDisposable _registerExportTargetSubscription; + private readonly IDisposable _updateExportTargetSubscription; private bool _disposedValue; - public RegisterExportTargetEventHandler( + public ExportTargetEventHandler( ApplicationContext applicationContext, FabricMirroringClient client, IDataLakeConstants constants, @@ -218,15 +220,18 @@ public RegisterExportTargetEventHandler( IDateTimeOffsetProvider dateTimeOffsetProvider) : base(applicationContext, constants, jobDataFactory, dateTimeOffsetProvider) { - _subscription = applicationContext.System.Events.Local.Subscribe(ProcessEvent); + _registerExportTargetSubscription = applicationContext.System.Events.Local.Subscribe(ProcessEvent); + _updateExportTargetSubscription = applicationContext.System.Events.Local.Subscribe(ProcessEvent); } - private void ProcessEvent(RegisterExportTargetEvent eventData) + private void ProcessEvent(TEvent eventData) + where TEvent : RemoteEvent { ProcessEventAsync(eventData).GetAwaiter().GetResult(); } - private async Task ProcessEventAsync(RegisterExportTargetEvent eventData) + private async Task ProcessEventAsync(TEvent eventData) + where TEvent : RemoteEvent { await UpdateOrCreateMirroredDatabaseAsync(eventData); } @@ -237,7 +242,7 @@ protected virtual void Dispose(bool disposing) { if (disposing) { - _subscription.Dispose(); + _registerExportTargetSubscription.Dispose(); } _disposedValue = true; diff --git a/src/Connector.FabricMirroring/EventHandlers/UpdateExportTargetEventHandler.cs b/src/Connector.FabricMirroring/EventHandlers/UpdateExportTargetEventHandler.cs deleted file mode 100644 index 209384b..0000000 --- a/src/Connector.FabricMirroring/EventHandlers/UpdateExportTargetEventHandler.cs +++ /dev/null @@ -1,67 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Threading.Tasks; - -using Azure.Identity; - -using CluedIn.Connector.DataLake.Common; -using CluedIn.Connector.FabricMirroring.Connector; -using CluedIn.Core; -using CluedIn.Core.Data.Relational; -using CluedIn.Core.Events; -using CluedIn.Core.Events.Types; - -using Microsoft.Fabric.Api; -using Microsoft.Fabric.Api.Core.Models; -using Microsoft.Fabric.Api.MirroredDatabase.Models; - -using Newtonsoft.Json.Linq; - -namespace CluedIn.Connector.FabricMirroring.EventHandlers; - -internal class UpdateExportTargetEventHandler : UpdateMirroredDatabaseBase, IDisposable -{ - private readonly IDisposable _subscription; - private bool _disposedValue; - - public UpdateExportTargetEventHandler( - ApplicationContext applicationContext, - FabricMirroringClient client, - IDataLakeConstants constants, - IDataLakeJobDataFactory jobDataFactory, - IDateTimeOffsetProvider dateTimeOffsetProvider) - : base(applicationContext, constants, jobDataFactory, dateTimeOffsetProvider) - { - _subscription = applicationContext.System.Events.Local.Subscribe(ProcessEvent); - } - - private void ProcessEvent(UpdateExportTargetEvent eventData) - { - ProcessEventAsync(eventData).GetAwaiter().GetResult(); - } - - private async Task ProcessEventAsync(UpdateExportTargetEvent eventData) - { - await UpdateOrCreateMirroredDatabaseAsync(eventData); - } - - protected virtual void Dispose(bool disposing) - { - if (!_disposedValue) - { - if (disposing) - { - _subscription.Dispose(); - } - - _disposedValue = true; - } - } - - public void Dispose() - { - // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method - Dispose(disposing: true); - GC.SuppressFinalize(this); - } -} diff --git a/src/Connector.FabricMirroring/FabricMirroringConnectorComponent.cs b/src/Connector.FabricMirroring/FabricMirroringConnectorComponent.cs index bafb06c..6e1295c 100644 --- a/src/Connector.FabricMirroring/FabricMirroringConnectorComponent.cs +++ b/src/Connector.FabricMirroring/FabricMirroringConnectorComponent.cs @@ -1,5 +1,6 @@ using CluedIn.Connector.DataLake.Common; using CluedIn.Connector.FabricMirroring.Connector; +using CluedIn.Connector.FabricMirroring.EventHandlers; using CluedIn.Core; using ComponentHost; @@ -11,6 +12,7 @@ namespace CluedIn.Connector.FabricMirroring; Components.Server, Components.DataStores, Isolation = ComponentIsolation.NotIsolated)] public sealed class FabricMirroringConnectorComponent : DataLakeConnectorComponentBase { + private ExportTargetEventHandler _exportTargetEventHandler; public FabricMirroringConnectorComponent(ComponentInfo componentInfo) : base(componentInfo) { Container.Install(new InstallComponents()); @@ -27,4 +29,12 @@ public override void Start() protected override string ConnectorComponentName => ComponentName; protected override string ShortConnectorComponentName => ComponentName; + + private protected override void SubscribeToEvents(IDataLakeConstants constants, IDataLakeJobDataFactory jobDataFactory, IScheduledJobQueue jobQueue) + { + var dateTimeProvider = Container.Resolve(); + var fabricClient = Container.Resolve(); + _exportTargetEventHandler = new(ApplicationContext, fabricClient, constants, jobDataFactory, dateTimeProvider); + base.SubscribeToEvents(constants, jobDataFactory, jobQueue); + } } diff --git a/src/Connector.FabricMirroring/FabricMirroringConnectorJobData.cs b/src/Connector.FabricMirroring/FabricMirroringConnectorJobData.cs index 50e6b8d..4a68e1c 100644 --- a/src/Connector.FabricMirroring/FabricMirroringConnectorJobData.cs +++ b/src/Connector.FabricMirroring/FabricMirroringConnectorJobData.cs @@ -14,21 +14,21 @@ public FabricMirroringConnectorJobData( { } - public string WorkspaceName => Configurations[FabricMirroringConstants.WorkspaceName] as string; - public string ItemName => Configurations[FabricMirroringConstants.ItemName] as string; - public string ClientId => Configurations[FabricMirroringConstants.ClientId] as string; - public string ClientSecret => Configurations[FabricMirroringConstants.ClientSecret] as string; - public string TenantId => Configurations[FabricMirroringConstants.TenantId] as string; + public string WorkspaceName => GetConfigurationValue(FabricMirroringConstants.WorkspaceName) as string; + public string MirroredDatabaseName => GetConfigurationValue(FabricMirroringConstants.MirroredDatabaseName) as string ?? string.Empty; + public string ClientId => GetConfigurationValue(FabricMirroringConstants.ClientId) as string; + public string ClientSecret => GetConfigurationValue(FabricMirroringConstants.ClientSecret) as string; + public string TenantId => GetConfigurationValue(FabricMirroringConstants.TenantId) as string; public override bool ShouldWriteGuidAsString => true; public override bool ShouldEscapeVocabularyKeys => true; public override bool IsDeltaMode => true; public override bool IsOverwriteEnabled => false; - public virtual bool ShouldCreateMirroredDatabase => false; + public virtual bool ShouldCreateMirroredDatabase => GetConfigurationValue(FabricMirroringConstants.ShouldCreateMirroredDatabase) as bool? ?? false; protected override void AddToHashCode(HashCode hash) { hash.Add(WorkspaceName); - hash.Add(ItemName); + hash.Add(MirroredDatabaseName); hash.Add(ClientId); hash.Add(ClientSecret); hash.Add(TenantId); @@ -46,7 +46,7 @@ public bool Equals(FabricMirroringConnectorJobData other) { return other != null && WorkspaceName == other.WorkspaceName && - ItemName == other.ItemName && + MirroredDatabaseName == other.MirroredDatabaseName && ClientId == other.ClientId && ClientSecret == other.ClientSecret && TenantId == other.TenantId && diff --git a/src/Connector.FabricMirroring/FabricMirroringConnectorProvider.cs b/src/Connector.FabricMirroring/FabricMirroringConnectorProvider.cs index f97cf0d..c09c56c 100644 --- a/src/Connector.FabricMirroring/FabricMirroringConnectorProvider.cs +++ b/src/Connector.FabricMirroring/FabricMirroringConnectorProvider.cs @@ -16,7 +16,7 @@ public FabricMirroringConnectorProvider([NotNull] ApplicationContext appContext, protected override IEnumerable ProviderNameParts => new[] { FabricMirroringConstants.WorkspaceName, - FabricMirroringConstants.ItemName, + FabricMirroringConstants.MirroredDatabaseName, FabricMirroringConstants.ClientId, FabricMirroringConstants.TenantId, }; diff --git a/src/Connector.FabricMirroring/FabricMirroringConstants.cs b/src/Connector.FabricMirroring/FabricMirroringConstants.cs index 8fcced4..1b5dc25 100644 --- a/src/Connector.FabricMirroring/FabricMirroringConstants.cs +++ b/src/Connector.FabricMirroring/FabricMirroringConstants.cs @@ -12,7 +12,7 @@ public class FabricMirroringConstants : DataLakeConstants, IFabricMirroringConst internal static readonly Guid DataLakeProviderId = Guid.Parse("75CD5880-0537-4C6D-AE14-511C273ACD68"); public const string WorkspaceName = nameof(WorkspaceName); - public const string ItemName = nameof(ItemName); + public const string MirroredDatabaseName = nameof(MirroredDatabaseName); public const string ClientId = nameof(ClientId); public const string ClientSecret = nameof(ClientSecret); public const string TenantId = nameof(TenantId); @@ -45,10 +45,11 @@ private static AuthMethods GetFabricMirroringAuthMethods(ApplicationContext appl }, new () { - Name = ItemName, - DisplayName = ItemName, + Name = MirroredDatabaseName, + DisplayName = MirroredDatabaseName, Type = "input", IsRequired = false, + Help = "When left blank, one will be auto created" }, new () { @@ -73,7 +74,7 @@ private static AuthMethods GetFabricMirroringAuthMethods(ApplicationContext appl }, }; - controls.AddRange(GetAuthMethods(applicationContext)); + controls.AddRange(GetAuthMethods(applicationContext, isCustomFileNamePatternSupported: false)); return new AuthMethods { diff --git a/src/Connector.FabricMirroring/FabricMirroringJobDataFactory.cs b/src/Connector.FabricMirroring/FabricMirroringJobDataFactory.cs index 491b58b..8a365fe 100644 --- a/src/Connector.FabricMirroring/FabricMirroringJobDataFactory.cs +++ b/src/Connector.FabricMirroring/FabricMirroringJobDataFactory.cs @@ -5,6 +5,7 @@ using CluedIn.Connector.DataLake.Common; using CluedIn.Core; +using CluedIn.Core.Data.Relational; namespace CluedIn.Connector.FabricMirroring; @@ -18,18 +19,35 @@ protected override Task CreateJobData( return Task.FromResult(new FabricMirroringConnectorJobData(authenticationDetails, containerName)); } - public override async Task GetConfiguration(ExecutionContext executionContext, Guid providerDefinitionId, string containerName) - { - var authenticationDetails = await GetAuthenticationDetails(executionContext, providerDefinitionId); - var dictionary = authenticationDetails.Authentication.ToDictionary(detail => detail.Key, detail => detail.Value); + //public override async Task GetConfiguration(ExecutionContext executionContext, Guid providerDefinitionId, string containerName) + //{ + // var baseResult = await base.GetConfiguration(executionContext, providerDefinitionId, containerName); + //} - if (!dictionary.TryGetValue(FabricMirroringConstants.ItemName, out var value) + public override async Task GetConfiguration(ExecutionContext executionContext, IDictionary authenticationDetails, string containerName = null) + { + if (!authenticationDetails.TryGetValue(FabricMirroringConstants.MirroredDatabaseName, out var value) || string.IsNullOrWhiteSpace(value?.ToString())) { - dictionary[FabricMirroringConstants.ItemName] = $"CluedIn_ExportTarget_{providerDefinitionId:N}"; - dictionary[FabricMirroringConstants.ShouldCreateMirroredDatabase] = true; + if (!authenticationDetails.TryGetValue("__ProviderDefinitionId__", out var providerDefinitionId)) + { + throw new ApplicationException("Failed to get provider definition id from authentication details."); + } + + authenticationDetails[FabricMirroringConstants.MirroredDatabaseName] = $"CluedIn_ExportTarget_{providerDefinitionId:N}"; + authenticationDetails[FabricMirroringConstants.ShouldCreateMirroredDatabase] = true; } - return await GetConfiguration(executionContext, dictionary, containerName); + return await base.GetConfiguration(executionContext, authenticationDetails, containerName); } + + //private static void UpdateMirroredDatabaseName(IDictionary authenticationDetails, Guid providerDefinitionId, bool shouldCreate) + //{ + // if (!authenticationDetails.TryGetValue(FabricMirroringConstants.MirroredDatabaseName, out var value) + // || string.IsNullOrWhiteSpace(value?.ToString())) + // { + // authenticationDetails[FabricMirroringConstants.MirroredDatabaseName] = $"CluedIn_ExportTarget_{providerDefinitionId:N}"; + // authenticationDetails[FabricMirroringConstants.ShouldCreateMirroredDatabase] = true; + // } + //} } From a4e60dea929b62dfc9c46c058d1f25f087d99dc5 Mon Sep 17 00:00:00 2001 From: Ho Peng Foong Date: Thu, 13 Feb 2025 12:07:35 +0800 Subject: [PATCH 4/6] update description and logo --- .../FabricMirroringConstants.cs | 6 +- .../Resources/fabricMirroring.svg | 8470 ++++++++++++++++- 2 files changed, 8434 insertions(+), 42 deletions(-) diff --git a/src/Connector.FabricMirroring/FabricMirroringConstants.cs b/src/Connector.FabricMirroring/FabricMirroringConstants.cs index 1b5dc25..4c7e589 100644 --- a/src/Connector.FabricMirroring/FabricMirroringConstants.cs +++ b/src/Connector.FabricMirroring/FabricMirroringConstants.cs @@ -19,13 +19,13 @@ public class FabricMirroringConstants : DataLakeConstants, IFabricMirroringConst public const string ShouldCreateMirroredDatabase = nameof(ShouldCreateMirroredDatabase); public FabricMirroringConstants(ApplicationContext applicationContext) : base(DataLakeProviderId, - providerName: "FabricMirroring Connector", + providerName: "Fabric Mirroring Connector", componentName: "FabricMirroringConnector", icon: "Resources.fabricMirroring.svg", domain: "https://azure.microsoft.com/en-us/services/data-lake-analytics/", - about: "Supports publishing of data to FabricMirroring.", + about: "Supports publishing of data to Microsoft Fabric Mirrored Database.", authMethods: GetFabricMirroringAuthMethods(applicationContext), - guideDetails: "Supports publishing of data to FabricMirroring.", + guideDetails: "Supports publishing of data to Microsoft Fabric Mirrored Database.", guideInstructions: "Provide authentication instructions here, if applicable") // TODO: ROK: { } diff --git a/src/Connector.FabricMirroring/Resources/fabricMirroring.svg b/src/Connector.FabricMirroring/Resources/fabricMirroring.svg index 5c92217..9c48622 100644 --- a/src/Connector.FabricMirroring/Resources/fabricMirroring.svg +++ b/src/Connector.FabricMirroring/Resources/fabricMirroring.svg @@ -1,42 +1,8434 @@ - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + Icon-databases-136 + + + + + + + + + + + + + + + + + + + + + + + Icon-databases-136 + + + + + + + + + + + + + + + + + + + + + + + Icon-databases-136 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 87cb04367f3f7f86944009753b27f989d3947285 Mon Sep 17 00:00:00 2001 From: Ho Peng Foong Date: Thu, 13 Feb 2025 15:27:58 +0800 Subject: [PATCH 5/6] update logo --- .../Resources/fabricMirroring.svg | 8485 +---------------- 1 file changed, 51 insertions(+), 8434 deletions(-) diff --git a/src/Connector.FabricMirroring/Resources/fabricMirroring.svg b/src/Connector.FabricMirroring/Resources/fabricMirroring.svg index 9c48622..296b77c 100644 --- a/src/Connector.FabricMirroring/Resources/fabricMirroring.svg +++ b/src/Connector.FabricMirroring/Resources/fabricMirroring.svg @@ -1,8434 +1,51 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Icon-databases-136 - - - - - - - - - - - - - - - - - - - - - - - Icon-databases-136 - - - - - - - - - - - - - - - - - - - - - - - Icon-databases-136 - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file From 84ce1368b0e550a8a1570d043106317727bd6514 Mon Sep 17 00:00:00 2001 From: Ho Peng Foong Date: Thu, 13 Feb 2025 15:28:05 +0800 Subject: [PATCH 6/6] add index --- .../Connector/DataLakeConnector.cs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/Connector.DataLake.Common/Connector/DataLakeConnector.cs b/src/Connector.DataLake.Common/Connector/DataLakeConnector.cs index 4f9a79f..70a2bee 100644 --- a/src/Connector.DataLake.Common/Connector/DataLakeConnector.cs +++ b/src/Connector.DataLake.Common/Connector/DataLakeConnector.cs @@ -198,14 +198,17 @@ private async Task EnsureCacheTableExists( }); var createTableSql = $""" IF NOT EXISTS (SELECT * FROM SYSOBJECTS WHERE NAME='{tableName}' AND XTYPE='U') - CREATE TABLE [{tableName}] ( - {DataLakeConstants.IdKey} UNIQUEIDENTIFIER NOT NULL, - {string.Join(string.Empty, propertiesColumns.Select(prop => $"{prop},\n "))} - [ValidFrom] DATETIME2 GENERATED ALWAYS AS ROW START HIDDEN, - [ValidTo] DATETIME2 GENERATED ALWAYS AS ROW END HIDDEN, - PERIOD FOR SYSTEM_TIME(ValidFrom, ValidTo), - CONSTRAINT [PK_{tableName}] PRIMARY KEY CLUSTERED ({DataLakeConstants.IdKey}) - ) WITH (SYSTEM_VERSIONING = ON (HISTORY_TABLE = dbo.[{tableName}_History])); + BEGIN + CREATE TABLE [{tableName}] ( + {DataLakeConstants.IdKey} UNIQUEIDENTIFIER NOT NULL, + {string.Join(string.Empty, propertiesColumns.Select(prop => $"{prop},\n "))} + [ValidFrom] DATETIME2 GENERATED ALWAYS AS ROW START HIDDEN, + [ValidTo] DATETIME2 GENERATED ALWAYS AS ROW END HIDDEN, + PERIOD FOR SYSTEM_TIME(ValidFrom, ValidTo), + CONSTRAINT [PK_{tableName}] PRIMARY KEY CLUSTERED ({DataLakeConstants.IdKey}) + ) WITH (SYSTEM_VERSIONING = ON (HISTORY_TABLE = dbo.[{tableName}_History])); + CREATE INDEX [ValidFromValidTo] ON [{tableName}] ([ValidFrom], [ValidTo]); + END """; var command = new SqlCommand(createTableSql, connection) {