diff --git a/FileProcessor.cs b/FileProcessor.cs index 4af3364..d986374 100644 --- a/FileProcessor.cs +++ b/FileProcessor.cs @@ -1,10 +1,5 @@ -using System.Security.Cryptography; -using System.Text; using Org.BouncyCastle.Bcpg; -using Org.BouncyCastle.Bcpg.OpenPgp; -using Org.BouncyCastle.Crypto; -using Org.BouncyCastle.Crypto.Parameters; -using Org.BouncyCastle.Security; +using PgpCore; using SharpCompress.Common; using SharpCompress.Writers; @@ -19,178 +14,36 @@ public void CreateTarGz(string outputPath, string inputDirectory) writer.WriteAll(inputDirectory, searchPattern: "*", SearchOption.AllDirectories); } - public static void DecryptFileSymetric(string inputFile, string outputFile, string password) + public async Task EncryptFilePgp(string inputFile, string publicKeyFile, string outputFile) { - byte[] passwordBytes = System.Text.Encoding.UTF8.GetBytes(password); - byte[] salt = new byte[32]; - - FileStream fsCrypt = new FileStream(inputFile, FileMode.Open); - fsCrypt.Read(salt, 0, salt.Length); - - Rfc2898DeriveBytes derivedBytes = new Rfc2898DeriveBytes(passwordBytes, salt, 50000, HashAlgorithmName.SHA256); - Aes aes = Aes.Create(); - aes.KeySize = 256; - aes.BlockSize = 128; - aes.Key = derivedBytes.GetBytes(aes.KeySize / 8); - aes.IV = derivedBytes.GetBytes(aes.BlockSize / 8); - - CryptoStream cs = new CryptoStream(fsCrypt, aes.CreateDecryptor(), CryptoStreamMode.Read); - FileStream fsOut = new FileStream(outputFile, FileMode.Create); - - int read; - byte[] buffer = new byte[1048576]; - - try - { - while ((read = cs.Read(buffer, 0, buffer.Length)) > 0) - { - fsOut.Write(buffer, 0, read); - } - } - catch (Exception ex) - { - Console.WriteLine("Error: " + ex.Message); - } - - try - { - cs.Close(); - } - catch (Exception ex) - { - Console.WriteLine("Error by closing CryptoStream: " + ex.Message); - } - finally - { - fsOut.Close(); - fsCrypt.Close(); - } + // Load key + FileInfo publicKey = new FileInfo(publicKeyFile); + EncryptionKeys encryptionKeys = new EncryptionKeys(publicKey); + + // Get file infos + FileInfo inputFileInfo = new FileInfo(inputFile); + FileInfo encryptedSignedFile = new FileInfo(outputFile); + // Encrypt and Sign + PGP pgp = new PGP(encryptionKeys); + pgp.SymmetricKeyAlgorithm = SymmetricKeyAlgorithmTag.Aes256; + await pgp.EncryptFileAsync(inputFileInfo, encryptedSignedFile); } - - public void EncryptFileSymetric(string inputFile, string outputFile, string pgpKeyFile) + public async Task DecryptFilePgp(string inputFilePath, string outputFilePath, string privateKeyPath, string password) { - byte[] salt = RandomNumberGenerator.GetBytes(32); - string randomPassword = RandomNumberGenerator.GetHexString(128); + // Load keys + EncryptionKeys encryptionKeys; + await using (Stream privateKeyStream = new FileStream(privateKeyPath, FileMode.Open)) + encryptionKeys = new EncryptionKeys(privateKeyStream, password); - // Encrypt password with Pgp - Console.WriteLine($"Password used: {randomPassword}"); - - EncryptStringPgp(randomPassword, pgpKeyFile, $"{outputFile}.key"); - - FileStream fsCrypt = new FileStream(outputFile, FileMode.Create); - - byte[] passwordBytes = System.Text.Encoding.UTF8.GetBytes(randomPassword); - Rfc2898DeriveBytes derivedBytes = new Rfc2898DeriveBytes(passwordBytes, salt, 50000, HashAlgorithmName.SHA256); - Aes aes = Aes.Create(); - aes.KeySize = 256; - aes.BlockSize = 128; - aes.Key = derivedBytes.GetBytes(aes.KeySize / 8); - aes.IV = derivedBytes.GetBytes(aes.BlockSize / 8); - - fsCrypt.Write(salt, 0, salt.Length); + // Get file infos + FileInfo inputFileInfo = new FileInfo(inputFilePath); + FileInfo outputFileInfo = new FileInfo(outputFilePath); - CryptoStream cs = new CryptoStream(fsCrypt, aes.CreateEncryptor(), CryptoStreamMode.Write); - FileStream fsIn = new FileStream(inputFile, FileMode.Open); + PGP pgp = new PGP(encryptionKeys); + pgp.SymmetricKeyAlgorithm = SymmetricKeyAlgorithmTag.Aes256; - byte[] buffer = new byte[1048576]; - - Console.Write("Encrypting file."); - try - { - int read; - while ((read = fsIn.Read(buffer, 0, buffer.Length)) > 0) - { - cs.Write(buffer, 0, read); - Console.Write("."); - } - - fsIn.Close(); - } - catch (Exception ex) - { - Console.WriteLine("Error: " + ex.Message); - } - finally - { - cs.Close(); - fsCrypt.Close(); - } - - Console.WriteLine("done"); + // Reference input/output files + await pgp.DecryptFileAsync(inputFileInfo, outputFileInfo); } - - - public void EncryptFilePGP(string inputFile, string publicKeyFile, string outputFile, bool armor, bool withIntegrityCheck) - { - using Stream publicKeyStream = File.OpenRead(publicKeyFile); - PgpPublicKey encKey = ReadPublicKey(publicKeyStream); - - using MemoryStream bOut = new MemoryStream(); - PgpCompressedDataGenerator comData = new PgpCompressedDataGenerator(CompressionAlgorithmTag.Zip); - - PgpUtilities.WriteFileToLiteralData( - comData.Open(bOut), - PgpLiteralData.Binary, - new FileInfo(inputFile)); - - comData.Close(); - - PgpEncryptedDataGenerator cPk = new PgpEncryptedDataGenerator(SymmetricKeyAlgorithmTag.Cast5, withIntegrityCheck, new SecureRandom()); - - cPk.AddMethod(encKey); - - byte[] bytes = bOut.ToArray(); - - using Stream outputStream = File.Create(outputFile); - if (armor) - { - using Stream armoredStream = new ArmoredOutputStream(outputStream); - WriteBytesToStream(cPk.Open(armoredStream, bytes.Length), bytes); - } - else - { - WriteBytesToStream(cPk.Open(outputStream, bytes.Length), bytes); - } - } - - public void EncryptStringPgp(string inputString, string publicKeyFile, string outputFile) - { - using Stream publicKeyStream = File.OpenRead(publicKeyFile); - PgpPublicKey encKey = ReadPublicKey(publicKeyStream); - - PgpEncryptedDataGenerator cPk = new PgpEncryptedDataGenerator(SymmetricKeyAlgorithmTag.Cast5, true, new SecureRandom()); - cPk.AddMethod(encKey); - - byte[] bytes = Encoding.UTF8.GetBytes(inputString); - - using Stream outputStream = File.Create(outputFile); - using Stream armoredStream = new ArmoredOutputStream(outputStream); - WriteBytesToStream(cPk.Open(armoredStream, bytes.Length), bytes); - } - private static void WriteBytesToStream(Stream outputStream, byte[] bytes) - { - using Stream encryptedOut = outputStream; - encryptedOut.Write(bytes, 0, bytes.Length); - } - - private static PgpPublicKey ReadPublicKey(Stream inputStream) - { - using Stream keyIn = inputStream; - PgpPublicKeyRingBundle pgpPub = new PgpPublicKeyRingBundle(PgpUtilities.GetDecoderStream(keyIn)); - - foreach (PgpPublicKeyRing keyRing in pgpPub.GetKeyRings()) - { - foreach (PgpPublicKey key in keyRing.GetPublicKeys()) - { - if (key.IsEncryptionKey) - { - return key; - } - } - } - - throw new ArgumentException("Can't find encryption key in key ring."); - } - } \ No newline at end of file diff --git a/MailGrabber.cs b/MailGrabber.cs index c8ba03f..e8cad68 100644 --- a/MailGrabber.cs +++ b/MailGrabber.cs @@ -17,43 +17,50 @@ public MailGrabber(string host, string user, string password) _password = password; } - internal List FindUrl(string sender, string regexPattern) + internal List> FindUrl(string sender, string regexPatternDownload, string regexPatternWorkspace) { - List urls = new List(); - using (var client = new ImapClient()) - { - client.ServerCertificateValidationCallback = (s, c, h, e) => true; - client.Connect(_host, 993, true); - client.Authenticate(_user, _password); + using var client = new ImapClient(); + client.ServerCertificateValidationCallback = (s, c, h, e) => true; + client.Connect(_host, 993, true); + client.Authenticate(_user, _password); + List> urls = new List>(); - var inbox = client.Inbox; - inbox.Open(MailKit.FolderAccess.ReadOnly); + var inbox = client.Inbox; + inbox.Open(MailKit.FolderAccess.ReadOnly); - for (int i = 0; i < inbox.Count; i++) - { - MimeMessage? message = inbox.GetMessage(i); + for (int i = 0; i < inbox.Count; i++) + { + MimeMessage? message = inbox.GetMessage(i); - if((message.From[0] as MailboxAddress)?.Address != sender) - continue; + if((message.From[0] as MailboxAddress)?.Address != sender) + continue; - string body = message.HtmlBody; - - // Regex to extract URLs - var regex = new Regex(regexPattern, RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.Multiline ); + string body = message.HtmlBody; - // Find matches - MatchCollection matches = regex.Matches(body); + // Regex to extract URLs + var regexDl = new Regex(regexPatternDownload, RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.Multiline ); + var regexWorkspace = new Regex(regexPatternWorkspace, RegexOptions.Compiled | RegexOptions.IgnoreCase | RegexOptions.Multiline ); - // Report on each match. - foreach (Match match in matches) - { - urls.Add(match.Value); - } + string dlUrl = String.Empty; + string workspaceUrl = String.Empty; + // Report on each match. + foreach (Match match in regexDl.Matches(body)) + { + dlUrl = match.Value; + break; } - - client.Disconnect(true); + + foreach (Match match in regexWorkspace.Matches(body)) + { + workspaceUrl = match.Value; + break; + } + + urls.Add(new Tuple(dlUrl,workspaceUrl)); } + client.Disconnect(true); + return urls; } diff --git a/NotionBackupTool.csproj b/NotionBackupTool.csproj index 1823d58..7f6960c 100644 --- a/NotionBackupTool.csproj +++ b/NotionBackupTool.csproj @@ -12,6 +12,7 @@ + diff --git a/NotionWebsitePuppeteer.cs b/NotionWebsitePuppeteer.cs index 2891539..ed70060 100644 --- a/NotionWebsitePuppeteer.cs +++ b/NotionWebsitePuppeteer.cs @@ -38,11 +38,12 @@ void TryLoginWithLink() { Thread.Sleep(2000); Console.WriteLine("Waiting for login code email..."); - List loginUrls = mg.FindUrl("notify@mail.notion.so", "https://www\\.notion\\.so/loginwithemail.*?(?=\")"); + List> loginUrls = mg.FindUrl("notify@mail.notion.so", + "https://www\\.notion\\.so/loginwithemail.*?(?=\")", String.Empty); if (loginUrls.Count == 0) continue; - loginUrl = loginUrls.First(); + loginUrl = loginUrls.First().Item1; foundUrl = true; mg.Purge("notify@mail.notion.so","login code"); diff --git a/Program.cs b/Program.cs index 18a32fb..6e4affb 100644 --- a/Program.cs +++ b/Program.cs @@ -1,8 +1,4 @@ -using System.Runtime; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using OpenQA.Selenium; -using OpenQA.Selenium.Remote; +using OpenQA.Selenium; namespace NotionBackupTool; @@ -20,8 +16,10 @@ static string GetConfig(string envName, string defaultValue = "") return string.IsNullOrEmpty(envVar) ? defaultValue : envVar; } - static async Task Main(string[] args) + static async Task Main() { + string mode = GetConfig("MODE"); + string webDriverEndpoint = GetConfig("WEBDRIVER_URL", "http://localhost:4444"); string s3Host = GetConfig("S3_HOST"); @@ -37,10 +35,12 @@ static async Task Main(string[] args) string mailPassword = GetConfig("IMAP_PASSWORD"); string cachePath = GetConfig("CACHE_PATH"); - string mode = GetConfig("MODE"); string temporaryDir = GetConfig("TMP_DIR"); string gpgPublicKey = GetConfig("GPG_PUBKEY_FILE"); - + + List workspaces = GetConfig("WORKSPACES").Split(',').Select(w => w.Split(':')).Select(ws => new Workspace { Name = ws[0], Id = ws[1] }).ToList(); + + var pingHttp = new HttpClient(); string hcUrl = GetConfig("HEALTHCHECK_URL"); @@ -93,8 +93,8 @@ async Task FileCookie() // Look in Mail inbox for completed exports Console.WriteLine("Checking emails..."); MailGrabber mail = new MailGrabber(mailHost, mailUser, mailPassword); - List downloadUrls = mail.FindUrl("export-noreply@mail.notion.so", @"https://file\.notion\.so/.+\.zip"); - downloadUrls.AddRange(mail.FindUrl("notify@mail.notion.so", @"https://file\.notion\.so/.+\.zip")); + List> downloadUrls = mail.FindUrl("export-noreply@mail.notion.so", @"https://file\.notion\.so/.+\.zip","https://www\\.notion\\.so/space/[a-z0-9]+"); + downloadUrls.AddRange(mail.FindUrl("notify@mail.notion.so", @"https://file\.notion\.so/.+\.zip","https://www\\.notion\\.so/space/[a-z0-9]+")); // Download the exports using the session cookies @@ -102,35 +102,45 @@ async Task FileCookie() using HttpClient hc = new HttpClient(); int ct = 0; - foreach (string url in downloadUrls) + foreach (Tuple urls in downloadUrls) { Console.WriteLine($"Processing download #{ct}"); - var req = new HttpRequestMessage(HttpMethod.Get, url); + var req = new HttpRequestMessage(HttpMethod.Get, urls.Item1); req.Headers.Add("cookie", $"file_token={fileCookieValue}"); var response = await hc.SendAsync(req, HttpCompletionOption.ResponseHeadersRead); response.EnsureSuccessStatusCode(); string datetime = DateTime.UtcNow.ToString("s").Replace(':', '-'); - string dlFilename = Path.Combine(temporaryDir, $"notion-{ct}-{datetime}.zip"); + + // map workspace + string workspaceId = urls.Item2.Split('/').Last(); + string workspaceName = workspaces.FirstOrDefault(x => x.Id == workspaceId)?.Name ?? String.Empty; + + if (string.IsNullOrEmpty(workspaceName)) + { + Console.WriteLine($"Warning: Workspace ID {workspaceId} unknown"); + workspaceName = workspaceId; + } + + string dlFilename = Path.Combine(temporaryDir, $"notion-{workspaceName}-{datetime}.zip"); Console.WriteLine("Downloading..."); await using Stream dlStream = await response.Content.ReadAsStreamAsync(); await using var fileStream = File.Create(dlFilename); await dlStream.CopyToAsync(fileStream, 1048576); await fileStream.FlushAsync(); fileStream.Close(); - + // Encrypt backup FileProcessor fp = new FileProcessor(); Console.WriteLine("==> Encrypting..."); string encryptedFile = $"{dlFilename}.enc"; - fp.EncryptFileSymetric(dlFilename, encryptedFile, gpgPublicKey); + await fp.EncryptFilePgp(dlFilename, gpgPublicKey, encryptedFile); // Upload to S3 Console.WriteLine("==> Send to S3 storage"); var s3 = new S3Uploader(s3Host, s3AccessKey, s3SecretKey, s3Bucket); await s3.UploadFileAsync(encryptedFile); - await s3.UploadFileAsync($"{encryptedFile}.key"); ct++; } @@ -142,12 +152,9 @@ async Task FileCookie() } else if (mode == "trigger") { - string workspace = GetConfig("WORKSPACES"); - List workspaces = new List(workspace.Split(",")); - grabber.TriggerExport(workspaces); + grabber.TriggerExport(workspaces.Select(x => x.Name).ToList()); } Console.WriteLine("done."); } - } \ No newline at end of file diff --git a/Workspace.cs b/Workspace.cs new file mode 100644 index 0000000..f966e9d --- /dev/null +++ b/Workspace.cs @@ -0,0 +1,7 @@ +namespace NotionBackupTool; + +public record Workspace +{ + public string Name { get; set; } = String.Empty; + public string Id { get; set; } = String.Empty; +} \ No newline at end of file