diff --git a/src/FMBot.Bot/Models/ImportModels.cs b/src/FMBot.Bot/Models/ImportModels.cs index 7487fac6..5b427f7d 100644 --- a/src/FMBot.Bot/Models/ImportModels.cs +++ b/src/FMBot.Bot/Models/ImportModels.cs @@ -100,5 +100,6 @@ public enum ImportStatus { Success, UnknownFailure, - WrongPackageFailure + WrongPackageFailure, + WrongCsvFailure } diff --git a/src/FMBot.Bot/Services/ImportService.cs b/src/FMBot.Bot/Services/ImportService.cs index 03d6ca0f..8c114c13 100644 --- a/src/FMBot.Bot/Services/ImportService.cs +++ b/src/FMBot.Bot/Services/ImportService.cs @@ -35,7 +35,8 @@ public class ImportService private readonly ArtistEnrichment.ArtistEnrichmentClient _artistEnrichment; - public ImportService(HttpClient httpClient, TimeService timeService, IOptions botSettings, TimeEnrichment.TimeEnrichmentClient timeEnrichment, ArtistEnrichment.ArtistEnrichmentClient artistEnrichment) + public ImportService(HttpClient httpClient, TimeService timeService, IOptions botSettings, + TimeEnrichment.TimeEnrichmentClient timeEnrichment, ArtistEnrichment.ArtistEnrichmentClient artistEnrichment) { this._httpClient = httpClient; this._timeService = timeService; @@ -44,7 +45,8 @@ public ImportService(HttpClient httpClient, TimeService timeService, IOptions result)> HandleAppleMusicFiles(User user, IAttachment attachment) + public async Task<(ImportStatus status, List result)> HandleAppleMusicFiles(User user, + IAttachment attachment) { var appleMusicPlays = new List(); @@ -58,12 +60,15 @@ public ImportService(HttpClient httpClient, TimeService timeService, IOptions a.FullName.EndsWith("Apple Music Play Activity.csv", StringComparison.OrdinalIgnoreCase))) + if (!zip.Entries.Any(a => + a.FullName.EndsWith("Apple Music Play Activity.csv", StringComparison.OrdinalIgnoreCase))) { - var innerZipEntry = zip.Entries.FirstOrDefault(f => f.FullName.EndsWith("Apple_Media_Services.zip", StringComparison.OrdinalIgnoreCase)); + var innerZipEntry = zip.Entries.FirstOrDefault(f => + f.FullName.EndsWith("Apple_Media_Services.zip", StringComparison.OrdinalIgnoreCase)); if (innerZipEntry == null) { - Log.Information("Importing: {userId} / {discordUserId} - HandleAppleMusicFiles - Could not find 'Apple_Media_Services.zip' inside first zip - {zipName}", + Log.Information( + "Importing: {userId} / {discordUserId} - HandleAppleMusicFiles - Could not find 'Apple_Media_Services.zip' inside first zip - {zipName}", user.UserId, user.DiscordUserId, attachment.Filename); return (ImportStatus.UnknownFailure, null); @@ -72,7 +77,8 @@ public ImportService(HttpClient httpClient, TimeService timeService, IOptions f.FullName.EndsWith("Apple Music Play Activity.csv", StringComparison.OrdinalIgnoreCase)); + var csvEntry = innerZip.Entries.FirstOrDefault(f => + f.FullName.EndsWith("Apple Music Play Activity.csv", StringComparison.OrdinalIgnoreCase)); if (csvEntry != null) { await ExtractPlaysFromCsv(csvEntry); @@ -80,26 +86,39 @@ public ImportService(HttpClient httpClient, TimeService timeService, IOptions f.FullName.EndsWith("Apple Music Play Activity.csv", StringComparison.OrdinalIgnoreCase)); + var csvEntry = zip.Entries.FirstOrDefault(f => + f.FullName.EndsWith("Apple Music Play Activity.csv", StringComparison.OrdinalIgnoreCase)); if (csvEntry != null) { await ExtractPlaysFromCsv(csvEntry); } } } + if (attachment.Filename.EndsWith(".csv")) { await using var stream = await this._httpClient.GetStreamAsync(attachment.Url); using var innerCsvStreamReader = new StreamReader(stream); - using var csv = new CsvReader(innerCsvStreamReader, csvConfig); + try + { + using var csv = new CsvReader(innerCsvStreamReader, csvConfig); - var records = csv.GetRecords(); - if (records.Any()) + + var records = csv.GetRecords(); + if (records.Any()) + { + appleMusicPlays.AddRange(records.Where(w => w.PlayDurationMs > 0 && + !string.IsNullOrWhiteSpace(w.AlbumName) && + !string.IsNullOrWhiteSpace(w.SongName)).ToList()); + } + } + catch (Exception e) { - appleMusicPlays.AddRange(records.Where(w => w.PlayDurationMs > 0 && - !string.IsNullOrWhiteSpace(w.AlbumName) && - !string.IsNullOrWhiteSpace(w.SongName)).ToList()); + Log.Information( + "Importing: {userId} / {discordUserId} - HandleAppleMusicFiles - Failed to parse csv - {zipName}", + user.UserId, user.DiscordUserId, attachment.Filename); + return (ImportStatus.WrongCsvFailure, appleMusicPlays); } } @@ -127,7 +146,8 @@ async Task ExtractPlaysFromCsv(ZipArchiveEntry csvEntry) } } - public async Task<(RepeatedField userPlays, string matchFoundPercentage)> AppleMusicImportAddArtists(User user, List appleMusicPlays) + public async Task<(RepeatedField userPlays, string matchFoundPercentage)> + AppleMusicImportAddArtists(User user, List appleMusicPlays) { var simplePlays = appleMusicPlays .Where(w => w.AlbumName != null && @@ -158,24 +178,29 @@ async Task ExtractPlaysFromCsv(ZipArchiveEntry csvEntry) var playsWithArtist = await this._artistEnrichment.AddArtistToPlaysAsync(appleMusicImports); - Log.Information("Importing: {userId} / {discordUserId} - AppleMusicImportToUserPlays - Total {totalPlays} - Artist found {artistFound} - Artist not found {artistNotFound}", - user.UserId, user.DiscordUserId, playsWithArtist.Plays.Count, playsWithArtist.ArtistFound, playsWithArtist.ArtistNotFound); + Log.Information( + "Importing: {userId} / {discordUserId} - AppleMusicImportToUserPlays - Total {totalPlays} - Artist found {artistFound} - Artist not found {artistNotFound}", + user.UserId, user.DiscordUserId, playsWithArtist.Plays.Count, playsWithArtist.ArtistFound, + playsWithArtist.ArtistNotFound); var validPlays = playsWithArtist.Plays .Where(w => IsValidScrobble(w.MsPlayed, w.MediaLength)).ToList(); - Log.Information("Importing: {userId} / {discordUserId} - AppleMusicImportToUserPlays - {validScrobbles} plays left after listening time filter", + Log.Information( + "Importing: {userId} / {discordUserId} - AppleMusicImportToUserPlays - {validScrobbles} plays left after listening time filter", user.UserId, user.DiscordUserId, validPlays.Count); return (playsWithArtist.Plays, $"{(decimal)playsWithArtist.ArtistFound / playsWithArtist.Plays.Count:0%}"); } - public static List AppleMusicImportsToValidUserPlays(User user, RepeatedField appleMusicPlays) + public static List AppleMusicImportsToValidUserPlays(User user, + RepeatedField appleMusicPlays) { var validPlays = appleMusicPlays .Where(w => IsValidScrobble(w.MsPlayed, w.MediaLength)).ToList(); - Log.Information("Importing: {userId} / {discordUserId} - AppleMusicImportToUserPlays - {validScrobbles} plays left after listening time filter", + Log.Information( + "Importing: {userId} / {discordUserId} - AppleMusicImportToUserPlays - {validScrobbles} plays left after listening time filter", user.UserId, user.DiscordUserId, validPlays.Count); return validPlays.Select(s => new UserPlay @@ -200,14 +225,16 @@ private static bool IsValidScrobble(long msPlayed, long mediaLength) }; } - public async Task<(ImportStatus status, List result, List processedFiles)> HandleSpotifyFiles(User user, IEnumerable attachments) + public async Task<(ImportStatus status, List result, List processedFiles)> + HandleSpotifyFiles(User user, IEnumerable attachments) { try { var spotifyPlays = new List(); var processedFiles = new List(); - foreach (var attachment in attachments.Where(w => w?.Url != null && w.Filename.Contains(".json")).GroupBy(g => g.Filename)) + foreach (var attachment in attachments.Where(w => w?.Url != null && w.Filename.Contains(".json")) + .GroupBy(g => g.Filename)) { await using var stream = await this._httpClient.GetStreamAsync(attachment.First().Url); @@ -220,10 +247,13 @@ private static bool IsValidScrobble(long msPlayed, long mediaLength) } catch (Exception e) { - Log.Error("Importing: {userId} / {discordUserId} - Error in import .zip file ({fileName})", user.UserId, user.DiscordUserId, attachment.Key, e); + Log.Error("Importing: {userId} / {discordUserId} - Error in import .zip file ({fileName})", + user.UserId, user.DiscordUserId, attachment.Key, e); } } - foreach (var attachment in attachments.Where(w => w?.Url != null && w.Filename.Contains(".zip")).GroupBy(g => g.Filename)) + + foreach (var attachment in attachments.Where(w => w?.Url != null && w.Filename.Contains(".zip")) + .GroupBy(g => g.Filename)) { await using var stream = await this._httpClient.GetStreamAsync(attachment.First().Url); @@ -246,7 +276,8 @@ private static bool IsValidScrobble(long msPlayed, long mediaLength) } catch (Exception e) { - Log.Error("Importing: {userId} / {discordUserId} - Error in import .zip file ({fileName})", user.UserId, user.DiscordUserId, entry.Name, e); + Log.Error("Importing: {userId} / {discordUserId} - Error in import .zip file ({fileName})", + user.UserId, user.DiscordUserId, entry.Name, e); } } } @@ -255,7 +286,8 @@ private static bool IsValidScrobble(long msPlayed, long mediaLength) } catch (Exception e) { - Log.Error("Importing: {userId} / {discordUserId} - Error while attempting to process Spotify import file", user.UserId, user.DiscordUserId, e); + Log.Error("Importing: {userId} / {discordUserId} - Error while attempting to process Spotify import file", + user.UserId, user.DiscordUserId, e); return (ImportStatus.UnknownFailure, null, null); } } @@ -284,7 +316,8 @@ public async Task> SpotifyImportToUserPlays(User user, List new UserPlay @@ -292,7 +325,9 @@ public async Task> SpotifyImportToUserPlays(User user, List plays) @@ -353,11 +389,14 @@ public async Task InsertImportPlays(User user, IList plays) await connection.OpenAsync(); var inserted = await PlayRepository.InsertTimeSeriesPlays(plays, connection); - Log.Information("Importing: {userId} / {discordUserId} - Inserted {insertCount} plays (Should be {importCount})", user.UserId, user.DiscordUserId, inserted, plays.Count); + Log.Information( + "Importing: {userId} / {discordUserId} - Inserted {insertCount} plays (Should be {importCount})", + user.UserId, user.DiscordUserId, inserted, plays.Count); } else { - Log.Error("Importing: {userId} / {discordUserId} Tried to insert 0 import plays!", user.UserId, user.DiscordUserId); + Log.Error("Importing: {userId} / {discordUserId} Tried to insert 0 import plays!", user.UserId, + user.DiscordUserId); } } @@ -368,7 +407,8 @@ public async Task RemoveImportedSpotifyPlays(User user) await PlayRepository.RemoveAllImportedSpotifyPlays(user.UserId, connection); - Log.Information("Importing: {userId} / {discordUserId} - Removed imported Spotify plays", user.UserId, user.DiscordUserId); + Log.Information("Importing: {userId} / {discordUserId} - Removed imported Spotify plays", user.UserId, + user.DiscordUserId); } public async Task RemoveImportedAppleMusicPlays(User user) @@ -378,7 +418,8 @@ public async Task RemoveImportedAppleMusicPlays(User user) await PlayRepository.RemoveAllImportedAppleMusicPlays(user.UserId, connection); - Log.Information("Importing: {userId} / {discordUserId} - Removed imported Apple Music plays", user.UserId, user.DiscordUserId); + Log.Information("Importing: {userId} / {discordUserId} - Removed imported Apple Music plays", user.UserId, + user.DiscordUserId); } public async Task HasImported(int userId) @@ -395,6 +436,7 @@ public static string AddImportDescription(StringBuilder stringBuilder, List