Skip to content

Commit

Permalink
add indication for wrong apple music file
Browse files Browse the repository at this point in the history
  • Loading branch information
@th0mk
th0mk committed Nov 10, 2024
1 parent 998bf57 commit 718c494
Show file tree
Hide file tree
Showing 3 changed files with 85 additions and 33 deletions.
3 changes: 2 additions & 1 deletion src/FMBot.Bot/Models/ImportModels.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,5 +100,6 @@ public enum ImportStatus
{
Success,
UnknownFailure,
WrongPackageFailure
WrongPackageFailure,
WrongCsvFailure
}
106 changes: 74 additions & 32 deletions src/FMBot.Bot/Services/ImportService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@ public class ImportService
private readonly ArtistEnrichment.ArtistEnrichmentClient _artistEnrichment;


public ImportService(HttpClient httpClient, TimeService timeService, IOptions<BotSettings> botSettings, TimeEnrichment.TimeEnrichmentClient timeEnrichment, ArtistEnrichment.ArtistEnrichmentClient artistEnrichment)
public ImportService(HttpClient httpClient, TimeService timeService, IOptions<BotSettings> botSettings,
TimeEnrichment.TimeEnrichmentClient timeEnrichment, ArtistEnrichment.ArtistEnrichmentClient artistEnrichment)
{
this._httpClient = httpClient;
this._timeService = timeService;
Expand All @@ -44,7 +45,8 @@ public ImportService(HttpClient httpClient, TimeService timeService, IOptions<Bo
this._botSettings = botSettings.Value;
}

public async Task<(ImportStatus status, List<AppleMusicCsvImportModel> result)> HandleAppleMusicFiles(User user, IAttachment attachment)
public async Task<(ImportStatus status, List<AppleMusicCsvImportModel> result)> HandleAppleMusicFiles(User user,
IAttachment attachment)
{
var appleMusicPlays = new List<AppleMusicCsvImportModel>();

Expand All @@ -58,12 +60,15 @@ public ImportService(HttpClient httpClient, TimeService timeService, IOptions<Bo
await using var stream = await this._httpClient.GetStreamAsync(attachment.Url);
using var zip = new ZipArchive(stream, ZipArchiveMode.Read);

if (!zip.Entries.Any(a => a.FullName.EndsWith("Apple Music Play Activity.csv", StringComparison.OrdinalIgnoreCase)))
if (!zip.Entries.Any(a =>
a.FullName.EndsWith("Apple Music Play Activity.csv", StringComparison.OrdinalIgnoreCase)))
{
var innerZipEntry = zip.Entries.FirstOrDefault(f => f.FullName.EndsWith("Apple_Media_Services.zip", StringComparison.OrdinalIgnoreCase));
var innerZipEntry = zip.Entries.FirstOrDefault(f =>
f.FullName.EndsWith("Apple_Media_Services.zip", StringComparison.OrdinalIgnoreCase));
if (innerZipEntry == null)
{
Log.Information("Importing: {userId} / {discordUserId} - HandleAppleMusicFiles - Could not find 'Apple_Media_Services.zip' inside first zip - {zipName}",
Log.Information(
"Importing: {userId} / {discordUserId} - HandleAppleMusicFiles - Could not find 'Apple_Media_Services.zip' inside first zip - {zipName}",
user.UserId, user.DiscordUserId, attachment.Filename);

return (ImportStatus.UnknownFailure, null);
Expand All @@ -72,34 +77,48 @@ public ImportService(HttpClient httpClient, TimeService timeService, IOptions<Bo
await using var innerZipStream = innerZipEntry.Open();
using var innerZip = new ZipArchive(innerZipStream, ZipArchiveMode.Read);

var csvEntry = innerZip.Entries.FirstOrDefault(f => f.FullName.EndsWith("Apple Music Play Activity.csv", StringComparison.OrdinalIgnoreCase));
var csvEntry = innerZip.Entries.FirstOrDefault(f =>
f.FullName.EndsWith("Apple Music Play Activity.csv", StringComparison.OrdinalIgnoreCase));
if (csvEntry != null)
{
await ExtractPlaysFromCsv(csvEntry);
}
}
else
{
var csvEntry = zip.Entries.FirstOrDefault(f => f.FullName.EndsWith("Apple Music Play Activity.csv", StringComparison.OrdinalIgnoreCase));
var csvEntry = zip.Entries.FirstOrDefault(f =>
f.FullName.EndsWith("Apple Music Play Activity.csv", StringComparison.OrdinalIgnoreCase));
if (csvEntry != null)
{
await ExtractPlaysFromCsv(csvEntry);
}
}
}

if (attachment.Filename.EndsWith(".csv"))
{
await using var stream = await this._httpClient.GetStreamAsync(attachment.Url);
using var innerCsvStreamReader = new StreamReader(stream);

using var csv = new CsvReader(innerCsvStreamReader, csvConfig);
try
{
using var csv = new CsvReader(innerCsvStreamReader, csvConfig);

var records = csv.GetRecords<AppleMusicCsvImportModel>();
if (records.Any())

var records = csv.GetRecords<AppleMusicCsvImportModel>();
if (records.Any())
{
appleMusicPlays.AddRange(records.Where(w => w.PlayDurationMs > 0 &&
!string.IsNullOrWhiteSpace(w.AlbumName) &&
!string.IsNullOrWhiteSpace(w.SongName)).ToList());
}
}
catch (Exception e)
{
appleMusicPlays.AddRange(records.Where(w => w.PlayDurationMs > 0 &&
!string.IsNullOrWhiteSpace(w.AlbumName) &&
!string.IsNullOrWhiteSpace(w.SongName)).ToList());
Log.Information(
"Importing: {userId} / {discordUserId} - HandleAppleMusicFiles - Failed to parse csv - {zipName}",
user.UserId, user.DiscordUserId, attachment.Filename);
return (ImportStatus.WrongCsvFailure, appleMusicPlays);
}
}

Expand Down Expand Up @@ -127,7 +146,8 @@ async Task ExtractPlaysFromCsv(ZipArchiveEntry csvEntry)
}
}

public async Task<(RepeatedField<PlayWithoutArtist> userPlays, string matchFoundPercentage)> AppleMusicImportAddArtists(User user, List<AppleMusicCsvImportModel> appleMusicPlays)
public async Task<(RepeatedField<PlayWithoutArtist> userPlays, string matchFoundPercentage)>
AppleMusicImportAddArtists(User user, List<AppleMusicCsvImportModel> appleMusicPlays)
{
var simplePlays = appleMusicPlays
.Where(w => w.AlbumName != null &&
Expand Down Expand Up @@ -158,24 +178,29 @@ async Task ExtractPlaysFromCsv(ZipArchiveEntry csvEntry)

var playsWithArtist = await this._artistEnrichment.AddArtistToPlaysAsync(appleMusicImports);

Log.Information("Importing: {userId} / {discordUserId} - AppleMusicImportToUserPlays - Total {totalPlays} - Artist found {artistFound} - Artist not found {artistNotFound}",
user.UserId, user.DiscordUserId, playsWithArtist.Plays.Count, playsWithArtist.ArtistFound, playsWithArtist.ArtistNotFound);
Log.Information(
"Importing: {userId} / {discordUserId} - AppleMusicImportToUserPlays - Total {totalPlays} - Artist found {artistFound} - Artist not found {artistNotFound}",
user.UserId, user.DiscordUserId, playsWithArtist.Plays.Count, playsWithArtist.ArtistFound,
playsWithArtist.ArtistNotFound);

var validPlays = playsWithArtist.Plays
.Where(w => IsValidScrobble(w.MsPlayed, w.MediaLength)).ToList();

Log.Information("Importing: {userId} / {discordUserId} - AppleMusicImportToUserPlays - {validScrobbles} plays left after listening time filter",
Log.Information(
"Importing: {userId} / {discordUserId} - AppleMusicImportToUserPlays - {validScrobbles} plays left after listening time filter",
user.UserId, user.DiscordUserId, validPlays.Count);

return (playsWithArtist.Plays, $"{(decimal)playsWithArtist.ArtistFound / playsWithArtist.Plays.Count:0%}");
}

public static List<UserPlay> AppleMusicImportsToValidUserPlays(User user, RepeatedField<PlayWithoutArtist> appleMusicPlays)
public static List<UserPlay> AppleMusicImportsToValidUserPlays(User user,
RepeatedField<PlayWithoutArtist> appleMusicPlays)
{
var validPlays = appleMusicPlays
.Where(w => IsValidScrobble(w.MsPlayed, w.MediaLength)).ToList();

Log.Information("Importing: {userId} / {discordUserId} - AppleMusicImportToUserPlays - {validScrobbles} plays left after listening time filter",
Log.Information(
"Importing: {userId} / {discordUserId} - AppleMusicImportToUserPlays - {validScrobbles} plays left after listening time filter",
user.UserId, user.DiscordUserId, validPlays.Count);

return validPlays.Select(s => new UserPlay
Expand All @@ -200,14 +225,16 @@ private static bool IsValidScrobble(long msPlayed, long mediaLength)
};
}

public async Task<(ImportStatus status, List<SpotifyEndSongImportModel> result, List<string> processedFiles)> HandleSpotifyFiles(User user, IEnumerable<IAttachment> attachments)
public async Task<(ImportStatus status, List<SpotifyEndSongImportModel> result, List<string> processedFiles)>
HandleSpotifyFiles(User user, IEnumerable<IAttachment> attachments)
{
try
{
var spotifyPlays = new List<SpotifyEndSongImportModel>();
var processedFiles = new List<string>();

foreach (var attachment in attachments.Where(w => w?.Url != null && w.Filename.Contains(".json")).GroupBy(g => g.Filename))
foreach (var attachment in attachments.Where(w => w?.Url != null && w.Filename.Contains(".json"))
.GroupBy(g => g.Filename))
{
await using var stream = await this._httpClient.GetStreamAsync(attachment.First().Url);

Expand All @@ -220,10 +247,13 @@ private static bool IsValidScrobble(long msPlayed, long mediaLength)
}
catch (Exception e)
{
Log.Error("Importing: {userId} / {discordUserId} - Error in import .zip file ({fileName})", user.UserId, user.DiscordUserId, attachment.Key, e);
Log.Error("Importing: {userId} / {discordUserId} - Error in import .zip file ({fileName})",
user.UserId, user.DiscordUserId, attachment.Key, e);
}
}
foreach (var attachment in attachments.Where(w => w?.Url != null && w.Filename.Contains(".zip")).GroupBy(g => g.Filename))

foreach (var attachment in attachments.Where(w => w?.Url != null && w.Filename.Contains(".zip"))
.GroupBy(g => g.Filename))
{
await using var stream = await this._httpClient.GetStreamAsync(attachment.First().Url);

Expand All @@ -246,7 +276,8 @@ private static bool IsValidScrobble(long msPlayed, long mediaLength)
}
catch (Exception e)
{
Log.Error("Importing: {userId} / {discordUserId} - Error in import .zip file ({fileName})", user.UserId, user.DiscordUserId, entry.Name, e);
Log.Error("Importing: {userId} / {discordUserId} - Error in import .zip file ({fileName})",
user.UserId, user.DiscordUserId, entry.Name, e);
}
}
}
Expand All @@ -255,7 +286,8 @@ private static bool IsValidScrobble(long msPlayed, long mediaLength)
}
catch (Exception e)
{
Log.Error("Importing: {userId} / {discordUserId} - Error while attempting to process Spotify import file", user.UserId, user.DiscordUserId, e);
Log.Error("Importing: {userId} / {discordUserId} - Error while attempting to process Spotify import file",
user.UserId, user.DiscordUserId, e);
return (ImportStatus.UnknownFailure, null, null);
}
}
Expand Down Expand Up @@ -284,15 +316,18 @@ public async Task<List<UserPlay>> SpotifyImportToUserPlays(User user, List<Spoti

var filterInvalidPlays = await this._timeEnrichment.FilterInvalidSpotifyImportsAsync(spotifyImports);

Log.Information("Importing: {userId} / {discordUserId} - SpotifyImportToUserPlays found {validPlays} valid plays and {invalidPlays} invalid plays",
Log.Information(
"Importing: {userId} / {discordUserId} - SpotifyImportToUserPlays found {validPlays} valid plays and {invalidPlays} invalid plays",
user.UserId, user.DiscordUserId, filterInvalidPlays.ValidImports.Count, filterInvalidPlays.InvalidPlays);

return filterInvalidPlays.ValidImports.Select(s => new UserPlay
{
UserId = user.UserId,
TimePlayed = DateTime.SpecifyKind(s.Ts.ToDateTime(), DateTimeKind.Utc),
ArtistName = s.MasterMetadataAlbumArtistName,
AlbumName = !string.IsNullOrWhiteSpace(s.MasterMetadataAlbumAlbumName) ? s.MasterMetadataAlbumAlbumName : null,
AlbumName = !string.IsNullOrWhiteSpace(s.MasterMetadataAlbumAlbumName)
? s.MasterMetadataAlbumAlbumName
: null,
TrackName = s.MasterMetadataTrackName,
PlaySource = PlaySource.SpotifyImport,
MsPlayed = s.MsPlayed
Expand Down Expand Up @@ -342,7 +377,8 @@ public async Task UpdateExistingPlays(User user)
await connection.OpenAsync();

await PlayRepository.SetDefaultSourceForPlays(user.UserId, connection);
Log.Information("Importing: {userId} / {discordUserId} - Set default data source", user.UserId, user.DiscordUserId);
Log.Information("Importing: {userId} / {discordUserId} - Set default data source", user.UserId,
user.DiscordUserId);
}

public async Task InsertImportPlays(User user, IList<UserPlay> plays)
Expand All @@ -353,11 +389,14 @@ public async Task InsertImportPlays(User user, IList<UserPlay> plays)
await connection.OpenAsync();

var inserted = await PlayRepository.InsertTimeSeriesPlays(plays, connection);
Log.Information("Importing: {userId} / {discordUserId} - Inserted {insertCount} plays (Should be {importCount})", user.UserId, user.DiscordUserId, inserted, plays.Count);
Log.Information(
"Importing: {userId} / {discordUserId} - Inserted {insertCount} plays (Should be {importCount})",
user.UserId, user.DiscordUserId, inserted, plays.Count);
}
else
{
Log.Error("Importing: {userId} / {discordUserId} Tried to insert 0 import plays!", user.UserId, user.DiscordUserId);
Log.Error("Importing: {userId} / {discordUserId} Tried to insert 0 import plays!", user.UserId,
user.DiscordUserId);
}
}

Expand All @@ -368,7 +407,8 @@ public async Task RemoveImportedSpotifyPlays(User user)

await PlayRepository.RemoveAllImportedSpotifyPlays(user.UserId, connection);

Log.Information("Importing: {userId} / {discordUserId} - Removed imported Spotify plays", user.UserId, user.DiscordUserId);
Log.Information("Importing: {userId} / {discordUserId} - Removed imported Spotify plays", user.UserId,
user.DiscordUserId);
}

public async Task RemoveImportedAppleMusicPlays(User user)
Expand All @@ -378,7 +418,8 @@ public async Task RemoveImportedAppleMusicPlays(User user)

await PlayRepository.RemoveAllImportedAppleMusicPlays(user.UserId, connection);

Log.Information("Importing: {userId} / {discordUserId} - Removed imported Apple Music plays", user.UserId, user.DiscordUserId);
Log.Information("Importing: {userId} / {discordUserId} - Removed imported Apple Music plays", user.UserId,
user.DiscordUserId);
}

public async Task<bool> HasImported(int userId)
Expand All @@ -395,6 +436,7 @@ public static string AddImportDescription(StringBuilder stringBuilder, List<Play
{
return null;
}

if (playSources.Contains(PlaySource.SpotifyImport) && playSources.Contains(PlaySource.AppleMusicImport))
{
stringBuilder.AppendLine("Contains imported Spotify and Apple Music plays");
Expand Down
9 changes: 9 additions & 0 deletions src/FMBot.Bot/SlashCommands/ImportSlashCommands.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,15 @@ await UpdateImportEmbed(message, embed, description, $"❌ Invalid Apple Music i
this.Context.LogCommandUsed(CommandResponse.WrongInput);
return;
}
if (imports.status == ImportStatus.WrongCsvFailure)
{
embed.WithColor(DiscordConstants.WarningColorOrange);
await UpdateImportEmbed(message, embed, description, $"❌ We couldn't read the `.csv` file that was provided.\n\n" +
$"We can only read a `Apple Music Play Activity.csv` file. Other files do not contain the data required for importing.\n\n" +
$"Still having issues? You can also open a help thread on [our server](https://discord.gg/fmbot).", true);
this.Context.LogCommandUsed(CommandResponse.WrongInput);
return;
}

await UpdateImportEmbed(message, embed, description, $"- **{imports.result.Count}** Apple Music imports found");

Expand Down

0 comments on commit 718c494

Please sign in to comment.