diff --git a/Command/ImportDownloadVideosFromYoutubeChannel.php b/Command/DownloadVideosFromYouTubeChannel.php similarity index 93% rename from Command/ImportDownloadVideosFromYoutubeChannel.php rename to Command/DownloadVideosFromYouTubeChannel.php index c8a12f0..e187b3d 100644 --- a/Command/ImportDownloadVideosFromYoutubeChannel.php +++ b/Command/DownloadVideosFromYouTubeChannel.php @@ -20,7 +20,7 @@ use YouTube\Utils\Utils; use YouTube\YouTubeDownloader; -final class ImportDownloadVideosFromYoutubeChannel extends Command +final class DownloadVideosFromYouTubeChannel extends Command { public const BASE_URL_YOUTUBE_VIDEO = 'https://www.youtube.com/watch?v='; @@ -39,7 +39,7 @@ public function __construct(DocumentManager $documentManager, GoogleAccountServi protected function configure(): void { $this - ->setName('pumukit:youtube:import:videos:from:channel') + ->setName('pumukit:youtube:download:videos:from:channel') ->addOption( 'account', null, @@ -61,11 +61,12 @@ protected function configure(): void ->setDescription('Import all videos from Youtube channel') ->setHelp( <<<'EOT' -Import all videos from Youtube channel + +Download all videos "published" and "hidden" from Youtube channel on storage. Limit is optional to test the command. If you don't set it, all videos will be downloaded. -Usage: php bin/console pumukit:youtube:import:videos:from:channel --account={ACCOUNT} --channel={CHANNEL_ID} --limit={LIMIT} +Usage: php bin/console pumukit:youtube:download:videos:from:channel --account={ACCOUNT} --channel={CHANNEL_ID} --limit={LIMIT} EOT ) @@ -76,7 +77,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int { $channel = $input->getOption('channel'); - $youtubeAccount = $this->getYoutubeAccount($input); + $youtubeAccount = $this->ensureYouTubeAccountExists($input); $service = $this->googleAccountService->googleServiceFromAccount($youtubeAccount); $channelId = $this->channelId($channel, $service); @@ -206,7 +207,7 @@ private function moveFileToStorage($item, $url, DownloadOptions $downloadOptions $file = $this->tempDir.'/'.$channelId.'/'.$videoId.'.'.$mimeType; $multimediaObject = $this->documentManager->getRepository(MultimediaObject::class)->findOneBy([ - 'properties.youtube_video_id' => $videoId, + 'properties.youtube_import_id' => $videoId, ]); $failedJobs = $this->documentManager->getRepository(Job::class)->findOneBy([ @@ -230,7 +231,7 @@ private function createChannelDir(string $channelId): void } } - private function getYoutubeAccount(InputInterface $input): Tag + private function ensureYouTubeAccountExists(InputInterface $input): Tag { $youtubeAccount = $this->documentManager->getRepository(Tag::class)->findOneBy([ 'properties.login' => $input->getOption('account'), diff --git a/Command/ImportPlaylistFromYouTubeChannel.php b/Command/ImportPlaylistFromYouTubeChannel.php new file mode 100644 index 0000000..6a1d079 --- /dev/null +++ b/Command/ImportPlaylistFromYouTubeChannel.php @@ -0,0 +1,236 @@ +documentManager = $documentManager; + $this->googleAccountService = $googleAccountService; + $this->factoryService = $factoryService; + $this->i18nService = $i18nService; + $this->seriesPicService = $seriesPicService; + $this->picDir = $picDir; + parent::__construct(); + } + + protected function configure(): void + { + $this + ->setName('pumukit:youtube:import:playlist:from:channel') + ->addOption( + 'account', + null, + InputOption::VALUE_REQUIRED, + 'Account' + ) + ->addOption( + 'channel', + null, + InputOption::VALUE_REQUIRED, + 'Channel ID' + ) + ->setDescription('Import playlist from YouTube to create series.') + ->setHelp( + <<<'EOT' + +Create series on PuMuKIT based on YouTube playlist. + +1 Series for each playlist and 1 default series for channel. + +Usage: php bin/console pumukit:youtube:import:playlist:from:channel --account={ACCOUNT} --channel={CHANNEL_ID} + +EOT + ) + ; + } + + protected function execute(InputInterface $input, OutputInterface $output): int + { + $channel = $input->getOption('channel'); + $youtubeAccount = $this->ensureYouTubeAccountExists($input); + + $service = $this->googleAccountService->googleServiceFromAccount($youtubeAccount); + $this->channelId = $this->channelId($channel, $service); + + $this->createDefaultSeriesForChannel($service); + + $nextPageToken = null; + $count = 0; + $queryParams = [ + 'channelId' => $this->channelId, + 'maxResults' => 50, + ]; + + $response = $service->playlists->listPlaylists('snippet', $queryParams); + + $progressBar = new ProgressBar($output, $response->pageInfo->getTotalResults()); + $progressBar->start(); + do { + if (null !== $nextPageToken) { + $queryParams['pageToken'] = $nextPageToken; + } + + $response = $service->playlists->listPlaylists('snippet', $queryParams); + $nextPageToken = $response->getNextPageToken(); + foreach ($response->getItems() as $item) { + $progressBar->advance(); + + try { + $series = $this->ensureSeriesExists($item->id, $item->snippet->title); + $series = $this->autocompleteSeriesMetadata($series, $item); + } catch (\Exception $exception) { + $output->writeln('There was error creating series by playlist: '.$item->snippet->title.'('.$item->id.')'); + + continue; + } + + $this->mongoDBFlush($count); + } + } while (null !== $nextPageToken); + + $this->mongoDBFlush($count); + + $progressBar->finish(); + $output->writeln(' '); + + return 0; + } + + private function mongoDBFlush(int $count): void + { + if (0 == $count % 50) { + $this->documentManager->flush(); + $this->documentManager->clear(); + } + } + + private function ensureYouTubeAccountExists(InputInterface $input): Tag + { + $youtubeAccount = $this->documentManager->getRepository(Tag::class)->findOneBy([ + 'properties.login' => $input->getOption('account'), + ]); + + if (!$youtubeAccount) { + throw new \Exception('Account not found'); + } + + return $youtubeAccount; + } + + private function channelId(string $channel, \Google_Service_YouTube $service): string + { + $channels = $this->channelInfo($channel, $service); + + return $channels->getItems()[0]->getId(); + } + + private function channelInfo(string $channel, \Google_Service_YouTube $service): ChannelListResponse + { + $queryParams = ['id' => $channel]; + + return $service->channels->listChannels('snippet', $queryParams); + } + + private function ensureSeriesExists(string $id, string $text): Series + { + $series = $this->documentManager->getRepository(Series::class)->findOneBy([ + 'properties.youtube_import_id' => $id, + ]); + + if ($series instanceof Series) { + return $series; + } + + $text = $this->i18nService->generateI18nText($text); + + return $this->factoryService->createSeries(null, $text); + } + + private function autocompleteSeriesMetadata(Series $series, Playlist $item): Series + { + $series->setProperty('youtube_import_id', $item->id); + $series->setProperty('youtube_import_raw', json_encode($item->snippet)); + $series->setProperty('youtube_import_type', 'playlist'); + + $series->setI18nTitle($this->i18nService->generateI18nText($item->snippet->title)); + $series->setI18nDescription($this->i18nService->generateI18nText($item->snippet->description)); + $series->setPublicDate(new \DateTime($item->snippet->publishedAt)); + + if (null !== $item->snippet->thumbnails->getMaxres()) { + $filePath = $this->downloadThumbnail($item, $series); + $this->seriesPicService->addPicFromPath($series, $filePath); + } + + return $series; + } + + private function downloadThumbnail(Playlist $item, Series $series): string + { + $seriesStoragePath = $this->picDir.'/series/'.$series->getId().'/'; + if (!is_dir($seriesStoragePath)) { + mkdir($seriesStoragePath, 0775, true); + } + + $fileName = basename(parse_url($item->snippet->thumbnails->getMaxres()->getUrl(), PHP_URL_PATH)); + $path = $seriesStoragePath.$fileName; + + $content = file_get_contents($item->snippet->thumbnails->getMaxres()->getUrl()); + file_put_contents($path, $content); + + return $path; + } + + private function createDefaultSeriesForChannel(\Google_Service_YouTube $service): void + { + $channelInfo = $this->channelInfo($this->channelId, $service); + $series = $this->ensureSeriesExists( + $channelInfo->getItems()[0]->getId(), + $channelInfo->getItems()[0]->getSnippet()->getTitle() + ); + + $channelData = $channelInfo->getItems()[0]; + $series->setProperty('youtube_import_id', $channelData->id); + $series->setProperty('youtube_import_raw', json_encode($channelData)); + $series->setProperty('youtube_import_type', 'channel'); + + $series->setI18nTitle($this->i18nService->generateI18nText('Huerfanos')); + $series->setI18nDescription($this->i18nService->generateI18nText($channelData->snippet->localized->description)); + $series->setPublicDate(new \DateTime($channelData->snippet->publishedAt)); + + $this->documentManager->flush(); + } +} diff --git a/Command/GenerateMultimediaObjectByYouTubeAPICommand.php b/Command/ImportVideosFromYouTubeChannel.php similarity index 50% rename from Command/GenerateMultimediaObjectByYouTubeAPICommand.php rename to Command/ImportVideosFromYouTubeChannel.php index 95379cb..9a1585c 100644 --- a/Command/GenerateMultimediaObjectByYouTubeAPICommand.php +++ b/Command/ImportVideosFromYouTubeChannel.php @@ -5,56 +5,74 @@ namespace Pumukit\YoutubeBundle\Command; use Doctrine\ODM\MongoDB\DocumentManager; +use Google\Service\YouTube\Video; +use Google\Service\YouTube\VideoListResponse; +use Pumukit\CoreBundle\Services\i18nService; use Pumukit\CoreBundle\Utils\FinderUtils; use Pumukit\EncoderBundle\Services\JobService; use Pumukit\SchemaBundle\Document\MultimediaObject; use Pumukit\SchemaBundle\Document\Series; use Pumukit\SchemaBundle\Document\Tag; use Pumukit\SchemaBundle\Services\FactoryService; +use Pumukit\SchemaBundle\Services\MultimediaObjectPicService; +use Pumukit\SchemaBundle\Services\TagService; +use Pumukit\WebTVBundle\PumukitWebTVBundle; use Pumukit\YoutubeBundle\Services\GoogleAccountService; +use Pumukit\YoutubeBundle\Services\PlaylistListService; use Symfony\Component\Console\Command\Command; use Symfony\Component\Console\Helper\ProgressBar; use Symfony\Component\Console\Input\InputInterface; use Symfony\Component\Console\Input\InputOption; use Symfony\Component\Console\Output\OutputInterface; -use YouTube\DownloadOptions; -use YouTube\YouTubeDownloader; -final class GenerateMultimediaObjectByYouTubeAPICommand extends Command +final class ImportVideosFromYouTubeChannel extends Command { + public const YOUTUBE_STATUS_MAPPING = [ + 'public' => MultimediaObject::STATUS_PUBLISHED, + 'hidden' => MultimediaObject::STATUS_HIDDEN, + ]; + public const DEFAULT_PROFILE_ENCODER = 'broadcastable_master'; private DocumentManager $documentManager; private GoogleAccountService $googleAccountService; private FactoryService $factoryService; private JobService $jobService; + private i18nService $i18nService; + + private TagService $tagService; + + private MultimediaObjectPicService $multimediaObjectPicService; + + private PlaylistListService $playlistListService; private string $tempDir; + private string $channelId; public function __construct( DocumentManager $documentManager, GoogleAccountService $googleAccountService, FactoryService $factoryService, JobService $jobService, + i18nService $i18nService, + TagService $tagService, + MultimediaObjectPicService $multimediaObjectPicService, + PlaylistListService $playlistListService, string $tempDir ) { $this->documentManager = $documentManager; $this->googleAccountService = $googleAccountService; $this->factoryService = $factoryService; $this->jobService = $jobService; + $this->i18nService = $i18nService; + $this->tagService = $tagService; + $this->multimediaObjectPicService = $multimediaObjectPicService; + $this->playlistListService = $playlistListService; $this->tempDir = $tempDir; parent::__construct(); } - public function mongoDBFlush(int $count): void - { - if (0 == $count % 50) { - $this->documentManager->flush(); - $this->documentManager->clear(); - } - } - protected function configure(): void { $this - ->setName('pumukit:youtube:generate:multimedia:from:api') + ->setName('pumukit:youtube:import:videos:from:channel') ->addOption( 'account', null, @@ -76,11 +94,12 @@ protected function configure(): void ->setDescription('Import all videos from Youtube channel') ->setHelp( <<<'EOT' + Import all videos from Youtube channel -Limit is optional to test the command. If you don't set it, all videos will be downloaded. +Limit is optional to test the command. -Usage: php bin/console pumukit:youtube:generate:multimedia:from:api --account={ACCOUNT} --channel={CHANNEL_ID} --limit={LIMIT} +Usage: php bin/console pumukit:youtube:import:videos:from:channel --account={ACCOUNT} --channel={CHANNEL_ID} --limit={LIMIT} EOT ) @@ -91,10 +110,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int { $channel = $input->getOption('channel'); - $youtubeAccount = $this->getYoutubeAccount($input); + $youtubeAccount = $this->ensureYouTubeAccountExists($input); $service = $this->googleAccountService->googleServiceFromAccount($youtubeAccount); - $channelId = $this->channelId($channel, $service); + $this->channelId = $this->channelId($channel, $service); $nextPageToken = null; $count = 0; @@ -125,19 +144,20 @@ protected function execute(InputInterface $input, OutputInterface $output): int } ++$count; $videoId = $item->getId()->getVideoId(); - $youtubeDownloader = new YouTubeDownloader(); try { - $youtubeURL = 'https://www.youtube.com/watch?v='.$videoId; - $downloadOptions = $youtubeDownloader->getDownloadLinks($youtubeURL); + $videoInfo = $this->videoInfo($service, $videoId); + $series = $this->obtainSeriesToSave($service, $videoId); - $series = $this->ensureSeriesExists($channelId); - $series = $this->autocompleteSeriesMetadata($series, $downloadOptions, $channelId); $multimediaObject = $this->ensureMultimediaObjectExists($series, $videoId); - $multimediaObject = $this->autocompleteMultimediaObjectMetadata($multimediaObject, $downloadOptions); - $this->addJob($multimediaObject, $videoId, $channelId); + $multimediaObject = $this->autocompleteMultimediaObjectMetadata($multimediaObject, $videoInfo); + + // Download and add maxRes PIC. + + $this->addJob($multimediaObject, $videoId); } catch (\Exception $exception) { $output->writeln('There was error downloaded video with title '.$item->snippet->title.' and id '.$videoId); + $output->writeln($exception->getMessage()); continue; } @@ -154,7 +174,15 @@ protected function execute(InputInterface $input, OutputInterface $output): int return 0; } - private function getYoutubeAccount(InputInterface $input): Tag + private function mongoDBFlush(int $count): void + { + if (0 == $count % 50) { + $this->documentManager->flush(); + $this->documentManager->clear(); + } + } + + private function ensureYouTubeAccountExists(InputInterface $input): Tag { $youtubeAccount = $this->documentManager->getRepository(Tag::class)->findOneBy([ 'properties.login' => $input->getOption('account'), @@ -178,42 +206,24 @@ private function channelId(string $channel, \Google_Service_YouTube $service): s return $channels->getItems()[0]->getId(); } - private function ensureSeriesExists(string $channelId): Series + private function defaultSeries(): Series { $series = $this->documentManager->getRepository(Series::class)->findOneBy([ - 'properties.youtube_channel_id' => $channelId, + 'properties.youtube_import_id' => $this->channelId, + 'properties.youtube_import_type' => 'channel', ]); if ($series instanceof Series) { return $series; } - $text = $this->generateTextWithLocales($channelId); - - return $this->factoryService->createSeries(null, $text); - } - - private function generateTextWithLocales(string $channelId): array - { - $text = []; - foreach ($this->factoryService->getLocales() as $locale) { - $text[$locale] = $channelId; - } - - return $text; - } - - private function autocompleteSeriesMetadata(Series $series, DownloadOptions $downloadOptions, string $channelId): Series - { - $series->setProperty('youtube_channel_id', $channelId); - - return $series; + throw new \Exception('Default series for import not found. Execute pumukit:youtube:import:playlist:from:channel first.'); } private function ensureMultimediaObjectExists(Series $series, string $videoId): MultimediaObject { $multimediaObject = $this->documentManager->getRepository(MultimediaObject::class)->findOneBy([ - 'properties.youtube_video_id' => $videoId, + 'properties.youtube_import_id' => $videoId, ]); if ($multimediaObject instanceof MultimediaObject) { @@ -223,29 +233,42 @@ private function ensureMultimediaObjectExists(Series $series, string $videoId): return $this->factoryService->createMultimediaObject($series); } - private function autocompleteMultimediaObjectMetadata(MultimediaObject $multimediaObject, DownloadOptions $downloadOptions) + private function autocompleteMultimediaObjectMetadata(MultimediaObject $multimediaObject, VideoListResponse $videoInfo): MultimediaObject { - $youtubeInfo = $downloadOptions->getInfo(); - $text = $this->generateTextWithLocales($youtubeInfo->title); + $youtubeInfo = $videoInfo->getItems()[0]; + + $text = $this->i18nService->generateI18nText($youtubeInfo->snippet->title); $multimediaObject->setI18nTitle($text); - $text = $this->generateTextWithLocales($youtubeInfo->description); + $text = $this->i18nService->generateI18nText($youtubeInfo->snippet->description); $multimediaObject->setI18nDescription($text); - $multimediaObject->setProperty('youtube_metadata', $youtubeInfo); - $multimediaObject->setProperty('youtube_video_id', $youtubeInfo->id); + $multimediaObject->setProperty('youtube_import_raw', $youtubeInfo); + $multimediaObject->setProperty('youtube_import_id', $youtubeInfo->id); + $multimediaObject->setProperty('youtube_import_type', 'video'); + $multimediaObject->setProperty('youtube_import_channel', $youtubeInfo->id); + + $multimediaObject->setPublicDate(new \DateTime($youtubeInfo->snippet->publishedAt)); + $multimediaObject->setStatus($this->convertYouTubeStatus($youtubeInfo->status->privacyStatus)); + $this->addBasicTags($multimediaObject); + $multimediaObject = $this->addKeywords($multimediaObject, $youtubeInfo); + + if (null !== $youtubeInfo->snippet->thumbnails->getMaxres()) { + $filePath = $this->downloadThumbnail($youtubeInfo, $multimediaObject); + $this->multimediaObjectPicService->addPicFromPath($multimediaObject, $filePath); + } return $multimediaObject; } - private function addJob(MultimediaObject $multimediaObject, string $youtubeId, string $channelId): MultimediaObject + private function addJob(MultimediaObject $multimediaObject, string $youtubeId): MultimediaObject { - $path = $this->tempDir.'/'.$channelId.'/'; + $path = $this->tempDir.'/'.$this->channelId.'/'; $trackUrl = FinderUtils::findFilePathname($path, $youtubeId); if (!$trackUrl) { return $multimediaObject; } - $profile = 'broadcastable_master'; + $profile = self::DEFAULT_PROFILE_ENCODER; $priority = 0; $language = null; @@ -261,4 +284,72 @@ private function addJob(MultimediaObject $multimediaObject, string $youtubeId, s $flags = 0 ); } + + private function videoInfo(\Google_Service_YouTube $service, string $videoId): VideoListResponse + { + return $service->videos->listVideos('snippet, status', ['id' => $videoId]); + } + + private function convertYouTubeStatus(string $status): int + { + return self::YOUTUBE_STATUS_MAPPING[strtolower($status)] ?? MultimediaObject::STATUS_HIDDEN; + } + + private function addBasicTags(MultimediaObject $multimediaObject): void + { + $this->tagService->addTagByCodToMultimediaObject($multimediaObject, PumukitWebTVBundle::WEB_TV_TAG); + } + + private function addKeywords(MultimediaObject $multimediaObject, Video $video): MultimediaObject + { + if (null === $video->snippet->tags) { + return $multimediaObject; + } + + foreach ($video->snippet->tags as $tag) { + $multimediaObject->addKeyword($tag); + } + + return $multimediaObject; + } + + private function downloadThumbnail(Video $video, MultimediaObject $multimediaObject): string + { + $multimediaObjectStoragePath = $this->multimediaObjectPicService->getTargetPath($multimediaObject).'/'; + if (!is_dir($multimediaObjectStoragePath)) { + mkdir($multimediaObjectStoragePath, 0775, true); + } + + $fileName = basename(parse_url($video->snippet->thumbnails->getMaxres()->getUrl(), PHP_URL_PATH)); + $path = $multimediaObjectStoragePath.$fileName; + + $content = file_get_contents($video->snippet->thumbnails->getMaxres()->getUrl()); + file_put_contents($path, $content); + + return $path; + } + + private function obtainSeriesToSave(\Google_Service_YouTube $service, string $videoId): Series + { + $playlists = $this->documentManager->getRepository(Series::class)->findBy([ + 'properties.youtube_import_type' => 'playlist', + ]); + + foreach ($playlists as $playlist) { + $response = $service->playlistItems->listPlaylistItems('snippet', [ + 'playlistId' => $playlist->getProperty('youtube_import_id'), + 'videoId' => $videoId, + ]); + + if (0 === count($response->getItems())) { + continue; + } + + if (1 === count($response->getItems())) { + return $playlist; + } + } + + return $this->defaultSeries(); + } } diff --git a/Resources/config/pumukit_youtube.yaml b/Resources/config/pumukit_youtube.yaml index f5cbef8..7ebac27 100644 --- a/Resources/config/pumukit_youtube.yaml +++ b/Resources/config/pumukit_youtube.yaml @@ -6,6 +6,7 @@ services: bind: $pumukitLocales: "%pumukit.locales%" $tempDir: "%pumukit.tmp%" + $picDir: "%pumukit.uploads_pic_dir%" Pumukit\YoutubeBundle\Controller\: resource: '../../Controller' diff --git a/Resources/doc/ImportFromYoutube.md b/Resources/doc/ImportFromYoutube.md new file mode 100644 index 0000000..2d555ba --- /dev/null +++ b/Resources/doc/ImportFromYoutube.md @@ -0,0 +1,41 @@ +IMPORT FROM YOUTUBE +=================== + +## Configure YouTube API account + +Create YouTube API account [Guide](AccountsGuide.md) + +### 1. Import playlist as series from YouTube and create default series for channel + +``` + php bin/console pumukit:youtube:import:playlist:from:channel --account={ACCOUNT} --channel={CHANNEL_ID} +``` + +where ACCOUNT is the name added for YouTube tag created on PuMuKIT and CHANNEL_ID is the channel id of the YouTube channel. + +### 2. Download videos from YouTube + +This process will be downloaded all videos from YouTube channel on status PUBLISH or HIDDEN. BLOCKED videos will be ignored. + +Max resolution will be downloaded. + +``` + php bin/console pumukit:youtube:download:videos:from:channel --account={ACCOUNT} --channel={CHANNEL_ID} +``` + +where ACCOUNT is the name added for YouTube tag created on PuMuKIT and CHANNEL_ID is the channel id of the YouTube channel. + +You can use limit to test the download using optional parameter --limit={NUMBER_OF_VIDEOS}. + +### 3. Import videos from YouTube + +After download videos you will be able to import videos from YouTube to PuMuKIT using this command. + +The command will be autocomplete metadata from YouTube and create a new video on PuMuKIT and move the video to the series created on step 1. + +``` + php bin/console pumukit:youtube:import:videos:from:channel --account={ACCOUNT} --channel={CHANNEL_ID} +``` + +where ACCOUNT is the name added for YouTube tag created on PuMuKIT and CHANNEL_ID is the channel id of the YouTube channel. +