diff --git a/charmcraft/commands/store/__init__.py b/charmcraft/commands/store/__init__.py index e5debe28b..fd61fd8dd 100644 --- a/charmcraft/commands/store/__init__.py +++ b/charmcraft/commands/store/__init__.py @@ -827,6 +827,94 @@ def run(self, parsed_args): lib_data.full_name, lib_data.api, lib_data.patch) +class FetchLibCommand(BaseCommand): + """Fetch one or more charm libraries.""" + name = 'fetch-lib' + help_msg = "Fetch one or more charm libraries" + overview = textwrap.dedent(""" + Fetch charm libraries. + + The first time a library is downloaded the command will create the needed + directories to place it, subsequent fetches will just update the local copy. + """) + + def fill_parser(self, parser): + """Add own parameters to the general parser.""" + parser.add_argument( + 'library', nargs='?', + help="Library to fetch (e.g. charms.mycharm.v2.foo.); optional, default to all") + + def run(self, parsed_args): + """Run the command.""" + if parsed_args.library: + local_libs_data = [_get_lib_info(full_name=parsed_args.library)] + else: + local_libs_data = _get_libs_from_tree() + + # get tips from the Store + store = Store() + to_query = [] + for lib in local_libs_data: + if lib.lib_id is None: + item = dict(charm_name=lib.charm_name, lib_name=lib.lib_name) + else: + item = dict(lib_id=lib.lib_id) + item['api'] = lib.api + to_query.append(item) + libs_tips = store.get_libraries_tips(to_query) + + # check if something needs to be done + to_fetch = [] + for lib_data in local_libs_data: + logger.debug("Verifying local lib %s", lib_data) + # fix any missing lib id using the Store info + if lib_data.lib_id is None: + for tip in libs_tips.values(): + if lib_data.charm_name == tip.charm_name and lib_data.lib_name == tip.lib_name: + lib_data = lib_data._replace(lib_id=tip.lib_id) + break + + tip = libs_tips.get((lib_data.lib_id, lib_data.api)) + logger.debug("Store tip: %s", tip) + if tip is None: + logger.info("Library %s not found in Charmhub.", lib_data.full_name) + continue + + if tip.patch > lib_data.patch: + # the store has a higher version than local + to_fetch.append(lib_data) + elif tip.patch < lib_data.patch: + # the store has a lower version numbers than local + logger.info( + "Library %s has local changes, can not be updated.", lib_data.full_name) + else: + # same versions locally and in the store + if tip.content_hash == lib_data.content_hash: + logger.info( + "Library %s was already up to date in version %d.%d.", + lib_data.full_name, tip.api, tip.patch) + else: + logger.info( + "Library %s has local changes, can not be updated.", lib_data.full_name) + + for lib_data in to_fetch: + downloaded = store.get_library(lib_data.charm_name, lib_data.lib_id, lib_data.api) + if lib_data.content is None: + # locally new + lib_data.path.parent.mkdir(parents=True, exist_ok=True) + lib_data.path.write_text(downloaded.content) + logger.info( + "Library %s version %d.%d downloaded.", + lib_data.full_name, downloaded.api, downloaded.patch) + else: + # XXX Facundo 2020-12-17: manage the case where the library was renamed + # (related GH issue: #214) + lib_data.path.write_text(downloaded.content) + logger.info( + "Library %s updated to version %d.%d.", + lib_data.full_name, downloaded.api, downloaded.patch) + + class ListLibCommand(BaseCommand): """List all libraries belonging to a charm.""" name = 'list-lib' diff --git a/charmcraft/commands/store/store.py b/charmcraft/commands/store/store.py index 96e5b13c0..0ef443590 100644 --- a/charmcraft/commands/store/store.py +++ b/charmcraft/commands/store/store.py @@ -211,6 +211,13 @@ def create_library_revision(self, charm_name, lib_id, api, patch, content, conte result = _build_library(response) return result + def get_library(self, charm_name, lib_id, api): + """Get the library tip by id for a given api version.""" + endpoint = '/v1/charm/libraries/{}/{}?api={}'.format(charm_name, lib_id, api) + response = self._client.get(endpoint) + result = _build_library(response) + return result + def get_libraries_tips(self, libraries): """Get the tip details for several libraries at once. diff --git a/charmcraft/main.py b/charmcraft/main.py index a2b402a97..e88336468 100644 --- a/charmcraft/main.py +++ b/charmcraft/main.py @@ -93,6 +93,7 @@ def run(self, parsed_args, all_commands): store.ReleaseCommand, store.StatusCommand, # libraries support store.CreateLibCommand, store.PublishLibCommand, store.ListLibCommand, + store.FetchLibCommand, ]), ] @@ -181,6 +182,7 @@ def _load_command(self, command_name, cmd_args): parser = CustomArgumentParser(prog=cmd.name) cmd.fill_parser(parser) parsed_args = parser.parse_args(cmd_args) + logger.debug("Command parsed sysargs: %s", parsed_args) return cmd, parsed_args diff --git a/tests/commands/test_store_api.py b/tests/commands/test_store_api.py index 7f9fa1cd8..c6fcfc12c 100644 --- a/tests/commands/test_store_api.py +++ b/tests/commands/test_store_api.py @@ -520,6 +520,41 @@ def test_create_library_revision(client_mock): assert result_lib.patch == test_patch +def test_get_library(client_mock): + """Get all the information (including content) for a library revision.""" + test_charm_name = 'test-charm-name' + test_lib_name = 'test-lib-name' + test_lib_id = 'test-lib-id' + test_api = 'test-api-version' + test_patch = 'test-patch-version' + test_content = 'test content with quite a lot of funny Python code :p' + test_hash = '1234' + + store = Store() + client_mock.get.return_value = { + 'api': test_api, + 'content': test_content, + 'hash': test_hash, + 'library-id': test_lib_id, + 'library-name': test_lib_name, + 'charm-name': test_charm_name, + 'patch': test_patch, + } + + result_lib = store.get_library(test_charm_name, test_lib_id, test_api) + + assert client_mock.mock_calls == [ + call.get('/v1/charm/libraries/test-charm-name/{}?api={}'.format(test_lib_id, test_api)), + ] + assert result_lib.api == test_api + assert result_lib.content == test_content + assert result_lib.content_hash == test_hash + assert result_lib.lib_id == test_lib_id + assert result_lib.lib_name == test_lib_name + assert result_lib.charm_name == test_charm_name + assert result_lib.patch == test_patch + + def test_get_tips_simple(client_mock): """Get info for a lib, simple case with successful result.""" test_charm_name = 'test-charm-name' diff --git a/tests/commands/test_store_commands.py b/tests/commands/test_store_commands.py index 62a78cfea..e5f328178 100644 --- a/tests/commands/test_store_commands.py +++ b/tests/commands/test_store_commands.py @@ -32,6 +32,7 @@ from charmcraft.commands.store import ( _get_lib_info, CreateLibCommand, + FetchLibCommand, ListLibCommand, ListNamesCommand, ListRevisionsCommand, @@ -1591,6 +1592,205 @@ def test_getlibinfo_libid_empty(tmp_path, monkeypatch): "Library {} metadata field LIBID must be a non-empty ASCII string.".format(test_path)) +# -- tests for fetch libraries command + +def test_fetchlib_simple_downloaded(caplog, store_mock, tmp_path, monkeypatch): + """Happy path fetching the lib for the first time (downloading it).""" + caplog.set_level(logging.INFO, logger="charmcraft.commands") + monkeypatch.chdir(tmp_path) + + lib_id = 'test-example-lib-id' + lib_content = 'some test content with uñicode ;)' + store_mock.get_libraries_tips.return_value = { + (lib_id, 0): Library( + lib_id=lib_id, content=None, content_hash='abc', api=0, patch=7, + lib_name='testlib', charm_name='testcharm'), + } + store_mock.get_library.return_value = Library( + lib_id=lib_id, content=lib_content, content_hash='abc', api=0, patch=7, + lib_name='testlib', charm_name='testcharm') + + FetchLibCommand('group').run(Namespace(library='charms.testcharm.v0.testlib')) + + assert store_mock.mock_calls == [ + call.get_libraries_tips( + [{'charm_name': 'testcharm', 'lib_name': 'testlib', 'api': 0}]), + call.get_library('testcharm', lib_id, 0), + ] + expected = "Library charms.testcharm.v0.testlib version 0.7 downloaded." + assert [expected] == [rec.message for rec in caplog.records] + saved_file = tmp_path / 'lib' / 'charms' / 'testcharm' / 'v0' / 'testlib.py' + assert saved_file.read_text() == lib_content + + +def test_fetchlib_simple_updated(caplog, store_mock, tmp_path, monkeypatch): + """Happy path fetching the lib for Nth time (updating it).""" + caplog.set_level(logging.INFO, logger="charmcraft.commands") + monkeypatch.chdir(tmp_path) + + lib_id = 'test-example-lib-id' + content, content_hash = factory.create_lib_filepath( + 'testcharm', 'testlib', api=0, patch=1, lib_id=lib_id) + + new_lib_content = 'some test content with uñicode ;)' + store_mock.get_libraries_tips.return_value = { + (lib_id, 0): Library( + lib_id=lib_id, content=None, content_hash='abc', api=0, patch=2, + lib_name='testlib', charm_name='testcharm'), + } + store_mock.get_library.return_value = Library( + lib_id=lib_id, content=new_lib_content, content_hash='abc', api=0, patch=2, + lib_name='testlib', charm_name='testcharm') + + FetchLibCommand('group').run(Namespace(library='charms.testcharm.v0.testlib')) + + assert store_mock.mock_calls == [ + call.get_libraries_tips([{'lib_id': lib_id, 'api': 0}]), + call.get_library('testcharm', lib_id, 0), + ] + expected = "Library charms.testcharm.v0.testlib updated to version 0.2." + assert [expected] == [rec.message for rec in caplog.records] + saved_file = tmp_path / 'lib' / 'charms' / 'testcharm' / 'v0' / 'testlib.py' + assert saved_file.read_text() == new_lib_content + + +def test_fetchlib_all(caplog, store_mock, tmp_path, monkeypatch): + """Update all the libraries found in disk.""" + caplog.set_level(logging.DEBUG, logger="charmcraft.commands") + monkeypatch.chdir(tmp_path) + + c1, h1 = factory.create_lib_filepath( + 'testcharm1', 'testlib1', api=0, patch=1, lib_id='lib_id_1') + c2, h2 = factory.create_lib_filepath( + 'testcharm2', 'testlib2', api=3, patch=5, lib_id='lib_id_2') + + store_mock.get_libraries_tips.return_value = { + ('lib_id_1', 0): Library( + lib_id='lib_id_1', content=None, content_hash='abc', api=0, patch=2, + lib_name='testlib1', charm_name='testcharm1'), + ('lib_id_2', 3): Library( + lib_id='lib_id_2', content=None, content_hash='def', api=3, patch=14, + lib_name='testlib2', charm_name='testcharm2'), + } + _store_libs_info = [ + Library( + lib_id='lib_id_1', content='new lib content 1', content_hash='xxx', api=0, patch=2, + lib_name='testlib1', charm_name='testcharm1'), + Library( + lib_id='lib_id_2', content='new lib content 2', content_hash='yyy', api=3, patch=14, + lib_name='testlib2', charm_name='testcharm2'), + ] + store_mock.get_library.side_effect = lambda *a: _store_libs_info.pop(0) + + FetchLibCommand('group').run(Namespace(library=None)) + + assert store_mock.mock_calls == [ + call.get_libraries_tips([ + {'lib_id': 'lib_id_1', 'api': 0}, + {'lib_id': 'lib_id_2', 'api': 3}, + ]), + call.get_library('testcharm1', 'lib_id_1', 0), + call.get_library('testcharm2', 'lib_id_2', 3), + ] + names = [ + 'charms.testcharm1.v0.testlib1', + 'charms.testcharm2.v3.testlib2', + ] + expected = [ + "Libraries found under lib/charms: " + str(names), + "Library charms.testcharm1.v0.testlib1 updated to version 0.2.", + "Library charms.testcharm2.v3.testlib2 updated to version 3.14.", + ] + + records = [rec.message for rec in caplog.records] + assert all(e in records for e in expected) + saved_file = tmp_path / 'lib' / 'charms' / 'testcharm1' / 'v0' / 'testlib1.py' + assert saved_file.read_text() == 'new lib content 1' + saved_file = tmp_path / 'lib' / 'charms' / 'testcharm2' / 'v3' / 'testlib2.py' + assert saved_file.read_text() == 'new lib content 2' + + +def test_fetchlib_store_not_found(caplog, store_mock): + """The indicated library is not found in the store.""" + caplog.set_level(logging.INFO, logger="charmcraft.commands") + + store_mock.get_libraries_tips.return_value = {} + FetchLibCommand('group').run(Namespace(library='charms.testcharm.v0.testlib')) + + assert store_mock.mock_calls == [ + call.get_libraries_tips( + [{'charm_name': 'testcharm', 'lib_name': 'testlib', 'api': 0}]), + ] + expected = "Library charms.testcharm.v0.testlib not found in Charmhub." + assert [expected] == [rec.message for rec in caplog.records] + + +def test_fetchlib_store_is_old(caplog, store_mock, tmp_path, monkeypatch): + """The store has an older version that what is found locally.""" + caplog.set_level(logging.DEBUG, logger="charmcraft.commands") + monkeypatch.chdir(tmp_path) + + lib_id = 'test-example-lib-id' + factory.create_lib_filepath('testcharm', 'testlib', api=0, patch=7, lib_id=lib_id) + + store_mock.get_libraries_tips.return_value = { + (lib_id, 0): Library( + lib_id=lib_id, content=None, content_hash='abc', api=0, patch=6, + lib_name='testlib', charm_name='testcharm'), + } + FetchLibCommand('group').run(Namespace(library='charms.testcharm.v0.testlib')) + + assert store_mock.mock_calls == [ + call.get_libraries_tips([{'lib_id': lib_id, 'api': 0}]), + ] + expected = "Library charms.testcharm.v0.testlib has local changes, can not be updated." + assert expected in [rec.message for rec in caplog.records] + + +def test_fetchlib_store_same_versions_same_hash(caplog, store_mock, tmp_path, monkeypatch): + """The store situation is the same than locally.""" + caplog.set_level(logging.DEBUG, logger="charmcraft.commands") + monkeypatch.chdir(tmp_path) + + lib_id = 'test-example-lib-id' + _, c_hash = factory.create_lib_filepath('testcharm', 'testlib', api=0, patch=7, lib_id=lib_id) + + store_mock.get_libraries_tips.return_value = { + (lib_id, 0): Library( + lib_id=lib_id, content=None, content_hash=c_hash, api=0, patch=7, + lib_name='testlib', charm_name='testcharm'), + } + FetchLibCommand('group').run(Namespace(library='charms.testcharm.v0.testlib')) + + assert store_mock.mock_calls == [ + call.get_libraries_tips([{'lib_id': lib_id, 'api': 0}]), + ] + expected = "Library charms.testcharm.v0.testlib was already up to date in version 0.7." + assert expected in [rec.message for rec in caplog.records] + + +def test_fetchlib_store_same_versions_differnt_hash(caplog, store_mock, tmp_path, monkeypatch): + """The store has the lib in the same version, but with different content.""" + caplog.set_level(logging.DEBUG, logger="charmcraft.commands") + monkeypatch.chdir(tmp_path) + + lib_id = 'test-example-lib-id' + factory.create_lib_filepath('testcharm', 'testlib', api=0, patch=7, lib_id=lib_id) + + store_mock.get_libraries_tips.return_value = { + (lib_id, 0): Library( + lib_id=lib_id, content=None, content_hash='abc', api=0, patch=7, + lib_name='testlib', charm_name='testcharm'), + } + FetchLibCommand('group').run(Namespace(library='charms.testcharm.v0.testlib')) + + assert store_mock.mock_calls == [ + call.get_libraries_tips([{'lib_id': lib_id, 'api': 0}]), + ] + expected = "Library charms.testcharm.v0.testlib has local changes, can not be updated." + assert expected in [rec.message for rec in caplog.records] + + # -- tests for list libraries command def test_listlib_simple(caplog, store_mock):