Merge pull request #215 from facundobatista/charmlibs-5

(charmlibs) Fetch the specified library

charmcraft/commands/store/__init__.py

@@ -827,6 +827,94 @@ def run(self, parsed_args):
                 lib_data.full_name, lib_data.api, lib_data.patch)
 
 
+class FetchLibCommand(BaseCommand):
+    """Fetch one or more charm libraries."""
+    name = 'fetch-lib'
+    help_msg = "Fetch one or more charm libraries"
+    overview = textwrap.dedent("""
+        Fetch charm libraries.
+
+        The first time a library is downloaded the command will create the needed
+        directories to place it, subsequent fetches will just update the local copy.
+    """)
+
+    def fill_parser(self, parser):
+        """Add own parameters to the general parser."""
+        parser.add_argument(
+            'library', nargs='?',
+            help="Library to fetch (e.g. charms.mycharm.v2.foo.); optional, default to all")
+
+    def run(self, parsed_args):
+        """Run the command."""
+        if parsed_args.library:
+            local_libs_data = [_get_lib_info(full_name=parsed_args.library)]
+        else:
+            local_libs_data = _get_libs_from_tree()
+
+        # get tips from the Store
+        store = Store()
+        to_query = []
+        for lib in local_libs_data:
+            if lib.lib_id is None:
+                item = dict(charm_name=lib.charm_name, lib_name=lib.lib_name)
+            else:
+                item = dict(lib_id=lib.lib_id)
+            item['api'] = lib.api
+            to_query.append(item)
+        libs_tips = store.get_libraries_tips(to_query)
+
+        # check if something needs to be done
+        to_fetch = []
+        for lib_data in local_libs_data:
+            logger.debug("Verifying local lib %s", lib_data)
+            # fix any missing lib id using the Store info
+            if lib_data.lib_id is None:
+                for tip in libs_tips.values():
+                    if lib_data.charm_name == tip.charm_name and lib_data.lib_name == tip.lib_name:
+                        lib_data = lib_data._replace(lib_id=tip.lib_id)
+                        break
+
+            tip = libs_tips.get((lib_data.lib_id, lib_data.api))
+            logger.debug("Store tip: %s", tip)
+            if tip is None:
+                logger.info("Library %s not found in Charmhub.", lib_data.full_name)
+                continue
+
+            if tip.patch > lib_data.patch:
+                # the store has a higher version than local
+                to_fetch.append(lib_data)
+            elif tip.patch < lib_data.patch:
+                # the store has a lower version numbers than local
+                logger.info(
+                    "Library %s has local changes, can not be updated.", lib_data.full_name)
+            else:
+                # same versions locally and in the store
+                if tip.content_hash == lib_data.content_hash:
+                    logger.info(
+                        "Library %s was already up to date in version %d.%d.",
+                        lib_data.full_name, tip.api, tip.patch)
+                else:
+                    logger.info(
+                        "Library %s has local changes, can not be updated.", lib_data.full_name)
+
+        for lib_data in to_fetch:
+            downloaded = store.get_library(lib_data.charm_name, lib_data.lib_id, lib_data.api)
+            if lib_data.content is None:
+                # locally new
+                lib_data.path.parent.mkdir(parents=True, exist_ok=True)
+                lib_data.path.write_text(downloaded.content)
+                logger.info(
+                    "Library %s version %d.%d downloaded.",
+                    lib_data.full_name, downloaded.api, downloaded.patch)
+            else:
+                # XXX Facundo 2020-12-17: manage the case where the library was renamed
+                # (related GH issue: #214)
+                lib_data.path.write_text(downloaded.content)
+                logger.info(
+                    "Library %s updated to version %d.%d.",
+                    lib_data.full_name, downloaded.api, downloaded.patch)
+
+
 class ListLibCommand(BaseCommand):
     """List all libraries belonging to a charm."""
     name = 'list-lib'

charmcraft/commands/store/store.py

@@ -211,6 +211,13 @@ def create_library_revision(self, charm_name, lib_id, api, patch, content, conte
         result = _build_library(response)
         return result
 
+    def get_library(self, charm_name, lib_id, api):
+        """Get the library tip by id for a given api version."""
+        endpoint = '/v1/charm/libraries/{}/{}?api={}'.format(charm_name, lib_id, api)
+        response = self._client.get(endpoint)
+        result = _build_library(response)
+        return result
+
     def get_libraries_tips(self, libraries):
         """Get the tip details for several libraries at once.
 

charmcraft/main.py

@@ -93,6 +93,7 @@ def run(self, parsed_args, all_commands):
         store.ReleaseCommand, store.StatusCommand,
         # libraries support
         store.CreateLibCommand, store.PublishLibCommand, store.ListLibCommand,
+        store.FetchLibCommand,
     ]),
 ]
 
@@ -181,6 +182,7 @@ def _load_command(self, command_name, cmd_args):
         parser = CustomArgumentParser(prog=cmd.name)
         cmd.fill_parser(parser)
         parsed_args = parser.parse_args(cmd_args)
+        logger.debug("Command parsed sysargs: %s", parsed_args)
 
         return cmd, parsed_args
 

tests/commands/test_store_api.py

@@ -520,6 +520,41 @@ def test_create_library_revision(client_mock):
     assert result_lib.patch == test_patch
 
 
+def test_get_library(client_mock):
+    """Get all the information (including content) for a library revision."""
+    test_charm_name = 'test-charm-name'
+    test_lib_name = 'test-lib-name'
+    test_lib_id = 'test-lib-id'
+    test_api = 'test-api-version'
+    test_patch = 'test-patch-version'
+    test_content = 'test content with quite a lot of funny Python code :p'
+    test_hash = '1234'
+
+    store = Store()
+    client_mock.get.return_value = {
+        'api': test_api,
+        'content': test_content,
+        'hash': test_hash,
+        'library-id': test_lib_id,
+        'library-name': test_lib_name,
+        'charm-name': test_charm_name,
+        'patch': test_patch,
+    }
+
+    result_lib = store.get_library(test_charm_name, test_lib_id, test_api)
+
+    assert client_mock.mock_calls == [
+        call.get('/v1/charm/libraries/test-charm-name/{}?api={}'.format(test_lib_id, test_api)),
+    ]
+    assert result_lib.api == test_api
+    assert result_lib.content == test_content
+    assert result_lib.content_hash == test_hash
+    assert result_lib.lib_id == test_lib_id
+    assert result_lib.lib_name == test_lib_name
+    assert result_lib.charm_name == test_charm_name
+    assert result_lib.patch == test_patch
+
+
 def test_get_tips_simple(client_mock):
     """Get info for a lib, simple case with successful result."""
     test_charm_name = 'test-charm-name'

tests/commands/test_store_commands.py

@@ -32,6 +32,7 @@
 from charmcraft.commands.store import (
     _get_lib_info,
     CreateLibCommand,
+    FetchLibCommand,
     ListLibCommand,
     ListNamesCommand,
     ListRevisionsCommand,
@@ -1591,6 +1592,205 @@ def test_getlibinfo_libid_empty(tmp_path, monkeypatch):
         "Library {} metadata field LIBID must be a non-empty ASCII string.".format(test_path))
 
 
+# -- tests for fetch libraries command
+
+def test_fetchlib_simple_downloaded(caplog, store_mock, tmp_path, monkeypatch):
+    """Happy path fetching the lib for the first time (downloading it)."""
+    caplog.set_level(logging.INFO, logger="charmcraft.commands")
+    monkeypatch.chdir(tmp_path)
+
+    lib_id = 'test-example-lib-id'
+    lib_content = 'some test content with uñicode ;)'
+    store_mock.get_libraries_tips.return_value = {
+        (lib_id, 0): Library(
+            lib_id=lib_id, content=None, content_hash='abc', api=0, patch=7,
+            lib_name='testlib', charm_name='testcharm'),
+    }
+    store_mock.get_library.return_value = Library(
+        lib_id=lib_id, content=lib_content, content_hash='abc', api=0, patch=7,
+        lib_name='testlib', charm_name='testcharm')
+
+    FetchLibCommand('group').run(Namespace(library='charms.testcharm.v0.testlib'))
+
+    assert store_mock.mock_calls == [
+        call.get_libraries_tips(
+            [{'charm_name': 'testcharm', 'lib_name': 'testlib', 'api': 0}]),
+        call.get_library('testcharm', lib_id, 0),
+    ]
+    expected = "Library charms.testcharm.v0.testlib version 0.7 downloaded."
+    assert [expected] == [rec.message for rec in caplog.records]
+    saved_file = tmp_path / 'lib' / 'charms' / 'testcharm' / 'v0' / 'testlib.py'
+    assert saved_file.read_text() == lib_content
+
+
+def test_fetchlib_simple_updated(caplog, store_mock, tmp_path, monkeypatch):
+    """Happy path fetching the lib for Nth time (updating it)."""
+    caplog.set_level(logging.INFO, logger="charmcraft.commands")
+    monkeypatch.chdir(tmp_path)
+
+    lib_id = 'test-example-lib-id'
+    content, content_hash = factory.create_lib_filepath(
+        'testcharm', 'testlib', api=0, patch=1, lib_id=lib_id)
+
+    new_lib_content = 'some test content with uñicode ;)'
+    store_mock.get_libraries_tips.return_value = {
+        (lib_id, 0): Library(
+            lib_id=lib_id, content=None, content_hash='abc', api=0, patch=2,
+            lib_name='testlib', charm_name='testcharm'),
+    }
+    store_mock.get_library.return_value = Library(
+        lib_id=lib_id, content=new_lib_content, content_hash='abc', api=0, patch=2,
+        lib_name='testlib', charm_name='testcharm')
+
+    FetchLibCommand('group').run(Namespace(library='charms.testcharm.v0.testlib'))
+
+    assert store_mock.mock_calls == [
+        call.get_libraries_tips([{'lib_id': lib_id, 'api': 0}]),
+        call.get_library('testcharm', lib_id, 0),
+    ]
+    expected = "Library charms.testcharm.v0.testlib updated to version 0.2."
+    assert [expected] == [rec.message for rec in caplog.records]
+    saved_file = tmp_path / 'lib' / 'charms' / 'testcharm' / 'v0' / 'testlib.py'
+    assert saved_file.read_text() == new_lib_content
+
+
+def test_fetchlib_all(caplog, store_mock, tmp_path, monkeypatch):
+    """Update all the libraries found in disk."""
+    caplog.set_level(logging.DEBUG, logger="charmcraft.commands")
+    monkeypatch.chdir(tmp_path)
+
+    c1, h1 = factory.create_lib_filepath(
+        'testcharm1', 'testlib1', api=0, patch=1, lib_id='lib_id_1')
+    c2, h2 = factory.create_lib_filepath(
+        'testcharm2', 'testlib2', api=3, patch=5, lib_id='lib_id_2')
+
+    store_mock.get_libraries_tips.return_value = {
+        ('lib_id_1', 0): Library(
+            lib_id='lib_id_1', content=None, content_hash='abc', api=0, patch=2,
+            lib_name='testlib1', charm_name='testcharm1'),
+        ('lib_id_2', 3): Library(
+            lib_id='lib_id_2', content=None, content_hash='def', api=3, patch=14,
+            lib_name='testlib2', charm_name='testcharm2'),
+    }
+    _store_libs_info = [
+        Library(
+            lib_id='lib_id_1', content='new lib content 1', content_hash='xxx', api=0, patch=2,
+            lib_name='testlib1', charm_name='testcharm1'),
+        Library(
+            lib_id='lib_id_2', content='new lib content 2', content_hash='yyy', api=3, patch=14,
+            lib_name='testlib2', charm_name='testcharm2'),
+    ]
+    store_mock.get_library.side_effect = lambda *a: _store_libs_info.pop(0)
+
+    FetchLibCommand('group').run(Namespace(library=None))
+
+    assert store_mock.mock_calls == [
+        call.get_libraries_tips([
+            {'lib_id': 'lib_id_1', 'api': 0},
+            {'lib_id': 'lib_id_2', 'api': 3},
+        ]),
+        call.get_library('testcharm1', 'lib_id_1', 0),
+        call.get_library('testcharm2', 'lib_id_2', 3),
+    ]
+    names = [
+        'charms.testcharm1.v0.testlib1',
+        'charms.testcharm2.v3.testlib2',
+    ]
+    expected = [
+        "Libraries found under lib/charms: " + str(names),
+        "Library charms.testcharm1.v0.testlib1 updated to version 0.2.",
+        "Library charms.testcharm2.v3.testlib2 updated to version 3.14.",
+    ]
+
+    records = [rec.message for rec in caplog.records]
+    assert all(e in records for e in expected)
+    saved_file = tmp_path / 'lib' / 'charms' / 'testcharm1' / 'v0' / 'testlib1.py'
+    assert saved_file.read_text() == 'new lib content 1'
+    saved_file = tmp_path / 'lib' / 'charms' / 'testcharm2' / 'v3' / 'testlib2.py'
+    assert saved_file.read_text() == 'new lib content 2'
+
+
+def test_fetchlib_store_not_found(caplog, store_mock):
+    """The indicated library is not found in the store."""
+    caplog.set_level(logging.INFO, logger="charmcraft.commands")
+
+    store_mock.get_libraries_tips.return_value = {}
+    FetchLibCommand('group').run(Namespace(library='charms.testcharm.v0.testlib'))
+
+    assert store_mock.mock_calls == [
+        call.get_libraries_tips(
+            [{'charm_name': 'testcharm', 'lib_name': 'testlib', 'api': 0}]),
+    ]
+    expected = "Library charms.testcharm.v0.testlib not found in Charmhub."
+    assert [expected] == [rec.message for rec in caplog.records]
+
+
+def test_fetchlib_store_is_old(caplog, store_mock, tmp_path, monkeypatch):
+    """The store has an older version that what is found locally."""
+    caplog.set_level(logging.DEBUG, logger="charmcraft.commands")
+    monkeypatch.chdir(tmp_path)
+
+    lib_id = 'test-example-lib-id'
+    factory.create_lib_filepath('testcharm', 'testlib', api=0, patch=7, lib_id=lib_id)
+
+    store_mock.get_libraries_tips.return_value = {
+        (lib_id, 0): Library(
+            lib_id=lib_id, content=None, content_hash='abc', api=0, patch=6,
+            lib_name='testlib', charm_name='testcharm'),
+    }
+    FetchLibCommand('group').run(Namespace(library='charms.testcharm.v0.testlib'))
+
+    assert store_mock.mock_calls == [
+        call.get_libraries_tips([{'lib_id': lib_id, 'api': 0}]),
+    ]
+    expected = "Library charms.testcharm.v0.testlib has local changes, can not be updated."
+    assert expected in [rec.message for rec in caplog.records]
+
+
+def test_fetchlib_store_same_versions_same_hash(caplog, store_mock, tmp_path, monkeypatch):
+    """The store situation is the same than locally."""
+    caplog.set_level(logging.DEBUG, logger="charmcraft.commands")
+    monkeypatch.chdir(tmp_path)
+
+    lib_id = 'test-example-lib-id'
+    _, c_hash = factory.create_lib_filepath('testcharm', 'testlib', api=0, patch=7, lib_id=lib_id)
+
+    store_mock.get_libraries_tips.return_value = {
+        (lib_id, 0): Library(
+            lib_id=lib_id, content=None, content_hash=c_hash, api=0, patch=7,
+            lib_name='testlib', charm_name='testcharm'),
+    }
+    FetchLibCommand('group').run(Namespace(library='charms.testcharm.v0.testlib'))
+
+    assert store_mock.mock_calls == [
+        call.get_libraries_tips([{'lib_id': lib_id, 'api': 0}]),
+    ]
+    expected = "Library charms.testcharm.v0.testlib was already up to date in version 0.7."
+    assert expected in [rec.message for rec in caplog.records]
+
+
+def test_fetchlib_store_same_versions_differnt_hash(caplog, store_mock, tmp_path, monkeypatch):
+    """The store has the lib in the same version, but with different content."""
+    caplog.set_level(logging.DEBUG, logger="charmcraft.commands")
+    monkeypatch.chdir(tmp_path)
+
+    lib_id = 'test-example-lib-id'
+    factory.create_lib_filepath('testcharm', 'testlib', api=0, patch=7, lib_id=lib_id)
+
+    store_mock.get_libraries_tips.return_value = {
+        (lib_id, 0): Library(
+            lib_id=lib_id, content=None, content_hash='abc', api=0, patch=7,
+            lib_name='testlib', charm_name='testcharm'),
+    }
+    FetchLibCommand('group').run(Namespace(library='charms.testcharm.v0.testlib'))
+
+    assert store_mock.mock_calls == [
+        call.get_libraries_tips([{'lib_id': lib_id, 'api': 0}]),
+    ]
+    expected = "Library charms.testcharm.v0.testlib has local changes, can not be updated."
+    assert expected in [rec.message for rec in caplog.records]
+
+
 # -- tests for list libraries command
 
 def test_listlib_simple(caplog, store_mock):