Feature/update resolution (#17642)

* update resolution new conditional logic

* failing test

* made it work for the 2 remotes case too

* deprecation warning

Author: memsharded

conan/internal/conan_app.py

@@ -8,7 +8,6 @@
 from conans.client.loader import ConanFileLoader, load_python_file
 from conans.client.remote_manager import RemoteManager
 from conans.client.rest.auth_manager import ConanApiAuthManager
-from conans.client.rest.conan_requester import ConanRequester
 from conan.internal.api.remotes.localdb import LocalDB
 
 

conan/internal/model/conf.py

@@ -28,6 +28,7 @@
     "core.download:retry_wait": "Seconds to wait between download attempts from Conan server",
     "core.download:download_cache": "Define path to a file download cache",
     "core.cache:storage_path": "Absolute path where the packages and database are stored",
+    "core:update_policy": "(Legacy). If equal 'legacy' when multiple remotes, update based on order of remotes, only the timestamp of the first occurrence of each revision counts.",
     # Sources backup
     "core.sources:download_cache": "Folder to store the sources backup",
     "core.sources:download_urls": "List of URLs to download backup sources from",

conan/test/utils/tools.py

@@ -11,7 +11,6 @@
 import traceback
 import uuid
 import zipfile
-from collections import OrderedDict
 from contextlib import contextmanager
 from inspect import getframeinfo, stack
 from urllib.parse import urlsplit, urlunsplit
@@ -416,15 +415,6 @@ def __init__(self, cache_folder=None, current_folder=None, servers=None, inputs=
         self.cache_folder = cache_folder or os.path.join(temp_folder(path_with_spaces), ".conan2")
 
         self.requester_class = requester_class
-
-        if servers and len(servers) > 1 and not isinstance(servers, OrderedDict):
-            raise Exception(textwrap.dedent("""
-                Testing framework error: Servers should be an OrderedDict. e.g:
-                    servers = OrderedDict()
-                    servers["r1"] = server
-                    servers["r2"] = TestServer()
-            """))
-
         self.servers = servers or {}
         if servers is not False:  # Do not mess with registry remotes
             self.update_servers()

conans/client/graph/proxy.py

@@ -14,6 +14,7 @@ def __init__(self, conan_app, editable_packages):
         self._cache = conan_app.cache
         self._remote_manager = conan_app.remote_manager
         self._resolved = {}  # Cache of the requested recipes to optimize calls
+        self._legacy_update = conan_app.global_conf.get("core:update_policy", choices=["legacy"])
 
     def get_recipe(self, ref, remotes, update, check_update):
         """
@@ -97,18 +98,27 @@ def _find_newest_recipe_in_remotes(self, reference, remotes, update, check_updat
         output = ConanOutput(scope=str(reference))
 
         results = []
+        need_update = should_update_reference(reference, update) or check_update
         for remote in remotes:
             if remote.allowed_packages and not any(reference.matches(f, is_consumer=False)
                                                    for f in remote.allowed_packages):
                 output.debug(f"Excluding remote {remote.name} because recipe is filtered out")
                 continue
             output.info(f"Checking remote: {remote.name}")
             try:
+                if self._legacy_update and need_update:
+                    if not getattr(ConanProxy, "update_policy_legacy_warning", None):
+                        ConanProxy.update_policy_legacy_warning = True
+                        ConanOutput().warning("The 'core:update_policy' conf is deprecated and will "
+                                              "be removed in future versions", warn_tag="deprecated")
+                    refs = self._remote_manager.get_recipe_revisions_references(reference, remote)
+                    results.extend([{'remote': remote, 'ref': ref} for ref in refs])
+                    continue
                 if not reference.revision:
                     ref = self._remote_manager.get_latest_recipe_reference(reference, remote)
                 else:
                     ref = self._remote_manager.get_recipe_revision_reference(reference, remote)
-                if not should_update_reference(reference, update) and not check_update:
+                if not need_update:
                     return remote, ref
                 results.append({'remote': remote, 'ref': ref})
             except NotFoundException:
@@ -117,6 +127,17 @@ def _find_newest_recipe_in_remotes(self, reference, remotes, update, check_updat
         if len(results) == 0:
             return None, None
 
+        if self._legacy_update and need_update:
+            # Use only the first occurence of each revision in the remotes
+            filtered_results = []
+            revisions = set()
+            for r in results:
+                ref = r["ref"]
+                if ref.revision not in revisions:
+                    revisions.add(ref.revision)
+                    filtered_results.append(r)
+            results = filtered_results
+
         remotes_results = sorted(results, key=lambda k: k['ref'].timestamp, reverse=True)
         # get the latest revision from all remotes
         found_rrev = remotes_results[0]

test/integration/command/install/install_update_test.py

@@ -3,7 +3,7 @@
 from time import sleep
 
 from conan.api.model import RecipeReference
-from conan.test.utils.tools import TestClient, GenConanfile
+from conan.test.utils.tools import TestClient, GenConanfile, TestServer
 from conans.util.files import load
 
 
@@ -165,3 +165,71 @@ def test_install_update_repeated_tool_requires():
     c.run("create libc")
     c.run("install libc --update -pr=profile")
     assert 1 == str(c.out).count("tool/0.1: Checking remote")
+
+
+class TestUpdateOldPolicy:
+    def test_multi_remote_update_resolution(self):
+        c = TestClient(servers={"r1": TestServer(), "r2": TestServer(), "r3": TestServer()},
+                       inputs=["admin", "password"] * 3, light=True)
+        c.save({"conanfile.py": GenConanfile("pkg", "0.1")})
+        c.run("export .")
+        rev1 = c.exported_recipe_revision()
+        c.run("upload * -r=r1 -c")
+        # second revision
+        c.save({"conanfile.py": GenConanfile("pkg", "0.1").with_class_attribute("auther = 'me'")})
+        c.run("export .")
+        rev2 = c.exported_recipe_revision()
+        assert rev1 != rev2
+        c.run("upload * -r=r2 -c")  # By default uploads latest revisions only
+        assert rev1 not in c.out
+        assert rev2 in c.out
+        # going back to the previous revision
+        c.save({"conanfile.py": GenConanfile("pkg", "0.1")})
+        c.run("export .")  # Makes it the latest
+        rev3 = c.exported_recipe_revision()
+        assert rev1 == rev3
+        c.run("upload * -r=r3 -c")  # By default uploads latest revisions only
+        assert rev3 in c.out
+        assert rev2 not in c.out
+
+        # now test the --update, it will pick up the latest revision, which is r3
+        c.run("remove * -c")
+        c.run("graph info --requires=pkg/0.1 --update")
+        assert f"pkg/0.1#{rev3} - Downloaded (r3)" in c.out
+
+        # But if we enable order-based first found timestamp, it will pick up r2
+        c.run("remove * -c")
+        c.run("graph info --requires=pkg/0.1 --update -cc core:update_policy=legacy")
+        assert "The 'core:update_policy' conf is deprecated and will be removed" in c.out
+        assert f"pkg/0.1#{rev2} - Downloaded (r2)" in c.out
+
+    def test_multi_remote_update_resolution_2_remotes(self):
+        c = TestClient(servers={"r1": TestServer(), "r2": TestServer()},
+                       inputs=["admin", "password"] * 2, light=True)
+        c.save({"conanfile.py": GenConanfile("pkg", "0.1")})
+        c.run("export .")
+        rev1 = c.exported_recipe_revision()
+        c.run("upload * -r=r1 -c")
+        # second revision
+        c.save({"conanfile.py": GenConanfile("pkg", "0.1").with_class_attribute("auther = 'me'")})
+        c.run("export .")
+        rev2 = c.exported_recipe_revision()
+        assert rev1 != rev2
+        c.run("upload * -r=r1 -c")
+        c.run("list *#* -r=r1")
+
+        # Upload the first, old revision to the other remote
+        c.run(f"upload pkg/0.1#{rev1} -r=r2 -c")
+        assert rev1 in c.out
+        assert rev2 not in c.out
+        c.run("list *#* -r=r2")
+
+        # now test the --update, it will pick up the latest revision, which is r3
+        c.run("remove * -c")
+        c.run("graph info --requires=pkg/0.1 --update")
+        assert f"pkg/0.1#{rev1} - Downloaded (r2)" in c.out
+
+        # But if we enable order-based first found timestamp, it will pick up r2
+        c.run("remove * -c")
+        c.run("graph info --requires=pkg/0.1 --update -cc core:update_policy=legacy")
+        assert f"pkg/0.1#{rev2} - Downloaded (r1)" in c.out