WIP: Add internal BitTorrent v2 support

src/tribler/core/libtorrent/download_manager/download.py

@@ -434,7 +434,8 @@ def on_save_resume_data_alert(self, alert: lt.save_resume_data_alert) -> None:
         self.config.set_engineresumedata(resume_data)
 
         # Save it to file
-        basename = hexlify(resume_data[b"info-hash"]).decode() + ".conf"
+        # Note resume_data[b"info-hash"] can be b"\x00" * 32, so we use the tdef.
+        basename = hexlify(self.tdef.get_infohash()).decode() + ".conf"
         Path(self.download_manager.get_checkpoint_dir()).mkdir(parents=True, exist_ok=True)
         filename = self.download_manager.get_checkpoint_dir() / basename
         self.config.config["download_defaults"]["name"] = self.tdef.get_name_as_unicode()  # store name (for debugging)
@@ -523,7 +524,7 @@ def on_metadata_received_alert(self, alert: lt.metadata_received_alert) -> None:
             metadata[b"announce"] = tracker_urls[0]
 
         try:
-            self.tdef = TorrentDef.load_from_dict(metadata)
+            self.set_def(TorrentDef.load_from_dict(metadata))
             with suppress(RuntimeError):
                 # Try to load the torrent info in the background if we have a loop.
                 get_running_loop().run_in_executor(None, self.tdef.load_torrent_info)
@@ -895,6 +896,11 @@ def set_def(self, tdef: TorrentDef) -> None:
         """
         Set the torrent definition for this download.
         """
+        if (isinstance(self.tdef, TorrentDefNoMetainfo) and not isinstance(tdef, TorrentDefNoMetainfo)
+                and len(self.tdef.infohash) != 20):
+            # We store SHA-1 conf files. v2 torrents start with SHA-256 infohashes.
+            basename = hexlify(self.tdef.get_infohash()).decode() + ".conf"
+            Path(self.download_manager.get_checkpoint_dir() / basename).unlink(missing_ok=True)
         self.tdef = tdef
 
     @check_handle(None)

src/tribler/core/libtorrent/download_manager/download_manager.py

@@ -1024,7 +1024,7 @@ async def load_checkpoints(self) -> None:
         Load the checkpoint files in the checkpoint directory.
         """
         self._logger.info("Load checkpoints...")
-        checkpoint_filenames = list(self.get_checkpoint_dir().glob("*.conf"))
+        checkpoint_filenames = sorted(self.get_checkpoint_dir().glob("*.conf"), key=lambda p: len(p.parts[-1]))
         self.checkpoints_count = len(checkpoint_filenames)
         for i, filename in enumerate(checkpoint_filenames, start=1):
             await self.load_checkpoint(filename)

src/tribler/core/libtorrent/download_manager/download_state.py

@@ -287,6 +287,12 @@ def get_files_completion(self) -> list[tuple[Path, float]]:
                 for index, (path, size) in enumerate(files):
                     completion_frac = (float(progress[index]) / size) if size > 0 else 1
                     completion.append((path, completion_frac))
+            elif progress and len(progress) > len(files) and self.download.tdef.torrent_info_loaded():
+                # We need to remap
+                remapping = self.download.tdef.get_file_indices()
+                for index, (path, size) in enumerate(files):
+                    completion_frac = (float(progress[remapping[index]]) / size) if size > 0 else 1
+                    completion.append((path, completion_frac))
 
         return completion
 

src/tribler/core/libtorrent/restapi/downloads_endpoint.py

@@ -158,13 +158,14 @@ def get_files_info_json(download: Download) -> list[JSONFilesInfo]:
         files_json = []
         files_completion = dict(download.get_state().get_files_completion())
         selected_files = download.config.get_selected_files()
+        index_mapping = download.get_def().get_file_indices()
         for file_index, (fn, size) in enumerate(download.get_def().get_files_with_length()):
             files_json.append(cast(JSONFilesInfo, {
-                "index": file_index,
+                "index": index_mapping[file_index],
                 # We always return files in Posix format to make GUI independent of Core and simplify testing
                 "name": str(PurePosixPath(fn)),
                 "size": size,
-                "included": (file_index in selected_files or not selected_files),
+                "included": (index_mapping[file_index] in selected_files or not selected_files),
                 "progress": files_completion.get(fn, 0.0)
             }))
         return files_json
@@ -581,8 +582,8 @@ async def update_download(self, request: Request) -> RESTResponse:  # noqa: C901
 
         if "selected_files" in parameters:
             selected_files_list = parameters["selected_files"]
-            num_files = len(download.tdef.get_files())
-            if not all(0 <= index < num_files for index in selected_files_list):
+            max_index = max(download.tdef.get_file_indices())
+            if not all(0 <= index <= max_index for index in selected_files_list):
                 return RESTResponse({"error": {
                                         "handled": True,
                                         "message": "index out of range"
@@ -1115,7 +1116,7 @@ async def stream(self, request: Request) -> web.StreamResponse:
             return DownloadsEndpoint.return_404()
 
         file_index = int(request.match_info["fileindex"])
-        if not 0 <= file_index < len(download.get_def().get_files()):
+        if not 0 <= file_index <= max(download.get_def().get_file_indices()):
             return DownloadsEndpoint.return_404()
 
         return TorrentStreamResponse(download, file_index)

src/tribler/core/libtorrent/restapi/torrentinfo_endpoint.py

@@ -4,9 +4,8 @@
 import logging
 from asyncio.exceptions import TimeoutError as AsyncTimeoutError
 from binascii import hexlify, unhexlify
-from copy import deepcopy
 from ssl import SSLError
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any, TypedDict
 
 import libtorrent as lt
 from aiohttp import (
@@ -25,7 +24,7 @@
 from yarl import URL
 
 from tribler.core.database.orm_bindings.torrent_metadata import tdef_to_metadata_dict
-from tribler.core.libtorrent.torrentdef import TorrentDef
+from tribler.core.libtorrent.torrentdef import TorrentDef, yield_v2_files
 from tribler.core.libtorrent.uris import unshorten, url_to_path
 from tribler.core.notifier import Notification
 from tribler.core.restapi.rest_endpoint import (
@@ -46,6 +45,16 @@
 logger = logging.getLogger(__name__)
 
 
+class JSONMiniFileInfo(TypedDict):
+    """
+    A minimal JSON dict to describe file info.
+    """
+
+    index: int
+    name: str
+    size: int
+
+
 def recursive_unicode(obj: Iterable, ignore_errors: bool = False) -> Iterable:
     """
     Converts any bytes within a data structure to unicode strings. Bytes are assumed to be UTF-8 encoded text.
@@ -102,6 +111,11 @@ def __init__(self, download_manager: DownloadManager) -> None:
         self.app.add_routes([web.get("", self.get_torrent_info),
                              web.put("", self.get_torrent_info_from_file)])
 
+    def get_files(self, tdef: TorrentDef) -> list[JSONMiniFileInfo]:
+        remapped_indices = tdef.get_file_indices()
+        return [{"index": remapped_indices[i], "name": str(f[0]), "size": f[1]}
+                for i, f in enumerate(tdef.get_files_with_length())]
+
     @docs(
         tags=["Libtorrent"],
         summary="Return metainfo from a torrent found at a provided URI.",
@@ -269,13 +283,8 @@ async def get_torrent_info(self, request: Request) -> RESTResponse:  # noqa: C90
         metainfo_download = metainfo_lookup.download if metainfo_lookup else None
         download_is_metainfo_request = download == metainfo_download
 
-        # Check if the torrent is already in the downloads
-        encoded_metainfo = deepcopy(metainfo)
-
-        ready_for_unicode = recursive_unicode(encoded_metainfo, ignore_errors=True)
-        json_dump = json.dumps(ready_for_unicode, ensure_ascii=False)
-
-        return RESTResponse({"metainfo": hexlify(json_dump.encode()).decode(),
+        return RESTResponse({"files": self.get_files(torrent_def),
+                             "name": torrent_def.get_name_utf8(),
                              "download_exists": download and not download_is_metainfo_request,
                              "valid_certificate": valid_cert})
 
@@ -302,8 +311,7 @@ async def get_torrent_info_from_file(self, request: web.Request) -> RESTResponse
         metainfo_download = metainfo_lookup.download if metainfo_lookup else None
         requesting_metainfo = download == metainfo_download
 
-        metainfo_unicode = recursive_unicode(deepcopy(tdef.get_metainfo()), ignore_errors=True)
-        metainfo_json = json.dumps(metainfo_unicode, ensure_ascii=False)
         return RESTResponse({"infohash": hexlify(infohash).decode(),
-                             "metainfo": hexlify(metainfo_json.encode('utf-8')).decode(),
+                             "files": self.get_files(tdef),
+                             "name": tdef.get_name_utf8(),
                              "download_exists": download and not requesting_metainfo})