binrepos.conf: Support custom download location

Download packages to a custom location if it is configured
in binrepos.conf, instead of PKGDIR. If a custom download
location is not configured then inject downloaded packages
into PKGDIR as usual.

The binarytree download_required method should now be used
instead of the isremote method to check if download is
required, since a remote package may or may not be cached
in the custom location. The get_local_repo_location method
should be used to check if there is a custom download
location, and if there is then downloaded packages must
not be injected into PKGDIR.

If any packages from a repo with a custom download location
were injected into PKGDIR in the past, their identity will
be recognized and will not be re-downloaded to the custom
location.

Bug: https://bugs.gentoo.org/934784
Signed-off-by: Zac Medico <zmedico@gentoo.org>

9 files changed, 106 insertions, 12 deletions

diff --git a/lib/_emerge/Binpkg.py b/lib/_emerge/Binpkg.py
index 299ae7fbc..437111fa1 100644
--- a/lib/_emerge/Binpkg.py
+++ b/lib/_emerge/Binpkg.py
@@ -170,7 +170,7 @@ class Binpkg(CompositeTask):
         pkg_count = self.pkg_count
         fetcher = None
 
-        if self.opts.getbinpkg and self._bintree.isremote(pkg.cpv):
+        if self.opts.getbinpkg and self._bintree.download_required(pkg.cpv):
             fetcher = BinpkgFetcher(
                 background=self.background,
                 logfile=self.settings.get("PORTAGE_LOG_FILE"),
@@ -245,7 +245,10 @@ class Binpkg(CompositeTask):
         pkg = self.pkg
         pkg_count = self.pkg_count
 
-        if self._fetched_pkg:
+        if self._fetched_pkg and self._bintree.get_local_repo_location(pkg.cpv):
+            os.rename(self._fetched_pkg, self._pkg_allocated_path)
+            pkg_path = self._pkg_allocated_path
+        elif self._fetched_pkg:
             stdout_orig = sys.stdout
             stderr_orig = sys.stderr
             out = io.StringIO()
diff --git a/lib/_emerge/BinpkgFetcher.py b/lib/_emerge/BinpkgFetcher.py
index 19d08359f..a357bac82 100644
--- a/lib/_emerge/BinpkgFetcher.py
+++ b/lib/_emerge/BinpkgFetcher.py
@@ -34,8 +34,14 @@ class BinpkgFetcher(CompositeTask):
             )
         binpkg_format = get_binpkg_format(binpkg_path)
 
+        getname_kwargs = {}
+        if not bintree.get_local_repo_location(pkg.cpv):
+            getname_kwargs.update(
+                dict(allocate_new=True, remote_binpkg_format=binpkg_format)
+            )
+
         self.pkg_allocated_path = pkg.root_config.trees["bintree"].getname(
-            pkg.cpv, allocate_new=True, remote_binpkg_format=binpkg_format
+            pkg.cpv, **getname_kwargs
         )
         self.pkg_path = self.pkg_allocated_path + ".partial"
 
diff --git a/lib/_emerge/BinpkgPrefetcher.py b/lib/_emerge/BinpkgPrefetcher.py
index a8af30ca8..f7204bcc1 100644
--- a/lib/_emerge/BinpkgPrefetcher.py
+++ b/lib/_emerge/BinpkgPrefetcher.py
@@ -51,6 +51,13 @@ class BinpkgPrefetcher(CompositeTask):
             self.wait()
             return
 
+        if self._bintree.get_local_repo_location(self.pkg.cpv):
+            os.rename(self.pkg_path, self.pkg_allocated_path)
+            self._current_task = None
+            self.returncode = os.EX_OK
+            self.wait()
+            return
+
         injected_pkg = None
         stdout_orig = sys.stdout
         stderr_orig = sys.stderr
diff --git a/lib/_emerge/Scheduler.py b/lib/_emerge/Scheduler.py
index 614df9e78..e23ebeb7a 100644
--- a/lib/_emerge/Scheduler.py
+++ b/lib/_emerge/Scheduler.py
@@ -830,7 +830,7 @@ class Scheduler(PollScheduler):
         elif (
             pkg.type_name == "binary"
             and "--getbinpkg" in self.myopts
-            and pkg.root_config.trees["bintree"].isremote(pkg.cpv)
+            and pkg.root_config.trees["bintree"].download_required(pkg.cpv)
         ):
             prefetcher = BinpkgPrefetcher(
                 background=True, pkg=pkg, scheduler=self._sched_iface
@@ -939,7 +939,7 @@ class Scheduler(PollScheduler):
 
                     # Display fetch on stdout, so that it's always clear what
                     # is consuming time here.
-                    if bintree.isremote(x.cpv):
+                    if bintree.download_required(x.cpv):
                         fetcher = self._get_prefetcher(x)
                         if fetcher is not None and not fetcher.isAlive():
                             # Cancel it because it hasn't started yet.
@@ -983,7 +983,9 @@ class Scheduler(PollScheduler):
                         continue
 
                     current_task = None
-                    if fetched:
+                    if fetched and bintree.get_local_repo_location(x.cpv):
+                        os.rename(fetched, fetcher.pkg_allocated_path)
+                    elif fetched:
                         if not bintree.inject(
                             x.cpv,
                             current_pkg_path=fetched,
diff --git a/lib/_emerge/actions.py b/lib/_emerge/actions.py
index 43d936fd1..455444599 100644
--- a/lib/_emerge/actions.py
+++ b/lib/_emerge/actions.py
@@ -1882,7 +1882,7 @@ def action_info(settings, trees, myopts, myfiles):
             matches.reverse()
             for match in matches:
                 if pkg_type == "binary":
-                    if db.bintree.isremote(match):
+                    if db.bintree.download_required(match):
                         continue
                 auxkeys = ["EAPI", "DEFINED_PHASES"]
                 metadata = dict(zip(auxkeys, db.aux_get(match, auxkeys)))
diff --git a/lib/portage/binrepo/config.py b/lib/portage/binrepo/config.py
index c744d1012..97207eb24 100644
--- a/lib/portage/binrepo/config.py
+++ b/lib/portage/binrepo/config.py
@@ -16,6 +16,7 @@ class BinRepoConfig:
         "name",
         "name_fallback",
         "fetchcommand",
+        "location",
         "priority",
         "resumecommand",
         "sync_uri",
@@ -40,6 +41,8 @@ class BinRepoConfig:
         indent = " " * 4
         repo_msg = []
         repo_msg.append(self.name or self.name_fallback)
+        if self.location:
+            repo_msg.append(indent + "location: " + self.location)
         if self.priority is not None:
             repo_msg.append(indent + "priority: " + str(self.priority))
         repo_msg.append(indent + "sync-uri: " + self.sync_uri)
diff --git a/lib/portage/dbapi/bintree.py b/lib/portage/dbapi/bintree.py
index b32dea1ea..6099bab96 100644
--- a/lib/portage/dbapi/bintree.py
+++ b/lib/portage/dbapi/bintree.py
@@ -468,7 +468,7 @@ class bindbapi(fakedbapi):
         pkg = getattr(pkg, "cpv", pkg)
 
         filesdict = {}
-        if not self.bintree.isremote(pkg):
+        if not self.bintree.download_required(pkg):
             pass
         else:
             metadata = self.bintree._remotepkgs[self._instance_key(pkg)]
@@ -1630,7 +1630,11 @@ class binarytree:
                 remote_base_uri = pkgindex.header.get("URI", base_url)
                 for d in pkgindex.packages:
                     cpv = _pkg_str(
-                        d["CPV"], metadata=d, settings=self.settings, db=self.dbapi
+                        d["CPV"],
+                        metadata=d,
+                        settings=self.settings,
+                        db=self.dbapi,
+                        repoconfig=repo,
                     )
                     # Local package instances override remote instances
                     # with the same instance_key.
@@ -2265,6 +2269,14 @@ class binarytree:
             path = self._pkg_paths.get(instance_key)
             if path is not None:
                 filename = os.path.join(self.pkgdir, path)
+            elif instance_key in self._remotepkgs:
+                remote_metadata = self._remotepkgs[instance_key]
+                location = self.get_local_repo_location(cpv)
+                if location:
+                    return (
+                        os.path.join(location, remote_metadata["PATH"]),
+                        int(remote_metadata["BUILD_ID"]),
+                    )
 
         if filename is None and not allocate_new:
             try:
@@ -2422,7 +2434,10 @@ class binarytree:
 
     def isremote(self, pkgname):
         """Returns true if the package is kept remotely and it has not been
-        downloaded (or it is only partially downloaded)."""
+        downloaded (or it is only partially downloaded), or if the package
+        is cached in a binrepo location (use download_required to check if
+        cached file has correct size and mtime).
+        """
         if self._remotepkgs is None:
             return False
         instance_key = self.dbapi._instance_key(pkgname)
@@ -2434,6 +2449,55 @@ class binarytree:
         # package is downloaded, state is updated by self.inject().
         return True
 
+    def download_required(self, pkgname):
+        """Returns True if package is remote and download is required."""
+        if not self._remotepkgs:
+            return False
+
+        instance_key = self.dbapi._instance_key(pkgname)
+        remote_metadata = self._remotepkgs.get(instance_key)
+        if remote_metadata is None:
+            return False
+
+        if not remote_metadata["CPV"]._repoconfig.location:
+            # In this case the package would have been removed from
+            # self._remotepkgs if it was already downloaded.
+            return True
+
+        pkg_path = self.getname(pkgname)
+        try:
+            st = os.stat(pkg_path)
+        except OSError:
+            return True
+
+        return (
+            int(remote_metadata["SIZE"]) != st.st_size
+            or int(remote_metadata["_mtime_"]) != st[stat.ST_MTIME]
+        )
+
+    def get_local_repo_location(self, pkgname):
+        """Returns local repo location associated with pkgname or None
+        if a location is not associated."""
+        # Since pkgname._repoconfig is not guaranteed to be present
+        # here, retrieve it from the remote metadata.
+        if not self._remotepkgs:
+            return None
+        instance_key = self.dbapi._instance_key(pkgname)
+        remote_metadata = self._remotepkgs.get(instance_key)
+        if remote_metadata is None:
+            return False
+        repoconfig = remote_metadata["CPV"]._repoconfig
+        if repoconfig is None:
+            return None
+        if repoconfig.location:
+            location = normalize_path(repoconfig.location)
+            if location == self.pkgdir:
+                # If the cache location is set to the same location as
+                # PKGDIR, then behave as though it is unset so that
+                # packages will be correctly injected into PKGDIR.
+                return None
+        return repoconfig.location
+
     def get_pkgindex_uri(self, cpv):
         """Returns the URI to the Packages file for a given package."""
         uri = None
diff --git a/lib/portage/versions.py b/lib/portage/versions.py
index 0e515ba5c..c7eb91b38 100644
--- a/lib/portage/versions.py
+++ b/lib/portage/versions.py
@@ -1,5 +1,5 @@
 # versions.py -- core Portage functionality
-# Copyright 1998-2023 Gentoo Authors
+# Copyright 1998-2024 Gentoo Authors
 # Distributed under the terms of the GNU General Public License v2
 
 __all__ = [
@@ -386,6 +386,7 @@ class _pkg_str(str):
         file_size: Optional[int] = None,
         mtime: Optional[int] = None,
         db: Any = None,
+        repoconfig: Any = None,
     ):
         return str.__new__(cls, cpv)
 
@@ -402,6 +403,7 @@ class _pkg_str(str):
         file_size: Optional[int] = None,
         mtime: Optional[int] = None,
         db: Any = None,
+        repoconfig: Any = None,
     ):
         if not isinstance(cpv, str):
             # Avoid TypeError from str.__init__ with PyPy.
@@ -420,6 +422,8 @@ class _pkg_str(str):
             self.__dict__["_settings"] = settings
         if db is not None:
             self.__dict__["_db"] = db
+        if repoconfig is not None:
+            self.__dict__["_repoconfig"] = repoconfig
         if eapi is not None:
             self.__dict__["eapi"] = eapi
 
diff --git a/man/portage.5 b/man/portage.5
index 66437d8f8..f5cbe0ad4 100644
--- a/man/portage.5
+++ b/man/portage.5
@@ -1,4 +1,4 @@
-.TH "PORTAGE" "5" "May 2024" "Portage @VERSION@" "Portage"
+.TH "PORTAGE" "5" "Jul 2024" "Portage @VERSION@" "Portage"
 .SH NAME
 portage \- the heart of Gentoo
 .SH "DESCRIPTION"
@@ -667,6 +667,11 @@ overriding the value from \fBmake.conf\fR(5).
 Specifies priority of given repository. When a package exists in multiple
 repositories, those with higher priority are preferred.
 .TP
+.B location
+Specify a cache location which serves as a partial local mirror of a
+remote repository. If unset then the effective default is PKGDIR
+(see \fBmake.conf\fR(5)).
+.TP
 .B sync\-uri
 Specifies URI of repository used for `emerge \-\-getbinpkg`.
 .RE