pym/portage/dbapi/bintree.py

   1 # Copyright 1998-2013 Gentoo Foundation
   2 # Distributed under the terms of the GNU General Public License v2
   3
   4 from __future__ import unicode_literals
   5
   6 __all__ = ["bindbapi", "binarytree"]
   7
   8 import portage
   9 portage.proxy.lazyimport.lazyimport(globals(),
  10         'portage.checksum:hashfunc_map,perform_multiple_checksums,' + \
  11                 'verify_all,_apply_hash_filter,_hash_filter',
  12         'portage.dbapi.dep_expand:dep_expand',
  13         'portage.dep:dep_getkey,isjustname,isvalidatom,match_from_list',
  14         'portage.output:EOutput,colorize',
  15         'portage.locks:lockfile,unlockfile',
  16         'portage.package.ebuild.fetch:_check_distfile,_hide_url_passwd',
  17         'portage.update:update_dbentries',
  18         'portage.util:atomic_ofstream,ensure_dirs,normalize_path,' + \
  19                 'writemsg,writemsg_stdout',
  20         'portage.util.listdir:listdir',
  21         'portage.util._urlopen:urlopen@_urlopen',
  22         'portage.versions:best,catpkgsplit,catsplit,_pkg_str',
  23 )
  24
  25 from portage.cache.mappings import slot_dict_class
  26 from portage.const import CACHE_PATH
  27 from portage.dbapi.virtual import fakedbapi
  28 from portage.dep import Atom, use_reduce, paren_enclose
  29 from portage.exception import AlarmSignal, InvalidData, InvalidPackageName, \
  30         PermissionDenied, PortageException
  31 from portage.localization import _
  32 from portage import _movefile
  33 from portage import os
  34 from portage import _encodings
  35 from portage import _unicode_decode
  36 from portage import _unicode_encode
  37
  38 import codecs
  39 import errno
  40 import io
  41 import stat
  42 import subprocess
  43 import sys
  44 import tempfile
  45 import textwrap
  46 import traceback
  47 import warnings
  48 from gzip import GzipFile
  49 from itertools import chain
  50 try:
  51         from urllib.parse import urlparse
  52 except ImportError:
  53         from urlparse import urlparse
  54
  55 if sys.hexversion >= 0x3000000:
  56         _unicode = str
  57         basestring = str
  58         long = int
  59 else:
  60         _unicode = unicode
  61
  62 class UseCachedCopyOfRemoteIndex(Exception):
  63         # If the local copy is recent enough
  64         # then fetching the remote index can be skipped.
  65         pass
  66
  67 class bindbapi(fakedbapi):
  68         _known_keys = frozenset(list(fakedbapi._known_keys) + \
  69                 ["CHOST", "repository", "USE"])
  70         def __init__(self, mybintree=None, **kwargs):
  71                 fakedbapi.__init__(self, **kwargs)
  72                 self.bintree = mybintree
  73                 self.move_ent = mybintree.move_ent
  74                 self.cpvdict={}
  75                 self.cpdict={}
  76                 # Selectively cache metadata in order to optimize dep matching.
  77                 self._aux_cache_keys = set(
  78                         ["BUILD_TIME", "CHOST", "DEPEND", "EAPI",
  79                         "HDEPEND", "IUSE", "KEYWORDS",
  80                         "LICENSE", "PDEPEND", "PROPERTIES", "PROVIDE",
  81                         "RDEPEND", "repository", "RESTRICT", "SLOT", "USE", "DEFINED_PHASES"
  82                         ])
  83                 self._aux_cache_slot_dict = slot_dict_class(self._aux_cache_keys)
  84                 self._aux_cache = {}
  85
  86         def match(self, *pargs, **kwargs):
  87                 if self.bintree and not self.bintree.populated:
  88                         self.bintree.populate()
  89                 return fakedbapi.match(self, *pargs, **kwargs)
  90
  91         def cpv_exists(self, cpv, myrepo=None):
  92                 if self.bintree and not self.bintree.populated:
  93                         self.bintree.populate()
  94                 return fakedbapi.cpv_exists(self, cpv)
  95
  96         def cpv_inject(self, cpv, **kwargs):
  97                 self._aux_cache.pop(cpv, None)
  98                 fakedbapi.cpv_inject(self, cpv, **kwargs)
  99
 100         def cpv_remove(self, cpv):
 101                 self._aux_cache.pop(cpv, None)
 102                 fakedbapi.cpv_remove(self, cpv)
 103
 104         def aux_get(self, mycpv, wants, myrepo=None):
 105                 if self.bintree and not self.bintree.populated:
 106                         self.bintree.populate()
 107                 cache_me = False
 108                 if not self._known_keys.intersection(
 109                         wants).difference(self._aux_cache_keys):
 110                         aux_cache = self._aux_cache.get(mycpv)
 111                         if aux_cache is not None:
 112                                 return [aux_cache.get(x, "") for x in wants]
 113                         cache_me = True
 114                 mysplit = mycpv.split("/")
 115                 mylist = []
 116                 tbz2name = mysplit[1]+".tbz2"
 117                 if not self.bintree._remotepkgs or \
 118                         not self.bintree.isremote(mycpv):
 119                         tbz2_path = self.bintree.getname(mycpv)
 120                         if not os.path.exists(tbz2_path):
 121                                 raise KeyError(mycpv)
 122                         metadata_bytes = portage.xpak.tbz2(tbz2_path).get_data()
 123                         def getitem(k):
 124                                 v = metadata_bytes.get(_unicode_encode(k,
 125                                         encoding=_encodings['repo.content'],
 126                                         errors='backslashreplace'))
 127                                 if v is not None:
 128                                         v = _unicode_decode(v,
 129                                                 encoding=_encodings['repo.content'], errors='replace')
 130                                 return v
 131                 else:
 132                         getitem = self.bintree._remotepkgs[mycpv].get
 133                 mydata = {}
 134                 mykeys = wants
 135                 if cache_me:
 136                         mykeys = self._aux_cache_keys.union(wants)
 137                 for x in mykeys:
 138                         myval = getitem(x)
 139                         # myval is None if the key doesn't exist
 140                         # or the tbz2 is corrupt.
 141                         if myval:
 142                                 mydata[x] = " ".join(myval.split())
 143
 144                 if not mydata.setdefault('EAPI', '0'):
 145                         mydata['EAPI'] = '0'
 146
 147                 if cache_me:
 148                         aux_cache = self._aux_cache_slot_dict()
 149                         for x in self._aux_cache_keys:
 150                                 aux_cache[x] = mydata.get(x, '')
 151                         self._aux_cache[mycpv] = aux_cache
 152                 return [mydata.get(x, '') for x in wants]
 153
 154         def aux_update(self, cpv, values):
 155                 if not self.bintree.populated:
 156                         self.bintree.populate()
 157                 tbz2path = self.bintree.getname(cpv)
 158                 if not os.path.exists(tbz2path):
 159                         raise KeyError(cpv)
 160                 mytbz2 = portage.xpak.tbz2(tbz2path)
 161                 mydata = mytbz2.get_data()
 162
 163                 for k, v in values.items():
 164                         k = _unicode_encode(k,
 165                                 encoding=_encodings['repo.content'], errors='backslashreplace')
 166                         v = _unicode_encode(v,
 167                                 encoding=_encodings['repo.content'], errors='backslashreplace')
 168                         mydata[k] = v
 169
 170                 for k, v in list(mydata.items()):
 171                         if not v:
 172                                 del mydata[k]
 173                 mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
 174                 # inject will clear stale caches via cpv_inject.
 175                 self.bintree.inject(cpv)
 176
 177         def cp_list(self, *pargs, **kwargs):
 178                 if not self.bintree.populated:
 179                         self.bintree.populate()
 180                 return fakedbapi.cp_list(self, *pargs, **kwargs)
 181
 182         def cp_all(self):
 183                 if not self.bintree.populated:
 184                         self.bintree.populate()
 185                 return fakedbapi.cp_all(self)
 186
 187         def cpv_all(self):
 188                 if not self.bintree.populated:
 189                         self.bintree.populate()
 190                 return fakedbapi.cpv_all(self)
 191
 192         def getfetchsizes(self, pkg):
 193                 """
 194                 This will raise MissingSignature if SIZE signature is not available,
 195                 or InvalidSignature if SIZE signature is invalid.
 196                 """
 197
 198                 if not self.bintree.populated:
 199                         self.bintree.populate()
 200
 201                 pkg = getattr(pkg, 'cpv', pkg)
 202
 203                 filesdict = {}
 204                 if not self.bintree.isremote(pkg):
 205                         pass
 206                 else:
 207                         metadata = self.bintree._remotepkgs[pkg]
 208                         try:
 209                                 size = int(metadata["SIZE"])
 210                         except KeyError:
 211                                 raise portage.exception.MissingSignature("SIZE")
 212                         except ValueError:
 213                                 raise portage.exception.InvalidSignature(
 214                                         "SIZE: %s" % metadata["SIZE"])
 215                         else:
 216                                 filesdict[os.path.basename(self.bintree.getname(pkg))] = size
 217
 218                 return filesdict
 219
 220 def _pkgindex_cpv_map_latest_build(pkgindex):
 221         """
 222         Given a PackageIndex instance, create a dict of cpv -> metadata map.
 223         If multiple packages have identical CPV values, prefer the package
 224         with latest BUILD_TIME value.
 225         @param pkgindex: A PackageIndex instance.
 226         @type pkgindex: PackageIndex
 227         @rtype: dict
 228         @return: a dict containing entry for the give cpv.
 229         """
 230         cpv_map = {}
 231
 232         for d in pkgindex.packages:
 233                 cpv = d["CPV"]
 234
 235                 try:
 236                         cpv = _pkg_str(cpv)
 237                 except InvalidData:
 238                         writemsg(_("!!! Invalid remote binary package: %s\n") % cpv,
 239                                 noiselevel=-1)
 240                         continue
 241
 242                 btime = d.get('BUILD_TIME', '')
 243                 try:
 244                         btime = int(btime)
 245                 except ValueError:
 246                         btime = None
 247
 248                 other_d = cpv_map.get(cpv)
 249                 if other_d is not None:
 250                         other_btime = other_d.get('BUILD_TIME', '')
 251                         try:
 252                                 other_btime = int(other_btime)
 253                         except ValueError:
 254                                 other_btime = None
 255                         if other_btime and (not btime or other_btime > btime):
 256                                 continue
 257
 258                 cpv_map[_pkg_str(cpv)] = d
 259
 260         return cpv_map
 261
 262 class binarytree(object):
 263         "this tree scans for a list of all packages available in PKGDIR"
 264         def __init__(self, _unused=DeprecationWarning, pkgdir=None,
 265                 virtual=DeprecationWarning, settings=None):
 266
 267                 if pkgdir is None:
 268                         raise TypeError("pkgdir parameter is required")
 269
 270                 if settings is None:
 271                         raise TypeError("settings parameter is required")
 272
 273                 if _unused is not DeprecationWarning:
 274                         warnings.warn("The first parameter of the "
 275                                 "portage.dbapi.bintree.binarytree"
 276                                 " constructor is now unused. Instead "
 277                                 "settings['ROOT'] is used.",
 278                                 DeprecationWarning, stacklevel=2)
 279
 280                 if virtual is not DeprecationWarning:
 281                         warnings.warn("The 'virtual' parameter of the "
 282                                 "portage.dbapi.bintree.binarytree"
 283                                 " constructor is unused",
 284                                 DeprecationWarning, stacklevel=2)
 285
 286                 if True:
 287                         self.pkgdir = normalize_path(pkgdir)
 288                         self.dbapi = bindbapi(self, settings=settings)
 289                         self.update_ents = self.dbapi.update_ents
 290                         self.move_slot_ent = self.dbapi.move_slot_ent
 291                         self.populated = 0
 292                         self.tree = {}
 293                         self._remote_has_index = False
 294                         self._remotepkgs = None # remote metadata indexed by cpv
 295                         self.invalids = []
 296                         self.settings = settings
 297                         self._pkg_paths = {}
 298                         self._pkgindex_uri = {}
 299                         self._populating = False
 300                         self._all_directory = os.path.isdir(
 301                                 os.path.join(self.pkgdir, "All"))
 302                         self._pkgindex_version = 0
 303                         self._pkgindex_hashes = ["MD5","SHA1"]
 304                         self._pkgindex_file = os.path.join(self.pkgdir, "Packages")
 305                         self._pkgindex_keys = self.dbapi._aux_cache_keys.copy()
 306                         self._pkgindex_keys.update(["CPV", "MTIME", "SIZE"])
 307                         self._pkgindex_aux_keys = \
 308                                 ["BUILD_TIME", "CHOST", "DEPEND", "DESCRIPTION", "EAPI",
 309                                 "HDEPEND", "IUSE", "KEYWORDS", "LICENSE", "PDEPEND", "PROPERTIES",
 310                                 "PROVIDE", "RESTRICT", "RDEPEND", "repository", "SLOT", "USE", "DEFINED_PHASES",
 311                                 "BASE_URI"]
 312                         self._pkgindex_aux_keys = list(self._pkgindex_aux_keys)
 313                         self._pkgindex_use_evaluated_keys = \
 314                                 ("DEPEND", "HDEPEND", "LICENSE", "RDEPEND",
 315                                 "PDEPEND", "PROPERTIES", "PROVIDE", "RESTRICT")
 316                         self._pkgindex_header_keys = set([
 317                                 "ACCEPT_KEYWORDS", "ACCEPT_LICENSE",
 318                                 "ACCEPT_PROPERTIES", "ACCEPT_RESTRICT", "CBUILD",
 319                                 "CONFIG_PROTECT", "CONFIG_PROTECT_MASK", "FEATURES",
 320                                 "GENTOO_MIRRORS", "INSTALL_MASK", "SYNC", "USE"])
 321                         self._pkgindex_default_pkg_data = {
 322                                 "BUILD_TIME"         : "",
 323                                 "DEFINED_PHASES"     : "",
 324                                 "DEPEND"  : "",
 325                                 "EAPI"    : "0",
 326                                 "HDEPEND" : "",
 327                                 "IUSE"    : "",
 328                                 "KEYWORDS": "",
 329                                 "LICENSE" : "",
 330                                 "PATH"    : "",
 331                                 "PDEPEND" : "",
 332                                 "PROPERTIES" : "",
 333                                 "PROVIDE" : "",
 334                                 "RDEPEND" : "",
 335                                 "RESTRICT": "",
 336                                 "SLOT"    : "0",
 337                                 "USE"     : "",
 338                         }
 339                         self._pkgindex_inherited_keys = ["CHOST", "repository"]
 340
 341                         # Populate the header with appropriate defaults.
 342                         self._pkgindex_default_header_data = {
 343                                 "CHOST"        : self.settings.get("CHOST", ""),
 344                                 "repository"   : "",
 345                         }
 346
 347                         # It is especially important to populate keys like
 348                         # "repository" that save space when entries can
 349                         # inherit them from the header. If an existing
 350                         # pkgindex header already defines these keys, then
 351                         # they will appropriately override our defaults.
 352                         main_repo = self.settings.repositories.mainRepo()
 353                         if main_repo is not None and not main_repo.missing_repo_name:
 354                                 self._pkgindex_default_header_data["repository"] = \
 355                                         main_repo.name
 356
 357                         self._pkgindex_translated_keys = (
 358                                 ("DESCRIPTION"   ,   "DESC"),
 359                                 ("repository"    ,   "REPO"),
 360                         )
 361
 362                         self._pkgindex_allowed_pkg_keys = set(chain(
 363                                 self._pkgindex_keys,
 364                                 self._pkgindex_aux_keys,
 365                                 self._pkgindex_hashes,
 366                                 self._pkgindex_default_pkg_data,
 367                                 self._pkgindex_inherited_keys,
 368                                 chain(*self._pkgindex_translated_keys)
 369                         ))
 370
 371         @property
 372         def root(self):
 373                 warnings.warn("The root attribute of "
 374                         "portage.dbapi.bintree.binarytree"
 375                         " is deprecated. Use "
 376                         "settings['ROOT'] instead.",
 377                         DeprecationWarning, stacklevel=3)
 378                 return self.settings['ROOT']
 379
 380         def move_ent(self, mylist, repo_match=None):
 381                 if not self.populated:
 382                         self.populate()
 383                 origcp = mylist[1]
 384                 newcp = mylist[2]
 385                 # sanity check
 386                 for atom in (origcp, newcp):
 387                         if not isjustname(atom):
 388                                 raise InvalidPackageName(str(atom))
 389                 mynewcat = catsplit(newcp)[0]
 390                 origmatches=self.dbapi.cp_list(origcp)
 391                 moves = 0
 392                 if not origmatches:
 393                         return moves
 394                 for mycpv in origmatches:
 395                         try:
 396                                 mycpv = self.dbapi._pkg_str(mycpv, None)
 397                         except (KeyError, InvalidData):
 398                                 continue
 399                         mycpv_cp = portage.cpv_getkey(mycpv)
 400                         if mycpv_cp != origcp:
 401                                 # Ignore PROVIDE virtual match.
 402                                 continue
 403                         if repo_match is not None \
 404                                 and not repo_match(mycpv.repo):
 405                                 continue
 406
 407                         # Use isvalidatom() to check if this move is valid for the
 408                         # EAPI (characters allowed in package names may vary).
 409                         if not isvalidatom(newcp, eapi=mycpv.eapi):
 410                                 continue
 411
 412                         mynewcpv = mycpv.replace(mycpv_cp, _unicode(newcp), 1)
 413                         myoldpkg = catsplit(mycpv)[1]
 414                         mynewpkg = catsplit(mynewcpv)[1]
 415
 416                         if (mynewpkg != myoldpkg) and os.path.exists(self.getname(mynewcpv)):
 417                                 writemsg(_("!!! Cannot update binary: Destination exists.\n"),
 418                                         noiselevel=-1)
 419                                 writemsg("!!! "+mycpv+" -> "+mynewcpv+"\n", noiselevel=-1)
 420                                 continue
 421
 422                         tbz2path = self.getname(mycpv)
 423                         if os.path.exists(tbz2path) and not os.access(tbz2path,os.W_OK):
 424                                 writemsg(_("!!! Cannot update readonly binary: %s\n") % mycpv,
 425                                         noiselevel=-1)
 426                                 continue
 427
 428                         moves += 1
 429                         mytbz2 = portage.xpak.tbz2(tbz2path)
 430                         mydata = mytbz2.get_data()
 431                         updated_items = update_dbentries([mylist], mydata, parent=mycpv)
 432                         mydata.update(updated_items)
 433                         mydata[b'PF'] = \
 434                                 _unicode_encode(mynewpkg + "\n",
 435                                 encoding=_encodings['repo.content'])
 436                         mydata[b'CATEGORY'] = \
 437                                 _unicode_encode(mynewcat + "\n",
 438                                 encoding=_encodings['repo.content'])
 439                         if mynewpkg != myoldpkg:
 440                                 ebuild_data = mydata.pop(_unicode_encode(myoldpkg + '.ebuild',
 441                                         encoding=_encodings['repo.content']), None)
 442                                 if ebuild_data is not None:
 443                                         mydata[_unicode_encode(mynewpkg + '.ebuild',
 444                                                 encoding=_encodings['repo.content'])] = ebuild_data
 445
 446                         mytbz2.recompose_mem(portage.xpak.xpak_mem(mydata))
 447
 448                         self.dbapi.cpv_remove(mycpv)
 449                         del self._pkg_paths[mycpv]
 450                         new_path = self.getname(mynewcpv)
 451                         self._pkg_paths[mynewcpv] = os.path.join(
 452                                 *new_path.split(os.path.sep)[-2:])
 453                         if new_path != mytbz2:
 454                                 self._ensure_dir(os.path.dirname(new_path))
 455                                 _movefile(tbz2path, new_path, mysettings=self.settings)
 456                                 self._remove_symlink(mycpv)
 457                                 if new_path.split(os.path.sep)[-2] == "All":
 458                                         self._create_symlink(mynewcpv)
 459                         self.inject(mynewcpv)
 460
 461                 return moves
 462
 463         def _remove_symlink(self, cpv):
 464                 """Remove a ${PKGDIR}/${CATEGORY}/${PF}.tbz2 symlink and also remove
 465                 the ${PKGDIR}/${CATEGORY} directory if empty.  The file will not be
 466                 removed if os.path.islink() returns False."""
 467                 mycat, mypkg = catsplit(cpv)
 468                 mylink = os.path.join(self.pkgdir, mycat, mypkg + ".tbz2")
 469                 if os.path.islink(mylink):
 470                         """Only remove it if it's really a link so that this method never
 471                         removes a real package that was placed here to avoid a collision."""
 472                         os.unlink(mylink)
 473                 try:
 474                         os.rmdir(os.path.join(self.pkgdir, mycat))
 475                 except OSError as e:
 476                         if e.errno not in (errno.ENOENT,
 477                                 errno.ENOTEMPTY, errno.EEXIST):
 478                                 raise
 479                         del e
 480
 481         def _create_symlink(self, cpv):
 482                 """Create a ${PKGDIR}/${CATEGORY}/${PF}.tbz2 symlink (and
 483                 ${PKGDIR}/${CATEGORY} directory, if necessary).  Any file that may
 484                 exist in the location of the symlink will first be removed."""
 485                 mycat, mypkg = catsplit(cpv)
 486                 full_path = os.path.join(self.pkgdir, mycat, mypkg + ".tbz2")
 487                 self._ensure_dir(os.path.dirname(full_path))
 488                 try:
 489                         os.unlink(full_path)
 490                 except OSError as e:
 491                         if e.errno != errno.ENOENT:
 492                                 raise
 493                         del e
 494                 os.symlink(os.path.join("..", "All", mypkg + ".tbz2"), full_path)
 495
 496         def prevent_collision(self, cpv):
 497                 """Make sure that the file location ${PKGDIR}/All/${PF}.tbz2 is safe to
 498                 use for a given cpv.  If a collision will occur with an existing
 499                 package from another category, the existing package will be bumped to
 500                 ${PKGDIR}/${CATEGORY}/${PF}.tbz2 so that both can coexist."""
 501                 if not self._all_directory:
 502                         return
 503
 504                 # Copy group permissions for new directories that
 505                 # may have been created.
 506                 for path in ("All", catsplit(cpv)[0]):
 507                         path = os.path.join(self.pkgdir, path)
 508                         self._ensure_dir(path)
 509                         if not os.access(path, os.W_OK):
 510                                 raise PermissionDenied("access('%s', W_OK)" % path)
 511
 512                 full_path = self.getname(cpv)
 513                 if "All" == full_path.split(os.path.sep)[-2]:
 514                         return
 515                 """Move a colliding package if it exists.  Code below this point only
 516                 executes in rare cases."""
 517                 mycat, mypkg = catsplit(cpv)
 518                 myfile = mypkg + ".tbz2"
 519                 mypath = os.path.join("All", myfile)
 520                 dest_path = os.path.join(self.pkgdir, mypath)
 521
 522                 try:
 523                         st = os.lstat(dest_path)
 524                 except OSError:
 525                         st = None
 526                 else:
 527                         if stat.S_ISLNK(st.st_mode):
 528                                 st = None
 529                                 try:
 530                                         os.unlink(dest_path)
 531                                 except OSError:
 532                                         if os.path.exists(dest_path):
 533                                                 raise
 534
 535                 if st is not None:
 536                         # For invalid packages, other_cat could be None.
 537                         other_cat = portage.xpak.tbz2(dest_path).getfile(b"CATEGORY")
 538                         if other_cat:
 539                                 other_cat = _unicode_decode(other_cat,
 540                                         encoding=_encodings['repo.content'], errors='replace')
 541                                 other_cat = other_cat.strip()
 542                                 other_cpv = other_cat + "/" + mypkg
 543                                 self._move_from_all(other_cpv)
 544                                 self.inject(other_cpv)
 545                 self._move_to_all(cpv)
 546
 547         def _ensure_dir(self, path):
 548                 """
 549                 Create the specified directory. Also, copy gid and group mode
 550                 bits from self.pkgdir if possible.
 551                 @param cat_dir: Absolute path of the directory to be created.
 552                 @type cat_dir: String
 553                 """
 554                 try:
 555                         pkgdir_st = os.stat(self.pkgdir)
 556                 except OSError:
 557                         ensure_dirs(path)
 558                         return
 559                 pkgdir_gid = pkgdir_st.st_gid
 560                 pkgdir_grp_mode = 0o2070 & pkgdir_st.st_mode
 561                 try:
 562                         ensure_dirs(path, gid=pkgdir_gid, mode=pkgdir_grp_mode, mask=0)
 563                 except PortageException:
 564                         if not os.path.isdir(path):
 565                                 raise
 566
 567         def _file_permissions(self, path):
 568                 try:
 569                         pkgdir_st = os.stat(self.pkgdir)
 570                 except OSError:
 571                         pass
 572                 else:
 573                         pkgdir_gid = pkgdir_st.st_gid
 574                         pkgdir_grp_mode = 0o0060 & pkgdir_st.st_mode
 575                         try:
 576                                 portage.util.apply_permissions(path, gid=pkgdir_gid,
 577                                         mode=pkgdir_grp_mode, mask=0)
 578                         except PortageException:
 579                                 pass
 580
 581         def _move_to_all(self, cpv):
 582                 """If the file exists, move it.  Whether or not it exists, update state
 583                 for future getname() calls."""
 584                 mycat, mypkg = catsplit(cpv)
 585                 myfile = mypkg + ".tbz2"
 586                 self._pkg_paths[cpv] = os.path.join("All", myfile)
 587                 src_path = os.path.join(self.pkgdir, mycat, myfile)
 588                 try:
 589                         mystat = os.lstat(src_path)
 590                 except OSError as e:
 591                         mystat = None
 592                 if mystat and stat.S_ISREG(mystat.st_mode):
 593                         self._ensure_dir(os.path.join(self.pkgdir, "All"))
 594                         dest_path = os.path.join(self.pkgdir, "All", myfile)
 595                         _movefile(src_path, dest_path, mysettings=self.settings)
 596                         self._create_symlink(cpv)
 597                         self.inject(cpv)
 598
 599         def _move_from_all(self, cpv):
 600                 """Move a package from ${PKGDIR}/All/${PF}.tbz2 to
 601                 ${PKGDIR}/${CATEGORY}/${PF}.tbz2 and update state from getname calls."""
 602                 self._remove_symlink(cpv)
 603                 mycat, mypkg = catsplit(cpv)
 604                 myfile = mypkg + ".tbz2"
 605                 mypath = os.path.join(mycat, myfile)
 606                 dest_path = os.path.join(self.pkgdir, mypath)
 607                 self._ensure_dir(os.path.dirname(dest_path))
 608                 src_path = os.path.join(self.pkgdir, "All", myfile)
 609                 _movefile(src_path, dest_path, mysettings=self.settings)
 610                 self._pkg_paths[cpv] = mypath
 611
 612         def populate(self, getbinpkgs=0):
 613                 "populates the binarytree"
 614
 615                 if self._populating:
 616                         return
 617
 618                 pkgindex_lock = None
 619                 try:
 620                         if os.access(self.pkgdir, os.W_OK):
 621                                 pkgindex_lock = lockfile(self._pkgindex_file,
 622                                         wantnewlockfile=1)
 623                         self._populating = True
 624                         self._populate(getbinpkgs)
 625                 finally:
 626                         if pkgindex_lock:
 627                                 unlockfile(pkgindex_lock)
 628                         self._populating = False
 629
 630         def _populate(self, getbinpkgs=0):
 631                 if (not os.path.isdir(self.pkgdir) and not getbinpkgs):
 632                         return 0
 633
 634                 # Clear all caches in case populate is called multiple times
 635                 # as may be the case when _global_updates calls populate()
 636                 # prior to performing package moves since it only wants to
 637                 # operate on local packages (getbinpkgs=0).
 638                 self._remotepkgs = None
 639                 self.dbapi._clear_cache()
 640                 self.dbapi._aux_cache.clear()
 641                 if True:
 642                         pkg_paths = {}
 643                         self._pkg_paths = pkg_paths
 644                         dirs = listdir(self.pkgdir, dirsonly=True, EmptyOnError=True)
 645                         if "All" in dirs:
 646                                 dirs.remove("All")
 647                         dirs.sort()
 648                         dirs.insert(0, "All")
 649                         pkgindex = self._load_pkgindex()
 650                         pf_index = None
 651                         if not self._pkgindex_version_supported(pkgindex):
 652                                 pkgindex = self._new_pkgindex()
 653                         header = pkgindex.header
 654                         metadata = {}
 655                         for d in pkgindex.packages:
 656                                 metadata[d["CPV"]] = d
 657                         update_pkgindex = False
 658                         for mydir in dirs:
 659                                 for myfile in listdir(os.path.join(self.pkgdir, mydir)):
 660                                         if not myfile.endswith(".tbz2"):
 661                                                 continue
 662                                         mypath = os.path.join(mydir, myfile)
 663                                         full_path = os.path.join(self.pkgdir, mypath)
 664                                         s = os.lstat(full_path)
 665                                         if stat.S_ISLNK(s.st_mode):
 666                                                 continue
 667
 668                                         # Validate data from the package index and try to avoid
 669                                         # reading the xpak if possible.
 670                                         if mydir != "All":
 671                                                 possibilities = None
 672                                                 d = metadata.get(mydir+"/"+myfile[:-5])
 673                                                 if d:
 674                                                         possibilities = [d]
 675                                         else:
 676                                                 if pf_index is None:
 677                                                         pf_index = {}
 678                                                         for mycpv in metadata:
 679                                                                 mycat, mypf = catsplit(mycpv)
 680                                                                 pf_index.setdefault(
 681                                                                         mypf, []).append(metadata[mycpv])
 682                                                 possibilities = pf_index.get(myfile[:-5])
 683                                         if possibilities:
 684                                                 match = None
 685                                                 for d in possibilities:
 686                                                         try:
 687                                                                 if long(d["MTIME"]) != s[stat.ST_MTIME]:
 688                                                                         continue
 689                                                         except (KeyError, ValueError):
 690                                                                 continue
 691                                                         try:
 692                                                                 if long(d["SIZE"]) != long(s.st_size):
 693                                                                         continue
 694                                                         except (KeyError, ValueError):
 695                                                                 continue
 696                                                         if not self._pkgindex_keys.difference(d):
 697                                                                 match = d
 698                                                                 break
 699                                                 if match:
 700                                                         mycpv = match["CPV"]
 701                                                         if mycpv in pkg_paths:
 702                                                                 # discard duplicates (All/ is preferred)
 703                                                                 continue
 704                                                         mycpv = _pkg_str(mycpv)
 705                                                         pkg_paths[mycpv] = mypath
 706                                                         # update the path if the package has been moved
 707                                                         oldpath = d.get("PATH")
 708                                                         if oldpath and oldpath != mypath:
 709                                                                 update_pkgindex = True
 710                                                         if mypath != mycpv + ".tbz2":
 711                                                                 d["PATH"] = mypath
 712                                                                 if not oldpath:
 713                                                                         update_pkgindex = True
 714                                                         else:
 715                                                                 d.pop("PATH", None)
 716                                                                 if oldpath:
 717                                                                         update_pkgindex = True
 718                                                         self.dbapi.cpv_inject(mycpv)
 719                                                         if not self.dbapi._aux_cache_keys.difference(d):
 720                                                                 aux_cache = self.dbapi._aux_cache_slot_dict()
 721                                                                 for k in self.dbapi._aux_cache_keys:
 722                                                                         aux_cache[k] = d[k]
 723                                                                 self.dbapi._aux_cache[mycpv] = aux_cache
 724                                                         continue
 725                                         if not os.access(full_path, os.R_OK):
 726                                                 writemsg(_("!!! Permission denied to read " \
 727                                                         "binary package: '%s'\n") % full_path,
 728                                                         noiselevel=-1)
 729                                                 self.invalids.append(myfile[:-5])
 730                                                 continue
 731                                         metadata_bytes = portage.xpak.tbz2(full_path).get_data()
 732                                         mycat = _unicode_decode(metadata_bytes.get(b"CATEGORY", ""),
 733                                                 encoding=_encodings['repo.content'], errors='replace')
 734                                         mypf = _unicode_decode(metadata_bytes.get(b"PF", ""),
 735                                                 encoding=_encodings['repo.content'], errors='replace')
 736                                         slot = _unicode_decode(metadata_bytes.get(b"SLOT", ""),
 737                                                 encoding=_encodings['repo.content'], errors='replace')
 738                                         mypkg = myfile[:-5]
 739                                         if not mycat or not mypf or not slot:
 740                                                 #old-style or corrupt package
 741                                                 writemsg(_("\n!!! Invalid binary package: '%s'\n") % full_path,
 742                                                         noiselevel=-1)
 743                                                 missing_keys = []
 744                                                 if not mycat:
 745                                                         missing_keys.append("CATEGORY")
 746                                                 if not mypf:
 747                                                         missing_keys.append("PF")
 748                                                 if not slot:
 749                                                         missing_keys.append("SLOT")
 750                                                 msg = []
 751                                                 if missing_keys:
 752                                                         missing_keys.sort()
 753                                                         msg.append(_("Missing metadata key(s): %s.") % \
 754                                                                 ", ".join(missing_keys))
 755                                                 msg.append(_(" This binary package is not " \
 756                                                         "recoverable and should be deleted."))
 757                                                 for line in textwrap.wrap("".join(msg), 72):
 758                                                         writemsg("!!! %s\n" % line, noiselevel=-1)
 759                                                 self.invalids.append(mypkg)
 760                                                 continue
 761                                         mycat = mycat.strip()
 762                                         slot = slot.strip()
 763                                         if mycat != mydir and mydir != "All":
 764                                                 continue
 765                                         if mypkg != mypf.strip():
 766                                                 continue
 767                                         mycpv = mycat + "/" + mypkg
 768                                         if mycpv in pkg_paths:
 769                                                 # All is first, so it's preferred.
 770                                                 continue
 771                                         if not self.dbapi._category_re.match(mycat):
 772                                                 writemsg(_("!!! Binary package has an " \
 773                                                         "unrecognized category: '%s'\n") % full_path,
 774                                                         noiselevel=-1)
 775                                                 writemsg(_("!!! '%s' has a category that is not" \
 776                                                         " listed in %setc/portage/categories\n") % \
 777                                                         (mycpv, self.settings["PORTAGE_CONFIGROOT"]),
 778                                                         noiselevel=-1)
 779                                                 continue
 780                                         mycpv = _pkg_str(mycpv)
 781                                         pkg_paths[mycpv] = mypath
 782                                         self.dbapi.cpv_inject(mycpv)
 783                                         update_pkgindex = True
 784                                         d = metadata.get(mycpv, {})
 785                                         if d:
 786                                                 try:
 787                                                         if long(d["MTIME"]) != s[stat.ST_MTIME]:
 788                                                                 d.clear()
 789                                                 except (KeyError, ValueError):
 790                                                         d.clear()
 791                                         if d:
 792                                                 try:
 793                                                         if long(d["SIZE"]) != long(s.st_size):
 794                                                                 d.clear()
 795                                                 except (KeyError, ValueError):
 796                                                         d.clear()
 797
 798                                         d["CPV"] = mycpv
 799                                         d["SLOT"] = slot
 800                                         d["MTIME"] = str(s[stat.ST_MTIME])
 801                                         d["SIZE"] = str(s.st_size)
 802
 803                                         d.update(zip(self._pkgindex_aux_keys,
 804                                                 self.dbapi.aux_get(mycpv, self._pkgindex_aux_keys)))
 805                                         try:
 806                                                 self._eval_use_flags(mycpv, d)
 807                                         except portage.exception.InvalidDependString:
 808                                                 writemsg(_("!!! Invalid binary package: '%s'\n") % \
 809                                                         self.getname(mycpv), noiselevel=-1)
 810                                                 self.dbapi.cpv_remove(mycpv)
 811                                                 del pkg_paths[mycpv]
 812
 813                                         # record location if it's non-default
 814                                         if mypath != mycpv + ".tbz2":
 815                                                 d["PATH"] = mypath
 816                                         else:
 817                                                 d.pop("PATH", None)
 818                                         metadata[mycpv] = d
 819                                         if not self.dbapi._aux_cache_keys.difference(d):
 820                                                 aux_cache = self.dbapi._aux_cache_slot_dict()
 821                                                 for k in self.dbapi._aux_cache_keys:
 822                                                         aux_cache[k] = d[k]
 823                                                 self.dbapi._aux_cache[mycpv] = aux_cache
 824
 825                         for cpv in list(metadata):
 826                                 if cpv not in pkg_paths:
 827                                         del metadata[cpv]
 828
 829                         # Do not bother to write the Packages index if $PKGDIR/All/ exists
 830                         # since it will provide no benefit due to the need to read CATEGORY
 831                         # from xpak.
 832                         if update_pkgindex and os.access(self.pkgdir, os.W_OK):
 833                                 del pkgindex.packages[:]
 834                                 pkgindex.packages.extend(iter(metadata.values()))
 835                                 self._update_pkgindex_header(pkgindex.header)
 836                                 self._pkgindex_write(pkgindex)
 837
 838                 if getbinpkgs and not self.settings["PORTAGE_BINHOST"]:
 839                         writemsg(_("!!! PORTAGE_BINHOST unset, but use is requested.\n"),
 840                                 noiselevel=-1)
 841
 842                 if not getbinpkgs or 'PORTAGE_BINHOST' not in self.settings:
 843                         self.populated=1
 844                         return
 845                 self._remotepkgs = {}
 846                 for base_url in self.settings["PORTAGE_BINHOST"].split():
 847                         parsed_url = urlparse(base_url)
 848                         host = parsed_url.netloc
 849                         port = parsed_url.port
 850                         user = None
 851                         passwd = None
 852                         user_passwd = ""
 853                         if "@" in host:
 854                                 user, host = host.split("@", 1)
 855                                 user_passwd = user + "@"
 856                                 if ":" in user:
 857                                         user, passwd = user.split(":", 1)
 858                         port_args = []
 859                         if port is not None:
 860                                 port_str = ":%s" % (port,)
 861                                 if host.endswith(port_str):
 862                                         host = host[:-len(port_str)]
 863                         pkgindex_file = os.path.join(self.settings["EROOT"], CACHE_PATH, "binhost",
 864                                 host, parsed_url.path.lstrip("/"), "Packages")
 865                         pkgindex = self._new_pkgindex()
 866                         try:
 867                                 f = io.open(_unicode_encode(pkgindex_file,
 868                                         encoding=_encodings['fs'], errors='strict'),
 869                                         mode='r', encoding=_encodings['repo.content'],
 870                                         errors='replace')
 871                                 try:
 872                                         pkgindex.read(f)
 873                                 finally:
 874                                         f.close()
 875                         except EnvironmentError as e:
 876                                 if e.errno != errno.ENOENT:
 877                                         raise
 878                         local_timestamp = pkgindex.header.get("TIMESTAMP", None)
 879                         remote_timestamp = None
 880                         rmt_idx = self._new_pkgindex()
 881                         proc = None
 882                         tmp_filename = None
 883                         try:
 884                                 # urlparse.urljoin() only works correctly with recognized
 885                                 # protocols and requires the base url to have a trailing
 886                                 # slash, so join manually...
 887                                 url = base_url.rstrip("/") + "/Packages"
 888                                 f = None
 889
 890                                 # Don't use urlopen for https, since it doesn't support
 891                                 # certificate/hostname verification (bug #469888).
 892                                 if parsed_url.scheme not in ('https',):
 893                                         try:
 894                                                 f = _urlopen(url, if_modified_since=local_timestamp)
 895                                                 if hasattr(f, 'headers') and f.headers.get('timestamp', ''):
 896                                                         remote_timestamp = f.headers.get('timestamp')
 897                                         except IOError as err:
 898                                                 if hasattr(err, 'code') and err.code == 304: # not modified (since local_timestamp)
 899                                                         raise UseCachedCopyOfRemoteIndex()
 900
 901                                                 if parsed_url.scheme in ('ftp', 'http', 'https'):
 902                                                         # This protocol is supposedly supported by urlopen,
 903                                                         # so apparently there's a problem with the url
 904                                                         # or a bug in urlopen.
 905                                                         if self.settings.get("PORTAGE_DEBUG", "0") != "0":
 906                                                                 traceback.print_exc()
 907
 908                                                         raise
 909
 910                                 if f is None:
 911
 912                                         path = parsed_url.path.rstrip("/") + "/Packages"
 913
 914                                         if parsed_url.scheme == 'ssh':
 915                                                 # Use a pipe so that we can terminate the download
 916                                                 # early if we detect that the TIMESTAMP header
 917                                                 # matches that of the cached Packages file.
 918                                                 ssh_args = ['ssh']
 919                                                 if port is not None:
 920                                                         ssh_args.append("-p%s" % (port,))
 921                                                 # NOTE: shlex evaluates embedded quotes
 922                                                 ssh_args.extend(portage.util.shlex_split(
 923                                                         self.settings.get("PORTAGE_SSH_OPTS", "")))
 924                                                 ssh_args.append(user_passwd + host)
 925                                                 ssh_args.append('--')
 926                                                 ssh_args.append('cat')
 927                                                 ssh_args.append(path)
 928
 929                                                 proc = subprocess.Popen(ssh_args,
 930                                                         stdout=subprocess.PIPE)
 931                                                 f = proc.stdout
 932                                         else:
 933                                                 setting = 'FETCHCOMMAND_' + parsed_url.scheme.upper()
 934                                                 fcmd = self.settings.get(setting)
 935                                                 if not fcmd:
 936                                                         fcmd = self.settings.get('FETCHCOMMAND')
 937                                                         if not fcmd:
 938                                                                 raise EnvironmentError("FETCHCOMMAND is unset")
 939
 940                                                 fd, tmp_filename = tempfile.mkstemp()
 941                                                 tmp_dirname, tmp_basename = os.path.split(tmp_filename)
 942                                                 os.close(fd)
 943
 944                                                 fcmd_vars = {
 945                                                         "DISTDIR": tmp_dirname,
 946                                                         "FILE": tmp_basename,
 947                                                         "URI": url
 948                                                 }
 949
 950                                                 for k in ("PORTAGE_SSH_OPTS",):
 951                                                         try:
 952                                                                 fcmd_vars[k] = self.settings[k]
 953                                                         except KeyError:
 954                                                                 pass
 955
 956                                                 success = portage.getbinpkg.file_get(
 957                                                         fcmd=fcmd, fcmd_vars=fcmd_vars)
 958                                                 if not success:
 959                                                         raise EnvironmentError("%s failed" % (setting,))
 960                                                 f = open(tmp_filename, 'rb')
 961
 962                                 f_dec = codecs.iterdecode(f,
 963                                         _encodings['repo.content'], errors='replace')
 964                                 try:
 965                                         rmt_idx.readHeader(f_dec)
 966                                         if not remote_timestamp: # in case it had not been read from HTTP header
 967                                                 remote_timestamp = rmt_idx.header.get("TIMESTAMP", None)
 968                                         if not remote_timestamp:
 969                                                 # no timestamp in the header, something's wrong
 970                                                 pkgindex = None
 971                                                 writemsg(_("\n\n!!! Binhost package index " \
 972                                                 " has no TIMESTAMP field.\n"), noiselevel=-1)
 973                                         else:
 974                                                 if not self._pkgindex_version_supported(rmt_idx):
 975                                                         writemsg(_("\n\n!!! Binhost package index version" \
 976                                                         " is not supported: '%s'\n") % \
 977                                                         rmt_idx.header.get("VERSION"), noiselevel=-1)
 978                                                         pkgindex = None
 979                                                 elif local_timestamp != remote_timestamp:
 980                                                         rmt_idx.readBody(f_dec)
 981                                                         pkgindex = rmt_idx
 982                                 finally:
 983                                         # Timeout after 5 seconds, in case close() blocks
 984                                         # indefinitely (see bug #350139).
 985                                         try:
 986                                                 try:
 987                                                         AlarmSignal.register(5)
 988                                                         f.close()
 989                                                 finally:
 990                                                         AlarmSignal.unregister()
 991                                         except AlarmSignal:
 992                                                 writemsg("\n\n!!! %s\n" % \
 993                                                         _("Timed out while closing connection to binhost"),
 994                                                         noiselevel=-1)
 995                         except UseCachedCopyOfRemoteIndex:
 996                                 writemsg_stdout("\n")
 997                                 writemsg_stdout(
 998                                         colorize("GOOD", _("Local copy of remote index is up-to-date and will be used.")) + \
 999                                         "\n")
1000                                 rmt_idx = pkgindex
1001                         except EnvironmentError as e:
1002                                 writemsg(_("\n\n!!! Error fetching binhost package" \
1003                                         " info from '%s'\n") % _hide_url_passwd(base_url))
1004                                 writemsg("!!! %s\n\n" % str(e))
1005                                 del e
1006                                 pkgindex = None
1007                         if proc is not None:
1008                                 if proc.poll() is None:
1009                                         proc.kill()
1010                                         proc.wait()
1011                                 proc = None
1012                         if tmp_filename is not None:
1013                                 try:
1014                                         os.unlink(tmp_filename)
1015                                 except OSError:
1016                                         pass
1017                         if pkgindex is rmt_idx:
1018                                 pkgindex.modified = False # don't update the header
1019                                 try:
1020                                         ensure_dirs(os.path.dirname(pkgindex_file))
1021                                         f = atomic_ofstream(pkgindex_file)
1022                                         pkgindex.write(f)
1023                                         f.close()
1024                                 except (IOError, PortageException):
1025                                         if os.access(os.path.dirname(pkgindex_file), os.W_OK):
1026                                                 raise
1027                                         # The current user doesn't have permission to cache the
1028                                         # file, but that's alright.
1029                         if pkgindex:
1030                                 # Organize remote package list as a cpv -> metadata map.
1031                                 remotepkgs = _pkgindex_cpv_map_latest_build(pkgindex)
1032                                 remote_base_uri = pkgindex.header.get("URI", base_url)
1033                                 for cpv, remote_metadata in remotepkgs.items():
1034                                         remote_metadata["BASE_URI"] = remote_base_uri
1035                                         self._pkgindex_uri[cpv] = url
1036                                 self._remotepkgs.update(remotepkgs)
1037                                 self._remote_has_index = True
1038                                 for cpv in remotepkgs:
1039                                         self.dbapi.cpv_inject(cpv)
1040                                 if True:
1041                                         # Remote package instances override local package
1042                                         # if they are not identical.
1043                                         hash_names = ["SIZE"] + self._pkgindex_hashes
1044                                         for cpv, local_metadata in metadata.items():
1045                                                 remote_metadata = self._remotepkgs.get(cpv)
1046                                                 if remote_metadata is None:
1047                                                         continue
1048                                                 # Use digests to compare identity.
1049                                                 identical = True
1050                                                 for hash_name in hash_names:
1051                                                         local_value = local_metadata.get(hash_name)
1052                                                         if local_value is None:
1053                                                                 continue
1054                                                         remote_value = remote_metadata.get(hash_name)
1055                                                         if remote_value is None:
1056                                                                 continue
1057                                                         if local_value != remote_value:
1058                                                                 identical = False
1059                                                                 break
1060                                                 if identical:
1061                                                         del self._remotepkgs[cpv]
1062                                                 else:
1063                                                         # Override the local package in the aux_get cache.
1064                                                         self.dbapi._aux_cache[cpv] = remote_metadata
1065                                 else:
1066                                         # Local package instances override remote instances.
1067                                         for cpv in metadata:
1068                                                 self._remotepkgs.pop(cpv, None)
1069
1070                 self.populated=1
1071
1072         def inject(self, cpv, filename=None):
1073                 """Add a freshly built package to the database.  This updates
1074                 $PKGDIR/Packages with the new package metadata (including MD5).
1075                 @param cpv: The cpv of the new package to inject
1076                 @type cpv: string
1077                 @param filename: File path of the package to inject, or None if it's
1078                         already in the location returned by getname()
1079                 @type filename: string
1080                 @rtype: None
1081                 """
1082                 mycat, mypkg = catsplit(cpv)
1083                 if not self.populated:
1084                         self.populate()
1085                 if filename is None:
1086                         full_path = self.getname(cpv)
1087                 else:
1088                         full_path = filename
1089                 try:
1090                         s = os.stat(full_path)
1091                 except OSError as e:
1092                         if e.errno != errno.ENOENT:
1093                                 raise
1094                         del e
1095                         writemsg(_("!!! Binary package does not exist: '%s'\n") % full_path,
1096                                 noiselevel=-1)
1097                         return
1098                 mytbz2 = portage.xpak.tbz2(full_path)
1099                 slot = mytbz2.getfile("SLOT")
1100                 if slot is None:
1101                         writemsg(_("!!! Invalid binary package: '%s'\n") % full_path,
1102                                 noiselevel=-1)
1103                         return
1104                 slot = slot.strip()
1105                 self.dbapi.cpv_inject(cpv)
1106
1107                 # Reread the Packages index (in case it's been changed by another
1108                 # process) and then updated it, all while holding a lock.
1109                 pkgindex_lock = None
1110                 created_symlink = False
1111                 try:
1112                         pkgindex_lock = lockfile(self._pkgindex_file,
1113                                 wantnewlockfile=1)
1114                         if filename is not None:
1115                                 new_filename = self.getname(cpv)
1116                                 try:
1117                                         samefile = os.path.samefile(filename, new_filename)
1118                                 except OSError:
1119                                         samefile = False
1120                                 if not samefile:
1121                                         self._ensure_dir(os.path.dirname(new_filename))
1122                                         _movefile(filename, new_filename, mysettings=self.settings)
1123                                 full_path = new_filename
1124
1125                         self._file_permissions(full_path)
1126
1127                         if self._all_directory and \
1128                                 self.getname(cpv).split(os.path.sep)[-2] == "All":
1129                                 self._create_symlink(cpv)
1130                                 created_symlink = True
1131                         pkgindex = self._load_pkgindex()
1132
1133                         if not self._pkgindex_version_supported(pkgindex):
1134                                 pkgindex = self._new_pkgindex()
1135
1136                         # Discard remote metadata to ensure that _pkgindex_entry
1137                         # gets the local metadata. This also updates state for future
1138                         # isremote calls.
1139                         if self._remotepkgs is not None:
1140                                 self._remotepkgs.pop(cpv, None)
1141
1142                         # Discard cached metadata to ensure that _pkgindex_entry
1143                         # doesn't return stale metadata.
1144                         self.dbapi._aux_cache.pop(cpv, None)
1145
1146                         try:
1147                                 d = self._pkgindex_entry(cpv)
1148                         except portage.exception.InvalidDependString:
1149                                 writemsg(_("!!! Invalid binary package: '%s'\n") % \
1150                                         self.getname(cpv), noiselevel=-1)
1151                                 self.dbapi.cpv_remove(cpv)
1152                                 del self._pkg_paths[cpv]
1153                                 return
1154
1155                         # If found, remove package(s) with duplicate path.
1156                         path = d.get("PATH", "")
1157                         for i in range(len(pkgindex.packages) - 1, -1, -1):
1158                                 d2 = pkgindex.packages[i]
1159                                 if path and path == d2.get("PATH"):
1160                                         # Handle path collisions in $PKGDIR/All
1161                                         # when CPV is not identical.
1162                                         del pkgindex.packages[i]
1163                                 elif cpv == d2.get("CPV"):
1164                                         if path == d2.get("PATH", ""):
1165                                                 del pkgindex.packages[i]
1166                                         elif created_symlink and not d2.get("PATH", ""):
1167                                                 # Delete entry for the package that was just
1168                                                 # overwritten by a symlink to this package.
1169                                                 del pkgindex.packages[i]
1170
1171                         pkgindex.packages.append(d)
1172
1173                         self._update_pkgindex_header(pkgindex.header)
1174                         self._pkgindex_write(pkgindex)
1175
1176                 finally:
1177                         if pkgindex_lock:
1178                                 unlockfile(pkgindex_lock)
1179
1180         def _pkgindex_write(self, pkgindex):
1181                 contents = codecs.getwriter(_encodings['repo.content'])(io.BytesIO())
1182                 pkgindex.write(contents)
1183                 contents = contents.getvalue()
1184                 atime = mtime = long(pkgindex.header["TIMESTAMP"])
1185                 output_files = [(atomic_ofstream(self._pkgindex_file, mode="wb"),
1186                         self._pkgindex_file, None)]
1187
1188                 if "compress-index" in self.settings.features:
1189                         gz_fname = self._pkgindex_file + ".gz"
1190                         fileobj = atomic_ofstream(gz_fname, mode="wb")
1191                         output_files.append((GzipFile(filename='', mode="wb",
1192                                 fileobj=fileobj, mtime=mtime), gz_fname, fileobj))
1193
1194                 for f, fname, f_close in output_files:
1195                         f.write(contents)
1196                         f.close()
1197                         if f_close is not None:
1198                                 f_close.close()
1199                         self._file_permissions(fname)
1200                         # some seconds might have elapsed since TIMESTAMP
1201                         os.utime(fname, (atime, mtime))
1202
1203         def _pkgindex_entry(self, cpv):
1204                 """
1205                 Performs checksums and evaluates USE flag conditionals.
1206                 Raises InvalidDependString if necessary.
1207                 @rtype: dict
1208                 @return: a dict containing entry for the give cpv.
1209                 """
1210
1211                 pkg_path = self.getname(cpv)
1212
1213                 d = dict(zip(self._pkgindex_aux_keys,
1214                         self.dbapi.aux_get(cpv, self._pkgindex_aux_keys)))
1215
1216                 d.update(perform_multiple_checksums(
1217                         pkg_path, hashes=self._pkgindex_hashes))
1218
1219                 d["CPV"] = cpv
1220                 st = os.stat(pkg_path)
1221                 d["MTIME"] = str(st[stat.ST_MTIME])
1222                 d["SIZE"] = str(st.st_size)
1223
1224                 rel_path = self._pkg_paths[cpv]
1225                 # record location if it's non-default
1226                 if rel_path != cpv + ".tbz2":
1227                         d["PATH"] = rel_path
1228
1229                 self._eval_use_flags(cpv, d)
1230                 return d
1231
1232         def _new_pkgindex(self):
1233                 return portage.getbinpkg.PackageIndex(
1234                         allowed_pkg_keys=self._pkgindex_allowed_pkg_keys,
1235                         default_header_data=self._pkgindex_default_header_data,
1236                         default_pkg_data=self._pkgindex_default_pkg_data,
1237                         inherited_keys=self._pkgindex_inherited_keys,
1238                         translated_keys=self._pkgindex_translated_keys)
1239
1240         def _update_pkgindex_header(self, header):
1241                 portdir = normalize_path(os.path.realpath(self.settings["PORTDIR"]))
1242                 profiles_base = os.path.join(portdir, "profiles") + os.path.sep
1243                 if self.settings.profile_path:
1244                         profile_path = normalize_path(
1245                                 os.path.realpath(self.settings.profile_path))
1246                         if profile_path.startswith(profiles_base):
1247                                 profile_path = profile_path[len(profiles_base):]
1248                         header["PROFILE"] = profile_path
1249                 header["VERSION"] = str(self._pkgindex_version)
1250                 base_uri = self.settings.get("PORTAGE_BINHOST_HEADER_URI")
1251                 if base_uri:
1252                         header["URI"] = base_uri
1253                 else:
1254                         header.pop("URI", None)
1255                 for k in self._pkgindex_header_keys:
1256                         v = self.settings.get(k, None)
1257                         if v:
1258                                 header[k] = v
1259                         else:
1260                                 header.pop(k, None)
1261
1262         def _pkgindex_version_supported(self, pkgindex):
1263                 version = pkgindex.header.get("VERSION")
1264                 if version:
1265                         try:
1266                                 if int(version) <= self._pkgindex_version:
1267                                         return True
1268                         except ValueError:
1269                                 pass
1270                 return False
1271
1272         def _eval_use_flags(self, cpv, metadata):
1273                 use = frozenset(metadata["USE"].split())
1274                 raw_use = use
1275                 iuse = set(f.lstrip("-+") for f in metadata["IUSE"].split())
1276                 use = [f for f in use if f in iuse]
1277                 use.sort()
1278                 metadata["USE"] = " ".join(use)
1279                 for k in self._pkgindex_use_evaluated_keys:
1280                         if k.endswith('DEPEND'):
1281                                 token_class = Atom
1282                         else:
1283                                 token_class = None
1284
1285                         try:
1286                                 deps = metadata[k]
1287                                 deps = use_reduce(deps, uselist=raw_use, token_class=token_class)
1288                                 deps = paren_enclose(deps)
1289                         except portage.exception.InvalidDependString as e:
1290                                 writemsg("%s: %s\n" % (k, str(e)),
1291                                         noiselevel=-1)
1292                                 raise
1293                         metadata[k] = deps
1294
1295         def exists_specific(self, cpv):
1296                 if not self.populated:
1297                         self.populate()
1298                 return self.dbapi.match(
1299                         dep_expand("="+cpv, mydb=self.dbapi, settings=self.settings))
1300
1301         def dep_bestmatch(self, mydep):
1302                 "compatibility method -- all matches, not just visible ones"
1303                 if not self.populated:
1304                         self.populate()
1305                 writemsg("\n\n", 1)
1306                 writemsg("mydep: %s\n" % mydep, 1)
1307                 mydep = dep_expand(mydep, mydb=self.dbapi, settings=self.settings)
1308                 writemsg("mydep: %s\n" % mydep, 1)
1309                 mykey = dep_getkey(mydep)
1310                 writemsg("mykey: %s\n" % mykey, 1)
1311                 mymatch = best(match_from_list(mydep,self.dbapi.cp_list(mykey)))
1312                 writemsg("mymatch: %s\n" % mymatch, 1)
1313                 if mymatch is None:
1314                         return ""
1315                 return mymatch
1316
1317         def getname(self, pkgname):
1318                 """Returns a file location for this package.  The default location is
1319                 ${PKGDIR}/All/${PF}.tbz2, but will be ${PKGDIR}/${CATEGORY}/${PF}.tbz2
1320                 in the rare event of a collision.  The prevent_collision() method can
1321                 be called to ensure that ${PKGDIR}/All/${PF}.tbz2 is available for a
1322                 specific cpv."""
1323                 if not self.populated:
1324                         self.populate()
1325                 mycpv = pkgname
1326                 mypath = self._pkg_paths.get(mycpv, None)
1327                 if mypath:
1328                         return os.path.join(self.pkgdir, mypath)
1329                 mycat, mypkg = catsplit(mycpv)
1330                 if self._all_directory:
1331                         mypath = os.path.join("All", mypkg + ".tbz2")
1332                         if mypath in self._pkg_paths.values():
1333                                 mypath = os.path.join(mycat, mypkg + ".tbz2")
1334                 else:
1335                         mypath = os.path.join(mycat, mypkg + ".tbz2")
1336                 self._pkg_paths[mycpv] = mypath # cache for future lookups
1337                 return os.path.join(self.pkgdir, mypath)
1338
1339         def isremote(self, pkgname):
1340                 """Returns true if the package is kept remotely and it has not been
1341                 downloaded (or it is only partially downloaded)."""
1342                 if self._remotepkgs is None or pkgname not in self._remotepkgs:
1343                         return False
1344                 # Presence in self._remotepkgs implies that it's remote. When a
1345                 # package is downloaded, state is updated by self.inject().
1346                 return True
1347
1348         def get_pkgindex_uri(self, pkgname):
1349                 """Returns the URI to the Packages file for a given package."""
1350                 return self._pkgindex_uri.get(pkgname)
1351
1352
1353
1354         def gettbz2(self, pkgname):
1355                 """Fetches the package from a remote site, if necessary.  Attempts to
1356                 resume if the file appears to be partially downloaded."""
1357                 tbz2_path = self.getname(pkgname)
1358                 tbz2name = os.path.basename(tbz2_path)
1359                 resume = False
1360                 if os.path.exists(tbz2_path):
1361                         if tbz2name[:-5] not in self.invalids:
1362                                 return
1363                         else:
1364                                 resume = True
1365                                 writemsg(_("Resuming download of this tbz2, but it is possible that it is corrupt.\n"),
1366                                         noiselevel=-1)
1367
1368                 mydest = os.path.dirname(self.getname(pkgname))
1369                 self._ensure_dir(mydest)
1370                 # urljoin doesn't work correctly with unrecognized protocols like sftp
1371                 if self._remote_has_index:
1372                         rel_url = self._remotepkgs[pkgname].get("PATH")
1373                         if not rel_url:
1374                                 rel_url = pkgname+".tbz2"
1375                         remote_base_uri = self._remotepkgs[pkgname]["BASE_URI"]
1376                         url = remote_base_uri.rstrip("/") + "/" + rel_url.lstrip("/")
1377                 else:
1378                         url = self.settings["PORTAGE_BINHOST"].rstrip("/") + "/" + tbz2name
1379                 protocol = urlparse(url)[0]
1380                 fcmd_prefix = "FETCHCOMMAND"
1381                 if resume:
1382                         fcmd_prefix = "RESUMECOMMAND"
1383                 fcmd = self.settings.get(fcmd_prefix + "_" + protocol.upper())
1384                 if not fcmd:
1385                         fcmd = self.settings.get(fcmd_prefix)
1386                 success = portage.getbinpkg.file_get(url, mydest, fcmd=fcmd)
1387                 if not success:
1388                         try:
1389                                 os.unlink(self.getname(pkgname))
1390                         except OSError:
1391                                 pass
1392                         raise portage.exception.FileNotFound(mydest)
1393                 self.inject(pkgname)
1394
1395         def _load_pkgindex(self):
1396                 pkgindex = self._new_pkgindex()
1397                 try:
1398                         f = io.open(_unicode_encode(self._pkgindex_file,
1399                                 encoding=_encodings['fs'], errors='strict'),
1400                                 mode='r', encoding=_encodings['repo.content'],
1401                                 errors='replace')
1402                 except EnvironmentError:
1403                         pass
1404                 else:
1405                         try:
1406                                 pkgindex.read(f)
1407                         finally:
1408                                 f.close()
1409                 return pkgindex
1410
1411         def _get_digests(self, pkg):
1412
1413                 try:
1414                         cpv = pkg.cpv
1415                 except AttributeError:
1416                         cpv = pkg
1417
1418                 digests = {}
1419                 metadata = None
1420                 if self._remotepkgs is None or cpv not in self._remotepkgs:
1421                         for d in self._load_pkgindex().packages:
1422                                 if d["CPV"] == cpv:
1423                                         metadata = d
1424                                         break
1425                 else:
1426                         metadata = self._remotepkgs[cpv]
1427                 if metadata is None:
1428                         return digests
1429
1430                 for k in hashfunc_map:
1431                         v = metadata.get(k)
1432                         if not v:
1433                                 continue
1434                         digests[k] = v
1435
1436                 if "SIZE" in metadata:
1437                         try:
1438                                 digests["size"] = int(metadata["SIZE"])
1439                         except ValueError:
1440                                 writemsg(_("!!! Malformed SIZE attribute in remote " \
1441                                 "metadata for '%s'\n") % cpv)
1442
1443                 return digests
1444
1445         def digestCheck(self, pkg):
1446                 """
1447                 Verify digests for the given package and raise DigestException
1448                 if verification fails.
1449                 @rtype: bool
1450                 @return: True if digests could be located, False otherwise.
1451                 """
1452
1453                 digests = self._get_digests(pkg)
1454
1455                 if not digests:
1456                         return False
1457
1458                 try:
1459                         cpv = pkg.cpv
1460                 except AttributeError:
1461                         cpv = pkg
1462
1463                 pkg_path = self.getname(cpv)
1464                 hash_filter = _hash_filter(
1465                         self.settings.get("PORTAGE_CHECKSUM_FILTER", ""))
1466                 if not hash_filter.transparent:
1467                         digests = _apply_hash_filter(digests, hash_filter)
1468                 eout = EOutput()
1469                 eout.quiet = self.settings.get("PORTAGE_QUIET") == "1"
1470                 ok, st = _check_distfile(pkg_path, digests, eout, show_errors=0)
1471                 if not ok:
1472                         ok, reason = verify_all(pkg_path, digests)
1473                         if not ok:
1474                                 raise portage.exception.DigestException(
1475                                         (pkg_path,) + tuple(reason))
1476
1477                 return True
1478
1479         def getslot(self, mycatpkg):
1480                 "Get a slot for a catpkg; assume it exists."
1481                 myslot = ""
1482                 try:
1483                         myslot = self.dbapi._pkg_str(mycatpkg, None).slot
1484                 except KeyError:
1485                         pass
1486                 return myslot