Source Code for Module conary.conaryclient.clone

   1  # 
   2  # Copyright (c) 2005-2008 rPath, Inc. 
   3  # 
   4  # This program is distributed under the terms of the Common Public License, 
   5  # version 1.0. A copy of this license should have been distributed with this 
   6  # source file in a file called LICENSE. If it is not present, the license 
   7  # is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. 
   8  # 
   9  # This program is distributed in the hope that it will be useful, but 
  10  # without any warranty; without even the implied warranty of merchantability 
  11  # or fitness for a particular purpose. See the Common Public License for 
  12  # full details. 
  13  """ 
  14  Implementation of "clone" + "promote" functionality. 
  15   
  16  Cloning creates a copy of a trove on a related branch, with the only link 
  17  back to the original branch being through the "clonedFrom" link. 
  18  """ 
  19  # NOTE FOR READING THE CODE: creating the copy is easy.  It's determining 
  20  # whether or not the clone is necessary that is complicated.  To that end  
  21  # we have: 
  22  # 
  23  #   The chooser: The chooser contains the algorithm for determining whether 
  24  #                a particular trove should be cloned or not, and where it 
  25  #                should be cloned. 
  26  # 
  27  #   The leafMap: keeps track of the relevant current state of the repository - 
  28  #                what troves are at the leaves, and where they were cloned 
  29  #                from. 
  30  # 
  31  #   The cloneMap: keeps track of the relationship between troves we might clone 
  32  #                 and where they would be cloned to. 
  33  # 
  34  #   The cloneJob: keeps track of the actual clones we're going to perform 
  35  #                 as well as the clones we would perform but aren't because 
  36  #                 they have already been cloned. 
  37  # 
  38  # I've been thinking about combining the cloneMap and leafMap. 
  39   
  40  import itertools 
  41  import os 
  42  import tempfile 
  43  import time 
  44   
  45  from conary import callbacks 
  46  from conary import errors, files 
  47  from conary import trove 
  48  from conary import versions 
  49  from conary.build.nextversion import nextVersions 
  50  from conary.conarycfg import selectSignatureKey 
  51  from conary.deps import deps 
  52  from conary.lib import log 
  53  from conary.repository import changeset, filecontents 
  54  from conary.repository import trovesource 
  55  from conary.repository import errors as neterrors 
  56   
  57  V_LOADED = 0 
  58  V_BREQ = 1 
  59  V_REFTRV = 2 
  60   
  61  # don't change  
  62  DEFAULT_MESSAGE = 1 
  63   
  64 -class CloneJob(object): 
  65 -    def __init__(self, options): 
  66          self.cloneJob = {} 
  67          self.preCloned = {} 
  68          self.options = options 
  69   
  70 -    def add(self, troveTup): 
  71          self.cloneJob[troveTup] = None 
  72   
  73 -    def alreadyCloned(self, troveTup): 
  74          self.cloneJob.pop(troveTup, False) 
  75          self.preCloned[troveTup] = True 
  76   
  77 -    def target(self, troveTup, targetVersion): 
  78          self.cloneJob[troveTup] = targetVersion 
  79   
  80 -    def iterTargetList(self): 
  81          return self.cloneJob.iteritems() 
  82   
  83 -    def getTrovesToClone(self): 
  84          return self.cloneJob.keys() 
  85   
  86 -    def getPreclonedTroves(self): 
  87          return self.preCloned.keys() 
  88   
  89 -    def isEmpty(self): 
  90          return not self.cloneJob 
  91   
  92  # target for the maximum number of files to handle in one pass 
  93  MAX_CLONE_FILES  = 5000 
  94  # threshhold for using a changeset instead of getting individual files 
  95  CHANGESET_MULTIPLE = 3 
  96   
  97 -class ClientClone: 
  98   
  99 -    def createCloneChangeSet(self, targetBranch, troveList, 
 100                               updateBuildInfo=True, message=DEFAULT_MESSAGE, 
 101                               infoOnly=False, fullRecurse=False, 
 102                               cloneSources=False, callback=None,  
 103                               trackClone=True, excludeGroups=False): 
 104          targetMap = dict((x[1].branch(), targetBranch) for x in troveList) 
 105          return self.createTargetedCloneChangeSet(targetMap, 
 106                                                 troveList, 
 107                                                 fullRecurse=fullRecurse, 
 108                                                 cloneSources=cloneSources, 
 109                                                 trackClone=trackClone, 
 110                                                 callback=callback, 
 111                                                 message=message, 
 112                                                 updateBuildInfo=updateBuildInfo, 
 113                                                 infoOnly=infoOnly, 
 114                                                 excludeGroups=excludeGroups) 
 115   
 116 -    def createTargetedCloneChangeSet(self, targetMap, troveList, 
 117                                       updateBuildInfo=True, infoOnly=False, 
 118                                       callback=None, message=DEFAULT_MESSAGE, 
 119                                       trackClone=True, fullRecurse=True, 
 120                                       cloneOnlyByDefaultTroves=False, 
 121                                       cloneSources=True, excludeGroups=False): 
 122          cloneOptions = CloneOptions(fullRecurse=fullRecurse, 
 123                              cloneSources=cloneSources, 
 124                              trackClone=trackClone, 
 125                              callback=callback, 
 126                              message=message, 
 127                              cloneOnlyByDefaultTroves=cloneOnlyByDefaultTroves, 
 128                              updateBuildInfo=updateBuildInfo, 
 129                              infoOnly=infoOnly, 
 130                              bumpGroupVersions=True, 
 131                              excludeGroups=excludeGroups) 
 132          chooser = CloneChooser(targetMap, troveList, cloneOptions) 
 133          return self._createCloneChangeSet(chooser, cloneOptions) 
 134      # bw compatibility 
 135      createSiblingCloneChangeSet = createTargetedCloneChangeSet 
 136   
 137 -    def createCloneChangeSetWithOptions(self, chooser, cloneOptions): 
 138          return self._createCloneChangeSet(chooser, cloneOptions) 
 139   
 140 -    def _createCloneChangeSet(self, chooser, cloneOptions): 
 141          callback = cloneOptions.callback 
 142          troveCache = TroveCache(self.repos, callback) 
 143   
 144          cloneJob, cloneMap, leafMap = self._createCloneJob(cloneOptions, 
 145                                                             chooser, 
 146                                                             troveCache) 
 147          if cloneJob.isEmpty(): 
 148              log.warning('Nothing to clone!') 
 149              return False, None 
 150   
 151          newTroveList = self._buildTroves(chooser, cloneMap, cloneJob, 
 152                                           leafMap, troveCache, callback) 
 153          if newTroveList is None: 
 154              return False, None 
 155   
 156          _logMe('new troves calculated') 
 157   
 158          if cloneOptions.infoOnly: 
 159              # build an absolute changeset. it's faster and easier. 
 160              cs = changeset.ChangeSet() 
 161              for oldVersion, newTrove in newTroveList: 
 162                  cs.newTrove(newTrove.diff(None, absolute = True)[0]) 
 163              callback.done() 
 164              return True, cs 
 165   
 166          finalCs = self._buildChangeSet(troveCache, newTroveList, callback) 
 167   
 168          callback.prefix = '' 
 169          callback.done() 
 170          return True, finalCs 
 171   
 172 -    def _buildChangeSet(self, troveCache, finalTroveList, callback): 
 173          def _sameHost(v1, v2): 
 174              return v1.trailingLabel().getHost() == v2.trailingLabel().getHost() 
 175   
 176          # What should each TroveChangeSet be relative to? If the original 
 177          # version was on the same server, great (because we don't have to 
 178          # include any file contents!). Otherwise, look for something on the 
 179          # target label because it is likely close to the new one. 
 180          # 
 181          # Note that what the diff is relative to is not the same as the 
 182          # fromVersion in the finalTroveList. fromVersion is the version 
 183          # we're cloning/shadowing from. The diff is always relative to 
 184          # something in the target repository. We call the version the diff 
 185          # is relative to the oldVersion. 
 186          searchDict = {} 
 187          for (fromVersion, finalTrove) in finalTroveList: 
 188              if not _sameHost(fromVersion, finalTrove.getVersion()): 
 189                  name, version, flavor = finalTrove.getNameVersionFlavor() 
 190                  label = version.trailingLabel() 
 191                  searchDict.setdefault(name, {}) 
 192                  searchDict[name].setdefault(label, []) 
 193                  searchDict[name][label].append(flavor) 
 194   
 195          matches = self.repos.getTroveLeavesByLabel(searchDict) 
 196   
 197          oldTrovesNeeded = [] 
 198          for (fromVersion, finalTrove) in finalTroveList: 
 199              name, version, flavor = finalTrove.getNameVersionFlavor() 
 200              if _sameHost(fromVersion, finalTrove.getVersion()): 
 201                  oldTrovesNeeded.append((name, fromVersion, flavor)) 
 202              else: 
 203                  match = None 
 204                  versionD = matches.get(name, {}) 
 205                  for matchVersion, flavorList in versionD.iteritems(): 
 206                      if (matchVersion.trailingLabel() == 
 207                                  version.trailingLabel() 
 208                              and flavor in flavorList): 
 209                          match = matchVersion 
 210   
 211                  if match is None: 
 212                      # keep oldTrovesNeeded parallel to finalTroveList 
 213                      oldTrovesNeeded.append(None) 
 214                  else: 
 215                      oldTrovesNeeded.append((name, match, flavor)) 
 216   
 217          oldTroves = troveCache.getTroves( 
 218              [ x for x in oldTrovesNeeded if x is not None ], withFiles=True) 
 219   
 220          # we periodically write file contents to disk and merge in a new 
 221          # changeset to save RAM. promotes can get large. 
 222          # 
 223          # Now to try and explain getting file streams and contents. If there 
 224          # are few contents changed, we're better off using getFileVersions, but 
 225          # if lots changes, we're better off just grabbing the whole bloody 
 226          # changeset. If more than 1/3rd of the files changed, let's grab the 
 227          # changeset. Note that this percent is completely arbitrary. We also 
 228          # want to consolidate getFileVersions() and createChangeSet() calls. 
 229          # Once we've found 5000 files to add to the current change set, we'll 
 230          # add those, write the change set, merge it, and start again. Got all 
 231          # that? 
 232          finalCs = changeset.ReadOnlyChangeSet() 
 233          cs = changeset.ChangeSet() 
 234          fileCount = 0 
 235          jobList = [