conary.lib.util

Source Code for Module conary.lib.util

1 # 2 # Copyright (c) 2004-2008 rPath, Inc. 3 # 4 # This program is distributed under the terms of the Common Public License, 5 # version 1.0. A copy of this license should have been distributed with this 6 # source file in a file called LICENSE. If it is not present, the license 7 # is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. 8 # 9 # This program is distributed in the hope that it will be useful, but 10 # without any warranty; without even the implied warranty of merchantability 11 # or fitness for a particular purpose. See the Common Public License for 12 # full details. 13 # 14 15 import bdb 16 import bz2 17 import debugger 18 import fcntl 19 import errno 20 import itertools 21 import log 22 import misc 23 import os 24 import re 25 import select 26 import shutil 27 import signal 28 import stat 29 import string 30 import StringIO 31 import subprocess 32 import struct 33 import sys 34 import tempfile 35 import time 36 import urllib 37 import urlparse 38 import weakref 39 import xmlrpclib 40 import zlib 41 42 from conary.lib import fixedglob, graph, log 43 44 # Simple ease-of-use extensions to python libraries 45

46 -def normpath(path):

47 s = os.path.normpath(path) 48 if s.startswith(os.sep + os.sep): 49 return s[1:] 50 return s

51

52 -def realpath(path):

53 # returns the real path of a file, if and only if it is not a symbolic 54 # link 55 if not os.path.exists(path): 56 return path 57 if stat.S_ISLNK(os.lstat(path)[stat.ST_MODE]): 58 return path 59 return os.path.realpath(path)

60

61 -def isregular(path):

62 return stat.S_ISREG(os.lstat(path)[stat.ST_MODE])

63

64 -def mkdirChain(*paths):

65 _ignoredErrors = set((errno.ENOENT, errno.ENOTDIR, errno.EACCES)) 66 for path in paths: 67 if path[0] != os.sep: 68 path = os.getcwd() + os.sep + path 69 normpath = os.path.normpath(path) 70 71 try: 72 misc.mkdirIfMissing(normpath) 73 continue 74 except OSError, exc: 75 if exc.errno not in _ignoredErrors: 76 raise 77 78 os.makedirs(normpath)

79

80 -def _searchVisit(arg, dirname, names):

81 file = arg[0] 82 path = arg[1] 83 testname = '%s%s%s' %(dirname, os.sep, file) 84 if file in names: 85 path[0] = testname 86 del names

87

88 -def searchPath(file, basepath):

89 path = [ None ] 90 # XXX replace with os.walk in python 2.3, to cut short properly 91 os.path.walk(basepath, _searchVisit, (file, path)) 92 return path[0]

93

94 -def searchFile(file, searchdirs, error=None):

95 for dir in searchdirs: 96 s = "%s%s%s" %(dir, os.sep, file) 97 if os.path.exists(s): 98 return s 99 if error: 100 raise OSError, (errno.ENOENT, os.strerror(errno.ENOENT)) 101 return None

102

103 -def findFile(file, searchdirs):

104 return searchFile(file, searchdirs, error=1)

105

106 -def which (filename):

107 if not os.environ.has_key('PATH') or os.environ['PATH'] == '': 108 p = os.defpath 109 else: 110 p = os.environ['PATH'] 111 112 pathlist = p.split (os.pathsep) 113 114 for path in pathlist: 115 f = os.path.join(path, filename) 116 if os.access(f, os.X_OK): 117 return f 118 return None

119

120 -def recurseDirectoryList(topdir, withDirs=False):

121 """Recursively list all files in the directory""" 122 items = [topdir] 123 while items: 124 item = items.pop() 125 if os.path.islink(item) or os.path.isfile(item): 126 yield item 127 continue 128 # Directory 129 listdir = os.listdir(item) 130 # Add the contents of the directory in reverse order (we use pop(), so 131 # last element in the list is the one popped out) 132 listdir.sort() 133 listdir.reverse() 134 listdir = [ os.path.join(item, x) for x in listdir ] 135 items.extend(listdir) 136 137 if withDirs: 138 # This is useful if one wants to catch empty directories 139 yield item

140

141 -def normurl(url):

142 surl = list(urlparse.urlsplit(url)) 143 if surl[2] == '': 144 surl[2] = '/' 145 elif surl[2] != '/': 146 tail = '' 147 if surl[2].endswith('/'): 148 tail = '/' 149 surl[2] = normpath(surl[2]) + tail 150 return urlparse.urlunsplit(surl)

151 152 errorMessage = ''' 153 ERROR: An unexpected condition has occurred in Conary. This is 154 most likely due to insufficient handling of erroneous input, but 155 may be some other bug. In either case, please report the error at 156 http://issues.rpath.com/ and attach to the issue the file 157 %(stackfile)s 158 159 Then, for more complete information, please run the following script: 160 conary-debug "%(command)s" 161 You can attach the resulting archive to your issue report at 162 http://issues.rpath.com/ For more information, or if you have 163 trouble with the conary-debug command, go to: 164 http://wiki.rpath.com/wiki/Conary:How_To_File_An_Effective_Bug_Report 165 166 To get a debug prompt, rerun the command with --debug-all 167 168 Error details follow: 169 170 %(filename)s:%(lineno)s 171 %(errtype)s: %(errmsg)s 172 173 The complete related traceback has been saved as %(stackfile)s 174 ''' 175 _debugAll = False 176

177 -def genExcepthook(debug=True, 178 debugCtrlC=False, prefix='conary-error-', 179 catchSIGUSR1=True, error=errorMessage):

180 def SIGUSR1Handler(signum, frame): 181 global _debugAll 182 _debugAll = True 183 print >>sys.stderr, '<Turning on KeyboardInterrupt catching>'

184 185 def excepthook(typ, value, tb): 186 if typ is bdb.BdbQuit: 187 sys.exit(1) 188 sys.excepthook = sys.__excepthook__ 189 if not _debugAll and (typ == KeyboardInterrupt and not debugCtrlC): 190 sys.exit(1) 191 192 out = BoundedStringIO() 193 formatTrace(typ, value, tb, stream = out, withLocals = False) 194 out.write("\nFull stack:\n") 195 formatTrace(typ, value, tb, stream = out, withLocals = True) 196 out.seek(0) 197 tbString = out.read() 198 del out 199 if log.syslog is not None: 200 log.syslog("command failed\n%s", tbString) 201 202 if debug or _debugAll: 203 formatTrace(typ, value, tb, stream = sys.stderr, 204 withLocals = False) 205 if sys.stdout.isatty() and sys.stdin.isatty(): 206 debugger.post_mortem(tb, typ, value) 207 else: 208 sys.exit(1) 209 elif log.getVerbosity() is log.DEBUG: 210 log.debug(tbString) 211 else: 212 cmd = sys.argv[0] 213 if cmd.endswith('/commands/conary'): 214 cmd = cmd[:len('/commands/conary')] + '/bin/conary' 215 elif cmd.endswith('/commands/cvc'): 216 cmd = cmd[:len('/commands/cvc')] + '/bin/cvc' 217 218 origTb = tb 219 cmd = normpath(cmd) 220 sys.argv[0] = cmd 221 while tb.tb_next: tb = tb.tb_next 222 lineno = tb.tb_frame.f_lineno 223 filename = tb.tb_frame.f_code.co_filename 224 tmpfd, stackfile = tempfile.mkstemp('.txt', prefix) 225 os.write(tmpfd, tbString) 226 os.close(tmpfd) 227 228 sys.stderr.write(error % dict(command=' '.join(sys.argv), 229 filename=filename, 230 lineno=lineno, 231 errtype=type.__name__, 232 errmsg=value, 233 stackfile=stackfile)) 234 235 #if catchSIGUSR1: 236 # signal.signal(signal.SIGUSR1, SIGUSR1Handler) 237 return excepthook 238 239 240

241 -def _handle_rc(rc, cmd):

242 if rc: 243 if not os.WIFEXITED(rc): 244 info = 'Shell command "%s" killed with signal %d' \ 245 %(cmd, os.WTERMSIG(rc)) 246 if os.WEXITSTATUS(rc): 247 info = 'Shell command "%s" exited with exit code %d' \ 248 %(cmd, os.WEXITSTATUS(rc)) 249 log.error(info) 250 raise RuntimeError, info

251

252 -def execute(cmd, destDir=None, verbose=True):

253 """ 254 similar to os.system, but raises errors if exit code != 0 and closes stdin 255 so processes can never block on user input 256 """ 257 if verbose: 258 log.info(cmd) 259 rc = subprocess.call(cmd, shell=True, cwd=destDir, stdin=open(os.devnull)) 260 # form the rc into a standard exit status 261 if rc < 0: 262 # turn rc positive 263 rc = rc * -1 264 else: 265 # shift the return code into the high bits 266 rc = rc << 8 267 _handle_rc(rc, cmd)

268

269 -class popen:

270 """ 271 Version of popen() that throws errors on close(), unlike os.popen() 272 """ 273 # unfortunately, can't derive from os.popen. Add methods as necessary.

274 - def __init__(self, *args):

275 self.p = os.popen(*args) 276 self.write = self.p.write 277 self.read = self.p.read 278 self.readline = self.p.readline 279 self.readlines = self.p.readlines 280 self.writelines = self.p.writelines

281

282 - def close(self, *args):

283 rc = self.p.close(*args) 284 _handle_rc(rc, self.p.