1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 import bdb
16 import bz2
17 import debugger
18 import errno
19 import fcntl
20 import fnmatch
21 import gzip
22 import misc
23 import os
24 import re
25 import select
26 import shutil
27 import stat
28 import string
29 import StringIO
30 import struct
31 import subprocess
32 import sys
33 import tempfile
34 import time
35 import types
36 import urllib
37 import weakref
38 import xmlrpclib
39 import zlib
40
41 from conary.lib import fixedglob, log, api, urlparse
42 from conary.lib import networking
43
44
45 from conary.lib.formattrace import formatTrace
46
47
48
49
51 s = os.path.normpath(path)
52 if s.startswith(os.sep + os.sep):
53 return s[1:]
54 return s
55
64
66 return stat.S_ISREG(os.lstat(path)[stat.ST_MODE])
67
68
79
80
81 @api.developerApi
83 """
84 Make one or more directories if they do not already exist, including any
85 needed parent directories. Similar to L{os.makedirs} except that it does
86 not error if the requested directory already exists, and it is more
87 resilient to race conditions.
88 """
89 for path in paths:
90 path = normpath(os.path.abspath(path))
91 if not os.path.exists(path):
92 _mkdirs(path)
93
99
101 for dir in searchdirs:
102 s = "%s%s%s" %(dir, os.sep, file)
103 if os.path.exists(s):
104 return s
105 if error:
106 raise OSError, (errno.ENOENT, os.strerror(errno.ENOENT))
107 return None
108
111
113 if not os.environ.has_key('PATH') or os.environ['PATH'] == '':
114 p = os.defpath
115 else:
116 p = os.environ['PATH']
117
118 pathlist = p.split (os.pathsep)
119
120 for path in pathlist:
121 f = os.path.join(path, filename)
122 if os.access(f, os.X_OK):
123 return f
124 return None
125
127 """Recursively list all files in the directory"""
128 items = [topdir]
129 while items:
130 item = items.pop()
131 if os.path.islink(item) or os.path.isfile(item):
132 yield item
133 continue
134
135 listdir = os.listdir(item)
136
137
138 listdir.sort()
139 listdir.reverse()
140 listdir = [ os.path.join(item, x) for x in listdir ]
141 items.extend(listdir)
142
143 if withDirs:
144
145 yield item
146
148 surl = list(urlparse.urlsplit(url))
149 if surl[2] == '':
150 surl[2] = '/'
151 elif surl[2] != '/':
152 tail = ''
153 if surl[2].endswith('/'):
154 tail = '/'
155 surl[2] = normpath(surl[2]) + tail
156 return urlparse.urlunsplit(surl)
157
158 errorMessage = '''
159 ERROR: An unexpected condition has occurred in Conary. This is
160 most likely due to insufficient handling of erroneous input, but
161 may be some other bug. In either case, please report the error at
162 http://issues.rpath.com/ and attach to the issue the file
163 %(stackfile)s
164
165 Then, for more complete information, please run the following script:
166 conary-debug "%(command)s"
167 You can attach the resulting archive to your issue report at
168 http://issues.rpath.com/ For more information, or if you have
169 trouble with the conary-debug command, go to:
170 http://wiki.rpath.com/wiki/Conary:How_To_File_An_Effective_Bug_Report
171
172 To get a debug prompt, rerun the command with --debug-all
173
174 Error details follow:
175
176 %(filename)s:%(lineno)s
177 %(errtype)s: %(errmsg)s
178
179 The complete related traceback has been saved as %(stackfile)s
180 '''
181 _debugAll = False
182
183 @api.developerApi
187 def SIGUSR1Handler(signum, frame):
188 global _debugAll
189 _debugAll = True
190 print >>sys.stderr, '<Turning on KeyboardInterrupt catching>'
191
192 def excepthook(typ, value, tb):
193 if typ is bdb.BdbQuit:
194 sys.exit(1)
195
196 sys.excepthook = sys.__excepthook__
197 if not _debugAll and (typ == KeyboardInterrupt and not debugCtrlC):
198 sys.exit(1)
199
200 out = BoundedStringIO()
201 formatTrace(typ, value, tb, stream = out, withLocals = False)
202 out.write("\nFull stack:\n")
203 formatTrace(typ, value, tb, stream = out, withLocals = True)
204 out.seek(0)
205 tbString = out.read()
206 del out
207 if log.syslog is not None:
208 log.syslog("command failed\n%s", tbString)
209
210 if debug or _debugAll:
211 formatTrace(typ, value, tb, stream = sys.stderr,
212 withLocals = False)
213 if sys.stdout.isatty() and sys.stdin.isatty():
214 debugger.post_mortem(tb, typ, value)
215 else:
216 sys.exit(1)
217 elif log.getVerbosity() is log.DEBUG:
218 log.debug(tbString)
219 else:
220 cmd = sys.argv[0]
221 if cmd.endswith('/commands/conary'):
222 cmd = cmd[:len('/commands/conary')] + '/bin/conary'
223 elif cmd.endswith('/commands/cvc'):
224 cmd = cmd[:len('/commands/cvc')] + '/bin/cvc'
225
226 origTb = tb
227 cmd = normpath(cmd)
228 sys.argv[0] = cmd
229 while tb.tb_next: tb = tb.tb_next
230 lineno = tb.tb_frame.f_lineno
231 filename = tb.tb_frame.f_code.co_filename
232 tmpfd, stackfile = tempfile.mkstemp('.txt', prefix)
233 os.write(tmpfd, tbString)
234 os.close(tmpfd)
235
236 sys.stderr.write(error % dict(command=' '.join(sys.argv),
237 filename=filename,
238 lineno=lineno,
239 errtype=typ.__name__,
240 errmsg=value,
241 stackfile=stackfile))
242
243
244
245 return excepthook
246
247
248
250 if rc:
251 if not os.WIFEXITED(rc):
252 info = 'Shell command "%s" killed with signal %d' \
253 %(cmd, os.WTERMSIG(rc))
254 if os.WEXITSTATUS(rc):
255 info = 'Shell command "%s" exited with exit code %d' \
256 %(cmd, os.WEXITSTATUS(rc))
257 log.error(info)
258 raise RuntimeError, info
259
260 -def execute(cmd, destDir=None, verbose=True):
261 """
262 similar to os.system, but raises errors if exit code != 0 and closes stdin
263 so processes can never block on user input
264 """
265 if verbose:
266 log.info(cmd)
267 rc = subprocess.call(cmd, shell=True, cwd=destDir, stdin=open(os.devnull))
268
269 if rc < 0:
270
271 rc = rc * -1
272 else:
273
274 rc = rc << 8
275 _handle_rc(rc, cmd)
276
278 """
279 Version of popen() that throws errors on close(), unlike os.popen()
280 """
281
289
294
295
296
297 -def find(s, subs, start=0):
298 ret = -1
299 found = None
300 for sub in subs:
301 this = string.find(s, sub, start)
302 if this > -1 and ( ret < 0 or this < ret):
303 ret = this
304 found = s[this:this+1]
305 return (ret, found)
306
309
310
311
313 """Class encapsulating the logic required by the brace expander parser"""
323 "Concatenation operator"
324
325 @classmethod
343
344 @classmethod
354
355 @classmethod
357 haveComma = False
358 haveText = False
359
360 listObj.append(None)
361 outputQ = []
362 operators = []
363 lastWasLiteral = False
364 for item in listObj:
365 if isinstance(item, basestring):
366 if not haveText:
367 text = []
368 outputQ.append(text)
369 haveText = True
370 else:
371 text = outputQ[-1]
372 text.append(item)
373 continue
374 if haveText:
375 topNode = outputQ.pop()
376 topNode = ''.join(topNode)
377 haveText = False
378 outputQ.append(topNode)
379 lastWasLiteral = True
380
381 if item is None:
382
383 break
384 if item is cls.Comma:
385 haveComma = True
386 lastWasLiteral = False
387 while operators:
388 op = operators.pop()
389 outputQ.append(op)
390 operators.append(item)
391 continue
392 outputQ.append(item)
393 if not lastWasLiteral:
394 lastWasLiteral = True
395 continue
396
397 while operators and operators[-1] is not cls.Comma:
398 op = operators.pop()
399 outputQ.append(op)
400 operators.append(cls.Concat)
401 while operators:
402 op = operators.pop()
403 outputQ.append(op)
404
405 stack = []
406 opMap = {
407 cls.Comma: cls.Alternative,
408 cls.Concat: cls.Product,
409 }
410 for item in outputQ:
411 if not (item is cls.Comma or item is cls.Concat):
412 stack.append(item)
413 continue
414 op2 = stack.pop()
415 op1 = stack.pop()
416 ncls = opMap[item]
417 if isinstance(op1, ncls):
418 op1.append(op2)
419 stack.append(op1)
420 elif isinstance(op2, ncls):
421 op2[0:0] = [op1]
422 stack.append(op2)
423 else:
424 nobj = ncls()
425 nobj.extend([op1, op2])
426 stack.append(nobj)
427 ret = stack[0]
428 if not haveComma:
429 ret = cls.Alternative([ret])
430 return ret
431
432 @classmethod
434 for item in l:
435 if item is cls.Comma:
436 yield ','
437 else:
438 yield item
439
440 @classmethod
488
491
492 @api.publicApi
494 """
495 @raises ValueError: raised if paths has unbalanced braces
496 @raises OSError: raised in some cases where lstat on a path fails
497 """
498 pathlist = []
499 for path in braceExpand(paths):
500 pathlist.extend(fixedglob.glob(path))
501 return pathlist
502
503 @api.developerApi
504 -def rmtree(paths, ignore_errors=False, onerror=None):
514
524
525 -def remove(paths, quiet=False):
535
536 -def copyfile(sources, dest, verbose=True):
541
542 -def copyfileobj(source, dest, callback = None, digest = None,
543 abortCheck = None, bufSize = 128*1024, rateLimit = None,
544 sizeLimit = None, total=0):
545 if hasattr(dest, 'send'):
546 write = dest.send
547 else:
548 write = dest.write
549
550 if rateLimit is None:
551 rateLimit = 0
552
553 if not rateLimit == 0:
554 if rateLimit < 8 * 1024:
555 bufSize = 4 * 1024
556 else:
557 bufSize = 8 * 1024
558
559 rateLimit = float(rateLimit)
560
561 starttime = time.time()
562
563 copied = 0
564
565 if abortCheck and hasattr(source, 'fileno'):
566 pollObj = select.poll()
567 pollObj.register(source.fileno(), select.POLLIN)
568 else:
569 pollObj = None
570
571 while True:
572 if sizeLimit and (sizeLimit - copied < bufSize):
573 bufSize = sizeLimit - copied
574
575 if abortCheck:
576
577
578 l = []
579 while not l:
580 if abortCheck():
581 return None
582 if pollObj:
583 l = pollObj.poll(5000)
584 else:
585 break
586
587 buf = source.read(bufSize)
588 if not buf:
589 break
590
591 total += len(buf)
592 copied += len(buf)
593 write(buf)
594
595 if digest:
596 digest.update(buf)
597
598 now = time.time()
599 if now == starttime:
600 rate = 0
601 else:
602 rate = copied / ((now - starttime))
603
604 if callback:
605 callback(total, rate)
606
607 if copied == sizeLimit:
608 break
609
610 if rateLimit > 0 and rate > rateLimit:
611 time.sleep((copied / rateLimit) - (copied / rate))
612
613 return copied
614
619
621 sourcelist = arg[0]
622 sourcelen = arg[1]
623 dest = arg[2]
624 filemode = arg[3]
625 dirmode = arg[4]
626 if dirmode:
627 os.chmod(dirname, dirmode)
628 for name in names:
629 if filemode:
630 os.chmod(dirname+os.sep+name, filemode)
631 sourcelist.append(os.path.normpath(
632 dest + os.sep + dirname[sourcelen:] + os.sep + name))
633
634 -def copytree(sources, dest, symlinks=False, filemode=None, dirmode=None):
635 """
636 Copies tree(s) from sources to dest, returning a list of
637 the filenames that it has written.
638 """
639 sourcelist = []
640 for source in braceGlob(sources):
641 if os.path.isdir(source):
642 if source[-1] == '/':
643 source = source[:-1]
644 thisdest = '%s%s%s' %(dest, os.sep, os.path.basename(source))
645 log.debug('copying [tree] %s to %s', source, thisdest)
646 shutil.copytree(source, thisdest, symlinks)
647 if dirmode:
648 os.chmod(thisdest, dirmode)
649 os.path.walk(source, _copyVisit,
650 (sourcelist, len(source), thisdest, filemode, dirmode))
651 else:
652 log.debug('copying [file] %s to %s', source, dest)
653 shutil.copy2(source, dest)
654 if dest.endswith(os.sep):
655 thisdest = dest + os.sep + os.path.basename(source)
656 else:
657 thisdest = dest
658 if filemode:
659 os.chmod(thisdest, filemode)
660 sourcelist.append(thisdest)
661 return sourcelist
662
689
692
694 """Split the path at the operating system's separators.
695 Returns a list with the path components in reverse order.
696 Empty path components are stripped out.
697 Example: 'a//b//c/d' -> ['d', 'c', 'b', 'a']
698 """
699 while 1:
700 path, tail = os.path.split(path)
701 if not tail:
702 break
703 yield tail
704
706 """Split the path at the operating system's separators
707 Empty path components are stripped out
708 Example: 'a//b//c/d' -> ['a', 'b', 'c', 'd']
709 """
710 ret = list(splitPathReverse(path))
711 ret.reverse()
712 return ret
713
715 try:
716 iter.next()
717 raise AssertionError
718 except StopIteration:
719 return True
720
721 ref = weakref.ref
723 """
724 Implements a cache of arbitrary (hashable) objects where an object
725 can be looked up and have its cached value retrieved. This allows
726 a single copy of immutable objects to be kept in memory.
727 """
729 dict.__init__(self, *args)
730
731 def remove(k, selfref=ref(self)):
732 self = selfref()
733 if self is not None:
734 return dict.__delitem__(self, k)
735 self._remove = remove
736
739
742
745
748
751
754
757
759 """Get the memory usage.
760 @param pid: Process to analyze (None for current process)
761 """
762 if pid is None:
763 pfn = "/proc/self/statm"
764 else:
765 pfn = "/proc/%d/statm" % pid
766 line = open(pfn).readline()
767
768
769 arr = [ 4 * int(x) for x in line.split()[:6] ]
770 vmsize, vmrss, vmshared, text, lib, data = arr
771
772
773
774
775 return vmsize, vmrss, vmshared, text, lib, data
776
786
788 """
789 Inserts newItem into haystack, maintaining the sorted order. The
790 cmpIdx is the item number in the list of tuples to base comparisons on.
791 Duplicates items aren't added. Returns True if the item was added,
792 False if it was already present.
793
794 @param haystack: list of tuples.
795 @type haystack: list
796 @param newItem: The item to be inserted
797 @type newItem: tuple
798 @param cmpFn: Comparison function
799 @type cmpFn: function
800 @rtype: bool
801 """
802 start = 0
803 finish = len(haystack) - 1
804 while start < finish:
805 i = (start + finish) / 2
806
807 rc = cmpFn(haystack[i], newItem)
808 if rc == 0:
809 start = i
810 finish = i
811 break
812 elif rc < 0:
813 start = i + 1
814 else:
815 finish = i - 1
816
817 if start >= len(haystack):
818 haystack.append(newItem)
819 else:
820 rc = cmpFn(haystack[start], newItem)
821 if rc < 0:
822 haystack.insert(start + 1, newItem)
823 elif rc > 0:
824 haystack.insert(start, newItem)
825 else:
826 return False
827
828 return True
829
830 _tempdir = tempfile.gettempdir()
835
837 """
838 a wrapper for tempfile.mkstemp that uses a common prefix which
839 is set through settempdir()
840 """
841 if dir is None:
842 global _tempdir
843 dir = _tempdir
844 return tempfile.mkstemp(suffix=suffix, prefix=prefix, dir=dir, text=text)
845
847
848 tags = {}
849 ptrSize = len(struct.pack("@P", 0))
850
851 @staticmethod
854
857
858 @staticmethod
860 sendmsg(sock, [ struct.pack("@" + ("P" * len(l)),
861 *[ id(x) for x in l] ) ])
862
863 @staticmethod
868
871
872 - def send(self, sock):
873 stack = self.l[:]
874 allFds = []
875 toSend = []
876 handled = set()
877
878 while stack:
879 f = stack.pop()
880
881 if f in handled:
882 continue
883
884 fd = None
885 objDepList = []
886
887 dependsOn, s = f._sendInfo()
888
889 if type(dependsOn) == int:
890 fd = dependsOn
891 elif dependsOn is not None:
892 assert(type(dependsOn) == list)
893
894 notHandled = list(set(dependsOn) - set(handled))
895 if notHandled:
896 stack.append(f)
897 stack.extend(notHandled)
898 continue
899
900
901 objDepList = dependsOn
902
903 toSend.append((f, fd, objDepList, s))
904 handled.add(f)
905
906 fds = list(set([ x[1] for x in toSend if x[1] is not None]))
907 objsById = dict( (id(x[2]), x[2]) for x in toSend )
908
909 sendmsg(sock, [ struct.pack("@I", len(fds)) ] )
910 sendmsg(sock, [ struct.pack("@II", len(self.l), len(toSend)) ], fds)
911
912 for f, fd, objDepList, s in toSend:
913 if fd is None:
914 fdIndex = 0xffffffff
915 else:
916 fdIndex = fds.index(fd)
917
918 depList = objDepList
919
920 sendmsg(sock, [ struct.pack("@BIIIP", len(f._tag), fdIndex,
921 len(s), len(depList), id(f)),
922 f._tag, s ])
923 self.sendObjIds(sock, depList)
924
925 self.sendObjIds(sock, self.l)
926
927 @staticmethod
929 hdrSize = len(struct.pack("@BIIPP", 0, 0, 0, 0, 0))
930
931 q = IterableQueue()
932 s = recvmsg(sock, 4)
933 fdCount = struct.unpack("@I", s)[0]
934 if fdCount:
935 s, fds = recvmsg(sock, 8, fdCount)
936 else:
937 s = recvmsg(sock, 8, 0)
938
939 objCount, fileCount = struct.unpack("@II", s)
940
941 fileList = []
942 objById = {}
943
944 for i in range(fileCount):
945 s = recvmsg(sock, hdrSize)
946 tagLen, fdIndex, dataLen, depLen, thisId = struct.unpack("@BIIIP", s)
947 tag = recvmsg(sock, tagLen)
948 if dataLen:
949 s = recvmsg(sock, dataLen)
950 else:
951 s = ''
952
953 if not depLen:
954 depList = []
955 else:
956 depList = SendableFileSet.recvObjIds(sock, depLen)
957
958 if fdIndex != 0xffffffff:
959 assert(not depList)
960 dep = fds[fdIndex]
961 elif depList:
962 assert(fdIndex == 0xffffffff)
963 dep = [ objById[x] for x in depList ]
964 else:
965 dep = None
966
967 f = SendableFileSet.tags[tag]._fromInfo(dep, s)
968 objById[thisId] = f
969
970 fileList.append(f)
971
972 fileIds = SendableFileSet.recvObjIds(sock, objCount)
973 files = [ objById[x] for x in fileIds]
974
975 return files
976
978
979 _tag = 'efd'
980 __slots__ = [ 'fd' ]
981
983 self.fd = fd
984
985 fcntl.fcntl(self.fd, fcntl.F_SETFD, 1)
986
987 @staticmethod
991
993 return (self.fd, '-')
994
997
1001
1003 if self.fd is not None:
1004 try:
1005 self.close()
1006 except OSError:
1007 self.fd = None
1008
1009 - def read(self, bytes = -1):
1010
1011
1012 if bytes == -1:
1013 bufSize = 8 * 1024
1014 l = []
1015 while 1:
1016 s = os.read(self.fd, bufSize)
1017 if not s:
1018 return ''.join(l)
1019 l.append(s)
1020 return os.read(self.fd, bytes)
1021
1023 return os.ftruncate(self.fd, offset)
1024
1027
1028 - def pread(self, bytes, offset):
1030
1031 - def seek(self, offset, whence = 0):
1032 return os.lseek(self.fd, offset, whence)
1033
1035
1036 return os.lseek(self.fd, 0, 1)
1037
1038 SendableFileSet._register(ExtendedFdopen)
1039
1041
1042 __slots__ = [ 'fObj', 'name' ]
1043
1045 if not self.fObj:
1046 return
1047 self.fObj.close()
1048 self.fd = None
1049 self.fObj = None
1050
1052 return '<ExtendedFile %r>' % (self.name,)
1053
1054 - def __init__(self, path, mode = "r", buffering = True):
1065
1067
1068 _tag = 'efs'
1069
1070 @staticmethod
1074
1076 return (None, self.getvalue())
1077
1078 - def pread(self, bytes, offset):
1084
1085 SendableFileSet._register(ExtendedStringIO)
1086
1088
1089 _tag = "snf"
1090
1091 - def __init__(self, file, size, start = -1):
1101
1102 @staticmethod
1107
1109 return ([ self.file ], struct.pack("!II", self.size, self.start))
1110
1112 if hasattr(self.file, '_fdInfo'):
1113 fd, start, size = self.file._fdInfo()
1114 start += self.start
1115 size = self.size
1116 elif hasattr(self.file, 'fileno'):
1117 fd, start, size = self.file.fileno(), self.start, self.size
1118 else:
1119 return (None, None, None)
1120
1121 return (fd, start, size)
1122
1125
1126 - def read(self, bytes = -1, offset = None):
1127 if offset is None:
1128 readPos = self.pos
1129 else:
1130 readPos = offset
1131
1132 if bytes < 0 or (self.end - readPos) <= bytes:
1133
1134 count = self.end - readPos
1135 newPos = self.end
1136 else:
1137 count = bytes
1138 newPos = readPos + bytes
1139
1140 buf = self.file.pread(count, readPos + self.start)
1141
1142 if offset is None:
1143 self.pos = newPos
1144
1145 return buf
1146
1147 pread = read
1148
1149 - def seek(self, offset, whence = 0):
1150 if whence == 0:
1151 newPos = offset
1152 elif whence == 1:
1153 newPos = self.pos + offset
1154 else:
1155 newPos = self.size + offset
1156
1157 if newPos > self.size or newPos < 0:
1158 raise IOError("Position %d is outside file (len %d)"
1159 % (newPos, self.size))
1160
1161 self.pos = newPos
1162 return self.pos
1163
1166 SendableFileSet._register(SeekableNestedFile)
1167
1170 self.decomp = bz2.BZ2Decompressor()
1171 self.fobj = fobj
1172 self.leftover = ''
1173
1174 - def read(self, bytes):
1175 while 1:
1176 buf = self.fobj.read(2048)
1177 if not buf:
1178
1179 if self.leftover:
1180
1181
1182 if len(self.leftover) > bytes:
1183 rc = self.leftover[:bytes]
1184 self.leftover = self.leftover[bytes:]
1185 else:
1186 rc = self.leftover[:]
1187 self.leftover = None
1188 return rc
1189
1190 return None
1191
1192 self.leftover += self.decomp.decompress(buf)
1193
1194 if len(self.leftover) > bytes:
1195 rc = self.leftover[:bytes]
1196 self.leftover = self.leftover[bytes:]
1197 return rc
1202
1203 - def push(self, val):
1205
1207 if self.head:
1208 val = self.head.pop(0)
1209 return val
1210
1211 return self.iter.next()
1212
1214 self.head = []
1215 self.iter = iter
1216
1218
1220 try:
1221 self.val = self.iter.next()
1222 except StopIteration:
1223 self.done = True
1224
1226 if self.done:
1227 raise StopIteration
1228
1229 return self.val
1230
1232 if self.done:
1233 raise StopIteration
1234
1235 val = self.val
1236 self._next()
1237 return val
1238
1240 while True:
1241 yield self.next()
1242
1247
1249
1250 - def add(self, item):
1252
1253 - def peekRemainder(self):
1255
1257 while self.l:
1258 yield self.l.pop(0)
1259
1260 raise StopIteration
1261
1264
1266 """
1267 Return None if the path doesn't exist.
1268 """
1269 if not misc.exists(path):
1270 return None
1271
1272 try:
1273 sb = os.lstat(path)
1274 except OSError, e:
1275 if e.errno != errno.ENOENT:
1276 raise
1277 return None
1278
1279 return sb
1280
1282
1284 s = os.read(self.fd, 4096)
1285 if not s:
1286 if self.buf:
1287 s = self.buf
1288 self.buf = ''
1289 return [ s ]
1290
1291 return None
1292
1293 self.buf += s
1294
1295 lines = self.buf.split('\n')
1296 self.buf = lines[-1]
1297 del lines[-1]
1298
1299 return [ x + "\n" for x in lines ]
1300
1302 self.fd = fd
1303 self.buf = ''
1304
1305 exists = misc.exists
1306 removeIfExists = misc.removeIfExists
1307 pread = misc.pread
1308 res_init = misc.res_init
1309 sha1Uncompress = misc.sha1Uncompress
1310 fchmod = misc.fchmod
1311 fopenIfExists = misc.fopenIfExists
1312
1314 """Decorator to perform the housekeeping of opening/closing of fds"""
1315 def wrapper(self, *args, **kwargs):
1316 if self._realFd is not None:
1317
1318
1319 self._timestamp = time.time()
1320
1321 return getattr(self._realFd, method.func_name)(*args, **kwargs)
1322 if self._cache is None:
1323 raise Exception("Cache object is closed")
1324 try:
1325 self._cache()._getSlot()
1326 except ReferenceError:
1327
1328 raise
1329 self._reopen()
1330 return getattr(self._realFd, method.func_name)(*args, **kwargs)
1331 return wrapper
1332
1333
1335 __slots__ = ['path', 'marker', 'mode', '_cache', '_hash', '_realFd',
1336 '_timestamp']
1337 - def __init__(self, cache, path, mode):
1345
1351
1354
1356 """Called when the cache object gets destroyed"""
1357 self._close()
1358 self._cache = None
1359
1360 @_LazyFile_reopen
1361 - def read(self, bytes):
1363
1364 @_LazyFile_reopen
1365 - def pread(self, bytes, offset):
1367
1368 @_LazyFile_reopen
1369 - def seek(self, loc, type):
1371
1372 @_LazyFile_reopen
1375
1376 @_LazyFile_reopen
1379
1380 @_LazyFile_reopen
1383
1390
1403
1406
1409
1411 """An object tracking open files. It will serve file-like objects that get
1412 closed behind the scene (and reopened on demand) if the number of open
1413 files in the current process exceeds a threshold.
1414 The objects will close automatically when they fall out of scope.
1415 """
1416
1417 threshold = 900
1418
1419 @api.publicApi
1426
1427 @api.publicApi
1428 - def open(self, path, mode="r"):
1429 """
1430 @raises IOError: raised if there's an I/O error opening the fd
1431 @raises OSError: raised on other errors opening the fd
1432 """
1433 fd = _LazyFile(self, path, mode=mode)
1434 self._fdMap[fd._hash] = fd
1435
1436 fd.tell()
1437 return fd
1438
1440 try:
1441 return countOpenFileDescriptors()
1442 except OSError, e:
1443
1444 if e.errno != errno.EINVAL:
1445 raise
1446
1447 return len([ x for x in self._fdMap.values()
1448 if x._realFd is not None])
1449
1451 ret = self._fdCounter;
1452 self._fdCounter += 1;
1453 return ret;
1454
1477
1480
1481 @api.publicApi
1483 """
1484 @raises IOError: could be raised if tell() fails prior to close()
1485 """
1486
1487 for fd in self._fdMap.values():
1488 fd._close()
1489 fd._cache = None
1490 self._fdMap.clear()
1491
1493 """Release the file descriptors kept open by the LazyFile objects"""
1494 for fd in self._fdMap.values():
1495 fd._close()
1496
1497 __del__ = close
1498
1500
1501
1502
1503 __slots__ = []
1504
1506 for flag in self.__slots__:
1507 setattr(self, flag, False)
1508
1509 for (flag, val) in kwargs.iteritems():
1510 setattr(self, flag, val)
1511
1513 if type(val) != bool:
1514 raise TypeError, 'bool expected'
1515 object.__setattr__(self, flag, val)
1516
1518 return "%s(%s)" % (self.__class__.__name__,
1519 "".join( flag for flag in self.__slots__
1520 if getattr(self, flag) ) )
1521
1528
1529
1531 def wrapper(*args, **kwargs):
1532 try:
1533 return fn(*args, **kwargs)
1534 except IOError, e:
1535 if e.errno != errno.EPIPE:
1536 raise
1537 return
1538 return wrapper
1539
1540
1542
1543 @_FileIgnoreEpipe_ignoreEpipe
1544 - def write(self, *args):
1546
1547 @_FileIgnoreEpipe_ignoreEpipe
1548 - def close(self, *args):
1550
1552 return getattr(self.f, name)
1553
1556
1558 """
1559 An IO object that behaves like a StringIO.
1560 Data is stored in memory (just like in a StringIO) if shorter than
1561 maxMemorySize, or in a temporary file.
1562 """
1563 defaultMaxMemorySize = 65536
1564 __slots__ = ['_backend', '_backendType', 'maxMemorySize']
1565 - def __init__(self, buf='', maxMemorySize=None):
1572
1574 backend = object.__getattribute__(self, '_backend')
1575 if isinstance(backend, file):
1576
1577 return backend.write(s)
1578
1579
1580 maxMemorySize = object.__getattribute__(self, 'maxMemorySize')
1581
1582
1583 curPos = backend.tell()
1584 if curPos + len(s) < maxMemorySize:
1585
1586 return backend.write(s)
1587
1588 fd, name = tempfile.mkstemp(suffix=".tmp", prefix="tmpBSIO")
1589
1590 os.unlink(name)
1591 fcntl.fcntl(fd, fcntl.F_SETFD, 1)
1592 backendFile = os.fdopen(fd, "w+")
1593
1594 backend.seek(0)
1595 backendFile.write(backend.read(curPos))
1596 ret = backendFile.write(s)
1597 self._backend = backendFile
1598 self._backendType = "file"
1599 return ret
1600
1627
1630
1632
1633 locs = ['_backend', '_backendType', 'getBackendType', 'maxMemorySize']
1634 if attr in locs:
1635 return object.__getattribute__(self, attr)
1636
1637 if attr == 'write':
1638
1639 return object.__getattribute__(self, '_writeImpl')
1640
1641 if attr == 'truncate':
1642
1643 return object.__getattribute__(self, '_truncateImpl')
1644
1645 backend = object.__getattribute__(self, '_backend')
1646 return getattr(backend, attr)
1647
1649 """A string that is not printed in tracebacks"""
1651 return "<Protected Value>"
1652
1653 __repr__ = __safe_str__
1654
1656 _substArgs = None
1657 _templ = None
1658
1659 """A string template that hides parts of its components.
1660 The first argument is a template (see string.Template for a complete
1661 documentation). The values that can be filled in are using the format
1662 ${VAR} or $VAR. The keyword arguments are expanding the template.
1663 If one of the keyword arguments has a __safe_str__ method, its value is
1664 going to be hidden when this object's __safe_str__ is called."""
1665 - def __new__(cls, templ, **kwargs):
1671
1673 nargs = {}
1674 for k, v in self._substArgs.iteritems():
1675 if hasattr(v, '__safe_str__'):
1676 v = "<%s>" % k.upper()
1677 nargs[k] = v
1678 return self._templ.safe_substitute(nargs)
1679
1680 __repr__ = __safe_str__
1681
1683 """A function to split a URL in the format
1684 <scheme>://<user>:<pass>@<host>:<port>/<path>;<params>#<fragment>
1685 into a tuple
1686 (<scheme>, <user>, <pass>, <host>, <port>, <path>, <params>, <fragment>)
1687 Any missing pieces (user/pass) will be set to None.
1688 If the port is missing, it will be set to defaultPort; otherwise, the port
1689 should be a numeric value.
1690 """
1691 scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
1692 userpass, hostport = urllib.splituser(netloc)
1693 host, port = networking.splitHostPort(hostport)
1694
1695 if userpass:
1696 user, passwd = urllib.splitpasswd(userpass)
1697 if passwd:
1698 passwd = ProtectedString(passwd)
1699 else:
1700 user, passwd = None, None
1701 return scheme, user, passwd, host, port, path, \
1702 query or None, fragment or None
1703
1705 """Recompose a split URL as returned by urlSplit into a single string
1706 """
1707 scheme, user, passwd, host, port, path, query, fragment = urlTuple
1708 userpass = None
1709 if user:
1710 if passwd:
1711 userpass = "%s:${passwd}" % (urllib.quote(user))
1712 else:
1713 userpass = urllib.quote(user)
1714 if host and ':' in host:
1715
1716 host = '[%s]' % (host,)
1717 if port is not None:
1718 hostport = urllib.quote("%s:%s" % (host, port), safe = ':[]')
1719 else:
1720 hostport = host
1721 netloc = hostport
1722 if userpass:
1723 netloc = "%s@%s" % (userpass, hostport)
1724 urlTempl = urlparse.urlunsplit((scheme, netloc, path, query, fragment))
1725 if passwd is None:
1726 return urlTempl
1727 return ProtectedTemplate(urlTempl, passwd = ProtectedString(urllib.quote(passwd)))
1728
1729
1731 """Marshaller for XMLRPC data"""
1732 dispatch = xmlrpclib.Marshaller.dispatch.copy()
1733 - def dump_string(self, value, write, escape=xmlrpclib.escape):
1734 try:
1735 value = value.encode("ascii")
1736 except UnicodeError:
1737 sio = StringIO.StringIO()
1738 xmlrpclib.Binary(value).encode(sio)
1739 write(sio.getvalue())
1740 return
1741 return xmlrpclib.Marshaller.dump_string(self, value, write, escape)
1742
1743 - def dump(self, values, stream):
1759
1760 - def dumps(self, values):
1761 sio = StringIO.StringIO()
1762 self.dump(values, sio)
1763 return sio.getvalue()
1764
1765 - def _dump(self, value, write):
1766
1767
1768 try:
1769 f = self.dispatch[type(value)]
1770 except KeyError:
1771
1772 try:
1773 value.__dict__
1774 except:
1775 raise TypeError, "cannot marshal %s objects" % type(value)
1776
1777
1778
1779 for type_ in type(value).__mro__:
1780 if type_ in self.dispatch.keys():
1781 raise TypeError, "cannot marshal %s objects" % type(value)
1782 f = self.dispatch[types.InstanceType]
1783 f(self, value, write)
1784
1785 dispatch[str] = dump_string
1786 dispatch[ProtectedString] = dump_string
1787 dispatch[ProtectedTemplate] = dump_string
1788
1804
1812
1813 -def xmlrpcDump(params, methodname=None, methodresponse=None, stream=None,
1814 encoding=None, allow_none=False):
1815 assert isinstance(params, tuple) or isinstance(params, xmlrpclib.Fault),\
1816 "argument must be tuple or Fault instance"
1817 if isinstance(params, xmlrpclib.Fault):
1818 methodresponse = 1
1819 elif methodresponse and isinstance(params, tuple):
1820 assert len(params) == 1, "response tuple must be a singleton"
1821
1822 if not encoding:
1823 encoding = "utf-8"
1824
1825 m = XMLRPCMarshaller(encoding, allow_none)
1826 if encoding != "utf-8":
1827 xmlheader = "<?xml version='1.0' encoding='%s'?>\n" % str(encoding)
1828 else:
1829 xmlheader = "<?xml version='1.0'?>\n"
1830
1831 if stream is None:
1832 io = StringIO.StringIO(stream)
1833 else:
1834 io = stream
1835
1836
1837 if methodname:
1838 if not isinstance(methodname, str):
1839 methodname = methodname.encode(encoding)
1840 io.write(xmlheader)
1841 io.write("<methodCall>\n")
1842 io.write("<methodName>%s</methodName>\n" % methodname)
1843 m.dump(params, io)
1844 io.write("</methodCall>\n")
1845 elif methodresponse:
1846 io.write(xmlheader)
1847 io.write("<methodResponse>\n")
1848 m.dump(params, io)
1849 io.write("</methodResponse>\n")
1850 else:
1851
1852 m.dump(params, io)
1853
1854 if stream is None:
1855 return io.getvalue()
1856 return ""
1857
1859 p, u = xmlrpcGetParser()
1860 if hasattr(stream, "read"):
1861
1862 while 1:
1863 data = stream.read(16384)
1864 if not data:
1865 break
1866 p.feed(data)
1867 else:
1868
1869 p.feed(stream)
1870
1871 if hasattr(xmlrpclib, 'expat'):
1872 try:
1873 p.close()
1874 except xmlrpclib.expat.ExpatError:
1875 raise xmlrpclib.ResponseError
1876 else:
1877 p.close()
1878 return u.close(), u.getmethodname()
1879
1880
1882
1884 self._send = send
1885 self._name = name
1886
1888 if name.startswith('_'):
1889 raise AttributeError(name)
1890 return self.__class__(self._send, "%s.%s" % (self._name, name))
1891
1893 return self._send(self._name, args)
1894
1895
1897
1898
1899 - def __init__(self, url, transport, encoding=None, allow_none=False):
1900 if isinstance(url, basestring):
1901
1902
1903 from conary.lib.http.request import URL
1904 url = URL.parse(url)
1905 self._url = url
1906 self._transport = transport
1907 self._encoding = encoding
1908 self._allow_none = allow_none
1909
1910 - def _request(self, methodname, params):
1911
1912 request = xmlrpcDump(params, methodname,
1913 encoding = self._encoding, allow_none=self._allow_none)
1914
1915 response = self._transport.request(self._url, request)
1916
1917 if len(response) == 1:
1918 response = response[0]
1919
1920 return response
1921
1927
1930
1932 return "<ServerProxy for %s>" % (self._url,)
1933
1934 __str__ = __repr__
1935
1936
1937 -def copyStream(src, dest, length = None, bufferSize = 16384):
1938 """Copy from one stream to another, up to a specified length"""
1939 amtread = 0
1940 while amtread != length:
1941 if length is None:
1942 bsize = bufferSize
1943 else:
1944 bsize = min(bufferSize, length - amtread)
1945 buf = src.read(bsize)
1946 if not buf:
1947 break
1948 dest.write(buf)
1949 amtread += len(buf)
1950 return amtread
1951
1953 sio = BoundedStringIO()
1954 z = zlib.decompressobj()
1955 while 1:
1956 buf = src.read(bufferSize)
1957 if not buf:
1958 break
1959 sio.write(z.decompress(buf))
1960 sio.write(z.flush())
1961 sio.seek(0)
1962 return sio
1963
1965 sio = BoundedStringIO()
1966 z = zlib.compressobj(level)
1967 while 1:
1968 buf = src.read(bufferSize)
1969 if not buf:
1970 break
1971 sio.write(z.compress(buf))
1972 sio.write(z.flush())
1973 return sio
1974
1976 return zlib.decompress(s, 31)
1977
1979 """Close all file descriptors starting with start, until we hit
1980 unusedCount consecutive file descriptors that were already closed"""
1981 return misc.massCloseFileDescriptors(start, unusedCount, 0);
1982
1984 """Connects the file descriptor to /dev/null or an open file (if /dev/null
1985 does not exist)"""
1986 try:
1987 fd = os.open('/dev/null', os.O_RDONLY)
1988 except OSError:
1989
1990 fd, fn = tempfile.mkstemp()
1991 os.unlink(fn)
1992 if fd != fdesc:
1993 os.dup2(fd, fdesc)
1994 os.close(fd)
1995
1996 -def sendmsg(sock, dataList, fdList = []):
1997 """
1998 Sends multiple strings and an optional list of file descriptors through
1999 a unix domain socket.
2000
2001 @param sock: Unix domain socket to send message through
2002 @type sock: socket
2003 @param dataList: List of strings to send
2004 @type dataList: list of str
2005 @param fdList: File descriptors to send
2006 @type fdList: list of int
2007 @rtype: None
2008 """
2009 return misc.sendmsg(sock.fileno(), dataList, fdList)
2010
2011 -def recvmsg(sock, dataSize, fdCount = 0):
2012 """
2013 Receives data and optional file descriptors from a unix domain socket.
2014 Returns a (data, fdList) tuple.
2015
2016 @param sock: Unix domain socket to send message through
2017 @type sock: socket
2018 @param dataSize: Number of bytes to try to read from the socket.
2019 @type dataSize: int
2020 @param fdCount: Exact number of file descriptors to read from the socket
2021 @type fdCount: int
2022 @rtype: tuple
2023 """
2024 return misc.recvmsg(sock.fileno(), dataSize, fdCount)
2025
2027
2029 self.started = time.time()
2030
2032 self.total += (time.time() - self.started)
2033 self.started = None
2034
2036 if self.started:
2037 running = time.time() - self.started
2038 else:
2039 running = 0
2040
2041 return self.total + running
2042
2044 self.started = None
2045 self.total = 0
2046 if start:
2047 self.start()
2048
2052
2054 return ''.join([ x.capitalize() for x in pkgname.split('-') ])
2055
2057
2058 - def read(self, limit = 4096):
2059
2060
2061
2062 buffers = []
2063 pos = 0
2064 while pos < limit:
2065 buf = os.read(self.infd, limit - pos)
2066 if not buf:
2067 break
2068 buffers.append(buf)
2069 pos += len(buf)
2070 return ''.join(buffers)
2071
2073 if self.childpid:
2074 os.close(self.infd)
2075 os.waitpid(self.childpid, 0)
2076 self.childpid = None
2077
2080
2082 self.executable = None
2083 for executable, args in (('xz', ('-dc',)), ('unlzma', ('-dc',))):
2084 for pathElement in os.getenv('PATH', '').split(os.path.pathsep):
2085 fullpath = os.sep.join((pathElement, executable))
2086 if os.path.exists(fullpath):
2087 self.executable = fullpath
2088 commandLine = (executable,) + args
2089 break
2090 if self.executable:
2091 break
2092 if self.executable is None:
2093 raise RuntimeError('xz or unlzma is required to decompress this file')
2094
2095 [ self.infd, outfd ] = os.pipe()
2096 self.childpid = os.fork()
2097 if self.childpid == 0:
2098 try:
2099 os.close(self.infd)
2100 if isinstance(fileobj, gzip.GzipFile):
2101
2102
2103
2104
2105 f = tempfile.TemporaryFile()
2106 copyfileobj(fileobj, f)
2107 f.seek(0)
2108 fileobj.close()
2109 fileobj = f
2110 os.close(0)
2111 os.close(1)
2112
2113 fd = fileobj.fileno()
2114
2115 os.lseek(fd, fileobj.tell(), 0)
2116
2117 os.dup2(fd, 0)
2118 fileobj.close()
2119 os.dup2(outfd, 1)
2120 os.close(outfd)
2121 os.execv(self.executable, commandLine)
2122 finally:
2123 os._exit(1)
2124
2125 os.close(outfd)
2126
2127
2129
2131 if not exc_info:
2132 exc_info = sys.exc_info()
2133 elif isinstance(exc_info, Exception):
2134 exc_info = exc_info.__class__, exc_info, None
2135 self.type, self.value, self.tb = exc_info
2136
2138 return "<saved %s exception>" % self.getName()
2139
2141 return '.'.join((self.type.__module__, self.type.__name__))
2142
2145
2147 raise self.type, self.value, self.tb
2148
2150 """Free the exception and traceback to avoid reference loops."""
2151 self.value = self.tb = None
2152
2154 """Replace the saved exception with a new one. The traceback is
2155 preserved.
2156 """
2157 self.type = value.__class__
2158 self.value = value
2159
2160 - def check(self, *types):
2161 for type_ in types:
2162 if issubclass(self.type, type_):
2163 return True
2164 return False
2165
2166
2167 -def rethrow(newClassOrInstance, prependClassName=True, oldTup=None):
2168 '''
2169 Re-throw an exception, either from C{sys.exc_info()} (the default)
2170 or from C{oldTup} (when set). If C{newClassOrInstance} is a class,
2171 the original traceback will be stringified and used as the parameter
2172 to the new exception, otherwise it should be an instance which will
2173 be thrown as-is. In either case, the original traceback will be
2174 preserved. Additionally, if it is a class and C{prependClassName} is
2175 C{True} (the default), the resulting exception will after
2176 stringification be prepended with the name of the original class.
2177
2178 Note that C{prependClassName} should typically be set to C{False}
2179 when re-throwing a re-thrown exception so that the intermediate
2180 class is not prepended to a value that already has the original
2181 class name in it.
2182
2183 @param newClassOrInstance: Class of the new exception to be thrown,
2184 or the exact exception instance to be thrown.
2185 @type newClassOrInstance: subclass or instance of Exception
2186 @param prependClassName: If C{True}, prepend the original class
2187 name to the new exception
2188 @type prependClassName: bool
2189 @param oldTup: Exception triple to use instead of the current
2190 exception
2191 @type oldTup: (exc_class, exc_value, exc_traceback)
2192 '''
2193
2194 if oldTup is None:
2195 oldTup = sys.exc_info()
2196 exc_class, exc_value, exc_traceback = oldTup
2197
2198 if isinstance(newClassOrInstance, Exception):
2199 newClass = newClassOrInstance.__class__
2200 newValue = newClassOrInstance
2201 else:
2202 newClass = newClassOrInstance
2203 newStr = str(exc_value)
2204 if prependClassName:
2205 exc_name = getattr(exc_class, '__name__', 'Unknown Error')
2206 newStr = '%s: %s' % (exc_name, newStr)
2207 newValue = newClass(newStr)
2208
2209 raise newClass, newValue, exc_traceback
2210
2214 - def log(self, m = ''):
2215 now = time.time()
2216 print "tick: +%.2f %s total=%.3f" % (now-self.last, m, now-self.start)
2217 self.last = now
2220
2221
2222
2226
2228 magic = self.fileobj.read(2)
2229 if magic == '':
2230 return False
2231
2232 elif magic != '\037\213':
2233 raise IOError, 'Not a gzipped file'
2234 method = ord( self.fileobj.read(1) )
2235 if method != 8:
2236 raise IOError, 'Unknown compression method'
2237 flag = ord( self.fileobj.read(1) )
2238
2239
2240
2241 self.fileobj.read(6)
2242
2243 if flag & gzip.FEXTRA:
2244
2245 xlen = ord(self.fileobj.read(1))
2246 xlen = xlen + 256*ord(self.fileobj.read(1))
2247 self.fileobj.read(xlen)
2248 if flag & gzip.FNAME:
2249
2250 while True:
2251 s = self.fileobj.read(1)
2252 if not s or s=='\000':
2253 break
2254 if flag & gzip.FCOMMENT:
2255
2256 while True:
2257 s = self.fileobj.read(1)
2258 if not s or s=='\000':
2259 break
2260 if flag & gzip.FHCRC:
2261 self.fileobj.read(2)
2262
2263 return True
2264
2265 - def _read(self, size=1024):
2266 if self.fileobj is None:
2267 raise EOFError, "Reached EOF"
2268
2269 if self._new_member:
2270
2271
2272 self._init_read()
2273 if not self._read_gzip_header():
2274 raise EOFError, "Reached EOF"
2275 self.decompress = zlib.decompressobj(-zlib.MAX_WBITS)
2276 self._new_member = False
2277
2278
2279 buf = self.fileobj.read(size)
2280
2281
2282
2283
2284 if buf == "":
2285 uncompress = self.decompress.flush()
2286 self._read_eof()
2287 self._add_read_data( uncompress )
2288 raise EOFError, 'Reached EOF'
2289
2290 uncompress = self.decompress.decompress(buf)
2291 self._add_read_data( uncompress )
2292
2293 if self.decompress.unused_data != "":
2294 eof = self.decompress.unused_data
2295 eof += self.fileobj.read(8 - len(eof))
2296
2297
2298
2299 self._read_eof(eof)
2300 self._new_member = True
2301
2303
2304
2305
2306
2307
2308
2309 crc32, isize = struct.unpack("<LL", eof)
2310
2311 actualCrc = (self.crc & 0xffffffff)
2312 if crc32 != actualCrc:
2313 raise IOError("CRC check failed %s != %s" % (hex(crc32),
2314 hex(actualCrc)))
2315 elif isize != (self.size & 0xffffffffL):
2316 raise IOError, "Incorrect length of data produced"
2317
2318
2319 -def walkiter(dirNameList, skipPathSet = set(), root = '/'):
2320 dirNameList.sort()
2321
2322 for dirName in dirNameList:
2323 try:
2324 entries = os.listdir(root + dirName)
2325 except:
2326 return
2327
2328 entries.sort()
2329 for entry in entries:
2330 fullPath = os.path.join(dirName, entry)
2331 if fullPath in skipPathSet:
2332 continue
2333
2334 sb = os.lstat(root + fullPath)
2335 yield fullPath, sb
2336
2337 if stat.S_ISDIR(sb.st_mode):
2338 for x in walkiter([fullPath], root = root,
2339 skipPathSet = skipPathSet):
2340 yield x
2341
2343 '''Reads the no-proxy environment variable and can be used to decide
2344 if the proxy should be bypassed for a specific URL'''
2345 alwayBypass = False
2346 no_proxy_list = []
2348
2349 no_proxy = os.environ.get('no_proxy', '') or \
2350 os.environ.get('NO_PROXY', '')
2351
2352 self.alwaysBypass = no_proxy == '*'
2353
2354 for name in no_proxy.split(','):
2355 name = name.strip()
2356 if name:
2357 self.no_proxy_list.append(name)
2358
2360 if self.alwaysBypass:
2361 return True
2362 for x in self.no_proxy_list:
2363 if urlStr.endswith(x):
2364 return True
2365 return False
2366
2368 "Like fnmatch.translate, but do not add the end-of-string character(s)"
2369 patt = fnmatch.translate(pattern)
2370
2371 if patt.endswith('$'):
2372 return patt[:-1]
2373 if patt.endswith(r'\Z(?ms)'):
2374 return patt[:-7]
2375 raise RuntimeError("Unrecognized end-of-string in %s" % patt)
2376
2378 """
2379 A file protected by a lock.
2380 To use it::
2381
2382 l = LockedFile("filename")
2383 fileobj = l.open()
2384 if fileobj is None:
2385 # The target file does not exist. Create it.
2386 l.write("Some content")
2387 fileobj = l.commit()
2388 else:
2389 # The target file exists
2390 pass
2391 print fileobj.read()
2392 """
2393 __slots__ = ('fileName', 'lockFileName', '_lockfobj', '_tmpfobj')
2394
2400
2401 - def open(self, shouldLock = True):
2402 """
2403 Attempt to open the file.
2404
2405 Returns a file object if the file exists.
2406
2407 Returns None if the file does not exist, and needs to be created.
2408 At this point the lock is acquired. Use write() and commit() to
2409 have the file created and the lock released.
2410 """
2411
2412 if self._lockfobj is not None:
2413 self.close()
2414
2415 fobj = fopenIfExists(self.fileName, "r")
2416 if fobj is not None or not shouldLock:
2417 return fobj
2418
2419 self._lockfobj = open(self.lockFileName, "w")
2420
2421
2422 fcntl.lockf(self._lockfobj, fcntl.LOCK_EX)
2423
2424
2425 fobj = fopenIfExists(self.fileName, "r")
2426 if fobj is not None:
2427
2428
2429 self.unlock()
2430 return fobj
2431
2432 if not os.path.exists(self.lockFileName):
2433
2434
2435
2436
2437
2438 return self.open()
2439
2440 return None
2441
2447
2449
2450
2451
2452 if self._tmpfobj is None:
2453 fileobj = None
2454 else:
2455 fileobj = self._tmpfobj.commit(returnHandle = True)
2456 self._tmpfobj = None
2457 self.unlock()
2458 return fileobj
2459
2463
2473
2474 __del__ = close
2475
2477 """
2478 Open a temporary file adjacent to C{path} for writing. When
2479 C{f.commit()} is called, the temporary file will be flushed and
2480 renamed on top of C{path}, constituting an atomic file write.
2481 """
2482
2483 fObj = None
2484
2485 - def __init__(self, path, mode='w+b', chmod=0644, tmpsuffix = "",
2486 tmpprefix = None):
2487 self.finalPath = os.path.realpath(path)
2488 self.finalMode = chmod
2489
2490 if tmpprefix is None:
2491 tmpprefix = os.path.basename(self.finalPath) + '.tmp.'
2492 fDesc, self.name = tempfile.mkstemp(dir=os.path.dirname(self.finalPath),
2493 suffix=tmpsuffix, prefix=tmpprefix)
2494 self.fObj = os.fdopen(fDesc, mode)
2495
2498
2499 - def commit(self, returnHandle=False):
2500 """
2501 C{flush()}, C{chmod()}, and C{rename()} to the target path.
2502 C{close()} afterwards.
2503 """
2504 if self.fObj.closed:
2505 raise RuntimeError("Can't commit a closed file")
2506
2507
2508
2509 self.fObj.flush()
2510 os.chmod(self.name, self.finalMode)
2511 os.fsync(self.fObj)
2512
2513
2514
2515 os.rename(self.name, self.finalPath)
2516
2517
2518 if returnHandle:
2519 fObj, self.fObj = self.fObj, None
2520 return fObj
2521 return fObj
2522 else:
2523 self.fObj.close()
2524
2529 __del__ = close
2530
2531
2533 """
2534 A map that timestamps entries, to cycle them out after delta seconds.
2535 If delta is set to None, new entries will never go stale.
2536 """
2537 __slots__ = [ 'delta', '_map' ]
2538 _MISSING = object()
2542
2543 - def get(self, key, default = None, stale = False):
2544 v = self._map.get(key, None)
2545 if v is not None:
2546 v, ts = v
2547 if stale or ts is None or time.time() <= ts:
2548 return v
2549 return default
2550
2551 - def set(self, key, value):
2552 if self.delta is None:
2553 ts = None
2554 else:
2555 ts = time.time() + self.delta
2556 self._map[key] = (value, ts)
2557 return self
2558
2561
2563 now = time.time()
2564 ret = sorted(self._map.items(), key = lambda x: x[1][1])
2565 ret = [ (k, v[0]) for (k, v) in ret
2566 if stale or now <= v[1] ]
2567 return ret
2568
2569
2570 -def statFile(pathOrFile, missingOk=False, inodeOnly=False):
2571 """Return a (dev, inode, size, mtime, ctime) tuple of the given file.
2572
2573 Accepts paths, file descriptors, and file-like objects with a C{fileno()}
2574 method.
2575
2576 @param pathOrFile: A file path or file-like object
2577 @type pathOrFile: C{basestring} or file-like object or C{int}
2578 @param missingOk: If C{True}, return C{None} if the file is missing.
2579 @type missingOk: C{bool}
2580 @param inodeOnly: If C{True}, return just (dev, inode).
2581 @type inodeOnly: C{bool}
2582 @rtype: C{tuple}
2583 """
2584 try:
2585 if isinstance(pathOrFile, basestring):
2586 st = os.stat(pathOrFile)
2587 else:
2588 if hasattr(pathOrFile, 'fileno'):
2589 pathOrFile = pathOrFile.fileno()
2590 st = os.fstat(pathOrFile)
2591 except OSError, err:
2592 if err.errno == errno.ENOENT and missingOk:
2593 return None
2594 raise
2595
2596 if inodeOnly:
2597 return (st.st_dev, st.st_ino)
2598 else:
2599 return (st.st_dev, st.st_ino, st.st_size, st.st_mtime, st.st_ctime)
2600
2601
2603 """Yield chunks of data from the given file object."""
2604 while True:
2605 data = fobj.read(16384)
2606 if not data:
2607 break
2608 yield data
2609