conary.metadata

1 # 2 # Copyright (c) 2004-2008 rPath, Inc. 3 # 4 # This program is distributed under the terms of the Common Public License, 5 # version 1.0. A copy of this license should have been distributed with this 6 # source file in a file called LICENSE. If it is not present, the license 7 # is always available at http://www.rpath.com/permanent/licenses/CPL-1.0. 8 # 9 # This program is distributed in the hope that it will be useful, but 10 # without any warranty; without even the implied warranty of merchantability 11 # or fitness for a particular purpose. See the Common Public License for 12 # full details. 13 # 14 from httplib import HTTPConnection 15 from urllib2 import urlopen 16 import textwrap 17 import time 18 import urlparse 19 import xml.dom.minidom 20 import xml.parsers.expat 21 22 from conary.fmtroves import TroveCategories, LicenseCategories 23 from conary.local import schema 24

25 -class MDClass:

26 (SHORT_DESC, LONG_DESC, 27 URL, LICENSE, CATEGORY, 28 SOURCE) = range(6) 29 30 # mapping from enum id to real name 31 className = {SHORT_DESC: "shortDesc", 32 LONG_DESC: "longDesc", 33 URL: "url", 34 LICENSE: "license", 35 CATEGORY: "category", 36 SOURCE: "source"} 37 38 (STRING, LIST) = range(2) 39 40 types = {SHORT_DESC: STRING, 41 LONG_DESC: STRING, 42 URL: LIST, 43 LICENSE: LIST, 44 CATEGORY: LIST, 45 SOURCE: STRING}

46

47 -class MetadataTable:

48 - def __init__(self, db, create = True):

49 self.db = db 50 if create: 51 schema.createMetadata(db)

52

53 - def add(self, itemId, versionId, branchId, shortDesc, longDesc, 54 urls, licenses, categories, source="", language="C"):

55 cu = self.db.cursor() 56 57 if language == "C": 58 cu.execute(""" 59 INSERT INTO Metadata (itemId, versionId, branchId, timestamp) 60 VALUES(?, ?, ?, ?)""", itemId, versionId, branchId, time.time()) 61 mdId = cu.lastrowid 62 else: 63 cu.execute(""" 64 SELECT metadataId FROM Metadata 65 WHERE itemId=? AND versionId=? AND branchId=? ORDER BY timestamp DESC LIMIT 1""", 66 itemId, versionId, branchId) 67 mdId = cu.fetchone()[0] 68 69 for mdClass, data in (MDClass.SHORT_DESC, [shortDesc]),\ 70 (MDClass.LONG_DESC, [longDesc]),\ 71 (MDClass.URL, urls),\ 72 (MDClass.LICENSE, licenses),\ 73 (MDClass.CATEGORY, categories),\ 74 (MDClass.SOURCE, [source]): 75 for d in data: 76 cu.execute(""" 77 INSERT INTO MetadataItems (metadataId, class, data, language) 78 VALUES(?, ?, ?, ?)""", mdId, mdClass, d, language) 79 80 # XXX should I be calling commit here? 81 self.db.commit() 82 return mdId

83

84 - def get(self, itemId, versionId, branchId, language="C"):

85 cu = self.db.cursor() 86 87 cu.execute("SELECT metadataId FROM Metadata WHERE itemId=? AND versionId=? AND branchId=? ORDER BY timestamp DESC LIMIT 1", 88 itemId, versionId, branchId) 89 metadataId = cu.fetchone() 90 if metadataId: 91 metadataId = metadataId[0] 92 else: 93 return None 94 95 # URL, LICENSE, and CATEGORY are not translated 96 cu.execute(""" 97 SELECT class, data FROM MetadataItems 98 WHERE metadataId=? and (language=? OR class IN (?, ?, ?)) 99 """, (metadataId, language, 100 MDClass.URL, MDClass.LICENSE, MDClass.CATEGORY)) 101 102 # create a dictionary of metadata classes 103 # each key points to a list of metadata items 104 105 items = {} 106 for mdClass, className in MDClass.className.items(): 107 classType = MDClass.types[mdClass] 108 109 if classType == MDClass.STRING: 110 items[className] = "" 111 elif classType == MDClass.LIST: 112 items[className] = [] 113 else: 114 items[className] = None 115 116 for mdClass, data in cu: 117 className = MDClass.className[mdClass] 118 classType = MDClass.types[mdClass] 119 120 if classType == MDClass.STRING: 121 items[className] = data 122 elif classType == MDClass.LIST: 123 items[className].append(data) 124 125 for key, value in items.iteritems(): 126 if isinstance(value, list): 127 items[key] = sorted(value) 128 return items

129

130 - def getLatestVersion(self, itemId, branchId):

131 cu = self.db.cursor() 132 cu.execute("""SELECT 133 Versions.version 134 FROM 135 Metadata, Branches, Versions 136 WHERE 137 Metadata.itemId=? AND Metadata.branchId=? 138 AND Metadata.branchId=Branches.branchId 139 AND Metadata.versionId=Versions.versionId 140 ORDER BY 141 Metadata.timeStamp DESC LIMIT 1""", 142 (itemId, branchId)) 143 144 item = cu.fetchone() 145 if item: 146 return item[0] 147 else: 148 return None

149

150 -def resolveUrl(url):

151 """Follows a redirect one level and returns the location of the HTTP 302 redirect""" 152 url = urlparse.urlparse(url) 153 connection = HTTPConnection(url[1]) 154 connection.request("GET", url[2]) 155 request = connection.getresponse() 156 if request.status == 302: # header "Found:", might need more here 157 realUrl = request.getheader("Location") 158 else: 159 realUrl = urlparse.urlunparse(url) 160 return realUrl

161

162 -class Metadata:

163 shortDesc = "" 164 longDesc = "" 165 urls = [] 166 licenses = [] 167 categories = [] 168 language = "C" 169 version = None 170 source = "local" 171

172 - def __init__(self, md):

173 if md: 174 self.shortDesc = md["shortDesc"] 175 self.longDesc = md["longDesc"] 176 self.urls = md["url"] 177 self.licenses = md["license"] 178 self.categories = md["category"] 179 if "version" in md: 180 self.version = md["version"] 181 if "source" in md and md["source"]: 182 self.source = md["source"] 183 if "language" in md: 184 self.language = md["language"]

185

186 - def freeze(self):

187 return {"shortDesc": self.shortDesc, 188 "longDesc": self.longDesc, 189 "url": self.urls, 190 "license": self.licenses, 191 "category": self.categories, 192 "version": self.version, 193 "source": self.source, 194 "language": self.language}

195

196 - def getShortDesc(self):

197 return self.shortDesc

198

199 - def getLongDesc(self):

200 return self.longDesc

201

202 - def getUrls(self):

203 return self.urls

204

205 - def getLicenses(self):

206 return self.licenses

207

208 - def getCategories(self):

209 return self.categories

210

211 - def getVersion(self):

212 return self.version

213

214 - def getSource(self):

215 return self.source

216

217 - def getLanguage(self):

218 return self.language

219

220 -class NoFreshmeatRecord(xml.parsers.expat.ExpatError):

221 pass

222

223 -def fetchFreshmeat(troveName, xmlDocStream=None):

224 if xmlDocStream: 225 # Most likely part of a test suite, although one may want a different 226 # plugin for xml metadata - a stream will work just fine 227 source = xmlDocStream 228 else: 229 source = urlopen('http://freshmeat.net/projects-xml/%s/%s.xml' % (troveName, troveName)) 230 231 try: 232 doc = xml.dom.minidom.parse(source) 233 metadata = {} 234 235 shortDesc = doc.getElementsByTagName("desc_short")[0] 236 if shortDesc.childNodes: 237 metadata["shortDesc"] = shortDesc.childNodes[0].data 238 239 longDesc = doc.getElementsByTagName("desc_full")[0] 240 if longDesc.childNodes: 241 metadata["longDesc"] = longDesc.childNodes[0].data 242 243 metadata["url"] = [] 244 urlHomepage = doc.getElementsByTagName("url_homepage")[0] 245 if urlHomepage.childNodes: 246 metadata["url"].append(resolveUrl(urlHomepage.childNodes[0].data)) 247 metadata["url"].append("http://freshmeat.net/projects/%s/" % troveName) 248 249 metadata["license"] = [] 250 metadata["category"] = [] 251 252 for node in doc.getElementsByTagName("trove_id"): 253 id = node.childNodes[0].data 254 if id in LicenseCategories: 255 name = LicenseCategories[id] 256 metadata["license"].append(name) 257 else: 258 name = TroveCategories[id] 259 if name.startswith('Topic ::'): 260 metadata["category"].append(name) 261 262 metadata["source"] = "freshmeat" 263 metadata["language"] = "C" 264 # Free the DOM - CNY-2674 265

Source Code for Module conary.metadata