from time import localtime, strftime, time, mktime import datetime import re, sys import operator import cherrypy from pkgcore.ebuild.cpv import unversioned_CPV from pkgcore.ebuild.cpv import versioned_CPV from pkgcore.ebuild.errors import InvalidCPV from DBUtils.PersistentDB import PersistentDB from DBUtils.PooledDB import PooledDB from etc.const_data import ConstData from web.lib.changelog_formatter import pretty_changelog, \ optimal_collapse from web.lib.links import viewcvs_link, \ bugzilla_search_link, \ forums_search_link # We use short variable names! # pylint: disable-msg=C0103 def get_atom(cpvstr): """Nasty hack to work around not knowing if an atom is versioned or not""" v_atom = u_atom = None try: v_atom = versioned_CPV(str(cpvstr)) except InvalidCPV: pass try: u_atom = unversioned_CPV(str(cpvstr)) except InvalidCPV: pass if u_atom is not None and v_atom is not None: #raise Exception('Ambiguous package atom, could be CAT/PN or CAT/PN-PV: %s' % (cpvstr)) return v_atom if v_atom: return v_atom if u_atom: return u_atom raise Exception('Not a valid package atom at all: %s' % (cpvstr)) def caller_name(): """Return the name of the function that this was called from""" return sys._getframe(1).f_code.co_name def mc_expiretime(granularity): t = time() return t - (t % granularity) + granularity def latest_per_day(cpvlist, daycount, pkgcount = 110): """generate list of sets with days and pv's as value""" dayno = 0 pkgno = 0 result = [] last_day = "" last_day_cpvs = [] for cpvstr, timestamp in cpvlist: cur_day = strftime("%A", localtime(timestamp)) if last_day != cur_day: if last_day: dayno += 1 if dayno == daycount: break result.append([last_day, last_day_cpvs]) last_day = cur_day last_day_cpvs = [] atom = get_atom(str(cpvstr)) pnpv = optimal_collapse(atom, 16, 6, '…') last_day_cpvs.append((pnpv, atom.key)) pkgno += 1 if pkgno == pkgcount: break if last_day_cpvs: result.append([last_day, last_day_cpvs]) return result def build_centerpkg_list(latest_entries, get_package_details_cpv, count, **kwds): center_pkgs = [] shown_cp = [] # dummy is mtime for cpv, dummy in latest_entries: atom = get_atom(str(cpv)) if 'use_fullver' in kwds and kwds['use_fullver']: version_check = atom.cpvstr else: version_check = atom.key if version_check not in shown_cp: pkg_details = get_package_details_cpv(version_check) if 'filter_func' in kwds: if not kwds['filter_func'](pkg_details, kwds['filter_value']): continue center_pkgs.append(pkg_details) shown_cp.append(version_check) if count and len(center_pkgs) == count: break return center_pkgs class PackageDB(object): """Wrapper for all SQL functionality to read from the database""" # This should match /usr/portage/profiles/arch.list arches = frozenset(ConstData.arches['all']) # DB fun dbconfig = {} db = None syntax_placeholder = None columns_category_pn = None columns_category_pn_pv = None sql = {} mc = None def __init__(self, config=None): # Do not complain about correct usage of ** magic # pylint: disable-msg=W0142 if config is None or 'dbconn' not in config: print "No configuration available!" sys.exit(1) self.dbconfig.update(config) if 'memcached' in config and config['memcached']: cherrypy.log("Thread started", "THREAD") import memcache servers = config['memcached']['servers'] args = config['memcached']['args'] mc = memcache.Client(servers, **args) dummy = mc.get_stats() if dummy and len(dummy) > 0: self.mc = mc def cursor(self): """Provide a DB-cursor for the given DB implementation""" # Uncomment this if you have too many SQL queries and # you want to trace some of them down. #print "\n\nTraceback:" #import traceback #traceback.print_stack(file=sys.stdout) db = self.db.connection() return db.cursor() def close_db(self): """Force the DB conenction to be closed""" self.db._closeable = True self.db.connection.close() def close_mc(self): """Force the Memcached connection to be closed""" if 'memcached' in self.dbconfig and self.dbconfig['memcached']: self.mc.disconnect_all() def _preparesql(self): """Prepare all SQL statements for the relevant DB backend""" reps = [] reps.append(('__CP__', self.columns_category_pn)) reps.append(('__CPV__', self.columns_category_pn_pv)) reps.append(('?', self.syntax_placeholder)) # Only used once spacematch = re.compile(r'(\s+|\n)') for k in self.sql.keys(): s = self.sql[k] for o, n in reps: s = s.replace(o, n) s = spacematch.sub(' ', s) self.sql[k] = s #sql = FrozenDict(sql) def mc_wrap(self, key, func, **kwopts): # Do not complain about correct usage of ** magic # pylint: disable-msg=W0142 obj = None if self.mc: obj = self.mc.get(key) if not obj: obj = func() if self.mc and obj: kwopts['time'] = mc_expiretime(60*30) self.mc.set(key, obj, **kwopts) return obj def get_latest_timestamp(self): """return modified cpvs""" key = '%s' % (caller_name(), ) def f(): return self._get_latest_timestamp() return self.mc_wrap(key, f) sql['SELECT_get_latest_timestamp'] = """ SELECT mtime FROM versions ORDER BY mtime DESC LIMIT 0,1 """ def _get_latest_timestamp(self): """return timestamp of latest modification""" sql = self.sql['SELECT_get_latest_timestamp'] cursor = self.cursor() cursor.execute(sql) row = cursor.fetchone() if row is not None: row = row[0] cursor.close() return row def get_latest_cpvs(self, limit=None): """return modified cpvs""" key = '%s_%s' % (caller_name(), limit) def f(): return self._get_latest_cpvs(limit) return self.mc_wrap(key, f) sql['SELECT_get_latest_cpvs'] = """ SELECT __CPV__, versions.mtime FROM versions JOIN packages USING (cp) JOIN categories USING (c) WHERE versions.mtime > ? ORDER by versions.mtime DESC LIMIT 0, ?""" def _get_latest_cpvs(self, limit=None): """return modified cpvs (uncached)""" sql = self.sql['SELECT_get_latest_cpvs'] i = self.get_latest_timestamp() if i is None: i = time.time() timerange = i - 86400*10 if limit is None: limit = 1000 try: params = (timerange, int(limit), ) except ValueError: return [] cursor = self.cursor() cursor.execute(sql, params) entries = cursor.fetchall() cursor.close() return entries def get_latest_cpvs_by_verbump(self, limit=None): """return modified cpvs""" key = '%s_%s' % (caller_name(), limit) def f(): return self._get_latest_cpvs_by_verbump(limit) return self.mc_wrap(key, f) sql['SELECT_get_latest_cpvs_by_verbump'] = """ SELECT __CPV__, verbumps.mtime FROM verbumps JOIN versions USING(cpv) JOIN metadata USING(cp) JOIN packages USING (cp) JOIN categories USING (c) WHERE verbumps.mtime > ? ORDER by verbumps.mtime DESC LIMIT 0, ?""" def _get_latest_cpvs_by_verbump(self, limit=None): """return modified cpvs (uncached)""" # We cannot rely on the contents of ChangeLog here # otherwise we could add this bit of magic: # AND SUBSTR(metadata.changelog,1,1)='*' sql = self.sql['SELECT_get_latest_cpvs_by_verbump'] i = self.get_latest_timestamp() if i is None: i = time.time() timerange = i - 86400*5 if limit is None: limit = 1000 try: params = (timerange, int(limit)) except ValueError: return [] cursor = self.cursor() cursor.execute(sql, params) entries = cursor.fetchall() cursor.close() return entries def get_latest_cpvs_by_newpkg(self, limit=None): """return modified cpvs""" key = '%s_%s' % (caller_name(), limit) def f(): return self._get_latest_cpvs_by_newpkg(limit) return self.mc_wrap(key, f) sql['SELECT_get_latest_cpvs_by_newpkg'] = """ SELECT __CPV__, verbumps.mtime FROM verbumps JOIN versions USING(cpv) JOIN metadata USING(cp) JOIN packages USING (cp) JOIN categories USING (c) WHERE verbumps.mtime > ? AND verbumps.newpkg = 1 ORDER by verbumps.mtime DESC LIMIT 0, ? """ def _get_latest_cpvs_by_newpkg(self, limit=None): """return modified cpvs (uncached)""" # We cannot rely on the contents of ChangeLog here # otherwise we could add this bit of magic: # AND SUBSTR(metadata.changelog,1,1)='*' sql = self.sql['SELECT_get_latest_cpvs_by_newpkg'] i = self.get_latest_timestamp() if i is None: i = time.time() timerange = i - 86400*10 if limit is None: limit = 1000 try: params = (timerange, int(limit), ) except ValueError: return [] cursor = self.cursor() cursor.execute(sql, params) entries = cursor.fetchall() cursor.close() return entries def get_latest_cpvs_by_pkgname(self, pkgname, limit=None): """return modified cpvs""" key = '%s_%s_%s' % (caller_name(), pkgname, limit) def f(): return self._get_latest_cpvs_by_pkgname(pkgname, limit) return self.mc_wrap(key, f) def get_latest_cpvs_by_date(self, date, limit=None): """return new cpvs since date""" key = '%s_%s_%s' % (caller_name(), strftime("%Y%m%d",date), limit) def f(): return self._get_latest_cpvs_by_date(date, limit) return self.mc_wrap(key, f) sql['SELECT_get_latest_cpvs_by_date'] = """ SELECT __CPV__, verbumps.mtime FROM verbumps JOIN versions USING(cpv) JOIN metadata USING(cp) JOIN packages USING (cp) JOIN categories USING (c) WHERE verbumps.mtime > ? AND verbumps.newpkg = 1 ORDER by verbumps.mtime DESC LIMIT 0, ? """ def _get_latest_cpvs_by_date(self, date, limit=None): """return new cpvs since date""" # We cannot rely on the contents of ChangeLog here # otherwise we could add this bit of magic: # AND SUBSTR(metadata.changelog,1,1)='*' sql = self.sql['SELECT_get_latest_cpvs_by_date'] i = self.get_latest_timestamp() if i is None: i = time.time() datetime = mktime(date) if limit is None: limit = 300 try: params = (datetime, int(limit), ) except ValueError: return [] cursor = self.cursor() cursor.execute(sql, params) entries = cursor.fetchall() cursor.close() return entries sql['SELECT_get_latest_cpvs_by_pkgname'] = """ SELECT __CPV__, versions.mtime FROM versions JOIN packages USING (cp) JOIN categories USING (c) WHERE pn = ? ORDER by versions.mtime DESC LIMIT 0, ? """ def _get_latest_cpvs_by_pkgname(self, pkgname, limit=None): """return modified cpvs (uncached)""" sql = self.sql['SELECT_get_latest_cpvs_by_pkgname'] if limit is None: limit = 1000 try: params = (pkgname, int(limit), ) except ValueError: return [] cursor = self.cursor() cursor.execute(sql, params) entries = cursor.fetchall() cursor.close() return entries def get_latest_cpvs_by_category(self, cat, limit=None): """return modified cpvs""" key = '%s_%s_%s' % (caller_name(), cat, limit) def f(): return self._get_latest_cpvs_by_category(cat, limit) return self.mc_wrap(key, f) sql['SELECT_get_latest_cpvs_by_category'] = """ SELECT __CPV__, versions.mtime FROM versions JOIN packages USING (cp) JOIN categories USING (c) WHERE category = ? ORDER by versions.mtime DESC LIMIT 0, ? """ def _get_latest_cpvs_by_category(self, cat, limit=None): """return modified cpvs (uncached)""" sql = self.sql['SELECT_get_latest_cpvs_by_category'] if limit is None: limit = 1000 try: params = (cat, int(limit), ) except ValueError: return [] cursor = self.cursor() cursor.execute(sql, params) entries = cursor.fetchall() cursor.close() return entries def get_latest_cpvs_by_category_pkgname(self, cat, pn, limit=None): """return modified cpvs""" key = '%s_%s/%s_%s' % (caller_name(), cat, pn, limit) def f(): return self._get_latest_cpvs_by_category_pkgname(cat, pn, limit) return self.mc_wrap(key, f) sql['SELECT_get_latest_cpvs_by_category_pkgname'] = """ SELECT __CPV__, versions.mtime FROM versions JOIN packages USING (cp) JOIN categories USING (c) WHERE category = ? AND pn = ? ORDER by versions.mtime DESC LIMIT 0, ? """ def _get_latest_cpvs_by_category_pkgname(self, cat, pn, limit=None): """return modified cpvs (uncached)""" sql = self.sql['SELECT_get_latest_cpvs_by_category_pkgname'] if limit is None: limit = 1000 try: params = (cat, pn, int(limit), ) except ValueError: return [] cursor = self.cursor() cursor.execute(sql, params) entries = cursor.fetchall() cursor.close() return entries def get_latest_cpvs_by_arch(self, arch, mode, limit=None): """return modified cpvs limited by arch and mode""" key = '%s_%s_%s_%s' % (caller_name(), arch, mode, limit) def f(): return self._get_latest_cpvs_by_arch(arch, mode, limit) return self.mc_wrap(key, f) sql['SELECT_get_latest_cpvs_by_arch_mode'] = """ SELECT __CPV__, versions.mtime FROM arches JOIN keywords USING (a) RIGHT OUTER JOIN versions USING (cpv) JOIN packages USING (cp) JOIN categories USING (c) WHERE arches.arch=? AND SUBSTR(keywords.mode,1,1)=? ORDER by versions.mtime DESC LIMIT 0, ? """ sql['SELECT_get_latest_cpvs_by_arch_nomode'] = """ SELECT __CPV__, versions.mtime FROM arches JOIN keywords USING (a) RIGHT OUTER JOIN versions USING (cpv) JOIN packages USING (cp) JOIN categories USING (c) WHERE arches.arch=? ORDER by versions.mtime DESC LIMIT 0, ? """ def _get_latest_cpvs_by_arch(self, arch, mode, limit=None): """return modified cpvs limited by arch and mode (uncached)""" valid_modes = ['+', '~', 'M', 'M+', 'M~', ''] params = () if mode not in valid_modes or arch not in self.arches: return [] if limit is None: limit = 1000 try: limit = int(limit) except ValueError: return [] if mode != '': sql = self.sql['SELECT_get_latest_cpvs_by_arch_mode'] params = (arch, str(mode)[0], limit) else: sql = self.sql['SELECT_get_latest_cpvs_by_arch_nomode'] params = (arch, limit) cursor = self.cursor() cursor.execute(sql, params) entries = cursor.fetchall() cursor.close() return entries def get_package_details_cpv(self, cpvstr): """get package details with versionfiltered keywords""" key = '%s_%s' % (caller_name(), str(cpvstr)) def f(): return self._get_package_details_cpv(cpvstr) return self.mc_wrap(key, f) def _get_package_details_cpv(self, cpvstr): """get package details with versionfiltered keywords (uncached)""" atom = get_atom(str(cpvstr)) (cat, pn) = (atom.category, atom.package) pkg_metadata = self.get_package_details(cat, pn) if pkg_metadata is None: raise Exception('No metadata available for %s/%s' % (cat, pn)) pkg = {} pkg['atom'] = atom pkg['meta'] = pkg_metadata pkg['keywords'] = self.get_package_keywords(atom) pkg['license'] = [] if pkg_metadata['license']: for l in pkg_metadata['license'].split(): if l[0].isalnum() and not l.endswith("?"): url = viewcvs_link('licenses/%s' % l) l = '%s' % (url, l) pkg['license'].append(l) pkg['license'] = ' '.join(pkg['license']) pkg['homepage'] = [] homepages = pkg_metadata['homepage'].split() if len(homepages) == 1: url = homepages[0] l = '%s' % (url, 'Homepage') pkg['homepage'].append(l) else: pkg['homepage'].append('Homepages:') i = 1 for l in homepages: l = '%s' % (l, i) pkg['homepage'].append(l) i += 1 pkg['homepage'] = ' '.join(pkg['homepage']) pkg['changelog'] = viewcvs_link('%s/ChangeLog' % (atom.key)) pkg['metadataxml'] = viewcvs_link('%s/metadata.xml' % (atom.key)) pkg['bugzilla'] = bugzilla_search_link(pn) pkg['forums'] = forums_search_link(pn) return pkg def get_package_details(self, cat, pn): """get package details""" key = '%s_%s/%s' % (caller_name(), cat, pn) def f(): return self._get_package_details(cat, pn) return self.mc_wrap(key, f) sql['SELECT_get_package_details_metadata'] = """ SELECT license, homepage, description, changelog FROM metadata JOIN packages USING (cp) JOIN categories USING (c) WHERE category=? AND pn=? """ sql['SELECT_get_package_details_mtime'] = """ SELECT mtime FROM versions JOIN packages USING (cp) JOIN categories USING (c) WHERE category = ? AND pn = ? ORDER BY mtime DESC """ def _get_package_details(self, cat, pn): """get package details (uncached)""" cp = '%s/%s' % (cat, pn) # Returns exactly one row sql = self.sql['SELECT_get_package_details_metadata'] cursor = self.cursor() cursor.execute(sql, (cat, pn)) row = cursor.fetchone() if not row: return pkglicense, homepage, description, changelog = row (changelog, authorname, authorid) = \ pretty_changelog(cat, pn, changelog) # Returns exactly one column and one row sql = self.sql['SELECT_get_package_details_mtime'] cursor.execute(sql, (cat, pn)) row = cursor.fetchone() if not row: return ebuildchangeutc = datetime.datetime.utcfromtimestamp(row[0]) package_details = { 'atom': get_atom(str(cp)), 'license': pkglicense, 'homepage': homepage, 'description': description, 'commitmsg': changelog, 'authorname': authorname, 'authorid': authorid, #'changelogdate': date, 'ebuildchange': ebuildchangeutc } cursor.close() return package_details def get_package_keywords(self, query): """return cpv keywords""" key = '%s_%s' % (caller_name(), query, ) def f(): return self._get_package_keywords(query) return self.mc_wrap(key, f) sql['SELECT_get_package_keywords_cat_pn'] = """ SELECT __CPV__, arches.arch, mode FROM arches JOIN keywords USING (a) RIGHT OUTER JOIN versions USING (cpv) JOIN packages USING (cp) JOIN categories USING (c) WHERE category = ? AND pn = ? ORDER BY pv ASC""" sql['SELECT_get_package_keywords_cat_pn_pv'] = """ SELECT __CPV__, arches.arch, mode FROM arches JOIN keywords USING (a) RIGHT OUTER JOIN versions USING (cpv) JOIN packages USING (cp) JOIN categories USING (c) WHERE category = ? AND pn = ? AND pv = ? ORDER BY pv ASC""" def _get_package_keywords(self, query): """return cpv keywords (uncached)""" if query.fullver: sql = self.sql['SELECT_get_package_keywords_cat_pn_pv'] params = (query.category, query.package, query.fullver) else: sql = self.sql['SELECT_get_package_keywords_cat_pn'] params = (query.category, query.package) cursor = self.cursor() cursor.execute(sql, params) last_cpv = None ret_list = [] pkg = {} for row in cursor.fetchall(): cpv = str(row[0]) arch = str(row[1]) mode = str(row[2]) if not query.fullver: current_cpv = get_atom(cpv) else: current_cpv = query if last_cpv != current_cpv: if last_cpv is not None: ret_list.append(pkg) pkg = {'atom': current_cpv} for i in frozenset(ConstData.arches['all']): pkg[i] = {'mode':'', 'name': 'blank'} last_cpv = current_cpv if mode.startswith("M"): name = "hardmask" elif mode == "~": name = "unstable" elif mode =="+": name = "stable" elif mode =="-": name = "blocked" else: # mode == "": name = "blank" # this might be 'None' which is not a valid arch... if arch != 'None': pkg[arch]['name'] = name pkg[arch]['mode'] = mode # catch most recent package ret_list.append(pkg) # Give it a nice order if len(ret_list) > 1: ret_list.sort(key=operator.itemgetter('atom'), reverse=True) cursor.close() return ret_list def get_category_packages(self, cat): """get a list of all packages for a category""" key = '%s_%s' % (caller_name(), cat, ) def f(): return self._get_category_packages(cat) return self.mc_wrap(key, f) sql['SELECT_get_category_packages'] = """ SELECT __CP__, 0 FROM packages JOIN categories USING (c) WHERE category = ? ORDER BY pn""" def _get_category_packages(self, cat): """get a list of all packages for a category (uncached)""" sql = self.sql['SELECT_get_category_packages'] cursor = self.cursor() cursor.execute(sql, (cat, )) entries = cursor.fetchall() cursor.close() return entries def get_category_list(self): """get a list of all categories with their package and ebuild counts""" key = '%s' % (caller_name(), ) def f(): return self._get_category_list() return self.mc_wrap(key, f) sql['SELECT_get_category_list'] = """ SELECT category, COUNT(DISTINCT cp), COUNT(DISTINCT cpv) FROM categories JOIN packages USING (c) JOIN versions USING (cp) GROUP BY c ORDER BY category""" def _get_category_list(self): """get a list of all categories with their package and ebuild counts (uncached)""" sql = self.sql['SELECT_get_category_list'] cursor = self.cursor() cursor.execute(sql) entries = cursor.fetchall() cursor.close() return entries class SQLitePackageDB(PackageDB): """override for sqlite backend""" columns_category_pn = "category || '/' || pn" columns_category_pn_pv = "category || '/' || pn || '-' || pv" syntax_placeholder = '?' def __init__(self, config=None): # Do not complain about correct usage of ** magic # pylint: disable-msg=W0142 PackageDB.__init__(self, config) try: import sqlite3 as sqlite except ImportError: try: import pysqlite2.dbapi2 as sqlite except ImportError: print "Please install PySQLite or use Python 2.5 with sqlite" sys.exit(1) self.db = PersistentDB(sqlite, 100, **self.dbconfig['dbconn']) self._preparesql() class MySQLPackageDB(PackageDB): """override for MySQL backend""" columns_category_pn = "CONCAT(category, '/', pn)" columns_category_pn_pv = "CONCAT(category, '/', pn, '-', pv)" syntax_placeholder = '%s' def __init__(self, config=None): # Do not complain about correct usage of ** magic # pylint: disable-msg=W0142 PackageDB.__init__(self, config) try: import MySQLdb except ImportError: print "Please install a recent version of MySQLdb for Python" sys.exit(1) self.db = PooledDB(MySQLdb, mincached=1, maxcached=3, maxshared=3, maxconnections=3, **self.dbconfig['dbconn']) self._preparesql() # vim:ts=4 et ft=python: