from time import localtime, strftime, time, mktime
import datetime
import re, sys
import operator
import cherrypy
from pkgcore.ebuild.cpv import unversioned_CPV
from pkgcore.ebuild.cpv import versioned_CPV
from pkgcore.ebuild.errors import InvalidCPV
from DBUtils.PersistentDB import PersistentDB
from DBUtils.PooledDB import PooledDB
from etc.const_data import ConstData
from web.lib.changelog_formatter import pretty_changelog, \
optimal_collapse
from web.lib.links import viewcvs_link, \
bugzilla_search_link, \
forums_search_link
# We use short variable names!
# pylint: disable-msg=C0103
def get_atom(cpvstr):
"""Nasty hack to work around not knowing if an atom is versioned or not"""
v_atom = u_atom = None
try:
v_atom = versioned_CPV(str(cpvstr))
except InvalidCPV:
pass
try:
u_atom = unversioned_CPV(str(cpvstr))
except InvalidCPV:
pass
if u_atom is not None and v_atom is not None:
#raise Exception('Ambiguous package atom, could be CAT/PN or CAT/PN-PV: %s' % (cpvstr))
return v_atom
if v_atom:
return v_atom
if u_atom:
return u_atom
raise Exception('Not a valid package atom at all: %s' % (cpvstr))
def caller_name():
"""Return the name of the function that this was called from"""
return sys._getframe(1).f_code.co_name
def mc_expiretime(granularity):
t = time()
return t - (t % granularity) + granularity
def latest_per_day(cpvlist, daycount, pkgcount = 110):
"""generate list of sets with days and pv's as value"""
dayno = 0
pkgno = 0
result = []
last_day = ""
last_day_cpvs = []
for cpvstr, timestamp in cpvlist:
cur_day = strftime("%A", localtime(timestamp))
if last_day != cur_day:
if last_day:
dayno += 1
if dayno == daycount:
break
result.append([last_day, last_day_cpvs])
last_day = cur_day
last_day_cpvs = []
atom = get_atom(str(cpvstr))
pnpv = optimal_collapse(atom, 16, 6, '…')
last_day_cpvs.append((pnpv, atom.key))
pkgno += 1
if pkgno == pkgcount:
break
if last_day_cpvs:
result.append([last_day, last_day_cpvs])
return result
def build_centerpkg_list(latest_entries,
get_package_details_cpv,
count, **kwds):
center_pkgs = []
shown_cp = []
# dummy is mtime
for cpv, dummy in latest_entries:
atom = get_atom(str(cpv))
if 'use_fullver' in kwds and kwds['use_fullver']:
version_check = atom.cpvstr
else:
version_check = atom.key
if version_check not in shown_cp:
pkg_details = get_package_details_cpv(version_check)
if 'filter_func' in kwds:
if not kwds['filter_func'](pkg_details, kwds['filter_value']):
continue
center_pkgs.append(pkg_details)
shown_cp.append(version_check)
if count and len(center_pkgs) == count:
break
return center_pkgs
class PackageDB(object):
"""Wrapper for all SQL functionality to read from the database"""
# This should match /usr/portage/profiles/arch.list
arches = frozenset(ConstData.arches['all'])
# DB fun
dbconfig = {}
db = None
syntax_placeholder = None
columns_category_pn = None
columns_category_pn_pv = None
sql = {}
mc = None
def __init__(self, config=None):
# Do not complain about correct usage of ** magic
# pylint: disable-msg=W0142
if config is None or 'dbconn' not in config:
print "No configuration available!"
sys.exit(1)
self.dbconfig.update(config)
if 'memcached' in config and config['memcached']:
cherrypy.log("Thread started", "THREAD")
import memcache
servers = config['memcached']['servers']
args = config['memcached']['args']
mc = memcache.Client(servers, **args)
dummy = mc.get_stats()
if dummy and len(dummy) > 0:
self.mc = mc
def cursor(self):
"""Provide a DB-cursor for the given DB implementation"""
# Uncomment this if you have too many SQL queries and
# you want to trace some of them down.
#print "\n\nTraceback:"
#import traceback
#traceback.print_stack(file=sys.stdout)
db = self.db.connection()
return db.cursor()
def close_db(self):
"""Force the DB conenction to be closed"""
self.db._closeable = True
self.db.connection.close()
def close_mc(self):
"""Force the Memcached connection to be closed"""
if 'memcached' in self.dbconfig and self.dbconfig['memcached']:
self.mc.disconnect_all()
def _preparesql(self):
"""Prepare all SQL statements for the relevant DB backend"""
reps = []
reps.append(('__CP__', self.columns_category_pn))
reps.append(('__CPV__', self.columns_category_pn_pv))
reps.append(('?', self.syntax_placeholder))
# Only used once
spacematch = re.compile(r'(\s+|\n)')
for k in self.sql.keys():
s = self.sql[k]
for o, n in reps:
s = s.replace(o, n)
s = spacematch.sub(' ', s)
self.sql[k] = s
#sql = FrozenDict(sql)
def mc_wrap(self, key, func, **kwopts):
# Do not complain about correct usage of ** magic
# pylint: disable-msg=W0142
obj = None
if self.mc:
obj = self.mc.get(key)
if not obj:
obj = func()
if self.mc and obj:
kwopts['time'] = mc_expiretime(60*30)
self.mc.set(key, obj, **kwopts)
return obj
def get_latest_timestamp(self):
"""return modified cpvs"""
key = '%s' % (caller_name(), )
def f():
return self._get_latest_timestamp()
return self.mc_wrap(key, f)
sql['SELECT_get_latest_timestamp'] = """
SELECT mtime
FROM versions
ORDER BY mtime DESC
LIMIT 0,1
"""
def _get_latest_timestamp(self):
"""return timestamp of latest modification"""
sql = self.sql['SELECT_get_latest_timestamp']
cursor = self.cursor()
cursor.execute(sql)
row = cursor.fetchone()
if row is not None:
row = row[0]
cursor.close()
return row
def get_latest_cpvs(self, limit=None):
"""return modified cpvs"""
key = '%s_%s' % (caller_name(), limit)
def f():
return self._get_latest_cpvs(limit)
return self.mc_wrap(key, f)
sql['SELECT_get_latest_cpvs'] = """
SELECT __CPV__, versions.mtime
FROM versions
JOIN packages USING (cp)
JOIN categories USING (c)
WHERE versions.mtime > ?
ORDER by versions.mtime DESC
LIMIT 0, ?"""
def _get_latest_cpvs(self, limit=None):
"""return modified cpvs (uncached)"""
sql = self.sql['SELECT_get_latest_cpvs']
i = self.get_latest_timestamp()
if i is None:
i = time.time()
timerange = i - 86400*10
if limit is None:
limit = 1000
try:
params = (timerange, int(limit), )
except ValueError:
return []
cursor = self.cursor()
cursor.execute(sql, params)
entries = cursor.fetchall()
cursor.close()
return entries
def get_latest_cpvs_by_verbump(self, limit=None):
"""return modified cpvs"""
key = '%s_%s' % (caller_name(), limit)
def f():
return self._get_latest_cpvs_by_verbump(limit)
return self.mc_wrap(key, f)
sql['SELECT_get_latest_cpvs_by_verbump'] = """
SELECT __CPV__, verbumps.mtime
FROM verbumps
JOIN versions USING(cpv)
JOIN metadata USING(cp)
JOIN packages USING (cp)
JOIN categories USING (c)
WHERE verbumps.mtime > ?
ORDER by verbumps.mtime DESC
LIMIT 0, ?"""
def _get_latest_cpvs_by_verbump(self, limit=None):
"""return modified cpvs (uncached)"""
# We cannot rely on the contents of ChangeLog here
# otherwise we could add this bit of magic:
# AND SUBSTR(metadata.changelog,1,1)='*'
sql = self.sql['SELECT_get_latest_cpvs_by_verbump']
i = self.get_latest_timestamp()
if i is None:
i = time.time()
timerange = i - 86400*5
if limit is None:
limit = 1000
try:
params = (timerange, int(limit))
except ValueError:
return []
cursor = self.cursor()
cursor.execute(sql, params)
entries = cursor.fetchall()
cursor.close()
return entries
def get_latest_cpvs_by_newpkg(self, limit=None):
"""return modified cpvs"""
key = '%s_%s' % (caller_name(), limit)
def f():
return self._get_latest_cpvs_by_newpkg(limit)
return self.mc_wrap(key, f)
sql['SELECT_get_latest_cpvs_by_newpkg'] = """
SELECT __CPV__, verbumps.mtime
FROM verbumps
JOIN versions USING(cpv)
JOIN metadata USING(cp)
JOIN packages USING (cp)
JOIN categories USING (c)
WHERE verbumps.mtime > ? AND verbumps.newpkg = 1
ORDER by verbumps.mtime DESC
LIMIT 0, ?
"""
def _get_latest_cpvs_by_newpkg(self, limit=None):
"""return modified cpvs (uncached)"""
# We cannot rely on the contents of ChangeLog here
# otherwise we could add this bit of magic:
# AND SUBSTR(metadata.changelog,1,1)='*'
sql = self.sql['SELECT_get_latest_cpvs_by_newpkg']
i = self.get_latest_timestamp()
if i is None:
i = time.time()
timerange = i - 86400*10
if limit is None:
limit = 1000
try:
params = (timerange, int(limit), )
except ValueError:
return []
cursor = self.cursor()
cursor.execute(sql, params)
entries = cursor.fetchall()
cursor.close()
return entries
def get_latest_cpvs_by_pkgname(self, pkgname, limit=None):
"""return modified cpvs"""
key = '%s_%s_%s' % (caller_name(), pkgname, limit)
def f():
return self._get_latest_cpvs_by_pkgname(pkgname, limit)
return self.mc_wrap(key, f)
def get_latest_cpvs_by_date(self, date, limit=None):
"""return new cpvs since date"""
key = '%s_%s_%s' % (caller_name(), strftime("%Y%m%d",date), limit)
def f():
return self._get_latest_cpvs_by_date(date, limit)
return self.mc_wrap(key, f)
sql['SELECT_get_latest_cpvs_by_date'] = """
SELECT __CPV__, verbumps.mtime
FROM verbumps
JOIN versions USING(cpv)
JOIN metadata USING(cp)
JOIN packages USING (cp)
JOIN categories USING (c)
WHERE verbumps.mtime > ? AND verbumps.newpkg = 1
ORDER by verbumps.mtime DESC
LIMIT 0, ?
"""
def _get_latest_cpvs_by_date(self, date, limit=None):
"""return new cpvs since date"""
# We cannot rely on the contents of ChangeLog here
# otherwise we could add this bit of magic:
# AND SUBSTR(metadata.changelog,1,1)='*'
sql = self.sql['SELECT_get_latest_cpvs_by_date']
i = self.get_latest_timestamp()
if i is None:
i = time.time()
datetime = mktime(date)
if limit is None:
limit = 300
try:
params = (datetime, int(limit), )
except ValueError:
return []
cursor = self.cursor()
cursor.execute(sql, params)
entries = cursor.fetchall()
cursor.close()
return entries
sql['SELECT_get_latest_cpvs_by_pkgname'] = """
SELECT __CPV__, versions.mtime
FROM versions
JOIN packages USING (cp)
JOIN categories USING (c)
WHERE pn = ?
ORDER by versions.mtime DESC
LIMIT 0, ?
"""
def _get_latest_cpvs_by_pkgname(self, pkgname, limit=None):
"""return modified cpvs (uncached)"""
sql = self.sql['SELECT_get_latest_cpvs_by_pkgname']
if limit is None:
limit = 1000
try:
params = (pkgname, int(limit), )
except ValueError:
return []
cursor = self.cursor()
cursor.execute(sql, params)
entries = cursor.fetchall()
cursor.close()
return entries
def get_latest_cpvs_by_category(self, cat, limit=None):
"""return modified cpvs"""
key = '%s_%s_%s' % (caller_name(), cat, limit)
def f():
return self._get_latest_cpvs_by_category(cat, limit)
return self.mc_wrap(key, f)
sql['SELECT_get_latest_cpvs_by_category'] = """
SELECT __CPV__, versions.mtime
FROM versions
JOIN packages USING (cp)
JOIN categories USING (c)
WHERE category = ?
ORDER by versions.mtime DESC
LIMIT 0, ?
"""
def _get_latest_cpvs_by_category(self, cat, limit=None):
"""return modified cpvs (uncached)"""
sql = self.sql['SELECT_get_latest_cpvs_by_category']
if limit is None:
limit = 1000
try:
params = (cat, int(limit), )
except ValueError:
return []
cursor = self.cursor()
cursor.execute(sql, params)
entries = cursor.fetchall()
cursor.close()
return entries
def get_latest_cpvs_by_category_pkgname(self, cat, pn, limit=None):
"""return modified cpvs"""
key = '%s_%s/%s_%s' % (caller_name(), cat, pn, limit)
def f():
return self._get_latest_cpvs_by_category_pkgname(cat, pn, limit)
return self.mc_wrap(key, f)
sql['SELECT_get_latest_cpvs_by_category_pkgname'] = """
SELECT __CPV__, versions.mtime
FROM versions
JOIN packages USING (cp)
JOIN categories USING (c)
WHERE category = ? AND pn = ?
ORDER by versions.mtime DESC
LIMIT 0, ?
"""
def _get_latest_cpvs_by_category_pkgname(self, cat, pn, limit=None):
"""return modified cpvs (uncached)"""
sql = self.sql['SELECT_get_latest_cpvs_by_category_pkgname']
if limit is None:
limit = 1000
try:
params = (cat, pn, int(limit), )
except ValueError:
return []
cursor = self.cursor()
cursor.execute(sql, params)
entries = cursor.fetchall()
cursor.close()
return entries
def get_latest_cpvs_by_arch(self, arch, mode, limit=None):
"""return modified cpvs limited by arch and mode"""
key = '%s_%s_%s_%s' % (caller_name(), arch, mode, limit)
def f():
return self._get_latest_cpvs_by_arch(arch, mode, limit)
return self.mc_wrap(key, f)
sql['SELECT_get_latest_cpvs_by_arch_mode'] = """
SELECT __CPV__, versions.mtime
FROM arches
JOIN keywords USING (a)
RIGHT OUTER JOIN versions USING (cpv)
JOIN packages USING (cp)
JOIN categories USING (c)
WHERE arches.arch=? AND SUBSTR(keywords.mode,1,1)=?
ORDER by versions.mtime DESC
LIMIT 0, ?
"""
sql['SELECT_get_latest_cpvs_by_arch_nomode'] = """
SELECT __CPV__, versions.mtime
FROM arches
JOIN keywords USING (a)
RIGHT OUTER JOIN versions USING (cpv)
JOIN packages USING (cp)
JOIN categories USING (c)
WHERE arches.arch=?
ORDER by versions.mtime DESC
LIMIT 0, ?
"""
def _get_latest_cpvs_by_arch(self, arch, mode, limit=None):
"""return modified cpvs limited by arch and mode (uncached)"""
valid_modes = ['+', '~', 'M', 'M+', 'M~', '']
params = ()
if mode not in valid_modes or arch not in self.arches:
return []
if limit is None:
limit = 1000
try:
limit = int(limit)
except ValueError:
return []
if mode != '':
sql = self.sql['SELECT_get_latest_cpvs_by_arch_mode']
params = (arch, str(mode)[0], limit)
else:
sql = self.sql['SELECT_get_latest_cpvs_by_arch_nomode']
params = (arch, limit)
cursor = self.cursor()
cursor.execute(sql, params)
entries = cursor.fetchall()
cursor.close()
return entries
def get_package_details_cpv(self, cpvstr):
"""get package details with versionfiltered keywords"""
key = '%s_%s' % (caller_name(), str(cpvstr))
def f():
return self._get_package_details_cpv(cpvstr)
return self.mc_wrap(key, f)
def _get_package_details_cpv(self, cpvstr):
"""get package details with versionfiltered keywords (uncached)"""
atom = get_atom(str(cpvstr))
(cat, pn) = (atom.category, atom.package)
pkg_metadata = self.get_package_details(cat, pn)
if pkg_metadata is None:
raise Exception('No metadata available for %s/%s' % (cat, pn))
pkg = {}
pkg['atom'] = atom
pkg['meta'] = pkg_metadata
pkg['keywords'] = self.get_package_keywords(atom)
pkg['license'] = []
if pkg_metadata['license']:
for l in pkg_metadata['license'].split():
if l[0].isalnum() and not l.endswith("?"):
url = viewcvs_link('licenses/%s' % l)
l = '%s' % (url, l)
pkg['license'].append(l)
pkg['license'] = ' '.join(pkg['license'])
pkg['homepage'] = []
homepages = pkg_metadata['homepage'].split()
if len(homepages) == 1:
url = homepages[0]
l = '%s' % (url, 'Homepage')
pkg['homepage'].append(l)
else:
pkg['homepage'].append('Homepages:')
i = 1
for l in homepages:
l = '%s' % (l, i)
pkg['homepage'].append(l)
i += 1
pkg['homepage'] = ' '.join(pkg['homepage'])
pkg['changelog'] = viewcvs_link('%s/ChangeLog' % (atom.key))
pkg['metadataxml'] = viewcvs_link('%s/metadata.xml' % (atom.key))
pkg['bugzilla'] = bugzilla_search_link(pn)
pkg['forums'] = forums_search_link(pn)
return pkg
def get_package_details(self, cat, pn):
"""get package details"""
key = '%s_%s/%s' % (caller_name(), cat, pn)
def f():
return self._get_package_details(cat, pn)
return self.mc_wrap(key, f)
sql['SELECT_get_package_details_metadata'] = """
SELECT license, homepage, description, changelog
FROM metadata
JOIN packages USING (cp)
JOIN categories USING (c)
WHERE category=? AND pn=?
"""
sql['SELECT_get_package_details_mtime'] = """
SELECT mtime
FROM versions
JOIN packages USING (cp)
JOIN categories USING (c)
WHERE category = ? AND pn = ?
ORDER BY mtime DESC
"""
def _get_package_details(self, cat, pn):
"""get package details (uncached)"""
cp = '%s/%s' % (cat, pn)
# Returns exactly one row
sql = self.sql['SELECT_get_package_details_metadata']
cursor = self.cursor()
cursor.execute(sql, (cat, pn))
row = cursor.fetchone()
if not row:
return
pkglicense, homepage, description, changelog = row
(changelog, authorname, authorid) = \
pretty_changelog(cat, pn, changelog)
# Returns exactly one column and one row
sql = self.sql['SELECT_get_package_details_mtime']
cursor.execute(sql, (cat, pn))
row = cursor.fetchone()
if not row:
return
ebuildchangeutc = datetime.datetime.utcfromtimestamp(row[0])
package_details = { 'atom': get_atom(str(cp)),
'license': pkglicense,
'homepage': homepage,
'description': description,
'commitmsg': changelog,
'authorname': authorname,
'authorid': authorid,
#'changelogdate': date,
'ebuildchange': ebuildchangeutc }
cursor.close()
return package_details
def get_package_keywords(self, query):
"""return cpv keywords"""
key = '%s_%s' % (caller_name(), query, )
def f():
return self._get_package_keywords(query)
return self.mc_wrap(key, f)
sql['SELECT_get_package_keywords_cat_pn'] = """
SELECT __CPV__, arches.arch, mode
FROM arches
JOIN keywords USING (a)
RIGHT OUTER JOIN versions USING (cpv)
JOIN packages USING (cp)
JOIN categories USING (c)
WHERE category = ? AND pn = ?
ORDER BY pv ASC"""
sql['SELECT_get_package_keywords_cat_pn_pv'] = """
SELECT __CPV__, arches.arch, mode
FROM arches
JOIN keywords USING (a)
RIGHT OUTER JOIN versions USING (cpv)
JOIN packages USING (cp)
JOIN categories USING (c)
WHERE category = ? AND pn = ? AND pv = ?
ORDER BY pv ASC"""
def _get_package_keywords(self, query):
"""return cpv keywords (uncached)"""
if query.fullver:
sql = self.sql['SELECT_get_package_keywords_cat_pn_pv']
params = (query.category, query.package, query.fullver)
else:
sql = self.sql['SELECT_get_package_keywords_cat_pn']
params = (query.category, query.package)
cursor = self.cursor()
cursor.execute(sql, params)
last_cpv = None
ret_list = []
pkg = {}
for row in cursor.fetchall():
cpv = str(row[0])
arch = str(row[1])
mode = str(row[2])
if not query.fullver:
current_cpv = get_atom(cpv)
else:
current_cpv = query
if last_cpv != current_cpv:
if last_cpv is not None:
ret_list.append(pkg)
pkg = {'atom': current_cpv}
for i in frozenset(ConstData.arches['all']):
pkg[i] = {'mode':'', 'name': 'blank'}
last_cpv = current_cpv
if mode.startswith("M"):
name = "hardmask"
elif mode == "~":
name = "unstable"
elif mode =="+":
name = "stable"
elif mode =="-":
name = "blocked"
else:
# mode == "":
name = "blank"
# this might be 'None' which is not a valid arch...
if arch != 'None':
pkg[arch]['name'] = name
pkg[arch]['mode'] = mode
# catch most recent package
ret_list.append(pkg)
# Give it a nice order
if len(ret_list) > 1:
ret_list.sort(key=operator.itemgetter('atom'), reverse=True)
cursor.close()
return ret_list
def get_category_packages(self, cat):
"""get a list of all packages for a category"""
key = '%s_%s' % (caller_name(), cat, )
def f():
return self._get_category_packages(cat)
return self.mc_wrap(key, f)
sql['SELECT_get_category_packages'] = """
SELECT __CP__, 0
FROM packages
JOIN categories USING (c)
WHERE category = ?
ORDER BY pn"""
def _get_category_packages(self, cat):
"""get a list of all packages for a category (uncached)"""
sql = self.sql['SELECT_get_category_packages']
cursor = self.cursor()
cursor.execute(sql, (cat, ))
entries = cursor.fetchall()
cursor.close()
return entries
def get_category_list(self):
"""get a list of all categories with their package and ebuild counts"""
key = '%s' % (caller_name(), )
def f():
return self._get_category_list()
return self.mc_wrap(key, f)
sql['SELECT_get_category_list'] = """
SELECT category, COUNT(DISTINCT cp), COUNT(DISTINCT cpv)
FROM categories
JOIN packages USING (c)
JOIN versions USING (cp)
GROUP BY c
ORDER BY category"""
def _get_category_list(self):
"""get a list of all categories with their package and ebuild counts (uncached)"""
sql = self.sql['SELECT_get_category_list']
cursor = self.cursor()
cursor.execute(sql)
entries = cursor.fetchall()
cursor.close()
return entries
class SQLitePackageDB(PackageDB):
"""override for sqlite backend"""
columns_category_pn = "category || '/' || pn"
columns_category_pn_pv = "category || '/' || pn || '-' || pv"
syntax_placeholder = '?'
def __init__(self, config=None):
# Do not complain about correct usage of ** magic
# pylint: disable-msg=W0142
PackageDB.__init__(self, config)
try:
import sqlite3 as sqlite
except ImportError:
try:
import pysqlite2.dbapi2 as sqlite
except ImportError:
print "Please install PySQLite or use Python 2.5 with sqlite"
sys.exit(1)
self.db = PersistentDB(sqlite, 100, **self.dbconfig['dbconn'])
self._preparesql()
class MySQLPackageDB(PackageDB):
"""override for MySQL backend"""
columns_category_pn = "CONCAT(category, '/', pn)"
columns_category_pn_pv = "CONCAT(category, '/', pn, '-', pv)"
syntax_placeholder = '%s'
def __init__(self, config=None):
# Do not complain about correct usage of ** magic
# pylint: disable-msg=W0142
PackageDB.__init__(self, config)
try:
import MySQLdb
except ImportError:
print "Please install a recent version of MySQLdb for Python"
sys.exit(1)
self.db = PooledDB(MySQLdb, mincached=1, maxcached=3, maxshared=3, maxconnections=3, **self.dbconfig['dbconn'])
self._preparesql()
# vim:ts=4 et ft=python: