#!/usr/bin/env python """Packages2 CherryPy Application and launcher""" import os, sys, math, time import cherrypy from web.model import latest_per_day, \ build_centerpkg_list, \ get_atom from web.lib.headers import lastmodified_httpheader, \ lastmodified_rightcontent from web.lib.query_filter import sanitize_query_string from web.lib import template, filters from pkgcore.ebuild.cpv import unversioned_CPV from pkgcore.ebuild.cpv import versioned_CPV # We use short variable names! # pylint: disable-msg=C0103 class Root(object): """Packages2 CherryPy Application""" database = None def __init__(self, db): self.database = db @property def cache_latest(self): def f(): entry_filter = filters.EntryFilters(self.database) return entry_filter.unfiltered() return self.database.mc_wrap('cache_latest', f, time=300) @property def cache_newpkgs(self): def f(): entry_filter = filters.EntryFilters(self.database) return entry_filter.newpkgs_filter() return self.database.mc_wrap('cache_newpkgs', f, time=300) @property def cache_verbumps(self): def f(): entry_filter = filters.EntryFilters(self.database) return entry_filter.verbumps_filter() return self.database.mc_wrap('cache_verbumps', f, time=300) @property def cache_date(self): def f(): entry_filter = filters.EntryFilters(self.database) return entry_filter.date_filter() return self.database.mc_wrap('cache_date', f, time=300) @property def cache_categories(self): def f(): return self.database.get_category_list() return self.database.mc_wrap('cache_categories', f, time=300) @cherrypy.expose @template.expire_on_30_min() @template.output('index.html', method='xhtml') def index(self, *args, **kwds): """Render the root page as HTML""" # Do not complain about correct usage of ** magic # pylint: disable-msg=W0142 return self._index(*args, **kwds) def _index(self, *args, **kwds): """Render the root page per the caller""" if len(args) > 0: raise cherrypy.HTTPRedirect("/") db = self.database entry_filter = filters.EntryFilters(db) latest_entries = self.cache_latest left_daycount = filters.limit_leftcount(kwds) arches = filters.limit_arches(kwds) center_pkgs = build_centerpkg_list(latest_entries, db.get_package_details_cpv, filters.limit_centercount(kwds), use_fullver=True) day_list = latest_per_day(latest_entries, left_daycount) latest_entry = entry_filter.latest_entry() http_lastmodified = lastmodified_httpheader(latest_entry) cherrypy.response.headers['Last-Modified'] = http_lastmodified pagetitle = '' if pagetitle in kwds and len(kwds['pagetitle']) > 0: pagetitle = kwds['pagetitle'] kwds = sanitize_query_string(kwds) db.close_mc() return template.render(arches = arches, daylist = day_list, center_pkgs = center_pkgs, lastupdate = latest_entry, safeqs = kwds, pagetitle = pagetitle) @cherrypy.expose @template.expire_on_30_min() @template.output('faq.html', method='xhtml') def faq(self, *args, **kwds): """Render the FAQ page as HTML""" # Do not complain about correct usage of ** magic # pylint: disable-msg=W0142 kwds['pagetitle'] = 'FAQ' return self._index(*args, **kwds) @cherrypy.expose @template.expire_on_30_min() @template.output('index.html', method='xhtml') def verbump(self, *args, **kwds): """Render the /verbump/ page as HTML""" # Do not complain about correct usage of ** magic # pylint: disable-msg=W0142 return self._verbump(*args, **kwds) def _verbump(self, *args, **kwds): """Render the /verbump/ page per the caller""" if len(args) > 0: raise cherrypy.HTTPRedirect("/") db = self.database entry_filter = filters.EntryFilters(db) filtered_latest_entries = self.cache_verbumps left_daycount = filters.limit_leftcount(kwds) arches = filters.limit_arches(kwds) center_pkgs = build_centerpkg_list(filtered_latest_entries, db.get_package_details_cpv, filters.limit_centercount(kwds), use_fullver=True) left_entrylist = self.cache_latest day_list = latest_per_day(left_entrylist, left_daycount) pagetitle = "/verbump/" latest_entry = entry_filter.latest_entry() http_lastmodified = lastmodified_httpheader(latest_entry) cherrypy.response.headers['Last-Modified'] = http_lastmodified kwds = sanitize_query_string(kwds) db.close_mc() return template.render(arches = arches, daylist = day_list, center_pkgs = center_pkgs, lastupdate = latest_entry, pagetitle = pagetitle, safeqs = kwds) @cherrypy.expose @template.expire_on_30_min() @template.output('index.html', method='xhtml') def newpackage(self, *args, **kwds): """Render the /newpackage/ page as HTML""" # Do not complain about correct usage of ** magic # pylint: disable-msg=W0142 return self._newpackage(*args, **kwds) def _newpackage(self, *args, **kwds): """Render the /newpackage/ page per the caller""" if len(args) > 0: raise cherrypy.HTTPRedirect("/") db = self.database entry_filter = filters.EntryFilters(db) filtered_latest_entries = self.cache_newpkgs left_daycount = filters.limit_leftcount(kwds) arches = filters.limit_arches(kwds) center_pkgs = build_centerpkg_list(filtered_latest_entries, db.get_package_details_cpv, filters.limit_centercount(kwds), use_fullver=True) left_entrylist = self.cache_latest day_list = latest_per_day(left_entrylist, left_daycount) pagetitle = "/newpackage/" latest_entry = entry_filter.latest_entry() http_lastmodified = lastmodified_httpheader(latest_entry) cherrypy.response.headers['Last-Modified'] = http_lastmodified kwds = sanitize_query_string(kwds) db.close_mc() return template.render(arches = arches, daylist = day_list, center_pkgs = center_pkgs, lastupdate = latest_entry, pagetitle = pagetitle, safeqs = kwds) @cherrypy.expose @template.expire_on_30_min() @template.output('index.html', method='xhtml') def arch(self, *args, **kwds): """Render the /arch/[/] page as HTML""" # Do not complain about correct usage of ** magic # pylint: disable-msg=W0142 return self._arch(*args, **kwds) def _arch(self, arch = "", mode="", *args, **kwds): """Render the /arch/[/] page per the caller""" if arch == "" or len(args) > 0: raise cherrypy.HTTPRedirect("/") db = self.database entry_filter = filters.EntryFilters(db) # Default first limit = filters.limit_centercount(kwds) if mode == 'stable': filtered_latest_entries = entry_filter.stable_filter(arch, limit) elif mode == 'unstable': filtered_latest_entries = entry_filter.unstable_filter(arch, limit) elif mode == 'hardmask': filtered_latest_entries = entry_filter.hardmasked_filter(arch, limit) else: filtered_latest_entries = entry_filter.arch_filter(arch, limit) center_pkgs = build_centerpkg_list(filtered_latest_entries, db.get_package_details_cpv, limit, use_fullver=True) if not center_pkgs: raise cherrypy.HTTPRedirect("/") if 'local_latest' in kwds: left_entrylist = filtered_latest_entries else: left_entrylist = self.cache_latest left_daycount = filters.limit_leftcount(kwds) arches = filters.limit_arches(kwds) day_list = latest_per_day(left_entrylist, left_daycount) pagetitle = "/arch/%s/" % (arch, ) if mode: pagetitle += "/%s/" % (mode, ) latest_entry = entry_filter.latest_entry() http_lastmodified = lastmodified_httpheader(latest_entry) cherrypy.response.headers['Last-Modified'] = http_lastmodified kwds = sanitize_query_string(kwds) db.close_mc() return template.render(arches = arches, daylist = day_list, center_pkgs = center_pkgs, lastupdate = latest_entry, pagetitle = pagetitle, safeqs = kwds) @cherrypy.expose @template.expire_on_30_min() @template.output('categories.html', method='xhtml') def categories(self, *args, **kwds): """Render the /categories/ page as HTML""" # Do not complain about correct usage of ** magic # pylint: disable-msg=W0142 return self._categories(*args, **kwds) def _categories(self, *args, **kwds): """Render the /categories/ page per the caller""" if len(args) > 0: raise cherrypy.HTTPRedirect("/") db = self.database entry_filter = filters.EntryFilters(db) latest_entries = self.cache_latest left_daycount = filters.limit_leftcount(kwds) arches = filters.limit_arches(kwds) categories_raw = self.cache_categories cols = 5 total_len = len(categories_raw) table_rows = int(math.floor(total_len / cols)) if total_len % cols > 0: table_rows += 1 rows = [] sums = {'cats':0, 'pkgs':0, 'ebuilds':0} for i in range(table_rows): row = [] for j in range(cols): if i*cols + j < total_len: cat = categories_raw[i*cols + j] row.append(cat) sums['cats'] += 1 sums['pkgs'] += cat[1] sums['ebuilds'] += cat[2] else: row.append((None, -1, -1)) rows.append(row) day_list = latest_per_day(latest_entries, left_daycount) latest_entry = entry_filter.latest_entry() http_lastmodified = lastmodified_httpheader(latest_entry) cherrypy.response.headers['Last-Modified'] = http_lastmodified pagetitle = 'Categories' kwds = sanitize_query_string(kwds) db.close_mc() return template.render(arches = arches, daylist = day_list, center_categories = rows, lastupdate = latest_entry, safeqs = kwds, sums = sums, pagetitle = pagetitle) @cherrypy.expose @template.expire_on_30_min() @template.output('index.html', method='xhtml') def category(self, *args, **kwds): """Render the /category/ page as HTML""" # Do not complain about correct usage of ** magic # pylint: disable-msg=W0142 return self._category(*args, **kwds) def _category(self, category="", *args, **kwds): """Render the /category/ page per the caller""" if category == '' or len(args) > 0: raise cherrypy.HTTPRedirect("/") db = self.database entry_filter = filters.EntryFilters(db) cat_entries = entry_filter.category_filter(category) if 'full_cat' in kwds: cat_entries = entry_filter.category_filter(category, False) center_pkgs = build_centerpkg_list(cat_entries, db.get_package_details_cpv, None) else: center_pkgs = build_centerpkg_list(cat_entries, db.get_package_details_cpv, filters.limit_centercount(kwds)) if not center_pkgs: raise cherrypy.HTTPRedirect("/") if 'local_latest' in kwds: left_entrylist = cat_entries else: left_entrylist = self.cache_latest left_daycount = filters.limit_leftcount(kwds) arches = filters.limit_arches(kwds) day_list = latest_per_day(left_entrylist, left_daycount) pagetitle = "/category/%s/" % (category, ) latest_entry = entry_filter.latest_entry() http_lastmodified = lastmodified_httpheader(latest_entry) cherrypy.response.headers['Last-Modified'] = http_lastmodified kwds = sanitize_query_string(kwds) db.close_mc() return template.render(arches = arches, daylist = day_list, center_pkgs = center_pkgs, lastupdate = latest_entry, pagetitle = pagetitle, safeqs = kwds) @cherrypy.expose @template.expire_on_30_min() @template.output('index.html', method='xhtml') def package(self, *args, **kwds): """Render the /package/ page as HTML""" # Do not complain about correct usage of ** magic # pylint: disable-msg=W0142 return self._package(*args, **kwds) def _package(self, *args, **kwds): """Render the /package/ page per the caller""" pn = '' cat = '' if len(args) == 1: pn = args[0] cpvtmp = get_atom(str('%s/%s' % ('tmp', pn))) pn = cpvtmp.package pagetitle = "/package/%s" % (pn) elif len(args) == 2: cat = args[0] pn = args[1] cpvstr = '%s/%s' % (cat, pn) cpvtmp = get_atom(cpvstr) pn = cpvtmp.package cat = cpvtmp.category pagetitle = "/package/%s/%s" % (cat, pn) if pn == '': raise cherrypy.HTTPRedirect("/") db = self.database entry_filter = filters.EntryFilters(db) if cat == '': package_entries = entry_filter.package_filter(pn) else: package_entries = entry_filter.category_package_filter(cat, pn) center_pkgs = build_centerpkg_list(package_entries, db.get_package_details_cpv, None) if not center_pkgs: raise cherrypy.HTTPRedirect("/") left_entrylist = self.cache_latest left_daycount = filters.limit_leftcount(kwds) arches = filters.limit_arches(kwds) day_list = latest_per_day(left_entrylist, left_daycount) latest_entry = entry_filter.latest_entry() http_lastmodified = lastmodified_httpheader(latest_entry) cherrypy.response.headers['Last-Modified'] = http_lastmodified kwds = sanitize_query_string(kwds) db.close_mc() return template.render(arches = arches, daylist = day_list, center_pkgs = center_pkgs, lastupdate = latest_entry, pagetitle = pagetitle, safeqs = kwds) @cherrypy.expose @template.expire_on_30_min() @template.output('index.html', method='xhtml') def date(self, *args, **kwds): """Render the /date/ page as HTML""" # Do not complain about correct usage of ** magic # pylint: disable-msg=W0142 return self._date(*args, **kwds) def _date(self, *args, **kwds): """Render the /date/ page per the caller""" date = '' if len(args) == 1: try: date = time.strptime(args[0][0:8]+" UTC", "%Y%m%d %Z") pagetitle = time.strftime("/date/%Y%m%d",date) except: date = '' if date == '': raise cherrypy.HTTPRedirect("/") db = self.database entry_filter = filters.EntryFilters(db) package_entries = entry_filter.date_filter(date) center_pkgs = build_centerpkg_list(package_entries, db.get_package_details_cpv, None) # Empty resultsets are allowed # if not center_pkgs: # raise cherrypy.HTTPRedirect("/") left_entrylist = self.cache_latest left_daycount = filters.limit_leftcount(kwds) arches = filters.limit_arches(kwds) day_list = latest_per_day(left_entrylist, left_daycount) latest_entry = entry_filter.latest_entry() http_lastmodified = lastmodified_httpheader(latest_entry) cherrypy.response.headers['Last-Modified'] = http_lastmodified kwds = sanitize_query_string(kwds) db.close_mc() return template.render(arches = arches, daylist = day_list, center_pkgs = center_pkgs, lastupdate = latest_entry, pagetitle = pagetitle, safeqs = kwds) @cherrypy.expose @template.expire_on_30_min() @template.output('index.xml', method='xml') def feed(self, *args, **kwds): """Render the /feed/ page as Atom""" # Do not complain about correct usage of ** magic # pylint: disable-msg=W0142 base = "" if len(args) > 0: base = args[0] args = args[1:] if base == "": return self._index(*args, **kwds) elif base == "arch": return self._arch(*args, **kwds) elif base == "category": return self._category(*args, **kwds) elif base == "package": return self._package(*args, **kwds) elif base == "verbump": return self._verbump(*args, **kwds) elif base == "newpackage": return self._newpackage(*args, **kwds) elif base == "date": return self._date(*args, **kwds) else: raise Exception('Unknown page!') def database_connect(): """Create a DB connection and store it in the current thread""" from etc.database_config import DatabaseConfig database = None if DatabaseConfig.mode == 'sqlite': from web.model import SQLitePackageDB config = { 'dbconn': DatabaseConfig.settings['sqlite'], 'memcached': DatabaseConfig.settings['memcached'] } database = SQLitePackageDB(config) if DatabaseConfig.mode == 'mysql': from web.model import MySQLPackageDB config = { 'dbconn': DatabaseConfig.settings['mysql_ro'], 'memcached': DatabaseConfig.settings['memcached'] } database = MySQLPackageDB(config) if database is None: print "choose database in core.py first" sys.exit(1) return database def handle_error(): """Handle any unhandled exceptions. It turns out a lot of our code raises exceptions that cause 500's for users Instead of raising a 500 (for say, an invalid atom) simply return a 404 instead. The exception should still get logged. """ cherrypy.response.status = 404 def main(): """Use this when we run standalone""" # site-wide config cherrypy.config.update({ #'environment': 'production', 'response.error_response': handle_error, 'log.screen': True, 'log.error_file': '/tmp/cherrypy_packages2.gentoo.log', 'request.show_tracebacks': True, 'tools.caching.on': False, 'tools.caching.cache_class': cherrypy.lib.caching.MemoryCache, 'tools.caching.delay': 1, 'tools.caching.maxobjsize': 1, 'tools.decode.on': True, 'tools.encode.on': True, 'tools.encode.encoding': 'utf-8', #'tools.expires.secs': 1800, 'tools.staticdir.on': False, 'tools.staticdir.root': os.path.abspath(os.path.dirname(__file__)), 'tools.trailing_slash.on': True, #'server.thread_pool': 10 # These are needed for speed #'engine.autoreload_on': False, #'checker.on': False, #'tools.log_headers.on': False, #'request.show_tracebacks': False, #'log.screen': False, }) static_config = { '/media' : { 'tools.staticdir.on': True, 'tools.staticdir.dir': 'static' } } app = Root(database_connect()) cherrypy.server.socket_port = 8081 cherrypy.quickstart(app, '/', static_config) #app = cherrypy.Application(Root(), '') #X#cherrypy.config.update(static_config) #X#cherrypy.tree.mount(Root(), '/', static_config) #X#cherrypy.server.quickstart() #X#cherrypy.engine.start() def setup_server(): """within mod_python use me""" # Set up site-wide config. Do this first so that, # if something goes wrong, we get a log. cherrypy.config.update({ #'environment': 'production', 'response.error_response': handle_error, 'log.screen': False, 'log.error_file': '/tmp/cherrypy_packages2.gentoo.log', 'request.show_tracebacks': True, 'tools.caching.on': False, 'tools.caching.cache_class': cherrypy.lib.caching.MemoryCache, 'tools.caching.delay': 1, 'tools.caching.maxobjsize': 1, 'tools.decode.on': True, 'tools.encode.on': True, 'tools.encode.encoding': 'utf-8', #'tools.expires.secs': 1800, 'tools.staticdir.on': False, 'tools.staticdir.root': os.path.abspath(os.path.dirname(__file__)), 'tools.trailing_slash.on': True, #'server.thread_pool': 64, # These are needed for speed 'engine.autoreload_on': False, 'checker.on': False, 'tools.log_headers.on': False, 'request.show_tracebacks': False, #'request.show_errors': True, 'log.screen': False, }) # Static content is served externally app = Root(database_connect()) cherrypy.tree.mount(app) # Per http://cherrypy.org/wiki/ModPython cherrypy.engine.SIGHUP = None cherrypy.engine.SIGTERM = None if __name__ == '__main__': main() # vim:ts=4 et ft=python: