diff --git a/calc_timing.sql b/calc_timing.sql new file mode 100644 index 000000000..acf634a08 --- /dev/null +++ b/calc_timing.sql @@ -0,0 +1,16 @@ +select + t.measure_id, + t.measure_session, + t.measure_point, + t.call_count, + (t.time_function / t.call_count) as time_per_call, + t.recursive_call_count, + t.time_total, + t.time_function, + (t.time_total - t.time_function) as time_other, + t.code +from timings as t +order by + t.time_total desc, + t.time_function desc +limit 10; \ No newline at end of file diff --git a/calc_timing_calls.sql b/calc_timing_calls.sql new file mode 100644 index 000000000..5bbb89e09 --- /dev/null +++ b/calc_timing_calls.sql @@ -0,0 +1,17 @@ +select + tc.measure_id, + tc.measure_point, + tc.call_count, + (tc.time_function / tc.call_count) as time_per_call, + tc.recursive_call_count, + tc.time_total, + tc.time_function, + (tc.time_total - tc.time_function) as time_other, + tc.code +from timings_calls as tc +where tc.measure_id = 411 +order by + tc.time_total desc, + tc.time_function desc, + time_other desc +limit 5; diff --git a/etc/adhocracy.pyprol.ini b/etc/adhocracy.pyprol.ini new file mode 100644 index 000000000..17e1b1205 --- /dev/null +++ b/etc/adhocracy.pyprol.ini @@ -0,0 +1,441 @@ +# +# WARNING: Please make you changes in *.ini.in file and then run buildout to install it. +# + +############################################################## +# Generated adhocracy settings +############################################################## + +[DEFAULT] +debug = True + + +# Uncomment and replace with the address which should receive any error reports +email_to = +smtp_server = localhost +smtp_port = 8825 +error_email_from = mail@adhocracy.cc + +[server:main] +use = egg:Paste#http +host = 0.0.0.0 +port = 5001 +use_threadpool = False + +threadpool_workers = 20 + + +###################not themed adhocracy ################### + +[composite:main] +use = egg:Paste#urlmap +/ = default + +[pipeline:default] +pipeline = pyprol content + + +[filter:pyprol] +use = egg:pyprol +pyprol.storage = sqlite:///nobackup/koenen/adhocracy_buildout/var/perf/db +pyprol.instrumentations = adhocracy.instrumentations.model + + +###################Adhocracy conf ################### + +#WARNING changed name: #[app:main] + +[app:content] +use = egg:adhocracy +full_stack = true +static_files = true + +cache_dir = /nobackup/koenen/adhocracy_buildout/var/data +beaker.session.key = adhocracy_state + +beaker.session.secret = 0x4f65112ae94cafb2135031d8198aa9bda26fca79906a20ac2d7e7e48ecf857ffL +adhocracy.auth.secret = 0x4c26646ca5b72ba9a9022f8e280aa3e0796ecc54fa0c43ca9d0b9535944f7d3aL + +adhocracy.crypto.secret = 0xc04cd03c82e27356a66bf40318f9b4d56d6f1574a6bbdd687fe3997527bf4e06L + +# INSTALL: Use cookies instead of beaker(=file system) to store session values +# adhocracy.session.implementation = cookie + +# INSTALL: Use URLs like /i/meta/proposal/foo instead of +# http://meta.adhocracy.lan:5001/proposal/foo +# If this is set, make sure to comment out beaker.session.cookie_domain +# and session.domain below. +adhocracy.relative_urls = True + +# INSTALL: Insert your domain name here. Make sure to include the +# dot at the beginning of each domain to create wildcard cookies. +# Also make sure this is identical to the adhocracy.domains setting +# below. +# Should be commented out iff relative URLs are used. + + + +# If you'd like to fine-tune the individual locations of the cache data dirs +# for the Cache data, or the Session saves, un-comment the desired settings +# here: +beaker.cache.data_dir = /nobackup/koenen/adhocracy_buildout/var/data/cache +beaker.session.data_dir = /nobackup/koenen/adhocracy_buildout/var/data/sessions + +# INSTALL: Locate your Memcached server if installed. Otherwise comment this out. +memcached.server = 127.0.0.1:5005 + +# INSTALL: SQLAlchemy database URL +sqlalchemy.url = sqlite:////nobackup/koenen/adhocracy_buildout/var/development.db + +# INSTALL: Uncommenting this line will delete all locally saved data and drop +# the entire database. Please only use this if you have a backup ready. +# Yes, that's a magic string. +## adhocracy.setup.drop = KILL_EM_ALL +# + +adhocracy.solr.url = http://localhost:5007/solr + +# To avoid a race condition between main and background processes, delaying +# background process updates can be delayed (see issue #358). This option +# defines the delay in seconds (default: 1 second) + +# adhocracy.delay_update_queue_seconds = 1 + +# If debug is enabled, an web-based interactive debugger can be executed from +# the HTTP500 error page, if the following option is set to true. This option +# doesn't have any effect if debug is false. +adhocracy.interactive_debugging = true + +# WARNING: *THE LINE BELOW MUST BE UNCOMMENTED ON A PRODUCTION ENVIRONMENT* +# Debug mode will enable the interactive debugging tool, allowing ANYONE to +# execute malicious code after an exception is raised. +#set debug = false + + +# INSTALL: Set up the Adhocracy domain name. This will be used to determine the +# active instance, e.g. "Test Instance" for test.adhocracy.lan. + + +adhocracy.domain = adhocracy.lan:5001 +adhocracy.protocol = http + + + +# INSTALL: Set a site name for all instances. This will be used as a part of +# the html , notification E-Mails etc. +adhocracy.site.name = HHU Normsetzung +adhocracy.language = de_DE + +# INSTALL: Server timezone +# adhocracy.timezone = Europe/Berlin + +# INSTALL: The site directory contains your site modifications of Adhocracy +# (e.g. logos, CSS, html pages). A skeleton site directory will be created +# by the "setup-app" paster command. +adhocracy.site.dir = /nobackup/koenen/adhocracy_buildout/var/site + +# INSTALL: Location of your custom client (overrides adhocracy defaults) +# adhocracy.client_location = /nobackup/koenen/adhocracy_buildout/src/adhocracy.YOUR_theme/ + +# INSTALL: A nice E-Mail address. +adhocracy.email.from = mail@adhocracy.cc + +# INSTALL: Allow users to use the above email as sender address in mass messages +allow_system_email_in_mass_messages = True + +# Show fallback icons in instance overview if no instance icon is defined +adhocracy.show_instance_fallback_icons = False + +# INSTALL: Twitter. Since there is currently no option to take it out of the UI, +# you will want to use Twitter. Create an OAuth application to allow users to +# link their profiles: https://twitter.com/oauth_clients +adhocracy.twitter.username = +adhocracy.twitter.key = +adhocracy.twitter.secret = +adhocracy.twitter.consumer_key = +adhocracy.twitter.consumer_secret = + +# INSTALL: Full path to a installation wide twitter profile which is used in +# follow-us link, e.g. http://twitter.com/liqd_de +adhocracy.twitter.profile_url = + +# INSTALL: bit.ly is used as a URL shortener. Since their API requires auth, you +# can either leave the defaults here and give certain usage statistics to the +# Adhocracy developers, or you can create an account at http://api.bit.ly +#adhocracy.bitly.login = +#adhocracy.bitly.key = + + +# Redis is used to build a task queue that will be processed by the +# `worker` process. +# Note: host has to be an IP address, not a hostname +adhocracy.redis.host = 127.0.0.1 +adhocracy.redis.port = 5006 +adhocracy.redis.queue = adhocracy.queue + +# Background processing can be disabled entirely +adhocracy.background_processing = true + +# INSTALL: If you want to use recaptcha uncomment these lines +#recaptcha.public_key = 6LcICcgSAAAAAKNYq9iM4lIpB_GSjYItF21wS4NS +#recaptcha.private_key = 6LcICcgSAAAAAEJL1zhVbX6N0TGb0zVXD_g2lXSo + +# TUNING: Memcache page fragments? +adhocracy.cache_tiles = True + +# adhocracy.instance = adhocracy + +# INSTALL: Add an HTTP header that identifies the server we're running on +#adhocracy.include_machine_name_in_header = True + +# INSTALL: Feedback instance settings. +# If feedback shall be used, users need to be member of the feedback instance. +# This can be achieved with the `adhocracy.instances.autojoin` option below. If +# the feedback instance is enabled later, existing users can be added to the +# feedback instance with the following command: +# +# `bin/adhocpy src/adhocracy/scripts/add_all_users_to_instances.py etc/adhocracy.ini feedback` + +adhocracy.use_feedback_instance = true +adhocracy.feedback_instance_key = feedback +adhocracy.feedback_instance_label = Feedback + +# INSTALL: Instances which new users automatically join. This can be a comma +# separated list of instances, or ALL. If feedback is enabled above, the +# corresponding instance key or ALL should be part of this list. +adhocracy.instances.autojoin = feedback + +# INSTALL: Read client IP addresses from X-Forwarded-For header. +adhocracy.behind_proxy = False + +# INSTALL: Store statistic information about requests in the database +#adhocracy.requestlog_active = True +# ... but don't log IP addresses ('dontlog', default) +# ... but set the last 8 / 80 bits to zero ('anonymize') +# -.. but log the full IP address ('none') +#adhocracy.requestlog_ipanonymization = anonymize + +# INSTALL: Redirect outgoing links so that we can see when a user leaves our site. +# adhocracy.track_outgoing_links = True + +# Statistics via Piwik +piwik.site = +piwik.id = + +# INSTALL: Allow select users to include HTML +# If this option is set to false, we only allow markdown input. +# If it is set, we make a best effort to filter unsafe HTML, but allow iframes +# and other complex constructs. +#adhocracy.allow_user_html = false + +# INSTALL: Allow registration +#adhocracy.allow_registration = true + +# Require emails from users (default: true) +# adhocracy.require_email = true + +# INSTALL: If registration is disabled above, and adhocracy.login_style is +# "alternate", notify this address on registration attempts. +# adhocracy.registration_support_email = + +# INSTALL: How can users login? +# Available options are openid, username+password, email+password, shibboleth +# Note that shibboleth needs additional configuration +#adhocracy.login_type = openid,username+password,email+password + +# Allow password change for shibboleth users (default: false) +# adhocracy.allow_shibboleth_password_change = false + +# Hide local login form (e.g. when using Shibboleth or OpenID primarily) +# The normal login can still be found under `/login?locallogin` +# adhocracy.hide_locallogin = true + +# INSTALL: Allow administrators to let users log in from an invitation email +# adhocracy.enable_welcome = True + +# INSTALL: Hide individual votes and only show aggregate results (EXPERIMENTAL) +# adhocracy.hide_individual_votes = true + +# INSTALL: Allow users to lock themselves out of adhocracy +# adhocracy.allow_self_deletion = true + +# In the case that no came_from parameter is set, users are redirected to the +# locations defined in the following settings. You can leave any of these +# commented out. If instance is given, the url of the given instance is used +# to build the URL, otherwise the current instance is used. + +# adhocracy.post_register_url = +# adhocracy.post_register_instance = +# adhocracy.post_login_url = /proposal +# adhocracy.post_login_instance = test + +# activate geo features + + + +# show tutorials +adhocracy.show_tutorials = True + +# hide instance list in main navigation +adhocracy.hide_instance_list_in_navigation = False + +# hide watchlist in main navigation and put it in user menu +adhocracy.put_watchlist_in_user_menu = False + +# useful in settings where users can create instances +adhocracy.show_instance_creators = False + +# show next milestones in instance overview +adhocracy.show_instance_overview_milestones = True + +# number of upcoming milestones to show in instance overview +adhocracy.number_instance_overview_milestones = 3 + +# show newest proposals in instance overview +# adhocracy.show_instance_overview_proposals_new = False + +# hide categories in facet navigation if no proposal is assigned to it +adhocracy.hide_empty_categories_in_facet_list = True + +# Show global stats (user/proposal/comment count) on frontpage +#adhocracy.show_stats_on_frontpage = true + +# number of newest proposals to show on frontpage (0 to hide the list) +adhocracy.startpage.proposals.list_length = 0 + +# Source for imprint, details etc. ("database" or "filesystem" (default)) +#adhocracy.staticpage_backend = database + +# Configure paths to static content which is inserted + +# Static content path for start page (default: index) +adhocracy.static_index_path = index + +# Static content path for login page (default: empty) +# adhocracy.static_login_path = login + +# Static content paths for shibboleth register page (default: empty) +# Ontop defines the area on top of the form and the title of the page. +# adhocracy.static_shibboleth_register_ontop_path = shibboleth_register_ontop +# Below defines the area below the form, the title is ignored. +# adhocracy.static_shibboleth_register_below_path = shibboleth_register + +# Ask the user for email and then password, instead of form + links +# adhocracy.login_style = alternate + +# Force randomized user names (default: false) +# adhocracy.force_randomized_user_names = true + +# Set display name in register dialog (default: false) +# adhocracy.set_display_name_on_register = true + +# Automatically set new permissions their default value. This should be +# set to false in restrictive environments. +# adhocracy.autoassign_permissions = true + +# Turn on badge-specific UI modifications +# adhocracy.enable_behavior = True + +# Show Facebook / Twitter / G+ buttons +# adhocracy.show_social_buttons = False + +## Performance options + +# Check whether feedback instance exists before rendering a feedback button +# adhocracy.feedback_check_instance = False + +# Load feedback categories for the feedback form +# adhocracy.feedback_use_categories = False + +# Optional user attributes +# adhocracy.user.optional_attributes = +# year_of_birth, int, Year of Birth +# foo, unicode, Foo +# adhocracy.user.optional_attributes.year_of_birth = +# 2001, 2222, 1111 +## adhocracy.user.optional_attributes.foo = + + +# Logging configuration +[loggers] +keys = root, adhocracy, sqlalchemy + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = INFO +handlers = console + +[logger_routes] +level = INFO +handlers = +qualname = routes.middleware +# "level = DEBUG" logs the route matched and routing variables. + +[logger_adhocracy] +level = DEBUG + +handlers = +qualname = adhocracy + +[logger_sqlalchemy] +level = WARN +handlers = console +qualname = sqlalchemy.engine + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(asctime)s %(levelname)-5.5s [%(name)s] %(message)s + +############################################################## +# Custom OVERRIDE adhocracy settings +############################################################## + +[app:content] +adhocracy.client_location = /nobackup/koenen/adhocracy_buildout/src/adhocracy.hhu_theme/ +adhocracy.comment_wording = True +adhocracy.enable_behavior = True +adhocracy.enable_gender = True +adhocracy.enable_votedetail = True +adhocracy.enable_welcome = True +adhocracy.feedback_check_instance = False +adhocracy.feedback_use_categories = False +adhocracy.hide_individual_votes = true +adhocracy.include_machine_name_in_header = True +adhocracy.instances.autojoin = ALL +adhocracy.login_style = alternate +adhocracy.login_type = email+password +adhocracy.monitor_browser_values = True +adhocracy.monitor_comment_behavior = True +adhocracy.monitor_external_links = True +adhocracy.monitor_page_performance = True +adhocracy.monitor_page_time_interval = 10000 +adhocracy.monitor_pager_clicks = True +adhocracy.post_login_instance = +adhocracy.post_login_url = / +adhocracy.post_register_instance = +adhocracy.post_register_url = +adhocracy.registration_support_email = normsetzung-support@cs.uni-duesseldorf.de +adhocracy.requestlog_active = True +adhocracy.requestlog_ipanonymization = none +adhocracy.self_deletion_allowed = false +adhocracy.session.implementation = cookie +adhocracy.show_abuse_button = False +adhocracy.show_instance_overview_events = False +adhocracy.show_instance_overview_proposals_all = True +adhocracy.show_instance_overview_proposals_new = False +adhocracy.show_instance_overview_stats = False +adhocracy.show_social_buttons = False +adhocracy.show_stats_on_frontpage = False +adhocracy.startpage.instances.list_length = -1 +adhocracy.staticpage_backend = database +adhocracy.track_outgoing_links = True diff --git a/src/adhocracy/instrumentations/__init__.py b/src/adhocracy/instrumentations/__init__.py new file mode 100644 index 000000000..833a0ba11 --- /dev/null +++ b/src/adhocracy/instrumentations/__init__.py @@ -0,0 +1,2 @@ +from . import model + diff --git a/src/adhocracy/instrumentations/model.py b/src/adhocracy/instrumentations/model.py new file mode 100644 index 000000000..48e2d77c5 --- /dev/null +++ b/src/adhocracy/instrumentations/model.py @@ -0,0 +1,25 @@ +import importlib + +from pyprol.measurement import measurement +from logging import getLogger + +log = getLogger(u'adhocracy.instrumentations.model') + +def inject(config): + try: + _adhocracy_model = importlib.import_module("adhocracy.model") + _before_commit = _adhocracy_model.before_commit + + def before_commit(session): + measure = measurement.enable(u"adhocracy.model.before_commit") + result = _before_commit(session) + measurement.disable(measure) + return result + + _adhocracy_model.before_commit = before_commit + log.info("injected into adhocracy.model.before_commit") + + except ImportError as e: + log.info("Could not import `adhocracy.model`") + log.debug(e) + diff --git a/src/adhocracy/lib/importexport/transforms.py b/src/adhocracy/lib/importexport/transforms.py index 904dc139d..2aa1f78d3 100644 --- a/src/adhocracy/lib/importexport/transforms.py +++ b/src/adhocracy/lib/importexport/transforms.py @@ -1,6 +1,7 @@ import datetime import re +import hashlib from adhocracy.lib import votedetail from adhocracy import config @@ -459,12 +460,31 @@ class RequestLogTransform(_Transform): def __init__(self, options): super(RequestLogTransform, self).__init__(options) + self.hash_func = hashlib.sha1 def _export(self, obj): res = obj.to_dict() + res['access_time'] = encode_time(res['access_time']) + + # Filter out session codes from adhocracy login cookie + res['cookies'] = re.sub(r'(adhocracy_login\=)("?[a-f0-9]{40})', + lambda m: m.group(1) + self.hash_func(m.group(2)).hexdigest(), + res['cookies']) + + # Filter out welcome codes + res['request_url'] = self._url_filter(res['request_url']) + res['referer'] = self._url_filter(res['referer']) + return res + def _url_filter(self, url): + if url is not None: + url = re.sub(r'(/welcome/[^/]+/)([0-9a-f]+)(?=/|\?|$)', + lambda m: m.group(1) + self.hash_func(m.group(2)).hexdigest(), + url) + return url + class StaticPageTransform(_Transform): def __init__(self, options, backend=None):