diff --git a/README.md b/README.md
index 45e7609..5a5f5ff 100644
--- a/README.md
+++ b/README.md
@@ -89,7 +89,7 @@ ckanext-report.notes.dataset = ' '.join(('Unpublished' if asbool(pkg.extras.get(
A report has three key elements:
-1. Report Code - a python function that produces the report.
+1. Report Code - a python function that produces the report.
2. Template - HTML for displaying the report data.
3. Registration - containing the configuration of the report.
@@ -113,7 +113,7 @@ The returned data should be a dict like this:
'average_tags_per_package': 3.5,
}
```
-
+
There should be a `table` with the main body of the data, and any other totals or incidental pieces of data.
Note: the table is required because of the CSV download facility, and CSV demands a table. (The CSV download only includes the table, ignoring any other values in the data.) Although the data has to essentially be stored as a table, you do have the option to display it differently in the web page by using a clever template.
@@ -242,6 +242,7 @@ Info dict spec:
* option_defaults - dict of ALL option names and their default values. Use ckan.common.OrderedDict. If there are no options, you can return None.
* option_combinations - function returning a list of all the options combinations (reports for these combinations are generated by default). If there are no options, return None.
* authorize (optional) - a function that says if the user is allowed to view the report. Takes params: (user_object, options_dict) and should return a boolean - if the user is authorized or not. The default is that anyone can see reports.
+* paginate_by (optional) - number of items to paginate the report view by. If excluded, pagination will be disabled.
Finally we need to define the function that returns the option_combinations:
```python
diff --git a/ckanext/report/controllers.py b/ckanext/report/controllers.py
index cc2c9e0..9bcd491 100644
--- a/ckanext/report/controllers.py
+++ b/ckanext/report/controllers.py
@@ -7,7 +7,6 @@
from ckan.lib.render import TemplateNotFound
from ckan.common import OrderedDict
-
log = __import__('logging').getLogger(__name__)
c = t.c
@@ -33,11 +32,11 @@ def view(self, report_name, organization=None, refresh=False):
# ensure correct url is being used
if 'organization' in t.request.environ['pylons.routes_dict'] and \
- 'organization' not in report['option_defaults']:
+ 'organization' not in report['option_defaults']:
t.redirect_to(helpers.relative_url_for(organization=None))
elif 'organization' not in t.request.environ['pylons.routes_dict'] and\
'organization' in report['option_defaults'] and \
- report['option_defaults']['organization']:
+ report['option_defaults']['organization']:
org = report['option_defaults']['organization']
t.redirect_to(helpers.relative_url_for(organization=org))
if 'organization' in t.request.params:
@@ -46,7 +45,11 @@ def view(self, report_name, organization=None, refresh=False):
t.redirect_to(helpers.relative_url_for())
# options
- options = Report.add_defaults_to_options(t.request.params, report['option_defaults'])
+ options = Report.add_defaults_to_options(
+ t.request.params,
+ report['option_defaults']
+ )
+
option_display_params = {}
if 'format' in options:
format = options.pop('format')
@@ -59,19 +62,22 @@ def view(self, report_name, organization=None, refresh=False):
for option in options:
if option not in report['option_defaults']:
# e.g. 'refresh' param
- log.warn('Not displaying report option HTML for param %s as option not recognized')
+ log.warn('Not displaying report option HTML for param %s '
+ 'as option not recognized')
continue
- option_display_params = {'value': options[option],
- 'default': report['option_defaults'][option]}
+ option_display_params = {
+ 'value': options[option],
+ 'default': report['option_defaults'][option]
+ }
try:
options_html[option] = \
t.render_snippet('report/option_%s.html' % option,
data=option_display_params)
except TemplateNotFound:
- log.warn('Not displaying report option HTML for param %s as no template found')
+ log.warn('Not displaying report option HTML for param %s as '
+ 'no template found')
continue
-
# Alternative way to refresh the cache - not in the UI, but is
# handy for testing
try:
@@ -87,19 +93,24 @@ def view(self, report_name, organization=None, refresh=False):
if refresh:
try:
- t.get_action('report_refresh')({}, {'id': report_name, 'options': options})
+ t.get_action('report_refresh')(
+ {}, {'id': report_name, 'options': options})
except t.NotAuthorized:
- t.abort(401)
+ t.abort(401)
# Don't want the refresh=1 in the url once it is done
t.redirect_to(helpers.relative_url_for(refresh=None))
# Check for any options not allowed by the report
for key in options:
- if key not in report['option_defaults']:
+ if key not in report['option_defaults'] and \
+ key not in ['page', 'limit']:
t.abort(400, 'Option not allowed by report: %s' % key)
try:
- data, report_date = t.get_action('report_data_get')({}, {'id': report_name, 'options': options})
+ data, report_date = t.get_action('report_data_get')(
+ {},
+ {'id': report_name, 'options': options}
+ )
except t.ObjectNotFound:
t.abort(404)
except t.NotAuthorized:
@@ -107,15 +118,20 @@ def view(self, report_name, organization=None, refresh=False):
if format and format != 'html':
ensure_data_is_dicts(data)
- anonymise_user_names(data, organization=options.get('organization'))
+ anonymise_user_names(
+ data, organization=options.get('organization'))
if format == 'csv':
try:
- key = t.get_action('report_key_get')({}, {'id': report_name, 'options': options})
+ key = t.get_action('report_key_get')(
+ {},
+ {'id': report_name, 'options': options}
+ )
except t.NotAuthorized:
t.abort(401)
filename = 'report_%s.csv' % key
t.response.headers['Content-Type'] = 'application/csv'
- t.response.headers['Content-Disposition'] = str('attachment; filename=%s' % (filename))
+ t.response.headers['Content-Disposition'] = \
+ str('attachment; filename=%s' % (filename))
return make_csv_from_dicts(data['table'])
elif format == 'json':
t.response.headers['Content-Type'] = 'application/json'
@@ -126,15 +142,46 @@ def view(self, report_name, organization=None, refresh=False):
are_some_results = bool(data['table'] if 'table' in data
else data)
+
+ # Pagination
+ paginate_by = report.get('paginate_by')
+ total_results = len(data['table'])
+ if paginate_by:
+ page = int(options.get('page', 1))
+ limit = int(options.get('limit', paginate_by))
+ min_val = (page * limit) - limit
+ max_val = (page * limit)
+ data['table'] = data['table'][min_val:max_val]
+ options['page'] = page
+ options['limit'] = limit
+ else:
+ page = None
+ limit = None
+
+ pagination = {
+ 'total': total_results,
+ 'page': page,
+ 'limit': limit
+ }
+
# A couple of context variables for legacy genshi reports
c.data = data
c.options = options
- return t.render('report/view.html', extra_vars={
- 'report': report, 'report_name': report_name, 'data': data,
- 'report_date': report_date, 'options': options,
- 'options_html': options_html,
- 'report_template': report['template'],
- 'are_some_results': are_some_results})
+
+ return t.render(
+ 'report/view.html',
+ extra_vars={
+ 'report': report,
+ 'report_name': report_name,
+ 'data': data,
+ 'report_date': report_date,
+ 'options': options,
+ 'options_html': options_html,
+ 'report_template': report['template'],
+ 'are_some_results': are_some_results,
+ 'pagination': pagination
+ }
+ )
def make_csv_from_dicts(rows):
diff --git a/ckanext/report/logic/action/get.py b/ckanext/report/logic/action/get.py
index dce0225..1f6fd49 100644
--- a/ckanext/report/logic/action/get.py
+++ b/ckanext/report/logic/action/get.py
@@ -65,12 +65,13 @@ def report_data_get(context=None, data_dict=None):
:returns: A list containing the data and the date on which it was created
:rtype: list
"""
+
logic.check_access('report_data_get', context, data_dict)
- id = logic.get_or_bust(data_dict, 'id')
+ report_id = logic.get_or_bust(data_dict, 'id')
options = data_dict.get('options', {})
- report = ReportRegistry.instance().get_report(id)
+ report = ReportRegistry.instance().get_report(report_id)
data, date = report.get_fresh_report(**options)
diff --git a/ckanext/report/model.py b/ckanext/report/model.py
index 5f094af..d5d9666 100644
--- a/ckanext/report/model.py
+++ b/ckanext/report/model.py
@@ -10,6 +10,7 @@
from ckan import model
from ckan.lib.helpers import OrderedDict
+
log = logging.getLogger(__name__)
__all__ = ['DataCache', 'data_cache_table', 'init_tables']
@@ -39,7 +40,7 @@ class DataCache(object):
This model makes no assumptions on what is stored, and so it is up to the
producer and consumer to agree on a format for the value stored. It is
suggested that for data that is not a basic type (int, string etc) that
- json is used as decoding of JSON will still be a lot faster than performing
+ JSON is used as decoding of JSON will still be a lot faster than performing
the initial queries.
Example usage:
@@ -71,35 +72,35 @@ def get(cls, object_id, key, convert_json=False, max_age=None):
Retrieves the value and date that it was written if the record with
object_id/key exists. If not it will return None/None.
"""
+
item = model.Session.query(cls)\
.filter(cls.key == key)\
.filter(cls.object_id == object_id)\
.first()
if not item:
- #log.debug('Does not exist in cache: %s/%s', object_id, key)
return (None, None)
if max_age:
age = datetime.datetime.now() - item.created
if age > max_age:
- log.debug('Cache not returned - it is older than requested %s/%s %r > %r',
+ log.debug('Cache not returned - it is older than requested '
+ '%s/%s %r > %r',
object_id, key, age, max_age)
return (None, None)
value = item.value
if convert_json:
# Use OrderedDict instead of dict, so that the order of the columns
- # in the data is preserved from the data when it was written (assuming
- # it was written as an OrderedDict in the report's code).
+ # in the data is preserved from the data when it was written
+ # (assuming it was written as an OrderedDict in the report's code).
try:
# Python 2.7's json library has object_pairs_hook
import json
value = json.loads(value, object_pairs_hook=OrderedDict)
- except TypeError: # Untested
+ except TypeError: # Untested
# Python 2.4-2.6
import simplejson as json
value = json.loads(value, object_pairs_hook=OrderedDict)
- #log.debug('Cache load: %s/%s "%s"...', object_id, key, repr(value)[:40])
return value, item.created
@classmethod
@@ -133,5 +134,6 @@ def set(cls, object_id, key, value, convert_json=False):
mapper(DataCache, data_cache_table)
+
def init_tables():
metadata.create_all(model.meta.engine)
diff --git a/ckanext/report/plugin.py b/ckanext/report/plugin.py
index 36d5315..793cdc6 100644
--- a/ckanext/report/plugin.py
+++ b/ckanext/report/plugin.py
@@ -6,6 +6,7 @@
import ckanext.report.logic.auth.get as auth_get
import ckanext.report.logic.auth.update as auth_update
+
class ReportPlugin(p.SingletonPlugin):
p.implements(p.IRoutes, inherit=True)
p.implements(p.IConfigurer)
@@ -72,4 +73,3 @@ class TaglessReportPlugin(p.SingletonPlugin):
def register_reports(self):
import reports
return [reports.tagless_report_info]
-
diff --git a/ckanext/report/report_registry.py b/ckanext/report/report_registry.py
index 2585857..12e9886 100644
--- a/ckanext/report/report_registry.py
+++ b/ckanext/report/report_registry.py
@@ -12,20 +12,23 @@
REPORT_KEYS_REQUIRED = set(('name', 'generate', 'template', 'option_defaults',
'option_combinations'))
-REPORT_KEYS_OPTIONAL = set(('title', 'description', 'authorize'))
+REPORT_KEYS_OPTIONAL = set(('title', 'description', 'authorize',
+ 'paginate_by'))
class Report(object):
'''Represents a report that can be generated. Instances are generated by
ReportRegistry.'''
+
def __init__(self, report_info_dict, plugin):
# Check the report_info_dict has the correct keys
- missing_required_keys = REPORT_KEYS_REQUIRED - set(report_info_dict.keys())
+ missing_required_keys = REPORT_KEYS_REQUIRED - \
+ set(report_info_dict.keys())
assert not missing_required_keys, 'Report info dict missing keys %r: '\
'%r' % (missing_required_keys, report_info_dict)
unknown_keys = set(report_info_dict.keys()) - REPORT_KEYS_REQUIRED - \
- REPORT_KEYS_OPTIONAL
+ REPORT_KEYS_OPTIONAL
assert not unknown_keys, 'Report info dict has unrecognized keys %r: '\
'%r' % (unknown_keys, report_info_dict)
if not report_info_dict['option_defaults']:
@@ -46,6 +49,9 @@ def __init__(self, report_info_dict, plugin):
elif key == 'description':
self.description = ''
+ # Save the paginate_by setting
+ self.paginate_by = report_info_dict.get('paginate_by')
+
def generate_key(self, option_dict, defaults_for_missing_keys=True):
'''Returns a key that will identify the report and options when saved
in the DataCache. It looks like URL parameters for convenience.'''
@@ -73,10 +79,14 @@ def generate_key(self, option_dict, defaults_for_missing_keys=True):
return '%s' % self.name
def refresh_cache_for_all_options(self):
- '''Generates the report for all the option combinations and caches them.'''
+ '''Generates the report for all the option combinations
+
+ and caches them.
+ '''
+
log.info('Report: %s %s', self.plugin, self.name)
option_combinations = list(self.option_combinations()) \
- if self.option_combinations else [{}]
+ if self.option_combinations else [{}]
for option_dict in option_combinations:
self.refresh_cache(option_dict)
log.info(' report done')
@@ -105,6 +115,7 @@ def get_fresh_report(self, **option_dict):
entity_name, key, convert_json=True)
if data is None:
data, date = self.refresh_cache(option_dict)
+
return data, date
def get_cached_date(self, **option_dict):
@@ -130,7 +141,7 @@ def add_defaults_to_options(options, defaults):
'''
defaulted_options = copy.deepcopy(defaults)
for key in defaulted_options:
- if not key in options:
+ if key not in options:
if defaulted_options[key] is True:
# Checkboxes don't submit a value when False, so cannot
# default to True. i.e. to get a True value, you always
@@ -154,7 +165,8 @@ def as_dict(self):
'title': self.title,
'description': self.description,
'option_defaults': self.option_defaults,
- 'template': self.get_template()}
+ 'template': self.get_template(),
+ 'paginate_by': self.paginate_by}
def is_visible_to_user(self, user):
if hasattr(self, 'authorize'):
@@ -166,7 +178,11 @@ def is_visible_to_user(self, user):
def extract_entity_name(option_dict):
'''Hunts for an option key that is the entity name and returns its
value. Used in the DataCache storage.'''
- for entity_name in ('organization', 'publisher', 'group', 'package', 'resource'):
+ for entity_name in ('organization',
+ 'publisher',
+ 'group',
+ 'package',
+ 'resource'):
if entity_name in option_dict:
return option_dict[entity_name]
return None
@@ -184,6 +200,7 @@ def instance(cls):
def __init__(self):
# register all the reports
+
import ckan.plugins as p
self._reports = {} # this reset is needed for 'paster serve --restart'
for plugin in p.PluginImplementations(IReport):
@@ -198,7 +215,9 @@ def __init__(self):
def get_names(self):
return [(r.plugin, r.name, r.title)
- for r in sorted(self._reports.values(), key=lambda r: r.plugin)]
+ for r in sorted(
+ self._reports.values(),
+ key=lambda r: r.plugin)]
def get_reports(self):
return sorted(self._reports.values(), key=lambda r: r.title)
@@ -207,11 +226,10 @@ def get_report(self, report_name):
return self._reports[report_name]
def refresh_cache_for_all_reports(self):
- '''Generates all the reports for all the option combinations and caches them.'''
- for report in self._reports.values():
- report.refresh_cache_for_all_options()
+ '''Generates all the reports for all the option combinations
+ and caches them.
+ '''
-# 'name': 'feedback-report',
-# 'option_combinations': nii_report_combinations,
-# 'generate': nii_report,
+ for report in self._reports.values():
+ report.refresh_cache_for_all_options()
diff --git a/ckanext/report/reports.py b/ckanext/report/reports.py
index 9b9a349..fb880b9 100644
--- a/ckanext/report/reports.py
+++ b/ckanext/report/reports.py
@@ -13,8 +13,8 @@ def tagless_report(organization, include_sub_organizations=False):
Returns something like this:
{
'table': [
- {'name': 'river-levels', 'title': 'River levels', 'notes': 'Harvested', 'user': 'bob', 'created': '2008-06-13T10:24:59.435631'},
- {'name': 'co2-monthly', 'title' 'CO2 monthly', 'notes': '', 'user': 'bob', 'created': '2009-12-14T08:42:45.473827'},
+ {'name': 'river-levels', 'title': 'River levels', 'notes': 'Harvested', 'user': 'bob', 'created': '2008-06-13T10:24:59.435631'}, # noqa
+ {'name': 'co2-monthly', 'title' 'CO2 monthly', 'notes': '', 'user': 'bob', 'created': '2009-12-14T08:42:45.473827'}, # noqa
],
'num_packages': 56,
'packages_without_tags_percent': 4,
@@ -46,14 +46,16 @@ def tagless_report(organization, include_sub_organizations=False):
average_tags_per_package = round(float(num_taggings) / num_packages, 1)
else:
average_tags_per_package = None
- packages_without_tags_percent = lib.percent(len(tagless_pkgs), num_packages)
+ packages_without_tags_percent = lib.percent(
+ len(tagless_pkgs), num_packages)
return {
'table': tagless_pkgs,
'num_packages': num_packages,
'packages_without_tags_percent': packages_without_tags_percent,
'average_tags_per_package': average_tags_per_package,
- }
+ }
+
def tagless_report_option_combinations():
for organization in lib.all_organizations(include_none=True):
@@ -70,4 +72,5 @@ def tagless_report_option_combinations():
'option_combinations': tagless_report_option_combinations,
'generate': tagless_report,
'template': 'report/tagless-datasets.html',
+ 'paginate_by': 20
}
diff --git a/ckanext/report/templates/report/snippets/pagination.html b/ckanext/report/templates/report/snippets/pagination.html
new file mode 100644
index 0000000..48cde7a
--- /dev/null
+++ b/ckanext/report/templates/report/snippets/pagination.html
@@ -0,0 +1,23 @@
+
+{% set total = pagination.get('total') %}
+{% set page = pagination.get('page') %}
+{% set limit = pagination.get('limit') %}
+
+{% if total|int > 0 and limit > 0 %}
+ {% set total = (total|int / limit|int)|round(0, 'ceil')|int %}
+ {% set current = page|int %}
+
+
+
+{% endif %}
diff --git a/ckanext/report/templates/report/view.html b/ckanext/report/templates/report/view.html
index 48c7373..8c0e241 100644
--- a/ckanext/report/templates/report/view.html
+++ b/ckanext/report/templates/report/view.html
@@ -52,7 +52,8 @@ {{ _('Results') }}
{{ _('No results found.') }}
{% else %}
- {% snippet report_template, table=data['table'], data=data, report_name=report_name, options=options %}
+ {% snippet report_template, table=data['table'], data=data, report_name=report_name, options=options, pagination=pagination %}
+ {% snippet 'report/snippets/pagination.html', pagination=pagination %}
{% endif %}
@@ -77,4 +78,3 @@ {{ _('Results') }}
// ]]>
{% endblock%}
-