From d821a627fd6e454aeb91f1cd2d6e714fa36b0f47 Mon Sep 17 00:00:00 2001 From: Matthew Boehm Date: Thu, 26 Jan 2017 14:59:56 -0500 Subject: [PATCH 1/4] Add script for updating timestamp on resources This will allow us to automatically update last modified timestamps from the pipeline --- .../scripts/update_resource.py | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 ckanext/cfpb_extrafields/scripts/update_resource.py diff --git a/ckanext/cfpb_extrafields/scripts/update_resource.py b/ckanext/cfpb_extrafields/scripts/update_resource.py new file mode 100644 index 0000000..d0e0c86 --- /dev/null +++ b/ckanext/cfpb_extrafields/scripts/update_resource.py @@ -0,0 +1,58 @@ +"""Update CKAN Resources + +usage: CKAN_API_KEY= CKAN_URL= python update_resource.py [] + + CKAN_API_KEY: the API key on the bottom left of your ckan user settings page + CKAN_URL: url to the data catalog, i.e. http://catalog.data.cfpb.local + resource_id: the unique ID of a ckan resource + timestamp: ISO timestamp to use as the new update time. Defaults to now. +""" +from __future__ import print_function +import datetime as dt +import json +import os +import sys + +import requests + +import logging +logging.basicConfig(level=logging.DEBUG) + +API_KEY = os.environ["CKAN_API_KEY"] +CKAN_URL = os.environ["CKAN_URL"] + +def do_action(action, data, ckan_url=CKAN_URL, api_key=API_KEY): + response = requests.post( + ckan_url + "/api/3/action/" + action, + json=data, + headers={"Authorization": api_key}, + ) + response.raise_for_status() + return response + +def get_resource(resource_id, ckan_url=CKAN_URL, api_key=API_KEY): + response = do_action("resource_show", {"id": resource_id}, ckan_url, api_key) + return response.json()["result"] + +def update_resource(resource, ckan_url=CKAN_URL, api_key=API_KEY): + do_action("resource_update", resource, ckan_url, api_key) + +def main(args): + if len(args) < 2 or args[1] in ["help", "-h", "--help"]: + print(__doc__) + else: + resource_id = args[1] + + if len(args) > 2: + timestamp = args[2] + else: + timestamp = dt.datetime.now().isoformat() + + resource = get_resource(resource_id) + resource["last_modified"] = timestamp + update_resource(resource) + + +if __name__ == "__main__": + main(sys.argv) + From 752268c98a344de932055f71eea8b7d45618e013 Mon Sep 17 00:00:00 2001 From: Matthew Boehm Date: Thu, 26 Jan 2017 23:06:28 -0500 Subject: [PATCH 2/4] Add update_date field to resources --- ckanext/cfpb_extrafields/plugin.py | 2 ++ .../cfpb_extrafields/templates/package/resource_read.html | 6 ++++++ .../resource_type-specific/cfpb_extrafields_database.html | 4 ++++ .../resource_type-specific/cfpb_extrafields_datafile.html | 4 ++++ .../resource_type-specific/cfpb_extrafields_defaults.html | 2 ++ .../cfpb_extrafields_documentation.html | 4 ++++ 6 files changed, 22 insertions(+) diff --git a/ckanext/cfpb_extrafields/plugin.py b/ckanext/cfpb_extrafields/plugin.py index 16a761d..aeee0c9 100644 --- a/ckanext/cfpb_extrafields/plugin.py +++ b/ckanext/cfpb_extrafields/plugin.py @@ -280,6 +280,7 @@ def _modify_package_schema(self, schema): schema['resources'].update({ 'approximate_total_size' : [tk.get_validator('ignore_missing'),], 'intake_date' : [v.reasonable_date_validator, tk.get_validator('ignore_missing'),], + 'update_date' : [v.reasonable_date_validator, tk.get_validator('ignore_missing'),], 'resource_type' : [tk.get_validator('ignore_missing'),], 'storage_location' : [tk.get_validator('ignore_missing'),], 'storage_location_path' : [tk.get_validator('ignore_missing'),], @@ -395,6 +396,7 @@ def show_package_schema(self): schema['resources'].update({ 'approximate_total_size' : [ tk.get_validator('ignore_missing'),], 'intake_date' : [tk.get_validator('ignore_missing'),], + 'update_date' : [tk.get_validator('ignore_missing'),], 'resource_type' : [ tk.get_validator('ignore_missing'),], 'storage_location' : [ tk.get_validator('ignore_missing'),], 'storage_location_path' : [ tk.get_validator('ignore_missing'),], diff --git a/ckanext/cfpb_extrafields/templates/package/resource_read.html b/ckanext/cfpb_extrafields/templates/package/resource_read.html index d036df7..f1fd1a9 100644 --- a/ckanext/cfpb_extrafields/templates/package/resource_read.html +++ b/ckanext/cfpb_extrafields/templates/package/resource_read.html @@ -145,6 +145,12 @@

{{ h.resource_display_name(res) | truncate(50) }}

{{ res.intake_date }} {% endif %} + {% if res.update_date %} + + {{ _("update date") }} + {{ res.update_date }} + + {% endif %} {{ _('metadata resource ID') }} {{ res.id or _('unknown') }} diff --git a/ckanext/cfpb_extrafields/templates/package/snippets/resource_type-specific/cfpb_extrafields_database.html b/ckanext/cfpb_extrafields/templates/package/snippets/resource_type-specific/cfpb_extrafields_database.html index ef6db4d..f5cea63 100644 --- a/ckanext/cfpb_extrafields/templates/package/snippets/resource_type-specific/cfpb_extrafields_database.html +++ b/ckanext/cfpb_extrafields/templates/package/snippets/resource_type-specific/cfpb_extrafields_database.html @@ -7,6 +7,7 @@ X description X roles and descriptions (+ reveals more) X + intake_date +X + update_date X + hidden (required) url #} @@ -44,6 +45,9 @@ {{ form.input('intake_date', type='date', label=_("intake date"), placeholder="eg 2012-12-21",value=data.intake_date, error=errors.intake_date) }} + {{ form.input('update_date', type='date', label=_("update date"), + placeholder="eg 2012-12-21",value=data.update_date, error=errors.update_date) }} +

Roles and access descriptions:

diff --git a/ckanext/cfpb_extrafields/templates/package/snippets/resource_type-specific/cfpb_extrafields_datafile.html b/ckanext/cfpb_extrafields/templates/package/snippets/resource_type-specific/cfpb_extrafields_datafile.html index a7369e2..c117f43 100644 --- a/ckanext/cfpb_extrafields/templates/package/snippets/resource_type-specific/cfpb_extrafields_datafile.html +++ b/ckanext/cfpb_extrafields/templates/package/snippets/resource_type-specific/cfpb_extrafields_datafile.html @@ -8,6 +8,7 @@ X +storage location ; path (drop-down+free-text) X +approximate_size X +intake_date +X +update_date #} {% if location=="form" %} @@ -44,6 +45,9 @@ {{ form.input('intake_date', type='date', label=_("intake date"), placeholder="eg 2012-12-21",value=data.intake_date, error=errors.intake_date) }} + + {{ form.input('update_date', type='date', label=_("update date"), + placeholder="eg 2012-12-21",value=data.update_date, error=errors.update_date) }} {% elif location=="description" %} diff --git a/ckanext/cfpb_extrafields/templates/package/snippets/resource_type-specific/cfpb_extrafields_defaults.html b/ckanext/cfpb_extrafields/templates/package/snippets/resource_type-specific/cfpb_extrafields_defaults.html index b0de27e..484a1ef 100644 --- a/ckanext/cfpb_extrafields/templates/package/snippets/resource_type-specific/cfpb_extrafields_defaults.html +++ b/ckanext/cfpb_extrafields/templates/package/snippets/resource_type-specific/cfpb_extrafields_defaults.html @@ -42,4 +42,6 @@ {{ form.input('intake_date', type='date', label=_("intake date"), placeholder="eg 2012-12-21", value=data.intake_date, error=errors.intake_date) }} +{{ form.input('update_date', type='date', label=_("update date"), placeholder="eg 2012-12-21", value=data.update_date, error=errors.update_date) }} + {{ form.select('approximate_total_size', label=_('approximate total size'), id='field-approximate_total_size', options=h.options_approximate_total_size(), selected=data.approximate_total_size, error=errors.approximate_total_size, classes=['control-medium']) }} diff --git a/ckanext/cfpb_extrafields/templates/package/snippets/resource_type-specific/cfpb_extrafields_documentation.html b/ckanext/cfpb_extrafields/templates/package/snippets/resource_type-specific/cfpb_extrafields_documentation.html index 38e39ff..9e858e0 100644 --- a/ckanext/cfpb_extrafields/templates/package/snippets/resource_type-specific/cfpb_extrafields_documentation.html +++ b/ckanext/cfpb_extrafields/templates/package/snippets/resource_type-specific/cfpb_extrafields_documentation.html @@ -7,6 +7,7 @@ X +format X +storage location ; path (drop-down+free-text) X +intake_date +X +update_date #} {% if location=="form" %} @@ -43,6 +44,9 @@ {{ form.input('intake_date', type='date', label=_("intake date"), placeholder="eg 2012-12-21",value=data.intake_date, error=errors.intake_date) }} + + {{ form.input('update_date', type='date', label=_("update date"), + placeholder="eg 2012-12-21",value=data.update_date, error=errors.update_date) }} {% elif location=="description" %} From 647518182c39e338021707b0c2a58c9777a93029 Mon Sep 17 00:00:00 2001 From: Matthew Boehm Date: Thu, 26 Jan 2017 23:11:50 -0500 Subject: [PATCH 3/4] Change date field for update_resaurce and make configurable --- ckanext/cfpb_extrafields/scripts/update_resource.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ckanext/cfpb_extrafields/scripts/update_resource.py b/ckanext/cfpb_extrafields/scripts/update_resource.py index d0e0c86..46e15d7 100644 --- a/ckanext/cfpb_extrafields/scripts/update_resource.py +++ b/ckanext/cfpb_extrafields/scripts/update_resource.py @@ -1,9 +1,10 @@ """Update CKAN Resources -usage: CKAN_API_KEY= CKAN_URL= python update_resource.py [] +usage: CKAN_API_KEY= CKAN_URL= [CKAN_DATE_FIELD=] python update_resource.py [] CKAN_API_KEY: the API key on the bottom left of your ckan user settings page CKAN_URL: url to the data catalog, i.e. http://catalog.data.cfpb.local + CKAN_DATE_FIELD: the field on the resource metadata to update. defaults to update_date resource_id: the unique ID of a ckan resource timestamp: ISO timestamp to use as the new update time. Defaults to now. """ @@ -20,6 +21,7 @@ API_KEY = os.environ["CKAN_API_KEY"] CKAN_URL = os.environ["CKAN_URL"] +DATE_FIELD = os.environ.get("CKAN_DATE_FIELD", "update_date") def do_action(action, data, ckan_url=CKAN_URL, api_key=API_KEY): response = requests.post( @@ -49,7 +51,7 @@ def main(args): timestamp = dt.datetime.now().isoformat() resource = get_resource(resource_id) - resource["last_modified"] = timestamp + resource[DATE_FIELD] = timestamp update_resource(resource) From d71a16a5f18c00f8de421117393c5bcf423b07f0 Mon Sep 17 00:00:00 2001 From: Matthew Boehm Date: Thu, 26 Jan 2017 23:33:40 -0500 Subject: [PATCH 4/4] Change date format for update_resource to YYYY-MM-DD Now that update_resource defaults to a custom field, the expected date format has changed --- ckanext/cfpb_extrafields/scripts/update_resource.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ckanext/cfpb_extrafields/scripts/update_resource.py b/ckanext/cfpb_extrafields/scripts/update_resource.py index 46e15d7..4591de6 100644 --- a/ckanext/cfpb_extrafields/scripts/update_resource.py +++ b/ckanext/cfpb_extrafields/scripts/update_resource.py @@ -6,7 +6,7 @@ CKAN_URL: url to the data catalog, i.e. http://catalog.data.cfpb.local CKAN_DATE_FIELD: the field on the resource metadata to update. defaults to update_date resource_id: the unique ID of a ckan resource - timestamp: ISO timestamp to use as the new update time. Defaults to now. + timestamp: YYYY-MM-DD timestamp to use as the new update time. Defaults to now. """ from __future__ import print_function import datetime as dt @@ -48,7 +48,7 @@ def main(args): if len(args) > 2: timestamp = args[2] else: - timestamp = dt.datetime.now().isoformat() + timestamp = dt.datetime.now().strftime("%Y-%m-%d") resource = get_resource(resource_id) resource[DATE_FIELD] = timestamp