Merge branch 'master' into django-1.5

Conflicts:
	requirements.txt

Author: stefanw

.gitignore

@@ -7,8 +7,7 @@ local_*.py
 data/
 solr/
 files/
-artwork/
-documents/
 .coverage
 */_build/
-froide/htmlcov/
+htmlcov/
+public/

.travis.yml

@@ -1,19 +1,29 @@
+---
 language: python
 python:
-  - "2.6"
-  - "2.7"
+- '2.7'
+- '2.6'
 services:
-  - elasticsearch
+- elasticsearch
 before_install:
-  - export PIP_USE_MIRRORS=true
-  - export PIP_INDEX_URL=https://simple.crate.io/
-  - "export DISPLAY=:99.0"
-  - "sh -e /etc/init.d/xvfb start"
+- export PIP_USE_MIRRORS=true
+- export PIP_INDEX_URL=https://simple.crate.io/
+- phantomjs --version
+- phantomjs --webdriver= &
 install:
-  - pip install -e .
-  - pip install -r requirements-test.txt
+- pip install https://github.com/stefanw/terrarium/archive/develop.zip boto
+- terrarium key requirements-test.txt
+- terrarium --virtualenv-log-level 20 --pip-log-level 20 --target froideenv --remote-key-format "u/stwe/froideenv/%(arch)s-%(python_vmajor)s.%(python_vminor)s-%(digest)s" install requirements-test.txt || (wget "http://resources.opendatalabs.org.s3.amazonaws.com/u/stwe/froideenv/`terrarium key requirements-test.txt`" ; terrarium --pip-log-level 20 --virtualenv-log-level 20 --no-upload --s3-bucket "" --target froideenv --storage-dir . install requirements-test.txt)
+- source froideenv/bin/activate
+- which python
+- pip install -e .
 script:
-  - make test
+- make test
 branches:
   only:
-    - master
+  - master
+env:
+  global:
+  - S3_BUCKET=resources.opendatalabs.org
+  - S3_ACCESS_KEY=AKIAIY6JZKLHFPB3KTQA
+  - secure: "B0JCveS4EwWZybclbEpFN1e/Am6uTTBU9qFJTFs0fyj8LBmdUl14YcWK+24O\nBiX9C3QUsfzHP5SoMEo3GgYZzLMIKRX8wr/UdspyW6sBh8VyPqfaanWys9xn\nGETJBJA30cYpUEwfEsNko/xz7TpW3dfP6O2hweSFuPo/Uzjt1tg="

Makefile

@@ -2,5 +2,5 @@ export DJANGO_SETTINGS_MODULE=froide.test_settings
 
 test:
 	pep8 --ignore=E501,E124,E126,E127,E128 --exclude=migrations froide
-	coverage run --branch --source=froide `which django-admin.py` test froide.account.tests froide.foirequest.tests froide.foirequestfollower.tests froide.frontpage.tests froide.publicbody.tests
+	coverage run --branch --source=froide `which django-admin.py` test froide
 	coverage report --omit="*/migrations/*"

README.mkd renamed to README.md

@@ -1,7 +1,7 @@
 Froide
 ======
 
-[![Build Status](https://secure.travis-ci.org/stefanw/froide.png)](http://travis-ci.org/#!/stefanw/froide)
+[![Build Status](https://travis-ci.org/stefanw/froide.png?branch=master)](https://travis-ci.org/stefanw/froide)
 
 
 Froide is a Freedom Of Information Portal written in Python using Django 1.4.3.

docs/api.rst

@@ -0,0 +1,12 @@
+==========
+Froide API
+==========
+
+Froide has a RESTful API that allows structured access to the content
+inside your Froide instance.
+
+The Froide API is available at `/api/v1/` and the interactive Froide API documentation is available at `/api/v1/docs/`.
+
+There are additional search endpoints for Public Bodies and FOI Requests at `/api/v1/publicbody/search/` and `/api/v1/request/search/` respectively. Use `q` as the query parameter in a GET request.
+
+GET requests do not need to be authenticated. POST, PUT and DELETE requests have to either carry a valid session cookie and a CSRF token or provide username (you find your username on your profile) and password via Basic Authentication.

docs/gettingstarted.rst

@@ -28,23 +28,26 @@ Install the requirements inside the virtual env with `pip`::
 
 If only your global `pip` is available, run `easy_install pip`. The dependency installation takes a couple of seconds, but after that everything is in place.
 
-Copy `local_settings.py.example` to `local_settings.py`::
+Copy `custom_settings.py.example` to `custom_settings.py`::
 
     cd froide
-    cp local_settings.py.example local_settings.py
+    cp custom_settings.py.example custom_settings.py
+
+The development environment uses SQLite. You can override the default Django settings values for froide in your `custom_settings.py`, if you want, but you don't have to.
+
+Your `custom_settings.py` file (you can name it differently) is your Django settings file that you must give on each Django manage.py command. You can also set the `DJANGO_SETTINGS_MODULE` environment variable.
 
-The development environment uses SQLite. You can change that in `local_settings.py`, if you want, but you don't have to.
 Sync and migrate and *do NOT* create a superuser just yet::
 
-    python manage.py syncdb --noinput --migrate
+    python manage.py syncdb --noinput --migrate --settings=froide.custom_settings
 
 Now you can create a superuser account::
 
-    python manage.py createsuperuser
+    python manage.py createsuperuser --settings=froide.custom_settings
 
 That's it for a setup that basically works. Run this::
 
-    python manage.py runserver
+    python manage.py runserver --settings=froide.custom_settings
 
 and go to `http://localhost:8000 <http://localhost:8000>`_. You should
 be greeted by a working Froide installation. It doesn't have any data
@@ -55,13 +58,13 @@ For more information on the different models you find in the admin visit :doc:`m
 Run tests
 ---------
 
-Froide has a test suite. Copy `test_settings.py.example` to `test_settings.py`. `test_settings.py` does not import your `local_settings.py` changes.
+Froide has a test suite. Copy `test_settings.py.example` to `test_settings.py`. `test_settings.py` does not use your `custom_settings.py` changes.
 
-You can then run the shell script for tests::
+You can then run for tests::
 
-    sh runtests.sh
+    make test
 
-This also does timing and a test coverage analysis that you can then
+This also does test coverage analysis that you can then
 find at `htmlcov/index.html`.
 Note some tests will not work without a search engine like solr running.
 

docs/importpublicbodies.rst

@@ -0,0 +1,59 @@
+=======================
+Importing Public Bodies
+=======================
+
+While it is possible to create public bodies individually through the admin
+interface, it might be more advisable to scrape a list from a website or
+crowdsource them with other people in a Google doc. You can then import these
+public bodies as a CSV file into Froide.
+
+The format of the CSV file should be like `in this Google Docs Spreadsheet <https://docs.google.com/spreadsheet/ccc?key=0AhDkodM9ozpddGNTaGJoa203aEJaRXVfM0Q0d1RjNUE#gid=0>`_. You could for example copy it and either fill in public bodies collaboratively or programmatically.
+
+Format
+------
+
+The format is quite simple.
+
+name
+  (required) The name of the public body.
+email
+  (optional) If you give no email, users will not be able to make requests to this public body. You can fill in email addresses later through the admin.
+jurisdiction__slug
+  (required) Give the slug of the jurisdiction this public body belongs to.
+other_names
+  (optional) Possible other, alternative names for the public body separated by commas.
+description
+  (optional) A text description of the public body.
+topic__slug
+  (optional) Slug for this public body's topic. If not given, the public body topic with the highest rank attribute will be chosen.
+url
+  (optional) Website for this public body.
+parent__name
+  (optional) if this public body has a parent, give it's name here. The parent must be specified before in the CSV file.
+classification
+  (required) Give a classification (e.g. "ministry"). If you don't want have the information, just fill in something general like "public body".
+contact
+  (optional) Contact information apart from post address and email. E.g. phone or fax number. May contain line breaks.
+address
+  (optional) Postal address of this public body.
+website_dump
+  (optional) Any further text that can be used to described this public body. This is used for search indexing and will not be displayed publicly.
+request_note
+  (optional) Display this text for this public body when making a request to it.
+
+If during import a public body with the same slug is found, it is skipped and not overwritten.
+
+Importing through Admin
+-----------------------
+
+The admin interface that lists public bodies has an import form at the very bottom of the page. Give a HTTP or HTTPS url of your CSV file and press the import button. The file will be downloaded and imported. Any errors will be shown to you.
+
+
+Importing via command line
+--------------------------
+
+The management command `import_csv` takes a URL or a path to a CSV file::
+
+    python manage.py import_csv public_bodies.csv --settings=froide.custom_settings
+
+

docs/index.rst

@@ -17,8 +17,10 @@ Freedom of Information Portal.
 
    about
    gettingstarted
+   importpublicbodies
    models
    configuration
+   api
    development
    productionsetup
 

froide/account/models.py

@@ -1,7 +1,7 @@
 import hmac
 import re
 
-from django.db import models
+from django.db import models, transaction, IntegrityError
 from django.conf import settings
 from django import dispatch
 from django.utils.translation import ugettext_lazy as _
@@ -169,25 +169,40 @@ def create_user(cls, **data):
                 last_name=data['last_name'],
                 email=data['user_email'])
         username_base = cls.get_username_base(user.first_name, user.last_name)
-        count = 0
-        while True:
-            if count:
-                username = "%s_%s" % (username_base, count)
-            else:
-                username = username_base
-            try:
-                User.objects.get(username=username)
-            except User.DoesNotExist:
-                break
-            count += 1
-        user.username = username
+
         user.is_active = False
         if "password" in data:
             password = data['password']
         else:
             password = User.objects.make_random_password()
         user.set_password(password)
-        user.save()
+
+        # ensure username is unique
+        while True:
+            username = username_base
+            first_round = True
+            count = 0
+            postfix = ""
+            with transaction.commit_manually():
+                try:
+                    while True:
+                        if not first_round:
+                            postfix = "_%d" % count
+                        if not User.objects.filter(username=username + postfix).exists():
+                            break
+                        if first_round:
+                            first_round = False
+                            count = User.objects.filter(username__startswith=username).count()
+                        else:
+                            count += 1
+                    user.username = username + postfix
+                    user.save()
+                except IntegrityError:
+                    transaction.rollback()
+                else:
+                    transaction.commit()
+                    break
+
         profile = user.get_profile()
         for key in data:
             if key in ('first_name', 'last_name', 'user_email', 'password'):

froide/local_settings.py.example renamed to froide/custom_settings.py.example

@@ -1,6 +1,12 @@
 # -*- coding: utf-8 -*-
 import re
 
+from .settings import *  # noqa
+
+# Add your theme to the top of installed apps
+# INSTALLED_APPS = ['your_theme'] + INSTALLED_APPS
+
+
 DEBUG = True
 TEMPLATE_DEBUG = True
 
@@ -39,6 +45,7 @@ CACHES = {
 }
 
 SITE_NAME = "Your Site Name"
+SITE_EMAIL = "[email protected]"
 SITE_URL = 'http://localhost:8000'
 
 rec = re.compile
@@ -52,6 +59,11 @@ SECRET_URLS = {
     "databrows": "databrowse"
 }
 
+# Possibly set this for https deployments
+# see https://docs.djangoproject.com/en/1.4/ref/settings/#secure-proxy-ssl-header
+# SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTOCOL', 'https')
+
+
 FROIDE_CONFIG = {
     "create_new_publicbody": True,
     "publicbody_empty": True,

froide/foirequest/admin.py

@@ -9,7 +9,8 @@
 from taggit.utils import parse_tags
 
 from froide.foirequest.models import (FoiRequest, FoiMessage,
-        FoiAttachment, FoiEvent, PublicBodySuggestion)
+        FoiAttachment, FoiEvent, PublicBodySuggestion,
+        DeferredMessage)
 from froide.foirequest.tasks import count_same_foirequests
 
 
@@ -173,8 +174,59 @@ class PublicBodySuggestionAdmin(admin.ModelAdmin):
     raw_id_fields = ('request', 'public_body', 'user')
 
 
+class DeferredMessageAdmin(admin.ModelAdmin):
+    model = DeferredMessage
+
+    list_display = ('recipient', 'timestamp',)
+    raw_id_fields = ('request',)
+    actions = ['redeliver']
+
+    def redeliver(self, request, queryset):
+        """
+        Redeliver undelivered mails
+
+        """
+        opts = self.model._meta
+        # Check that the user has change permission for the actual model
+        if not self.has_change_permission(request):
+            raise PermissionDenied
+
+        # User has already chosen the other req
+        if request.POST.get('req_id'):
+            req_id = int(request.POST.get('req_id'))
+            try:
+                req = FoiRequest.objects.get(id=req_id)
+            except (ValueError, FoiRequest.DoesNotExist,):
+                raise PermissionDenied
+
+            for deferred in queryset:
+                deferred.redeliver(req)
+
+            self.message_user(request, _("Successfully triggered redelivery."))
+
+            return None
+
+        db = router.db_for_write(self.model)
+        context = {
+            'opts': opts,
+            'queryset': queryset,
+            'media': self.media,
+            'action_checkbox_name': helpers.ACTION_CHECKBOX_NAME,
+            'req_widget': mark_safe(admin.widgets.ForeignKeyRawIdWidget(
+                    self.model._meta.get_field(
+                        'request').rel, self.admin_site, using=db).render(
+                            'req_id', None).replace('../../..', '../..')),
+            'applabel': opts.app_label
+        }
+
+        # Display the confirmation page
+        return TemplateResponse(request, 'foirequest/admin_redeliver.html',
+            context, current_app=self.admin_site.name)
+    redeliver.short_description = _("Redeliver to...")
+
 admin.site.register(FoiRequest, FoiRequestAdmin)
 admin.site.register(FoiMessage, FoiMessageAdmin)
 admin.site.register(FoiAttachment, FoiAttachmentAdmin)
 admin.site.register(FoiEvent, FoiEventAdmin)
 admin.site.register(PublicBodySuggestion, PublicBodySuggestionAdmin)
+admin.site.register(DeferredMessage, DeferredMessageAdmin)