Merge branch 'master' into 232_implement_ftp_retry

README.md

@@ -67,7 +67,7 @@ Create the stack and start it:
 -   **notify:** Sends an e-mail for every new tender or winner(default) or an e-mail with all new tenders and winners(digest parameter specified);
 -   **notify_favourites:** Sends e-mails for favourites tenders, according to digest parameter described previously
 -   **notify_keywords:** Sends e-mails for tenders with keywords, according to digest parameter described previously
--   **remove_unnecessary_newlines:** Removes newlines from Winner vendor field
+-   **remove_unnecessary_newlines:** Removes newlines Vendors' names
 -   **update_ted:** Add new TED tenders, according to days_ago parameter or beginning with latest published date
     from database, if not specified
 -   **update_ungm:** Add new UNGM tenders, according to days_ago parameter or beginning with latest published date from

app/admin.py

@@ -27,21 +27,16 @@ def get_tender_title(self, obj):
 
 class WinnerAdmin(admin.ModelAdmin):
     list_display = [
-        'value', 'currency', 'award_date', 'notified', 'get_vendor_name',
+        'value', 'currency', 'award_date', 'notified', 'get_vendors',
         'get_tender_title', 'get_tender_organization', 'get_tender_source', 'get_tender_deadline',
     ]
-    search_fields = ['vendor__name', 'tender__title', 'award_date']
+
+    search_fields = ['vendors__name', 'tender__title', 'award_date']
     list_filter = (
-        'vendor__name', 'tender__title', 'tender__deadline', 'tender__source',
+        'vendors','tender__title', 'tender__deadline', 'tender__source',
         'tender__organization', 'currency', 'award_date',
     )
 
-    def get_vendor_name(self, obj):
-        return obj.vendor.name
-
-    get_vendor_name.short_description = 'Vendor Name'
-    get_vendor_name.admin_order_field = 'vendor__name'
-
     def get_tender_title(self, obj):
         return obj.tender.title
 

app/documents.py

@@ -73,8 +73,8 @@ class Meta:
 @winner.doc_type
 class WinnerDoc(DocType):
     tender_title = fields.KeywordField(attr='tender.title')
-    vendor_name = fields.KeywordField(attr='vendor.name')
-    value = fields.TextField(attr='convert_value_to_string')
+    vendors_name = fields.KeywordField(attr='get_vendors')
+    value = fields.TextField(attr="convert_value_to_string")
 
     class Meta:
         model = Winner

app/forms.py

@@ -89,7 +89,7 @@ def __init__(self, *args, **kwargs):
             (org, org) for org in organizations_list
         ]
         vendors_list = Winner.objects.values_list(
-            "vendor__name", flat=True
+            "vendors__name", flat=True
         ).distinct()
         self.fields["vendor"].choices = [("", "All vendors")] + [
             (vendor, vendor) for vendor in vendors_list

app/management/commands/add_winner.py

@@ -6,25 +6,25 @@
 import datetime
 from time import sleep
 from random import randint
-from app.models import Tender, Winner
+from app.models import Tender, Winner, Vendor
 from app.server_requests import PAYLOAD
 import logging
 from app.management.commands.base.params import BaseParamsUI
 
 logger = logging.getLogger(__name__)
-WINNERS_ENDPOINT_URI = "https://www.ungm.org/Public/ContractAward"
+WINNERS_ENDPOINT_URI = 'https://www.ungm.org/Public/ContractAward'
 
-CSS_TITLE = "Title"
-CSS_REFERENCE = "Reference"
-CSS_AWARD_DATE = "AwardDate"
-CSS_DESCRIPTION = "raw clear"
-CSS_ORGANIZATION = "AgencyId"
-CSS_VALUE = "ContractValue"
-CSS_VENDOR_LIST = "contractAwardVendorsContainer"
+CSS_TITLE = 'Title'
+CSS_REFERENCE = 'Reference'
+CSS_AWARD_DATE = 'AwardDate'
+CSS_DESCRIPTION = 'raw clear'
+CSS_ORGANIZATION = 'AgencyId'
+CSS_VALUE = 'ContractValue'
+CSS_VENDOR_LIST = 'contractAwardVendorsContainer'
 
 
 class Command(BaseCommand, BaseParamsUI):
-    help = "Gets all awards from the past day"
+    help = 'Gets all awards from the past day'
 
     def handle(self, *args, **options):
         expired_tenders = Tender.objects.filter(
@@ -41,7 +41,7 @@ def handle(self, *args, **options):
                 try:
                     contract_id = self.get_contract_id(tender.reference)
                 except TypeError:
-                    logger.warning(f"No winner was found for the corresponding tender reference ({ tender.reference })")
+                    logger.warning(f'No winner was found for the corresponding tender reference ({ tender.reference })')
                     continue
 
                 request_cls = get_request_class(public=True)
@@ -52,76 +52,80 @@ def handle(self, *args, **options):
                     winner_fields = self.parse_winner(html_data)
                     self.save_winner(tender.reference, winner_fields)
                 except TypeError:
-                    logger.error("Contract does not exist!")
+                    logger.error('Contract does not exist!')
 
     @staticmethod
     def find_by_label(soup, label):
         try:
-            return soup.find("label", attrs={"for": label}).next_sibling.next_sibling.string
+            return soup.find('label', attrs={'for': label}).next_sibling.next_sibling.string
         except AttributeError:
             return ''
 
     @staticmethod
     def get_contract_id(reference):
         if len(reference) < 3:
-            logger.error("The search text must be at least 3 characters long.")
+            logger.error('The search text must be at least 3 characters long.')
             return
 
         requester = get_request_class(public=True)
 
-        payload = PAYLOAD["winners"]
-        payload["Reference"] = reference
-
+        payload = PAYLOAD['winners']
+        payload['Reference'] = reference
         for i in range(0, 3):
             resp = requester.post_request(
                 WINNERS_ENDPOINT_URI,
-                WINNERS_ENDPOINT_URI + "/Search",
+                WINNERS_ENDPOINT_URI + '/Search',
                 json.dumps(payload),
             )
             if resp:
-                soup = BeautifulSoup(resp, "html.parser")
-                contract_id = soup.find("div", {"class": "tableRow dataRow"})["data-contractawardid"]
+                soup = BeautifulSoup(resp, 'html.parser')
+                contract_id = soup.find('div', {'class': 'tableRow dataRow'})['data-contractawardid']
                 return contract_id
             sleep(randint(10, 15))
 
-        logger.error("POST request failed.")
+        logger.error('POST request failed.')
         return None
 
     def parse_winner(self, html):
         """ Parse a contract award HTML and return a dictionary with information
          such as: title, reference, vendor etc
         """
 
-        soup = BeautifulSoup(html, "html.parser")
+        soup = BeautifulSoup(html, 'html.parser')
         vendor_list = []
         for vendors_div in soup.find_all(id=CSS_VENDOR_LIST):
             vendors = vendors_div.descendants
             for vendor in vendors:
-                if vendor.name == "div" and vendor.get("class", "") == [
-                    "editableListItem"
+                if vendor.name == 'div' and vendor.get('class', '') == [
+                    'editableListItem'
                 ]:
                     vendor_list.append(vendor.text.strip())
-        vendor_list = ", ".join(vendor_list)
+        vendor_list = vendor_list
         award_date = self.find_by_label(soup, CSS_AWARD_DATE)
         value = self.find_by_label(soup, CSS_VALUE)
         winner_fields = {
-            "award_date": self.string_to_date(award_date) or datetime.date.today(),
-            "vendor": vendor_list,
-            "value": float(value or 0) if value else '',
-            "currency": '',
+            'award_date': self.string_to_date(award_date) or datetime.date.today(),
+            'vendors': vendor_list,
+            'value': float(value or 0) if value else '',
+            'currency': '',
         }
 
-        if winner_fields["value"]:
-            winner_fields["currency"] = "USD"
+        if winner_fields['value']:
+            winner_fields['currency'] = 'USD'
 
         return winner_fields
 
     @staticmethod
     def save_winner(reference, winner_fields):
         tender_entry = Tender.objects.filter(reference=reference).first()
-
-        winner = Winner(tender=tender_entry, **winner_fields)
+        vendors = winner_fields.pop('vendors')
+        vendor_objects = []
+        for vendor in vendors:
+            vendor_object, _ = Vendor.objects.get_or_create(name=vendor)
+            vendor_objects.append(vendor_object)
+        winner = Winner.objects.update_or_create(tender=tender_entry, **winner_fields)
         winner.save()
+        winner.vendors.add(*vendor_objects)
 
         return winner
 
@@ -130,10 +134,10 @@ def to_unicode(string):
         if not string or isinstance(string, str):
             return string
         else:
-            return str(string, "utf8")
+            return str(string, 'utf8')
 
     @staticmethod
     def string_to_date(string_date):
         if string_date:
-            return datetime.datetime.strptime(string_date.strip(), "%d-%b-%Y")
+            return datetime.datetime.strptime(string_date.strip(), '%d-%b-%Y')
         return None

app/management/commands/deadline_notifications.py

@@ -12,7 +12,7 @@ class Command(BaseCommand, BaseParamsUI):
     help = 'Send deadline notifications to all users if there are tenders with 1, 3 or 7 days left'
 
     def handle(self, *args, **options):
-        tenders = Tender.objects.filter(favourite=True, winner=None).order_by('deadline')
+        tenders = Tender.objects.filter(favourite=True, winners=None).order_by('deadline')
 
         days_list = sorted(settings.DEADLINE_NOTIFICATIONS)
 

app/management/commands/remove_unnecessary_newlines.py

@@ -1,17 +1,17 @@
 from django.core.management.base import BaseCommand
 from app.management.commands.base.params import BaseParamsUI
-from app.models import Winner
+from app.models import Vendor
 
 
 class Command(BaseCommand, BaseParamsUI):
     help = 'Removes unnecessary newlines from award vendors name'
 
     def handle(self, *args, **options):
-        winners = Winner.objects.all()
+        vendors = Vendor.objects.all()
 
-        for winner in winners:
-            winner.vendor = winner.vendor.strip()
-            winner.save()
+        for vendor in vendors:
+            vendor.name = vendor.name.strip()
+            vendor.save()
 
         self.stdout.write(self.style.SUCCESS('Removed unnecessary new lines'))
         return 'Removed unnecessary new lines'

app/migrations/0025_change_vendor_winner_relation.py

@@ -0,0 +1,31 @@
+# Generated by Django 2.2.4 on 2019-10-07 15:07
+
+from django.db import migrations, models
+
+def move_vendors_link(apps, schema_editor):
+    vendor_model = apps.get_model('app', 'Vendor')
+    winner_model = apps.get_model('app', 'Winner')
+    for winner in winner_model.objects.all():
+        vendor = winner.vendor
+        winner.vendors.add(vendor)
+        winner.save()
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('app', '0024_split_winner_vendor'),
+    ]
+
+    operations = [
+
+        migrations.RunPython(move_vendors_link),
+        migrations.RemoveField(
+            model_name='winner',
+            name='vendor',
+        ),
+        migrations.AddField(
+            model_name='winner',
+            name='vendors',
+            field=models.ManyToManyField(related_name='winners', to='app.Vendor'),
+        ),
+    ]

app/models.py

@@ -105,11 +105,16 @@ class Winner(models.Model):
     currency = models.CharField(null=True, max_length=3)
     award_date = models.DateField()
     notified = models.BooleanField(default=False)
-    vendor = models.ForeignKey(Vendor, on_delete=models.DO_NOTHING)
-    tender = models.ForeignKey(Tender, on_delete=models.CASCADE)
+    tender = models.ForeignKey(Tender, on_delete=models.CASCADE, related_name='winners')
+    vendors = models.ManyToManyField('Vendor', related_name='winners')
 
     def __str__(self):
-        return '{} WON BY {}'.format(self.tender.title, self.vendor.name)
+        return '{} WON BY {}'.format(self.tender.title, self.get_vendors)
+
+    @property
+    def get_vendors(self):
+        return  ",".join(self.vendors.values_list('name', flat=True))
+    get_vendors.fget.short_description = "Vendors names"
 
     def convert_value_to_string(self):
         return str(self.value)