Merge pull request #763 from alonisser/master

Mostly vote issues + syncdata cleanup

deploy/crontab.txt

@@ -13,8 +13,10 @@
 30 04 * * * /oknesset_data/oknesset/Open-Knesset/manage.py okscrape lobbyists --dblog 2>&1 | /usr/bin/logger -t open_knesset
 #20 05 * * 1,3,5 /oknesset_data/oknesset/Open-Knesset/manage.py update_sitemap 2>&1 | /usr/bin/logger -t open_knesset
 26 04 * * * /oknesset_data/oknesset/Open-Knesset/manage.py scrape_votes 2>&1 | /usr/bin/logger -t open_knesset
+30 16 * * * /oknesset_data/oknesset/Open-Knesset/manage.py rescrape_missing_data_votes 2>&1 | /usr/bin/logger -t open_knesset
 43 04 * * * /oknesset_data/oknesset/Open-Knesset/manage.py update_links_from_kikar 2>&1 | /usr/bin/logger -t open_knesset
 53 03 * * * /oknesset_data/oknesset/Open-Knesset/manage.py okscrape events PersonsEventsScraper --dblog 2>&1 | /usr/bin/logger -t open_knesset
+
 # email handling
 02 01,05,09,13,17,21 * * * /oknesset_data/oknesset/Open-Knesset/manage.py send_mail 2>&1 | /usr/bin/logger -t open_knesset
 15 17,21 * * * /oknesset_data/oknesset/Open-Knesset/manage.py retry_deferred 2>&1 | /usr/bin/logger -t open_knesset

knesset/browser_cases.py

@@ -6,13 +6,17 @@
 # also, they must use the @on_platforms decorator. This decorator can run the test case several times - for different browser and platforms.
 
 @on_platforms()
-class MyTestCase(BrowserTestCase):
+class MainSIteBrowserTestCase(BrowserTestCase):
     """
     Simple demo test case - just makes sure the tidbit carousel appears on the homepage
     """
 
     def testHomepage(self):
         # inside the tests you can use self.drive which will have a ready selenium driver to use
-        self.driver.get(self.live_server_url+'/')
+        self.driver.get(self.live_server_url+'/main') # Until we return old page
         # most functions throw an exception if they don't find what their looking for, so you don't have to assert
         self.driver.find_element_by_id('tidbitCarousel')
+
+    def testHelpPageDisplayFacebookUpdates(self):
+        self.driver.get(self.live_server_url + '/help')  # Until we return old page
+        self.driver.find_element_by_id('kikar-facebook-updates-ul')

knesset/browser_test_runner.py

@@ -13,9 +13,9 @@
 sauce_accesskey = ''
 
 sauce_platforms = [
-    {"platform": "Mac OS X 10.9", "browserName": "chrome", "version": "35"},
+    {"platform": "MacOS El Capitan 10.11", "browserName": "chrome", "version": "52"},
     {"platform": "Windows 8.1", "browserName": "internet explorer", "version": "11"},
-    {"platform": "Linux", "browserName": "firefox", "version": "29"}
+    {"platform": "Linux", "browserName": "firefox", "version": "44"}
 ]
 
 

laws/admin.py

@@ -1,3 +1,4 @@
+from django.db.models import Q
 from import_export.admin import ImportExportModelAdmin
 
 from models import Vote, Law, PrivateProposal, KnessetProposal, GovProposal, Bill, GovLegislationCommitteeDecision
@@ -6,13 +7,48 @@
 from django.contrib import admin
 
 
+class MissingDataVotesFilter(admin.SimpleListFilter):
+    # Human-readable title which will be displayed in the
+    # right admin sidebar just above the filter options.
+    title = _('Missing data votes')
+
+    # Parameter for the filter that will be used in the URL query.
+    parameter_name = 'is_missing_data_vote'
+
+    def lookups(self, request, model_admin):
+        """
+        Returns a list of tuples. The first element in each
+        tuple is the coded value for the option that will
+        appear in the URL query. The second element is the
+        human-readable name for the option that will appear
+        in the right sidebar.
+        """
+        return (
+            ('is_missing_data_vote', _('Vote has missing data')),
+        )
+
+    def queryset(self, request, queryset):
+        """
+        Returns the filtered queryset based on the value
+        provided in the query string and retrievable via
+        `self.value()`.
+        """
+        # Compare the requested value
+        # to decide how to filter the queryset.
+        if self.value() == 'is_missing_data_vote':
+            return queryset.filter(Q(votes_count=0) | Q(votes_count=None))
+        else:
+            return queryset
+
+
 class VoteAdmin(ImportExportModelAdmin):
     #    filter_horizontal = ('voted_for','voted_against','voted_abstain','didnt_vote')
     list_display = (
         '__unicode__', 'short_summary', 'full_text_link', 'votes_count', 'for_votes_count', 'against_votes_count',
         'abstain_votes_count')
 
     search_fields = ('title', 'summary', 'full_text', 'id', 'src_id')
+    list_filter = (MissingDataVotesFilter, )
 
     def update_vote(self, request, queryset):
         vote_count = queryset.count()
@@ -25,7 +61,9 @@ def update_vote(self, request, queryset):
 
     def recreate_vote(self, request, queryset):
         recreated_votes = ScrapeVotesCommand().recreate_objects(queryset.values_list('pk', flat=True))
-        self.message_user(request, "successfully recreated {0} votes".format(len(recreated_votes), ', '.join([str(v.pk) for v in recreated_votes])))
+        recreated_vote_ids_string = ', '.join([str(v.pk) for v in recreated_votes])
+        self.message_user(request, "successfully recreated {0} votes: {1}".format(len(recreated_votes),
+                                                                                  recreated_vote_ids_string))
 
     recreate_vote.short_description = "recreate vote by deleting and then getting fresh data from knesset api"
 
@@ -36,7 +74,7 @@ def recreate_vote(self, request, queryset):
 
 
 class LawAdmin(ImportExportModelAdmin):
-    search_fields = ('title', )
+    search_fields = ('title',)
     list_display = ('title', 'merged_into')
 
 
@@ -60,7 +98,7 @@ class KnessetProposalAdmin(admin.ModelAdmin):
 class GovProposalAdmin(admin.ModelAdmin):
     search_fields = ('title', 'booklet_number')
     list_display = ('bill', 'booklet_number', 'knesset_id', 'date')
-    list_filter = ('knesset_id', )
+    list_filter = ('knesset_id',)
 
 
 admin.site.register(GovProposal, GovProposalAdmin)

laws/enums.py

@@ -1,3 +1,4 @@
+# encoding: utf-8
 from django.utils.translation import ugettext_lazy as _
 
 from knesset.enums import Enum
@@ -18,9 +19,13 @@ class BillStages(Enum):
     FAILED_APPROVAL = u'-6'
 
 
+VOTE_TYPES = {'law-approve': u'אישור החוק', 'second-call': u'קריאה שנייה', 'demurrer': u'הסתייגות',
+              'no-confidence': u'הצעת אי-אמון', 'pass-to-committee': u'להעביר את ',
+              'continuation': u'להחיל דין רציפות'}
+
 VOTE_ACTION_TYPE_CHOICES = (
     (u'for', _('For')),
     (u'against', _('Against')),
     (u'abstain', _('Abstain')),
     (u'no-vote', _('No Vote')),
-)
+)

laws/helpers.py

@@ -0,0 +1,26 @@
+# -*- coding: utf-8 -*
+from laws.enums import VOTE_TYPES
+
+
+def resolve_vote_type_by_title(title):
+    if type(title) == str:
+        transform_func = str.decode
+    else:  # its already unicode, do nothing
+        transform_func = lambda x, y: x
+    for vtype, vtype_prefix in VOTE_TYPES.items():
+        if transform_func(title, 'utf8').startswith(vtype_prefix):
+            return vtype
+    return ''
+
+
+class MissingVotePartyException(Exception):
+    pass
+
+
+def party_at_or_error(member, vote_date, vote_id):
+    party = member.party_at(vote_date)
+    if party:
+        return party
+    else:
+        raise MissingVotePartyException(
+            'could not find which party member %s belonged to during vote %s' % (member.pk, vote_id))

laws/management/commands/rescrape_missing_data_votes.py

@@ -0,0 +1,35 @@
+# encoding: utf-8
+from __future__ import print_function
+from laws.management.commands.scrape_votes import Command as ScrapeVotesCommand
+from django.core.management.base import BaseCommand
+from optparse import make_option
+
+from django.db.models import Q
+
+from laws.models import Vote
+
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = "Rescrape data for votes missing actual voting data"
+
+    option_list = BaseCommand.option_list + (
+        make_option(
+            '-n', action='store_true', dest="dryrun", default=False,
+            help='Dry run, changes nothing in the db, just display results'
+        ),
+    )
+
+    def handle(self, *args, **options):
+        votes_to_update = Vote.objects.filter(Q(votes_count=0) | Q(votes_count=None))
+        logger.info('Found %s votes with missing data' % votes_to_update.count())
+        if options['dryrun']:
+            logger.info("Not updating the db, dry run was specified")
+            return
+
+        votes_ids = votes_to_update.values_list('pk', flat=True)
+        ScrapeVotesCommand().recreate_objects(votes_ids)
+        logger.info(u'Completed re scraping votes %s' % votes_ids)

laws/management/commands/scrape_votes.py

@@ -1,14 +1,20 @@
 # encoding: utf-8
+from logging import getLogger
+
+from django.db import transaction
 from knesset_data.dataservice.votes import Vote as DataserviceVote
 from knesset_data.html_scrapers.votes import HtmlVote
 from laws.models import Vote, VoteAction
+from simple.constants import KNESSET_VOTE_PAGE
 from simple.scrapers import hebrew_strftime
 from simple.scrapers.management import BaseKnessetDataserviceCollectionCommand
 from mks.models import Member
 from simple.management.commands.syncdata import Command as SyncdataCommand
 from links.models import Link
 from django.contrib.contenttypes.models import ContentType
 
+logger = getLogger(__name__)
+
 
 class VoteScraperException(Exception):
     def __init__(self, *args, **kwargs):
@@ -27,42 +33,42 @@ class Command(BaseKnessetDataserviceCollectionCommand):
         'time': 'datetime',
         'meeting_number': "session_num",
         'vote_number': 'nbr_in_sess',
-        'src_url': lambda vote: "http://www.knesset.gov.il/vote/heb/Vote_Res_Map.asp?vote_id_t=%s" % vote.id
+        'src_url': lambda vote: KNESSET_VOTE_PAGE % vote.id
     }
 
     VALIDATE_FIELDS_TO_AUTOFIX = ['title', 'src_url']
 
     help = "Scrape votes data from the knesset"
 
+    @transaction.atomic
     def _update_or_create_vote(self, dataservice_vote, oknesset_vote=None):
         vote_kwargs = self._get_dataservice_model_kwargs(dataservice_vote)
         if oknesset_vote:
-            [setattr(oknesset_vote, k, v) for k, v in vote_kwargs.iteritems()]
+            [setattr(oknesset_vote, k, v) for k, v in vote_kwargs.items()]
             oknesset_vote.save()
         else:
             oknesset_vote = Vote.objects.create(**vote_kwargs)
         self._add_vote_actions(dataservice_vote, oknesset_vote)
         oknesset_vote.update_vote_properties()
         SyncdataCommand().find_synced_protocol(oknesset_vote)
-        Link.objects.create(
+
+        Link.objects.get_or_create(
             title=u'ההצבעה באתר הכנסת',
-            url='http://www.knesset.gov.il/vote/heb/Vote_Res_Map.asp?vote_id_t=%s' % oknesset_vote.src_id,
+            url=KNESSET_VOTE_PAGE % oknesset_vote.src_id,
             content_type=ContentType.objects.get_for_model(oknesset_vote), object_pk=str(oknesset_vote.id)
         )
         return oknesset_vote
-        # if v.full_text_url != None:
-        #     l = Link(title=u'מסמך הצעת החוק באתר הכנסת', url=v.full_text_url, content_type=ContentType.objects.get_for_model(v), object_pk=str(v.id))
-        #     l.save()
 
     def _add_vote_actions(self, dataservice_vote, oknesset_vote):
         for member_id, vote_result_code in HtmlVote.get_from_vote_id(dataservice_vote.id).member_votes:
             member_qs = Member.objects.filter(pk=member_id)
             if member_qs.exists():
                 member = member_qs.first()
-                vote_type = self._resolve_vote_type(vote_result_code)
+                vote_type = self._resolve_vote_type(vote_result_code)  # TODO: Move to static helper
+                member_party_at_time = member.party_at(oknesset_vote.time.date())
                 vote_action, created = VoteAction.objects.get_or_create(vote=oknesset_vote, member=member,
                                                                         defaults={'type': vote_type,
-                                                                                  'party': member.current_party})
+                                                                                  'party': member_party_at_time})
                 if created:
                     vote_action.save()
             else:
@@ -76,7 +82,10 @@ def _get_existing_object(self, dataservice_object):
         return Vote.objects.get(src_id=dataservice_object.id)
 
     def _create_new_object(self, dataservice_vote):
-        return self._update_or_create_vote(dataservice_vote)
+        try:
+            return self._update_or_create_vote(dataservice_vote)
+        except VoteScraperException:
+            logger.exception('Vote scraping exception for %s' % dataservice_vote)
 
     def _resolve_vote_type(cls, vote_result_code):
         return {
@@ -89,13 +98,24 @@ def _resolve_vote_type(cls, vote_result_code):
     def recreate_objects(self, vote_ids):
         recreated_votes = []
         for vote_id in vote_ids:
-            oknesset_vote = Vote.objects.get(id=int(vote_id))
-            vote_src_id = oknesset_vote.src_id
-            dataservice_vote = self.DATASERVICE_CLASS.get(vote_src_id)
-            VoteAction.objects.filter(vote=oknesset_vote).delete()
-            Link.objects.filter(content_type=ContentType.objects.get_for_model(oknesset_vote),
-                                object_pk=oknesset_vote.id).delete()
-            recreated_votes.append(self._update_or_create_vote(dataservice_vote, oknesset_vote))
+            logger.info('Attempting rescraping for vote id %s' % vote_id)
+            try:
+                try:
+                    oknesset_vote = Vote.objects.get(id=int(vote_id))
+                except Vote.DoesNotExist:
+                    raise VoteScraperException('Vote to recreate does not exist %s' % vote_id)
+                vote_src_id = oknesset_vote.src_id
+                try:
+                    dataservice_vote = self.DATASERVICE_CLASS.get(vote_src_id)
+                except Exception:
+                    raise VoteScraperException('Failure to fetch knesset data dto for vote id %s' % vote_id)
+                VoteAction.objects.filter(vote=oknesset_vote).delete()
+                Link.objects.filter(content_type=ContentType.objects.get_for_model(oknesset_vote),
+                                    object_pk=oknesset_vote.id).delete()
+                recreated_votes.append(self._update_or_create_vote(dataservice_vote, oknesset_vote))
+                logger.info('Success rescraping for vote id %s' % vote_id)
+            except VoteScraperException:
+                logger.exception('Vote scraper exception for vote %s' % vote_id)
         return recreated_votes
 
     def _get_validate_first_object_title(self, dataservice_object):