Merge pull request #531 from alonisser/master

Mainly fixing #513 (or allowing to fix after data updates with batch or specific triggering for updating vore properties)

knesset/settings.py

@@ -4,6 +4,8 @@
 from datetime import timedelta
 
 # dummy gettext, to get django-admin makemessages to find i18n texts in this file
+import sys
+
 gettext = lambda x: x
 
 DEBUG = True
@@ -121,7 +123,7 @@
     os.path.join(PROJECT_ROOT, 'locale'),
 )
 INSTALLED_APPS = (
-    'django.contrib.auth',          # django apps
+    'django.contrib.auth',  # django apps
     'django.contrib.contenttypes',
     'django.contrib.sessions',
     'django.contrib.sites',
@@ -156,8 +158,8 @@
     'storages',
     'corsheaders',
     'sslserver',
-    #'knesset',
-    'auxiliary',                  # knesset apps
+    # 'knesset',
+    'auxiliary',  # knesset apps
     'mks',
     'mmm',
     'laws',
@@ -197,7 +199,7 @@
 
 INTERNAL_IPS = ()
 # Add the following line to your local_settings.py files to enable django-debug-toolar:
-#INTERNAL_IPS = ('127.0.0.1',)
+# INTERNAL_IPS = ('127.0.0.1',)
 
 LOCAL_DEV = True
 
@@ -214,13 +216,23 @@
 USER_AGENT = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.2; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)"
 
 LOG_FILENAME = os.path.join(PROJECT_ROOT, 'open-knesset.log')
-logger = logging.getLogger("open-knesset")
-logger.setLevel(logging.DEBUG)  # override this in prod server to logging.ERROR
+file_logger = logging.getLogger("open-knesset")
+file_logger.setLevel(logging.DEBUG)  # override this in prod server to logging.ERROR
 h = logging.FileHandler(LOG_FILENAME)
 h.setLevel(logging.DEBUG)
 formatter = logging.Formatter("%(asctime)s\t%(name)s:%(lineno)d\t%(levelname)s\t%(message)s")
 h.setFormatter(formatter)
-logger.addHandler(h)
+file_logger.addHandler(h)
+
+# Console loggers, Best practice always log to stdout and stderr and let third party environment to deal with logging
+# See 12 factor app http://12factor.net/logs
+# Todo refactor this to support stderr and out and more dynamic config supporting sentry etc
+console_logger = logging.getLogger('') #root logger
+console_logger.setLevel(logging.INFO)
+stdout_handler = logging.StreamHandler(stream=sys.stdout)
+
+stdout_handler.setFormatter(formatter)
+console_logger.addHandler(stdout_handler)
 
 GOOGLE_CUSTOM_SEARCH = "007833092092208924626:1itz_l8x4a4"
 GOOGLE_MAPS_API_KEYS = {'dev': 'ABQIAAAAWCfW8hHVwzZc12qTG0qLEhQCULP4XOMyhPd8d_NrQQEO8sT8XBQdS2fOURLgU1OkrUWJE1ji1lJ-3w',
@@ -264,7 +276,6 @@
 TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
 NOSE_ARGS = ['--with-xunit']
 
-
 SERIALIZATION_MODULES = {
     'oknesset': 'auxiliary.serializers'
 }
@@ -273,7 +284,6 @@
 
 SOUTH_TESTS_MIGRATE = False
 
-
 TINYMCE_DEFAULT_CONFIG = {
     'mode': "textareas",
     'theme': "advanced",

laws/admin.py

@@ -6,7 +6,22 @@
 class VoteAdmin(admin.ModelAdmin):
     #    filter_horizontal = ('voted_for','voted_against','voted_abstain','didnt_vote')
     list_display = (
-    '__unicode__', 'short_summary', 'full_text_link', 'for_votes_count', 'against_votes_count', 'abstain_votes_count')
+        '__unicode__', 'short_summary', 'full_text_link', 'votes_count', 'for_votes_count', 'against_votes_count',
+        'abstain_votes_count')
+
+    search_fields = ('summary', 'full_text')
+
+    def update_vote(self, request, queryset):
+        vote_count = queryset.count() if queryset else 0
+        if queryset:
+            queryset = queryset.select_relate().prefetch_related('')
+        for vote in queryset:
+            vote.update_vote_properties()
+
+        self.message_user(request, "successfully updated {0} votes".format(vote_count))
+
+    update_vote.short_description = 'update vote properties and calculations'
+    actions = ['update_vote']
 
 
 admin.site.register(Vote, VoteAdmin)

laws/constants.py

@@ -0,0 +1 @@
+STANDS_FOR_THRESHOLD = 0.66

laws/management/commands/recalculate_votes_properties_current_knesset.py

@@ -0,0 +1,36 @@
+from __future__ import print_function
+
+from django.core.management.base import BaseCommand
+from optparse import make_option
+
+from laws.models import Vote
+from mks.models import Knesset
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    help = "Recalculate vote properties for current knesset"
+
+    option_list = BaseCommand.option_list + (
+        make_option(
+            '-n', action='store_true', dest="dryrun", default=False,
+            help='Dry run, changes nothing in the db, just display results'
+        ),
+    )
+
+    def handle(self, *args, **options):
+
+        start_date = Knesset.objects.current_knesset().start_date
+
+        votes_to_update = Vote.objects.filter(time__gte=start_date)
+        logger.info('Found {0} votes to update since {1}'.format(votes_to_update.count(), start_date))
+        if options['dryrun']:
+            logger.info("Not updating the db, dry run was specified")
+            return
+
+        for vote in votes_to_update:
+
+            vote.update_vote_properties()
+            logger.info(u'Recalculated vote properties for {0}'.format(vote))

laws/models.py

@@ -21,6 +21,7 @@
 from actstream import Action
 from actstream.models import action, Follow
 
+from laws import constants
 from mks.models import Party, Knesset
 from tagvotes.models import TagVote
 from knesset.utils import slugify_name, trans_clean
@@ -67,15 +68,15 @@ class PartyVotingStatistics(models.Model):
     def votes_against_party_count(self):
         d = Knesset.objects.current_knesset().start_date
         return VoteAction.objects.filter(
-                vote__time__gt=d,
-                member__current_party=self.party,
-                against_party=True).count()
+            vote__time__gt=d,
+            member__current_party=self.party,
+            against_party=True).count()
 
     def votes_count(self):
         d = Knesset.objects.current_knesset().start_date
         return VoteAction.objects.filter(
-                member__current_party=self.party,
-                vote__time__gt=d).exclude(type='no-vote').count()
+            member__current_party=self.party,
+            vote__time__gt=d).exclude(type='no-vote').count()
 
     def votes_per_seat(self):
         return round(float(self.votes_count()) / self.party.number_of_seats, 1)
@@ -95,14 +96,14 @@ def coalition_discipline(self):  # if party is in opposition this actually
         if total_votes:
             if self.party.is_coalition:
                 votes_against_coalition = VoteAction.objects.filter(
-                        vote__time__gt=d,
-                        member__current_party=self.party,
-                        against_coalition=True).count()
+                    vote__time__gt=d,
+                    member__current_party=self.party,
+                    against_coalition=True).count()
             else:
                 votes_against_coalition = VoteAction.objects.filter(
-                        vote__time__gt=d,
-                        member__current_party=self.party,
-                        against_opposition=True).count()
+                    vote__time__gt=d,
+                    member__current_party=self.party,
+                    against_opposition=True).count()
             return round(100.0 * (total_votes - votes_against_coalition) /
                          total_votes, 1)
         return _('N/A')
@@ -122,7 +123,7 @@ def votes_against_party_count(self, from_date=None):
     def votes_count(self, from_date=None):
         if from_date:
             return VoteAction.objects.filter(member=self.member, vote__time__gt=from_date).exclude(
-                    type='no-vote').count()
+                type='no-vote').count()
         vc = cache.get('votes_count_%d' % self.member.id)
         if not vc:
             vc = VoteAction.objects.filter(member=self.member).exclude(type='no-vote').count()
@@ -218,7 +219,7 @@ def filter_and_order(self, *args, **kwargs):
             # exclude votes that are ascribed to any of the given agendas
             from agendas.models import AgendaVote
             qs = qs.exclude(id__in=AgendaVote.objects.filter(
-                    agenda__in=exclude_agendas).values('vote__id'))
+                agenda__in=exclude_agendas).values('vote__id'))
 
         if 'order' in kwargs:
             if kwargs['order'] == 'controversy':
@@ -231,7 +232,7 @@ def filter_and_order(self, *args, **kwargs):
         if kwargs.get('exclude_ascribed', False):  # exclude votes ascribed to
             # any bill.
             qs = qs.exclude(bills_pre_votes__isnull=False).exclude(
-                    bills_first__isnull=False).exclude(bill_approved__isnull=False)
+                bills_first__isnull=False).exclude(bill_approved__isnull=False)
         return qs
 
 
@@ -363,12 +364,13 @@ def tag_form(self):
         return tf
 
     def update_vote_properties(self):
+        # TODO: this can be heavily optimized. somewhere sometimes..
         party_ids = Party.objects.values_list('id', flat=True)
         d = self.time.date()
         party_is_coalition = dict(zip(
-                party_ids,
-                [x.is_coalition_at(self.time.date())
-                 for x in Party.objects.all()]
+            party_ids,
+            [x.is_coalition_at(self.time.date())
+             for x in Party.objects.all()]
         ))
 
         def party_at_or_error(member):
@@ -377,16 +379,18 @@ def party_at_or_error(member):
                 return party
             else:
                 raise Exception(
-                        'could not find which party member %s belonged to during vote %s' % (member.pk, self.pk))
+                    'could not find which party member %s belonged to during vote %s' % (member.pk, self.pk))
 
         for_party_ids = [party_at_or_error(va.member).id for va in self.for_votes()]
         party_for_votes = [sum([x == id for x in for_party_ids]) for id in party_ids]
 
         against_party_ids = [party_at_or_error(va.member).id for va in self.against_votes()]
         party_against_votes = [sum([x == id for x in against_party_ids]) for id in party_ids]
 
-        party_stands_for = [float(fv) > 0.66 * (fv + av) for (fv, av) in zip(party_for_votes, party_against_votes)]
-        party_stands_against = [float(av) > 0.66 * (fv + av) for (fv, av) in zip(party_for_votes, party_against_votes)]
+        party_stands_for = [float(fv) > constants.STANDS_FOR_THRESHOLD * (fv + av) for (fv, av) in
+                            zip(party_for_votes, party_against_votes)]
+        party_stands_against = [float(av) > constants.STANDS_FOR_THRESHOLD * (fv + av) for (fv, av) in
+                                zip(party_for_votes, party_against_votes)]
 
         party_stands_for = dict(zip(party_ids, party_stands_for))
         party_stands_against = dict(zip(party_ids, party_stands_against))
@@ -395,14 +399,17 @@ def party_at_or_error(member):
         coalition_against_votes = sum([x for (x, y) in zip(party_against_votes, party_ids) if party_is_coalition[y]])
         opposition_for_votes = sum([x for (x, y) in zip(party_for_votes, party_ids) if not party_is_coalition[y]])
         opposition_against_votes = sum(
-                [x for (x, y) in zip(party_against_votes, party_ids) if not party_is_coalition[y]])
-
-        coalition_stands_for = (float(coalition_for_votes) > 0.66 * (coalition_for_votes + coalition_against_votes))
-        coalition_stands_against = float(coalition_against_votes) > 0.66 * (
-            coalition_for_votes + coalition_against_votes)
-        opposition_stands_for = float(opposition_for_votes) > 0.66 * (opposition_for_votes + opposition_against_votes)
-        opposition_stands_against = float(opposition_against_votes) > 0.66 * (
-            opposition_for_votes + opposition_against_votes)
+            [x for (x, y) in zip(party_against_votes, party_ids) if not party_is_coalition[y]])
+
+        coalition_total_votes = coalition_for_votes + coalition_against_votes
+        coalition_stands_for = (
+            float(coalition_for_votes) > constants.STANDS_FOR_THRESHOLD * (coalition_total_votes))
+        coalition_stands_against = float(coalition_against_votes) > constants.STANDS_FOR_THRESHOLD * (
+            coalition_total_votes)
+        opposition_total_votes = opposition_for_votes + opposition_against_votes
+        opposition_stands_for = float(opposition_for_votes) > constants.STANDS_FOR_THRESHOLD * opposition_total_votes
+        opposition_stands_against = float(
+            opposition_against_votes) > constants.STANDS_FOR_THRESHOLD * opposition_total_votes
 
         # a set of all MKs that proposed bills this vote is about.
         proposers = [set(b.proposers.all()) for b in self.bills()]
@@ -591,24 +598,24 @@ def filter_and_order(self, *args, **kwargs):
 
         if pp_id:
             pps = PrivateProposal.objects.filter(
-                    proposal_id=pp_id).values_list(
-                    'id', flat=True)
+                proposal_id=pp_id).values_list(
+                'id', flat=True)
             if pps:
                 qs = qs.filter(proposals__in=pps)
             else:
                 qs = qs.none()
 
         if knesset_booklet:
             kps = KnessetProposal.objects.filter(
-                    booklet_number=knesset_booklet).values_list(
-                    'id', flat=True)
+                booklet_number=knesset_booklet).values_list(
+                'id', flat=True)
             if kps:
                 qs = qs.filter(knesset_proposal__in=kps)
             else:
                 qs = qs.none()
         if gov_booklet:
             gps = GovProposal.objects.filter(
-                    booklet_number=gov_booklet).values_list('id', flat=True)
+                booklet_number=gov_booklet).values_list('id', flat=True)
             if gps:
                 qs = qs.filter(gov_proposal__in=gps)
             else:
@@ -729,7 +736,7 @@ def merge(self, another_bill):
         bill_ct = ContentType.objects.get_for_model(self)
         Comment.objects.filter(content_type=bill_ct,
                                object_pk=another_bill.id).update(
-                object_pk=self.id)
+            object_pk=self.id)
         for v in voting.models.Vote.objects.filter(content_type=bill_ct,
                                                    object_id=another_bill.id):
             if voting.models.Vote.objects.filter(content_type=bill_ct,

laws/views.py

@@ -29,7 +29,7 @@
 from hashnav import DetailView, ListView as HashnavListView
 from knesset.utils import notify_responsible_adult
 from mks.models import Member
-from models import Bill, BillBudgetEstimation, Vote, KnessetProposal
+from models import Bill, BillBudgetEstimation, Vote, KnessetProposal, VoteAction
 from models import BILL_STAGE_CHOICES
 
 logger = logging.getLogger("open-knesset.laws.views")

mks/admin.py

@@ -55,6 +55,7 @@ def queryset(self, request):
 
 class CoalitionMembershipAdmin(admin.ModelAdmin):
     list_display = ('party', 'start_date', 'end_date')
+
 admin.site.register(CoalitionMembership, CoalitionMembershipAdmin)