From 5eb027e11497c26c93d997b7fbb40176a2fed9e3 Mon Sep 17 00:00:00 2001 From: alonisser Date: Mon, 15 Feb 2016 01:11:44 +0200 Subject: [PATCH 1/6] pep8 --- mks/admin.py | 1 + mks/api.py | 85 ++++++++++++++++++++++++++++------------------------ 2 files changed, 47 insertions(+), 39 deletions(-) diff --git a/mks/admin.py b/mks/admin.py index 23d26b9a..f639b0f0 100644 --- a/mks/admin.py +++ b/mks/admin.py @@ -55,6 +55,7 @@ def queryset(self, request): class CoalitionMembershipAdmin(admin.ModelAdmin): list_display = ('party', 'start_date', 'end_date') + admin.site.register(CoalitionMembership, CoalitionMembershipAdmin) diff --git a/mks/api.py b/mks/api.py index 42bcd2d4..fb1d3ab9 100644 --- a/mks/api.py +++ b/mks/api.py @@ -27,6 +27,7 @@ logger = logging.getLogger(__name__) + class PartyResource(BaseResource): ''' Party API TBD: create a party app @@ -40,24 +41,23 @@ class Meta(BaseResource.Meta): include_absolute_url = True def get_object_list(self, request): - knesset = request.GET.get('knesset','current') + knesset = request.GET.get('knesset', 'current') if knesset == 'current': return super(PartyResource, self).get_object_list(request).filter( - knesset=Knesset.objects.current_knesset()) + knesset=Knesset.objects.current_knesset()) elif knesset == 'all': return super(PartyResource, self).get_object_list(request) else: return super(PartyResource, self).get_object_list(request).filter( - knesset=Knesset.objects.current_knesset()) + knesset=Knesset.objects.current_knesset()) class DictStruct: def __init__(self, **entries): - self.__dict__.update(entries) + self.__dict__.update(entries) class MemberBillsResource(BaseNonModelResource): - class Meta(BaseNonModelResource.Meta): allowed_methods = ['get'] resource_name = "member-bills" @@ -92,16 +92,16 @@ def get_member_data(self, member): # This prevents the need of using a forked django-tagging, and go # upstream tag_cloud = [{ - 'size': getattr(x, 'font_size', 1), - 'count':x.count, - 'name':x.name} for x in calculate_cloud(bills_tags)] + 'size': getattr(x, 'font_size', 1), + 'count': x.count, + 'name': x.name} for x in calculate_cloud(bills_tags)] - bills = map(lambda b: dict(title=b.full_title, - url=b.get_absolute_url(), - stage=b.stage, - stage_text=b.get_stage_display()), - member.bills.all()) - return DictStruct(id=member.id, tag_cloud=tag_cloud,bills=bills) + bills = map(lambda b: dict(title=b.full_title, + url=b.get_absolute_url(), + stage=b.stage, + stage_text=b.get_stage_display()), + member.bills.all()) + return DictStruct(id=member.id, tag_cloud=tag_cloud, bills=bills) def get_object_list(self, request): return map(self.get_member_data, Member.objects.all()) @@ -137,12 +137,14 @@ def dehydrate_agendas(self, bundle): friends = (mk.current_party.current_members() .values_list('id', flat=True)) else: - friends = [] + friends = [] agendas = [] - for a in Agenda.objects.filter(pk__in = agendas_values.keys(), - is_public = True): - amin = 200.0 ; amax = -200.0 - pmin = 200.0 ; pmax = -200.0 + for a in Agenda.objects.filter(pk__in=agendas_values.keys(), + is_public=True): + amin = 200.0; + amax = -200.0 + pmin = 200.0; + pmax = -200.0 av = agendas_values[a.id] for mk_id, values in a.get_mks_values_old(): score = values['score'] @@ -176,6 +178,7 @@ def dehydrate_agendas(self, bundle): class MemberResource(BaseResource): ''' The Parliament Member API ''' + class Meta(BaseResource.Meta): queryset = Member.objects.exclude( @@ -197,31 +200,32 @@ class Meta(BaseResource.Meta): ) excludes = ['website', 'backlinks_enabled', 'area_of_residence'] - list_fields = ['name', 'id', 'img_url', 'is_current', 'average_weekly_presence_hours', 'mmms_count', 'bills_stats_first', 'bills_stats_proposed',] + list_fields = ['name', 'id', 'img_url', 'is_current', 'average_weekly_presence_hours', 'mmms_count', + 'bills_stats_first', 'bills_stats_proposed', ] include_absolute_url = True party_name = fields.CharField() party_url = fields.CharField() mmms_count = fields.IntegerField(null=True) votes_count = fields.IntegerField(null=True) - video_about = fields.ToManyField(VideoResource, - attribute= lambda b: get_videos_queryset(b.obj,group='about'), - null = True, - full = True) + video_about = fields.ToManyField(VideoResource, + attribute=lambda b: get_videos_queryset(b.obj, group='about'), + null=True, + full=True) videos_related = fields.ToManyField(VideoResource, - attribute= lambda b: get_videos_queryset(b.obj,group='related'), - null = True) + attribute=lambda b: get_videos_queryset(b.obj, group='related'), + null=True) links = fields.ToManyField(LinkResource, - attribute = lambda b: Link.objects.for_model(b.obj), - full = True, - null = True) + attribute=lambda b: Link.objects.for_model(b.obj), + full=True, + null=True) bills_uri = fields.CharField() agendas_uri = fields.CharField() committees = fields.ListField() detailed_roles = fields.ToManyField(RoleResource, - attribute = lambda b: Person.objects.get(mk=b.obj).roles.all(), - full = True, - null = True) + attribute=lambda b: Person.objects.get(mk=b.obj).roles.all(), + full=True, + null=True) fields.ToOneField(PartyResource, 'current_party', full=True) average_weekly_presence_rank = fields.IntegerField() @@ -242,22 +246,25 @@ def obj_get_list(self, bundle, **kwargs): return simple return simple - def dehydrate_committees (self, bundle): + def dehydrate_committees(self, bundle): temp_list = bundle.obj.committee_meetings.exclude(committee__type='plenum') temp_list = temp_list.values("committee", "committee__name").annotate(Count("id")).order_by('-id__count')[:5] - return (map(lambda item: (item['committee__name'], reverse('committee-detail', args=[item['committee']])), temp_list)) + return ( + map(lambda item: (item['committee__name'], reverse('committee-detail', args=[item['committee']])), temp_list)) def dehydrate_bills_uri(self, bundle): return '%s?%s' % (reverse('api_dispatch_list', kwargs={'resource_name': 'bill', - 'api_name': 'v2', }), + 'api_name': 'v2',}), urllib.urlencode(dict(proposer=bundle.obj.id))) + def dehydrate_gender(self, bundle): return bundle.obj.get_gender_display() def dehydrate_agendas_uri(self, bundle): return reverse('api_dispatch_detail', kwargs={'resource_name': 'member-agendas', - 'api_name': 'v2', - 'pk' : bundle.obj.id}) + 'api_name': 'v2', + 'pk': bundle.obj.id}) + def dehydrate_party_name(self, bundle): party = bundle.obj.current_party return party.name if party else None @@ -286,7 +293,7 @@ def dehydrate_votes_count(self, bundle): return count - def dehydrate_average_weekly_presence_rank (self, bundle): + def dehydrate_average_weekly_presence_rank(self, bundle): ''' Calculate the distribution of presence and place the user on a 5 level scale ''' SCALE = 5 member = bundle.obj @@ -306,7 +313,7 @@ def dehydrate_average_weekly_presence_rank (self, bundle): else: mk_location = 0 - cache.set('average_presence_location_%d' % mk.id, mk_location, 60*60*24) + cache.set('average_presence_location_%d' % mk.id, mk_location, 60 * 60 * 24) if mk.id == member.id: rel_location = mk_location From a5217b74a8f18f5b289929ac3a3531e40d22c69f Mon Sep 17 00:00:00 2001 From: alonisser Date: Mon, 15 Feb 2016 01:12:17 +0200 Subject: [PATCH 2/6] missing import --- laws/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/laws/views.py b/laws/views.py index a9666c4c..65930e27 100644 --- a/laws/views.py +++ b/laws/views.py @@ -29,7 +29,7 @@ from hashnav import DetailView, ListView as HashnavListView from knesset.utils import notify_responsible_adult from mks.models import Member -from models import Bill, BillBudgetEstimation, Vote, KnessetProposal +from models import Bill, BillBudgetEstimation, Vote, KnessetProposal, VoteAction from models import BILL_STAGE_CHOICES logger = logging.getLogger("open-knesset.laws.views") From c00972b14fe7e0775a9d947fbf5bb383648e83ce Mon Sep 17 00:00:00 2001 From: alonisser Date: Mon, 15 Feb 2016 01:12:52 +0200 Subject: [PATCH 3/6] replace a magic number for named constants --- laws/constants.py | 1 + laws/models.py | 74 +++++++++++++++++++++++++---------------------- 2 files changed, 41 insertions(+), 34 deletions(-) create mode 100644 laws/constants.py diff --git a/laws/constants.py b/laws/constants.py new file mode 100644 index 00000000..6c75e42a --- /dev/null +++ b/laws/constants.py @@ -0,0 +1 @@ +STANDS_FOR_THRESHOLD = 0.66 diff --git a/laws/models.py b/laws/models.py index 868b08ca..648658dd 100755 --- a/laws/models.py +++ b/laws/models.py @@ -21,6 +21,7 @@ from actstream import Action from actstream.models import action, Follow +from laws import constants from mks.models import Party, Knesset from tagvotes.models import TagVote from knesset.utils import slugify_name, trans_clean @@ -67,15 +68,15 @@ class PartyVotingStatistics(models.Model): def votes_against_party_count(self): d = Knesset.objects.current_knesset().start_date return VoteAction.objects.filter( - vote__time__gt=d, - member__current_party=self.party, - against_party=True).count() + vote__time__gt=d, + member__current_party=self.party, + against_party=True).count() def votes_count(self): d = Knesset.objects.current_knesset().start_date return VoteAction.objects.filter( - member__current_party=self.party, - vote__time__gt=d).exclude(type='no-vote').count() + member__current_party=self.party, + vote__time__gt=d).exclude(type='no-vote').count() def votes_per_seat(self): return round(float(self.votes_count()) / self.party.number_of_seats, 1) @@ -95,14 +96,14 @@ def coalition_discipline(self): # if party is in opposition this actually if total_votes: if self.party.is_coalition: votes_against_coalition = VoteAction.objects.filter( - vote__time__gt=d, - member__current_party=self.party, - against_coalition=True).count() + vote__time__gt=d, + member__current_party=self.party, + against_coalition=True).count() else: votes_against_coalition = VoteAction.objects.filter( - vote__time__gt=d, - member__current_party=self.party, - against_opposition=True).count() + vote__time__gt=d, + member__current_party=self.party, + against_opposition=True).count() return round(100.0 * (total_votes - votes_against_coalition) / total_votes, 1) return _('N/A') @@ -122,7 +123,7 @@ def votes_against_party_count(self, from_date=None): def votes_count(self, from_date=None): if from_date: return VoteAction.objects.filter(member=self.member, vote__time__gt=from_date).exclude( - type='no-vote').count() + type='no-vote').count() vc = cache.get('votes_count_%d' % self.member.id) if not vc: vc = VoteAction.objects.filter(member=self.member).exclude(type='no-vote').count() @@ -218,7 +219,7 @@ def filter_and_order(self, *args, **kwargs): # exclude votes that are ascribed to any of the given agendas from agendas.models import AgendaVote qs = qs.exclude(id__in=AgendaVote.objects.filter( - agenda__in=exclude_agendas).values('vote__id')) + agenda__in=exclude_agendas).values('vote__id')) if 'order' in kwargs: if kwargs['order'] == 'controversy': @@ -231,7 +232,7 @@ def filter_and_order(self, *args, **kwargs): if kwargs.get('exclude_ascribed', False): # exclude votes ascribed to # any bill. qs = qs.exclude(bills_pre_votes__isnull=False).exclude( - bills_first__isnull=False).exclude(bill_approved__isnull=False) + bills_first__isnull=False).exclude(bill_approved__isnull=False) return qs @@ -366,9 +367,9 @@ def update_vote_properties(self): party_ids = Party.objects.values_list('id', flat=True) d = self.time.date() party_is_coalition = dict(zip( - party_ids, - [x.is_coalition_at(self.time.date()) - for x in Party.objects.all()] + party_ids, + [x.is_coalition_at(self.time.date()) + for x in Party.objects.all()] )) def party_at_or_error(member): @@ -377,7 +378,7 @@ def party_at_or_error(member): return party else: raise Exception( - 'could not find which party member %s belonged to during vote %s' % (member.pk, self.pk)) + 'could not find which party member %s belonged to during vote %s' % (member.pk, self.pk)) for_party_ids = [party_at_or_error(va.member).id for va in self.for_votes()] party_for_votes = [sum([x == id for x in for_party_ids]) for id in party_ids] @@ -385,8 +386,10 @@ def party_at_or_error(member): against_party_ids = [party_at_or_error(va.member).id for va in self.against_votes()] party_against_votes = [sum([x == id for x in against_party_ids]) for id in party_ids] - party_stands_for = [float(fv) > 0.66 * (fv + av) for (fv, av) in zip(party_for_votes, party_against_votes)] - party_stands_against = [float(av) > 0.66 * (fv + av) for (fv, av) in zip(party_for_votes, party_against_votes)] + party_stands_for = [float(fv) > constants.STANDS_FOR_THRESHOLD * (fv + av) for (fv, av) in + zip(party_for_votes, party_against_votes)] + party_stands_against = [float(av) > constants.STANDS_FOR_THRESHOLD * (fv + av) for (fv, av) in + zip(party_for_votes, party_against_votes)] party_stands_for = dict(zip(party_ids, party_stands_for)) party_stands_against = dict(zip(party_ids, party_stands_against)) @@ -395,14 +398,17 @@ def party_at_or_error(member): coalition_against_votes = sum([x for (x, y) in zip(party_against_votes, party_ids) if party_is_coalition[y]]) opposition_for_votes = sum([x for (x, y) in zip(party_for_votes, party_ids) if not party_is_coalition[y]]) opposition_against_votes = sum( - [x for (x, y) in zip(party_against_votes, party_ids) if not party_is_coalition[y]]) - - coalition_stands_for = (float(coalition_for_votes) > 0.66 * (coalition_for_votes + coalition_against_votes)) - coalition_stands_against = float(coalition_against_votes) > 0.66 * ( - coalition_for_votes + coalition_against_votes) - opposition_stands_for = float(opposition_for_votes) > 0.66 * (opposition_for_votes + opposition_against_votes) - opposition_stands_against = float(opposition_against_votes) > 0.66 * ( - opposition_for_votes + opposition_against_votes) + [x for (x, y) in zip(party_against_votes, party_ids) if not party_is_coalition[y]]) + + coalition_total_votes = coalition_for_votes + coalition_against_votes + coalition_stands_for = ( + float(coalition_for_votes) > constants.STANDS_FOR_THRESHOLD * (coalition_total_votes)) + coalition_stands_against = float(coalition_against_votes) > constants.STANDS_FOR_THRESHOLD * ( + coalition_total_votes) + opposition_total_votes = opposition_for_votes + opposition_against_votes + opposition_stands_for = float(opposition_for_votes) > constants.STANDS_FOR_THRESHOLD * opposition_total_votes + opposition_stands_against = float( + opposition_against_votes) > constants.STANDS_FOR_THRESHOLD * opposition_total_votes # a set of all MKs that proposed bills this vote is about. proposers = [set(b.proposers.all()) for b in self.bills()] @@ -591,8 +597,8 @@ def filter_and_order(self, *args, **kwargs): if pp_id: pps = PrivateProposal.objects.filter( - proposal_id=pp_id).values_list( - 'id', flat=True) + proposal_id=pp_id).values_list( + 'id', flat=True) if pps: qs = qs.filter(proposals__in=pps) else: @@ -600,15 +606,15 @@ def filter_and_order(self, *args, **kwargs): if knesset_booklet: kps = KnessetProposal.objects.filter( - booklet_number=knesset_booklet).values_list( - 'id', flat=True) + booklet_number=knesset_booklet).values_list( + 'id', flat=True) if kps: qs = qs.filter(knesset_proposal__in=kps) else: qs = qs.none() if gov_booklet: gps = GovProposal.objects.filter( - booklet_number=gov_booklet).values_list('id', flat=True) + booklet_number=gov_booklet).values_list('id', flat=True) if gps: qs = qs.filter(gov_proposal__in=gps) else: @@ -729,7 +735,7 @@ def merge(self, another_bill): bill_ct = ContentType.objects.get_for_model(self) Comment.objects.filter(content_type=bill_ct, object_pk=another_bill.id).update( - object_pk=self.id) + object_pk=self.id) for v in voting.models.Vote.objects.filter(content_type=bill_ct, object_id=another_bill.id): if voting.models.Vote.objects.filter(content_type=bill_ct, From f13346154abe5ab510482b111be478247da43e69 Mon Sep 17 00:00:00 2001 From: alonisser Date: Mon, 15 Feb 2016 09:57:37 +0200 Subject: [PATCH 4/6] Adding a default streamlogger, for future migration for environment based logging, also allows us to remove print statments from managment commands --- knesset/settings.py | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/knesset/settings.py b/knesset/settings.py index 705b2724..f70b55ee 100644 --- a/knesset/settings.py +++ b/knesset/settings.py @@ -4,6 +4,8 @@ from datetime import timedelta # dummy gettext, to get django-admin makemessages to find i18n texts in this file +import sys + gettext = lambda x: x DEBUG = True @@ -121,7 +123,7 @@ os.path.join(PROJECT_ROOT, 'locale'), ) INSTALLED_APPS = ( - 'django.contrib.auth', # django apps + 'django.contrib.auth', # django apps 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.sites', @@ -156,8 +158,8 @@ 'storages', 'corsheaders', 'sslserver', - #'knesset', - 'auxiliary', # knesset apps + # 'knesset', + 'auxiliary', # knesset apps 'mks', 'mmm', 'laws', @@ -197,7 +199,7 @@ INTERNAL_IPS = () # Add the following line to your local_settings.py files to enable django-debug-toolar: -#INTERNAL_IPS = ('127.0.0.1',) +# INTERNAL_IPS = ('127.0.0.1',) LOCAL_DEV = True @@ -214,13 +216,23 @@ USER_AGENT = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.2; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)" LOG_FILENAME = os.path.join(PROJECT_ROOT, 'open-knesset.log') -logger = logging.getLogger("open-knesset") -logger.setLevel(logging.DEBUG) # override this in prod server to logging.ERROR +file_logger = logging.getLogger("open-knesset") +file_logger.setLevel(logging.DEBUG) # override this in prod server to logging.ERROR h = logging.FileHandler(LOG_FILENAME) h.setLevel(logging.DEBUG) formatter = logging.Formatter("%(asctime)s\t%(name)s:%(lineno)d\t%(levelname)s\t%(message)s") h.setFormatter(formatter) -logger.addHandler(h) +file_logger.addHandler(h) + +# Console loggers, Best practice always log to stdout and stderr and let third party environment to deal with logging +# See 12 factor app http://12factor.net/logs +# Todo refactor this to support stderr and out and more dynamic config supporting sentry etc +console_logger = logging.getLogger('') #root logger +console_logger.setLevel(logging.INFO) +stdout_handler = logging.StreamHandler(stream=sys.stdout) + +stdout_handler.setFormatter(formatter) +console_logger.addHandler(stdout_handler) GOOGLE_CUSTOM_SEARCH = "007833092092208924626:1itz_l8x4a4" GOOGLE_MAPS_API_KEYS = {'dev': 'ABQIAAAAWCfW8hHVwzZc12qTG0qLEhQCULP4XOMyhPd8d_NrQQEO8sT8XBQdS2fOURLgU1OkrUWJE1ji1lJ-3w', @@ -264,7 +276,6 @@ TEST_RUNNER = 'django_nose.NoseTestSuiteRunner' NOSE_ARGS = ['--with-xunit'] - SERIALIZATION_MODULES = { 'oknesset': 'auxiliary.serializers' } @@ -273,7 +284,6 @@ SOUTH_TESTS_MIGRATE = False - TINYMCE_DEFAULT_CONFIG = { 'mode': "textareas", 'theme': "advanced", From 19a3191eec6e67b7ca9740ac7564527ad581571b Mon Sep 17 00:00:00 2001 From: alonisser Date: Mon, 15 Feb 2016 09:59:18 +0200 Subject: [PATCH 5/6] Adding command to recalculate vote properties, or for a specific vote to recalculate from django admin+ several improvments to admin allows fixing of #513 --- laws/admin.py | 17 ++++++++- ...culate_votes_properties_current_knesset.py | 36 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 laws/management/commands/recalculate_votes_properties_current_knesset.py diff --git a/laws/admin.py b/laws/admin.py index 9e3bd64a..5317f018 100644 --- a/laws/admin.py +++ b/laws/admin.py @@ -6,7 +6,22 @@ class VoteAdmin(admin.ModelAdmin): # filter_horizontal = ('voted_for','voted_against','voted_abstain','didnt_vote') list_display = ( - '__unicode__', 'short_summary', 'full_text_link', 'for_votes_count', 'against_votes_count', 'abstain_votes_count') + '__unicode__', 'short_summary', 'full_text_link', 'votes_count', 'for_votes_count', 'against_votes_count', + 'abstain_votes_count') + + search_fields = ('summary', 'full_text') + + def update_vote(self, request, queryset): + vote_count = queryset.count() if queryset else 0 + if queryset: + queryset = queryset.select_relate().prefetch_related('') + for vote in queryset: + vote.update_vote_properties() + + self.message_user(request, "successfully updated {0} votes".format(vote_count)) + + update_vote.short_description = 'update vote properties and calculations' + actions = ['update_vote'] admin.site.register(Vote, VoteAdmin) diff --git a/laws/management/commands/recalculate_votes_properties_current_knesset.py b/laws/management/commands/recalculate_votes_properties_current_knesset.py new file mode 100644 index 00000000..aead4f34 --- /dev/null +++ b/laws/management/commands/recalculate_votes_properties_current_knesset.py @@ -0,0 +1,36 @@ +from __future__ import print_function + +from django.core.management.base import BaseCommand +from optparse import make_option + +from laws.models import Vote +from mks.models import Knesset +import logging + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + help = "Recalculate vote properties for current knesset" + + option_list = BaseCommand.option_list + ( + make_option( + '-n', action='store_true', dest="dryrun", default=False, + help='Dry run, changes nothing in the db, just display results' + ), + ) + + def handle(self, *args, **options): + + start_date = Knesset.objects.current_knesset().start_date + + votes_to_update = Vote.objects.filter(time__gte=start_date) + logger.info('Found {0} votes to update since {1}'.format(votes_to_update.count(), start_date)) + if options['dryrun']: + logger.info("Not updating the db, dry run was specified") + return + + for vote in votes_to_update: + + vote.update_vote_properties() + logger.info(u'Recalculated vote properties for {0}'.format(vote)) From 51404b672fabf2ecf1996d137ae80a785f66edbd Mon Sep 17 00:00:00 2001 From: alonisser Date: Mon, 15 Feb 2016 09:59:36 +0200 Subject: [PATCH 6/6] commenting TODO --- laws/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/laws/models.py b/laws/models.py index 648658dd..36017789 100755 --- a/laws/models.py +++ b/laws/models.py @@ -364,6 +364,7 @@ def tag_form(self): return tf def update_vote_properties(self): + # TODO: this can be heavily optimized. somewhere sometimes.. party_ids = Party.objects.values_list('id', flat=True) d = self.time.date() party_is_coalition = dict(zip(