Merge pull request #405 from codeforIATI/404-empty-participating-org-…

…narratives

Organisation narratives can sometimes be empty

iati_datastore/iatilib/parse.py

@@ -103,17 +103,17 @@ def xpath_decimal(xpath, xml, resource=None, major_version='1'):
         return None
 
 
-def xvals_lang(xml, major_version, default_lang="default"):
+def xvals_lang(xml, major_version, default_lang="default", default=NODEFAULT):
      ret = {}
      if major_version == '1':
          for ele in xml.xpath("."):
              lang = xval(ele, "@xml:lang", default_lang)
-             value = xval(ele, "text()")
+             value = xval(ele, "text()", default)
              ret[lang] = value
      else:
          for ele in xml.xpath("./narrative"):
              lang = xval(ele, "@xml:lang", default_lang)
-             value = xval(ele, "text()")
+             value = xval(ele, "text()", default)
              ret[lang] = value
      return ret
 
@@ -122,7 +122,7 @@ def parse_org(xml, resource=no_resource, major_version='1', default_lang="defaul
     data = {
         "ref": xval(xml, "@ref", u""),
         "name": xval(xml, TEXT_ELEMENT[major_version], u""),
-        "name_all_values": xvals_lang(xml, TEXT_ELEMENT[major_version], default_lang=default_lang)
+        "name_all_values": xvals_lang(xml, TEXT_ELEMENT[major_version], default_lang=default_lang, default=u"")
     }
     try:
         data['type'] = codelists.by_major_version[major_version].OrganisationType.from_string(xval(xml, "@type"))

iati_datastore/iatilib/test/fixtures/activity_with_empty_participating_org_narrative.xml

@@ -0,0 +1,85 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<iati-activities version="2.03">
+ <iati-activity default-currency="SEK" dstore:dataset="sida-200" dstore:index="3299" hierarchy="2" humanitarian="0" iati-activities:version="2.03" xml:lang="EN" xmlns:dstore="http://d-portal.org/xmlns/dstore" xmlns:iati-activities="http://d-portal.org/xmlns/iati-activities">
+  <iati-identifier>SE-0-SE-6-SE-0-SE-6-71001016-LAO-21020-1</iati-identifier>
+  <reporting-org ref="SE-0" secondary-reporter="0" type="10">
+   <narrative xml:lang="EN">Sweden</narrative>
+  </reporting-org>
+  <title>
+   <narrative xml:lang="EN">Rehabilitering road 8, Laos</narrative>
+   <narrative xml:lang="SV">Rehabilitering road 8, Laos</narrative>
+  </title>
+  <description type="1">
+   <narrative xml:lang="EN">Rehabilitering road 8, Laos</narrative>
+   <narrative xml:lang="SV">Rehabilitering road 8, Laos</narrative>
+  </description>
+  <participating-org ref="SE-0" role="1" type="10">
+   <narrative xml:lang="EN">Sweden</narrative>
+  </participating-org>
+  <participating-org ref="SE-6" role="3" type="10">
+   <narrative xml:lang="EN">Swedish International Development Cooperation Agency</narrative>
+  </participating-org>
+  <participating-org ref="XM-DAC-12000" role="4" type="10">
+   <narrative xml:lang="EN"></narrative>
+  </participating-org>
+  <activity-status code="4"></activity-status>
+  <activity-date iso-date="2001-01-01" type="1"></activity-date>
+  <activity-date iso-date="2001-01-01" type="2"></activity-date>
+  <activity-date iso-date="2020-12-31" type="3"></activity-date>
+  <activity-date iso-date="2020-12-31" type="4"></activity-date>
+  <collaboration-type code="1"></collaboration-type>
+  <default-finance-type code="421"></default-finance-type>
+  <default-aid-type code="C01" vocabulary="1"></default-aid-type>
+  <default-tied-status code="4"></default-tied-status>
+  <capital-spend percentage="0"></capital-spend>
+  <transaction>
+   <transaction-type code="2"></transaction-type>
+   <transaction-date iso-date="2001-01-01"></transaction-date>
+   <value currency="SEK" value-date="2001-01-01">-22496000</value>
+   <sector code="210" vocabulary="2"></sector>
+   <sector code="21020" vocabulary="1"></sector>
+   <recipient-country code="LA"></recipient-country>
+  </transaction>
+  <transaction>
+   <transaction-type code="3"></transaction-type>
+   <transaction-date iso-date="2016-01-28"></transaction-date>
+   <value currency="SEK" value-date="2016-01-28">-5000000</value>
+   <sector code="210" vocabulary="2"></sector>
+   <sector code="21020" vocabulary="1"></sector>
+   <recipient-country code="LA"></recipient-country>
+  </transaction>
+  <transaction>
+   <transaction-type code="3"></transaction-type>
+   <transaction-date iso-date="2017-01-28"></transaction-date>
+   <value currency="SEK" value-date="2017-01-28">-5000000</value>
+   <sector code="210" vocabulary="2"></sector>
+   <sector code="21020" vocabulary="1"></sector>
+   <recipient-country code="LA"></recipient-country>
+  </transaction>
+  <transaction>
+   <transaction-type code="3"></transaction-type>
+   <transaction-date iso-date="2018-01-28"></transaction-date>
+   <value currency="SEK" value-date="2018-01-28">-5000000</value>
+   <sector code="210" vocabulary="2"></sector>
+   <sector code="21020" vocabulary="1"></sector>
+   <recipient-country code="LA"></recipient-country>
+  </transaction>
+  <transaction>
+   <transaction-type code="3"></transaction-type>
+   <transaction-date iso-date="2019-01-28"></transaction-date>
+   <value currency="SEK" value-date="2019-01-28">-5000000</value>
+   <sector code="210" vocabulary="2"></sector>
+   <sector code="21020" vocabulary="1"></sector>
+   <recipient-country code="LA"></recipient-country>
+  </transaction>
+  <transaction>
+   <transaction-type code="3"></transaction-type>
+   <transaction-date iso-date="2020-01-28"></transaction-date>
+   <value currency="SEK" value-date="2020-01-28">-2496000</value>
+   <sector code="210" vocabulary="2"></sector>
+   <sector code="21020" vocabulary="1"></sector>
+   <recipient-country code="LA"></recipient-country>
+  </transaction>
+  <related-activity ref="SE-0-SE-6-SE-0-SE-6-71001016-LAO-21020" type="1"></related-activity>
+ </iati-activity>
+</iati-activities>

iati_datastore/iatilib/test/test_parser_bugs.py

@@ -0,0 +1,72 @@
+import csv
+import datetime
+from decimal import Decimal
+from unittest import TestCase
+
+import mock
+from lxml import etree as ET
+
+from iatilib.test import db, AppTestCase, fixture_filename
+from iatilib import parse, codelists as cl
+from iatilib import model
+from iatilib.currency_conversion import update_exchange_rates
+
+cl2 = cl.by_major_version['2']
+
+def read_fixture(fix_name, encoding='utf-8'):
+    """Read and convert fixture from csv file"""
+    return csv.reader(open(fixture_filename(fix_name)).read().strip().split("\n"), delimiter=',')
+
+def fixture(fix_name, encoding='utf-8'):
+    return open(fixture_filename(fix_name), encoding=encoding).read()
+
+
+def parse_fixture(fix_name, encoding='utf-8'):
+    return ET.parse(fixture_filename(fix_name))
+
+
+class TestParseActivityParticipatingOrgs(AppTestCase):
+    def setUp(self):
+        super().setUp()
+        self.activities = list(parse.document_from_file(fixture_filename("activity_with_empty_participating_org_narrative.xml")))
+        self.act = self.activities[0]
+
+    def test_participating_org(self):
+        self.assertEquals(
+            cl2.OrganisationRole.funding,
+            self.act.participating_orgs[0].role)
+        self.assertEquals(
+            cl2.OrganisationType.government,
+            self.act.participating_orgs[0].organisation.type)
+        self.assertEquals(
+            u"SE-0",
+            self.act.participating_orgs[0].organisation.ref)
+        self.assertEquals(
+            u"Sweden",
+            self.act.participating_orgs[0].organisation.name)
+
+        self.assertEquals(
+            cl2.OrganisationRole.extending,
+            self.act.participating_orgs[1].role)
+        self.assertEquals(
+            cl2.OrganisationType.government,
+            self.act.participating_orgs[1].organisation.type)
+        self.assertEquals(
+            u"SE-6",
+            self.act.participating_orgs[1].organisation.ref)
+        self.assertEquals(
+            u"Swedish International Development Cooperation Agency",
+            self.act.participating_orgs[1].organisation.name)
+
+        self.assertEquals(
+            cl2.OrganisationRole.implementing,
+            self.act.participating_orgs[2].role)
+        self.assertEquals(
+            cl2.OrganisationType.government,
+            self.act.participating_orgs[2].organisation.type)
+        self.assertEquals(
+            u"XM-DAC-12000",
+            self.act.participating_orgs[2].organisation.ref)
+        self.assertEquals(
+            u"",
+            self.act.participating_orgs[2].organisation.name)