Merge pull request #2820 from zimmerman-team/feat/tests

Reintroduce tests for main python functionality of direct indexing

.gitignore

@@ -31,4 +31,4 @@ session.vim
 *.log
 
 # Python cache
-__pycache__
+__pycache__

README.MD

@@ -1,7 +1,7 @@
 # IATI.cloud
 
 [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=zimmerman-zimmerman_iati.cloud&metric=alert_status)](https://sonarcloud.io/dashboard?id=zimmerman-zimmerman_iati.cloud)
-[![License: AGPLv3](https://img.shields.io/badge/License-AGPL%20v3-blue.svg)](https://github.com/zimmerman-zimmerman/OIPA/blob/main/LICENSE.MD)
+[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT)
 [![Open issues](https://img.shields.io/github/issues/zimmerman-zimmerman/OIPA.svg?style=flat)](https://github.com/zimmerman-team/iati.cloud/issues)
 ---
 
@@ -95,6 +95,10 @@ test: Adding missing or correcting existing tests
 chore: Changes to the build process or auxiliary tools and libraries such as documentation generation
 ```
 
+## Testing
+We test with `pytest`,and use `coverage` to generage coverage reports.
+You can use `. scripts/cov.sh` to quickly run all tests and generate a coverage report. This also conveniently prints the location of the coverage HTML report, which can be viewed from your browser.
+
 ## Contributing
 ### Can I contribute?
 Yes! We are mainly looking for coders to help on the project. If you are a coder feel free to _Fork_ the repository and send us your amazing Pull Requests!

direct_indexing/cleaning/dataset.py

@@ -55,9 +55,8 @@ def extract_key_value_fields(data, add_fields, key, value):
             add_fields = extract_single_values(add_fields, value, key, data)
     # If the fields are not yet at the lowest level of key-value pair,
     # process the underlying field.
-    elif type(value) in [OrderedDict, list]:
+    elif type(value) in [OrderedDict, dict]:  # was list instead of dict
         data[key] = recursive_attribute_cleaning(value)
-
     return add_fields
 
 
@@ -130,7 +129,7 @@ def list_values(element, data, key, add_fields):
         data[key].append(' ')
     for string in ['@currency', '@value-date', '@year']:
         if string in element:
-            add_fields[f'{element}.{string[1:]}'].append(element[string])
+            add_fields[f'{key}.{string[1:]}'].append(element[string])
     if XML_LANG_STR in element:
         add_fields[f'{key}.{LANG_STR}'].append(
             element[XML_LANG_STR])
@@ -165,7 +164,7 @@ def extract_single_values(add_fields, value, key, data):
         data[key] = value
         return add_fields
     if type(value) is bool:
-        data[key] = 0
+        data[key] = 1 if value else 0
         return add_fields
     if '$' in value:
         data[key] = value['$']
@@ -179,6 +178,7 @@ def extract_single_values(add_fields, value, key, data):
     if XML_LANG_STR in value:
         add_fields[f'{key}.{LANG_STR}'] = value[
             XML_LANG_STR]
+    # assume the language is not provided
     else:
         if key != 'value':
             add_fields[f'{key}.{LANG_STR}'] = ' '

direct_indexing/custom_fields/currency_aggregation.py

@@ -23,6 +23,12 @@
 PDV_GBP_CURR = 'planned-disbursement.value-gbp.conversion-currency'
 TV_USD_CURR = 'transaction.value-usd.conversion-currency'
 TV_GBP_CURR = 'transaction.value-gbp.conversion-currency'
+T_TYPES = [None, "incoming-funds", "outgoing-commitment", "disbursement",
+           "expenditure", "interest-payment", "loan-repayment",
+           "reimbursement", "purchase-of-equity", "sale-of-equity",
+           "credit-guarantee", "incoming-commitment", "outgoing-pledge",
+           "incoming-pledge"]
+TT_U = [t.replace("-", "_") if t else None for t in T_TYPES]
 
 
 def currency_aggregation(data):
@@ -297,24 +303,18 @@ def process_activity_aggregations(data, activity_aggregations, activity_indexes,
     :param activity_indexes: the activity indexes
     :param aggregation_fields: the aggregation fields
     """
-    budget_agg = activity_aggregations['budget']
-    transaction_agg = activity_aggregations['transaction']
-    transaction_usd_agg = activity_aggregations['transaction-usd']
-    transaction_gbp_agg = activity_aggregations['transaction-gbp']
-    planned_disbursement_agg = activity_aggregations['planned-disbursement']
+    budget_agg = activity_aggregations.get('budget', [])
+    transaction_agg = activity_aggregations.get('transaction', [])
+    transaction_usd_agg = activity_aggregations.get('transaction-usd', [])
+    transaction_gbp_agg = activity_aggregations.get('transaction-gbp', [])
+    planned_disbursement_agg = activity_aggregations.get('planned-disbursement', [])
     # Process the aggregated data
     process_budget_agg(budget_agg, activity_indexes, aggregation_fields, data)
     process_planned_disbursement_agg(planned_disbursement_agg, activity_indexes, aggregation_fields, data)
     # Transaction types, starting with none to make array index match the transaction type code from the codelist
-    transaction_types = [None, "incoming-funds", "outgoing-commitment", "disbursement",
-                         "expenditure", "interest-payment", "loan-repayment",
-                         "reimbursement", "purchase-of-equity", "sale-of-equity",
-                         "credit-guarantee", "incoming-commitment", "outgoing-pledge",
-                         "incoming-pledge"]
-    tt_u = [t.replace("-", "_") if t else None for t in transaction_types]
-    process_transaction_agg(transaction_agg, activity_indexes, aggregation_fields, data, tt_u)
-    process_transaction_currency_agg(transaction_usd_agg, activity_indexes, aggregation_fields, data, tt_u, 'usd')
-    process_transaction_currency_agg(transaction_gbp_agg, activity_indexes, aggregation_fields, data, tt_u, 'gbp')
+    process_transaction_agg(transaction_agg, activity_indexes, aggregation_fields, data)
+    process_transaction_currency_agg(transaction_usd_agg, activity_indexes, aggregation_fields, data, 'usd')
+    process_transaction_currency_agg(transaction_gbp_agg, activity_indexes, aggregation_fields, data, 'gbp')
     return data
 
 
@@ -345,12 +345,13 @@ def get_child_aggregations(dba, aggregation_fields):
     for key in aggregation_fields:
         if "currency" not in aggregation_fields[key]:
             group_object[key] = {"$sum": f'${aggregation_fields[key]}'}
-
     # Get aggregations for all fields
     children_agg = list(dba.aggregate([
-        {MONGO_UNWIND: "$related-activity"},
+        # {MONGO_UNWIND: "$related-activity"},
+        {"$unwind": "$related-activity"},
         {"$match": {"related-activity.type": 1}},
-        {MONGO_GROUP: group_object}
+        {"$group": group_object}
+        # {MONGO_GROUP: group_object}
     ]))
     return children_agg
 
@@ -468,9 +469,9 @@ def process_budget_agg(budget_agg, activity_indexes, aggregation_fields, data):
             data[index_of_activity][aggregation_fields['budget_usd']] = data[index_of_activity]['budget.value-usd.sum']
         if 'budget.value-gbp.sum' in data[index_of_activity]:
             data[index_of_activity][aggregation_fields['budget_gbp']] = data[index_of_activity]['budget.value-gbp.sum']
+        # Get the original currency from which has been converted
         if BV_USD_CURR in data[index_of_activity]:
-            data[index_of_activity][aggregation_fields['budget_currency']] = data[index_of_activity][
-                BV_USD_CURR]
+            data[index_of_activity][aggregation_fields['budget_currency']] = data[index_of_activity][BV_USD_CURR]
 
 
 def process_planned_disbursement_agg(planned_disbursement_agg, activity_indexes, aggregation_fields, data):
@@ -494,34 +495,33 @@ def process_planned_disbursement_agg(planned_disbursement_agg, activity_indexes,
                 PDV_GBP_CURR]
 
 
-def process_transaction_agg(transaction_agg, activity_indexes, aggregation_fields, data, types):
+def process_transaction_agg(transaction_agg, activity_indexes, aggregation_fields, data):
     for agg in transaction_agg:
         # Find the index of the relevant activity
         if agg['_id'][0] not in activity_indexes:
             continue
         index_of_activity = activity_indexes[agg['_id'][0]]
         transaction_type = agg['_id'][1]
         if type(transaction_type) is int:
-            data[index_of_activity][f'{aggregation_fields[types[transaction_type]]}'] = \
+            data[index_of_activity][f'{aggregation_fields[TT_U[transaction_type]]}'] = \
                 agg['transaction-value-sum']
 
 
-def process_transaction_currency_agg(transaction_curr_agg, activity_indexes, aggregation_fields, data, types, currency):
+def process_transaction_currency_agg(transaction_curr_agg, activity_indexes, aggregation_fields, data, currency):
     for agg in transaction_curr_agg:
         if agg['_id'][0] not in activity_indexes:
             continue
         # Find the index of the relevant activity
         index_of_activity = activity_indexes[agg['_id'][0]]
         transaction_type = agg['_id'][1]
-        if not transaction_type:
+        if not transaction_type or type(transaction_type) is not int:
             continue
-        if type(transaction_type) is int:
-            data[index_of_activity][f'{aggregation_fields[types[transaction_type]]}-{currency}'] = agg[
-                f'transaction-value-{currency}-sum']
-            if f'transaction-value-{currency}-conversion-currency' in data[index_of_activity]:
-                if currency == 'gbp':
-                    selector = TV_GBP_CURR
-                else:
-                    selector = TV_USD_CURR
-                data[index_of_activity][f'{aggregation_fields[types[transaction_type]]}-currency'] = \
-                    data[index_of_activity][selector]
+        data[index_of_activity][f'{aggregation_fields[TT_U[transaction_type]]}-{currency}'] = agg[
+            f'transaction-value-{currency}-sum']
+        if f'transaction-value-{currency}-conversion-currency' in data[index_of_activity]:
+            if currency == 'gbp':
+                selector = TV_GBP_CURR
+            else:
+                selector = TV_USD_CURR
+            data[index_of_activity][f'{aggregation_fields[TT_U[transaction_type]]}-currency'] = \
+                data[index_of_activity][selector]

direct_indexing/custom_fields/currency_conversion.py

@@ -38,6 +38,9 @@ def currency_conversion(data, currencies):
 
 def convert_currencies_from_list(data, field, currencies, default_currency, value,
                                  rate, first_currency, t_type, curr_convert):
+    if field not in data:
+        return value, rate, first_currency, t_type
+
     for item in data[field]:
         c_value, c_rate, currency = convert(item, currencies,
                                             default_currency=default_currency,
@@ -56,6 +59,9 @@ def convert_currencies_from_list(data, field, currencies, default_currency, valu
 
 
 def convert_currencies_from_dict(data, field, currencies, default_currency, value, rate, t_type, curr_convert):
+    if field not in data:
+        return value, rate, "", t_type
+
     c_value, c_rate, first_currency = convert(data[field], currencies,
                                               default_currency=default_currency,
                                               target_currency=curr_convert)
@@ -123,6 +129,7 @@ def get_ym(data):
     now = datetime.datetime.now()
     if year > now.year:
         year = now.year
+        month = now.month
 
     if year == now.year and month > now.month:
         month = now.month

direct_indexing/custom_fields/date_quarters.py

@@ -96,8 +96,11 @@ def retrieve_date_quarter(date):
     The date object will always be a string object in the shape of an ISO date object,
     meaning YYYY-MM-DD
     """
-    if isinstance(date, str):
-        return ((int(date[5:7]) - 1) // 3) + 1
-    if hasattr(date, "strftime") and hasattr(date, "month"):
-        return ((date.month - 1) // 3) + 1
-    return None
+    try:
+        if isinstance(date, str):
+            return ((int(date[5:7]) - 1) // 3) + 1
+        if hasattr(date, "strftime") and hasattr(date, "month"):
+            return ((date.month - 1) // 3) + 1
+        return None
+    except Exception:
+        return None

direct_indexing/custom_fields/document_link_category_combined.py

@@ -21,11 +21,11 @@ def document_link_category_combined(data):
     data[final_field] = []
     for doc in data[dl]:
         codes = ''
+        if 'category' not in doc:
+            continue
         if type(doc['category']) is dict:
             doc['category'] = [doc['category']]
-        for category in doc['category']:
-            if codes != '':
-                codes += ','
-            codes += category["code"]
-        data[final_field].append(codes)
+        codes = ",".join(category["code"] for category in doc.get("category", []) if "code" in category)
+        if codes != '':
+            data[final_field].append(codes)
     return data

direct_indexing/custom_fields/indexing_manytomany_relations.py

@@ -7,13 +7,13 @@ def index_many_to_many_relations(activity):
     # Index result indicator, starting with baseline:
     # An indicator has 0 to N baselines, if 0, represent with index -1, else represent with index n.
     if 'result' in activity:
-        if type(activity['result']) != list:
+        if not isinstance(activity['result'], list):
             activity['result'] = [activity['result']]
         for result in activity['result']:
             add_result_child_indexes(result, 'indicator')
     # Index participating organisations.
     if 'participating-org' in activity:
-        if type(activity['participating-org']) != list:
+        if not isinstance(activity['participating-org'], list):
             activity['participating-org'] = [activity['participating-org']]
         add_participating_org_child_indexes(activity, 'participating-org')
 
@@ -47,7 +47,7 @@ def add_result_child_indexes(field, child):
     # Check if the child exists and make the child a list if it is a dict.
     if child not in field:
         return
-    if type(field[child]) != list:
+    if not isinstance(field[child], list):
         field[child] = [field[child]]
 
     add_field_child_field_indexes(field, child, 'baseline')
@@ -77,7 +77,7 @@ def add_field_child_field_indexes(data, target_field, field):
             continue
 
         # make sure the baseline is a list of baselines.
-        if type(target[field]) != list:
+        if not isinstance(target[field], list):
             target[field] = [target[field]]
 
         field_index = total_field
@@ -104,7 +104,7 @@ def add_field_child_field_children_indexes(data, target_field, field, children):
         for target in data[target_field]:
             if field in target:
                 # If the second level child is found, loop over this and check if the third level children are found.
-                if type(target[field]) != list:
+                if not isinstance(target[field], list):
                     target[field] = [target[field]]
                 iterate_third_level_children(child, data, field, target, target_field, total_field)
     return data
@@ -117,11 +117,12 @@ def iterate_third_level_children(child, data, field, target, target_field, total
     Use enumerate to only save the index of for the first occurrence.
     """
     for item in target[field]:
-        if type(item) != dict:
+        if not isinstance(item, dict):
             field_index = -1
         elif child in item:
             field_index = total_field
-            if type(item[child]) != list:
+            print(item[child])
+            if not isinstance(item[child], list):
                 total_field += 1
             else:
                 total_field += len(item[child])

direct_indexing/custom_fields/models/currencies.py

@@ -66,23 +66,26 @@ def convert_currency(self, source, target, value, month, year):
         :param year: int: the year
         :return: the converted value and the exchange rate from source to target
         """
-        if None in (source, target, value, month, year):
-            return None, None
+        try:
+            if None in (source, target, value, month, year):
+                return None, None
 
-        if source == target:
-            return value, 1  # 1 on 1 relation
+            if source == target:
+                return value, 1  # 1 on 1 relation
 
-        source_conversion = self.get_currency(month, year, source)
-        target_conversion = self.get_currency(month, year, target)
-        if not source_conversion or not target_conversion:
-            return None, None
-        source_to_xdr_rate = source_conversion['value']
-        xdr_to_target_rate = target_conversion['value']
+            source_conversion = self.get_currency(month, year, source)
+            target_conversion = self.get_currency(month, year, target)
+            if not source_conversion or not target_conversion:
+                return None, None
+            source_to_xdr_rate = source_conversion['value']
+            xdr_to_target_rate = target_conversion['value']
 
-        converted_value = value * source_to_xdr_rate
-        if target == 'XDR':
-            return converted_value, source_to_xdr_rate
+            converted_value = value * source_to_xdr_rate
+            if target == 'XDR':
+                return converted_value, source_to_xdr_rate
 
-        exchange_rate = xdr_to_target_rate / source_to_xdr_rate
+            exchange_rate = xdr_to_target_rate / source_to_xdr_rate
 
-        return converted_value / xdr_to_target_rate, exchange_rate
+            return converted_value / xdr_to_target_rate, exchange_rate
+        except TypeError:
+            return None, None

direct_indexing/custom_fields/organisation_custom_fields.py

@@ -25,7 +25,7 @@ def index_many_to_many_relations(organisation):
     # if 0, represent with index -1, else represent with index n.
     TE = 'total-expenditure'
     if TE in organisation:
-        if type(organisation[TE]) != list:
+        if not isinstance(organisation[TE], list):
             organisation[TE] = [organisation[TE]]
         index_total_expenditure(organisation, TE)
 
@@ -35,12 +35,7 @@ def index_total_expenditure(organisation, field):
     Go through the activity participating orgs and index the given child.
     Because this is currently used for results, we directly pass the required children.
 
-    :param field: a dataset containing the initial child of the activity
-    :param child: the second level child of the aforementioned field
-    """
-    # Check if the child exists and make the child a list if it is a dict.
-    # total-expenditure.value.currency
-    """
+    Total Expenditure
     0..*    total-expenditure
     1..1        period-start
     1..1        period-end
@@ -69,7 +64,11 @@ def index_total_expenditure(organisation, field):
         for every expense line, how many children value and ref are there
 
     -1 indicates there is no ref
+
+    :param field: a dataset containing the initial child of the activity
+    :param child: the second level child of the aforementioned field
     """
+    # Check if the child exists and make the child a list if it is a dict.
     EL_STR = 'expense-line'
     organisation['total-expenditure.expense-line-index'] = []
     organisation['total-expenditure.expense-line.ref-index'] = []