Merge branch 'main' into diet_set_fix

README.md

@@ -5,16 +5,32 @@ MammalBase is a database of recent mammals. The main focus of this database is t
 
 # Documentation
 
-[Instructions](documentation/instructions.md)
+Documentation has been compiled behind the documentation directory. There, the structure of the code, Django model objects used in the code, and, for example, the logic of import and export operations are explained. Various scripts related to the use of MammalBase are also described in their own documents.
 
-[Architecture](documentation/architecture.md)
+Below are a few direct links to key instructions.
 
-[Data Architecture](documentation/data-architecture.md)
+[Instructions](documentation/common/instructions.md)
+- Setting up the development environment and other important aspects in development work
+
+[Architecture](documentation/mammalbase/arch/architecture.md)
+- Overview of the software architecture
+
+[Data Architecture](documentation/mammalbase/arch/data-architecture.md)
+- Overview of data architecture
 
 [Master Habitats Script](documentation/scripts/master_habitat_scripts.md)
+- Retrieval of Master Habitats using a script
+
+[Features](documentation/mammalbase/features/)
+- Current features of MammalBase (e.g. importing and exporting tsv-files)
+
+[Models](documentation/mammalbase/models/)
+- Models used in MammalBase
 
 ## Testing
-[Testing Guide](documentation/testing.md)
+[Testing Guide](documentation/common/testing.md)
+- Instructions for testing the software
 
 ## Using Celery to run tasks in the background
-[Celery instructions](documentation/celery.md)
+[Celery instructions](documentation/common/celery.md)
+- Background tasks in the software are executed with the Celery library.

app/exports/query_sets/base_query.py

@@ -23,8 +23,11 @@ def base_query(measurement_choices):
             | Q(source_unit__master_unit__is_active=False)
     ) #filtteröi onko tiettyjen kysymysten arvot active vai non-active
 
-    #  measurement_choices is a list of user choices made in forms.py/views.py.  
-    query_filter_list = [Q(source_attribute__master_attribute__attributegrouprelation__group__name=value) for value in measurement_choices]
+    #  measurement_choices is a list of user choices made in forms.py/views.py.
+    query_filter_list = [
+        Q(
+            source_attribute__master_attribute__attributegrouprelation__group__name=value
+        ) for value in measurement_choices]
     measurement_choice_filter = Q()
     for query_filter in query_filter_list:
         measurement_choice_filter |= query_filter
@@ -65,4 +68,4 @@ def base_query(measurement_choices):
         | Q(source_attribute__reference__status=3)
     ).filter(measurement_choice_filter)
 
-    return query_filter
+    return query_filter

app/exports/query_sets/measurement_or_fact_query.py

@@ -37,6 +37,20 @@ def measurement_or_fact_query(
         | Q(source_entity__reference__is_active=False)
         | Q(source_entity__reference__master_reference__is_active=False)
         | Q(source_statistic__is_active=False)
+
+    )
+
+    nominal_non_active = (
+          Q(source_entity__master_entity__entity__is_active=False)
+        | Q(source_entity__reference__is_active=False)
+        | Q(source_entity__reference__master_reference__is_active=False)
+        | Q(source_choiceset_option__source_attribute__master_attribute__id=None)
+        | Q(source_choiceset_option__source_attribute__master_attribute__name='- Checked, Unlinked -')
+        | Q(source_choiceset_option__source_attribute__master_attribute__name__exact='')
+        | Q(source_entity__master_entity__name__exact='')
+        | Q(source_entity__master_entity__id__isnull=True)
+        | Q(source_choiceset_option__source_attribute__reference__status=1)
+        | Q(source_choiceset_option__source_attribute__reference__status=3)
     )
 
     now = datetime.now(tz=timezone(timedelta(hours=2)))
@@ -69,9 +83,9 @@ def measurement_or_fact_query(
                 default = mb_reference
         )
 
-    nominal_query = SourceChoiceSetOptionValue.objects.annotate(
+    nominal_query = SourceChoiceSetOptionValue.objects.exclude(nominal_non_active).annotate(
         entity_id=Concat(
-            Value('http://localhost:8000/sav/'),
+            Value('https://www.mammalbase.net/sav/'),
             'id',
             Value('/'),
             output_field=CharField()
@@ -80,6 +94,10 @@ def measurement_or_fact_query(
         basis_of_record_description=F('source_entity__reference__master_reference__type'),
         references=references,
         measurement_resolution=Case(
+            When(
+                source_entity__master_entity__entity__name__exact=None,
+                then=Value('NA')
+            ),
             When(
                 source_entity__master_entity__entity__name__iendswith='species',
                 then=Value('NA')
@@ -94,6 +112,10 @@ def measurement_or_fact_query(
                 source_choiceset_option__source_attribute__method__name__exact=None,
                 then=Value('NA')
             ),
+            When(
+                source_choiceset_option__source_attribute__method__name__exact='nan',
+                then=Value('NA')
+            ),
             default='source_choiceset_option__source_attribute__method__name',
             output_field=CharField()
         ),
@@ -104,7 +126,9 @@ def measurement_or_fact_query(
         individual_count=Value('NA'),
         dispersion=Value('NA'),
         measurement_value_min=Value('NA'),
-        measurement_value_max=Value('NA')
+        measurement_value_max=Value('NA'),
+        measurement_accuracy=Value('NA'),
+        statistical_method=Value('NA')
     )
 
 
@@ -140,9 +164,16 @@ def measurement_or_fact_query(
         measurement_remarks=Case(
             When(
                 remarks__exact=None,
-                then=Value('NA')
+                then=Concat(
+                    Value('Data quality score '),
+                    'data_quality_score',
+                    Value('/10'))
             ),
-            default='remarks',
+            default=Concat(
+                Value('Data quality score '),
+                'data_quality_score',
+                Value("/10 | "),
+                'remarks'),
             output_field=CharField()
         ),
         aggregate_measure=Case(
@@ -181,7 +212,7 @@ def measurement_or_fact_query(
     if measurement_choice == "Nominal traits":
         # TODO: Find fields in query and export to spreadsheet here
         fields = [
-            ('entity_id','traitID'),
+            ('entity_id','measurementID'),
             ('basis_of_record', 'basisOfRecord'),
             ('source_entity__reference__master_reference__type', 'basisOfRecordDescription'),
             ('references', 'references'),
@@ -194,7 +225,9 @@ def measurement_or_fact_query(
             ('individual_count', 'individualCount'),
             ('dispersion', 'dispersion'),
             ('measurement_value_min', 'measurementValue_min'),
-            ('measurement_value_max', 'measurementValue_max')
+            ('measurement_value_max', 'measurementValue_max'),
+            ('measurement_accuracy', 'measurementAccuracy'),
+            ('statistical_method', 'statisticalMethod')
         ]
         query = nominal_query
     elif measurement_choice in ('External measurements', 'Cranial measurements'):

app/exports/query_sets/metadata_query.py

@@ -27,12 +27,10 @@ def metadata_query(measurement_choices):
 
     query = base.exclude(non_active).annotate(
         dataset_id=Value('https://urn.fi/urn:nbn:fi:att:8dce459f-1401-4c6a-b2bb-c831bd8d3d6f'),
-        dataset_name=Value(
-            'MammalBase — Dataset 03: Trait Data in Ecological Trait-data Standard (ETS) format'),
+        dataset_name=Value('MammalBase — Dataset 03: Trait Data in Ecological Trait-data Standard (ETS) format'),
         dataset_description=Value(
             'MammalBase - www.mammalbase.net: '
-            'Trait dataset output in Ecological Trait-data Standard (ETS)'
-        ),
+            'Trait dataset output in Ecological Trait-data Standard (ETS)'),
         orcid_uid=Subquery(
             SocialAccount.objects.filter(
                 user_id=OuterRef('created_by__id')
@@ -54,9 +52,8 @@ def metadata_query(measurement_choices):
             'URL: https://doi.org/10.5281/zenodo.1485739')
         ),
         rights_holder=Value(
-            'Lintulaakso, ',
-            'Kari;https://orcid.org/0000-0001-9627-8821;Finnish Museum of Natural History LUOMUS'
-        ),
+            'Lintulaakso, '
+            'Kari;https://orcid.org/0000-0001-9627-8821;Finnish Museum of Natural History LUOMUS'),
         rights=Value('Attribution 4.0 International (CC BY 4.0)'),
         licence=Value('CC BY 4.0')
     ).annotate(
@@ -69,12 +66,10 @@ def metadata_query(measurement_choices):
 
     nominal_query = SourceChoiceSetOptionValue.objects.exclude(non_active).annotate(
         dataset_id=Value('https://urn.fi/urn:nbn:fi:att:8dce459f-1401-4c6a-b2bb-c831bd8d3d6f'),
-        dataset_name=Value(
-            'MammalBase — Dataset 03: Trait Data in Ecological Trait-data Standard (ETS) format'),
+        dataset_name=Value('MammalBase — Dataset 03: Trait Data in Ecological Trait-data Standard (ETS) format'),
         dataset_description=Value(
             'MammalBase - www.mammalbase.net: '
-            'Trait dataset output in Ecological Trait-data Standard (ETS)'
-        ),
+            'Trait dataset output in Ecological Trait-data Standard (ETS)'),
         orcid_uid=Subquery(
             SocialAccount.objects.filter(
                 user_id=OuterRef('created_by__id')
@@ -97,8 +92,7 @@ def metadata_query(measurement_choices):
         ),
         rights_holder=Value(
             'Lintulaakso, '
-            'Kari;https://orcid.org/0000-0001-9627-8821;Finnish Museum of Natural History LUOMUS'
-        ),
+            'Kari;https://orcid.org/0000-0001-9627-8821;Finnish Museum of Natural History LUOMUS'),
         rights=Value('Attribution 4.0 International (CC BY 4.0)'),
         licence=Value('CC BY 4.0')
     ).annotate(
@@ -110,7 +104,6 @@ def metadata_query(measurement_choices):
     ).order_by('author').distinct()
 
     fields = [
-        ('id', 'hmmm'),
         ('dataset_id', 'datasetID'),
         ('dataset_name', 'datasetName'),
         ('dataset_description', 'datasetDescription'),
@@ -128,10 +121,7 @@ def metadata_query(measurement_choices):
     if "Nominal traits" in measurement_choices:
         queries.append((nominal_query, fields))
 
-    if ("Cranial measurements" in
-        measurement_choices or
-        "External measurements" in
-        measurement_choices):
+    if "Cranial measurements" in measurement_choices or "External measurements" in measurement_choices:
         queries.append((query, fields))
 
     return queries

app/exports/query_sets/occurrence_query.py

@@ -1,4 +1,4 @@
-from django.db.models import Value, Case, When, CharField, Q
+from django.db.models import Value, Q
 
 from mb.models import SourceChoiceSetOptionValue
 from .base_query import base_query

app/exports/query_sets/taxon_query.py

@@ -16,6 +16,7 @@ def taxon_query(measurement_choices):
 
     non_active = (
             Q(source_entity__master_entity__entity__is_active=False)
+          | Q(source_entity__master_entity__id=None)
     )
 
     query = base.exclude(non_active).annotate(

app/exports/query_sets/traitdata_query.py

@@ -19,6 +19,23 @@ def traitdata_query(measurement_choices):
             Q(source_attribute__master_attribute__unit__is_active=False)
     )
 
+    nominal_non_active = (
+            Q(source_choiceset_option__source_attribute__master_attribute__unit__is_active=False)
+            | Q(source_choiceset_option__source_attribute__master_attribute__id=None)
+            | Q(source_choiceset_option__source_attribute__master_attribute__name='- Checked, Unlinked -')
+            | Q(source_choiceset_option__source_attribute__master_attribute__name__exact='')
+            | Q(source_entity__master_entity__name__exact='')
+            | Q(source_entity__master_entity__id__isnull=True)
+            | Q(source_choiceset_option__source_attribute__reference__status=1)
+            | Q(source_choiceset_option__source_attribute__reference__status=3)
+            | Q(source_choiceset_option__master_choiceset_option__name=None)
+    )
+    master_attribute_filter = (
+        Q(source_choiceset_option__master_choiceset_option__master_attribute_id =
+          F('source_choiceset_option__source_attribute__master_attribute__id')
+        )
+    )
+
     query = base.exclude(non_active).annotate(
         trait_id=Concat(
             Value('https://www.mammalbase.net/ma/'),
@@ -64,7 +81,8 @@ def traitdata_query(measurement_choices):
         'source_attribute__master_attribute__name'
     )
 
-    nominal_query = SourceChoiceSetOptionValue.objects.annotate(
+    nominal_query = SourceChoiceSetOptionValue.objects.filter(
+            master_attribute_filter).exclude(nominal_non_active).annotate(
         trait_id=Concat(
             Value('https://www.mammalbase.net/ma/'),
             'source_choiceset_option__source_attribute__master_attribute__id',
@@ -81,7 +99,12 @@ def traitdata_query(measurement_choices):
             Value('/'),
             output_field=CharField()
         ),
-        measurement_id=Value('NA'),
+        measurement_id=Concat(
+            Value('https://www.mammalbase.net/sav/'),
+            'id',
+            Value('/'),
+            output_field=CharField()
+        ),
         occurrence_id=Value('NA'),
         warnings=Value('NA'),
     ).order_by(

app/exports/query_sets/traitlist_query.py

@@ -1,7 +1,7 @@
-from django.db.models import Value, CharField, Q
+from django.db.models import Value, CharField, Q, Case, When
 from django.db.models.functions import Concat, Replace
 
-from mb.models import SourceChoiceSetOptionValue
+from mb.models import MasterAttribute
 from .base_query import base_query
 
 
@@ -14,11 +14,13 @@ def traitlist_query(measurement_choices):
     """
     base = base_query(measurement_choices)
 
+
     non_active = (
               Q(source_attribute__master_attribute__unit__is_active=False)
             | Q(source_attribute__master_attribute__reference__is_active=False)
     )
 
+
     query = base.exclude(non_active).annotate(
         identifier=Concat(
             Value('https://www.mammalbase.net/ma/'),
@@ -39,30 +41,44 @@ def traitlist_query(measurement_choices):
         'source_attribute__master_attribute__attributegrouprelation__display_order'
     ).distinct()
 
-    nominal_query = SourceChoiceSetOptionValue.objects.annotate(
+    attributelink = {}
+    attributes = MasterAttribute.objects.filter(groups__name='Nominal traits')
+    for i in attributes:
+        for j in i.masterchoicesetoption_set.all():
+            try:
+                attributelink[i.name].append(j.name)
+            except:
+                attributelink[i.name]=[j.name]
+
+    nominal_query = MasterAttribute.objects.prefetch_related(
+            'master_choiceset_option_set').filter(groups__name='Nominal traits').annotate(
         identifier=Concat(
             Value('https://www.mammalbase.net/ma/'),
-            'source_choiceset_option__source_attribute__master_attribute__id',
+            'id',
             Value('/'),
             output_field=CharField()
         ),
         trait=Replace(
-            'source_choiceset_option__source_attribute__master_attribute__name',
+            'name',
             Value(' '),
             Value('_')
         ),
         narrowerTerm=Value('NA'),
         relatedTerm=Value('NA'),
+        factorLevels=Case(
+            *[
+            When(name=attribute_name, then=Value(', '.join(attributelink[attribute_name]))) for attribute_name in attributelink
+            ],output_field=CharField()
+        ),
         broaderTerm=Value('NA'),
         expectedUnit=Value('NA'),
-        max_allowed_value=Value('NA'),
-        min_allowed_value=Value('NA'),
         comments=Value('NA'),
     ).order_by(
-        'source_choiceset_option__source_attribute__master_attribute__groups__name',
-        'source_choiceset_option__source_attribute__master_attribute__attributegrouprelation__display_order'
+        'groups__name',
+        'attributegrouprelation__display_order'
     ).distinct()
 
+
     fields = [
         ('identifier', 'identifier'),
         ('trait', 'trait'),
@@ -79,20 +95,21 @@ def traitlist_query(measurement_choices):
         ('source_attribute__master_attribute__reference__citation', 'source')
     ]
 
+
     nominal_fields = [
         ('identifier', 'identifier'),
         ('trait', 'trait'),
         ('broaderTerm', 'broaderTerm'),
         ('narrowerTerm', 'narrowerTerm'),
         ('relatedTerm', 'relatedTerm'),
-        ('source_choiceset_option__source_attribute__master_attribute__value_type', 'valueType'),
+        ('value_type', 'valueType'),
         ('expectedUnit', 'expectedUnit'),
-        ('source_choiceset_option__master_choiceset_option__name', 'factorLevels'),
+        ('factorLevels', 'factorLevels'),
         ('max_allowed_value', 'maxAllowedValue'),
         ('min_allowed_value', 'minAllowedValue'),
-        ('source_choiceset_option__source_attribute__master_attribute__description', 'traitDescription'),
+        ('description', 'traitDescription'),
         ('comments', 'comments'),
-        ('source_choiceset_option__source_attribute__master_attribute__reference__citation', 'source')
+        ('reference__citation', 'source')
     ]
 
     queries = []