feat: Add reports for DDDay.

Author: Nico-AP

.gitignore

@@ -26,4 +26,4 @@ staticfiles/
 # Ignore digital_meal folder
 digital_meal/
 
-*/temp
+temp/

config/settings/base.py

@@ -194,8 +194,23 @@
 
 # Reports
 # ------------------------------------------------------------------------------
-POLITICS_KEY_INSTAGRAM = os.getenv('POLITICS_KEY_INSTAGRAM', None)
-POLITICS_KEY_FACEBOOK = os.getenv('POLITICS_KEY_FACEBOOK', None)
-SEARCH_KEY = os.getenv('SEARCH_KEY', None)
-DIGITAL_MEAL_KEY = os.getenv('DIGITAL_MEAL_KEY', None)
-CHATGPT_KEY = os.getenv('CHATGPT_KEY', None)
+# Instagram Report
+INSTAGRAM_PROJECT_PK = os.getenv('INSTAGRAM_PROJECT_PK', None)
+INSTAGRAM_API_KEY = os.getenv('INSTAGRAM_API_KEY', None)
+BP_ID_FOLLOWED_ACCOUNTS = os.getenv('BP_ID_FOLLOWED_ACCOUNTS', None)
+
+# Facebook Report
+FACEBOOK_PROJECT_PK = os.getenv('FACEBOOK_PROJECT_PK', None)
+FACEBOOK_API_KEY = os.getenv('FACEBOOK_API_KEY', None)
+
+# Search Report
+SEARCH_PROJECT_PK = os.getenv('SEARCH_PROJECT_PK', None)
+SEARCH_API_KEY = os.getenv('SEARCH_API_KEY', None)
+
+# Digital Meal Report
+DIGITALMEAL_PROJECT_PK = os.getenv('DIGITALMEAL_PROJECT_PK', None)
+DIGITALMEAL_API_KEY = os.getenv('DIGITALMEAL_API_KEY', None)
+
+# ChatGPT Report
+CHATGPT_PROJECT_PK = os.getenv('CHATGPT_PROJECT_PK', None)
+CHATGPT_API_KEY = os.getenv('CHATGPT_API_KEY', None)

env.example

@@ -7,8 +7,23 @@ DJANGO_DB_USER=''
 DJANGO_DB_PW=''
 
 # Reports
-POLITICS_KEY_INSTAGRAM=''
-POLITICS_KEY_FACEBOOK=''
-SEARCH_KEY=''
-DIGITAL_MEAL_KEY=''
-CHATGPT_KEY=''
+## Instagram Report
+INSTAGRAM_PROJECT_PK=''
+INSTAGRAM_API_KEY=''
+BP_ID_FOLLOWED_ACCOUNTS=''
+
+## Facebook Report
+FACEBOOK_PROJECT_PK=''
+FACEBOOK_API_KEY=''
+
+## Search Report
+SEARCH_PROJECT_PK=''
+SEARCH_API_KEY=''
+
+## Digital Meal Report
+DIGITALMEAL_PROJECT_PK=''
+DIGITALMEAL_API_KEY=''
+
+## ChatGPT Report
+CHATGPT_PROJECT_PK=''
+CHATGPT_API_KEY=''

reports/migrations/0001_initial.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.2.17 on 2024-08-30 13:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='InstagramStatistics',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=30)),
+                ('follow_counts', models.JSONField(default=None, null=True)),
+                ('biodiversity_counts', models.JSONField(default=None, null=True)),
+                ('pension_counts', models.JSONField(default=None, null=True)),
+                ('party_counts', models.JSONField(default=None, null=True)),
+                ('social_media_use', models.JSONField()),
+                ('last_updated', models.DateTimeField(blank=True, null=True)),
+                ('project_pk', models.IntegerField(default=0)),
+            ],
+        ),
+    ]

reports/migrations/0002_alter_instagramstatistics_social_media_use.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.17 on 2024-08-30 13:31
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('reports', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='instagramstatistics',
+            name='social_media_use',
+            field=models.JSONField(default=None, null=True),
+        ),
+    ]

reports/models.py

@@ -6,10 +6,15 @@
 from ddm.models.core import DonationProject, QuestionnaireResponse, DataDonation, DonationBlueprint
 from ddm.models.serializers import ResponseSerializer, DonationSerializer
 
+from .utils import insta_data
+
 
 class InstagramStatistics (models.Model):
     name = models.CharField(max_length=30)
 
+    # Follow counts
+    follow_counts = models.JSONField(default=None, null=True)
+
     # Vote counts
     biodiversity_counts = models.JSONField(default=None, null=True)
     pension_counts = models.JSONField(default=None, null=True)
@@ -18,11 +23,41 @@ class InstagramStatistics (models.Model):
     party_counts = models.JSONField(default=None, null=True)
 
     # Use counts
-    social_media_use = models.JSONField()
+    social_media_use = models.JSONField(default=None, null=True)
 
     last_updated = models.DateTimeField(null=True, blank=True)
     project_pk = models.IntegerField(default=0)
 
+    def update_statistics(self):
+        new_responses = self.get_responses()
+        new_donations = self.get_blueprint_donations(settings.BP_ID_FOLLOWED_ACCOUNTS)
+
+        self.update_followed_accounts(new_donations)
+        self.update_vote_counts(new_responses)  # bio & pension
+        self.update_party_graphs(new_responses, new_donations)
+        self.last_updated = datetime.now()
+        self.save()
+
+    def update_followed_accounts(self, donations=None, bp_pk=None):
+        if donations is None:
+            donations = self.get_blueprint_donations(settings.BP_ID_FOLLOWED_ACCOUNTS)
+
+        insta_accounts = insta_data.load_political_account_list()
+        results = []
+        for p, d in donations.items():
+            data = {'Gefolgte Kanäle Instagram': [d]}
+            followed_accounts = insta_data.get_follows_insta(data, insta_accounts)
+            if followed_accounts:
+                results.append(followed_accounts.copy())
+
+        if self.follow_counts is None:
+            self.follow_counts = insta_data.TYPES_DICT_PLACEHOLDER.copy()
+
+        for r in results:
+            for k in r.keys():
+                self.follow_counts[k].append(len(r[k]))
+        return
+
     def update_vote_counts(self, responses=None):
         if responses is None:
             responses = self.get_responses()
@@ -51,10 +86,13 @@ def update_bio_count(self, responses, result_dummy, value_map):
             result = self.biodiversity_counts.copy()
 
         var = 'vote-1'
-        for response in responses:
-            # TODO: Add check that participant has answered question; otherwise skip
-            vote = response[var]
-            result[value_map[vote]] += 1
+        for p, r in responses.items():
+            if var in r.keys():
+                vote = r[var]
+            else:
+                continue
+            if vote in value_map.keys():
+                result[value_map[vote]] += 1
         self.biodiversity_counts = result
         return
 
@@ -65,10 +103,13 @@ def update_pension_count(self, responses, result_dummy, value_map):
             result = self.pension_counts.copy()
 
         var = 'vote-2'
-        for response in responses:
-            # TODO: Add check that participant has answered question; otherwise skip
-            vote = response[var]
-            result[value_map[vote]] += 1
+        for p, r in responses.items():
+            if var in r.keys():
+                vote = r[var]
+            else:
+                continue
+            if vote in value_map.keys():
+                result[value_map[vote]] += 1
         self.pension_counts = result
         return
 
@@ -88,25 +129,40 @@ def update_party_graphs(self, responses=None, donations=None, bp_pk=None):
                 'FDP': scale_dummy.copy()
             }
 
-        for response in responses:
-            # get participant id
-            participant = None
-            # Get var political left/right
-            var = 'lrsp'
-            # Check if participant has answered the question
-            # TODO: Add check that participant has answered question; otherwise skip
-            pol_stance = response[var]
+        for participant, response in responses.items():
+            if not (participant in responses.keys() and participant in donations.keys()):
+                continue
 
-            # get donation belonging to response
-            response_donation = None
-            # compute
-
-
-        # SP
-        # Mitte
-        # SVP
-        # FDP
-        pass
+            var = 'lrsp'
+            if var in response.keys():
+                pol_stance = response[var]
+            else:
+                continue
+
+            valid_responses = [str(i) for i in range(1, 11)]
+            if pol_stance not in valid_responses:
+                continue
+
+            donation = donations[participant]
+            political_accounts = insta_data.load_political_account_list()
+            parties = ['SP', 'SVP', 'Mitte', 'FDP']
+            p_follows_party = {p: False for p in parties}
+            for account in donation:
+                profile = account['string_list_data'][0]['href']
+                if profile in political_accounts.keys():
+                    insta_profile = political_accounts[profile]
+                    profile_type = insta_profile['type']
+                    if profile_type != 'party':
+                        continue
+                    profile_party = insta_profile['party']
+                    if profile_party in parties:
+                        p_follows_party[profile_party] = True
+
+            # Add to result
+            for party, follows in p_follows_party.items():
+                if follows:
+                    self.party_counts[party][pol_stance] += 1
+        return
 
     def update_sm_use(self, responses=None):
         var = 'media_use-4'
@@ -125,18 +181,28 @@ def get_decryptor(self, project):
         return Decryption(settings.SECRET_KEY, project.get_salt())
 
     def get_responses(self):
+        """
+        Returns dictionary with responses per participant.
+        {'participant_id': {'response-var': <response>, ...}}
+        """
         project = self.get_project()
         reference_date = self.get_reference_date()
 
         responses = QuestionnaireResponse.objects.filter(
             project=project, time_submitted__gte=reference_date)
 
         decryptor = self.get_decryptor(project)
-        decrypted_responses = [ResponseSerializer(r, decryptor=decryptor).data['responses'] for r in responses]
-
+        decrypted_responses = {}
+        for r in responses:
+            serialized_r = ResponseSerializer(r, decryptor=decryptor)
+            decrypted_responses[serialized_r.data['participant']] = serialized_r.data['responses']
         return decrypted_responses
 
     def get_blueprint_donations(self, bp_pk):
+        """
+        Returns dictionary with donations per participant.
+        {'participant_id': <extracted donation>}
+        """
         project = self.get_project()
         reference_date = self.get_reference_date()
         blueprint = DonationBlueprint.objects.get(pk=bp_pk)
@@ -145,6 +211,8 @@ def get_blueprint_donations(self, bp_pk):
             blueprint=blueprint, time_submitted__gte=reference_date)
 
         decryptor = self.get_decryptor(project)
-        decrypted_donations = [DonationSerializer(d, decryptor=decryptor).data['data'] for d in donations]
-
+        decrypted_donations = {}
+        for d in donations:
+            serialized_d = DonationSerializer(d, decryptor=decryptor)
+            decrypted_donations[serialized_d.data['participant']] = serialized_d.data['data']
         return decrypted_donations

reports/plots/__init__.py

@@ -0,0 +1,56 @@
+from bokeh.embed import components
+from bokeh.plotting import figure
+
+
+FIRE_PALETTE = ['#de0c1c', '#fe2d2d', '#fb7830', '#fecf02']  #, '#ffeea3']
+
+
+def get_language_plot(data, bar_color='#000'):
+    """
+    data = {'posemo': 2.060499780797895, 'negemo': 0.7452871547566856, 'anx': 0.131521262604121,
+                'anger': 0.08768084173608066, 'sad': 0.5260850504164841, 'social': 3.9017974572555874,
+                'cogproc': 9.294169224024545, 'bio': 0.7891275756247259, 'body': 0.08768084173608066},
+    """
+    original_categories = ['posemo', 'negemo', 'anx', 'anger', 'sad']  #, 'social', 'cogproc', 'bio', 'body']
+    new_names = {
+        'posemo': 'Positive Emotionen',
+        'negemo': 'Negative Emotionen',
+        'anx': 'Angst',
+        'anger': 'Wut',
+        'sad': 'Traurigkeit',
+        'social': 'Sozial (?)',
+        'cogproc': 'Kognitive Prozesse (?)',
+        'bio': 'Physisch (?)',
+        'body': 'Körper (?)'
+    }
+
+    categories = [new_names[c] for c in original_categories]
+    values = [data[c] for c in original_categories]
+
+    p = figure(x_range=categories, height=400, width=400,
+               toolbar_location=None, tools="")
+    p.vbar(x=categories, top=values, width=0.9,
+           fill_color=bar_color, line_color=bar_color)
+
+    p.border_fill_color = None
+    p.y_range.start = 0
+    p.x_range.range_padding = 0.1
+    p.xaxis.major_label_orientation = 1
+    p.axis.minor_tick_line_color = None
+    p.xgrid.grid_line_color = None
+    p.ygrid.grid_line_color = None
+
+    p.xaxis.group_text_color = '#000'
+    p.xaxis.group_text_font_size = '10pt'
+
+    p.xaxis.axis_label = 'Emotionalität Ihrer Suchanfragen'
+    p.yaxis.axis_label = f'Score'
+    p.xaxis.axis_label_text_font_size = '11pt'
+    p.yaxis.axis_label_text_font_size = '9pt'
+    p.xaxis.axis_label_text_font_style = 'normal'
+    p.yaxis.axis_label_text_font_style = 'normal'
+    p.xaxis.major_label_text_font_size = '10pt'
+    p.xaxis.major_tick_line_color = None
+
+    script, div = components(p)
+    return {'script': script, 'div': div}