Factscore replication and CommunityLM Analysis

examples/community_lm/community_lm_utils.py

@@ -2,10 +2,12 @@
 
 import os
 import tqdm
+import csv
 from pathlib import Path
 from community_lm_constants import anes_df
 import pandas as pd
 import numpy as np
+from typing import cast
 
 from llments.lm.lm import LanguageModel
 from llments.lm.rag import RAGLanguageModel
@@ -162,3 +164,66 @@ def compute_group_stance(
 
     df = pd.DataFrame(rows, columns=columns)
     df.to_csv(output_filename)
+
+def compute_group_stance_factscore(
+    evaluator: SentimentEvaluator,
+    input_filename: str,
+) -> dict[str, float]:
+    """Calculates group sentiment for the democratic and republican parties.
+
+    Args:
+        input_filename (str): The input filename.
+        evaluator: The sentiment evaluator.
+
+    Returns:
+        dict: A dictionary with keys 'democratic' and 'republican' containing their respective sentiments.
+    """
+    democratic_responses = []
+    republican_responses = []
+
+    try:
+        with open(input_filename, mode='r', newline='', encoding='utf-8') as csvfile:
+            reader = csv.DictReader(csvfile)
+
+            for row in reader:
+                party = row['Party'].strip().lower()
+                response = row['Response'].strip()
+                if not response:
+                    continue  # Skip empty responses
+                if party == 'democrats':
+                    democratic_responses.append(response)
+                elif party == 'republicans':
+                    republican_responses.append(response)
+                else:
+                    print(f"Warning: Unknown party '{party}' in row: {row}")
+
+        # Function to evaluate sentiments
+        def evaluate_sentiments(
+            responses: list[str],
+            party_name: str,
+        ) -> float:
+            """Calculates sentiment for given responses and party.
+
+            Args:
+                responses (list[str]): A list containing synthetic tweets for all politicians of a given party.
+                party_name (str): The party for which we calculate the sentiment.
+
+            Returns:
+                float: Group sentiment towards politicians of a given party.
+            """
+            sentiment_vals = evaluator.evaluate_batch(responses, minibatch_size=len(responses))
+            group_sentiment = np.mean(sentiment_vals) * 100
+            return cast(float, group_sentiment)
+
+        # Calculate sentiments for each group
+        sentiments = {}
+        sentiments['democratic'] = evaluate_sentiments(democratic_responses, 'democratic')
+        sentiments['republican'] = evaluate_sentiments(republican_responses, 'republican')
+        return sentiments
+
+    except FileNotFoundError:
+        print(f"Error: The file '{input_filename}' does not exist.")
+        return {}
+    except Exception as e:
+        print(f"An error occurred: {e}")
+        return {}

examples/factscore_eval/README.md

@@ -0,0 +1,83 @@
+# FActScore
+
+This is a replication of the experiments from
+[FActScore: Fine-grained Atomic Evaluation of Factual Precision in Long Form
+Text Generation](https://aclanthology.org/2023.emnlp-main.741) (Min et al., EMNLP
+2023).
+
+## Dependencies
+
+To better align with the original implementation in the paper,
+we recommend using the version of the dependencies in the `requirements.txt` file.
+You can install the dependencies by running the following command:
+
+```bash
+pip install -r requirements.txt
+```
+
+## Configuration
+
+### OpenAI API Key
+
+To access and use OpenAI's services (such as GPT models),
+you must obtain an API key from OpenAI.
+After acquiring your API key, store it in a txt file
+such as `key.txt` amd pass it when creating an instance
+of the `FactScorer` class.
+
+### Data Preparation
+
+Before running the code to generate the CommunityLM responses,
+make sure you have created the following directory to store the data:
+
+```bash
+mkdir -p factscore_data
+```
+
+This will be the data directory for the FactScore analysis
+and all CSV files must be inside this folder.
+
+## Reference
+
+Some of this code and data was derived from the
+[FActScore repo](https://github.com/shmsw25/FActScore).
+
+If you use this example, we would appreciate if you acknowledge
+[LLMents](https://github.com/neulab/llments) and the original paper.
+
+```bibtex
+@misc{
+    title = "{LLMents}: A Toolkit for Language Model Experiments",
+    author = "
+      Graham Neubig and
+      Aakriti Kinra and
+      Mihir Bansal and
+      Qingyang Liu and
+      Rohan Modi and
+      Xinran Wan
+    ",
+    year = "2024",
+    howpublished = "https://github.com/neulab/llments",
+}
+```
+
+```bibtex
+@inproceedings{min-etal-2023-factscore,
+    title = "{FA}ct{S}core: Fine-grained Atomic Evaluation of Factual
+    Precision in Long Form Text Generation",
+    author = "Min, Sewon  and
+      Krishna, Kalpesh  and
+      Lyu, Xinxi  and
+      Lewis, Mike  and
+      Yih, Wen-tau  and
+      Koh, Pang  and
+      Iyyer, Mohit  and
+      Zettlemoyer, Luke  and
+      Hajishirzi, Hannaneh",
+    booktitle = "Proceedings of the 2023 Conference on Empirical
+    Methods in Natural Language Processing",
+    year = "2023",
+    publisher = "Association for Computational Linguistics",
+    doi = "10.18653/v1/2023.emnlp-main.741",
+    }
+```