frank.yml

template_name: frank
template_label: FRANK
template_description: Factuality in Abstractive Summarization

# ========================================================================================
#
# While we adapt the annotation typology proposed in the FRANK paper, their annotation simply
# has annotators select whether the error occurs in the sentence, not a span which corresponds
# to the error.
#
# ========================================================================================

interface_text:
  typology:
    context_label: "Article"
    source_label: "Reference Summary"
    target_label: "Generated Summary"

disable:
- annotation

display:
- side-by-side

edits:
  - name: RelE
    label: "Relation Error"
    color: "#bd3dbf"
    icon: fa-people-arrows
  - name: EntE
    label: "Entity Error"
    color: blue
    icon: fa-box
  - name: CircE
    label: "Circumstance Error"
    color: teal
    icon: fa-person-falling
  - name: CorefE
    label: "Coreference Error"
    color: yellow
    icon: fa-user-minus
  - name: LinkE
    label: "Discourse Link Error"
    color: orange
    icon: fa-comment-slash
  - name: OutE
    label: "Out of Article Error"
    color: green
    icon: fa-file-export
  - name: GramE
    label: "Grammatical Error"
    color: red
    icon: fa-spell-check
  - name: OtherE
    label: "Other Error"
    color: grey
    icon: fa-minus

paper_link: https://aclanthology.org/2021.naacl-main.383/
citation: |
  @inproceedings{pagnoni-etal-2021-understanding,
    title = "Understanding Factuality in Abstractive Summarization with {FRANK}: A Benchmark for Factuality Metrics",
    author = "Pagnoni, Artidoro  and
      Balachandran, Vidhisha  and
      Tsvetkov, Yulia",
    booktitle = "Proceedings of the 2021 Conference of the North American Chapter of the Association for Computational Linguistics: Human Language Technologies",
    month = jun,
    year = "2021",
    address = "Online",
    publisher = "Association for Computational Linguistics",
    url = "https://www.aclweb.org/anthology/2021.naacl-main.383",
    doi = "10.18653/v1/2021.naacl-main.383",
    pages = "4812--4829",
  }