fix bug

ku-nlp · Apr 20, 2024 · 054ee32 · 054ee32
1 parent 043d883
commit 054ee32
Show file tree

Hide file tree

Showing 2 changed files with 5 additions and 2 deletions.
diff --git a/configs/word_module.yaml b/configs/word_module.yaml
@@ -1,4 +1,4 @@
-configs/word_module.yamldefaults:
+defaults:
   - base
   - callbacks: [word_module_writer, early_stopping, lr_monitor, model_checkpoint, model_summary, progress_bar]
   - datamodule: word

diff --git a/src/kwja/metrics/char.py b/src/kwja/metrics/char.py
@@ -68,7 +68,10 @@ def _build_documents(self) -> Tuple[List[Document], List[Document], List[Documen
         doc_id2predicted_sentences: Dict[str, List[Sentence]] = defaultdict(list)
         doc_id2partly_gold_sentences: Dict[str, List[Sentence]] = defaultdict(list)
         doc_id2gold_sentences: Dict[str, List[Sentence]] = defaultdict(list)
-        special_ids = set(self.dataset.tokenizer.all_special_ids) - {self.dataset.tokenizer.unk_token_id}
+        special_ids = {
+            getattr(self.dataset.tokenizer, f"{prefix}_token_id")
+            for prefix in ["bos", "eos", "sep", "pad", "cls", "mask"]
+        }
         for (
             example_id,
             sent_segmentation_predictions,