From f646930550462241d79b0e09b04941c189b1b9ec Mon Sep 17 00:00:00 2001
From: Mark Wardle <mark@wardle.org>
Date: Sat, 7 Dec 2024 07:53:24 +0000
Subject: [PATCH] Improve importer tests

---
 src/com/eldrix/hermes/importer.clj            | 11 ++++
 ...2_Description_Snapshot-en_INT_20230131.txt |  2 +
 test/src/com/eldrix/hermes/importer_test.clj  | 62 ++++++-------------
 3 files changed, 32 insertions(+), 43 deletions(-)

diff --git a/src/com/eldrix/hermes/importer.clj b/src/com/eldrix/hermes/importer.clj
index 366dbf0..c21cec6 100644
--- a/src/com/eldrix/hermes/importer.clj
+++ b/src/com/eldrix/hermes/importer.clj
@@ -136,6 +136,17 @@
               (log/debug "Processing cancelled (output channel closed)")
               (throw (InterruptedException. "process cancelled")))))))))
 
+(defn import-file
+  "Import a SNOMED file, returning a map containing :type :headings :parser 
+  and :data as per [[process-file]]. This is designed only for testing and
+  development purposes."
+  [f]
+  (let [ch (a/chan)]
+    (a/thread
+      (process-file f ch)
+      (a/close! ch))
+    (a/<!! ch)))
+
 (s/fdef load-snomed-files
   :args (s/cat :files (s/coll-of :info.snomed/ReleaseFile)
                :opts (s/keys* :opt-un [::nthreads ::batch-size])))
diff --git a/test/resources/example-snapshot/Terminology/sct2_Description_Snapshot-en_INT_20230131.txt b/test/resources/example-snapshot/Terminology/sct2_Description_Snapshot-en_INT_20230131.txt
index 646af88..ddb8f15 100644
--- a/test/resources/example-snapshot/Terminology/sct2_Description_Snapshot-en_INT_20230131.txt
+++ b/test/resources/example-snapshot/Terminology/sct2_Description_Snapshot-en_INT_20230131.txt
@@ -3,3 +3,5 @@ id	effectiveTime	active	moduleId	conceptId	languageCode	typeId	term	caseSignific
 102018	20170731	1	900000000000207008	126814004	en	900000000000013009	Neoplasm of junctional region of epiglottis	900000000000448009
 103011	20170731	1	900000000000207008	126815003	en	900000000000013009	Neoplasm of lateral wall of oropharynx	900000000000448009
 104017	20170731	1	900000000000207008	126816002	en	900000000000013009	Neoplasm of posterior wall of oropharynx	900000000000448009
+21377010        20020131        0       900000000000207008      12418009        en      900000000000013009      Transfer of "walking" tube graft        900000000000020002
+28578012        20020131        0       900000000000207008      16878007        en      900000000000013009      Blood group antibody rh"        900000000000020002
diff --git a/test/src/com/eldrix/hermes/importer_test.clj b/test/src/com/eldrix/hermes/importer_test.clj
index 6251f87..b2537fa 100644
--- a/test/src/com/eldrix/hermes/importer_test.clj
+++ b/test/src/com/eldrix/hermes/importer_test.clj
@@ -2,53 +2,46 @@
   (:require
    [clojure.core.async :as async]
    [clojure.java.io :as io]
-   [clojure.spec.gen.alpha :as gen]
    [clojure.test :refer [deftest is testing]]
-   [com.eldrix.hermes.importer :as importer]
-   [com.eldrix.hermes.rf2 :as rf2])
+   [com.eldrix.hermes.importer :as imp])
   (:import (java.time LocalDate)))
 
-(deftest parse-filename
+(deftest parse-filenames
   (testing "nil filename"
-    (is (nil? (importer/parse-filename nil))))
+    (is (nil? (imp/parse-filename nil))))
   (testing "concept filename as string"
-    (let [{:keys [format version-date content-subtype type country-code identifier component]} (importer/parse-filename "sct2_Concept_Snapshot_INT_20230131.txt")]
+    (let [{fmt :format t :type :keys [version-date country-code identifier component]} (imp/parse-filename "sct2_Concept_Snapshot_INT_20230131.txt")]
       (is (= "Concept" component))
       (is (= "INT" country-code))
       (is (= :info.snomed/Concept identifier))
-      (is (= "2" format))
-      (is (= "sct" type))
+      (is (= "2" fmt))
+      (is (= "sct" t))
       (is (= (LocalDate/of 2023 1 31) version-date))))
   (testing "description filename as URL"
-    (let [{:keys [identifier]} (importer/parse-filename (java.net.URL. "file://Terminology/sct2_Description_Snapshot-en_INT_20230131.txt"))]
+    (let [{:keys [identifier]} (imp/parse-filename (java.net.URL. "file://Terminology/sct2_Description_Snapshot-en_INT_20230131.txt"))]
       (is (= :info.snomed/Description identifier))))
   (testing "relationship concrete values filename as file"
-    (let [{:keys [identifier]} (importer/parse-filename (io/file "./Terminology/sct2_RelationshipConcreteValues_Snapshot_INT_20230131.txt"))]
+    (let [{:keys [identifier]} (imp/parse-filename (io/file "./Terminology/sct2_RelationshipConcreteValues_Snapshot_INT_20230131.txt"))]
       (is (= :info.snomed/RelationshipConcreteValues identifier)))))
 
-(defn import-file
-  "Import a SNOMED file"
-  [f]
-  (let [ch (async/chan)]
-    (async/thread
-      (importer/process-file f ch)
-      (async/close! ch))
-    (async/<!! ch)))
-
-(deftest import-concepts
-  (let [{:keys [type parser headings data]} (import-file (io/resource "example-snapshot/Terminology/sct2_Concept_Snapshot_INT_20230131.txt"))]
-    (is (= :info.snomed/Concept type))))
+(deftest import-terminology
+  (testing "concepts"
+    (let [{t :type} (imp/import-file (io/resource "example-snapshot/Terminology/sct2_Concept_Snapshot_INT_20230131.txt"))]
+      (is (= :info.snomed/Concept t))))
+  (testing "descriptions"
+    (let [{t :type} (imp/import-file (io/resource "example-snapshot/Terminology/sct2_Description_Snapshot-en_INT_20230131.txt"))]
+      (is (= :info.snomed/Description t)))))
 
 (deftest import-refset
-  (let [{:keys [type parser headings data] :as f} (import-file (io/resource "example-snapshot/Refset/Map/der2_iisssccRefset_ExtendedMapSnapshot_INT_20230131.txt"))]
-    (is (= :info.snomed/ExtendedMapRefset type))
+  (let [{t :type, headings :headings} (imp/import-file (io/resource "example-snapshot/Refset/Map/der2_iisssccRefset_ExtendedMapSnapshot_INT_20230131.txt"))]
+    (is (= :info.snomed/ExtendedMapRefset t))
     (is (= ["id" "effectiveTime" "active" "moduleId" "refsetId" "referencedComponentId"
             "mapGroup" "mapPriority" "mapRule" "mapAdvice" "mapTarget"
             "correlationId" "mapCategoryId"] headings))))
 
 (deftest import-custom-refset-nil-values
-  (let [{:keys [type parser headings data] :as f} (import-file  (io/resource "example-snapshot/Refset/Map/der2_ssRefset_SimpleMapWithDescriptionSnapshot_12345_20241021.txt"))]
-    (is (= :info.snomed/SimpleMapRefset type))
+  (let [{t :type :keys [headings data]} (imp/import-file (io/resource "example-snapshot/Refset/Map/der2_ssRefset_SimpleMapWithDescriptionSnapshot_12345_20241021.txt"))]
+    (is (= :info.snomed/SimpleMapRefset t))
     (is (= ["id" "effectiveTime" "active" "moduleId" "refsetId" "referencedComponentId"
             "mapTarget" "mapTargetDescription"] headings))
     (is (= ["000d91ce-aae4-4f9e-ad06-51be576becd6" "20241021"
@@ -56,22 +49,5 @@
             "1434181000001106" "ABC01256Q" ""] (first data))
         "Empty last column should be returned as empty string")))
 
-(comment
-  (require '[clojure.data.csv :as csv])
-  (csv/read-csv "hi\tthere\tand\thow\tare\tyou?\t" :separator \tab)
-  (def f (io/resource "example-snapshot/Terminology/sct2_Concept_Snapshot_INT_20230131.txt"))
-  (type f)
-  (io/as-file f)
-  (importer/parse-filename "sct2_Concept.txt")
-  (importer/parse-filename (java.net.URL. "https://wibble.com/sct_Concept_Snapshot_INT_20230131.txt"))
-  (importer/parse-filename f)
-  (importer/parse-filename nil)
-  (def ch (async/chan))
-  (async/thread
-    (importer/process-file f ch)
-    (async/close! ch))
-  (def ch (importer/load-snomed (io/resource "example-snapshot/")))
-  (async/<!! ch)
 
-  (gen/sample (rf2/gen-simple-map-refset {:fields [""]})))