Tests fixes, redis on github actions, check syntax before existence, …

…raw strings, tests on python^3.8,<3.14

.github/workflows/run_tests.yml

@@ -14,7 +14,6 @@
 # ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS
 # SOFTWARE.
 
-
 name: Run tests
 on:
   push:
@@ -27,7 +26,17 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
+    services:
+      redis:
+        image: redis
+        ports:
+          - 6379:6379
+        options: >-
+          --health-cmd "redis-cli ping"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     steps:
     - uses: actions/checkout@v3.0.2
     - name: Set up Python ${{ matrix.python-version }}
@@ -39,7 +48,7 @@ jobs:
         pip3 install poetry
         sudo apt install -y python-is-python3
         poetry install
-    - name: Check the overage
+    - name: Check the coverage
       run: |
         poetry run coverage run --rcfile=test/coverage/.coveragerc
     - name: Generate the badge

oc_ds_converter/datasource/redis.py

@@ -78,8 +78,11 @@ def mget(self, resources_id):
         #     for i, v in enumerate(self._r.mget(resources_id))
         # }
 
-    def flushall(self):
-        self._r.flushall()
+    def flushdb(self):
+        batch_size = 1000
+        keys = self._r.keys('*')
+        for i in range(0, len(keys), batch_size):
+            self._r.delete(*keys[i:i+batch_size])
 
     def delete(self, resource_id):
         self._r.delete(resource_id)

oc_ds_converter/lib/master_of_regex.py

@@ -1,116 +1,98 @@
-'''
-Split by ';' outside '[]' (any spaces before and after ';').
-'''
-semicolon_in_people_field = '\s*;\s*(?=[^\]]*(?:\[|$))'
+# Split by ';' outside '[]' (any spaces before and after ';').
+semicolon_in_people_field = r'\s*;\s*(?=[^\]]*(?:\[|$))'
 
-'''
-It gets string inside '[]' ignoring any space between (ex: [ TARGET  ] --> TARGET).
-An id schema must be present, followed by a colon.
-Before the colon, there must be any character that is not a square bracket 
-to prevent that in strings like 'Boezaart, Andr[eacute] [omid:123]' the id captured is '[eacute] [omid:123]'.
-Alternatively, empty square brackets containing one or more spaces also represent a valid match.
-'''
-ids_inside_square_brackets = '\[\s*((?:[^\s]+:[^\s]+)?(?:\s+[^\s]+:[^\s]+)*)\s*\]'
+# It gets string inside '[]' ignoring any space between (ex: [ TARGET  ] --> TARGET).
+# An id schema must be present, followed by a colon.
+# Before the colon, there must be any character that is not a square bracket 
+# to prevent that in strings like 'Boezaart, Andr[eacute] [omid:123]' the id captured is '[eacute] [omid:123]'.
+# Alternatively, empty square brackets containing one or more spaces also represent a valid match.
+ids_inside_square_brackets = r'\[\s*((?:[^\s]+:[^\s]+)?(?:\s+[^\s]+:[^\s]+)*)\s*\]'
 
-'''
-It gets the name and ids in two capturing groups.
-As for ids, it gets the string inside '[]' ignoring any space between (ex: [ TARGET  ] --> TARGET).
-An id schema must be present, followed by a colon.
-'''
-name_and_ids = f'\s*(.*?)\s*{ids_inside_square_brackets}'
+# It gets the name and ids in two capturing groups.
+# As for ids, it gets the string inside '[]' ignoring any space between (ex: [ TARGET  ] --> TARGET).
+# An id schema must be present, followed by a colon.
+name_and_ids = fr'\s*(.*?)\s*{ids_inside_square_brackets}'
 
-'''
-It captures a colon preceded and followed by zero or more spaces.
-'''
-colon_and_spaces = '\s*:\s*'
+# It captures a colon preceded and followed by zero or more spaces.
+colon_and_spaces = r'\s*:\s*'
 
-'''
-It captures a colon preceded and followed by zero or more spaces.
-'''
-comma_and_spaces = '\s*,\s*'
+# It captures a comma preceded and followed by zero or more spaces.
+comma_and_spaces = r'\s*,\s*'
 
-'''
-It captures one or more spaces.
-'''
-one_or_more_spaces = '\s+'
+# It captures one or more spaces.
+one_or_more_spaces = r'\s+'
 
-'''
-It captures any pages range separator.
-'''
-pages_separator = '[^A-Za-z\d]+(?=[A-Za-z\d]+)'
+# It captures any pages range separator.
+pages_separator = r'[^A-Za-z\d]+(?=[A-Za-z\d]+)'
 
-'''
-It captures an ORCID
-'''
-orcid_pattern = '([0-9]{4}-){3}[0-9]{3}[0-9X]'
+# It captures an ORCID
+orcid_pattern = r'([0-9]{4}-){3}[0-9]{3}[0-9X]'
 
-'''
-A series of patterns useful to clean invalid "volume" and "issue" fields
-'''
-good_sep = '\-'
-bad_sep = '&\/_,\.:+;\(\[\|'
+# A series of patterns useful to clean invalid "volume" and "issue" fields
+good_sep = r'\-'
+bad_sep = r'&\/_,\.:+;\(\[\|'
 separators = good_sep + bad_sep
-alphabets = 'a-zà-öø-ÿ\u0430-я\u0391-ω' # basic latin, latin-1 supplement, cyrillic, greek
-vi_pattern = f'((?:[{alphabets}]*\d[{alphabets}\d]*|[ivxlcdm]+)(?:\s?(?:[{separators}]|and|\()\s?[{alphabets}\d]+\)?)*?)'
-numero = '(?:№|no?(?!v)\.?|n[º°]\.?|n[uú]m(?:[eé]ro)?|number)'
-year_pattern = '(\d{4})'
+alphabets = r'a-zà-öø-ÿ\u0430-я\u0391-ω'  # basic latin, latin-1 supplement, cyrillic, greek
+vi_pattern = fr'((?:[{alphabets}]*\d[{alphabets}\d]*|[ivxlcdm]+)(?:\s?(?:[{separators}]|and|\()\s?[{alphabets}\d]+\)?)*?)'
+numero = r'(?:№|no?(?!v)\.?|n[º°]\.?|n[uú]m(?:[eé]ro)?|number)'
+year_pattern = r'(\d{4})'
 valid_vi_patterns = [
     vi_pattern,
-    f'[‹<\()]?[{alphabets}]+?([{separators}\s]?[{alphabets}])*[\)›>]?',
-    f'[{alphabets}{separators}\s]+{vi_pattern}',
-    f"[{alphabets}\d\-'/]*\d[{alphabets}\d\-'/]*(,?\s[{alphabets}\d\-'/]+)+",
-    f'\(?s(uppl([eéi]ment(ary|um)?))?\)?\s?(part)?\s?(s?{vi_pattern})?',
-    f'({vi_pattern}_)?({vi_pattern}\s)?[\(_]?supp?(plement|pl)?l?[\s\._]*({vi_pattern}|[{alphabets}])?\)?\.?',
-    f'{vi_pattern}*,?\s?part[\s_]{vi_pattern}(\sof\s{vi_pattern})?(,\sno?\.\s?{vi_pattern})?',
-    f'{vi_pattern}*[_\s]?pt?[_\s\.]{vi_pattern}',
+    fr'[‹<\()]?[{alphabets}]+?([{separators}\s]?[{alphabets}])*[\)›>]?',
+    fr'[{alphabets}{separators}\s]+{vi_pattern}',
+    fr"[{alphabets}\d\-'/]*\d[{alphabets}\d\-'/]*(,?\s[{alphabets}\d\-'/]+)+",
+    fr'\(?s(uppl([eéi]ment(ary|um)?))?\)?\s?(part)?\s?(s?{vi_pattern})?',
+    fr'({vi_pattern}_)?({vi_pattern}\s)?[\(_]?supp?(plement|pl)?l?[\s\._]*({vi_pattern}|[{alphabets}])?\)?\.?',
+    fr'{vi_pattern}*,?\s?part[\s_]{vi_pattern}(\sof\s{vi_pattern})?(,\sno?\.\s?{vi_pattern})?',
+    fr'{vi_pattern}*[_\s]?pt?[_\s\.]{vi_pattern}',
     '(ed|pt|d)\sinside(d|r)',
     'p(ublish\s)?a(head\sof\s)?p(rint)?',
     '預刊文章',
     '[\u0621-\u064A]+',
-    f'\[{year_pattern}\]\s(\d\s)?[{alphabets}]+',
-    f'{vi_pattern}\s\[\+CDROM\]',
-    f'{vi_pattern}[{separators}\s]?\({vi_pattern}\)(\s{vi_pattern})?',
-    f'([{alphabets}]+\.)?[{alphabets}]+\.?',
-    f'[{alphabets}]+-\d+',
-    f'[{alphabets}]+(_[{alphabets}]+)+',
-    f'{numero}:?\s?{vi_pattern}(,?\s({year_pattern}|\({vi_pattern}\)))?',
+    fr'\[{year_pattern}\]\s(\d\s)?[{alphabets}]+',
+    fr'{vi_pattern}\s\[\+CDROM\]',
+    fr'{vi_pattern}[{separators}\s]?\({vi_pattern}\)(\s{vi_pattern})?',
+    fr'([{alphabets}]+\.)?[{alphabets}]+\.?',
+    fr'[{alphabets}]+-\d+',
+    fr'[{alphabets}]+(_[{alphabets}]+)+',
+    fr'{numero}:?\s?{vi_pattern}(,?\s({year_pattern}|\({vi_pattern}\)))?',
     'historica\svol\.\s\d+(,\d+(-\d+)?)?',
     '\d+\(\d+\)\d{2,4}',
-    f'(\[{year_pattern}\]\s)?(\d+\s)?vl?r(\s\([a-z]+\))?',
-    f'\({vi_pattern}\/{vi_pattern}\)\s[{alphabets}]+(-[{alphabets}]+)?'
+    fr'(\[{year_pattern}\]\s)?(\d+\s)?vl?r(\s\([a-z]+\))?',
+    fr'\({vi_pattern}\/{vi_pattern}\)\s[{alphabets}]+(-[{alphabets}]+)?'
 ]
 volumes_valid_patterns = [
     'original\sseries,\svolume\s\d+',
-    f'(vol(ume)?|tome|cilt)\s?[{separators}]?\s?{vi_pattern}'
+    fr'(vol(ume)?|tome|cilt)\s?[{separators}]?\s?{vi_pattern}'
 ]
 issues_valid_patterns = [
-    f'issue[\.,]?\s{vi_pattern}',
-    f'({vi_pattern}\s)?e?sp?e?(ecial)?[\s_\-\.](issue)?(_number_)?[\s_-]?({vi_pattern})?(["“][{alphabets}\s]+?["”])?',
-    f'ö(zel)?(\ss(ayı)?|\(special\))?(\s?{vi_pattern})?',
-    f'({numero}[{separators}\s]?)?hors[{separators}\s]série[{separators}\s]{vi_pattern}',
+    fr'issue[\.,]?\s{vi_pattern}',
+    fr'({vi_pattern}\s)?e?sp?e?(ecial)?[\s_\-\.](issue)?(_number_)?[\s_-]?({vi_pattern})?(["“][{alphabets}\s]+?["”])?',
+    fr'ö(zel)?(\ss(ayı)?|\(special\))?(\s?{vi_pattern})?',
+    fr'({numero}[{separators}\s]?)?hors[{separators}\s]série[{separators}\s]{vi_pattern}',
     '특별호',
-    f'([{alphabets}]+\s{year_pattern}\s)?\(?(jan(uary)?|feb(ruary)?|mar(ch)?|apr(il)?|may|jun(e)?|jul(y)?|aug(ust)?|sep(tember)?|oct(ober)?|(nov|dec)(ember)?|spring|summer|autumn|winter)(\s{year_pattern})?\)?',
-    f'{vi_pattern},\spart\s{vi_pattern}\sof\s{vi_pattern}',
-    f'sayı[{separators}\s]\s?{vi_pattern}',
-    f'issues?\s{vi_pattern},\s(supplement|part)\s{vi_pattern}',
-    f'issues?\s{vi_pattern}\.?\spp\.\s[a-z\d]+[^a-z\d]+[a-z\d]+'
+    fr'([{alphabets}]+\s{year_pattern}\s)?\(?(jan(uary)?|feb(ruary)?|mar(ch)?|apr(il)?|may|jun(e)?|jul(y)?|aug(ust)?|sep(tember)?|oct(ober)?|(nov|dec)(ember)?|spring|summer|autumn|winter)(\s{year_pattern})?\)?',
+    fr'{vi_pattern},\spart\s{vi_pattern}\sof\s{vi_pattern}',
+    fr'sayı[{separators}\s]\s?{vi_pattern}',
+    fr'issues?\s{vi_pattern},\s(supplement|part)\s{vi_pattern}',
+    fr'issues?\s{vi_pattern}\.?\spp\.\s[a-z\d]+[^a-z\d]+[a-z\d]+'
 ]
 invalid_vi_patterns = {
-    f'.*?(?:vol\.?(?:ume)?|tome)(?:[{separators}]?\s?){vi_pattern}[\-&\/_,\.:+;\(\)\[\]|\s]*(?:{numero}|issues?)[{separators}|\s]*(?:sp[eé]cial\s)?{vi_pattern}': 'vol_iss',
-    f'{vi_pattern},\s?{numero}\s?{vi_pattern}': 'vol_iss',
-    f'tập\s?{vi_pattern},?\s?số\s?{vi_pattern}': 'vol_iss',
-    f'issues?\s{vi_pattern}\svol\.?(?:ume)?\s{vi_pattern}(?:.*?{year_pattern}.*?)?': 'iss_vol_year',
-    f"{vi_pattern}\s?\({vi_pattern}'{year_pattern}\)": 'vol_iss_year',
-    f'cilt[{separators}\s]\s?{vi_pattern}[{separators}\s]sayı[{separators}\s]\s?{vi_pattern}(?:[{separators}\s]\s?temmuz\s{year_pattern})?': 'vol_iss_year',
-    '&na;|n\/a|not\savailable': 'del',
-    '[\:\-\.`ё/]': 'del',
-    f'\${{[{alphabets}]+(\.[{alphabets}]+)?}}': 'del',
-    f"[&\/_,:+;\|`'#]\s*{vi_pattern}": 'all',
-    f'[\->+]{vi_pattern}': 'do_nothing',
-    f"{vi_pattern}[\.+]": "do_nothing",
-    f"{numero}?[{separators}]?\s?{vi_pattern}[&\/_,:;\|`'\(\[\{{]": 'all',
-    f'{vi_pattern}\(\)': 'all',
-    f'n[�?]+{vi_pattern}': 'all',
-    f'{vi_pattern}(?:â\x80[\x92\x93\x94]|�+|â|\?+){vi_pattern}': 'sep',
-    f'{vi_pattern}\s?\(first\sserie': 's)'
+    fr'.*?(?:vol\.?(?:ume)?|tome)(?:[{separators}]?\s?){vi_pattern}[\-&\/_,\.:+;\(\)\[\]|\s]*(?:{numero}|issues?)[{separators}|\s]*(?:sp[eé]cial\s)?{vi_pattern}': 'vol_iss',
+    fr'{vi_pattern},\s?{numero}\s?{vi_pattern}': 'vol_iss',
+    fr'tập\s?{vi_pattern},?\s?số\s?{vi_pattern}': 'vol_iss',
+    fr'issues?\s{vi_pattern}\svol\.?(?:ume)?\s{vi_pattern}(?:.*?{year_pattern}.*?)?': 'iss_vol_year',
+    fr"{vi_pattern}\s?\({vi_pattern}'{year_pattern}\)": 'vol_iss_year',
+    fr'cilt[{separators}\s]\s?{vi_pattern}[{separators}\s]sayı[{separators}\s]\s?{vi_pattern}(?:[{separators}\s]\s?temmuz\s{year_pattern})?': 'vol_iss_year',
+    r'&na;|n\/a|not\savailable': 'del',
+    r'[\:\-\.`ё/]': 'del',
+    fr'\${{[{alphabets}]+(\.[{alphabets}]+)?}}': 'del',
+    fr"[&\/_,:+;\|`'#]\s*{vi_pattern}": 'all',
+    fr'[\->+]{vi_pattern}': 'do_nothing',
+    fr"{vi_pattern}[\.+]": "do_nothing",
+    fr"{numero}?[{separators}]?\s?{vi_pattern}[&\/_,:;\|`'\(\[\{{]": 'all',
+    fr'{vi_pattern}\(\)': 'all',
+    fr'n[�?]+{vi_pattern}': 'all',
+    fr'{vi_pattern}(?:â\x80[\x92\x93\x94]|�+|â|\?+){vi_pattern}': 'sep',
+    fr'{vi_pattern}\s?\(first\sserie': 's)'
 }

oc_ds_converter/oc_idmanager/arxiv.py

@@ -83,7 +83,7 @@ def is_valid(self, id_string, get_extra_info=False):
                     info = self.exists(arxiv, get_extra_info=True)
                     self.storage_manager.set_full_value(arxiv,info[1])
                     return (info[0] and self.syntax_ok(arxiv)), info[1]
-                validity_check = self.exists(arxiv) and self.syntax_ok(arxiv)
+                validity_check = self.syntax_ok(arxiv) and self.exists(arxiv) 
                 self.storage_manager.set_value(arxiv, validity_check)
 
                 return validity_check

oc_ds_converter/oc_idmanager/crossref.py

@@ -53,7 +53,7 @@ def is_valid(self, cr_member_id, get_extra_info=False):
                     info = self.exists(cr_member_id, get_extra_info=True)
                     self.storage_manager.set_full_value(cr_member_id, info[1])
                     return (info[0] and self.syntax_ok(cr_member_id)), info[1]
-                validity_check = self.exists(cr_member_id) and self.syntax_ok(cr_member_id)
+                validity_check = self.syntax_ok(cr_member_id) and self.exists(cr_member_id) 
                 self.storage_manager.set_value(cr_member_id, validity_check)
 
                 return validity_check

oc_ds_converter/oc_idmanager/doi.py

@@ -124,14 +124,14 @@ def is_valid(self, id_string, get_extra_info=False):
                     info = self.exists(doi, get_extra_info=True)
                     self.storage_manager.set_full_value(doi,info[1])
                     return (info[0] and self.syntax_ok(doi)), info[1]
-                validity_check = self.exists(doi) and self.syntax_ok(doi)
+                validity_check = self.syntax_ok(doi) and self.exists(doi) 
                 self.storage_manager.set_value(doi, validity_check)
                 return validity_check
 
     def base_normalise(self, id_string):
         try:
             id_string = sub(
-                "\0+", "", sub("\s+", "", unquote(id_string[id_string.index("10.") :]))
+                r"\0+", "", sub(r"\s+", "", unquote(id_string[id_string.index("10.") :]))
             )
             return id_string.lower().strip() if id_string else None
         except:
@@ -191,7 +191,7 @@ def clean_doi(self, doi:str) -> Tuple[str, dict]:
     def syntax_ok(self, id_string):
         if not id_string.startswith(self._p):
             id_string = self._p+id_string
-        return True if match("^doi:10\.(\d{4,9}|[^\s/]+(\.[^\s/]+)*)/[^\s]+$", id_string, re.IGNORECASE) else False
+        return True if match(r"^doi:10\.(\d{4,9}|[^\s/]+(\.[^\s/]+)*)/[^\s]+$", id_string, re.IGNORECASE) else False
 
     def exists(self, doi_full, get_extra_info=False, allow_extra_api=None):
         valid_bool = True

oc_ds_converter/oc_idmanager/issn.py

@@ -65,7 +65,7 @@ def check_digit(self,issn):
             issn = issn[spl:]
         issn = issn.replace('-', '')
         if len(issn) != 8:
-            raise ValueError('ISSN of len 8 or 9 required (e.g. 00000949 or 0000-0949)')
+            return False
         ss = sum([int(digit) * f for digit, f in zip(issn, range(8, 1, -1))])
         _, mod = divmod(ss, 11)
         checkdigit = 0 if mod == 0 else 11 - mod

oc_ds_converter/oc_idmanager/jid.py

@@ -62,7 +62,7 @@ def is_valid(self, jid, get_extra_info=False):
                     info = self.exists(jid, get_extra_info=True)
                     self.storage_manager.set_full_value(jid, info[1])
                     return (info[0] and self.syntax_ok(jid)), info[1]
-                validity_check = self.exists(jid) and self.syntax_ok(jid)
+                validity_check = self.syntax_ok(jid) and self.exists(jid) 
                 self.storage_manager.set_value(jid, validity_check)
 
                 return validity_check

oc_ds_converter/oc_idmanager/oc_data_storage/redis_manager.py

@@ -118,7 +118,7 @@ def del_value(self, id: str) -> None:
 
 
     def delete_storage(self):
-        self.PROCESS_redis.flushall()
+        self.PROCESS_redis.flushdb()
 
     def get_all_keys(self):
         result = [x for x in self.PROCESS_redis.scan_iter('*')]

oc_ds_converter/oc_idmanager/openalex.py

@@ -60,7 +60,7 @@ def is_valid(self, oal_id, get_extra_info=False):
                     info = self.exists(oal_id, get_extra_info=True)
                     self.storage_manager.set_full_value(oal_id,info[1])
                     return (info[0] and self.syntax_ok(oal_id)), info[1]
-                validity_check = self.exists(oal_id) and self.syntax_ok(oal_id)
+                validity_check = self.syntax_ok(oal_id) and self.exists(oal_id) 
                 self.storage_manager.set_value(oal_id, validity_check)
 
                 return validity_check
@@ -72,7 +72,7 @@ def normalise(self, id_string, include_prefix=False):
             else:
                 oal_string = id_string
 
-            oal_string = sub("\0+", "", (sub("\s+", "", oal_string)))
+            oal_string = sub(r"\0+", "", (sub(r"\s+", "", oal_string)))
 
             oal_string = oal_string.replace(self._api_works_route, '', 1)
             oal_string = oal_string.replace(self._api_sources_route, '', 1)

oc_ds_converter/oc_idmanager/orcid.py

@@ -70,7 +70,7 @@ def is_valid(self, id_string, get_extra_info=False):
                     info = self.exists(orcid, get_extra_info=True)
                     self.storage_manager.set_full_value(orcid,info[1])
                     return (info[0] and self.check_digit(orcid) and self.syntax_ok(orcid)), info[1]
-                validity_check = self.exists(orcid) and self.syntax_ok(orcid) and self.check_digit(orcid)
+                validity_check = self.syntax_ok(orcid) and self.check_digit(orcid) and self.exists(orcid) 
                 self.storage_manager.set_value(orcid, validity_check)
                 return validity_check
 

oc_ds_converter/oc_idmanager/pmcid.py

@@ -72,7 +72,7 @@ def is_valid(self, pmcid, get_extra_info=False):
                     info = self.exists(pmcid, get_extra_info=True)
                     self.storage_manager.set_full_value(pmcid,info[1])
                     return (info[0] and self.syntax_ok(pmcid)), info[1]
-                validity_check = self.exists(pmcid) and self.syntax_ok(pmcid)
+                validity_check = self.syntax_ok(pmcid) and self.exists(pmcid) 
                 self.storage_manager.set_value(pmcid, validity_check)
 
                 return validity_check
@@ -85,7 +85,7 @@ def normalise(self, id_string, include_prefix=False):
                 id_string = id_string
 
             pmcid_string = sub(
-                "\0+", "", sub("\s+", "", unquote(id_string[id_string.index("PMC"):]))
+                r"\0+", "", sub(r"\s+", "", unquote(id_string[id_string.index("PMC"):]))
             )
             return "%s%s" % (
                 self._p if include_prefix else "",