Merge pull request #6 from eodcgmbh/update-readme

remove docker container and update client further

README.md

@@ -1,2 +1,10 @@
 # stac-rucio
 A python package for managing STAC asset data locality when those assets are managed by a RUCIO server.
+
+## Installation
+
+PyPI support coming.
+
+## Usage
+
+## Development

examples/demo_example.ipynb

@@ -27,7 +27,19 @@
    "cell_type": "code",
    "execution_count": 2,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "ename": "NameError",
+     "evalue": "name 'stac_rucio' is not defined",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[2], line 6\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mpystac_client\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m Client\n\u001b[1;32m      3\u001b[0m \u001b[38;5;66;03m# STAC client connection, with a modifier for updating items on the fly.\u001b[39;00m\n\u001b[1;32m      4\u001b[0m eodc \u001b[38;5;241m=\u001b[39m Client\u001b[38;5;241m.\u001b[39mopen(\n\u001b[1;32m      5\u001b[0m   \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mhttps://stac.eodc.eu/api/v1\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[0;32m----> 6\u001b[0m   modifier\u001b[38;5;241m=\u001b[39m\u001b[43mstac_rucio\u001b[49m\u001b[38;5;241m.\u001b[39mrucio_item\n\u001b[1;32m      7\u001b[0m )\n",
+      "\u001b[0;31mNameError\u001b[0m: name 'stac_rucio' is not defined"
+     ]
+    }
+   ],
    "source": [
     "from pystac_client import Client\n",
     "\n",
@@ -126,7 +138,7 @@
    "outputs": [],
    "source": [
     "# Create replication rules generates the rules that instruct rucio that a request has been made to copy data from one RSE to another.\n",
-    "stac_rucio.create_replication_rules(items=november_list, src_rse=\"EODC-DATA\", dst_rse=\"DESY-DCACHE\")"
+    "stac_rucio.create_replication_rules(items=november_list, dst_rse=\"DESY-DCACHE\")"
    ]
   },
   {
@@ -160,7 +172,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3",
+   "display_name": ".venv",
    "language": "python",
    "name": "python3"
   },

pyproject.toml

@@ -10,8 +10,8 @@ readme = "README.md"
 python = ">=3.9,<3.13"
 pystac = ">=1.10.0 "
 pystac-client = ">=0.8.3"
-rucio = ">=35.6.0"
-rucio_clients = ">=35.6.0"
+rucio = ">=36.3.0"
+rucio_clients = ">=36.3.0"
 
 [tool.poetry.group.dev.dependencies]
 pytest = ">=7.2.0"

stac_rucio/client.py

@@ -11,7 +11,6 @@ class ReplicaExists(Exception):
 class StacClient:
     def __init__(self, target):
         self.rucio = Client()
-        self.rucio.whoami()
         self.downloader = DownloadClient()
         self.scheme = "https"
 
@@ -32,11 +31,11 @@ def create_replicas(self, items: list, rse: str, target: str ):
             tmp = item.to_dict()
             config = RucioStac(**tmp["assets"][target]["rucio:config"])
 
-            replicas = self.get_existing_replicas(tmp, src_rse)
+            replicas = self.get_existing_replicas(tmp, rse)
 
             if not replicas:
                 self.rucio.add_replica(
-                    rse=src_rse,
+                    rse=rse,
                     scope=self.rucio.account,
                     name=item["id"],
                     pfn=self._ensure_port(tmp["assets"][target]["href"]),
@@ -46,7 +45,7 @@ def create_replicas(self, items: list, rse: str, target: str ):
 
         return True
 
-    def create_replication_rules(self, items: list, src_rse: str, dst_rse: str):
+    def create_replication_rules(self, items: list, dst_rse: str):
         """Create replication rules for stac items existing as a non-deterministic RSE to replication to a different rse location."""
 
         for item in items:
@@ -66,7 +65,7 @@ def create_replication_rules(self, items: list, src_rse: str, dst_rse: str):
 
     def delete_replication_rules(self, items: list, rse: str, state: str = None):
 
-        rules = self.get_item_replication_rules(items, rse, state)
+        rules = self.get_items_replication_rules(items, rse, state)
 
         for rule in rules:
             self.rucio.delete_replication_rule(rule["id"])
@@ -88,23 +87,14 @@ def download(self, item, rse):
 
     def download_available(self, items: list, rse: str):
 
-        available_names = self.get_available_names(items, rse)
+        available_names = self.get_items_replication_rules(items, rse, "OK")
         available_items = [ item for item in items if item.id in available_names ] 
 
         for item in available_items:
             self.download(item, rse)
 
         return True
 
-    def get_available_names(self, items: list, rse: str):
-
-        item_ids = [ item.id for item in items ]
-        rules = [ 
-            rule["name"] for rule in self.rucio.list_replication_rules(filters={"scope":self.rucio.account}) if rule["name"] in item_ids and rule["state"] == "OK"
-        ]
-
-        return rules
-
     # TODO This should check is replication rules exist for this item and rse. Found using filters in list_replication_rules somewhat difficult.
     def get_existing_replicas(self, item: dict, rse: str = None):
 
@@ -125,12 +115,12 @@ def get_existing_replicas(self, item: dict, rse: str = None):
 
         return replicas
 
-    def get_item_replication_rules(self, items: list, rse: str, state: str = None):
+    def get_items_replication_rules(self, items: list, rse: str, state: str = None):
         """Determine existing replication rules for a given item and state, if provided. """
 
         item_ids = [ item.id for item in items ]
         rules = [ 
-            x for x in self.rucio.list_replication_rules(filters={ "scope" : self.rucio.account })
+            x for x in self.rucio.list_replication_rules(filters={ "scope" : self.rucio.account, "rse_expression": rse })
             if x["name"] in item_ids and ( state is None or x["state"] == state )
         ]
 
@@ -139,9 +129,7 @@ def get_item_replication_rules(self, items: list, rse: str, state: str = None):
     def replication_availability(self, items: list, rse: str):
         """Determine the number of available replications based off the replication rule state. """
 
-        item_ids = [ item.id for item in items ]
-
-        rules = [ x for x in self.rucio.list_replication_rules(filters={"scope":self.rucio.account}) if x["name"] in item_ids ]
+        rules = self.get_items_replication_rules(items, rse)
 
         available = [ rule for rule in rules if rule["state"] == "OK"]
         stuck = [ rule for rule in rules if rule["state"] == "STUCK"]
@@ -166,5 +154,6 @@ def rucio_item(self, items: dict):
                     # TODO For multiple targets, link asset to original asset.
                     if not "alternate" in item["assets"][self.target]:
                         item["assets"][self.target]["alternate"] = {}
+
                     item["assets"][self.target]["alternate"][key] = value[0]