From 8d72e861190b9dbde117eaf7465bc13a2b396516 Mon Sep 17 00:00:00 2001 From: Todd Morse Date: Mon, 6 Mar 2023 13:47:26 -0500 Subject: [PATCH] Add support for http/https inputs (#103) --- miniwdl-plugins/s3upload/miniwdl_s3upload.py | 3 ++- test/test_wdl.py | 15 +++++++++++++++ version | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/miniwdl-plugins/s3upload/miniwdl_s3upload.py b/miniwdl-plugins/s3upload/miniwdl_s3upload.py index 02cea5ed..0b6cd38c 100644 --- a/miniwdl-plugins/s3upload/miniwdl_s3upload.py +++ b/miniwdl-plugins/s3upload/miniwdl_s3upload.py @@ -22,6 +22,7 @@ """ import os +import re import subprocess import threading import json @@ -119,7 +120,7 @@ def remove_temporary_flag(s3uri, retry=0): def inode(link: str): - if link.startswith("s3://"): + if re.match(r'^\w+://', link): return link st = os.stat(os.path.realpath(link)) return (st.st_dev, st.st_ino) diff --git a/test/test_wdl.py b/test/test_wdl.py index 1b183034..335633c0 100644 --- a/test/test_wdl.py +++ b/test/test_wdl.py @@ -387,6 +387,21 @@ def test_simple_sfn_wdl_workflow(self): json.loads(messages[0]["Body"])["detail"]["lastCompletedStage"], "run" ) + def test_https_inputs(self): + output_prefix = "out-https-1" + sfn_input: Dict[str, Any] = { + "RUN_WDL_URI": f"s3://{self.wdl_obj.bucket_name}/{self.wdl_obj.key}", + "OutputPrefix": f"s3://{self.input_obj.bucket_name}/{output_prefix}", + "Input": { + "Run": { + "hello": "https://raw.githubusercontent.com/chanzuckerberg/czid-workflows/main/README.md", + "docker_image_id": "ubuntu", + } + }, + } + + self._wait_sfn(sfn_input, self.single_sfn_arn) + def test_failing_wdl_workflow(self): output_prefix = "out-fail-1" sfn_input: Dict[str, Any] = { diff --git a/version b/version index 968e750a..ec7b9678 100644 --- a/version +++ b/version @@ -1 +1 @@ -v1.3.2 +v1.4.0 \ No newline at end of file