Merge pull request #16 from SKA-ScienceDataProcessor/feature_external…

…_data

Feature external data

.gitignore

@@ -41,3 +41,5 @@ docs/examples/ska_simulations/surface_simulation.rst
 examples/ska_simulations/atmospheric_simulation.rst
 
 docs/examples/ska_simulations/atmospheric_simulation.rst
+
+dist/

README.md

@@ -13,14 +13,21 @@ To acheive sufficient performance we take a dual pronged approach -
 using threaded libraries for shared memory processing, and the Dask
 library for distributed processing.
 
+Documentation is [here](https://timcornwell.gitlab.io/rascil/)
+
 RASCIL can be installed using pip::
 
     pip install rascil
-
-Other installation instructions such as for development are available from the [documentation](https://timcornwell.gitlab.io/rascil/)
-
-RASCIL code is hosted on [Github](https://github.com/SKA-ScienceDataProcessor/rascil)
-
+
+You may also need the rascil data for simulations:
+
+    wget https://timcornwell.gitlab.io/rascil/rascil_data.tgz
+    tar zxf rascil_data.tgz
+    export RASCIL_DATA=`pwd`/rascil_data
+
+Alternatively, if you wish to develop using RASCIL then you can 
+clone from the GitHub repository:
+
+    git clone https://github.com/SKA-ScienceDataProcessor/rascil
+
 RASCIL CI/CD occurs on  [Gitlab](https://gitlab.com/timcornwell/rascil)
-
-The documentation builds occur in the pipeline on GitLab.  

docker/rascil-v0.1.1/Dockerfile

@@ -0,0 +1,28 @@
+FROM ubuntu:18.04
+
+LABEL author="Tim Cornwell <realtimcornwell@gmail.com>" \
+      description="RASCIL reference image based on PyPI" \
+      license="Apache2.0"
+
+# Install required system packages.
+ENV DEBIAN_FRONTEND=noninteractive
+
+RUN apt-get update -y && apt-get install -y \
+    ca-certificates \
+    gosu git \
+    libxml2-dev liblapack-dev libcfitsio-dev libgtkmm-3.0-dev \
+    cmake g++ build-essential gfortran libpng-dev casacore-dev \
+    python3 python3-dev python3-pip python3-setuptools && \
+    apt-get clean -y && \
+    rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
+
+# Set runtime environment variables.
+ENV LC_ALL=C.UTF-8
+ENV LANG=C.UTF-8
+
+RUN pip3 install Cython setuptools && \
+    pip3 install "numpy>=1.16.0" && \
+    pip3 install rascil && \
+    rm -rf /root/.cache
+
+CMD ["/bin/bash"]

examples/comparisons/compare_imaging_sim2.py

@@ -2,8 +2,7 @@
 
 """
 import sys
-from rascil.data_models.parameters import rascil_path
-
+from rascil.data_models.parameters import rascil_path, rascil_data_path
 import numpy
 
 from rascil.processing_components.visibility.base import create_blockvisibility_from_ms
@@ -31,7 +30,7 @@
 
     # Test requires that casa be installed
     try:
-        bvt = create_blockvisibility_from_ms(rascil_path('data/vis/sim-2.ms'), channum=[35, 36, 37, 38, 39])[0]
+        bvt = create_blockvisibility_from_ms(rascil_data_path('vis/sim-2.ms'), channum=[35, 36, 37, 38, 39])[0]
         bvt.configuration.diameter[...] = 35.0
         vt = convert_blockvisibility_to_visibility(bvt)
         vt = convert_visibility_to_stokes(vt)
@@ -63,4 +62,4 @@
         assert numpy.abs(qa.data['medianabs'] - 0.0005590537883509844) < 1e-7, qa
     except ModuleNotFoundError as error:
         print(error.__class__.__name__ + ": " + error.msg)
-        pass
+        pass

examples/notebooks/mosaic_serial.ipynb

@@ -42,7 +42,7 @@
     "\n",
     "from rascil.data_models.polarisation import PolarisationFrame\n",
     "\n",
-    "from rascil.data_models.parameters import rascil_path\n",
+    "from rascil.data_models.parameters import rascil_path, rascil_data_path\n",
     "\n",
     "from rascil.processing_components import copy_image\n",
     "from rascil.processing_components import  create_visibility_from_ms, show_image, create_pb, create_image_from_visibility, \\\n",
@@ -90,7 +90,7 @@
    },
    "outputs": [],
    "source": [
-    "vis_list=create_visibility_from_ms(rascil_path('data/vis/xcasa.ms'))"
+    "vis_list=create_visibility_from_ms(rascil_data_path('vis/xcasa.ms'))"
    ]
   },
   {
@@ -208,4 +208,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}

examples/notebooks/mpc_simulation.ipynb

@@ -25,7 +25,7 @@
     "%matplotlib inline\n",
     "\n",
     "import sys\n",
-    "from rascil.data_models import rascil_path\n",
+    "from rascil.data_models import rascil_path, rascil_data_path\n",
     "\n",
     "results_dir = '/tmp/'\n",
     "\n",
@@ -268,4 +268,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 2
-}
+}

examples/notebooks/plot-imaging-results.ipynb

@@ -29,7 +29,7 @@
     "import numpy\n",
     "from matplotlib import pyplot as plt\n",
     "\n",
-    "from rascil.data_models.parameters import rascil_path\n",
+    "from rascil.data_models.parameters import rascil_path, rascil_data_path\n",
     "from rascil.processing_components.image.operations import show_image, qa_image, import_image_from_fits\n",
     "\n",
     "import logging\n",

examples/pipelines/mpccal/mpccal_arlexecute_pipeline.py

@@ -19,7 +19,7 @@
 from astropy import units as u
 from astropy.coordinates import SkyCoord
 
-from rascil.data_models import SkyModel, rascil_path, PolarisationFrame
+from rascil.data_models import SkyModel, rascil_path, rascil_data_path, PolarisationFrame
 from rascil.processing_components import create_gaintable_from_blockvisibility, \
     export_image_to_fits, \
     remove_neighbouring_components, find_skycomponents, \
@@ -161,7 +161,7 @@
                             reverse=True)
     print("Number of components in simulation %d" % len(all_components))
 
-    screen = import_image_from_fits(rascil_path('data/models/test_mpc_screen.fits'))
+    screen = import_image_from_fits(rascil_data_path('models/test_mpc_screen.fits'))
     all_gaintables = create_gaintable_from_screen(block_vis, all_components, screen)
 
     all_skymodel = [SkyModel(components=[all_components[i]], gaintable=all_gaintables[i])

examples/pipelines/ska-pipelines/ska_continuum_imaging_rsexecute_pipeline.py

@@ -3,8 +3,7 @@
 """
 # # Pipeline processing using Dask
 
-from rascil.data_models.parameters import rascil_path
-
+from rascil.data_models.parameters import rascil_path, rascil_data_path
 results_dir = rascil_path('test_results')
 dask_dir = rascil_path('test_results/dask-work-space')
 

examples/pipelines/ska-pipelines/ska_ical_rsexecute_pipeline.py

@@ -4,8 +4,7 @@
 
 # # Pipeline processing using Dask
 
-from rascil.data_models.parameters import rascil_path
-
+from rascil.data_models.parameters import rascil_path, rascil_data_path
 results_dir = rascil_path('test_results')
 dask_dir = rascil_path('test_results/dask-work-space')
 

examples/pipelines/ska-pipelines/ska_imaging_rsexecute_pipeline.py

@@ -4,8 +4,7 @@
 
 # # Pipeline processing using Dask
 
-from rascil.data_models.parameters import rascil_path
-
+from rascil.data_models.parameters import rascil_path, rascil_data_path
 results_dir = rascil_path('test_results')
 dask_dir = rascil_path('test_results/dask-work-space')
 

examples/scripts/dprepb_rsexecute_pipeline.py

@@ -9,7 +9,7 @@
 from dask.distributed import Client
 
 # These are the RASCIL functions we need
-from rascil.data_models import PolarisationFrame, rascil_path
+from rascil.data_models import PolarisationFrame, rascil_path, rascil_data_path
 from rascil.processing_components import create_visibility_from_ms, \
     create_visibility_from_rows, append_visibility, convert_visibility_to_stokes, \
     vis_select_uvrange, deconvolve_cube, restore_cube, export_image_to_fits, qa_image, \
@@ -85,7 +85,7 @@ def init_logging():
         context = '2d'
         vis_slices = 1
 
-    input_vis = [rascil_path('data/vis/sim-1.ms'), rascil_path('data/vis/sim-2.ms')]
+    input_vis = [rascil_data_path('vis/sim-1.ms'), rascil_data_path('vis/sim-2.ms')]
 
     import time
 

examples/scripts/imaging_sim2.py

@@ -23,7 +23,7 @@
 if __name__ == '__main__':
     results_dir = './'
 
-    bvt = create_blockvisibility_from_ms(rascil_path('data/vis/sim-2.ms'), start_chan=35,
+    bvt = create_blockvisibility_from_ms(rascil_data_path('vis/sim-2.ms'), start_chan=35,
                                          end_chan=39)[0]
     bvt.configuration.diameter[...] = 35.0
     vt = convert_blockvisibility_to_visibility(bvt)

examples/ska_simulations/atmospheric_simulation.py

@@ -21,7 +21,7 @@
 from astropy import units as u
 from astropy.coordinates import SkyCoord
 
-from rascil.data_models.parameters import rascil_path
+from rascil.data_models.parameters import rascil_path, rascil_data_path
 from rascil.data_models.polarisation import PolarisationFrame
 from rascil.processing_components import plot_azel, \
     plot_uvcoverage, find_pb_width_null, create_simulation_components, \
@@ -99,7 +99,7 @@ def cli_parser():
                      help='Export images in fits format?')
     par.add_argument('--use_agg', type=str, default="True",
                      help='Use Agg matplotlib backend?')
-    default_shared_path = rascil_path("data/configurations")
+    default_shared_path = rascil_data_path("configurations")
     par.add_argument('--shared_directory', type=str, default=default_shared_path,
                      help='Location of configuration files (default is RASCIL data/configurations)')
     # Dask parameters; matched to P3

examples/ska_simulations/pointing_simulation.py

@@ -21,7 +21,7 @@
 from astropy import units as u
 from astropy.coordinates import SkyCoord
 
-from rascil.data_models.parameters import rascil_path
+from rascil.data_models.parameters import rascil_path, rascil_data_path
 from rascil.data_models.polarisation import PolarisationFrame
 from rascil.processing_components import plot_azel, \
     plot_uvcoverage, find_pb_width_null, create_simulation_components, \
@@ -95,7 +95,7 @@ def cli_parser():
                      help='Use Agg matplotlib backend?')
     par.add_argument('--use_radec', type=str, default="False",
                      help='Calculate primary beams in RADEC?')
-    default_shared_path = rascil_path("data/configurations")
+    default_shared_path = rascil_data_path("configurations")
     par.add_argument('--shared_directory', type=str, default=default_shared_path,
                      help='Location of configuration files (default is RASCIL data/configurations)')
     # Dask parameters; matched to P3
@@ -121,7 +121,7 @@ def cli_parser():
                      help='Scale (arcsec) for random dynamic errors (varies with time and dish)')
     par.add_argument('--pointing_file', type=str, default=None, help="Pointing file")
     par.add_argument('--pointing_directory', type=str,
-                     default=rascil_path('data/models'),
+                     default=rascil_data_path('models'),
                      help='Location of wind-induced pointing files')
     return par
 

examples/ska_simulations/surface_simulation.py

@@ -21,7 +21,7 @@
 from astropy import units as u
 from astropy.coordinates import SkyCoord
 
-from rascil.data_models.parameters import rascil_path
+from rascil.data_models.parameters import rascil_path, rascil_data_path
 from rascil.data_models.polarisation import PolarisationFrame
 from rascil.processing_components import show_image, qa_image, export_image_to_fits, \
     create_vp, \

rascil/data_models/parameters.py

@@ -53,7 +53,7 @@ def processing_function(idatastruct1, idatastruct2, ..., *kwargs):
 
 """
 
-__all__ = ['rascil_path', 'get_parameter']
+__all__ = ['rascil_path', 'rascil_data_path', 'get_parameter']
 
 import logging
 import os
@@ -65,7 +65,7 @@ def rascil_path(path):
     """Converts a path that might be relative to RASCIL root into an
     absolute path::
 
-        rascil_path('data/models/SKA1_LOW_beam.fits')
+        rascil_data_path('models/SKA1_LOW_beam.fits')
         '/Users/timcornwell/Code/rascil/data/models/SKA1_LOW_beam.fits'
 
     :param path:
@@ -75,6 +75,22 @@ def rascil_path(path):
     rascilhome = os.getenv('RASCIL', project_root)
     return os.path.join(rascilhome, path)
 
+def rascil_data_path(path):
+    """Converts a path that might be relative to RASCIL root into an
+    absolute path::
+
+        rascil_data_path('models/SKA1_LOW_beam.fits')
+        '/Users/timcornwell/Code/rascil/data/models/SKA1_LOW_beam.fits'
+
+    :param path:
+    :return: absolute path
+    """
+    rascil_data_home = os.getenv('RASCIL_DATA', None)
+    if rascil_data_home is None:
+        return rascil_path('data/'+path)
+    else:
+        return os.path.join(rascil_data_home, path)
+
 
 def get_parameter(kwargs, key, default=None):
     """ Get a specified named value for this (calling) function

rascil/processing_components/imaging/primary_beams.py

@@ -12,7 +12,7 @@
 from astropy import constants as const
 
 from rascil.data_models.memory_data_models import Image
-from rascil.data_models.parameters import rascil_path
+from rascil.data_models.parameters import rascil_path, rascil_data_path
 from rascil.processing_components.image.operations import import_image_from_fits, reproject_image
 from rascil.processing_components.image.operations import create_image_from_array, create_empty_image_like, fft_image, pad_image
 
@@ -91,29 +91,29 @@ def create_vp(model, telescope='MID', pointingcentre=None, padding=4, use_local=
         return create_vp_generic(model, pointingcentre=pointingcentre, diameter=15.0, blockage=0.0, use_local=use_local)
     elif telescope == 'MID_GRASP':
         log.debug("create_vp: Using GRASP model for MID voltage pattern")
-        real_vp = import_image_from_fits(rascil_path('data/models/MID_GRASP_VP_real.fits'))
-        imag_vp = import_image_from_fits(rascil_path('data/models/MID_GRASP_VP_imag.fits'))
+        real_vp = import_image_from_fits(rascil_data_path('models/MID_GRASP_VP_real.fits'))
+        imag_vp = import_image_from_fits(rascil_data_path('models/MID_GRASP_VP_imag.fits'))
         real_vp.data = real_vp.data + 1j * imag_vp.data
         real_vp.data /= numpy.max(numpy.abs(real_vp.data))
         return real_vp
     elif telescope == 'MID_FEKO_B1':
         log.debug("create_vp: Using FEKO model for MID voltage pattern")
-        real_vp = import_image_from_fits(rascil_path('data/models/MID_FEKO_VP_B1_45_0765_real.fits'))
-        imag_vp = import_image_from_fits(rascil_path('data/models/MID_FEKO_VP_B1_45_0765_imag.fits'))
+        real_vp = import_image_from_fits(rascil_data_path('models/MID_FEKO_VP_B1_45_0765_real.fits'))
+        imag_vp = import_image_from_fits(rascil_data_path('models/MID_FEKO_VP_B1_45_0765_imag.fits'))
         real_vp.data = real_vp.data + 1j * imag_vp.data
         real_vp.data /= numpy.max(numpy.abs(real_vp.data))
         return real_vp
     elif telescope == 'MID_FEKO_B2':
         log.debug("create_vp: Using FEKO model for MID voltage pattern")
-        real_vp = import_image_from_fits(rascil_path('data/models/MID_FEKO_VP_B2_45_1360_real.fits'))
-        imag_vp = import_image_from_fits(rascil_path('data/models/MID_FEKO_VP_B2_45_1360_imag.fits'))
+        real_vp = import_image_from_fits(rascil_data_path('models/MID_FEKO_VP_B2_45_1360_real.fits'))
+        imag_vp = import_image_from_fits(rascil_data_path('models/MID_FEKO_VP_B2_45_1360_imag.fits'))
         real_vp.data = real_vp.data + 1j * imag_vp.data
         real_vp.data /= numpy.max(numpy.abs(real_vp.data))
         return real_vp
     elif telescope == 'MID_FEKO_Ku':
         log.debug("create_vp: Using FEKO model for MID voltage pattern")
-        real_vp = import_image_from_fits(rascil_path('data/models/MID_FEKO_VP_Ku_45_12179_real.fits'))
-        imag_vp = import_image_from_fits(rascil_path('data/models/MID_FEKO_VP_Ku_45_12179_imag.fits'))
+        real_vp = import_image_from_fits(rascil_data_path('models/MID_FEKO_VP_Ku_45_12179_real.fits'))
+        imag_vp = import_image_from_fits(rascil_data_path('models/MID_FEKO_VP_Ku_45_12179_imag.fits'))
         real_vp.data = real_vp.data + 1j * imag_vp.data
         real_vp.data /= numpy.max(numpy.abs(real_vp.data))
         return real_vp
@@ -331,7 +331,7 @@ def create_low_test_beam(model: Image, use_local=True) -> Image:
     :param model: Template image
     :return: Image
     """
-    beam = import_image_from_fits(rascil_path('data/models/SKA1_LOW_beam.fits'))
+    beam = import_image_from_fits(rascil_data_path('models/SKA1_LOW_beam.fits'))
 
     # Scale the image cellsize to account for the different in frequencies. Eventually we will want to
     # use a frequency cube
@@ -381,7 +381,7 @@ def create_low_test_vp(model: Image, use_local=True) -> Image:
     :return: Image
     """
 
-    beam = import_image_from_fits(rascil_path('data/models/SKA1_LOW_beam.fits'))
+    beam = import_image_from_fits(rascil_data_path('models/SKA1_LOW_beam.fits'))
     beam.data = numpy.sqrt(beam.data).astype('complex')
 
     # Scale the image cellsize to account for the different in frequencies. Eventually we will want to

rascil/processing_components/simulation/configurations.py

@@ -11,7 +11,7 @@
 
 from rascil.processing_components.util.coordinate_support import xyz_at_latitude
 from rascil.data_models.memory_data_models import Configuration
-from rascil.data_models.parameters import rascil_path, get_parameter
+from rascil.data_models.parameters import rascil_path, rascil_data_path, get_parameter
 from rascil.processing_components.util.installation_checks import check_data_directory
 
 import logging