From b6414d8084d9e5784bb170654e882116719640d0 Mon Sep 17 00:00:00 2001 From: timcornwell Date: Sat, 7 Mar 2020 12:10:31 +0000 Subject: [PATCH 1/2] Separate data dir --- .gitignore | 2 ++ README.md | 21 +++++++++----- docker/rascil-v0.1.1/Dockerfile | 28 +++++++++++++++++++ examples/comparisons/compare_imaging_sim2.py | 7 ++--- examples/notebooks/mosaic_serial.ipynb | 6 ++-- examples/notebooks/mpc_simulation.ipynb | 4 +-- examples/notebooks/plot-imaging-results.ipynb | 2 +- .../mpccal/mpccal_arlexecute_pipeline.py | 4 +-- ...ka_continuum_imaging_rsexecute_pipeline.py | 3 +- .../ska_ical_rsexecute_pipeline.py | 3 +- .../ska_imaging_rsexecute_pipeline.py | 3 +- examples/scripts/dprepb_rsexecute_pipeline.py | 4 +-- examples/scripts/imaging_sim2.py | 2 +- .../ska_simulations/atmospheric_simulation.py | 4 +-- .../ska_simulations/pointing_simulation.py | 6 ++-- .../ska_simulations/surface_simulation.py | 2 +- rascil/data_models/parameters.py | 20 +++++++++++-- .../imaging/primary_beams.py | 22 +++++++-------- .../simulation/configurations.py | 2 +- .../simulation/pointing.py | 4 +-- .../processing_components/simulation/rfi.py | 15 ++++++---- .../simulation/testing_support.py | 14 +++++----- tests/data_models/test_buffer_data_models.py | 2 +- tests/data_models/test_data_model_helpers.py | 2 +- .../test_atmospheric_screen.py | 10 +++---- .../test_configurations.py | 2 +- .../test_export_ms_rascil.py | 5 ++-- .../test_griddata_gridding.py | 2 +- .../test_griddata_kernels.py | 2 +- .../test_griddata_operations.py | 2 +- .../test_image_deconvolution.py | 2 +- .../test_image_deconvolution_msmfs.py | 2 +- .../test_image_gather_scatter.py | 2 +- .../test_image_gradients.py | 6 ++-- .../test_image_operations.py | 2 +- tests/processing_components/test_imaging.py | 2 +- .../test_imaging_functions.py | 2 +- .../processing_components/test_imaging_ms.py | 2 +- .../processing_components/test_imaging_ng.py | 2 +- .../test_imaging_params.py | 2 +- tests/processing_components/test_noise.py | 2 +- tests/processing_components/test_pointing.py | 3 +- .../test_pointing_circlecut.py | 3 +- .../test_pointing_time_series.py | 3 +- .../test_primary_beam_illumination.py | 2 +- .../test_primary_beam_zernikes.py | 2 +- .../test_primary_beams.py | 2 +- .../test_skycomponent.py | 2 +- .../test_skycomponent_insert.py | 2 +- tests/processing_components/test_skymodel.py | 2 +- .../test_skymodel_mpc.py | 2 +- tests/processing_components/test_surface.py | 3 +- .../test_testing_support.py | 2 +- .../test_visibility_ms.py | 3 +- .../test_visibility_uvfits.py | 2 +- tests/processing_components/test_weighting.py | 2 +- tests/workflows/test_calibrate_rsexecute.py | 2 +- tests/workflows/test_calibrate_serial.py | 2 +- tests/workflows/test_image_rsexecute.py | 2 +- .../test_imaging_deconvolve_rsexecute.py | 2 +- .../test_imaging_deconvolve_serial.py | 2 +- tests/workflows/test_imaging_rsexecute.py | 2 +- .../test_mid_simulations_rsexecute.py | 9 +++--- tests/workflows/test_mpc_rsexecute.py | 2 +- .../workflows/test_pipelines_mpc_rsexecute.py | 4 +-- tests/workflows/test_rcal_serial.py | 2 +- 66 files changed, 171 insertions(+), 124 deletions(-) create mode 100644 docker/rascil-v0.1.1/Dockerfile diff --git a/.gitignore b/.gitignore index b7cd6394..7ce1a26e 100644 --- a/.gitignore +++ b/.gitignore @@ -41,3 +41,5 @@ docs/examples/ska_simulations/surface_simulation.rst examples/ska_simulations/atmospheric_simulation.rst docs/examples/ska_simulations/atmospheric_simulation.rst + +dist/ diff --git a/README.md b/README.md index 17c51b13..992829a7 100644 --- a/README.md +++ b/README.md @@ -13,14 +13,21 @@ To acheive sufficient performance we take a dual pronged approach - using threaded libraries for shared memory processing, and the Dask library for distributed processing. +Documentation is [here](https://timcornwell.gitlab.io/rascil/) + RASCIL can be installed using pip:: pip install rascil - -Other installation instructions such as for development are available from the [documentation](https://timcornwell.gitlab.io/rascil/) - -RASCIL code is hosted on [Github](https://github.com/SKA-ScienceDataProcessor/rascil) - + +You may also need the rascil data for simulations: + + wget https://timcornwell.gitlab.io/rascil/rascil_data.tgz + tar zxf rascil_data.tgz + export RASCIL_DATA=`pwd`/rascil_data + +Alternatively, if you wish to develop using RASCIL then you can +clone from the GitHub repository: + + git clone https://github.com/SKA-ScienceDataProcessor/rascil + RASCIL CI/CD occurs on [Gitlab](https://gitlab.com/timcornwell/rascil) - -The documentation builds occur in the pipeline on GitLab. diff --git a/docker/rascil-v0.1.1/Dockerfile b/docker/rascil-v0.1.1/Dockerfile new file mode 100644 index 00000000..ed0c2d50 --- /dev/null +++ b/docker/rascil-v0.1.1/Dockerfile @@ -0,0 +1,28 @@ +FROM ubuntu:18.04 + +LABEL author="Tim Cornwell " \ + description="RASCIL reference image based on PyPI" \ + license="Apache2.0" + +# Install required system packages. +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update -y && apt-get install -y \ + ca-certificates \ + gosu git \ + libxml2-dev liblapack-dev libcfitsio-dev libgtkmm-3.0-dev \ + cmake g++ build-essential gfortran libpng-dev casacore-dev \ + python3 python3-dev python3-pip python3-setuptools && \ + apt-get clean -y && \ + rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* + +# Set runtime environment variables. +ENV LC_ALL=C.UTF-8 +ENV LANG=C.UTF-8 + +RUN pip3 install Cython setuptools && \ + pip3 install "numpy>=1.16.0" && \ + pip3 install rascil && \ + rm -rf /root/.cache + +CMD ["/bin/bash"] diff --git a/examples/comparisons/compare_imaging_sim2.py b/examples/comparisons/compare_imaging_sim2.py index 109b56a9..3d1ff52a 100644 --- a/examples/comparisons/compare_imaging_sim2.py +++ b/examples/comparisons/compare_imaging_sim2.py @@ -2,8 +2,7 @@ """ import sys -from rascil.data_models.parameters import rascil_path - +from rascil.data_models.parameters import rascil_path, rascil_data_path import numpy from rascil.processing_components.visibility.base import create_blockvisibility_from_ms @@ -31,7 +30,7 @@ # Test requires that casa be installed try: - bvt = create_blockvisibility_from_ms(rascil_path('data/vis/sim-2.ms'), channum=[35, 36, 37, 38, 39])[0] + bvt = create_blockvisibility_from_ms(rascil_data_path('vis/sim-2.ms'), channum=[35, 36, 37, 38, 39])[0] bvt.configuration.diameter[...] = 35.0 vt = convert_blockvisibility_to_visibility(bvt) vt = convert_visibility_to_stokes(vt) @@ -63,4 +62,4 @@ assert numpy.abs(qa.data['medianabs'] - 0.0005590537883509844) < 1e-7, qa except ModuleNotFoundError as error: print(error.__class__.__name__ + ": " + error.msg) - pass \ No newline at end of file + pass diff --git a/examples/notebooks/mosaic_serial.ipynb b/examples/notebooks/mosaic_serial.ipynb index 3ce1c682..6dd5262c 100644 --- a/examples/notebooks/mosaic_serial.ipynb +++ b/examples/notebooks/mosaic_serial.ipynb @@ -42,7 +42,7 @@ "\n", "from rascil.data_models.polarisation import PolarisationFrame\n", "\n", - "from rascil.data_models.parameters import rascil_path\n", + "from rascil.data_models.parameters import rascil_path, rascil_data_path\n", "\n", "from rascil.processing_components import copy_image\n", "from rascil.processing_components import create_visibility_from_ms, show_image, create_pb, create_image_from_visibility, \\\n", @@ -90,7 +90,7 @@ }, "outputs": [], "source": [ - "vis_list=create_visibility_from_ms(rascil_path('data/vis/xcasa.ms'))" + "vis_list=create_visibility_from_ms(rascil_data_path('vis/xcasa.ms'))" ] }, { @@ -208,4 +208,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/examples/notebooks/mpc_simulation.ipynb b/examples/notebooks/mpc_simulation.ipynb index 5d71b8a0..d77ae240 100644 --- a/examples/notebooks/mpc_simulation.ipynb +++ b/examples/notebooks/mpc_simulation.ipynb @@ -25,7 +25,7 @@ "%matplotlib inline\n", "\n", "import sys\n", - "from rascil.data_models import rascil_path\n", + "from rascil.data_models import rascil_path, rascil_data_path\n", "\n", "results_dir = '/tmp/'\n", "\n", @@ -268,4 +268,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/examples/notebooks/plot-imaging-results.ipynb b/examples/notebooks/plot-imaging-results.ipynb index 53a3941e..7c2367b7 100644 --- a/examples/notebooks/plot-imaging-results.ipynb +++ b/examples/notebooks/plot-imaging-results.ipynb @@ -29,7 +29,7 @@ "import numpy\n", "from matplotlib import pyplot as plt\n", "\n", - "from rascil.data_models.parameters import rascil_path\n", + "from rascil.data_models.parameters import rascil_path, rascil_data_path\n", "from rascil.processing_components.image.operations import show_image, qa_image, import_image_from_fits\n", "\n", "import logging\n", diff --git a/examples/pipelines/mpccal/mpccal_arlexecute_pipeline.py b/examples/pipelines/mpccal/mpccal_arlexecute_pipeline.py index 5641900f..fb7cf787 100644 --- a/examples/pipelines/mpccal/mpccal_arlexecute_pipeline.py +++ b/examples/pipelines/mpccal/mpccal_arlexecute_pipeline.py @@ -19,7 +19,7 @@ from astropy import units as u from astropy.coordinates import SkyCoord -from rascil.data_models import SkyModel, rascil_path, PolarisationFrame +from rascil.data_models import SkyModel, rascil_path, rascil_data_path, PolarisationFrame from rascil.processing_components import create_gaintable_from_blockvisibility, \ export_image_to_fits, \ remove_neighbouring_components, find_skycomponents, \ @@ -161,7 +161,7 @@ reverse=True) print("Number of components in simulation %d" % len(all_components)) - screen = import_image_from_fits(rascil_path('data/models/test_mpc_screen.fits')) + screen = import_image_from_fits(rascil_data_path('models/test_mpc_screen.fits')) all_gaintables = create_gaintable_from_screen(block_vis, all_components, screen) all_skymodel = [SkyModel(components=[all_components[i]], gaintable=all_gaintables[i]) diff --git a/examples/pipelines/ska-pipelines/ska_continuum_imaging_rsexecute_pipeline.py b/examples/pipelines/ska-pipelines/ska_continuum_imaging_rsexecute_pipeline.py index 207c0103..821d7423 100644 --- a/examples/pipelines/ska-pipelines/ska_continuum_imaging_rsexecute_pipeline.py +++ b/examples/pipelines/ska-pipelines/ska_continuum_imaging_rsexecute_pipeline.py @@ -3,8 +3,7 @@ """ # # Pipeline processing using Dask -from rascil.data_models.parameters import rascil_path - +from rascil.data_models.parameters import rascil_path, rascil_data_path results_dir = rascil_path('test_results') dask_dir = rascil_path('test_results/dask-work-space') diff --git a/examples/pipelines/ska-pipelines/ska_ical_rsexecute_pipeline.py b/examples/pipelines/ska-pipelines/ska_ical_rsexecute_pipeline.py index d27c53ab..181c88b0 100644 --- a/examples/pipelines/ska-pipelines/ska_ical_rsexecute_pipeline.py +++ b/examples/pipelines/ska-pipelines/ska_ical_rsexecute_pipeline.py @@ -4,8 +4,7 @@ # # Pipeline processing using Dask -from rascil.data_models.parameters import rascil_path - +from rascil.data_models.parameters import rascil_path, rascil_data_path results_dir = rascil_path('test_results') dask_dir = rascil_path('test_results/dask-work-space') diff --git a/examples/pipelines/ska-pipelines/ska_imaging_rsexecute_pipeline.py b/examples/pipelines/ska-pipelines/ska_imaging_rsexecute_pipeline.py index e61316f7..f94dcc40 100644 --- a/examples/pipelines/ska-pipelines/ska_imaging_rsexecute_pipeline.py +++ b/examples/pipelines/ska-pipelines/ska_imaging_rsexecute_pipeline.py @@ -4,8 +4,7 @@ # # Pipeline processing using Dask -from rascil.data_models.parameters import rascil_path - +from rascil.data_models.parameters import rascil_path, rascil_data_path results_dir = rascil_path('test_results') dask_dir = rascil_path('test_results/dask-work-space') diff --git a/examples/scripts/dprepb_rsexecute_pipeline.py b/examples/scripts/dprepb_rsexecute_pipeline.py index 9e910e47..b2d58351 100644 --- a/examples/scripts/dprepb_rsexecute_pipeline.py +++ b/examples/scripts/dprepb_rsexecute_pipeline.py @@ -9,7 +9,7 @@ from dask.distributed import Client # These are the RASCIL functions we need -from rascil.data_models import PolarisationFrame, rascil_path +from rascil.data_models import PolarisationFrame, rascil_path, rascil_data_path from rascil.processing_components import create_visibility_from_ms, \ create_visibility_from_rows, append_visibility, convert_visibility_to_stokes, \ vis_select_uvrange, deconvolve_cube, restore_cube, export_image_to_fits, qa_image, \ @@ -85,7 +85,7 @@ def init_logging(): context = '2d' vis_slices = 1 - input_vis = [rascil_path('data/vis/sim-1.ms'), rascil_path('data/vis/sim-2.ms')] + input_vis = [rascil_data_path('vis/sim-1.ms'), rascil_data_path('vis/sim-2.ms')] import time diff --git a/examples/scripts/imaging_sim2.py b/examples/scripts/imaging_sim2.py index bd107c7a..98bdf977 100644 --- a/examples/scripts/imaging_sim2.py +++ b/examples/scripts/imaging_sim2.py @@ -23,7 +23,7 @@ if __name__ == '__main__': results_dir = './' - bvt = create_blockvisibility_from_ms(rascil_path('data/vis/sim-2.ms'), start_chan=35, + bvt = create_blockvisibility_from_ms(rascil_data_path('vis/sim-2.ms'), start_chan=35, end_chan=39)[0] bvt.configuration.diameter[...] = 35.0 vt = convert_blockvisibility_to_visibility(bvt) diff --git a/examples/ska_simulations/atmospheric_simulation.py b/examples/ska_simulations/atmospheric_simulation.py index bccd2bc8..405dcdc3 100644 --- a/examples/ska_simulations/atmospheric_simulation.py +++ b/examples/ska_simulations/atmospheric_simulation.py @@ -21,7 +21,7 @@ from astropy import units as u from astropy.coordinates import SkyCoord -from rascil.data_models.parameters import rascil_path +from rascil.data_models.parameters import rascil_path, rascil_data_path from rascil.data_models.polarisation import PolarisationFrame from rascil.processing_components import plot_azel, \ plot_uvcoverage, find_pb_width_null, create_simulation_components, \ @@ -99,7 +99,7 @@ def cli_parser(): help='Export images in fits format?') par.add_argument('--use_agg', type=str, default="True", help='Use Agg matplotlib backend?') - default_shared_path = rascil_path("data/configurations") + default_shared_path = rascil_data_path("configurations") par.add_argument('--shared_directory', type=str, default=default_shared_path, help='Location of configuration files (default is RASCIL data/configurations)') # Dask parameters; matched to P3 diff --git a/examples/ska_simulations/pointing_simulation.py b/examples/ska_simulations/pointing_simulation.py index c0ddc721..649fd380 100644 --- a/examples/ska_simulations/pointing_simulation.py +++ b/examples/ska_simulations/pointing_simulation.py @@ -21,7 +21,7 @@ from astropy import units as u from astropy.coordinates import SkyCoord -from rascil.data_models.parameters import rascil_path +from rascil.data_models.parameters import rascil_path, rascil_data_path from rascil.data_models.polarisation import PolarisationFrame from rascil.processing_components import plot_azel, \ plot_uvcoverage, find_pb_width_null, create_simulation_components, \ @@ -95,7 +95,7 @@ def cli_parser(): help='Use Agg matplotlib backend?') par.add_argument('--use_radec', type=str, default="False", help='Calculate primary beams in RADEC?') - default_shared_path = rascil_path("data/configurations") + default_shared_path = rascil_data_path("configurations") par.add_argument('--shared_directory', type=str, default=default_shared_path, help='Location of configuration files (default is RASCIL data/configurations)') # Dask parameters; matched to P3 @@ -121,7 +121,7 @@ def cli_parser(): help='Scale (arcsec) for random dynamic errors (varies with time and dish)') par.add_argument('--pointing_file', type=str, default=None, help="Pointing file") par.add_argument('--pointing_directory', type=str, - default=rascil_path('data/models'), + default=rascil_data_path('models'), help='Location of wind-induced pointing files') return par diff --git a/examples/ska_simulations/surface_simulation.py b/examples/ska_simulations/surface_simulation.py index dc1e9a2d..3a33d45b 100644 --- a/examples/ska_simulations/surface_simulation.py +++ b/examples/ska_simulations/surface_simulation.py @@ -21,7 +21,7 @@ from astropy import units as u from astropy.coordinates import SkyCoord -from rascil.data_models.parameters import rascil_path +from rascil.data_models.parameters import rascil_path, rascil_data_path from rascil.data_models.polarisation import PolarisationFrame from rascil.processing_components import show_image, qa_image, export_image_to_fits, \ create_vp, \ diff --git a/rascil/data_models/parameters.py b/rascil/data_models/parameters.py index 594f7afa..e1e1b031 100644 --- a/rascil/data_models/parameters.py +++ b/rascil/data_models/parameters.py @@ -53,7 +53,7 @@ def processing_function(idatastruct1, idatastruct2, ..., *kwargs): """ -__all__ = ['rascil_path', 'get_parameter'] +__all__ = ['rascil_path', 'rascil_data_path', 'get_parameter'] import logging import os @@ -65,7 +65,7 @@ def rascil_path(path): """Converts a path that might be relative to RASCIL root into an absolute path:: - rascil_path('data/models/SKA1_LOW_beam.fits') + rascil_data_path('models/SKA1_LOW_beam.fits') '/Users/timcornwell/Code/rascil/data/models/SKA1_LOW_beam.fits' :param path: @@ -75,6 +75,22 @@ def rascil_path(path): rascilhome = os.getenv('RASCIL', project_root) return os.path.join(rascilhome, path) +def rascil_data_path(path): + """Converts a path that might be relative to RASCIL root into an + absolute path:: + + rascil_data_path('models/SKA1_LOW_beam.fits') + '/Users/timcornwell/Code/rascil/data/models/SKA1_LOW_beam.fits' + + :param path: + :return: absolute path + """ + rascil_data_home = os.getenv('RASCIL_DATA', None) + if rascil_data_home is None: + return rascil_path('data/'+path) + else: + return os.path.join(rascil_data_home, path) + def get_parameter(kwargs, key, default=None): """ Get a specified named value for this (calling) function diff --git a/rascil/processing_components/imaging/primary_beams.py b/rascil/processing_components/imaging/primary_beams.py index 578ae579..0fbd25fa 100644 --- a/rascil/processing_components/imaging/primary_beams.py +++ b/rascil/processing_components/imaging/primary_beams.py @@ -12,7 +12,7 @@ from astropy import constants as const from rascil.data_models.memory_data_models import Image -from rascil.data_models.parameters import rascil_path +from rascil.data_models.parameters import rascil_path, rascil_data_path from rascil.processing_components.image.operations import import_image_from_fits, reproject_image from rascil.processing_components.image.operations import create_image_from_array, create_empty_image_like, fft_image, pad_image @@ -91,29 +91,29 @@ def create_vp(model, telescope='MID', pointingcentre=None, padding=4, use_local= return create_vp_generic(model, pointingcentre=pointingcentre, diameter=15.0, blockage=0.0, use_local=use_local) elif telescope == 'MID_GRASP': log.debug("create_vp: Using GRASP model for MID voltage pattern") - real_vp = import_image_from_fits(rascil_path('data/models/MID_GRASP_VP_real.fits')) - imag_vp = import_image_from_fits(rascil_path('data/models/MID_GRASP_VP_imag.fits')) + real_vp = import_image_from_fits(rascil_data_path('models/MID_GRASP_VP_real.fits')) + imag_vp = import_image_from_fits(rascil_data_path('models/MID_GRASP_VP_imag.fits')) real_vp.data = real_vp.data + 1j * imag_vp.data real_vp.data /= numpy.max(numpy.abs(real_vp.data)) return real_vp elif telescope == 'MID_FEKO_B1': log.debug("create_vp: Using FEKO model for MID voltage pattern") - real_vp = import_image_from_fits(rascil_path('data/models/MID_FEKO_VP_B1_45_0765_real.fits')) - imag_vp = import_image_from_fits(rascil_path('data/models/MID_FEKO_VP_B1_45_0765_imag.fits')) + real_vp = import_image_from_fits(rascil_data_path('models/MID_FEKO_VP_B1_45_0765_real.fits')) + imag_vp = import_image_from_fits(rascil_data_path('models/MID_FEKO_VP_B1_45_0765_imag.fits')) real_vp.data = real_vp.data + 1j * imag_vp.data real_vp.data /= numpy.max(numpy.abs(real_vp.data)) return real_vp elif telescope == 'MID_FEKO_B2': log.debug("create_vp: Using FEKO model for MID voltage pattern") - real_vp = import_image_from_fits(rascil_path('data/models/MID_FEKO_VP_B2_45_1360_real.fits')) - imag_vp = import_image_from_fits(rascil_path('data/models/MID_FEKO_VP_B2_45_1360_imag.fits')) + real_vp = import_image_from_fits(rascil_data_path('models/MID_FEKO_VP_B2_45_1360_real.fits')) + imag_vp = import_image_from_fits(rascil_data_path('models/MID_FEKO_VP_B2_45_1360_imag.fits')) real_vp.data = real_vp.data + 1j * imag_vp.data real_vp.data /= numpy.max(numpy.abs(real_vp.data)) return real_vp elif telescope == 'MID_FEKO_Ku': log.debug("create_vp: Using FEKO model for MID voltage pattern") - real_vp = import_image_from_fits(rascil_path('data/models/MID_FEKO_VP_Ku_45_12179_real.fits')) - imag_vp = import_image_from_fits(rascil_path('data/models/MID_FEKO_VP_Ku_45_12179_imag.fits')) + real_vp = import_image_from_fits(rascil_data_path('models/MID_FEKO_VP_Ku_45_12179_real.fits')) + imag_vp = import_image_from_fits(rascil_data_path('models/MID_FEKO_VP_Ku_45_12179_imag.fits')) real_vp.data = real_vp.data + 1j * imag_vp.data real_vp.data /= numpy.max(numpy.abs(real_vp.data)) return real_vp @@ -331,7 +331,7 @@ def create_low_test_beam(model: Image, use_local=True) -> Image: :param model: Template image :return: Image """ - beam = import_image_from_fits(rascil_path('data/models/SKA1_LOW_beam.fits')) + beam = import_image_from_fits(rascil_data_path('models/SKA1_LOW_beam.fits')) # Scale the image cellsize to account for the different in frequencies. Eventually we will want to # use a frequency cube @@ -381,7 +381,7 @@ def create_low_test_vp(model: Image, use_local=True) -> Image: :return: Image """ - beam = import_image_from_fits(rascil_path('data/models/SKA1_LOW_beam.fits')) + beam = import_image_from_fits(rascil_data_path('models/SKA1_LOW_beam.fits')) beam.data = numpy.sqrt(beam.data).astype('complex') # Scale the image cellsize to account for the different in frequencies. Eventually we will want to diff --git a/rascil/processing_components/simulation/configurations.py b/rascil/processing_components/simulation/configurations.py index 99a19d55..32c21565 100644 --- a/rascil/processing_components/simulation/configurations.py +++ b/rascil/processing_components/simulation/configurations.py @@ -11,7 +11,7 @@ from rascil.processing_components.util.coordinate_support import xyz_at_latitude from rascil.data_models.memory_data_models import Configuration -from rascil.data_models.parameters import rascil_path, get_parameter +from rascil.data_models.parameters import rascil_path, rascil_data_path, get_parameter from rascil.processing_components.util.installation_checks import check_data_directory import logging diff --git a/rascil/processing_components/simulation/pointing.py b/rascil/processing_components/simulation/pointing.py index 5060dc71..b8881b3a 100644 --- a/rascil/processing_components/simulation/pointing.py +++ b/rascil/processing_components/simulation/pointing.py @@ -13,7 +13,7 @@ from rascil.data_models.memory_data_models import BlockVisibility from rascil.data_models.memory_data_models import PointingTable -from rascil.data_models.parameters import rascil_path +from rascil.data_models.parameters import rascil_path, rascil_data_path from rascil.processing_components.calibration.operations import create_gaintable_from_blockvisibility from rascil.processing_components.util.coordinate_support import hadec_to_azel from rascil.processing_components.visibility.base import create_visibility_from_rows @@ -251,7 +251,7 @@ def simulate_pointingtable_from_timeseries(pt, type='wind', time_series_type='pr numpy.random.seed(seed) if pointing_directory is None: - pointing_directory = rascil_path("data/models/%s" % time_series_type) + pointing_directory = rascil_data_path("models/%s" % time_series_type) pt.data['pointing'] = numpy.zeros(pt.data['pointing'].shape) diff --git a/rascil/processing_components/simulation/rfi.py b/rascil/processing_components/simulation/rfi.py index f0343d93..486198ce 100644 --- a/rascil/processing_components/simulation/rfi.py +++ b/rascil/processing_components/simulation/rfi.py @@ -92,19 +92,24 @@ def calculate_rfi_at_station(propagators, emitter): rfi_at_station[numpy.abs(rfi_at_station)<1e-15] = 0. return rfi_at_station + def calculate_station_correlation_rfi(rfi_at_station): """ Form the correlation from the rfi at the station - + :param rfi_at_station: :return: Correlation(nant, nants, ntimes, nchan] in Jy """ ntimes, nants, nchan = rfi_at_station.shape - correlation = numpy.zeros([ntimes, nants, nants, nchan], dtype='complex') - + correlationt = numpy.zeros([ntimes, nchan, nants, nants], dtype='complex') + rfit = numpy.transpose(rfi_at_station, axes=[0, 2, 1]) + rfitc = numpy.conjugate(rfit) + for itime in range(ntimes): for chan in range(nchan): - correlation[itime, ..., chan] = numpy.outer(rfi_at_station[itime, :, chan], - numpy.conjugate(rfi_at_station[itime, :, chan])) + correlationt[itime, chan, ...] = numpy.outer(rfit[itime, chan, :], + rfitc[itime, chan, :]) + + correlation = numpy.transpose(correlationt, axes=[0, 2, 3, 1]) return correlation[..., numpy.newaxis] * 1e26 diff --git a/rascil/processing_components/simulation/testing_support.py b/rascil/processing_components/simulation/testing_support.py index 51708105..eac1b460 100644 --- a/rascil/processing_components/simulation/testing_support.py +++ b/rascil/processing_components/simulation/testing_support.py @@ -55,7 +55,7 @@ from scipy import interpolate from rascil.data_models.memory_data_models import Configuration, Image, GainTable, Skycomponent, SkyModel -from rascil.data_models.parameters import rascil_path +from rascil.data_models.parameters import rascil_path, rascil_data_path from rascil.data_models.polarisation import PolarisationFrame from rascil.processing_components.calibration.chain_calibration import create_calibration_controls from rascil.processing_components.calibration.operations import create_gaintable_from_blockvisibility, apply_gaintable @@ -205,13 +205,13 @@ def create_test_image_from_s3(npixel=16384, polarisation_frame=PolarisationFrame else: fovstr = '10' if flux_limit >= 1e-3: - csvfilename = rascil_path('data/models/S3_1400MHz_1mJy_%sdeg.csv' % fovstr) + csvfilename = rascil_data_path('models/S3_1400MHz_1mJy_%sdeg.csv' % fovstr) else: - csvfilename = rascil_path('data/models/S3_1400MHz_100uJy_%sdeg.csv' % fovstr) + csvfilename = rascil_data_path('models/S3_1400MHz_100uJy_%sdeg.csv' % fovstr) log.info('create_test_image_from_s3: Reading S3 sources from %s ' % csvfilename) else: assert fov in [10, 20, 40], "Field of view invalid: use one of %s" % ([10, 20, 40]) - csvfilename = rascil_path('data/models/S3_151MHz_%ddeg.csv' % (fov)) + csvfilename = rascil_data_path('models/S3_151MHz_%ddeg.csv' % (fov)) log.info('create_test_image_from_s3: Reading S3 sources from %s ' % csvfilename) with open(csvfilename) as csvfile: @@ -317,13 +317,13 @@ def create_test_skycomponents_from_s3(polarisation_frame=PolarisationFrame("stok else: fovstr = '10' if flux_limit >= 1e-3: - csvfilename = rascil_path('data/models/S3_1400MHz_1mJy_%sdeg.csv' % fovstr) + csvfilename = rascil_data_path('models/S3_1400MHz_1mJy_%sdeg.csv' % fovstr) else: - csvfilename = rascil_path('data/models/S3_1400MHz_100uJy_%sdeg.csv' % fovstr) + csvfilename = rascil_data_path('models/S3_1400MHz_100uJy_%sdeg.csv' % fovstr) log.info('create_test_skycomponents_from_s3: Reading S3-SEX sources from %s ' % csvfilename) else: assert fov in [10, 20, 40], "Field of view invalid: use one of %s" % ([10, 20, 40]) - csvfilename = rascil_path('data/models/S3_151MHz_%ddeg.csv' % (fov)) + csvfilename = rascil_data_path('models/S3_151MHz_%ddeg.csv' % (fov)) log.info('create_test_skycomponents_from_s3: Reading S3-SEX sources from %s ' % csvfilename) skycomps = list() diff --git a/tests/data_models/test_buffer_data_models.py b/tests/data_models/test_buffer_data_models.py index 25478658..7eb16c5d 100644 --- a/tests/data_models/test_buffer_data_models.py +++ b/tests/data_models/test_buffer_data_models.py @@ -27,7 +27,7 @@ class TestBufferDataModelHelpers(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results/') self.midcore = create_named_configuration('MID', rmax=3000.0) diff --git a/tests/data_models/test_data_model_helpers.py b/tests/data_models/test_data_model_helpers.py index a63619c6..13cd3740 100644 --- a/tests/data_models/test_data_model_helpers.py +++ b/tests/data_models/test_data_model_helpers.py @@ -34,7 +34,7 @@ class TestDataModelHelpers(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.mid = create_named_configuration('MID', rmax=1000.0) diff --git a/tests/processing_components/test_atmospheric_screen.py b/tests/processing_components/test_atmospheric_screen.py index bfd80926..fa852f72 100644 --- a/tests/processing_components/test_atmospheric_screen.py +++ b/tests/processing_components/test_atmospheric_screen.py @@ -9,7 +9,7 @@ import numpy from astropy.coordinates import SkyCoord -from rascil.data_models.parameters import rascil_path +from rascil.data_models.parameters import rascil_path, rascil_data_path from rascil.data_models.polarisation import PolarisationFrame from rascil.processing_components.image.operations import import_image_from_fits, qa_image, \ export_image_to_fits @@ -55,11 +55,11 @@ def setUp(self): phasecentre=self.phasecentre) def test_read_screen(self): - screen = import_image_from_fits(rascil_path('data/models/test_mpc_screen.fits')) + screen = import_image_from_fits(rascil_data_path('models/test_mpc_screen.fits')) assert screen.data.shape == (1, 3, 2000, 2000), screen.data.shape def test_create_gaintable_from_screen(self): - screen = import_image_from_fits(rascil_path('data/models/test_mpc_screen.fits')) + screen = import_image_from_fits(rascil_data_path('models/test_mpc_screen.fits')) beam = create_test_image(cellsize=0.0015, phasecentre=self.vis.phasecentre, frequency=self.frequency) @@ -81,7 +81,7 @@ def test_create_gaintable_from_screen(self): assert gaintables[0].gain.shape == (3, 94, 3, 1, 1), gaintables[0].gain.shape def test_grid_gaintable_to_screen(self): - screen = import_image_from_fits(rascil_path('data/models/test_mpc_screen.fits')) + screen = import_image_from_fits(rascil_data_path('models/test_mpc_screen.fits')) beam = create_test_image(cellsize=0.0015, phasecentre=self.vis.phasecentre, frequency=self.frequency) @@ -110,7 +110,7 @@ def test_grid_gaintable_to_screen(self): if self.persist: export_image_to_fits(weights, rascil_path('test_results/test_mpc_screen_gridded_weights.fits')) def test_plot_gaintable_to_screen(self): - screen = import_image_from_fits(rascil_path('data/models/test_mpc_screen.fits')) + screen = import_image_from_fits(rascil_data_path('models/test_mpc_screen.fits')) beam = create_test_image(cellsize=0.0015, phasecentre=self.vis.phasecentre, frequency=self.frequency) diff --git a/tests/processing_components/test_configurations.py b/tests/processing_components/test_configurations.py index 6f4164e9..44349118 100644 --- a/tests/processing_components/test_configurations.py +++ b/tests/processing_components/test_configurations.py @@ -24,7 +24,7 @@ class TestConfigurations(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.frequency = numpy.linspace(0.8e8, 1.2e8, 3) diff --git a/tests/processing_components/test_export_ms_rascil.py b/tests/processing_components/test_export_ms_rascil.py index 99729a68..5c2709be 100644 --- a/tests/processing_components/test_export_ms_rascil.py +++ b/tests/processing_components/test_export_ms_rascil.py @@ -8,8 +8,7 @@ import logging import numpy -from rascil.data_models.parameters import rascil_path - +from rascil.data_models.parameters import rascil_path, rascil_data_path from rascil.processing_components.visibility.coalesce import convert_visibility_to_blockvisibility, convert_blockvisibility_to_visibility log = logging.getLogger('logger') @@ -119,4 +118,4 @@ def __init__(self): self.addTests(loader.loadTestsFromTestCase(export_ms_RASCIL_test)) if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/tests/processing_components/test_griddata_gridding.py b/tests/processing_components/test_griddata_gridding.py index 9a5ca9bb..bad812ea 100644 --- a/tests/processing_components/test_griddata_gridding.py +++ b/tests/processing_components/test_griddata_gridding.py @@ -46,7 +46,7 @@ class TestGridDataGridding(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.persist = os.getenv("RASCIL_PERSIST", False) self.persist = True diff --git a/tests/processing_components/test_griddata_kernels.py b/tests/processing_components/test_griddata_kernels.py index 69714c04..97138ee7 100644 --- a/tests/processing_components/test_griddata_kernels.py +++ b/tests/processing_components/test_griddata_kernels.py @@ -30,7 +30,7 @@ class TestGridDataKernels(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.phasecentre = SkyCoord(ra=+180.0 * u.deg, dec=-60.0 * u.deg, frame='icrs', equinox='J2000') diff --git a/tests/processing_components/test_griddata_operations.py b/tests/processing_components/test_griddata_operations.py index 3056fe45..55c28ef0 100644 --- a/tests/processing_components/test_griddata_operations.py +++ b/tests/processing_components/test_griddata_operations.py @@ -17,7 +17,7 @@ class TestGridData(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.m31image = create_test_image(cellsize=0.0001) diff --git a/tests/processing_components/test_image_deconvolution.py b/tests/processing_components/test_image_deconvolution.py index 70dca185..254b44f9 100644 --- a/tests/processing_components/test_image_deconvolution.py +++ b/tests/processing_components/test_image_deconvolution.py @@ -33,7 +33,7 @@ def setUp(self): self.persist = os.getenv("RASCIL_PERSIST", False) - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.lowcore = create_named_configuration('LOWBD2-CORE') self.times = (numpy.pi / (12.0)) * numpy.linspace(-3.0, 3.0, 7) diff --git a/tests/processing_components/test_image_deconvolution_msmfs.py b/tests/processing_components/test_image_deconvolution_msmfs.py index aee2700b..82f95700 100644 --- a/tests/processing_components/test_image_deconvolution_msmfs.py +++ b/tests/processing_components/test_image_deconvolution_msmfs.py @@ -28,7 +28,7 @@ class TestImageDeconvolutionMSMFS(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.persist = os.getenv("RASCIL_PERSIST", False) self.niter = 1000 diff --git a/tests/processing_components/test_image_gather_scatter.py b/tests/processing_components/test_image_gather_scatter.py index 5c97dbe6..b555edcc 100644 --- a/tests/processing_components/test_image_gather_scatter.py +++ b/tests/processing_components/test_image_gather_scatter.py @@ -23,7 +23,7 @@ class TestImageGatherScatters(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.persist = os.getenv("RASCIL_PERSIST", False) diff --git a/tests/processing_components/test_image_gradients.py b/tests/processing_components/test_image_gradients.py index 45585a4c..075ca9ca 100644 --- a/tests/processing_components/test_image_gradients.py +++ b/tests/processing_components/test_image_gradients.py @@ -5,7 +5,7 @@ import logging import unittest -from rascil.data_models.parameters import rascil_path +from rascil.data_models.parameters import rascil_path, rascil_data_path from rascil.processing_components.image.gradients import image_gradients from rascil.processing_components.image.operations import export_image_to_fits, show_image, import_image_from_fits @@ -15,14 +15,14 @@ class TestPrimaryBeams(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.show = False self.persist = False def test_create_gradient(self): - real_vp = import_image_from_fits(rascil_path('data/models/MID_GRASP_VP_real.fits')) + real_vp = import_image_from_fits(rascil_data_path('models/MID_GRASP_VP_real.fits')) gradx, grady = image_gradients(real_vp) gradxx, gradxy = image_gradients(gradx) diff --git a/tests/processing_components/test_image_operations.py b/tests/processing_components/test_image_operations.py index a658aef9..5592637e 100644 --- a/tests/processing_components/test_image_operations.py +++ b/tests/processing_components/test_image_operations.py @@ -24,7 +24,7 @@ class TestImage(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.m31image = create_test_image(cellsize=0.0001) diff --git a/tests/processing_components/test_imaging.py b/tests/processing_components/test_imaging.py index bed1cd3b..6e6186aa 100644 --- a/tests/processing_components/test_imaging.py +++ b/tests/processing_components/test_imaging.py @@ -35,7 +35,7 @@ class TestImaging(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.persist = os.getenv("RASCIL_PERSIST", False) diff --git a/tests/processing_components/test_imaging_functions.py b/tests/processing_components/test_imaging_functions.py index cd1b2148..7e2a543d 100644 --- a/tests/processing_components/test_imaging_functions.py +++ b/tests/processing_components/test_imaging_functions.py @@ -25,7 +25,7 @@ class TestImagingFunctions(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') def actualSetUp(self, add_errors=False, freqwin=1, block=False, dospectral=True, dopol=False): diff --git a/tests/processing_components/test_imaging_ms.py b/tests/processing_components/test_imaging_ms.py index c350f469..8da92c45 100644 --- a/tests/processing_components/test_imaging_ms.py +++ b/tests/processing_components/test_imaging_ms.py @@ -39,7 +39,7 @@ class TestImaging(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') def actualSetUp(self, freqwin=1, block=True, dopol=False): diff --git a/tests/processing_components/test_imaging_ng.py b/tests/processing_components/test_imaging_ng.py index ee546d64..d9dec706 100644 --- a/tests/processing_components/test_imaging_ng.py +++ b/tests/processing_components/test_imaging_ng.py @@ -38,7 +38,7 @@ class TestImagingNG(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.persist = os.getenv("RASCIL_PERSIST", False) diff --git a/tests/processing_components/test_imaging_params.py b/tests/processing_components/test_imaging_params.py index c59d8a18..9f715e02 100644 --- a/tests/processing_components/test_imaging_params.py +++ b/tests/processing_components/test_imaging_params.py @@ -22,7 +22,7 @@ class TestImagingParams(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.vnchan = 7 diff --git a/tests/processing_components/test_noise.py b/tests/processing_components/test_noise.py index 45fc8ded..1f9eac8e 100644 --- a/tests/processing_components/test_noise.py +++ b/tests/processing_components/test_noise.py @@ -25,7 +25,7 @@ class TestNoise(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.frequency = numpy.linspace(0.8e8, 1.2e8, 5) diff --git a/tests/processing_components/test_pointing.py b/tests/processing_components/test_pointing.py index 5b538c50..186b3271 100644 --- a/tests/processing_components/test_pointing.py +++ b/tests/processing_components/test_pointing.py @@ -27,8 +27,7 @@ class TestPointing(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path - + from rascil.data_models.parameters import rascil_path, rascil_data_path self.doplot = False self.midcore = create_named_configuration('MID', rmax=300.0) diff --git a/tests/processing_components/test_pointing_circlecut.py b/tests/processing_components/test_pointing_circlecut.py index ce9e0feb..3f6eadf1 100644 --- a/tests/processing_components/test_pointing_circlecut.py +++ b/tests/processing_components/test_pointing_circlecut.py @@ -28,8 +28,7 @@ class TestPointing(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path - + from rascil.data_models.parameters import rascil_path, rascil_data_path self.doplot = True self.midcore = create_named_configuration('MID', rmax=300.0) diff --git a/tests/processing_components/test_pointing_time_series.py b/tests/processing_components/test_pointing_time_series.py index 9aa1c43b..0254486a 100644 --- a/tests/processing_components/test_pointing_time_series.py +++ b/tests/processing_components/test_pointing_time_series.py @@ -25,8 +25,7 @@ class TestPointing(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path - + from rascil.data_models.parameters import rascil_path, rascil_data_path self.doplot = True self.midcore = create_named_configuration('MID', rmax=100.0) diff --git a/tests/processing_components/test_primary_beam_illumination.py b/tests/processing_components/test_primary_beam_illumination.py index 6724f677..128c6a63 100644 --- a/tests/processing_components/test_primary_beam_illumination.py +++ b/tests/processing_components/test_primary_beam_illumination.py @@ -26,7 +26,7 @@ class TestPrimaryBeams(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.persist = os.getenv("RASCIL_PERSIST", False) diff --git a/tests/processing_components/test_primary_beam_zernikes.py b/tests/processing_components/test_primary_beam_zernikes.py index f0f4e88b..62270c83 100644 --- a/tests/processing_components/test_primary_beam_zernikes.py +++ b/tests/processing_components/test_primary_beam_zernikes.py @@ -27,7 +27,7 @@ class TestPrimaryBeams(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.persist = os.getenv("RASCIL_PERSIST", False) diff --git a/tests/processing_components/test_primary_beams.py b/tests/processing_components/test_primary_beams.py index c21eb469..2702a8a3 100644 --- a/tests/processing_components/test_primary_beams.py +++ b/tests/processing_components/test_primary_beams.py @@ -25,7 +25,7 @@ class TestPrimaryBeams(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.persist = os.getenv("RASCIL_PERSIST", False) diff --git a/tests/processing_components/test_skycomponent.py b/tests/processing_components/test_skycomponent.py index 02c5b182..b2696ee6 100644 --- a/tests/processing_components/test_skycomponent.py +++ b/tests/processing_components/test_skycomponent.py @@ -25,7 +25,7 @@ class TestSkycomponent(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.frequency = numpy.array([1e8]) diff --git a/tests/processing_components/test_skycomponent_insert.py b/tests/processing_components/test_skycomponent_insert.py index 9a414c43..d7c2f4aa 100644 --- a/tests/processing_components/test_skycomponent_insert.py +++ b/tests/processing_components/test_skycomponent_insert.py @@ -28,7 +28,7 @@ def setUp(self): self.persist = os.getenv("RASCIL_PERSIST", False) - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.lowcore = create_named_configuration('LOWBD2-CORE') self.dir = rascil_path('test_results') self.times = (numpy.pi / 12.0) * numpy.linspace(-3.0, 3.0, 7) diff --git a/tests/processing_components/test_skymodel.py b/tests/processing_components/test_skymodel.py index 0c70cd9e..cd2fbeb4 100644 --- a/tests/processing_components/test_skymodel.py +++ b/tests/processing_components/test_skymodel.py @@ -24,7 +24,7 @@ class TestSkyModel(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.lowcore = create_named_configuration('LOWBD2', rmax=300.0) self.dir = rascil_path('test_results') self.times = (numpy.pi / 12.0) * numpy.linspace(-3.0, 3.0, 7) diff --git a/tests/processing_components/test_skymodel_mpc.py b/tests/processing_components/test_skymodel_mpc.py index 4ba6d7ee..cf405640 100644 --- a/tests/processing_components/test_skymodel_mpc.py +++ b/tests/processing_components/test_skymodel_mpc.py @@ -28,7 +28,7 @@ class TestSkymodelMPC(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path dec = -40.0 * u.deg self.lowcore = create_named_configuration('LOWBD2', rmax=300.0) diff --git a/tests/processing_components/test_surface.py b/tests/processing_components/test_surface.py index d7d5f53d..f17af9a2 100644 --- a/tests/processing_components/test_surface.py +++ b/tests/processing_components/test_surface.py @@ -23,8 +23,7 @@ class TestSurface(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path - + from rascil.data_models.parameters import rascil_path, rascil_data_path self.doplot = True self.midcore = create_named_configuration('MID', rmax=100.0) diff --git a/tests/processing_components/test_testing_support.py b/tests/processing_components/test_testing_support.py index 7cca8be0..fd0d8e3d 100644 --- a/tests/processing_components/test_testing_support.py +++ b/tests/processing_components/test_testing_support.py @@ -34,7 +34,7 @@ class TestTesting_Support(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.persist = os.getenv("RASCIL_PERSIST", False) diff --git a/tests/processing_components/test_visibility_ms.py b/tests/processing_components/test_visibility_ms.py index 5e63f398..f6a2aaa9 100644 --- a/tests/processing_components/test_visibility_ms.py +++ b/tests/processing_components/test_visibility_ms.py @@ -8,8 +8,7 @@ import numpy -from rascil.data_models.parameters import rascil_path - +from rascil.data_models.parameters import rascil_path, rascil_data_path from rascil.processing_components.visibility.base import create_blockvisibility_from_ms, create_visibility_from_ms from rascil.processing_components.visibility.operations import integrate_visibility_by_channel diff --git a/tests/processing_components/test_visibility_uvfits.py b/tests/processing_components/test_visibility_uvfits.py index 0d02754f..d6cd6a1e 100644 --- a/tests/processing_components/test_visibility_uvfits.py +++ b/tests/processing_components/test_visibility_uvfits.py @@ -8,7 +8,7 @@ import logging import numpy -from rascil.data_models.parameters import rascil_path +from rascil.data_models.parameters import rascil_path, rascil_data_path from rascil.data_models.polarisation import PolarisationFrame from rascil.processing_components.visibility.base import create_blockvisibility_from_uvfits, create_visibility_from_uvfits diff --git a/tests/processing_components/test_weighting.py b/tests/processing_components/test_weighting.py index f3435028..d81baf02 100644 --- a/tests/processing_components/test_weighting.py +++ b/tests/processing_components/test_weighting.py @@ -28,7 +28,7 @@ class TestWeighting(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.npixel = 512 diff --git a/tests/workflows/test_calibrate_rsexecute.py b/tests/workflows/test_calibrate_rsexecute.py index c33f4920..11d98388 100644 --- a/tests/workflows/test_calibrate_rsexecute.py +++ b/tests/workflows/test_calibrate_rsexecute.py @@ -35,7 +35,7 @@ def setUp(self): rsexecute.set_client(verbose=False, memory_limit=4 * 1024 * 1024 * 1024, n_workers=4, dashboard_address=None) - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.persist = os.getenv("RASCIL_PERSIST", False) diff --git a/tests/workflows/test_calibrate_serial.py b/tests/workflows/test_calibrate_serial.py index 65862cac..a0a93054 100644 --- a/tests/workflows/test_calibrate_serial.py +++ b/tests/workflows/test_calibrate_serial.py @@ -30,7 +30,7 @@ class TestCalibrateGraphs(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.persist = os.getenv("RASCIL_PERSIST", False) diff --git a/tests/workflows/test_image_rsexecute.py b/tests/workflows/test_image_rsexecute.py index 2ffc51b9..a8ecfbf2 100644 --- a/tests/workflows/test_image_rsexecute.py +++ b/tests/workflows/test_image_rsexecute.py @@ -29,7 +29,7 @@ class TestImageGraph(unittest.TestCase): def setUp(self): rsexecute.set_client(verbose=False, memory_limit=4 * 1024 * 1024 * 1024, n_workers=4, dashboard_address=None) - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.frequency = numpy.linspace(1e8, 1.5e8, 3) diff --git a/tests/workflows/test_imaging_deconvolve_rsexecute.py b/tests/workflows/test_imaging_deconvolve_rsexecute.py index 6f474d03..2e32139a 100644 --- a/tests/workflows/test_imaging_deconvolve_rsexecute.py +++ b/tests/workflows/test_imaging_deconvolve_rsexecute.py @@ -35,7 +35,7 @@ class TestImagingDeconvolveGraph(unittest.TestCase): def setUp(self): rsexecute.set_client(verbose=False, memory_limit=4 * 1024 * 1024 * 1024, n_workers=4, dashboard_address=None) - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.persist = os.getenv("RASCIL_PERSIST", False) diff --git a/tests/workflows/test_imaging_deconvolve_serial.py b/tests/workflows/test_imaging_deconvolve_serial.py index a479d685..27bbdc4a 100644 --- a/tests/workflows/test_imaging_deconvolve_serial.py +++ b/tests/workflows/test_imaging_deconvolve_serial.py @@ -32,7 +32,7 @@ class TestImagingDeconvolveGraph(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.persist = os.getenv("RASCIL_PERSIST", False) diff --git a/tests/workflows/test_imaging_rsexecute.py b/tests/workflows/test_imaging_rsexecute.py index c901571a..1f34b466 100644 --- a/tests/workflows/test_imaging_rsexecute.py +++ b/tests/workflows/test_imaging_rsexecute.py @@ -43,7 +43,7 @@ def setUp(self): rsexecute.set_client(use_dask=True) - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.persist = os.getenv("RASCIL_PERSIST", False) diff --git a/tests/workflows/test_mid_simulations_rsexecute.py b/tests/workflows/test_mid_simulations_rsexecute.py index 61557def..87149eaf 100644 --- a/tests/workflows/test_mid_simulations_rsexecute.py +++ b/tests/workflows/test_mid_simulations_rsexecute.py @@ -13,8 +13,7 @@ import sys import unittest -from rascil.data_models.parameters import rascil_path - +from rascil.data_models.parameters import rascil_path, rascil_data_path results_dir = rascil_path('test_results') import numpy @@ -215,7 +214,7 @@ def get_args(self): # Control parameters parser.add_argument('--use_radec', type=str, default="False", help='Calculate in RADEC (false)?') - parser.add_argument('--shared_directory', type=str, default=rascil_path('data/configurations'), + parser.add_argument('--shared_directory', type=str, default=rascil_data_path('configurations'), help='Location of configuration files') # Dask parameters @@ -233,9 +232,9 @@ def get_args(self): help='Multipliers for static errors') parser.add_argument('--dynamic_pe', type=float, default=1.0, help='Multiplier for dynamic errors') parser.add_argument('--pointing_file', type=str, default=None, help="Pointing file") - parser.add_argument('--pointing_directory', type=str, default=rascil_path('data/models'), + parser.add_argument('--pointing_directory', type=str, default=rascil_data_path('models'), help='Location of pointing files') - parser.add_argument('--vp_directory', type=str, default=rascil_path('data/models/interpolated'), + parser.add_argument('--vp_directory', type=str, default=rascil_data_path('models/interpolated'), help='Location of pointing files') args = parser.parse_args([]) diff --git a/tests/workflows/test_mpc_rsexecute.py b/tests/workflows/test_mpc_rsexecute.py index 020db3b1..d9529eae 100644 --- a/tests/workflows/test_mpc_rsexecute.py +++ b/tests/workflows/test_mpc_rsexecute.py @@ -36,7 +36,7 @@ def setUp(self): rsexecute.set_client(memory_limit=4 * 1024 * 1024 * 1024, n_workers=4, dashboard_address=None) - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.plot = False self.persist = os.getenv("RASCIL_PERSIST", False) diff --git a/tests/workflows/test_pipelines_mpc_rsexecute.py b/tests/workflows/test_pipelines_mpc_rsexecute.py index 36750412..83448398 100644 --- a/tests/workflows/test_pipelines_mpc_rsexecute.py +++ b/tests/workflows/test_pipelines_mpc_rsexecute.py @@ -7,7 +7,7 @@ from astropy import units as u from astropy.coordinates import SkyCoord -from rascil.data_models import SkyModel, rascil_path, PolarisationFrame +from rascil.data_models import SkyModel, rascil_path, rascil_data_path, PolarisationFrame from rascil.processing_components.image.operations import create_empty_image_like @@ -145,7 +145,7 @@ def max_flux(elem): if nvoronoi is not None: voronoi_components = [voronoi_components[0]] - self.screen = import_image_from_fits(rascil_path('data/models/test_mpc_screen.fits')) + self.screen = import_image_from_fits(rascil_data_path('models/test_mpc_screen.fits')) all_gaintables = create_gaintable_from_screen(blockvis, all_components, self.screen) diff --git a/tests/workflows/test_rcal_serial.py b/tests/workflows/test_rcal_serial.py index 48e9adb6..bf509a57 100644 --- a/tests/workflows/test_rcal_serial.py +++ b/tests/workflows/test_rcal_serial.py @@ -35,7 +35,7 @@ class TestPipelinesFunctions(unittest.TestCase): def setUp(self): - from rascil.data_models.parameters import rascil_path + from rascil.data_models.parameters import rascil_path, rascil_data_path self.dir = rascil_path('test_results') self.setupVis(add_errors=False, block=True) From b4d19b1202f733905c8c3ef6f9177f93fc2304a0 Mon Sep 17 00:00:00 2001 From: timcornwell Date: Sat, 7 Mar 2020 15:28:22 +0000 Subject: [PATCH 2/2] Uprev to 0.1.2 --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 23c58daf..2785722b 100644 --- a/setup.py +++ b/setup.py @@ -25,13 +25,14 @@ package_data = [i for p in packages for i in glob.glob(p + '/*/') + glob.glob(p + '/*/*/') + glob.glob(p + '/*/*/*/')] setup(name='rascil', - version='0.1.1', + version='0.1.2', python_requires='>=3.6', description='Radio Astronomy Simulation, Calibration, and Imaging Library', long_description=open('README.md').read(), author='Tim Cornwell, Peter Wortmann, Bojan Nikolic, Feng Wang, Vlad Stolyarov', author_email='realtimcornwell@gmail.com', url='https://github.com/SKA-ScienceDataProcessor/rascil', + download_url = 'https://github.com/SKA-ScienceDataProcessor/rascil/archive/v.0.1.2.tar.gz', license='Apache License Version 2.0', packages=(packages + package_data), test_suite="tests",