Release 0.5 (#124)

* Fix edit docs link (GH #112) (#114)

* Fix bug GH #115 (#117)

* Geographical Queries Proposal (#111)

* define ql geo-query api.

* implement backbone of geo-queries.

* document slf package

* implement location attr handling.

* implement pr suggestions.

* use ngsi attr to represent centroid.

* implement translation of ngsi geo queries to sql.

* hook geo-query code into quantum leap.

* close line string when converting box to polygon

* use wkt for crate geo queries

* missed this file in prev commit :-)

* implement end-to-end tests.

* enable geo queries independently of geo coding settings.

* accept notifs with missing attr values. (#122)

closes #110

* optmize docker image (issue #113) (#116)

* optmize docker image

now 170mb

* Avoid pinned pipenv

* bump up version number and add 0.5 release notes.

Author: c0c0n3

Dockerfile

@@ -1,18 +1,20 @@
-FROM python:3.6
-RUN pip install 'pipenv==2018.10.13'
-
+FROM python:3.6-alpine as base
+FROM base as builder
+RUN apk --no-cache --update-cache add gcc gfortran python python-dev py-pip build-base wget freetype-dev libpng-dev openblas-dev
+RUN ln -s /usr/include/locale.h /usr/include/xlocale.h
+RUN pip install pipenv
 RUN mkdir -p /src/ngsi-timeseries-api
-
 COPY Pipfile /src/ngsi-timeseries-api/Pipfile
 COPY Pipfile.lock /src/ngsi-timeseries-api/Pipfile.lock
+RUN mkdir /install
+WORKDIR /install
+RUN cd /src/ngsi-timeseries-api && { pipenv lock -r > /requirements.txt; }
+RUN pip install --install-option="--prefix=/install" -r /requirements.txt
 
-RUN cd /src/ngsi-timeseries-api && { pipenv lock -r > requirements.txt; }
-RUN pip install -r /src/ngsi-timeseries-api/requirements.txt
-
+FROM base
+COPY --from=builder /install /usr/local
 COPY . /src/ngsi-timeseries-api/
-
 WORKDIR /src/ngsi-timeseries-api/src
-
 ENV PYTHONPATH=$PWD:$PYTHONPATH
 
 CMD python app.py

Pipfile

@@ -8,6 +8,7 @@ connexion = "~=1.1"
 crate = "~=0.22"
 flask = "~=0.12"
 geocoder = "~=1.33"
+geojson = "~=2.4"
 influxdb = "~=4.0"
 pytest = "~=3.0"
 redis = "~=2.10"

Pipfile.lock

@@ -1,5 +1,16 @@
 # QuantumLeap Release Notes
 
+## 0.5
+Release 0.5 of QuantumLeap adds support for geographical queries and features
+a streamlined, much smaller docker image as well as several bug fixes.
+
+- Full support for geographical queries as specified by the FIWARE-NGSI v2
+  Specification except for equality queries (#111)
+- Optimised docker image, size is now down to 170 MB (#116)
+- Support for missing entity attributes (#122)
+- Metadata query fixes (#115)
+- Documentation fixes (#112)
+
 ## 0.4.1
 
 - Add: /health API endpoint (#68)

RELEASE_NOTES.md

@@ -1,8 +1,8 @@
 site_name: QuantumLeap
 site_url: https://quantumleap.readthedocs.org
 site_description: A NGSI to TSDB adapter to allow processing of Historical Data.
-repo_url: https://github.com/smartsdk/ngsi-timeseries-api.git
-edit_uri: 'edit/master/docs/'
+repo_url: https://github.com/smartsdk/ngsi-timeseries-api
+edit_uri: 'edit/master/docs/manuals'
 docs_dir: docs/manuals
 theme: readthedocs
 markdown_extensions: [toc,fenced_code]

mkdocs.yml

@@ -280,6 +280,38 @@ parameters:
     example, if the query was to return 10 results and you use an offset of
     1, the response will return the last 9 values. Make sure you don't give
     more offset than the number of results."
+  georel:
+    in: query
+    name: georel
+    type: string
+    pattern: '^coveredBy$|^intersects$|^equals$|^disjoint$|^near;maxDistance:(0|([1-9][0-9]*))(\.[0-9]+)?$|^near;minDistance:(0|([1-9][0-9]*))(\.[0-9]+)?$|^near;maxDistance:(0|([1-9][0-9]*))(\.[0-9]+)?;minDistance:(0|([1-9][0-9]*))(\.[0-9]+)?$|^near;minDistance:(0|([1-9][0-9]*))(\.[0-9]+)?;maxDistance:(0|([1-9][0-9]*))(\.[0-9]+)?$'
+    description: "Optional. It specifies a spatial relationship between matching
+    entities and a reference shape (geometry). This parameter is used to perform
+    geographical queries with the same semantics as in the FIWARE-NGSI v2 Specification.
+    Full details can be found in the Geographical Queries section of the specification:
+    http://fiware.github.io/specifications/ngsiv2/stable/."
+  geometry:
+    in: query
+    name: geometry
+    type: string
+    enum: [point, line, polygon, box]
+    description: "Optional but required if georel is specified. This parameter
+    defines the reference shape to be used for geographical queries and has the
+    same semantics as in the FIWARE-NGSI v2 Specification.
+    Full details can be found in the Geographical Queries section of the specification:
+    http://fiware.github.io/specifications/ngsiv2/stable/."
+  coords:
+    in: query
+    name: coords
+    type: string
+    pattern: '^[+,-]?(0|([1-9][0-9]*))(\.[0-9]+)?,[+,-]?(0|([1-9][0-9]*))(\.[0-9]+)?(;[+,-]?(0|([1-9][0-9]*))(\.[0-9]+)?,[+,-]?(0|([1-9][0-9]*))(\.[0-9]+)?)*$'
+    description: "Optional but required if georel is specified. This parameter
+    defines the reference shape (geometry) in terms of WGS 84 coordinates and has
+    the same semantics as in the FIWARE-NGSI v2 Specification, except we only
+    accept coordinates in decimal degrees---e.g. `40.714,-74.006` is okay, but not
+    `40 42' 51'',74 0' 21''`.
+    Full details can be found in the Geographical Queries section of the specification:
+    http://fiware.github.io/specifications/ngsiv2/stable/."
 
 ################################################################################
 # PATHS: META
@@ -612,6 +644,9 @@ paths:
         - $ref: '#/parameters/lastN'
         - $ref: '#/parameters/limit'
         - $ref: '#/parameters/offset'
+        - $ref: '#/parameters/georel'
+        - $ref: '#/parameters/geometry'
+        - $ref: '#/parameters/coords'
         # In Header...
         - $ref: '#/parameters/fiwareService'
         - $ref: '#/parameters/fiwareServicePath'
@@ -683,6 +718,9 @@ paths:
         - $ref: '#/parameters/lastN'
         - $ref: '#/parameters/limit'
         - $ref: '#/parameters/offset'
+        - $ref: '#/parameters/georel'
+        - $ref: '#/parameters/geometry'
+        - $ref: '#/parameters/coords'
         # In Header...
         - $ref: '#/parameters/fiwareService'
         - $ref: '#/parameters/fiwareServicePath'
@@ -741,6 +779,9 @@ paths:
         - $ref: '#/parameters/lastN'
         - $ref: '#/parameters/limit'
         - $ref: '#/parameters/offset'
+        - $ref: '#/parameters/georel'
+        - $ref: '#/parameters/geometry'
+        - $ref: '#/parameters/coords'
         # In Header...
         - $ref: '#/parameters/fiwareService'
         - $ref: '#/parameters/fiwareServicePath'
@@ -857,6 +898,9 @@ paths:
         - $ref: '#/parameters/lastN'
         - $ref: '#/parameters/limit'
         - $ref: '#/parameters/offset'
+        - $ref: '#/parameters/georel'
+        - $ref: '#/parameters/geometry'
+        - $ref: '#/parameters/coords'
         # In Header...
         - $ref: '#/parameters/fiwareService'
         - $ref: '#/parameters/fiwareServicePath'
@@ -936,6 +980,9 @@ paths:
         - $ref: '#/parameters/lastN'
         - $ref: '#/parameters/limit'
         - $ref: '#/parameters/offset'
+        - $ref: '#/parameters/georel'
+        - $ref: '#/parameters/geometry'
+        - $ref: '#/parameters/coords'
         # In Header...
         - $ref: '#/parameters/fiwareService'
         - $ref: '#/parameters/fiwareServicePath'
@@ -1005,6 +1052,9 @@ paths:
         - $ref: '#/parameters/lastN'
         - $ref: '#/parameters/limit'
         - $ref: '#/parameters/offset'
+        - $ref: '#/parameters/georel'
+        - $ref: '#/parameters/geometry'
+        - $ref: '#/parameters/coords'
         # In Header...
         - $ref: '#/parameters/fiwareService'
         - $ref: '#/parameters/fiwareServicePath'
@@ -1068,6 +1118,9 @@ paths:
         - $ref: '#/parameters/lastN'
         - $ref: '#/parameters/limit'
         - $ref: '#/parameters/offset'
+        - $ref: '#/parameters/georel'
+        - $ref: '#/parameters/geometry'
+        - $ref: '#/parameters/coords'
         # In Header...
         - $ref: '#/parameters/fiwareService'
         - $ref: '#/parameters/fiwareServicePath'
@@ -1186,6 +1239,9 @@ paths:
         - $ref: '#/parameters/lastN'
         - $ref: '#/parameters/limit'
         - $ref: '#/parameters/offset'
+        - $ref: '#/parameters/georel'
+        - $ref: '#/parameters/geometry'
+        - $ref: '#/parameters/coords'
         # In Header...
         - $ref: '#/parameters/fiwareService'
         - $ref: '#/parameters/fiwareServicePath'
@@ -1384,6 +1440,9 @@ paths:
         - $ref: '#/parameters/lastN'
         - $ref: '#/parameters/limit'
         - $ref: '#/parameters/offset'
+        - $ref: '#/parameters/georel'
+        - $ref: '#/parameters/geometry'
+        - $ref: '#/parameters/coords'
         # In Header...
         - $ref: '#/parameters/fiwareService'
         - $ref: '#/parameters/fiwareServicePath'
@@ -1483,6 +1542,9 @@ paths:
         - $ref: '#/parameters/lastN'
         - $ref: '#/parameters/limit'
         - $ref: '#/parameters/offset'
+        - $ref: '#/parameters/georel'
+        - $ref: '#/parameters/geometry'
+        - $ref: '#/parameters/coords'
         # In Header...
         - $ref: '#/parameters/fiwareService'
         - $ref: '#/parameters/fiwareServicePath'
@@ -1593,6 +1655,9 @@ paths:
         - $ref: '#/parameters/lastN'
         - $ref: '#/parameters/limit'
         - $ref: '#/parameters/offset'
+        - $ref: '#/parameters/georel'
+        - $ref: '#/parameters/geometry'
+        - $ref: '#/parameters/coords'
         # In Header...
         - $ref: '#/parameters/fiwareService'
         - $ref: '#/parameters/fiwareServicePath'

specification/quantumleap.yml

@@ -0,0 +1,89 @@
+from numbers import Real
+from typing import Sequence
+from geojson.utils import coords
+
+
+def centroid2d(points):
+    """
+    Calculate the centroid of a finite set of 2D points.
+
+    :param points: the set of points as an iterable. Each point is assumed to
+        be a sequence of (at least) two real numbers ``[x, y]``, respectively
+        the x (first element) and y coordinate (second element).
+    :return: the centroid or ``None`` if the input is empty.
+    :raise IndexError: if any point is an empty list.
+    :raise TypeError: if the input is ``None`` or a point has a ``None`` coord
+        or the coord isn't a number.
+    """
+    number_of_points = 0
+    centroid = [0, 0]
+
+    for p in points:
+        centroid = (centroid[0] + p[0], centroid[1] + p[1])
+        number_of_points += 1
+
+    centroid = [centroid[0]/number_of_points, centroid[1]/number_of_points] \
+        if number_of_points else None
+
+    return centroid
+#
+# NOTE. Performance.
+# This function can compute the centroid of a few thousands points in
+# microseconds but runtime goes up to millis on huge input lists---over
+# 100,000 points.
+# Even though we could use NumPy to bring the runtime back to microseconds on
+# huge lists (see e.g. https://stackoverflow.com/questions/23020659), the time
+# it takes to convert the input to a NumPy array is about 10 times what this
+# function actually takes to compute the result.
+#
+# NOTE. Typing.
+# Should we use complex numbers instead of lists?
+#
+
+
+def is_point(coords_list):
+    return coords_list and \
+           isinstance(coords_list, Sequence) and \
+           len(coords_list) > 1 and \
+           isinstance(coords_list[0], Real) and \
+           isinstance(coords_list[1], Real)
+
+
+def best_effort_centroid2d(points):
+    """
+    Calculate the centroid of the finite set of 2D points found in the input.
+    A 2D point is assumed to be a list of at least two numbers, the first
+    being the x coordinate and the second the y. This function first filters
+    any non-2D point element out of the input and then computes the centroid
+    of the remaining set of 2D points if such set isn't empty. If the filtered
+    set is empty, then ``None`` is returned.
+
+    :param points: the set of points as an iterable.
+    :return: the centroid or ``None`` if it couldn't be computed.
+    """
+    ps = filter(is_point, points if points else [])
+    return centroid2d(ps)
+
+
+def maybe_centroid2d(points):
+    """
+    Same as ``centroid2d`` but returns ``None`` instead of raising an exception
+    if the input isn't in the expected format.
+    """
+    try:
+        return centroid2d(points)
+    except (ZeroDivisionError, TypeError, IndexError):
+        return None
+
+
+def geojson_centroid(obj):
+    """
+    Compute the centroid of the set of points that define the shapes in the
+    input GeoJSON object.
+
+    :param obj: a GeoJSON object.
+    :return: the centroid if it could be calculated or ``None`` if there was
+    an error---e.g. the input GeoJSON contains invalid points.
+    """
+    points = coords(obj)
+    return best_effort_centroid2d(points)