Merge branch 'develop'

package-rfcx/rfcx/__init__.py

@@ -4,9 +4,8 @@
 
 2. Install with pip: `pip install rfcx-x.x.x-py3-none-any.whl`
 
-3. Use `import rfcx` and [try an example](https://gist.github.com/antonyharfield/93231b3df86cd58fecee4f4d1ec9cc5b)
+3. Use `import rfcx`
 """
 
-from .audio import save_audio_file
 from .client import Client
 name = "rfcx"

package-rfcx/rfcx/_api_rfcx.py

@@ -9,7 +9,7 @@
 host = 'https://api.rfcx.org'  # TODO move to configuration
 
 
-def streamSegments(token, stream_id, start, end, limit, offset):
+def stream_segments(token, stream_id, start, end, limit, offset):
     data = {
         'id': stream_id,
         'start': start,
@@ -29,12 +29,7 @@ def annotations(token,
                 stream=None,
                 limit=50,
                 offset=0):
-    data = {
-        'start': start,
-        'end': end,
-        'limit': limit,
-        'offset': offset
-    }
+    data = {'start': start, 'end': end, 'limit': limit, 'offset': offset}
     if (classifications):
         data['classifications[]'] = classifications
     if (stream):
@@ -48,18 +43,16 @@ def detections(token,
                start,
                end,
                classifications=None,
+               classifiers=None,
                streams=None,
                min_confidence=None,
                limit=50,
                offset=0):
-    data = {
-        'start': start,
-        'end': end,
-        'limit': limit,
-        'offset': offset
-    }
+    data = {'start': start, 'end': end, 'limit': limit, 'offset': offset}
     if (classifications):
         data['classifications[]'] = classifications
+    if (classifiers):
+        data['classifiers[]'] = classifiers
     if (streams):
         data['streams[]'] = streams
     if (min_confidence):
@@ -79,15 +72,19 @@ def streams(token,
             limit=1000,
             offset=0):
     data = {
-        'organizations[]': organizations,
-        'projects[]': projects,
-        'created_by': created_by,
-        'keyword': keyword,
         'is_public': is_public,
         'is_deleted': is_deleted,
         'limit': limit,
         'offset': offset
     }
+    if (organizations):
+        data['organizations[]'] = organizations
+    if (projects):
+        data['projects[]'] = projects
+    if (created_by):
+        data['created_by'] = created_by
+    if (keyword):
+        data['keyword'] = keyword
     path = '/streams'
     url = '{}{}?{}'.format(host, path, urllib.parse.urlencode(data, True))
     return _request(url, token=token)
@@ -109,4 +106,7 @@ def _request(url, method='GET', token=None):
 
     logger.error(f'HTTP status: {resp.status}')
 
+    if (resp.status == 403):
+        logger.error('No permission on given parameter(s)')
+
     return None

package-rfcx/rfcx/audio.py → package-rfcx/rfcx/_audio.py

@@ -3,7 +3,7 @@
 import shutil
 import os
 import concurrent.futures
-from rfcx._api_rfcx import streamSegments
+from rfcx._api_rfcx import stream_segments
 
 def __save_file(url, local_path, token):
     """ Download the file from `url` and save it locally under `local_path` """
@@ -31,7 +31,7 @@ def __generate_date_in_isoformat(date):
     """ Generate date in iso format ending with `Z` """
     return date.replace(microsecond=0).isoformat() + 'Z'
 
-def save_audio_file(token, dest_path, stream_id, start_time, end_time, gain=1, file_ext='wav'):
+def download_file(token, dest_path, stream_id, start_time, end_time, gain=1, file_ext='wav'):
     """ Prepare `url` and `local_path` and save it using function `__save_file` 
         Args:
             dest_path: Audio save path.
@@ -48,8 +48,8 @@ def save_audio_file(token, dest_path, stream_id, start_time, end_time, gain=1, f
             TypeError: if missing required arguements.
     
     """
-    start = iso_to_rfcx_custom_format(__generate_date_in_isoformat(start_time))
-    end = iso_to_rfcx_custom_format(__generate_date_in_isoformat(end_time))
+    start = __iso_to_rfcx_custom_format(__generate_date_in_isoformat(start_time))
+    end = __iso_to_rfcx_custom_format(__generate_date_in_isoformat(end_time))
     audio_name = "{stream_id}_t{start_time}.{end_time}_g{gain}_f{file_ext}".format(stream_id=stream_id,
                                                                                     start_time=start,
                                                                                     end_time=end,
@@ -59,7 +59,7 @@ def save_audio_file(token, dest_path, stream_id, start_time, end_time, gain=1, f
     local_path = __local_audio_file_path(dest_path, audio_name, file_ext)
     __save_file(url, local_path, token)
 
-def iso_to_rfcx_custom_format(time):
+def __iso_to_rfcx_custom_format(time):
     """Convert RFCx iso format to RFCx custom format"""
     return time.replace('-', '').replace(':', '').replace('.', '')
 
@@ -71,7 +71,7 @@ def __get_all_segments(token, stream_id, start, end):
 
     while not empty_segment:
         # No data will return empty array from server
-        segments = streamSegments(token, stream_id, start, end, limit=1000, offset=offset)
+        segments = stream_segments(token, stream_id, start, end, limit=1000, offset=offset)
         if segments:
             all_segments.extend(segments)
             offset = offset + 1000
@@ -80,11 +80,11 @@ def __get_all_segments(token, stream_id, start, end):
 
     return all_segments
 
-def __segmentDownload(save_path, gain, file_ext, segment, token):
+def __segment_download(save_path, gain, file_ext, segment, token):
     """Download audio using the core api(v2)"""
     stream_id = segment['stream']['id']
-    start = iso_to_rfcx_custom_format(segment['start'])
-    end = iso_to_rfcx_custom_format(segment['end'])
+    start = __iso_to_rfcx_custom_format(segment['start'])
+    end = __iso_to_rfcx_custom_format(segment['end'])
     custom_time_range = start + '.' + end
     rfcx_audio_format = "{stream_id}_t{time}_rfull_g{gain}_f{file_ext}".format(stream_id=stream_id,
                                                                                                 time=custom_time_range,
@@ -95,8 +95,8 @@ def __segmentDownload(save_path, gain, file_ext, segment, token):
     local_path = __local_audio_file_path(save_path, audio_name, file_ext)
     __save_file(url, local_path, token)
 
-def downloadStreamSegments(token, dest_path, stream, min_date, max_date, gain=1, file_ext='wav', parallel=True):
-    """ Download RFCx audio on specific time range using `streamSegments` to get audio segments information
+def download_file_segments(token, dest_path, stream, min_date, max_date, gain=1, file_ext='wav', parallel=True):
+    """ Download RFCx audio on specific time range using `stream_segments` to get audio segments information
         and save it using function `__save_file`
         Args:
             token: RFCx client token.
@@ -129,12 +129,12 @@ def downloadStreamSegments(token, dest_path, stream, min_date, max_date, gain=1,
             with concurrent.futures.ThreadPoolExecutor(max_workers=100) as executor:
                 futures = []
                 for segment in segments:
-                    futures.append(executor.submit(__segmentDownload, save_path, gain, file_ext, segment, token))
+                    futures.append(executor.submit(__segment_download, save_path, gain, file_ext, segment, token))
 
                 futures, _ = concurrent.futures.wait(futures)
         else:
             for segment in segments:
-                __segmentDownload(save_path, gain, file_ext, segment, token)
+                __segment_download(save_path, gain, file_ext, segment, token)
         print("Finish download on {}".format(stream))
     else:
         print("No data found on {} - {} at {}".format(start[:-10], end[:-10], stream))

package-rfcx/rfcx/ingest.py → package-rfcx/rfcx/_ingest.py

@@ -26,7 +26,7 @@ def _get_file_status(token, upload_url, upload_id):
     resp = requests.get(url, headers=headers, timeout=90)
     return resp.json()
 
-def ingest_audio(token, stream_id, filepath, timestamp):
+def ingest_file(token, stream_id, filepath, timestamp):
     """ Ingest an audio to RFCx
         Args:
             token: RFCx client token.
@@ -65,4 +65,4 @@ def ingest_audio(token, stream_id, filepath, timestamp):
 
         else:
             print('Success ingested file:', filepath)
-            break
+            break

package-rfcx/rfcx/client.py

@@ -2,8 +2,8 @@
 import datetime
 import os
 import re
-import rfcx.audio as audio
-import rfcx.ingest as ingest
+import rfcx._audio as audio
+import rfcx._ingest as ingest
 import rfcx._util as util
 import rfcx._pkce as pkce
 import rfcx._api_rfcx as api_rfcx
@@ -128,7 +128,7 @@ def _persist_credentials(self):
             f.write(c.id_token + '\n')
 
 
-    def saveAudioFile(self,
+    def download_file(self,
                       dest_path,
                       stream,
                       start_time,
@@ -151,6 +151,9 @@ def saveAudioFile(self,
             TypeError: if missing required arguements.
 
         """
+        if not os.path.exists(dest_path):
+            os.makedirs(dest_path)
+
         if not isinstance(start_time, datetime.datetime):
             print("start_time is not type datetime")
             return
@@ -159,11 +162,11 @@ def saveAudioFile(self,
             print("end_time is not type datetime")
             return
 
-        return audio.save_audio_file(self.credentials.id_token, dest_path,
-                                     stream, start_time, end_time, file_ext)
+        return audio.download_file(self.credentials.id_token, dest_path,
+                                     stream, start_time, end_time, gain, file_ext)
 
 
-    def streamSegments(self, stream, start, end, limit=50, offset=0):
+    def stream_segments(self, stream, start, end, limit=50, offset=0):
         """Retrieve audio information about a specific stream
 
         Args:
@@ -189,19 +192,19 @@ def streamSegments(self, stream, start, end, limit=50, offset=0):
         if end == None:
             end = util.date_now()
 
-        return api_rfcx.streamSegments(self.credentials.id_token, stream,
+        return api_rfcx.stream_segments(self.credentials.id_token, stream,
                                        start, end, limit, offset)
 
 
-    def downloadStreamSegments(self,
+    def download_file_segments(self,
                                dest_path=None,
                                stream=None,
                                min_date=None,
                                max_date=None,
                                gain=1,
                                file_ext='wav',
                                parallel=True):
-        """Download audio using audio information from `guardianAudio`
+        """Download audio using audio information from `stream_segments`
 
         Args:
             dest_path: (Required) Path to save audio.
@@ -245,7 +248,7 @@ def downloadStreamSegments(self,
                     '`audios` directory is already exits. Please specific the directory to save audio path or remove `audios` directoy'
                 )
                 return
-        return audio.downloadStreamSegments(self.credentials.id_token,
+        return audio.download_file_segments(self.credentials.id_token,
                                             dest_path, stream, min_date,
                                             max_date, gain, file_ext, parallel)
 
@@ -263,7 +266,7 @@ def streams(self,
 
         Args:
             organizations: List of organization ids
-            projects: List of organization ids
+            projects: List of project ids
             created_by: The stream owner. Have 3 options: None, me, or collaborators
             keyword: Match streams name with keyword
             is_public: (optional, default=True) Match public or private streams
@@ -279,11 +282,11 @@ def streams(self,
             return
 
         return api_rfcx.streams(self.credentials.id_token, organizations,
-                                projects, is_public, is_deleted, created_by,
-                                keyword, limit, offset)
+                                projects, created_by, keyword,
+                                is_public, is_deleted, limit, offset)
 
 
-    def ingest_audio(self, stream, filepath, timestamp):
+    def ingest_file(self, stream, filepath, timestamp):
         """ Ingest an audio to RFCx
         Args:
             stream: Identifies a stream/site
@@ -300,7 +303,7 @@ def ingest_audio(self, stream, filepath, timestamp):
 
         iso_timestamp = timestamp.replace(microsecond=0).isoformat() + 'Z'
 
-        return ingest.ingest_audio(self.credentials.id_token, stream, filepath, iso_timestamp)
+        return ingest.ingest_file(self.credentials.id_token, stream, filepath, iso_timestamp)
 
 
     def annotations(self, start=None, end=None, classifications=None, stream=None, limit=50, offset=0):
@@ -328,13 +331,14 @@ def annotations(self, start=None, end=None, classifications=None, stream=None, l
         return api_rfcx.annotations(self.credentials.id_token, start, end, classifications, stream, limit, offset)
 
 
-    def detections(self, start=None, end=None, classifications=None, streams=None, min_confidence=None, limit=50, offset=0):
+    def detections(self, start=None, end=None, classifications=None, classifiers=None, streams=None, min_confidence=None, limit=50, offset=0):
         """Retrieve a list of detections
 
         Args:
             start: Minimum timestamp of the audio. If None then defaults to exactly 30 days ago.
             end: Maximum timestamp of the audio. If None then defaults to now.
             classifications: (optional, default=None) List of classification names e.g. orca, chainsaw.
+            classifiers: (optional, default=None) List of classifier ids (integer) e.g. 93, 94.
             streams: (optional, default=None) List of stream ids.
             min_confidence (optional, default=None): Return the detection which equal or greater than given value. If None, it will use default in event strategy.
             limit: (optional, default=50) Maximum number of results to be return. The maximum value is 1000.
@@ -351,4 +355,4 @@ def detections(self, start=None, end=None, classifications=None, streams=None, m
         if end == None:
             end = util.date_now()
 
-        return api_rfcx.detections(self.credentials.id_token, start, end, classifications, streams, min_confidence, limit, offset)
+        return api_rfcx.detections(self.credentials.id_token, start, end, classifications, classifiers, streams, min_confidence, limit, offset)