Vberenz/ftp command (#3)

command snapshots: using ftp instead of websockets for tiff images

nightskycam/cams/runner.py

@@ -87,15 +87,10 @@ def __init__(
         self._last_picture_filename = ""
 
     @staticmethod
-    def _wait_duration(
-        frequency: float, now: datetime.time
-    ) -> Tuple[float, float]:
+    def _wait_duration(frequency: float, now: datetime.time) -> Tuple[float, float]:
         period = 1.0 / frequency
         now_seconds = (
-            now.hour * 3600
-            + now.minute * 60
-            + now.second
-            + now.microsecond * 1e-6
+            now.hour * 3600 + now.minute * 60 + now.second + now.microsecond * 1e-6
         )
         nb_pictures_since_midnight = int(now_seconds / period)
         next_picture_time = period * (nb_pictures_since_midnight + 1)
@@ -166,15 +161,11 @@ def _update_status(
         # is not available
         issues: List[str] = []
         if use_sun_alt and night is None:
-            issues.append(
-                "should use sun altitude, but information not available"
-            )
+            issues.append("should use sun altitude, but information not available")
 
         # report issue: should use weather, but weather information not available
         if use_weather and cloud_cover is None:
-            issues.append(
-                "should use sun weather, but information not available"
-            )
+            issues.append("should use sun weather, but information not available")
 
         # period at which pictures are taken
         period = 1.0 / frequency
@@ -208,9 +199,7 @@ def _update_status(
         # sharing the status dictionary
         self._status.entries(status_dict)
 
-    def get_camera(
-        self, active: bool, config: Config
-    ) -> Tuple[Camera, List[str]]:
+    def get_camera(self, active: bool, config: Config) -> Tuple[Camera, List[str]]:
         """
         Arguments
           active: true if the camera will need to take picture
@@ -305,7 +294,6 @@ def iterate(self):
             pause,
         )
 
-
         if bad_weather:
             # bad weather means that pictures are not taken because
             # of cloud coverage. We create a toml meta data file

nightskycam/cams/utils.py

@@ -273,8 +273,7 @@ def get_local_info(
         return None, None, None
     try:
         return tuple(
-            memory[key]  # type: ignore
-            for key in ("night", "weather", "cloud_cover")
+            memory[key] for key in ("night", "weather", "cloud_cover")  # type: ignore
         )
     except KeyError:
         return None, None, None

nightskycam/commands/runner.py

@@ -11,6 +11,7 @@
 from nightskyrunner.wait_interrupts import RunnerWaitInterruptors
 
 from ..utils.commands import CommandDB
+from ..utils.ftp import FtpConfig
 
 
 @status_error
@@ -44,6 +45,32 @@ def __init__(
         super().__init__(name, config_getter, interrupts, core_frequency)
         self._command_db: Optional[CommandDB] = None
 
+    def _get_ftp_config(self, config) -> Optional[FtpConfig]:
+
+        try:
+            ftp_host = config["ftp_host"]
+        except KeyError:
+            return None
+        remote_subdir_: Optional[str]
+        remote_subdir: Optional[Path]
+        try:
+            remote_subdir_ = str(config["ftp_remote_subdir"])
+        except KeyError:
+            remote_subdir_ = None
+        if remote_subdir_ in ("None", ""):
+            remote_subdir = None
+        else:
+            remote_subdir = Path(str(remote_subdir_))
+        nightskycam = str(config["nightskycam"])
+        remote_subdir = remote_subdir / nightskycam / "snapshot"
+        return FtpConfig(
+            str(config["ftp_username"]),
+            str(config["ftp_password"]),
+            str(config["ftp_host"]),
+            int(config["ftp_port"]),  # type: ignore
+            folder=remote_subdir,
+        )
+
     def on_exit(self) -> None:
         # closing websockets
         if self._command_db is not None:
@@ -78,6 +105,8 @@ def iterate(self) -> None:
                 f"failed to find the public certificate: {cert_file}"
             )
 
+        ftp_config = self._get_ftp_config(config)
+
         if self._command_db:
             # "iterating" command db, i.e.
             # read new commands, monitor the command currently
@@ -86,6 +115,7 @@ def iterate(self) -> None:
             status_dict: CommandRunnerEntries = self._command_db.iterate(
                 command_file,
                 url,
+                ftp_config=ftp_config,
                 token=token,
                 cert_file=cert_file,
                 status=self._status,

nightskycam/config_update/runner.py

@@ -181,9 +181,7 @@ def iterate(self) -> None:
 
         for message in messages:
             # which runner should be updated with which config
-            runner_name, new_config = deserialize_config_update(
-                message, token=token
-            )
+            runner_name, new_config = deserialize_config_update(message, token=token)
 
             # getting the current configuration used by this runner.
             # This is for checking the differences with the new confuration.
@@ -207,10 +205,7 @@ def iterate(self) -> None:
                         tomli_w.dump(new_config, f)
                 except Exception as e:
                     self._status.set_issue(
-                        str(
-                            "failed to update configuration "
-                            f"for {runner_name}: {e}"
-                        )
+                        str("failed to update configuration " f"for {runner_name}: {e}")
                     )
                     error = str(e)
                 else:
@@ -221,19 +216,15 @@ def iterate(self) -> None:
                     # "computing" the differences between the current and the
                     # new config (will be shared in the status, to keep the
                     # user updated via the website)
-                    differences = self._config_differences(
-                        current_config, new_config
-                    )
+                    differences = self._config_differences(current_config, new_config)
 
                     # sharing the path the new config file with the runner, which will
                     # overwrite its own configuration file.
                     SharedMemory.get(runner_name)["path"] = filepath
 
                 # keeping all configuration update in memory (so that they can be
                 # shared in the status)
-                self._track_updates(
-                    runner_name, error=error, differences=differences
-                )
+                self._track_updates(runner_name, error=error, differences=differences)
 
         # sharing the configuration updateds in the status
         self._update_status()

nightskycam/ftp/runner.py

@@ -15,9 +15,9 @@
 from nightskyrunner.wait_interrupts import RunnerWaitInterruptors
 
 from ..utils.filename import sort_by_night
-from ..utils.formating import bits_to_human
 from ..utils.folder_stats import list_nb_files
-from .ftp import FtpConfig, get_ftp
+from ..utils.formating import bits_to_human
+from ..utils.ftp import FtpConfig, get_ftp
 
 
 def _get_remote_dir(remote_subdir: str, date: str, system_name: str) -> Path:
@@ -164,10 +164,7 @@ def _update_status(
         total = sum(list(self._nb_files.values()))
 
         files_to_upload = ", ".join(
-            [
-                f"{filetype}: {nb}"
-                for filetype, nb in list_nb_files(local_dir).items()
-            ]
+            [f"{filetype}: {nb}" for filetype, nb in list_nb_files(local_dir).items()]
         )
 
         latest_uploaded = str(files[0].stem) if files else ""
@@ -247,9 +244,7 @@ def iterate(self):
                 )
 
                 # uploading data
-                uploaded_size = self._upload_files(
-                    ftp_config, remote_dir, files
-                )
+                uploaded_size = self._upload_files(ftp_config, remote_dir, files)
                 self._upload_speed.add(uploaded_size)
                 self._update_status(files, uploaded_size, local_dir)
             else:

nightskycam/location_info/runner.py

@@ -2,24 +2,24 @@
 Module defining LocationInfoRunner.
 """
 
-from contextlib import suppress
 import subprocess
 import time
+from contextlib import suppress
 from datetime import datetime
 from typing import List, Optional
 
-from nightskycam.utils.location_info import LocationInfo, get_location_info
-from nightskycam.utils.night import is_night
-from nightskycam.utils.weather import Weather, get_weather
 from nightskycam_serialization.status import LocationInfoRunnerEntries
 from nightskyrunner.config_getter import ConfigGetter
 from nightskyrunner.runner import ThreadRunner, status_error
 from nightskyrunner.shared_memory import SharedMemory
 from nightskyrunner.status import Level
 from nightskyrunner.wait_interrupts import RunnerWaitInterruptors
 
-from .ip import get_IPs
+from nightskycam.utils.location_info import LocationInfo, get_location_info
+from nightskycam.utils.night import is_night
+from nightskycam.utils.weather import Weather, get_weather
 
+from .ip import get_IPs
 
 # must be the same value as in nightskycam_images.constants.TIME_FORMAT
 TIME_FORMAT: str = "%H:%M:%S"
@@ -128,9 +128,7 @@ def iterate(self) -> None:
         place_id = str(config["place_id"])
         api_key = str(config["weather_api_key"])
         try:
-            sun_alt_threshold = float(
-                config["sun_altitude_threshold"]  # type: ignore
-            )
+            sun_alt_threshold = float(config["sun_altitude_threshold"])  # type: ignore
         except KeyError:
             sun_alt_threshold = -0.1
 
@@ -179,9 +177,7 @@ def iterate(self) -> None:
                 location["country"] = info["country"]
                 location["timezone"] = info["timezone"]
         except Exception as e:
-            errors.append(
-                f"{type(e)}: failed to read info related to {place_id} ({e})"
-            )
+            errors.append(f"{type(e)}: failed to read info related to {place_id} ({e})")
 
         # what is the current sun altitude ? is it night time ?
         if info:
@@ -195,14 +191,10 @@ def iterate(self) -> None:
                 location["night"] = night
                 location["sun_alt"] = sun_alt
             except Exception as e:
-                errors.append(
-                    f"{type(e)}: failed to determine if night time: {e}"
-                )
+                errors.append(f"{type(e)}: failed to determine if night time: {e}")
 
         # network IP(s) of the system
-        location["IPs"] = ", ".join(
-            [ip for ip in get_IPs() if "127.0.0.1" not in ip]
-        )
+        location["IPs"] = ", ".join([ip for ip in get_IPs() if "127.0.0.1" not in ip])
 
         # local time
         location["local_time"] = datetime.now().strftime(TIME_FORMAT)

nightskycam/main.py

@@ -10,7 +10,6 @@
 from nightskyrunner.manager import Manager
 from nightskyrunner.status import Level
 
-
 _logger = logging.Logger("executable")
 
 
@@ -45,9 +44,7 @@ def _get_config_path() -> Path:
 
 
 def _run(config_path: Path) -> None:
-    _logger.info(
-        f"starting nightskycam using the configuration file {config_path}"
-    )
+    _logger.info(f"starting nightskycam using the configuration file {config_path}")
     manager_config_getter = DynamicTomlManagerConfigGetter(config_path)
     with Manager(manager_config_getter, name="nightskycam"):
         while True:

nightskycam/process/debayer.py

@@ -4,18 +4,14 @@
 DEBAYER_CODE = "COLOR_BAYER_BG2BGR"
 
 
-def debayer(
-    image: npt.NDArray, conversion_code: str = DEBAYER_CODE
-) -> npt.NDArray:
+def debayer(image: npt.NDArray, conversion_code: str = DEBAYER_CODE) -> npt.NDArray:
     """
     debayer the image using cv2.COLOR_BAYER_BG3BGR
     """
     try:
         code = getattr(cv2, conversion_code)
     except AttributeError:
-        raise ValueError(
-            f"{conversion_code} is not a valid cv2 conversion code."
-        )
+        raise ValueError(f"{conversion_code} is not a valid cv2 conversion code.")
     try:
         r = cv2.cvtColor(image, code)
     except Exception as e:

nightskycam/process/runner.py

@@ -18,14 +18,8 @@
 from nightskyrunner.status import Level
 from nightskyrunner.wait_interrupts import RunnerWaitInterruptors
 
-from ..utils.file_saving import (
-    extend_meta,
-    get_cv2_config,
-    read_files,
-    save,
-    save_npy,
-    supported_file_formats,
-)
+from ..utils.file_saving import (extend_meta, get_cv2_config, read_files, save,
+                                 save_npy, supported_file_formats)
 from .bits_conversion import to_8bits
 from .darkframes import darkframes
 from .debayer import debayer
@@ -153,9 +147,7 @@ def _process(
         ratio = float(config["resize"])  # type: ignore
         interpolation = str(config["resize_interpolation"])
         if ratio != 1.0:
-            extra_meta.append(
-                f"resize with ratio {config['resize']} ({interpolation})"
-            )
+            extra_meta.append(f"resize with ratio {config['resize']} ({interpolation})")
             image = resize(image, ratio, interpolation=interpolation)
 
     # 8 bits conversion
@@ -308,9 +300,7 @@ def iterate(self):
             # Note: in nightskycam.utils.file_saving, functions save and save_npy,
             # image files are created before meta data toml file.
             origin = source_folder / f"{filename}.toml"
-            destination = (
-                Path(config["destination_folder"]) / f"{filename}.toml"
-            )
+            destination = Path(config["destination_folder"]) / f"{filename}.toml"
             origin.rename(destination)
 
         for filename in npy_files:

nightskycam/sleepypi/runner.py

@@ -2,11 +2,11 @@
 Module defining SleepyPiRunner.
 """
 
-from datetime import timedelta
 import datetime
 import subprocess
 import time
 from datetime import datetime as dtime
+from datetime import timedelta
 from pathlib import Path
 from typing import Optional, Tuple
 
@@ -125,9 +125,7 @@ def _should_sleep(
     if time_now is None:
         time_now = _now()
 
-    min_to_sleep, reason = _time_to_sleep(
-        start_sleep, stop_sleep, time_now=time_now
-    )
+    min_to_sleep, reason = _time_to_sleep(start_sleep, stop_sleep, time_now=time_now)
 
     if min_to_sleep < 0:
         if not _ftp_empty(ftp_folder):
@@ -284,9 +282,7 @@ def iterate(self) -> None:
                     sleep_blocked = True
 
         # informing users about what is going on.
-        self._update_status(
-            config, should_sleep_, sleep_blocked, info, min_to_sleep
-        )
+        self._update_status(config, should_sleep_, sleep_blocked, info, min_to_sleep)
 
         # should sleep, but runner just started, so not sleeping yet
         if sleep_blocked and should_sleep_: