Add hotspot (#1245)

* initial comitup hotspot implementation
* fix tryUntil helper
* check URL *below* serverRoot because ANY hostname resolves to router if comitup running

Signed-off-by: Markus Storm <markus.storm@gmx.net>

NEWS.md

@@ -3,6 +3,13 @@ PGUP/PGDN.
 All announcements will be stored in /opt/openhabian/docs/NEWSLOG.md for you to
 lookup.
 
+## November 14, 2020
+## WiFi Hotspot
+Whenever your system has a WiFi interface that fails to initialize on installation or startup,
+openHABian will now launch a [WiFi hotspot](docs/openhabian.md#WiFi-Hotspot) you can use to bootstrap WiFi i.e. to connect your
+system to an existing WiFi network.
+
+
 ## October 28, 2020
 ## openHAB3 readiness (BETA)
 openHABian now provides menu options 4X to upgrade your system to openHAB3 and to downgrade

build-image/first-boot.bash

@@ -31,6 +31,7 @@ if ! cp /boot/openhabian.conf "$CONFIGFILE"; then echo "FAILED (copy)"; fail_inp
 if ! sed -i 's|\r$||' "$CONFIGFILE"; then echo "FAILED (Unix line endings)"; fail_inprogress; fi
 if ! source "$CONFIGFILE"; then echo "FAILED (source config)"; fail_inprogress; fi
 if ! source "/opt/openhabian/functions/helpers.bash"; then echo "FAILED (source helpers)"; fail_inprogress; fi
+if ! source "/opt/openhabian/functions/wifi.bash"; then echo "FAILED (source wifi)"; fail_inprogress; fi
 if source "/opt/openhabian/functions/openhabian.bash"; then echo "OK"; else echo "FAILED (source openhabian)"; fail_inprogress; fi
 
 if [[ "${debugmode:-on}" == "on" ]]; then
@@ -75,11 +76,23 @@ fi
 # While setup: show log to logged in user, will be overwritten by openhabian-setup.sh
 echo "watch cat /boot/first-boot.log" > "$HOME/.bash_profile"
 
+# setup networking
+apt install --yes network-manager &>/dev/null
 # shellcheck source=/etc/openhabian.conf disable=SC2154
 if [[ -z $wifi_ssid ]]; then
   # Actually check if ethernet is working
   echo -n "$(timestamp) [openHABian] Setting up Ethernet connection... "
   if grep -qs "up" /sys/class/net/eth0/operstate; then echo "OK"; else echo "FAILED"; fi
+
+  if tryUntil "ping -c1 8.8.8.8 &> /dev/null || curl --silent --head http://www.openhab.org/docs |& grep -qs 'HTTP/1.1 200 OK'" 5 1; then
+    if [[ "$hotspot" == "enable" ]] && ! [[ -x $(command -v comitup) ]]; then
+      echo -n "$(timestamp) [openHABian] Installing comitup hotspot (will reboot after)... "
+      setup_hotspot "install"
+      cp "${BASEDIR:-/opt/openhabian}"/includes/interfaces /etc/network/
+      echo "$(timestamp) [openHABian] hotspot software installed. Rebooting your system to make it take effect!"
+      reboot
+    fi
+  fi
 elif grep -qs "openHABian" /etc/wpa_supplicant/wpa_supplicant.conf && ! grep -qsE "^[[:space:]]*dtoverlay=(pi3-)?disable-wifi" /boot/config.txt; then
   echo -n "$(timestamp) [openHABian] Checking if WiFi is working... "
   if iwlist wlan0 scan |& grep -qs "Interface doesn't support scanning"; then
@@ -91,13 +104,22 @@ elif grep -qs "openHABian" /etc/wpa_supplicant/wpa_supplicant.conf && ! grep -qs
       echo -e "\\nI was not able to turn on the WiFi\\nHere is some more information:\\n"
       rfkill list all
       ip a
-      fail_inprogress
+      echo -e "FAILED.\\n$(timestamp) [openHABian] Starting hotspot as a desperate last attempt; see if you can connect there...\\n"
     else
       echo "OK"
     fi
   else
     echo "OK"
   fi
+  if tryUntil "ping -c1 8.8.8.8 &> /dev/null || curl --silent --head http://www.openhab.org/docs |& grep -qs 'HTTP/1.1 200 OK'" 5 1; then
+    if [[ "$hotspot" == "enable" ]] && ! [[ -x $(command -v comitup) ]]; then
+      echo -n "$(timestamp) [openHABian] Installing comitup hotspot (will reboot after)... "
+      setup_hotspot "install"
+      cp "${BASEDIR:-/opt/openhabian}"/includes/interfaces /etc/network/
+      echo "$(timestamp) [openHABian] hotspot software installed. Rebooting your system to make it take effect!"
+      reboot
+    fi
+  fi
 else
   echo -n "$(timestamp) [openHABian] Setting up Wi-Fi connection... "
 
@@ -118,6 +140,13 @@ else
 
   sed -i 's|REGDOMAIN=.*$|REGDOMAIN='"${wifiCountry}"'|g' /etc/default/crda
 
+  echo -n "$(timestamp) [openHABian] Configuring network... "
+  if grep -qs "wlan0" /etc/network/interfaces; then
+    cond_echo "\\nNot writing to '/etc/network/interfaces', wlan0 entry already available. You might need to check, adopt or remove these lines."
+  else
+    echo -e "\\nallow-hotplug wlan0\\niface wlan0 inet manual\\nwpa-roam /etc/wpa_supplicant/wpa_supplicant.conf\\niface default inet dhcp" >> /etc/network/interfaces
+  fi
+
   if is_pi; then
     echo "OK (rebooting)"
     reboot
@@ -128,20 +157,24 @@ else
 fi
 
 echo -n "$(timestamp) [openHABian] Ensuring network connectivity... "
-if tryUntil "ping -c1 www.example.com &> /dev/null || curl --silent --head http://www.example.com |& grep -qs 'HTTP/1.1 200 OK'" 30 1; then
-    echo "FAILED"
-    if grep -qs "openHABian" /etc/wpa_supplicant/wpa_supplicant.conf && iwconfig |& grep -qs "ESSID:off"; then
-      echo "$(timestamp) [openHABian] I was not able to connect to the configured Wi-Fi. Please check your signal quality. Reachable Wi-Fi networks are:"
-      iwlist wlan0 scanning | grep "ESSID" | sed 's/^\s*ESSID:/\t- /g'
-      echo "$(timestamp) [openHABian] Please try again with your correct SSID and password. The following Wi-Fi configuration was used:"
-      cat /etc/wpa_supplicant/wpa_supplicant.conf
-      rm -f /etc/wpa_supplicant/wpa_supplicant.conf
-    else
-      echo "$(timestamp) [openHABian] The public internet is not reachable. Please check your local network environment."
-      echo "$(timestamp) [openHABian] We will continue trying to get your system installed, but without proper Internet connectivity this is not guaranteed to work."
-    fi
+if tryUntil "ping -c1 8.8.8.8 &> /dev/null || curl --silent --head http://www.openhab.org/docs |& grep -qs 'HTTP/1.1 200 OK'" 86400 1; then
+  echo "FAILED"
+  if grep -qs "openHABian" /etc/wpa_supplicant/wpa_supplicant.conf && iwconfig |& grep -qs "ESSID:off"; then
+    echo "$(timestamp) [openHABian] I was not able to connect to the configured Wi-Fi. Please check your signal quality. Reachable Wi-Fi networks are:"
+    iwlist wlan0 scanning | grep "ESSID" | sed 's/^\s*ESSID:/\t- /g'
+    echo "$(timestamp) [openHABian] Please try again with your correct SSID and password. The following Wi-Fi configuration was used:"
+    cat /etc/wpa_supplicant/wpa_supplicant.conf
+    rm -f /etc/wpa_supplicant/wpa_supplicant.conf
   fi
-echo "OK"
+  echo "$(timestamp) [openHABian] The public internet is not reachable. Please check your local network environment."
+  echo "$(timestamp)              We have launched a publicly accessible hotspot named openHABian-<n>."
+  echo "$(timestamp)              Use your mobile to connect and go to http://raspberrypi.local or http://10.42.0.1/"
+  echo "$(timestamp)              and select the WiFi network you want to connect your openHABian system to."
+  echo "$(timestamp)              After about an hour, we will continue trying to get your system installed,"
+  echo "$(timestamp)              but without proper Internet connectivity this is not guaranteed to work."
+else
+  echo "OK"
+fi
 
 echo -n "$(timestamp) [openHABian] Waiting for dpkg/apt to get ready... "
 if wait_for_apt_to_be_ready; then echo "OK"; else echo "FAILED"; fi
@@ -202,7 +235,6 @@ else
   echo "$(timestamp) [openHABian] We tried to get your system installed, but without proper internet connectivity this may not have worked properly."
 fi
 echo "$(timestamp) [openHABian] Execution of 'openhabian-config unattended' completed."
-
 echo -n "$(timestamp) [openHABian] Waiting for openHAB to become ready on ${HOSTNAME:-openhab}... "
 
 # this took ~130 seconds on a RPi2

build-image/openhabian.conf

@@ -67,6 +67,9 @@ java_opt=Zulu8-64
 # install ZRAM per default, set to "disable" to skip installation
 #zraminstall=enable
 
+# start comitup hotspot if internet is not reachable
+hotspot=enable
+
 # external SD card device to backup and mirror the internal SD card to
 #backupdrive=/dev/sda
 #storageconfig=openhab-dir

build-image/openhabian.pi-raspios32.conf

@@ -69,6 +69,9 @@ java_opt=Zulu8-32
 # install ZRAM per default, set to "disable" to skip installation
 #zraminstall=enable
 
+# start comitup hotspot if internet is not reachable
+hotspot=enable
+
 # external SD card device to backup and mirror the internal SD card to
 #backupdrive=/dev/sda
 #storageconfig=openhab-dir

build-image/openhabian.pi-raspios64beta.conf

@@ -70,6 +70,9 @@ java_opt=Zulu11-64
 # install ZRAM per default, set to "disable" to skip installation
 #zraminstall=enable
 
+# start comitup hotspot if internet is not reachable
+hotspot=enable
+
 # external SD card device to backup and mirror the internal SD card to
 #backupdrive=/dev/sda
 #storageconfig=openhab-dir

build.bash

@@ -2,7 +2,7 @@
 set -e
 
 ####################################################################
-#### dummy: changed this line 10 times to force another image build
+#### dummy: changed this line 14 times to force another image build
 ####################################################################
 
 usage() {
@@ -28,10 +28,10 @@ cleanup_build() {
 source "$(dirname "$0")"/functions/helpers.bash
 # shellcheck source=functions/java-jre.bash
 source "$(dirname "$0")"/functions/java-jre.bash
-# shellcheck source=functions/zram.bash
-source "$(dirname "$0")"/functions/zram.bash
 # shellcheck source=functions/packages.bash
 source "$(dirname "$0")"/functions/packages.bash
+# shellcheck source=functions/zram.bash
+source "$(dirname "$0")"/functions/zram.bash
 
 ## This function formats log messages
 ##

docs/NEWSLOG.md

@@ -1,3 +1,10 @@
+## November 14, 2020
+## WiFi hotspot
+Whenever your system has a WiFi interface that fails to initialize on installation or startup,
+openHABian will now launch a [WiFi hotspot](docs/openhabian.md#WiFi-Hotspot) you can use to
+bootstrap WiFi i.e. to connect your system to an existing WiFi network.
+
+
 ## October 28, 2020
 ## openHAB3 readiness (BETA)
 openHABian now provides menu options 4X to upgrade your system to openHAB3 and to downgrade

docs/openhabian.md

@@ -266,6 +266,19 @@ If you own a RPi3, RPi3+, RPi4, a RPi0W or any other model with a compatible WiF
 For the WiFi based setup to work, you'll need to make your SSID and password known to the system before the first boot.
 So in addition to the setup instructions given above, uncomment and complete the lines reading `wifi_ssid="My WiFi SSID"` and `wifi_psk="password123"` in `openhabian.conf`.
 
+#### WiFi Hotspot
+Whenever the WiFi interface wlan0 exists but does not have connectivity, openHABian will launch a **Hotspot**.
+When you use your mobile phone to scan for WiFi networks, you should be seeing a new network called `openHABian-<n>`.
+Connecting will work without a password. Once connected, open your browser and point it at `http://raspberrypi.local` or `http://comitup-<n>`.
+This may or may not work for your mobile browser as it requires Bonjour/ZeroConf abilities. If you cannot connect to this address, go to `http://10.42.0.1`.
+On that page you can select the SSID of the network you want to connect your system to. Provide the password and press the button.
+Note that as soon as you do, the wlan0 IP address changes so your mobile browser will not receive/priovide you any more feedback.
+Try to ping the new system's hostname (default is `openHABianDevice`) or check DHCP on your router if your openHABian system appeared there.
+For more information on this feature see [comitup-cli](https://davesteele.github.io/comitup/).
+You can use `sudo comitup-cli` inside openHABian to change networks and eventually remove network credentials.
+Note the hotspot may not only become available during installation: it will remain on standby and will show up again every time your `wlan0` interface is losing connectivity.
+The hotspot feature is known to work on RPi 0W, 3 and 4 but is known to often expose problems with WiFi USB adapters.
+
 #### Disable ZRAM
 ZRAM is activated by default on fresh installations on ARM hardware except on a 8GB RPi4 as that is known to be incompatible at the time of writing, leading to kernel crashes.
 If you want to disable ZRAM for a different reason, use `zraminstall=disable` in `openhabian.conf` to install without.

functions/helpers.bash

@@ -361,7 +361,9 @@ tryUntil() {
 
   until [[ $attempts -le 0 ]]; do
     cond_echo "\\nexecuting $cmd \\c"
-    if [[ $(eval "$cmd") -eq 0 ]]; then break; fi
+    eval "$cmd"
+    out=$?
+    if [[ $out -eq 0 ]]; then break; fi
     sleep "$interval"
     if [[ -z $SILENT ]]; then
       echo -e "#${attempts}. $COL_DEF"
@@ -458,6 +460,7 @@ select_blkdev() {
     retval="$(whiptail --title "$2" --cancel-button Cancel --ok-button Select --menu "\\n${3}" "${count}" 76 0 "${array[@]}" 3>&1 1>&2 2>&3)"
   fi
 }
+
 ## install bind9-dnsutils package if available (currently only in sid and focal)
 ## else resort to dnsutils
 ##
@@ -472,3 +475,20 @@ install_dnsutils() {
 
   return $?
 }
+
+## is the comitup WiFi hotspot active
+##
+##    is_hotspot_connected()
+is_hotspot() {
+  if ! [[ -x $(command -v "comitup-cli") ]]; then return 1; fi
+  if echo "q" | comitup-cli | grep -q 'State: HOTSPOT'; then return 0; else return 1; fi
+}
+
+## has WiFi been connected to a wireless network by means of a comitup hotspot
+##
+##    is_wifi_connected()
+is_wifi_connected() {
+  if ! [[ -x $(command -v "comitup-cli") ]]; then return 1; fi
+  if echo "q" | comitup-cli | grep -q 'State: CONNECTED'; then return 0; else return 1; fi
+}
+

functions/wifi.bash

@@ -106,11 +106,37 @@ configure_wifi() {
     if (whiptail --title "WiFi is currently enabled" --defaultno --yesno "$enabledText" 10 80); then
       cond_redirect enable_disable_wifi "disable"
       echo -n "$(timestamp) [openHABian] Cleaning up old WiFi configuration... "
-      if cond_redirect sed -i -e '/allow-hotplug wlan0/d; /iface wlan0 inet manual/d; /wpa-roam \/etc\/wpa_supplicant\/wpa_supplicant.conf/d; /iface default inet dhcp/d' /etc/network/interfaces; then echo "OK (reboot now)"; else echo "FAILED"; return 1; fi
+      if cond_redirect sed -i -e '/allow-hotplug wlan0/d; /iface wlan0 inet manual/d; /wpa-roam \/etc\/wpa_supplicant\/wpa_supplicant.conf/d; /iface default inet dhcp/d' /etc/network/interfaces; then echo "OK (will reboot now)"; else echo "FAILED"; return 1; fi
       whiptail --title "Operation Successful!" --msgbox "Setup was successful. Please reboot now." 7 80
     else
       echo "CANCELED"
       return 0
     fi
   fi
 }
+
+
+## Install comitup WiFi hotspot demon
+## Valid arguments: "setup" or "disable"
+## see https://davesteele.github.io/comitup/ppa.html and
+## https://gist.github.com/jjsanderson/ab2407ab5fd07feb2bc5e681b14a537a
+##
+##    setup_hotspot(String option)
+##
+setup_hotspot() {
+
+  if [[ $1 == "install" ]]; then
+    echo -n "$(timestamp) [openHABian] Installing Comitup hotspot... "
+    # get from source - the comitup package in Buster is 2yrs old
+    echo "deb http://davesteele.github.io/comitup/repo comitup main" > /etc/apt/sources.list.d/comitup.list
+    if ! cond_redirect apt-get --quiet update; then echo "FAILED (update apt lists)"; return 1; fi
+
+    if ! cp "${BASEDIR:-/opt/openhabian}"/includes/comitup.conf /etc/comitup.conf; then echo "FAILED (comitup config)"; return 1; fi
+    if cond_redirect apt install --yes -o Dpkg::Options::=--force-confdef comitup; then echo "OK"; else echo "FAILED"; return 1; fi
+    echo "denyinterfaces wlan0 eth0" >> /etc/dhcpcd.conf
+    sed -i '3 i dhcp=internal' /etc/NetworkManager/NetworkManager.conf
+  elif [[ $1 == "disable" ]]; then
+    echo -n "$(timestamp) [openHABian] Uninstalling hotspot... "
+    if cond_redirect apt purge --yes comitup; then echo "OK"; else echo "FAILED"; return 1; fi
+  fi
+}

includes/comitup.conf

@@ -0,0 +1,47 @@
+#
+# Comitup configuration
+#
+
+
+# ap_name
+#
+# This defines the name used for the AP hotspot, and for the ZeroConf
+# host name. The "<nnn>" string, if present, will be replaced with an
+# instance-unique, persistent number. There may be up to four "n's" in
+# the string.
+#
+ap_name: openHABian-<n>
+
+# web_service
+#
+# The name of a systemd service to be disabled when comitup is managing a
+# hotspot, and enabled when there is a normal wifi connection.
+#
+# web_service: httpd.service
+
+# enable_appliance_mode
+#
+# If enabled (true), and if two wifi adapters are available, comitup will
+# maintain the comitup-<nnn> hotspot on the first, and make other AP
+# connections on the second. IP forwarding and NAT are enabled, so that
+# hosts on the comitup hotspot will be able to access external networks.
+#
+# enable_appliance_mode: true
+
+# external_callback
+#
+# An external script that is called on comitup state changes. It will
+# include a single argument, either 'HOTSPOT', 'CONNECTING', or
+# 'CONNECTED'.
+#
+# The script must be executable. It will be run with the permissions of the 
+# owning user.
+#
+# external_callback: /usr/local/bin/comitup-callback
+
+# primary_wifi_device
+#
+# By default, the first wifi device returned by NetworkManager is used as
+# the primary wifi device. This allows you to override this choice.
+#
+# primary_wifi_device: wlan0

includes/interfaces

@@ -0,0 +1,7 @@
+# interfaces(5) file used by ifup(8) and ifdown(8)
+
+# Please note that this file is written to be used with dhcpcd
+# For static IP, consult /etc/dhcpcd.conf and 'man dhcpcd.conf'
+
+# Include files from /etc/network/interfaces.d:
+source-directory /etc/network/interfaces.d

includes/offline-install-modifications.bash

@@ -7,6 +7,8 @@ export PREOFFLINE="1"
 source /opt/openhabian/functions/helpers.bash
 add_keys "https://bintray.com/user/downloadSubjectPublicKey?username=openhab"
 echo "deb https://dl.bintray.com/openhab/apt-repo2 stable main" > /etc/apt/sources.list.d/openhab2.list
+add_keys https://davesteele.github.io/key-366150CE.pub.txt
+echo "deb http://davesteele.github.io/comitup/repo comitup main" > /etc/apt/sources.list.d/comitup.list
 apt-get --quiet update
 apt-get --quiet upgrade --yes
 apt-get --quiet install --download-only --yes libattr1-dev libc6 libstdc++6 \
@@ -15,7 +17,14 @@ apt-get --quiet install --download-only --yes libattr1-dev libc6 libstdc++6 \
   bash-completion htop curl wget multitail git util-linux bzip2 zip unzip \
   xz-utils software-properties-common man-db whiptail acl usbutils dirmngr \
   arping apt-transport-https bc sysstat jq moreutils avahi-daemon python3 \
-  python3-pip python3-wheel python3-setuptools avahi-autoipd fontconfig
+  python3-pip python3-wheel python3-setuptools avahi-autoipd fontconfig \
+  comitup dns-root-data dnsmasq-base javascript-common libcairo2 \
+  libgudev-1.0-0 libjs-jquery libmbim-glib4 libmbim-proxy libmm-glib0 \
+  libndp0 libnm0 libpixman-1-0 libqmi-glib5 libqmi-proxy libteamdctl0 \
+  libxcb-render0 libxcb-shm0 libxrender1 modemmanager  network-manager \
+  python3-blinker python3-cairo python3-click python3-colorama \
+  python3-flask python3-itsdangerous python3-jinja2 python3-markupsafe \
+  python3-networkmanager python3-pyinotify python3-simplejson python3-werkzeug
 source /opt/openhabian/functions/nodejs-apps.bash
 nodejs_setup
 apt-get --quiet autoremove --yes

openhabian.conf.dist

@@ -67,6 +67,9 @@ java_opt=Zulu8-64
 # install ZRAM per default, set to "disable" to skip installation
 #zraminstall=enable
 
+# start comitup hotspot if internet is not reachable
+hotspot=enable
+
 # external SD card device to backup and mirror the internal SD card to
 #backupdrive=/dev/sda
 #storageconfig=openhab-dir