ICE adapter support (#411)

* #233 remove peer address & port
Those are superseded by the ICE adapter

* #233 Forward SDP records

* #237 Clean up code in preparation for reconnect

* #237 Restore game connection on reconnect

* #233 remove connectivity related code

* #271 Implement `ice_servers` API for Twilio NTS and Coturn servers

* Creates a Twilio REST API client that fetches NTS tokens
* TTL of tokens made configurable
* implement Coturns TURN REST API using HMAC auth for the --static-auth-secret coturn option

* #340 prevent a trailing lobby_connection to remove the player from player_service

* Fix test_game.py unawaited clear_data() (#365)

* Fixed memory leak when StatsD is disabled (#364)

* try to remove db submodule to make the server run on test server

* try to not get kicked from the game upon closing of the old lobby_connection

* Post-merge fixes

* Fixed cyclic dependancy

* Formatted strings and used kwargs

* Replaced mock with the dummy class - reverted the revert

* IceMessage is now IceMsg

* Added nts_client to LobbyConnection

* Fixes post-merge atrocities

* Fixing further post-merge atrocities

* add import

* Update lobbyconnection.py

* address requested changes

* Some formatting cleanup

* Remove NatPacketServer from conftest

* GameConnection doesn't take nts_client

* More formatting cleanup

* Games don't need port any more

* Remove connectivity from integration tests

* Clean up some of the CoturnHMAC and TwilioNTS stuff

* Explicitly pass ttl to nts_client.server_tokens

* Remove unnecessary variable

* do not disconnect on Bottleneck gpg message

* handle GameEnded gpnet message

* WIP debugging

* WIP debugging

* prevent non ice clients from joining

* cleanup

* cleanup

README.md

@@ -92,6 +92,10 @@ Can be combined !, e.g. `{command: social, teaminvite: <...>, friends: <..>}`
 * `{command: avatar, action: list_avatar}`: Send a list of available avatars
 * `{command: avatar, action: select, avatar: <avatar_url>}`: Select a valid avatar for the player
 
+##### ICE Servers
+
+* `{command: ice_servers}`: Send ICE TURN/STUN servers - Returns: `{command: ice_servers, : <ice servers>, date_created: <date token was created in ISO 8601 format>, ttl: <ttl in seconds>}`
+
 ##### Misc
 
 * [deprecated] `{command: ask_session}`: response with an welcome command and an valid session (can be delayed)

requirements.txt

@@ -20,3 +20,4 @@ oauthlib
 requests_oauthlib
 mock
 SQLAlchemy
+twilio

server.py

@@ -16,13 +16,13 @@
 import server
 import server.config as config
 from server.api.api_accessor import ApiAccessor
-from server.config import DB_SERVER, DB_PORT, DB_LOGIN, DB_PASSWORD, DB_NAME
+from server.config import DB_SERVER, DB_PORT, DB_LOGIN, DB_PASSWORD, DB_NAME, TWILIO_ACCOUNT_SID
 from server.game_service import GameService
 from server.geoip_service import GeoIpService
 from server.matchmaker import MatchmakerQueue
-from server.natpacketserver import NatPacketServer
 from server.player_service import PlayerService
 from server.stats.game_stats_service import GameStatsService, EventService, AchievementService
+from server.ice_servers.nts import TwilioNTS
 
 if __name__ == '__main__':
     logger = logging.getLogger()
@@ -66,6 +66,13 @@ def signal_handler(signal, frame):
 
         players_online = PlayerService()
 
+        twilio_nts = None
+        if TWILIO_ACCOUNT_SID:
+            twilio_nts = TwilioNTS()
+        else:
+            logger.warning(
+                "Twilio is not set up. You must set TWILIO_ACCOUNT_SID and TWILIO_TOKEN to use the Twilio ICE servers.")
+
         api_accessor = None
         if config.USE_API:
             api_accessor = ApiAccessor()
@@ -74,10 +81,6 @@ def signal_handler(signal, frame):
         achievement_service = AchievementService(api_accessor)
         game_stats_service = GameStatsService(event_service, achievement_service)
 
-        natpacket_server = NatPacketServer(addresses=config.LOBBY_NAT_ADDRESSES, loop=loop)
-        loop.run_until_complete(natpacket_server.listen())
-        server.NatPacketServer.instance = natpacket_server
-
         games = GameService(players_online, game_stats_service)
 
         ctrl_server = loop.run_until_complete(server.run_control_server(loop, players_online, games))
@@ -87,6 +90,7 @@ def signal_handler(signal, frame):
             geoip_service=GeoIpService(),
             player_service=players_online,
             games=games,
+            nts_client=twilio_nts,
             matchmaker_queue=MatchmakerQueue('ladder1v1', game_service=games),
             loop=loop
         )

server/__init__.py

@@ -8,15 +8,15 @@
 """
 import json
 import logging
-from typing import Any, Dict
+from typing import Any, Dict, Optional
 
 import aiomeasures
 
 from . import config as config
 from .games.game import GameState, VisibilityState
 from .stats.game_stats_service import GameStatsService
 from .gameconnection import GameConnection
-from .natpacketserver import NatPacketServer
+from .ice_servers.nts import TwilioNTS
 from .lobbyconnection import LobbyConnection
 from .protocol import QDataStreamProtocol
 from .servercontext import ServerContext
@@ -39,7 +39,6 @@
     'GameStatsService',
     'GameService',
     'LadderService',
-    'NatPacketServer',
     'run_lobby_server',
     'run_control_server',
     'games',
@@ -85,6 +84,7 @@ def run_lobby_server(
     player_service: PlayerService,
     games: GameService,
     loop,
+    nts_client: Optional[TwilioNTS],
     geoip_service: GeoIpService,
     matchmaker_queue: MatchmakerQueue
 ) -> ServerContext:
@@ -139,10 +139,12 @@ def make_connection() -> LobbyConnection:
         return LobbyConnection(
             geoip=geoip_service,
             games=games,
+            nts_client=nts_client,
             players=player_service,
             matchmaker_queue=matchmaker_queue
         )
     ctx = ServerContext(make_connection, name="LobbyServer")
+
     loop.call_later(5, report_dirties)
     loop.call_soon(ping_broadcast)
     loop.run_until_complete(ctx.listen(*address))

server/config.py

@@ -4,8 +4,6 @@
 import os
 
 LOBBY_IP = os.getenv('LOBBY_IP', '37.58.123.3')
-LOBBY_UDP_PORTS = [int(port) for port in os.getenv('LOBBY_UDP_PORTS', '7,53,67,80,123,194,547,3478,3535,6112,30351').split(',')]
-LOBBY_NAT_ADDRESSES = list(map(lambda p: ('0.0.0.0', p), LOBBY_UDP_PORTS))
 
 logging.getLogger('aiomeasures').setLevel(logging.INFO)
 
@@ -39,7 +37,7 @@
 DB_PORT = int(os.getenv("DB_PORT_3306_TCP_PORT", "3306"))
 DB_LOGIN = os.getenv("FAF_DB_LOGIN", "root")
 DB_PASSWORD = os.getenv("FAF_DB_PASSWORD", "banana")
-DB_NAME = os.getenv("FAF_DB_NAME", "faf_test")
+DB_NAME = os.getenv("FAF_DB_NAME", "faf")
 
 CHALLONGE_KEY = "challonge_key"
 CHALLONGE_USER = "challonge_user"
@@ -57,6 +55,13 @@
 NEWBIE_BASE_MEAN = int(os.getenv('NEWBIE_BASE_MEAN', 500))
 NEWBIE_MIN_GAMES = int(os.getenv('NEWBIE_MIN_GAMES', 10))
 
+TWILIO_ACCOUNT_SID = os.getenv("TWILIO_ACCOUNT_SID", "")
+TWILIO_TOKEN = os.getenv("TWILIO_TOKEN", "")
+TWILIO_TTL = os.getenv("TWILIO_TTL", 3600*24)
+
+COTURN_HOSTS = os.getenv('COTURN_HOSTS', "").split(',')
+COTURN_KEYS = os.getenv('COTURN_KEYS', "").split(',')
+
 GEO_IP_DATABASE_PATH = os.getenv("GEO_IP_DATABASE_PATH", "GeoLite2-Country.mmdb")
 GEO_IP_DATABASE_URL = os.getenv(
     "GEO_IP_DATABASE_URL",