Modernize Python 2 code to prepare for Python 3

.travis.yml

@@ -1,14 +1,29 @@
 dist: xenial
 language: python
 addons:
-  apt: 
+  apt:
     packages:
       - libmysqlclient-dev
 
 python:
   - "2.7"
+  - "3.6"
+  - "3.7"
+
+matrix:
+  allow_failures:
+    - python "3.7"  # blocked until rkern/line_profiler#153 is merged and pushed to pypi
 
 install:
   - pip install .
 
+before_script:
+  - pip install flake8
+  - LEGACY="./gryphon/lib/analysis/legacy/"
+  - UTIL="./gryphon/dashboards/util/"
+  - EXCLUDE="${LEGACY}standard_deviation.py,${LEGACY}bollinger_bands.py,${UTIL}queries.py"
+  - flake8 . --count --exclude=${EXCLUDE} --select=E9,F63,F7,F82 --show-source --statistics
+  # TODO: Fix the flake8 tests and the remove both --exclude and --exit-zero
+  - flake8 ${EXCLUDE} --count --exit-zero --select=E9,F63,F7,F82 --show-source --statistics
+
 script: gryphon-runtests

gryphon/dashboards/app.py

@@ -13,7 +13,7 @@
 from gryphon.lib import environment
 environment.load_environment_variables()
 
-import pyximport; pyximport.install()
+import pyximport; pyximport.install(language_level=2 if bytes == str else 3)
 import gryphon.lib; gryphon.lib.prepare()
 
 from gryphon.dashboards.routes import url_patterns
@@ -50,15 +50,15 @@ def __init__(self):
             self.gds_db = None
 
         self.configuration = configuration.read_config_from_file('dashboards.conf')
-        
+
 
 def main():
     tornadotoad.register(
       api_key=os.environ.get('AIRBRAKE_API_KEY'),
       environment=os.environ.get('APP_ENV'),
     )
 
-    app = PentecostApp() 
+    app = PentecostApp()
     http_server = tornado.httpserver.HTTPServer(app)
     http_server.listen(options.port)
     tornado.ioloop.IOLoop.instance().start()

gryphon/dashboards/handlers/admin_base.py

@@ -2,6 +2,7 @@
 import logging
 import traceback
 
+from six import text_type
 import tornado.web
 
 from gryphon.dashboards.handlers.base import BaseHandler
@@ -27,7 +28,7 @@ def write_error(self, status_code, **kwargs):
             exc_info = kwargs.get('exc_info')
 
             error_readout = [
-                unicode(line, 'utf8') for line in traceback.format_exception(*exc_info)
+                text_type(line, 'utf8') for line in traceback.format_exception(*exc_info)
             ]
 
             logger.critical(u''.join(error_readout))
@@ -47,7 +48,7 @@ def get_current_user(self):
 
     def get_secure_cookie(self, name, include_name=True, value=None):
         cookie_value = super(AdminBaseHandler, self).get_secure_cookie(name)
-        return unicode(cookie_value or '', 'utf8')
+        return text_type(cookie_value or b'', 'utf8')
 
     def show_error_message(self, message):
         self.set_secure_cookie('error_message', message)

gryphon/dashboards/handlers/block_times.py

@@ -1,7 +1,8 @@
-from admin_base import AdminBaseHandler
+from __future__ import absolute_import
+from .admin_base import AdminBaseHandler
 import tornado.web
 
-from mixins.start_and_end_time import StartAndEndTimeMixin
+from .mixins.start_and_end_time import StartAndEndTimeMixin
 import util.tick_times as tick_times
 
 

gryphon/dashboards/handlers/home.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import
 import tornado.web
 
-from admin_base import AdminBaseHandler
+from .admin_base import AdminBaseHandler
 
 
 class HomeHandler(AdminBaseHandler):

gryphon/dashboards/handlers/ledger.py

@@ -142,7 +142,7 @@ def get_exchange_for_address(self, address):
         """
 
         if address and address in self.address_map:
-            return address_map[address]
+            return self.address_map[address]
         else:
             return 'External transfer'
 
@@ -261,7 +261,7 @@ def table_entries_from_transaction(self, transaction):
             entry['date'] = date
             entry['details'] = ''.join([
                 '%s:%s ' % (k, v)
-                for k, v in transaction.transaction_details.iteritems()
+                for k, v in transaction.transaction_details.items()
                 if k in ['external_transaction_id', 'notes'] and v not in ['xxx']
             ])
 

gryphon/dashboards/handlers/status.py

@@ -140,7 +140,7 @@ def get_system_balances(self, exchanges):
             system_balance += e.exchange_account_db_object(self.trading_db).balance
 
         total_fiat = sum([
-            balance.to("USD") for currency, balance in system_balance.iteritems()
+            balance.to("USD") for currency, balance in system_balance.items()
             if currency not in Money.CRYPTO_CURRENCIES
         ])
 

gryphon/dashboards/handlers/tick_times.py

@@ -1,7 +1,8 @@
-from admin_base import AdminBaseHandler
+from __future__ import absolute_import
+from .admin_base import AdminBaseHandler
 import tornado.web
 
-from mixins.start_and_end_time import StartAndEndTimeMixin
+from .mixins.start_and_end_time import StartAndEndTimeMixin
 from tinkerpy.exchange.exchange_factory import all_exchanges
 import util.tick_times as tick_times
 

gryphon/dashboards/models/base.py

@@ -1,13 +1,14 @@
 # -*- coding: utf-8 -*-
+from six import text_type
 from sqlalchemy.ext.declarative import declarative_base
 
 Base = declarative_base()
 metadata = Base.metadata
 
 def unicode_string(self):
-    return unicode(self).encode('utf-8')
+    return text_type(self).encode('utf-8')
 
-Base.__str__ == unicode_string   
+Base.__str__ == unicode_string
 
 
 # How to migrate a database

gryphon/dashboards/settings.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import
 import logging
 import tornado
 import tornado.template
 import os
 from os.path import dirname, abspath
 from tornado.options import define, options
 from gryphon.lib.logperf import log_request_perf
-import uimodules
+from . import uimodules
 
 # Make filepaths relative to settings.
 path = lambda root,*a: os.path.join(root, *a)

gryphon/dashboards/util/balances.py

@@ -12,7 +12,7 @@
 def get_balance_time_series_from_audits(audits):
     fiat_balances = []
     btc_balances = []
-    
+
     for audit in audits:
         timestamp = int(Delorean(audit.time_created, "UTC").epoch) * 1000
 
@@ -34,7 +34,7 @@ def get_balance_time_series_from_audits(audits):
                 continue
 
             # convert to Money objects
-            for currency, balance_str in balance_data.iteritems():
+            for currency, balance_str in balance_data.items():
                 balance_data[currency] = Money.loads(balance_str)
 
             balance = Balance(balance_data)
@@ -103,7 +103,7 @@ def get_drift_from_audits(audits):
         if 'drift' in audit.data:
             data = json.loads(audit.data)
 
-            for currency, str_amount in data['drift'].iteritems():
+            for currency, str_amount in data['drift'].items():
                 drift_by_currency += Money.loads(str_amount)
 
     return drift_by_currency

gryphon/data_service/auditors/orderbook_auditor.py

@@ -242,15 +242,15 @@ def audit_orderbook(self, orderbook, orderbook_timestamp):
             if not our_orderbooks:
                 log.msg('No orderbooks to audit against')
 
-            for key, value in fundamental_values.iteritems():
+            for key, value in fundamental_values.items():
                 log.msg(
                     '------ Fundamental Value Closeness:%.6f, DBfv:%s, HTTPfv:%s' % (
                     key,
                     value['db_fundamental_value'],
                     value['http_fundamental_value']
                 ))
 
-            for key, value in change_dict.iteritems():
+            for key, value in change_dict.items():
                 log.msg('------ Change Count: %s' % key)
 
         log.msg(

gryphon/data_service/exchange_volume_consumer.py

@@ -1,4 +1,4 @@
-import pyximport; pyximport.install()
+import pyximport; pyximport.install(language_level=2 if bytes == str else 3)
 
 import json
 import os

gryphon/data_service/migrations/env.py

@@ -1,5 +1,5 @@
 from __future__ import with_statement
-import pyximport; pyximport.install()
+import pyximport; pyximport.install(language_level=2 if bytes == str else 3)
 from logging.config import fileConfig
 
 from alembic import context

gryphon/data_service/orderbook_consumer.py

@@ -1,4 +1,4 @@
-import pyximport; pyximport.install()
+import pyximport; pyximport.install(language_level=2 if bytes == str else 3)
 import json
 import os
 import subprocess

gryphon/data_service/pollers/orderbook/kraken_cad_orderbook_poller.py

@@ -1,4 +1,5 @@
-from kraken_orderbook_poller import KrakenOrderbook
+from __future__ import absolute_import
+from .kraken_orderbook_poller import KrakenOrderbook
 
 
 class KrakenCADOrderbook(KrakenOrderbook):

gryphon/data_service/pollers/orderbook/kraken_usd_orderbook_poller.py

@@ -1,4 +1,5 @@
-from kraken_orderbook_poller import KrakenOrderbook
+from __future__ import absolute_import
+from .kraken_orderbook_poller import KrakenOrderbook
 
 
 class KrakenUSDOrderbook(KrakenOrderbook):

gryphon/data_service/pollers/orderbook/websocket/bitfinex_orderbook_websocket.py

@@ -130,11 +130,11 @@ def get_orderbook_to_publish(self):
         bids = []
         asks = []
 
-        for price, volume in sorted(self.bids_dict.iteritems(), reverse=True):
+        for price, volume in sorted(self.bids_dict.items(), reverse=True):
             if volume > 0:
                 bids.append([str(price), str(volume), ''])
 
-        for price, volume in sorted(self.asks_dict.iteritems()):
+        for price, volume in sorted(self.asks_dict.items()):
             if volume > 0:
                 asks.append([str(price), str(volume), ''])
 

gryphon/data_service/pollers/orderbook/websocket/bitstamp_orderbook_websocket.py

@@ -211,26 +211,26 @@ def apply_change_to_orderbook(self, change):
         # Remove the 0 volumes from the orderbook.
         self.orderbook['bids'].update(bids_changes)
 
-        for k, v in self.orderbook['bids'].iteritems():
+        for k, v in self.orderbook['bids'].items():
             if v == "0":
                 self.orderbook['bids'].pop(k)
 
         # Re-sort the bids.
         self.orderbook['bids'] = OrderedDict(sorted(
-            self.orderbook['bids'].iteritems(),
-            key=lambda (k, v): float(k),
+            self.orderbook['bids'].items(),
+            key=lambda k_v1: float(k_v1[0]),
             reverse=True,
         ))
 
         self.orderbook['asks'].update(asks_changes)
 
-        for k, v in self.orderbook['asks'].iteritems():
+        for k, v in self.orderbook['asks'].items():
             if v == "0":
                 self.orderbook['asks'].pop(k)
 
         # Re-sort the asks.
         self.orderbook['asks'] = OrderedDict(
-            sorted(self.orderbook['asks'].iteritems(), key=lambda (k, v): float(k)),
+            sorted(self.orderbook['asks'].items(), key=lambda k_v: float(k_v[0])),
         )
 
     def parse_orders(self, orders):
@@ -247,7 +247,7 @@ def get_orderbook_to_publish(self):
         price_key_orderbook = self.orderbook
 
         return {
-            'bids': [[k, v, ''] for k, v in price_key_orderbook['bids'].iteritems()],
-            'asks': [[k, v, ''] for k, v in price_key_orderbook['asks'].iteritems()],
+            'bids': [[k, v, ''] for k, v in price_key_orderbook['bids'].items()],
+            'asks': [[k, v, ''] for k, v in price_key_orderbook['asks'].items()],
         }
 

gryphon/data_service/pollers/orderbook/websocket/coinbase_orderbook_websocket.py

@@ -309,13 +309,13 @@ def get_orderbook_to_publish(self):
 
         sorted_bid_keys = sorted(
             fancy_orderbook['bids'].keys(),
-            key=lambda (k): float(k),
+            key=lambda k: float(k),
             reverse=True,
         )
 
         sorted_ask_keys = sorted(
             fancy_orderbook['asks'].keys(),
-            key=lambda (k): float(k),
+            key=lambda k: float(k),
         )
 
         bids = [[k, str(fancy_orderbook['bids'][k]), ''] for k in sorted_bid_keys]

gryphon/data_service/pollers/trades/trades_poller.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 import json
 
+from six import text_type
 import termcolor as tc
 from twisted.internet import defer
 from twisted.internet.task import LoopingCall
@@ -50,9 +51,9 @@ def parse_response(self, resp_obj):
         for trade in trades:
             if trade['trade_id'] > self.most_recent_trade_id:
                 trade['price_currency'] = trade['price'].currency
-                trade['price'] = unicode(trade['price'].amount)
+                trade['price'] = text_type(trade['price'].amount)
                 trade['volume_currency'] = trade['volume'].currency
-                trade['volume'] = unicode(trade['volume'].amount)
+                trade['volume'] = text_type(trade['volume'].amount)
                 trade['timestamp'] = int(trade['timestamp'])
                 trade_string = json.dumps(trade, ensure_ascii=False)
                 self.producer.publish_message(trade_string)

gryphon/data_service/runt.py

@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-import pyximport; pyximport.install()
+import pyximport; pyximport.install(language_level=2 if bytes == str else 3)
 import logging
 import logging.handlers
 import os

gryphon/data_service/scripts/autobahn-tester.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 ###############################################################################
 ##
 ##  Copyright (C) 2011-2013 Tavendo GmbH

gryphon/data_service/scripts/benchmark.py

@@ -1,3 +1,4 @@
+from __future__ import print_function
 # Simple test script for benchmarking regular python logging vs twisted's logging.
 
 import logging
@@ -25,7 +26,7 @@ def tx_log():
 
 
 def stop():
-    print "Log Counter: %s" % log_counter
+    print("Log Counter: %s" % log_counter)
     reactor.stop()