Dashboard dbtest

.travis.yml

@@ -1,14 +1,29 @@
 dist: xenial
 language: python
 addons:
-  apt: 
+  apt:
     packages:
       - libmysqlclient-dev
 
 python:
-  - "2.7"
+  - 2.7
+  - 3.6
+  - 3.7
+
+matrix:
+  allow_failures:
+    - python: 3.7  # blocked until rkern/line_profiler#153 is merged and pushed to pypi
 
 install:
   - pip install .
 
+before_script:
+  - pip install flake8
+  - LEGACY="./gryphon/lib/analysis/legacy/"
+  - UTIL="./gryphon/dashboards/util/"
+  - EXCLUDE="${LEGACY}standard_deviation.py,${LEGACY}bollinger_bands.py,${UTIL}queries.py"
+  - flake8 . --count --exclude=${EXCLUDE} --select=E9,F63,F7,F82 --show-source --statistics
+  # TODO: Fix the flake8 tests and then remove "|| true"
+  - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics || true
+
 script: gryphon-runtests

gryphon/dashboard_db_if.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+"""
+This module holds details regarding DB connection and is runnable standalone to verify integration with DB.
+"""
+
+from __future__ import absolute_import
+
+import sqlalchemy
+import sqlalchemy.orm as orm
+
+# Session holds the SQLAlchemy session
+# first a sessionmaker, it then gets configured by setting an engine, and finally turned into a scoped_session.
+Session = orm.sessionmaker()
+# TODO : better API design here...
+
+
+def db_creds():
+    import os
+    try:
+        creds = os.environ['DASHBOARD_DB_CRED']
+    except KeyError as ke:
+        print("DASHBOARD_DB_CRED environment variable not found.")
+        print("You probably need to setup your .env file. See https://gryphon.readthedocs.io/en/latest/usage.html#dotenv-files ")
+        raise
+    return creds
+
+
+def setup_engine(creds, **kwargs):
+    engine = sqlalchemy.create_engine(
+        creds, ** kwargs
+    )
+
+    # early engine bind, in case a client uses the ORM directly to create a session
+    Session.configure(bind=engine)
+
+    return engine
+
+
+def scoped_session(engine=None):
+    global Session
+
+    if engine is not None:
+        # binding engine as argument present can mean override of previous create_engine calls
+        Session.configure(bind=engine)
+
+    # creating session
+    Session = orm.scoped_session(Session)
+    return Session
+
+
+if __name__ == '__main__':
+
+    import os
+    # TODO : integrate with configuration
+    # TODO : make sure we connect to existing one, not creating a new one (careful with sqlite !)
+    os.environ['DASHBOARD_DB_CRED'] = 'sqlite:///test_dashboard.db'
+
+    try:
+        # basic integration test verifying the database connection is successful.
+        s = scoped_session(db_creds())
+        s.query("1").from_statement(sqlalchemy.text("SELECT 1")).all()
+        print('Works !')
+    except:
+        print("Broken !")

gryphon/dashboards/app.py

@@ -13,7 +13,7 @@
 from gryphon.lib import environment
 environment.load_environment_variables()
 
-import pyximport; pyximport.install()
+import pyximport; pyximport.install(language_level=2 if bytes == str else 3)
 import gryphon.lib; gryphon.lib.prepare()
 
 from gryphon.dashboards.routes import url_patterns
@@ -50,15 +50,15 @@ def __init__(self):
             self.gds_db = None
 
         self.configuration = configuration.read_config_from_file('dashboards.conf')
-        
+
 
 def main():
     tornadotoad.register(
       api_key=os.environ.get('AIRBRAKE_API_KEY'),
       environment=os.environ.get('APP_ENV'),
     )
 
-    app = PentecostApp() 
+    app = PentecostApp()
     http_server = tornado.httpserver.HTTPServer(app)
     http_server.listen(options.port)
     tornado.ioloop.IOLoop.instance().start()

gryphon/dashboards/handlers/admin_base.py

@@ -2,6 +2,7 @@
 import logging
 import traceback
 
+from six import text_type
 import tornado.web
 
 from gryphon.dashboards.handlers.base import BaseHandler
@@ -27,7 +28,7 @@ def write_error(self, status_code, **kwargs):
             exc_info = kwargs.get('exc_info')
 
             error_readout = [
-                unicode(line, 'utf8') for line in traceback.format_exception(*exc_info)
+                text_type(line, 'utf8') for line in traceback.format_exception(*exc_info)
             ]
 
             logger.critical(u''.join(error_readout))
@@ -47,7 +48,7 @@ def get_current_user(self):
 
     def get_secure_cookie(self, name, include_name=True, value=None):
         cookie_value = super(AdminBaseHandler, self).get_secure_cookie(name)
-        return unicode(cookie_value or '', 'utf8')
+        return text_type(cookie_value or b'', 'utf8')
 
     def show_error_message(self, message):
         self.set_secure_cookie('error_message', message)

gryphon/dashboards/handlers/block_times.py

@@ -1,7 +1,8 @@
-from admin_base import AdminBaseHandler
+from __future__ import absolute_import
+from .admin_base import AdminBaseHandler
 import tornado.web
 
-from mixins.start_and_end_time import StartAndEndTimeMixin
+from .mixins.start_and_end_time import StartAndEndTimeMixin
 import util.tick_times as tick_times
 
 

gryphon/dashboards/handlers/home.py

@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import
 import tornado.web
 
-from admin_base import AdminBaseHandler
+from .admin_base import AdminBaseHandler
 
 
 class HomeHandler(AdminBaseHandler):

gryphon/dashboards/handlers/ledger.py

@@ -142,7 +142,7 @@ def get_exchange_for_address(self, address):
         """
 
         if address and address in self.address_map:
-            return address_map[address]
+            return self.address_map[address]
         else:
             return 'External transfer'
 
@@ -261,7 +261,7 @@ def table_entries_from_transaction(self, transaction):
             entry['date'] = date
             entry['details'] = ''.join([
                 '%s:%s ' % (k, v)
-                for k, v in transaction.transaction_details.iteritems()
+                for k, v in transaction.transaction_details.items()
                 if k in ['external_transaction_id', 'notes'] and v not in ['xxx']
             ])
 

gryphon/dashboards/handlers/status.py

@@ -140,7 +140,7 @@ def get_system_balances(self, exchanges):
             system_balance += e.exchange_account_db_object(self.trading_db).balance
 
         total_fiat = sum([
-            balance.to("USD") for currency, balance in system_balance.iteritems()
+            balance.to("USD") for currency, balance in system_balance.items()
             if currency not in Money.CRYPTO_CURRENCIES
         ])
 

gryphon/dashboards/handlers/tick_times.py

@@ -1,7 +1,8 @@
-from admin_base import AdminBaseHandler
+from __future__ import absolute_import
+from .admin_base import AdminBaseHandler
 import tornado.web
 
-from mixins.start_and_end_time import StartAndEndTimeMixin
+from .mixins.start_and_end_time import StartAndEndTimeMixin
 from tinkerpy.exchange.exchange_factory import all_exchanges
 import util.tick_times as tick_times
 

gryphon/dashboards/models/__init__.py

@@ -0,0 +1,10 @@
+from __future__ import absolute_import
+from .base import Base
+from .user import User
+from .columns import Password
+
+__all__ = [
+    'Base',
+    'User',
+    'Password',
+]

gryphon/dashboards/models/base.py

@@ -1,13 +1,14 @@
 # -*- coding: utf-8 -*-
+from six import text_type
 from sqlalchemy.ext.declarative import declarative_base
 
 Base = declarative_base()
 metadata = Base.metadata
 
 def unicode_string(self):
-    return unicode(self).encode('utf-8')
+    return text_type(self).encode('utf-8')
 
-Base.__str__ == unicode_string   
+Base.__str__ == unicode_string
 
 
 # How to migrate a database

gryphon/dashboards/models/columns/__init__.py

@@ -0,0 +1,7 @@
+from .password_column import Password, PasswordColumn
+
+
+__all__ = [
+    'Password',
+    'PasswordColumn',
+]

gryphon/dashboards/models/columns/password_column.py

@@ -1,5 +1,6 @@
 import bcrypt
 import re
+import sys
 import logging
 import sqlalchemy.types as types
 
@@ -46,7 +47,10 @@ def salt(self):
             if self._hashed:
                 self._salt = self._hashed[:30]
             else:
-                self._salt = bcrypt.gensalt(log_rounds=self.DIFFICULTY)
+                if sys.version_info[0] < 3:
+                    self._salt = bcrypt.gensalt(log_rounds=self.DIFFICULTY)
+                else:
+                    self._salt = bcrypt.gensalt(rounds=self.DIFFICULTY)
         return self._salt
 
     @property

gryphon/dashboards/settings.py

@@ -1,12 +1,13 @@
 # -*- coding: utf-8 -*-
+from __future__ import absolute_import
 import logging
 import tornado
 import tornado.template
 import os
 from os.path import dirname, abspath
 from tornado.options import define, options
 from gryphon.lib.logperf import log_request_perf
-import uimodules
+from . import uimodules
 
 # Make filepaths relative to settings.
 path = lambda root,*a: os.path.join(root, *a)

gryphon/dashboards/util/balances.py

@@ -12,7 +12,7 @@
 def get_balance_time_series_from_audits(audits):
     fiat_balances = []
     btc_balances = []
-    
+
     for audit in audits:
         timestamp = int(Delorean(audit.time_created, "UTC").epoch) * 1000
 
@@ -34,7 +34,7 @@ def get_balance_time_series_from_audits(audits):
                 continue
 
             # convert to Money objects
-            for currency, balance_str in balance_data.iteritems():
+            for currency, balance_str in balance_data.items():
                 balance_data[currency] = Money.loads(balance_str)
 
             balance = Balance(balance_data)
@@ -103,7 +103,7 @@ def get_drift_from_audits(audits):
         if 'drift' in audit.data:
             data = json.loads(audit.data)
 
-            for currency, str_amount in data['drift'].iteritems():
+            for currency, str_amount in data['drift'].items():
                 drift_by_currency += Money.loads(str_amount)
 
     return drift_by_currency

gryphon/data_service/auditors/orderbook_auditor.py

@@ -242,15 +242,15 @@ def audit_orderbook(self, orderbook, orderbook_timestamp):
             if not our_orderbooks:
                 log.msg('No orderbooks to audit against')
 
-            for key, value in fundamental_values.iteritems():
+            for key, value in fundamental_values.items():
                 log.msg(
                     '------ Fundamental Value Closeness:%.6f, DBfv:%s, HTTPfv:%s' % (
                     key,
                     value['db_fundamental_value'],
                     value['http_fundamental_value']
                 ))
 
-            for key, value in change_dict.iteritems():
+            for key, value in change_dict.items():
                 log.msg('------ Change Count: %s' % key)
 
         log.msg(

gryphon/data_service/exchange_volume_consumer.py

@@ -1,4 +1,4 @@
-import pyximport; pyximport.install()
+import pyximport; pyximport.install(language_level=2 if bytes == str else 3)
 
 import json
 import os

gryphon/data_service/migrations/env.py

@@ -1,5 +1,5 @@
 from __future__ import with_statement
-import pyximport; pyximport.install()
+import pyximport; pyximport.install(language_level=2 if bytes == str else 3)
 from logging.config import fileConfig
 
 from alembic import context

gryphon/data_service/orderbook_consumer.py

@@ -1,4 +1,4 @@
-import pyximport; pyximport.install()
+import pyximport; pyximport.install(language_level=2 if bytes == str else 3)
 import json
 import os
 import subprocess

gryphon/data_service/pollers/orderbook/kraken_cad_orderbook_poller.py

@@ -1,4 +1,5 @@
-from kraken_orderbook_poller import KrakenOrderbook
+from __future__ import absolute_import
+from .kraken_orderbook_poller import KrakenOrderbook
 
 
 class KrakenCADOrderbook(KrakenOrderbook):

gryphon/data_service/pollers/orderbook/kraken_usd_orderbook_poller.py

@@ -1,4 +1,5 @@
-from kraken_orderbook_poller import KrakenOrderbook
+from __future__ import absolute_import
+from .kraken_orderbook_poller import KrakenOrderbook
 
 
 class KrakenUSDOrderbook(KrakenOrderbook):

gryphon/data_service/pollers/orderbook/websocket/bitfinex_orderbook_websocket.py

@@ -130,11 +130,11 @@ def get_orderbook_to_publish(self):
         bids = []
         asks = []
 
-        for price, volume in sorted(self.bids_dict.iteritems(), reverse=True):
+        for price, volume in sorted(self.bids_dict.items(), reverse=True):
             if volume > 0:
                 bids.append([str(price), str(volume), ''])
 
-        for price, volume in sorted(self.asks_dict.iteritems()):
+        for price, volume in sorted(self.asks_dict.items()):
             if volume > 0:
                 asks.append([str(price), str(volume), ''])