Skip to content

Commit

Permalink
fix: fix flake8 findings (#35)
Browse files Browse the repository at this point in the history 
Signed-off-by: Ricky Moorhouse <MOORH@uk.ibm.com>

Signed-off-by: Ricky Moorhouse <MOORH@uk.ibm.com>
  • Loading branch information
@rickymoorhouse
rickymoorhouse authored Aug 12, 2022
1 parent 085e29a commit eaa68e3
Show file tree
Hide file tree
Showing 2 changed files with 31 additions and 27 deletions.
26 changes: 15 additions & 11 deletions analytics_net.py
View file
Original file line number Diff line number Diff line change
@@ -1,17 +1,16 @@
import alog
import logging
import tempfile
from kubernetes import client, config
import urllib3
import base64
import requests
import json

urllib3.disable_warnings()
logger = alog.use_channel("analytics")


class AnalyticsNet(object):
""" Analytics Subsystem data """
namespace = ''
token = None
token_expires = 0
Expand Down Expand Up @@ -41,6 +40,7 @@ def __init__(self, config, trawler):
self.find_hostname_and_certs()

def find_hostname_and_certs(self):
""" Lookup hostname and certs for communicating with analytics subsystem"""
try:
# Initialise the k8s API
if self.use_kubeconfig:
Expand All @@ -56,11 +56,12 @@ def find_hostname_and_certs(self):
for service in servicelist.items:
if 'analytics-storage' in service.metadata.name:
for port_object in service.spec.ports:
if port_object.name == 'http-es':
if port_object.name == 'http-es' or port_object.name == 'http':
port = 9200 # default
if port_object.port:
port = port_object.port
self.hostname = "{}.{}.svc.cluster.local.:{}".format(service.metadata.name, self.namespace, port)
self.hostname = "{}.{}.svc.cluster.local.:{}".format(
service.metadata.name, self.namespace, port)
if self.hostname:
logger.info("Identified service host: {}".format(self.hostname))
# Get certificates to communicate with analytics
Expand All @@ -85,6 +86,7 @@ def find_hostname_and_certs(self):
logger.exception(e)

def buildQuery(self):
""" Build search query """
return """{
"size": 0,
"query": {
Expand All @@ -104,16 +106,19 @@ def buildQuery(self):

@alog.timed_function(logger.trace)
def fish(self):
""" main metrics gathering function """
errored = False
if self.hostname:
try:
r = requests.get('https://{}/_cluster/health'.format(self.hostname), verify=False,
cert=self.certificates.name)

r = requests.get('https://{}/_cluster/health'.format(self.hostname),
verify=False,
cert=self.certificates.name)

health_obj = r.json()
logger.debug(r.text)
except requests.exceptions.ConnectionError as e:
logger.error("Connection error on attempt to get cluster health from 'https://{}/_cluster/health'".format(self.hostname))
except requests.exceptions.ConnectionError:
logger.error("Error getting cluster health from 'https://{}/_cluster/health'".format(
self.hostname))
errored = True
health_obj = {}

Expand All @@ -134,7 +139,7 @@ def fish(self):
self.trawler.set_gauge('analytics', 'initializing_shards_total', health_obj['initializing_shards'])
self.trawler.set_gauge('analytics', 'pending_tasks_total', health_obj['number_of_pending_tasks'])
calls_req = requests.get('https://{}/apic-api-r/_search'.format(self.hostname), verify=False,
cert=self.certificates.name, data=self.buildQuery())
cert=self.certificates.name, data=self.buildQuery())

summary = calls_req.json()
self.trawler.set_gauge('analytics', 'apicalls_lasthour.total', summary['hits']['total'])
Expand All @@ -145,7 +150,6 @@ def fish(self):
logger.info("Cluster health failed, so no data and no point querying for calls")



if __name__ == "__main__":
net = AnalyticsNet({"namespace": "apic-management"}, None)
net.fish()
32 changes: 16 additions & 16 deletions trawler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@
import os
import time
import alog
import logging
import logging.config
import threading
import yaml
import click
Expand Down Expand Up @@ -43,9 +41,9 @@ def __init__(self, config_file=None):
self.load_config(config_file)
if 'logging' in self.config:
alog.configure(
default_level=self.config['logging'].get('level', 'debug'),
filters=self.config['logging'].get('filters', None),
formatter=self.config['logging'].get('format', 'json')
default_level=self.config['logging'].get('level', 'debug'),
filters=self.config['logging'].get('filters', None),
formatter=self.config['logging'].get('format', 'json')
)
else:
alog.configure(default_level='info', formatter='json')
Expand All @@ -72,7 +70,7 @@ def __init__(self, config_file=None):
self.watcher = Watcher()

def read_secret(self, key):
# Helper function read secrets from mounted k8s secrets
""" Helper function read secrets from mounted k8s secrets """
try:
with open("{}/{}".format(self.secrets_path, key), 'r') as secret:
value = secret.read().rstrip()
Expand All @@ -96,7 +94,7 @@ def set_gauge(self, component, target_name, value, pod_name=None, labels=None):
if pod_name:
labels['pod'] = pod_name
if 'labels' in self.config['prometheus']:
labels = {**self.config['prometheus']['labels'],**labels}
labels = {**self.config['prometheus']['labels'], **labels}
logger.debug("Entering set_gauge - params: ({}, {}, {}, {})".format(component, target_name, value, pod_name))
logger.debug(labels)
if type(value) is float or type(value) is int:
Expand All @@ -115,18 +113,18 @@ def set_gauge(self, component, target_name, value, pod_name=None, labels=None):
prometheus_target)

logger.debug("Setting gauge %s to %f",
self.gauges[prometheus_target]._name, value)
self.gauges[prometheus_target]._name, value)
try:
if labels:
self.gauges[prometheus_target].labels(**labels).set(value)
else:
self.gauges[prometheus_target].set(value)
except ValueError as valueException:
self.logger.exception(valueException)
except ValueError as value_exception:
self.logger.exception(value_exception)
if self.config['graphite']['enabled']:
if pod_name:
metric_name = "{}.{}.{}".format(component, pod_name, target_name)
else:
else:
metric_name = "{}.{}".format(component, target_name)
self.graphite.stage(metric_name, value)

Expand All @@ -137,7 +135,7 @@ def inc_counter(self, component, target_name, value, pod_name=None, labels=None)
if pod_name:
labels['pod'] = pod_name
if 'labels' in self.config['prometheus']:
labels = {**self.config['prometheus']['labels'],**labels}
labels = {**self.config['prometheus']['labels'], **labels}
logger.debug("Entering inc_counter - params: ({}, {}, {}, {})".format(component, target_name, value, pod_name))
logger.debug(labels)
if type(value) is float or type(value) is int:
Expand All @@ -156,20 +154,21 @@ def inc_counter(self, component, target_name, value, pod_name=None, labels=None)
prometheus_target)

logger.debug("Setting gauge %s to %f",
self.gauges[prometheus_target]._name, value)
self.gauges[prometheus_target]._name, value)
if labels:
self.gauges[prometheus_target].labels(**labels).inc()
else:
self.gauges[prometheus_target].inc()
if self.config['graphite']['enabled']:
if pod_name:
metric_name = "{}.{}.{}".format(component, pod_name, target_name)
else:
else:
metric_name = "{}.{}".format(component, target_name)
self.graphite.stage(metric_name, value)

@alog.timed_function(logger.trace)
def trawl_metrics(self):
""" Main loop to trawl for metrics """
# Initialise
logger.info("Laying nets...")
nets = []
Expand All @@ -183,7 +182,7 @@ def trawl_metrics(self):
nets.append(ManagerNet(self.config['nets']['manager'], self))
if 'analytics' in self.config['nets'] and self.config['nets']['analytics'].get('enabled', True):
nets.append(AnalyticsNet(self.config['nets']['analytics'], self))

# Start thread to watch if needed (nets need to call watcher.register)
if self.watcher.enabled:
watchThread = threading.Thread(target=self.watcher.watch_pods, daemon=True)
Expand All @@ -197,17 +196,18 @@ def trawl_metrics(self):
self.graphite.store()
time.sleep(self.frequency)


@click.command()
@click.version_option()
@click.option('-c', '--config', required=False, envvar='CONFIG',
help="Specifies an alternative config file",
default=None,
type=click.Path())
def cli(config=None):
""" run main trawler application """
trawler = Trawler(config)
trawler.trawl_metrics()


if __name__ == '__main__':
cli()

0 comments on commit eaa68e3

Please sign in to comment.