Add parallel exec

Author: YoranSys

Dockerfile

@@ -5,4 +5,5 @@ WORKDIR /usr/src/app
 COPY requirements.txt ./
 RUN  pip install --no-cache-dir -r requirements.txt
 COPY scoutapm_exporter.py .
+COPY scout_data.py .
 CMD [ "python", "scoutapm_exporter.py" ]

scoutapm_exporter.py

@@ -7,13 +7,16 @@
 from prometheus_client import start_http_server, Gauge
 from scout_data import ScoutAPMAPI
 import logging
+from concurrent.futures import ThreadPoolExecutor, as_completed
 
 base_url = "https://scoutapm.com"
 apikey = os.environ.get('scoutapm_apikey')
 scout_email = os.environ.get("scout_email")
 scout_password = os.environ.get("scout_password")
 
 monitored_apps = os.environ.get('monitored_apps','disabled').split(',')
+raw_metrics_enabled = bool(os.environ.get('raw_metrics_enabled', False))
+default_metrics_enabled = bool(os.environ.get('default_metrics_enabled', True))
 time_between_polls = os.environ.get("time_between_polls", 60)
 prefix = os.environ.get("metric_name_prefix", "scoutapm")
 port = os.environ.get("port", 8000)
@@ -54,6 +57,8 @@ def extract_metrics(metric_info):
     return dataset["summaries"][metric_name]
 
 def extract_last_value_metrics(app_id):
+    print(f"Extracting raw metrics for {app_id}")
+
     data = json.loads(raw_api.get_data(app_id=app_id))
 
     series_raw_metrics = {}
@@ -70,10 +75,24 @@ def extract_last_value_metrics(app_id):
                     last_value = metric['data'][-1][1]
                     series_raw_metrics[series_name][metric_name.lower()] = last_value
                 except:
-                    logging.error(f"Can't extract latest data of {series_name} -> {metric_name}")
+                    logging.error(f"Can't extract latest data for app: {app_id} of {series_name} -> {metric_name}")
 
     return series_raw_metrics
 
+def process_metric(metric_info):
+    app_name, app_id, metric_name = metric_info
+    value = extract_metrics(metric_info)
+    if value is not None:
+        metrics[metric_name].labels(app_name=app_name).set(value)
+
+def process_raw_metric(metric_info):
+    app_name, app_id, metric_name = metric_info
+    values = extract_last_value_metrics(app_id=app_id)
+    for metric in values:
+        for component in values[metric]:
+            if values[metric][component] is not None:
+                metrics[metric_name].labels(app_name=app_name,component=component).set(values[metric_name][component])
+
 if __name__ == '__main__':
     print("Extracting apps")
     apps = scoutapm_call('/api/v0/apps')
@@ -83,42 +102,42 @@ def extract_last_value_metrics(app_id):
         if app['name'] not in monitored_apps and monitored_apps[0] != 'disabled':
             # We don't need to extract these metrics"
             continue
-        print(f"Extracting available metrics for {app}")
-
-        # result: {'name': 'MyPlex', 'id': 119}
-        available_metrics = scoutapm_call('/api/v0/apps/%d/metrics' % app['id'])
-        for metric in available_metrics['availableMetrics']:
-            data.append((app['name'], app['id'], metric))
-            if not metrics.get(metric):
-                metrics[metric] = Gauge("%s_%s" % (prefix,metric), metric, ['app_name'])
+        if default_metrics_enabled:
+            print(f"Extracting available metrics for {app}")
+
+            # result: {'name': 'MyPlex', 'id': 119}
+            available_metrics = scoutapm_call('/api/v0/apps/%d/metrics' % app['id'])
+            for metric in available_metrics['availableMetrics']:
+                data.append((app['name'], app['id'], metric))
+                if not metrics.get(metric):
+                    metrics[metric] = Gauge("%s_%s" % (prefix,metric), metric, ['app_name'])
 
         # Get raw metrics
-        raw_metrics = extract_last_value_metrics(app_id=app['id'])
+        if raw_metrics_enabled:
+            raw_metrics = extract_last_value_metrics(app_id=app['id'])
 
-        for metric in raw_metrics:
-            data_raw.append((app['name'], app['id'], metric))
-            if not metrics.get(metric):
-                metrics[metric] = Gauge("%s_%s" % (prefix,metric), metric, ['app_name','component'])
+            for metric in raw_metrics:
+                data_raw.append((app['name'], app['id'], metric))
+                if not metrics.get(metric):
+                    metrics[metric] = Gauge("%s_%s" % (prefix,metric), metric, ['app_name','component'])
 
     print("Starting HTTP Server on port %s" % port)
     # Start up the server to expose the metrics.
     start_http_server(int(port))
 
-    while True:
-        for metric_info in data:
-            app_name, app_id, metric_name = metric_info
-            value = extract_metrics(metric_info)
-            if value is not None:
-                metrics[metric_name].labels(app_name=app_name).set(value)
-
-        for metric_info in data_raw:
-            app_name, app_id, metric_name = metric_info
-            values = extract_last_value_metrics(app_id=app_id)
-            for metric in values:
-                for component in values[metric]:
-                    if values[metric][component] is not None:
-                        metrics[metric_name].labels(app_name=app_name,component=component).set(values[metric_name][component])                
-
-        # We don't get data updated every seconds, so let's wait 1 minute between polls
-        print("Waiting %d seconds" % time_between_polls)
-        sleep(time_between_polls)
+    with ThreadPoolExecutor(max_workers=10) as executor:
+        while True:
+            # Parallelize the first loop
+            if raw_metrics_enabled:
+                futures = [executor.submit(process_raw_metric, metric_info) for metric_info in data_raw]
+                for future in as_completed(futures):
+                    future.result()
+
+            # Parallelize the second loop
+            if default_metrics_enabled:
+                futures = [executor.submit(process_metric, metric_info) for metric_info in data]
+                for future in as_completed(futures):
+                    future.result()
+
+            print("Waiting %d seconds" % time_between_polls)
+            sleep(time_between_polls)