Add support for Elasticsearch 7.x

CHANGELOG.md

@@ -1,3 +1,6 @@
+# 1.2.0
+
+- Add support for Elasticsearch 7.x
 
 # 1.1.1
 

README.md

@@ -154,7 +154,12 @@ st2 run elasticsearch.snapshots.delete host=elk repository=my_backup all_indices
 
 ### Querying Elasticsearch
 
-On successful search (*total hits > 0*) query actions exit with *return code* == 0, if no documents have been found *return code* == 1. In all other case such as execution exceptions *return code* is 99.
+| Return Code | Reason                                                      |
+|-------------|-------------------------------------------------------------|
+| `0`         | Successful search (total hits > 0 or returned hits > 0)     |
+| `1`         | No documents found (total hits == 0)                        |
+| `2`         | `hits.total` not present in response and returned hits == 0 |
+| `99`        | Other execution errors                                      |
 
 Let's look at a few examples:
 

actions/lib/utils.py

@@ -7,7 +7,7 @@
 from curator.utils import get_version, is_master_node
 
 # Elasticsearch versions supported
-version_max = (7, 0, 0)
+version_max = (8, 0, 0)
 version_min = (5, 0, 0)
 logger = logging.getLogger(__name__)
 

actions/search.py

@@ -87,7 +87,21 @@ def _pp_exit(self, data):
             kwargs = {'indent': 4}
         print(json.dumps(data, **kwargs))
 
-        if data['hits']['total'] > 0:
-            sys.exit(0)
-        else:
-            sys.exit(1)
+        # in ElasticSearch 7.0 hits.total becomes an object and may not even
+        # be present when track_total_hits is false see:
+        # https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#hits-total-now-object-search-response # noqa
+        if 'total' in data['hits']:
+            if isinstance(data['hits']['total'], int):
+                hit_value = data['hits']['total']
+            elif isinstance(data['hits']['total'], dict):
+                hit_value = data['hits']['total']['value']
+            else:
+                print('Unsupported data type for `hits.total`', file=sys.stderr)
+                sys.exit(99)
+
+            if hit_value == 0:
+                sys.exit(1)
+        elif len(data['hits']['hits']) == 0:
+            sys.exit(2)
+
+        sys.exit(0)

pack.yaml

@@ -6,7 +6,7 @@ keywords:
   - elasticsearch
   - curator
   - databases
-version: 1.1.1
+version: 1.2.0
 author : StackStorm, Inc.
 email : [email protected]
 python_versions:

requirements.txt

@@ -1,2 +1,2 @@
-elasticsearch-curator==5.4.0
 easydict
+elasticsearch-curator==5.8.1

sensors/count_sensor.py

@@ -8,9 +8,9 @@ class ElasticsearchCountSensor(PollingSensor):
 
     def setup(self):
         self.host = self.config.get('host', None)
-        self.port = self.config.get('port', None)
-        self.http_auth = self.config.get('http_auth', None)
-        self.use_ssl = self.config.get('use_ssl', False)
+        self.port = self.config.get('port', None)
+        self.http_auth = self.config.get('http_auth', None)
+        self.use_ssl = self.config.get('use_ssl', False)
         self.query_window = self.config.get('query_window', 60)
         self.query_string = self.config.get('query_string', '{}')
         self.cooldown_multiplier = self.config.get('cooldown_multiplier', 0)
@@ -22,9 +22,9 @@ def setup(self):
         self.es = None
 
         try:
-            self.es = Elasticsearch([{'host': self.host, 'port': self.port,
-                                    'http_auth': self.http_auth,
-                                    'use_ssl': self.use_ssl}])
+            self.es = Elasticsearch([{'host': self.host, 'port': self.port,
+                                    'http_auth': self.http_auth,
+                                    'use_ssl': self.use_ssl}])
         except Exception:
             raise
 
@@ -40,7 +40,23 @@ def poll(self):
         data = self.es.search(index=self.index, body=query_payload)
 
         hits = data.get('hits', None)
-        if hits.get('total', 0) > self.count_threshold:
+        if hits is None:
+            raise RuntimeError('No hits in search response.')
+
+        total = hits.get('total', None)
+        if total is None:
+            raise RuntimeError('Total not present in search response.' +
+                               'Did you disable track_total_hits?\n' +
+                               'https://www.elastic.co/guide/en/elasticsearch/reference/current/breaking-changes-7.0.html#hits-total-omitted-if-disabled')  # noqa
+
+        if isinstance(int, total):
+            hit_count = total
+        elif isinstance(dict, total):  # for Elasticsearch >= 7.0
+            hit_count = total['value']
+        else:
+            raise RuntimeError('Unsupported type of hit.total')
+
+        if hit_count > self.count_threshold:
             payload = dict()
             payload['results'] = hits
             payload['results']['query'] = query_payload