handle all indexes when reindexing and on init (#2466)

* handle all indexes when reindexing and on init

changes are in eve-elastic

SDESK-6805 SDESK-6780

* bump eve-elastic version

apps/prepopulate/app_initialize.py

@@ -280,13 +280,14 @@ def run(self, entity_name=None, path=None, sample_data=False, force=False, init_
         # As we want the rest of this command to still execute
         app.init_indexes(ignore_duplicate_keys=True)
 
+        rebuild_elastic_on_init_data_error = app.config.get("REBUILD_ELASTIC_ON_INIT_DATA_ERROR")
+
         # put mapping to elastic
         try:
-            app.data.init_elastic(app)
+            app.data.init_elastic(app, raise_on_mapping_error=rebuild_elastic_on_init_data_error)
         except elasticsearch.exceptions.TransportError as err:
             logger.error(err)
-
-            if app.config.get("REBUILD_ELASTIC_ON_INIT_DATA_ERROR"):
+            if rebuild_elastic_on_init_data_error:
                 logger.warning("Can't update the mapping, running app:flush_elastic_index command now.")
                 FlushElasticIndex().run(sd_index=True, capi_index=True)
             else:

setup.py

@@ -19,7 +19,7 @@
     # and https://github.com/psf/requests/pull/5651)
     "urllib3<1.26",
     "eve==1.1.2",
-    "eve-elastic>=7.3.0,<7.4.0",
+    "eve-elastic>=7.3.2,<7.4.0",
     "flask>=1.1,<1.2",
     "flask-mail>=0.9,<0.10",
     "flask-script>=2.0.5,<3.0",

superdesk/commands/rebuild_elastic_index.py

@@ -44,10 +44,7 @@ def run(self, resource_name=None):
         elif resource_name:
             raise ValueError("Resource {} is not configured".format(resource_name))
         for resource in resources:
-            try:
-                current_app.data.elastic.reindex(resource)
-            except elasticsearch.exceptions.NotFoundError as nfe:
-                print(nfe)
+            current_app.data.elastic.reindex(resource)
             print('Index {} rebuilt successfully.'.format(resource))
 
 

superdesk/datalayer.py

@@ -40,7 +40,7 @@ def init_app(self, app):
     def pymongo(self, resource=None, prefix=None):
         return self.mongo.pymongo(resource, prefix)
 
-    def init_elastic(self, app):
+    def init_elastic(self, app, raise_on_mapping_error=False):
         """Init elastic index.
 
         It will create index and put mapping. It should run only once so locks are in place.
@@ -49,7 +49,7 @@ def init_elastic(self, app):
         with app.app_context():
             if lock("elastic", expire=10):
                 try:
-                    self.elastic.init_index()
+                    self.elastic.init_index(raise_on_mapping_error=raise_on_mapping_error)
                 finally:
                     unlock("elastic")
 

tests/commands/rebuild_elastic_index_tests.py

@@ -13,26 +13,65 @@
 from eve.utils import ParsedRequest
 from superdesk import get_resource_service
 from superdesk.commands.rebuild_elastic_index import RebuildElasticIndex
-from time import sleep
+
+
+RESOURCE = 'ingest'
 
 
 class RebuildIndexTestCase(TestCase):
 
+    @property
+    def data(self):
+        return [
+            {'headline': 'test {}'.format(i), 'slugline': 'rebuild {}'.format(i),
+             'type': 'text' if (i % 2 == 0) else 'picture'} for i in range(11, 21)
+        ]
+
+    def query_items(self):
+        req = ParsedRequest()
+        req.args = {}
+        req.max_results = 25
+        return get_resource_service('ingest').get(req, {})
+
     def test_retrieve_items_after_index_rebuilt(self):
-        RESOURCE = 'ingest'
         elastic = self.app.data.elastic
         alias = elastic._resource_index(RESOURCE)
         alias_info = elastic.elastic(RESOURCE).indices.get_alias(name=alias)
 
-        data = [{'headline': 'test {}'.format(i), 'slugline': 'rebuild {}'.format(i),
-                 'type': 'text' if (i % 2 == 0) else 'picture'} for i in range(11, 21)]
-        get_resource_service(RESOURCE).post(data)
+        get_resource_service(RESOURCE).post(self.data)
         RebuildElasticIndex().run()
         alias_info_new = elastic.elastic(RESOURCE).indices.get_alias(name=alias)
         self.assertNotEqual(alias_info, alias_info_new)
 
-        req = ParsedRequest()
-        req.args = {}
-        req.max_results = 25
-        items = get_resource_service('ingest').get(req, {})
+        items = self.query_items()
+        self.assertEqual(10, items.count())
+
+    def test_rebuild_with_missing_index_wont_fail(self):
+        elastic = self.app.data.elastic
+        alias = elastic._resource_index(RESOURCE)
+        index = elastic.get_index(RESOURCE)
+        es = elastic.elastic(RESOURCE)
+        es.indices.delete_alias(index, alias)
+        es.indices.delete(index)
+
+        RebuildElasticIndex().run(RESOURCE)
+
+        assert es.indices.exists_alias(alias)
+
+    def test_rebuild_with_index_without_alias(self):
+        elastic = self.app.data.elastic
+        alias = elastic._resource_index(RESOURCE)
+        index = elastic.get_index(RESOURCE)
+        es = elastic.elastic(RESOURCE)
+        es.indices.delete_alias(index, alias)
+        es.indices.delete(index)
+
+        es.indices.create(alias)
+        get_resource_service(RESOURCE).post(self.data)
+
+        RebuildElasticIndex().run(RESOURCE)
+
+        assert es.indices.exists_alias(alias)
+
+        items = self.query_items()
         self.assertEqual(10, items.count())