11477 refactor append dsl

arches/app/models/migrations/11477_permitted_graphs_policy.py

@@ -0,0 +1,38 @@
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("models", "11042_update__arches_staging_to_tile"),
+    ]
+    sql = """
+    INSERT INTO search_component
+    VALUES (
+        '56b3feba-ffd4-445e-bd82-377fca29ff92',
+        'Permited Graphs Policy',
+        '',
+        'permitted_graphs_policy.py',
+        'PermittedGraphsPolicy',
+        'search-policy',
+        NULL,
+        'permitted-graphs-policy',
+        '{}'
+    );
+    UPDATE search_component
+    SET config = '{"default":true,"linkedSearchFilters":[{"componentname":"map-filter","searchcomponentid":"09d97fc6-8c83-4319-9cef-3aaa08c3fbec","layoutSortorder":1},{"componentname":"advanced-search","searchcomponentid":"f0e56205-acb5-475b-9c98-f5e44f1dbd2c","layoutSortorder":2},{"componentname":"related-resources-filter","searchcomponentid":"59f28272-d1f1-4805-af51-227771739aed","layoutSortorder":3},{"componentname":"provisional-filter","searchcomponentid":"073406ed-93e5-4b5b-9418-b61c26b3640f","layoutSortorder":4},{"componentname":"resource-type-filter","searchcomponentid":"f1c46b7d-0132-421b-b1f3-95d67f9b3980","layoutSortorder":5},{"componentname":"saved-searches","searchcomponentid":"6dc29637-43a1-4fba-adae-8d9956dcd3b9","layoutSortorder":6},{"componentname":"search-export","searchcomponentid":"9c6a5a9c-a7ec-48d2-8a25-501b55b8eff6","layoutSortorder":7},{"componentname":"search-result-details","searchcomponentid":"f5986dae-8b01-11ea-b65a-77903936669c","layoutSortorder":8},{"componentname":"sort-results","searchcomponentid":"6a2fe122-de54-4e44-8e93-b6a0cda7955c","layoutSortorder":9},{"componentname":"term-filter","searchcomponentid":"1f42f501-ed70-48c5-bae1-6ff7d0d187da","layoutSortorder":10},{"componentname":"time-filter","searchcomponentid":"7497ed4f-2085-40da-bee5-52076a48bcb1","layoutSortorder":11},{"componentname":"paging-filter","searchcomponentid":"7aff5819-651c-4390-9b9a-a61221ba52c6","required":true,"layoutSortorder":12,"executionSortorder":2},{"componentname":"search-results","searchcomponentid":"00673743-8c1c-4cc0-bd85-c073a52e03ec","required":true,"layoutSortorder":13,"executionSortorder":1},{"componentname":"permitted-graphs-policy","searchcomponentid":"56b3feba-ffd4-445e-bd82-377fca29ff92","required":true,"layoutSortorder":14,"executionSortorder":3}]}'
+    WHERE searchcomponentid = '69695d63-6f03-4536-8da9-841b07116381';
+    """
+    reverse_sql = """
+    UPDATE search_component
+    SET config = '{"default":true,"linkedSearchFilters":[{"componentname":"map-filter","searchcomponentid":"09d97fc6-8c83-4319-9cef-3aaa08c3fbec","layoutSortorder":1},{"componentname":"advanced-search","searchcomponentid":"f0e56205-acb5-475b-9c98-f5e44f1dbd2c","layoutSortorder":2},{"componentname":"related-resources-filter","searchcomponentid":"59f28272-d1f1-4805-af51-227771739aed","layoutSortorder":3},{"componentname":"provisional-filter","searchcomponentid":"073406ed-93e5-4b5b-9418-b61c26b3640f","layoutSortorder":4},{"componentname":"resource-type-filter","searchcomponentid":"f1c46b7d-0132-421b-b1f3-95d67f9b3980","layoutSortorder":5},{"componentname":"saved-searches","searchcomponentid":"6dc29637-43a1-4fba-adae-8d9956dcd3b9","layoutSortorder":6},{"componentname":"search-export","searchcomponentid":"9c6a5a9c-a7ec-48d2-8a25-501b55b8eff6","layoutSortorder":7},{"componentname":"search-result-details","searchcomponentid":"f5986dae-8b01-11ea-b65a-77903936669c","layoutSortorder":8},{"componentname":"sort-results","searchcomponentid":"6a2fe122-de54-4e44-8e93-b6a0cda7955c","layoutSortorder":9},{"componentname":"term-filter","searchcomponentid":"1f42f501-ed70-48c5-bae1-6ff7d0d187da","layoutSortorder":10},{"componentname":"time-filter","searchcomponentid":"7497ed4f-2085-40da-bee5-52076a48bcb1","layoutSortorder":11},{"componentname":"paging-filter","searchcomponentid":"7aff5819-651c-4390-9b9a-a61221ba52c6","required":true,"layoutSortorder":12,"executionSortorder":2},{"componentname":"search-results","searchcomponentid":"00673743-8c1c-4cc0-bd85-c073a52e03ec","required":true,"layoutSortorder":13,"executionSortorder":1}]}'
+    WHERE searchcomponentid = '69695d63-6f03-4536-8da9-841b07116381';
+    DELETE FROM search_component WHERE searchcomponentid = '56b3feba-ffd4-445e-bd82-377fca29ff92';
+    """
+
+    operations = [
+        migrations.RunSQL(
+            sql=sql,
+            reverse_sql=reverse_sql,
+        ),
+    ]

arches/app/search/components/advanced_search.py

@@ -25,9 +25,10 @@
 
 
 class AdvancedSearch(BaseSearchFilter):
-    def append_dsl(self, search_query_object, **kwargs):
-        querystring_params = kwargs.get("querystring", "")
-        advanced_filters = JSONDeserializer().deserialize(querystring_params)
+
+    def generate_dsl(self, search_query_object, **kwargs):
+        querysting_params = kwargs.get("querystring", "[]")
+        advanced_filters = JSONDeserializer().deserialize(querysting_params)
         datatype_factory = DataTypeFactory()
         search_query = Bool()
         advanced_query = Bool()
@@ -70,7 +71,11 @@ def append_dsl(self, search_query_object, **kwargs):
         for grouped_query in grouped_queries:
             advanced_query.should(grouped_query)
         search_query.must(advanced_query)
-        search_query_object["query"].add_query(search_query)
+        return search_query
+
+    def append_dsl(self, search_query_object, **kwargs):
+        dsl = self.generate_dsl(search_query_object, **kwargs)
+        search_query_object["query"].add_query(dsl)
 
     def view_data(self):
         ret = {}

arches/app/search/components/base.py

@@ -14,9 +14,19 @@ def __init__(self, request=None, user=None, componentname=None):
         self.user = user
         self.componentname = componentname
 
+    def generate_dsl(self, search_query_object, **kwargs):
+        """
+        used to generate ES query dsl
+
+        """
+
+        pass
+
     def append_dsl(self, search_query_object, **kwargs):
         """
-        used to append ES query dsl to the search request
+        used to append ES query dsl to the search request,
+        also used to modify the search query object
+        if custom index hooks are present
 
         """
 

arches/app/search/components/lifecycle_state_filter.py

@@ -18,13 +18,11 @@
 
 
 class LifecycleStateFilter(BaseSearchFilter):
-    def append_dsl(
-        self, search_results_object, permitted_nodegroups, include_provisional
-    ):
+    def generate_dsl(self, search_query_object, **kwargs):
         search_query = Bool()
         resource_instance_lifecycle_state_filter = Bool()
 
-        querystring_params = self.request.GET.get(details["componentname"], "")
+        querystring_params = self.request.GET.get(self.componentname, "")
         resource_instance_lifecycle_state_filter_term = JSONDeserializer().deserialize(
             querystring_params
         )[0]
@@ -54,4 +52,8 @@ def append_dsl(
             )
 
         search_query.must(resource_instance_lifecycle_state_filter)
-        search_results_object["query"].add_query(search_query)
+        return search_query
+
+    def append_dsl(self, search_results_object, **kwargs):
+        dsl = self.generate_dsl(search_results_object, kwargs)
+        search_results_object["query"].add_query(dsl)

arches/app/search/components/map_filter.py

@@ -23,7 +23,8 @@
 
 
 class MapFilter(BaseSearchFilter):
-    def append_dsl(self, search_query_object, **kwargs):
+
+    def generate_dsl(self, search_query_object, **kwargs):
         permitted_nodegroups = kwargs.get("permitted_nodegroups")
         include_provisional = kwargs.get("include_provisional")
         search_query = Bool()
@@ -72,17 +73,21 @@ def append_dsl(self, search_query_object, **kwargs):
                     )
 
                 search_query.filter(Nested(path="geometries", query=spatial_query))
+                try:
+                    search_query_object[self.componentname][
+                        "search_buffer"
+                    ] = feature_geom
+                except NameError:
+                    logger.info(_("Feature geometry is not defined"))
+        return search_query
 
-        search_query_object["query"].add_query(search_query)
+    def append_dsl(self, search_query_object, **kwargs):
+        dsl = self.generate_dsl(search_query_object, **kwargs)
+        search_query_object["query"].add_query(dsl)
 
         if self.componentname not in search_query_object:
             search_query_object[self.componentname] = {}
 
-        try:
-            search_query_object[self.componentname]["search_buffer"] = feature_geom
-        except NameError:
-            logger.info(_("Feature geometry is not defined"))
-
 
 def _buffer(geojson, width=0, unit="ft"):
     geojson = JSONSerializer().serialize(geojson)

arches/app/search/components/permitted_graphs_policy.py

@@ -0,0 +1,37 @@
+from arches.app.search.elasticsearch_dsl_builder import Bool, Terms
+from arches.app.search.components.base import BaseSearchFilter
+from arches.app.search.components.resource_type_filter import get_permitted_graphids
+
+details = {
+    "searchcomponentid": "",
+    "name": "Permitted Graphs Policy",
+    "icon": "",
+    "modulename": "permitted_graphs_policy.py",
+    "classname": "PermittedGraphsPolicy",
+    "type": "search-policy",
+    "componentpath": None,
+    "componentname": "permitted-graphs-policy",
+    "config": {},
+}
+
+
+class PermittedGraphsPolicy(BaseSearchFilter):
+    def generate_dsl(self, search_query_object, **kwargs):
+        resource_model_filter = None
+        permitted_nodegroups = kwargs.get("permitted_nodegroups")
+        try:
+            search_query_object["query"].dsl["query"]["bool"]["filter"][0]["terms"][
+                "graph_id"
+            ]  # check if resource_type filter is already applied
+        except (KeyError, IndexError):
+            resource_model_filter = Bool()
+            permitted_graphids = get_permitted_graphids(permitted_nodegroups)
+            terms = Terms(field="graph_id", terms=list(permitted_graphids))
+            resource_model_filter.filter(terms)
+            search_query_object["query"].add_query(resource_model_filter)
+        return resource_model_filter
+
+    def append_dsl(self, search_query_object, **kwargs):
+        dsl = self.generate_dsl(search_query_object, **kwargs)
+        if dsl:
+            search_query_object["query"].add_query(dsl)

arches/app/search/components/provisional_filter.py

@@ -15,7 +15,8 @@
 
 
 class ProvisionalFilter(BaseSearchFilter):
-    def append_dsl(self, search_query_object, **kwargs):
+
+    def generate_dsl(self, search_query_object, **kwargs):
         include_provisional = kwargs.get("include_provisional")
         search_query = Bool()
 
@@ -33,4 +34,8 @@ def append_dsl(self, search_query_object, **kwargs):
                 )
 
             search_query.must(provisional_resource_filter)
-            search_query_object["query"].add_query(search_query)
+        return search_query
+
+    def append_dsl(self, search_query_object, **kwargs):
+        dsl = self.generate_dsl(search_query_object, **kwargs)
+        search_query_object["query"].add_query(dsl)

arches/app/search/components/resource_type_filter.py

@@ -25,7 +25,8 @@ def get_permitted_graphids(permitted_nodegroups):
 
 
 class ResourceTypeFilter(BaseSearchFilter):
-    def append_dsl(self, search_query_object, **kwargs):
+
+    def generate_dsl(self, search_query_object, **kwargs):
         permitted_nodegroups = kwargs.get("permitted_nodegroups")
         search_query = Bool()
         querystring_params = kwargs.get("querystring", "")
@@ -49,8 +50,11 @@ def append_dsl(self, search_query_object, **kwargs):
             terms = Terms(field="graph_id", terms=graph_ids)
 
         search_query.filter(terms)
+        return search_query
 
-        search_query_object["query"].add_query(search_query)
+    def append_dsl(self, search_query_object, **kwargs):
+        dsl = self.generate_dsl(search_query_object, **kwargs)
+        search_query_object["query"].add_query(dsl)
 
     def view_data(self):
         return {

arches/app/search/components/search_results.py

@@ -30,24 +30,14 @@
 
 
 class SearchResultsFilter(BaseSearchFilter):
-    def append_dsl(self, search_query_object, **kwargs):
+
+    def generate_dsl(self, search_query_object, **kwargs):
         permitted_nodegroups = kwargs.get("permitted_nodegroups")
         include_provisional = kwargs.get("include_provisional")
         nested_agg = NestedAgg(path="points", name="geo_aggs")
         nested_agg_filter = FiltersAgg(name="inner")
         geo_agg_filter = Bool()
 
-        try:
-            search_query_object["query"].dsl["query"]["bool"]["filter"][0]["terms"][
-                "graph_id"
-            ]  # check if resource_type filter is already applied
-        except (KeyError, IndexError):
-            resource_model_filter = Bool()
-            permitted_graphids = get_permitted_graphids(permitted_nodegroups)
-            terms = Terms(field="graph_id", terms=list(permitted_graphids))
-            resource_model_filter.filter(terms)
-            search_query_object["query"].add_query(resource_model_filter)
-
         if include_provisional is True:
             geo_agg_filter.filter(
                 Terms(field="points.provisional", terms=["false", "true"])
@@ -89,9 +79,12 @@ def append_dsl(self, search_query_object, **kwargs):
                 )
             )
             search_query.must(subsearch_query)
-            search_query_object["query"].add_query(search_query)
+        return search_query, nested_agg
 
-        search_query_object["query"].add_aggregation(nested_agg)
+    def append_dsl(self, search_query_object, **kwargs):
+        dsl, aggregation = self.generate_dsl(search_query_object, **kwargs)
+        search_query_object["query"].add_query(dsl)
+        search_query_object["query"].add_aggregation(aggregation)
 
     def post_search_hook(self, search_query_object, response_object, **kwargs):
         permitted_nodegroups = kwargs.get("permitted_nodegroups")

arches/app/search/components/sort_results.py

@@ -15,17 +15,26 @@
 
 
 class SortResults(BaseSearchFilter):
-    def append_dsl(self, search_query_object, **kwargs):
-        sort_param = self.request.GET.get(self.componentname, None)
 
-        if sort_param is not None and sort_param != "":
-            search_query_object["query"].sort(
-                field="displayname.value",
-                dsl={
-                    "order": sort_param,
-                    "nested": {
-                        "path": "displayname",
-                        "filter": {"term": {"displayname.language": get_language()}},
-                    },
+    def generate_dsl(self, search_query_object, **kwargs):
+        sort_param = self.request.GET.get(self.componentname, "")
+        field = None
+        dsl = None
+        if sort_param != "":
+            field = "displayname.value"
+            dsl = {
+                "order": sort_param,
+                "nested": {
+                    "path": "displayname",
+                    "filter": {"term": {"displayname.language": get_language()}},
                 },
+            }
+        return field, dsl
+
+    def append_dsl(self, search_query_object, **kwargs):
+        sort_field, sort_dsl = self.generate_dsl(search_query_object, **kwargs)
+        if sort_field and sort_dsl:
+            search_query_object["query"].sort(
+                field=sort_field,
+                dsl=sort_dsl,
             )

arches/app/search/components/term_filter.py

@@ -28,7 +28,8 @@
 
 
 class TermFilter(BaseSearchFilter):
-    def append_dsl(self, search_query_object, **kwargs):
+
+    def generate_dsl(self, search_query_object, **kwargs):
         permitted_nodegroups = kwargs.get("permitted_nodegroups")
         include_provisional = kwargs.get("include_provisional")
         search_query = Bool()
@@ -148,8 +149,11 @@ def append_dsl(self, search_query_object, **kwargs):
                     search_query.must_not(nested_conceptid_filter)
                 else:
                     search_query.filter(nested_conceptid_filter)
+        return search_query
 
-        search_query_object["query"].add_query(search_query)
+    def append_dsl(self, search_query_object, **kwargs):
+        dsl = self.generate_dsl(search_query_object, **kwargs)
+        search_query_object["query"].add_query(dsl)
 
 
 def _get_child_concepts(conceptid):

arches/app/search/components/time_filter.py

@@ -18,7 +18,7 @@
 
 
 class TimeFilter(BaseSearchFilter):
-    def append_dsl(self, search_query_object, **kwargs):
+    def generate_dsl(self, search_query_object, **kwargs):
         permitted_nodegroups = kwargs.get("permitted_nodegroups")
         include_provisional = kwargs.get("include_provisional")
         search_query = Bool()
@@ -148,8 +148,11 @@ def append_dsl(self, search_query_object, **kwargs):
                 temporal_query.should(Nested(path="dates", query=date_query))
 
             search_query.filter(temporal_query)
+        return search_query
 
-            search_query_object["query"].add_query(search_query)
+    def append_dsl(self, search_query_object, **kwargs):
+        dsl = self.generate_dsl(search_query_object, **kwargs)
+        search_query_object["query"].add_query(dsl)
 
     def view_data(self):
         ret = {}