All geek questions in one place

ElasticSearch: filter by deeply embedded data

Our data is stored in MongoDB 2.4.8 and indexed on ElasticSearch 0.90.7 using ElasticSearch MongoDB River 1.7.3.

Our data indexes are correct, and I can successfully search for the fields we want to find. But I also need to filter the resolution - of course, we only want to return the results that the calling user can actually read.

In the code on our server, I have the authorization of the calling user in the form of an array, for example:

[ "Role:REGISTERED_USER", "Account:52c74b25da06f102c90d52f4", "Role:USER", "Group:52cb057cda06ca463e78f0d7" ]

Below is an example of the block data that we execute:

{
    "_id" : ObjectId("52dffbd6da06422559386f7d"),
    "content" : "various stuff",
    "ownerId" : ObjectId("52d96bfada0695fcbdb41daf"),
    "acls" : [
        {
            "accessMap" : {},
            "sourceClass" : "com.bulb.learn.domain.units.PublishedPageUnit",
            "sourceId" : ObjectId("52dffbd6da06422559386f7d")
        },
        {
            "accessMap" : {
                "Role:USER" : {
                    "allow" : [
                        "READ"
                    ]
                },
                "Account:52d96bfada0695fcbdb41daf" : {
                    "allow" : [
                        "CREATE",
                        "READ",
                        "UPDATE",
                        "DELETE",
                        "GRANT"
                    ]
                }
            },
            "sourceClass" : "com.bulb.learn.domain.units.CompositeUnit",
            "sourceId" : ObjectId("52dffb54da06422559386f57")
        }
    ]
}

In the above examples, I replaced all the searchable content with  "content" : "various stuff"

Authorization data is in the "acls" array. The filter I need to write will do the following (in English):

pass all units where the "acls" array
contains an "accessMap" object
that contains a property whose name is one of the user authorization strings
and whose "allow" property contains "READ"
and whose "deny" property does not contain "READ"

": ", , ": ", "", "READ" "Role: USER", "". , .

, , ElasticSearch.

, : "" "has_child" ( "has_parent" ).

"" , , -, , , - . , , .

, "has_child" "has_parent" ( ?), node . , ElasticSearch MongoDB River .

? ?

+4
2

, @Alex Brasetvik, , , "join-per-update", parent-child "join-per-query", .

, "" parent-child, .

, " bool".

:

{
    "_id" : ObjectId("52dffbd6da06422559386f7d"),
    "content" : "various stuff",
    "ownerId" : ObjectId("52d96bfada0695fcbdb41daf"),
    "accessMaps" : [
        {
            "sourceClass" : "com.bulb.learn.domain.units.PublishedPageUnit",
            "sourceId" : ObjectId("52dffbd6da06422559386f7d")
        },
        {
            "allow" : {
                "CREATE" : [
                    "Account:52d96bfada0695fcbdb41daf"
                ],
                "READ" : [
                    "Account:52d96bfada0695fcbdb41daf",
                    "Role:USER"
                ],
                "UPDATE" : [
                    "Account:52d96bfada0695fcbdb41daf"
                ],
                "DELETE" : [
                    "Account:52d96bfada0695fcbdb41daf"
                ],
                "GRANT" : [
                    "Account:52d96bfada0695fcbdb41daf"
                ]
            },
            "deny" : {},
            "sourceClass" : "com.bulb.learn.domain.units.CompositeUnit",
            "sourceId" : ObjectId("52dffb54da06422559386f57")
        }
    ]
}

:

{
  "unit": {
    "properties": {
      "accessMaps": {
        "type": "nested",
        "properties": {
          "allow": {
            "type": "nested",
            "properties": {
              "CREATE": {
                "type": "string",
                "index": "not_analyzed",
              },
              "DELETE": {
                "type": "string",
                "index": "not_analyzed",
              },
              "GRANT": {
                "type": "string",
                "index": "not_analyzed",
              },
              "READ": {
                "type": "string",
                "index": "not_analyzed",
              },
              "UPDATE": {
                "type": "string",
                "index": "not_analyzed",
              }
            } 
          },    
          "deny": {
            "type": "nested",
            "properties": {
              "CREATE": {
                "type": "string",
                "index": "not_analyzed",
              },
              "DELETE": {
                "type": "string",
                "index": "not_analyzed",
              },
              "GRANT": {
                "type": "string",
                "index": "not_analyzed",
              },
              "READ": {
                "type": "string",
                "index": "not_analyzed",
              },
              "UPDATE": {
                "type": "string",
                "index": "not_analyzed",
              } 
            }   
          },    
          "sourceClass": {
            "type": "string"
          },
          "sourceId": {
            "type": "string"
          }
        }
      }
    }
  }
}

:

{
  "query": {
    "filtered": {
      "query": {
        "match_all": {}
      },
      "filter": {
        "bool": {
          "must": {
            "nested": {
              "path": "accessMaps.allow",
              "filter": {
                "terms": {
                  "accessMaps.allow.READ": [
                    "Role:REGISTERED_USER",
                    "Account:52e6a361da06e4eb64172519",
                    "Role:USER",
                    "Group:52cb057cda06ca463e78f0d7"
                  ]
                }
              }
            }
          },
          "must_not": {
            "nested": {
              "path": "accessMaps.deny",
              "filter": {
                "terms": {
                  "accessMaps.deny.READ": [
                    "Role:REGISTERED_USER",
                    "Account:52e6a361da06e4eb64172519",
                    "Role:USER",
                    "Group:52cb057cda06ca463e78f0d7"
                  ]
                }
              }
            }
          }
        }
      }
    }
  }
}

, , - , "" . , ElasticSearch .

-5

.

Elasticsearch. , , , .

, , accessMap , , . . , accessMap .

ACL ( ), - , . , , - . -, .

, , : https://www.found.no/play/gist/8582654

, nested - bool -, erm, . , bool.

#!/bin/bash

export ELASTICSEARCH_ENDPOINT="http://localhost:9200"

# Create indexes

curl -XPUT "$ELASTICSEARCH_ENDPOINT/play" -d '{
    "settings": {
        "analysis": {}
    },
    "mappings": {
        "type": {
            "properties": {
                "acls": {
                    "type": "nested",
                    "properties": {
                        "accessMap": {
                            "type": "nested",
                            "properties": {
                                "allow": {
                                    "type": "string",
                                    "index": "not_analyzed"
                                },
                                "deny": {
                                    "type": "string",
                                    "index": "not_analyzed"
                                },
                                "key": {
                                    "type": "string",
                                    "index": "not_analyzed"
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}'


# Index documents
curl -XPOST "$ELASTICSEARCH_ENDPOINT/_bulk?refresh=true" -d '
{"index":{"_index":"play","_type":"type","_id":1}}
{"acls":[{"accessMap":[{"key":"Role:USER","allow":["READ"]},{"key":"Account:52d96bfada0695fcbdb41daf","allow":["READ","UPDATE"]}]}]}
{"index":{"_index":"play","_type":"type","_id":2}}
{"acls":[{"accessMap":[{"key":"Role:USER","allow":["READ"]},{"key":"Account:52d96bfada0695fcbdb41daf","deny":["READ","UPDATE"]}]}]}
{"index":{"_index":"play","_type":"type","_id":3}}
{"acls":[{"accessMap":[{"key":"Role:USER","allow":["READ"]},{"key":"Account:52d96bfada0695fcbdb41daf","allow":["READ","UPDATE"]}]}]}
'

# Do searches

curl -XPOST "$ELASTICSEARCH_ENDPOINT/_search?pretty" -d '
{
    "query": {
        "filtered": {
            "filter": {
                "nested": {
                    "path": "acls",
                    "filter": {
                        "bool": {
                            "must": {
                                "nested": {
                                    "path": "acls.accessMap",
                                    "filter": {
                                        "bool": {
                                            "must": [
                                                {
                                                    "term": {
                                                        "allow": "READ"
                                                    }
                                                },
                                                {
                                                    "terms": {
                                                        "key": [
                                                            "Role:USER",
                                                            "Account:52d96bfada0695fcbdb41daf"
                                                        ]
                                                    }
                                                }
                                            ]
                                        }
                                    }
                                }
                            },
                            "must_not": {
                                "nested": {
                                    "path": "acls.accessMap",
                                    "filter": {
                                        "bool": {
                                            "must": [
                                                {
                                                    "term": {
                                                        "deny": "READ"
                                                    }
                                                },
                                                {
                                                    "terms": {
                                                        "key": [
                                                            "Role:USER",
                                                            "Account:52d96bfada0695fcbdb41daf"
                                                        ]
                                                    }
                                                }
                                            ]
                                        }
                                    }
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}
'

parent-child: https://www.found.no/play/gist/8586840

#!/bin/bash

export ELASTICSEARCH_ENDPOINT="http://localhost:9200"

# Create indexes

curl -XPUT "$ELASTICSEARCH_ENDPOINT/play" -d '{
    "settings": {
        "analysis": {}
    },
    "mappings": {
        "acl": {
            "_parent": {
                "type": "document"
            },
            "properties": {
                "acls": {
                    "properties": {
                        "accessMap": {
                            "type": "nested",
                            "properties": {
                                "key": {
                                    "type": "string",
                                    "index": "not_analyzed"
                                },
                                "allow": {
                                    "type": "string",
                                    "index": "not_analyzed"
                                },
                                "deny": {
                                    "type": "string",
                                    "index": "not_analyzed"
                                }
                            }
                        }
                    }
                }
            }
        }
    }
}'


# Index documents
curl -XPOST "$ELASTICSEARCH_ENDPOINT/_bulk?refresh=true" -d '
{"index":{"_index":"play","_type":"document","_id":1}}
{"title":"Doc 1"}
{"index":{"_index":"play","_type":"acl","_parent":1}}
{"acls":[{"accessMap":[{"key":"Role:USER","allow":["READ"]},{"key":"Account:52d96bfada0695fcbdb41daf","allow":["READ","UPDATE"]}]}]}
{"index":{"_index":"play","_type":"document","_id":2}}
{"title":"Doc 2"}
{"index":{"_index":"play","_type":"acl","_parent":2}}
{"acls":[{"accessMap":[{"key":"Role:USER","allow":["READ"]},{"key":"Account:52d96bfada0695fcbdb41daf","deny":["READ","UPDATE"]}]}]}
'

# Do searches

curl -XPOST "$ELASTICSEARCH_ENDPOINT/_search?pretty" -d '
{
    "query": {
        "filtered": {
            "filter": {
                "has_child": {
                    "type": "acl",
                    "filter": {
                        "bool": {
                            "must": [
                                {
                                    "nested": {
                                        "path": "acls.accessMap",
                                        "filter": {
                                            "bool": {
                                                "must": [
                                                    {
                                                        "terms": {
                                                            "key": [
                                                                "Role:USER",
                                                                "Account:52d96bfada0695fcbdb41daf"
                                                            ]
                                                        }
                                                    },
                                                    {
                                                        "term": {
                                                            "allow": "READ"
                                                        }
                                                    }
                                                ]
                                            }
                                        }
                                    }
                                }
                            ],
                            "must_not": [
                                {
                                    "nested": {
                                        "path": "acls.accessMap",
                                        "filter": {
                                            "bool": {
                                                "must": [
                                                    {
                                                        "terms": {
                                                            "key": [
                                                                "Role:USER",
                                                                "Account:52d96bfada0695fcbdb41daf"
                                                            ]
                                                        }
                                                    },
                                                    {
                                                        "term": {
                                                            "deny": "READ"
                                                        }
                                                    }
                                                ]
                                            }
                                        }
                                    }
                                }
                            ]
                        }
                    }
                }
            }
        }
    }
}
'
+9
source

Source: https://habr.com/ru/post/1523385/

More articles:


All Articles