The fastest way to check if a string is a substring in a string list

Question

I have a static list of 4000 different first names: therefore, the length of the list is large (4000), but each line has from 4 to 12 characters (these are names).

Then I have a dynamic list of 10,000 rows retrieved from the database: these rows can be of arbitrary length.

I need to print for each of the 10,000 lines whether this line contains one of 4,000 names, and if so, which one. If it contains several names, I need only one of them (i.e. the First). Moreover, they are unlikely to find such names, so perhaps only 10 out of 10,000 will contain the name.

My code is:

names # list of 4000 short static names
fields # list of 10000 retrieved strings

def findit(element):
    for name in names:
        if name in element:
            return name
    return None

output = [findit(element) for element in fields]

, . , , , , (.. bisect , ). , . , 10000 x 4000 = 40 "" .

?

Answer 1

. , , :

names = ['AARON',
    'ABDUL',
    'ABE',
    'ABEL',
    'ABRAHAM',
    'ABRAM',
    'ADALBERTO',
    'ADAM',
    'ADAN',
    'ADOLFO',
    'ADOLPH',
    'ADRIAN',
]

:

\b(?:AARON|ABDUL|ABE|ABEL|ABRAHAM|ABRAM|ADALBERTO|ADAM|ADAN|ADOLFO|ADOLPH|ADRIAN)\b

. , , :

\b(?:A(?:B(?:E(?:|L)|RA(?:M|HAM)|DUL)|D(?:A(?:M|N|LBERTO)|OL(?:FO|PH)|RIAN)|ARON))\b

- , dict -tree , . :

{
    'A': {
        'A': {
            'R': {
                'O': {
                    'N': {
                        '': {}
                    }
                }
            }
        }, 
        'B': {
            'D': {
                'U': {
                    'L': {
                        '': {}
                    }
                }
            }, 
            'E': {
                '': {}, 
                'L': {
                    '': {}
                }
            }, 
... etc

... :

{
    'A': {
        'ARON': {
            '': {}
        }
        'B': {
            'DUL': {
                '': {}
            },
            'E': {
                '': {}, 
                'L': {
                    '': {}
                }
            },
            'RA': {
                'HAM': {
                    '': {}
                },
                'M': {
                    '': {}
                } 
            } 
        }, 

... etc

:

import re 

def addToTree(tree, name):
    if len(name) == 0:
        return
    if name[0] in tree.keys():
        addToTree(tree[name[0]], name[1:])
    else:
        for letter in name:
            tree[letter] = {}
            tree = tree[letter]
        tree[''] = {}

# Optional improvement of the tree: it combines several consecutive letters into 
# one key if there are no alternatives possible
def simplifyTree(tree):
    repeat = True
    while repeat:
        repeat = False
        for key, subtree in list(tree.items()):
            if key != '' and len(subtree) == 1 and '' not in subtree.keys():
                for letter, subsubtree in subtree.items():
                    tree[key + letter] = subsubtree
                del tree[key]
                repeat = True
    for key, subtree in tree.items():
        if key != '': 
            simplifyTree(subtree)

def treeToRegExp(tree):
    regexp = [re.escape(key) + treeToRegExp(subtree) for key, subtree in tree.items()]
    regexp = '|'.join(regexp)
    return '' if regexp == '' else '(?:' + regexp + ')'

def listToRegExp(names):
    tree = {}
    for name in names:
        addToTree(tree, name[:])
    simplifyTree(tree)
    return re.compile(r'\b' + treeToRegExp(tree) + r'\b', re.I)

# Demo
names = ['AARON',
'ABDUL',
'ABE',
'ABEL',
'ABRAHAM',
'ABRAM',
'ADALBERTO',
'ADAM',
'ADAN',
'ADOLFO',
'ADOLPH',
'ADRIAN',
]

fields = [
    'This is Aaron speaking',
    'Is Abex a name?',
    'Where did Abraham get the mustard from?'
]

regexp = listToRegExp(names)
# get the search result for each field, and link it with the index of the field
results = [[i, regexp.search(field)] for i, field in enumerate(fields)]
# remove non-matches from the results
results = [[i, match.group(0)] for [i, match] in results if match]
# print results
print(results)

, repl.it

Answer 2

, , set() :

names = ['AARON',
    'ABDUL',
    'ABE',
    'ABEL',
    'ABRAHAM',
    'ABRAM',
    'ADALBERTO',
    'ADAM',
    'ADAN',
    'ADOLFO',
    'ADOLPH',
    'ADRIAN',
]

search = {'BE', 'LFO', 'AB'}

def get_all_substrings(input_string):
  length = len(input_string)
  return {input_string[i:j+1] for i in range(length) for j in xrange(i,length)}

names_subs = {name: get_all_substrings(name)  for name in names}
result = [name for name, sub in names_subs.items() if bool(search.intersection(sub))]

Answer 3

Aho-Corasick (. https://en.wikipedia.org/wiki/Aho%E2%80%93Corasick_algorithm) python pyahocorasick (. http://pyahocorasick.readthedocs.io/en/latest/).

:

import ahocorasick

names # list of 4000 short static names
fields # list of 10000 retrieved strings

automaton = ahocorasick.Automaton()

for name in names:
    automaton.add_word(name, name)

automaton.make_automaton()

def findit_with_ahocorasick(element):
    try:
        return next(A.iter(element))[1]
    except StopIteration:
        return None


output = [findit_with_ahocorasick(element) for element in fields]

, (.. , 12 0,8 10000).

, docs, Automaton, , , , .