Performance is slow when replacing a string in a pandas frame using dict

Question

The following code works, but it needs to work faster. The dict has ~ 25K keys, and the dataframe has ~ 3M rows. Is there a way to get the same results, but with python code that will work faster? (without multiprocessing, processing will be 8 times slower).

miscdict={" isn't ": ' is not '," aren't ":' are not '," wasn't ":' was not '," snevada ":' Sierra Nevada '}

df=pd.DataFrame({"q1":["beer is ok","beer isn't ok","beer wasn't available"," snevada is good"]})

def parse_text(data):
    for key, replacement in miscdict.items():
        data['q1'] = data['q1'].str.replace( key, replacement )
    return data

if __name__ == '__main__':
    t1_1 = datetime.datetime.now()
    p = multiprocessing.Pool(processes=8)
    split_dfs = np.array_split(df,8)
    pool_results = p.map(parse_text, split_dfs)
    p.close()
    p.join()
    parts = pd.concat(pool_results, axis=0)
    df = pd.concat([parts], axis=1)
    t2_1 = datetime.datetime.now()
    print("done"+ str(t2_1-t1_1)) 

Answer 1

I checked a few of them. The @ A-Za-z proposal is a significant improvement, but it may be possible to do it even faster.

edit. , dict dataframe ( ). :

• : 11.71
• @A-Za-z: 4.72 , 60%.
• @piRSquared: 4,95 , 58%.
• : 2,81 , 76%.

, :

" . 15 , @A-Za-z 8-9 , 6 . . . ."

---

import pandas as pd
import re
import timeit

---

:

miscdict = {" isn't ": ' is not '," aren't ":' are not '," wasn't ":' was not '," snevada ":' Sierra Nevada '}
data=pd.DataFrame({"q1":["beer is ok","beer isn't ok","beer wasn't available"," snevada is good"]})
def org(printout=False):
    def parse_text(data):
        for key, replacement in miscdict.items():
            data['q1'] = data['q1'].str.replace( key, replacement )
        return data
    data2 = parse_text(data)
    if printout:
        print(data2)
org(printout=True)
print(timeit.timeit(org, number=10000))

11,7 :

                       q1
0              beer is ok
1          beer is not ok
2  beer was not available
3   Sierra Nevada is good
11.71043858179268

---

@A-Za-z:

miscdict = {" isn't ": ' is not '," aren't ":' are not '," wasn't ":' was not '," snevada ":' Sierra Nevada '}
data=pd.DataFrame({"q1":["beer is ok","beer isn't ok","beer wasn't available"," snevada is good"]})
def alt1(printout=False):
    data['q1'].replace(miscdict, regex = True, inplace = True)
    if printout:
        print(data)
alt1(printout=True)
print(timeit.timeit(alt1, number=10000))

4,7 :

                       q1
0              beer is ok
1          beer is not ok
2  beer was not available
3   Sierra Nevada is good
4.721581550644499

---

@piRSquared:

miscdict = {" isn't ": ' is not '," aren't ":' are not '," wasn't ":' was not '," snevada ":' Sierra Nevada '}
data=pd.DataFrame({"q1":["beer is ok","beer isn't ok","beer wasn't available"," snevada is good"]})
def alt2(printout=False):
    # regex = True is added later because it doesn't work without it.
    data = data.replace(miscdict, regex = True)
    if printout:
        print(data)
alt2(printout=True)
print(timeit.timeit(alt2, number=10000))

5,0 :

                       q1
0              beer is ok
1          beer is not ok
2  beer was not available
3   Sierra Nevada is good
4.951810616074919

---

miscdict = {" isn't ": ' is not '," aren't ":' are not '," wasn't ":' was not '," snevada ":' Sierra Nevada '}
miscdict_comp = {re.compile(k): v for k, v in miscdict.items()}
data=pd.DataFrame({"q1":["beer is ok","beer isn't ok","beer wasn't available"," snevada is good"]})
def alt3(printout=False):
    def parse_text(text):
        for pattern, replacement in miscdict_comp.items():
            text = pattern.sub(replacement, text)
        return text
    data["q1"] = data["q1"].apply(parse_text)
    if printout:
        print(data)
alt3(printout=True)
print(timeit.timeit(alt3, number=10000))

2,8 :

                       q1
0              beer is ok
1          beer is not ok
2  beer was not available
3   Sierra Nevada is good
2.810334940701157

, , .

: https://jerel.co/blog/2011/12/using-python-for-super-fast-regex-search-and-replace

Answer 2

, df.replace regex = True .

df['q1'].replace(miscdict, regex = True, inplace = True)
1000 loops, best of 3: 1.08 ms per loop

        q1
0   beer is ok
1   beer is not ok
2   beer was not available
3   Sierra Nevada is good

for key, replacement in miscdict.items(): df['q1'] = df['q1'].str.replace( key, replacement )
100 loops, best of 3: 2.35 ms per loop

Answer 3

WOW! We reinvented the wheel and designed some chic spokes and spikes and ...

... just do it

df.replace(miscdict)

                       q1
0              beer is ok
1          beer is not ok
2  beer was not available
3   Sierra Nevada is good

If I miss something obvious.