Add list items in long format in Python Pandas

Question

I have the following data:

study_id       list_value
1              ['aaa', 'bbb']
1              ['aaa']
1              ['ccc']
2              ['ddd', 'eee', 'aaa']
2              np.NaN
2              ['zzz', 'aaa', 'bbb']

How can I convert it to something like that?

study_id       list_value
1              ['aaa', 'bbb', 'ccc']
1              ['aaa', 'bbb', 'ccc']
1              ['aaa', 'bbb', 'ccc']
2              ['aaa', 'bbb', 'ddd', 'eee', 'zzz'] 
2              ['aaa', 'bbb', 'ddd', 'eee', 'zzz'] 
2              ['aaa', 'bbb', 'ddd', 'eee', 'zzz'] # order of list item doesn't matter

Answer 1

itertools.chainusing GroupBy.transform
First, get rid of NaN inside the column using list comprehension (randomly, I know, but this is the fastest way to do this).

df['list_value'] = [
    [] if not isinstance(x, list) else x for x in df.list_value
]

Next, group on study_idand flatten your lists inside GroupBy.transformand extract unique values ​​with set.

from itertools import chain

df['list_value'] = df.groupby('study_id').list_value.transform(
    lambda x: [list(set(chain.from_iterable(x)))]
)

As a last step, if you plan to change individual list items, you may want to do

df['list_value'] = [x[:] for x in df['list_value']]

If not, changes in one list will be reflected in all sublists in this group.

df
   study_id                 list_value
0         1            [aaa, ccc, bbb]
1         1            [aaa, ccc, bbb]
2         1            [aaa, ccc, bbb]
3         2  [bbb, ddd, eee, aaa, zzz]
4         2  [bbb, ddd, eee, aaa, zzz]
5         2  [bbb, ddd, eee, aaa, zzz]

Answer 2

defaultdict

from collections import defaultdict

d = defaultdict(set)

for t in df.dropna(subset=['list_value']).itertuples():
    d[t.study_id] |= set(t.list_value)

df.assign(list_value=df.study_id.map(pd.Series(d).apply(sorted)))


   study_id       list_value
0         1        [a, b, c]
1         1        [a, b, c]
2         1        [a, b, c]
3         2  [a, b, d, e, z]
4         2  [a, b, d, e, z]
5         2  [a, b, d, e, z]

---

np.unique and other other tricks

, ndarray

df.assign(
    list_value=df.study_id.map(
        df.set_index('study_id').list_value.dropna().sum(level=0).apply(np.unique)
    )
)

   study_id       list_value
0         1        [a, b, c]
1         1        [a, b, c]
2         1        [a, b, c]
3         2  [a, b, d, e, z]
4         2  [a, b, d, e, z]
5         2  [a, b, d, e, z]

sorted,

df.assign(
    list_value=df.study_id.map(
        df.set_index('study_id').list_value.dropna()
          .sum(level=0).apply(np.unique).apply(sorted)
    )
)

---

!

df.assign(
    list_value=df.study_id.map(
        df.list_value.str.join('|').groupby(df.study_id).apply(
            lambda x: sorted(set('|'.join(x.dropna()).split('|')))
        )
    )
)

   study_id       list_value
0         1        [a, b, c]
1         1        [a, b, c]
2         1        [a, b, c]
3         2  [a, b, d, e, z]
4         2  [a, b, d, e, z]
5         2  [a, b, d, e, z]

---

df = pd.DataFrame(dict(
    study_id=[1, 1, 1, 2, 2, 2],
    list_value=[['a', 'b'], ['a'], ['c'], ['d', 'e', 'a'], np.nan, ['z', 'a', 'b']]
), columns=['study_id', 'list_value'])

Answer 3

.

import pandas as pd, numpy as np
from itertools import chain

df = pd.DataFrame({'study_id': [1, 1, 1, 2, 2, 2],
                   'list_value': [['aaa', 'bbb',], ['aaa'], ['ccc'],['ddd', 'eee', 'aaa'],
                                  np.nan, ['zzz', 'aaa', 'bbb']]})

counts = df['study_id'].value_counts()

grp = df.dropna(subset=['list_value'])\
        .groupby('study_id')['list_value']\
        .apply(lambda x: sorted(set(chain.from_iterable(x))))\
        .reset_index()

res = pd.concat([pd.concat([grp[grp['study_id'] == x]]*counts[x]) for x in counts.index])\
        .sort_values('study_id')\
        .reset_index(drop=True)

#    study_id                 list_value
# 0         1            [aaa, bbb, ccc]
# 1         1            [aaa, bbb, ccc]
# 2         1            [aaa, bbb, ccc]
# 3         2  [aaa, bbb, ddd, eee, zzz]
# 4         2  [aaa, bbb, ddd, eee, zzz]
# 5         2  [aaa, bbb, ddd, eee, zzz]

Answer 4

Fill your null value with an empty list, then with transform

df.at[df.list_value.isnull().nonzero()[0][0],'list_value']=[]

df.groupby('study_id').list_value.transform(lambda x : [list(set(x.sum()))])
Out[160]: 
0          [b, c, a]
1          [b, c, a]
2          [b, c, a]
3    [b, e, d, z, a]
4    [b, e, d, z, a]
5    [b, e, d, z, a]
Name: list_value, dtype: object