Combine two lists of dicts of different lengths using the same key in Python

Question

I want to combine two dictionary lists by one key when the two lists have different lengths (using Python 3.6). For example, if we have a list of dicts called l1:

l1 = [{'pcd_sector': 'ABDC', 'coverage_2014': '100'},
       {'pcd_sector': 'DEFG', 'coverage_2014': '0'}]

and another dicts list called l2:

l2 = [{'pcd_sector': 'ABDC', 'asset': '3G', 'asset_id': '2gs'},
      {'pcd_sector': 'ABDC', 'asset': '4G', 'asset_id': '7jd'},
      {'pcd_sector': 'DEFG', 'asset': '3G', 'asset_id': '3je'},
      {'pcd_sector': 'DEFG', 'asset': '4G', 'asset_id': '8js'},
      {'pcd_sector': 'CDEF', 'asset': '3G', 'asset_id': '4jd'}]

How to combine them using pcd_sectorto get this (?):

result = [{'pcd_sector': 'ABDC', 'asset': '3G', 'asset_id': '2gs', 'coverage_2014': '100'},
          {'pcd_sector': 'ABDC', 'asset': '4G', 'asset_id': '7jd', 'coverage_2014': '100'},
          {'pcd_sector': 'DEFG', 'asset': '3G', 'asset_id': '3je', 'coverage_2014': '0'},
          {'pcd_sector': 'DEFG', 'asset': '4G', 'asset_id': '8js', 'coverage_2014': '0'},
          {'pcd_sector': 'CDEF', 'asset': '3G', 'asset_id': '4jd'}]

What have i tried so far

I used the following code to combine the two lists, but unfortunately I am running out of short version, and not the desired complete data structure.

import pprint
grouped = {}
for d in l1 + l2:
    grouped.setdefault(d['pcd_sector'], {'asset':0, 'asset_id':0, 'coverage_2014':0}).update(d)
result = [d for d in grouped.values()]
pprint.pprint(result)

Therefore, when I run the code, I end this short output:

result = [{'pcd_sector': 'ABDC', 'asset': '3G', 'asset_id': '2gs', 'coverage_2014': '100'},
         {'pcd_sector': 'DEFG', 'asset': '4G', 'asset_id': '8js', 'coverage_2014': '0'},
         {'pcd_sector': 'CDEF', 'asset': '3G', 'asset_id': '4jd'}]

Answer 1

Problem

, grouped dict 'pcd_sector' , l2 dicts 'pcd_sector'. 'pcd_sector', 'asset' l2, l1. , , , l1 + l2.

pcd_sector l1, dict dicts:

>>> d1 = {d['pcd_sector']:d for d in l1}
>>> d1
{'ABDC': {'pcd_sector': 'ABDC', 'coverage_2014': '100'}, 'DEFG': {'pcd_sector': 'DEFG', 'coverage_2014': '0'}}

pcd_sector:

>>> [dict(d, **d1.get(d['pcd_sector'], {})) for d in l2]
[{'asset_id': '2gs', 'coverage_2014': '100', 'pcd_sector': 'ABDC', 'asset': '3G'}, {'asset_id': '7jd', 'coverage_2014': '100', 'pcd_sector': 'ABDC', 'asset': '4G'}, {'asset_id': '3je', 'coverage_2014': '0', 'pcd_sector': 'DEFG', 'asset': '3G'}, {'asset_id': '8js', 'coverage_2014': '0', 'pcd_sector': 'DEFG', 'asset': '4G'}, {'asset_id': '4jd', 'pcd_sector': 'CDEF', 'asset': '3G'}]

, :

l1 = [{'pcd_sector': 'ABDC', 'coverage_2014': '100'},
       {'pcd_sector': 'DEFG', 'coverage_2014': '0'}]

l2 = [{'pcd_sector': 'ABDC', 'asset': '3G', 'asset_id': '2gs'},
      {'pcd_sector': 'ABDC', 'asset': '4G', 'asset_id': '7jd'},
      {'pcd_sector': 'DEFG', 'asset': '3G', 'asset_id': '3je'},
      {'pcd_sector': 'DEFG', 'asset': '4G', 'asset_id': '8js'},
      {'pcd_sector': 'CDEF', 'asset': '3G', 'asset_id': '4jd'}]

d1 = {d['pcd_sector']:d for d in l1}
result = [dict(d, **d1.get(d['pcd_sector'], {})) for d in l2]

import pprint
pprint.pprint(result)
#   [{'asset': '3G',
#     'asset_id': '2gs',
#     'coverage_2014': '100',
#     'pcd_sector': 'ABDC'},
#    {'asset': '4G',
#     'asset_id': '7jd',
#     'coverage_2014': '100',
#     'pcd_sector': 'ABDC'},
#    {'asset': '3G',
#     'asset_id': '3je',
#     'coverage_2014': '0',
#     'pcd_sector': 'DEFG'},
#    {'asset': '4G',
#     'asset_id': '8js',
#     'coverage_2014': '0',
#     'pcd_sector': 'DEFG'},
#    {'asset': '3G', 'asset_id': '4jd', 'pcd_sector': 'CDEF'}]

Answer 2

pcd_sector dicts :

>>> import copy
>>> lookup = { x['pcd_sector'] : x for x in l1 }
>>> result = copy.deepcopy(l2)
>>> for d in result:
...     d.update(lookup.get(d['pcd_sector'], {})) # golfed courtesy Ashwini Chaudhary
... 
>>> result
[{'pcd_sector': 'ABDC', 'asset': '3G', 'asset_id': '2gs', 'coverage_2014': '100'}, 
{'pcd_sector': 'ABDC', 'asset': '4G', 'asset_id': '7jd', 'coverage_2014': '100'}, 
{'pcd_sector': 'DEFG', 'asset': '3G', 'asset_id': '3je', 'coverage_2014': '0'}, 
{'pcd_sector': 'DEFG', 'asset': '4G', 'asset_id': '8js', 'coverage_2014': '0'},
{'pcd_sector': 'CDEF', 'asset': '3G', 'asset_id': '4jd'}]

Answer 3

pandas:

import pandas as pd

df1 = pd.DataFrame(l1)
df2 = pd.DataFrame(l2)
dfr = df1.join(df2, how='outer')
print(dfr)

:

  coverage_2014 pcd_sector asset asset_id
0           100       ABDC    3G      2gs
1           100       ABDC    4G      7jd
2             0       DEFG    3G      3je
3             0       DEFG    4G      8js
4           NaN       CDEF    3G      4jd

If you want it as a dictionary again:

result = dfr.to_dict('records')
print(result)

Output (with the addition of lines):

[{'coverage_2014': '100', 'pcd_sector': 'ABDC', 'asset': '3G', 'asset_id': '2gs'},
 {'coverage_2014': '100', 'pcd_sector': 'ABDC', 'asset': '4G', 'asset_id': '7jd'},
 {'coverage_2014': '0', 'pcd_sector': 'DEFG', 'asset': '3G', 'asset_id': '3je'},
 {'coverage_2014': '0', 'pcd_sector': 'DEFG', 'asset': '4G', 'asset_id': '8js'},
 {'coverage_2014': nan, 'pcd_sector': 'CDEF', 'asset': '3G', 'asset_id': '4jd'}]