How to execute for loop in packages?

Question

for x in records:
   data = {}
   for y in sObjectName.describe()['fields']
         data[y['name']] = x[y['name']]
   ls.append(adapter.insert_posts(collection, data))

I want to execute the ls.append code (adapter.insert_post (collection, x)) in the lot size of 500, where x should contain 500 dicts data. I could create a list a from 500 data dicts using a double loop and a list, and then insert it. I could do it as follows, is there a better way to do this?

for x in records:
    for i in xrange(0,len(records)/500):
        for j in xrange(0,500):
            l=[]
            data = {}
            for y in sObjectName.describe()['fields']:
                data[y['name']] = x[y['name']]
                #print data
            #print data
            l.append(data)
        ls.append(adapter.insert_posts(collection, data))

    for i in xrange(0,len(records)%500):
        l=[]
        data = {}
        for y in sObjectName.describe()['fields']:
            data[y['name']] = x[y['name']]
            #print data
        #print data
        l.append(data)
    ls.append(adapter.insert_posts(collection, data))

Answer 1

The general structure used by me is as follows:

worklist = [...]
batchsize = 500

for i in xrange(0, len(worklist), batchsize):
    batch = worklist[i:i+batchsize] # the result might be shorter than batchsize at the end
    # do stuff with batch

Note that we use the argument step xrangeto greatly simplify batch processing.

Answer 2

, @nneonneo , . , , itertools. :

def grouper(iterable, n, fillvalue=None):
    "Collect data into fixed-length chunks or blocks"
    # grouper('ABCDEFG', 3, 'x') --> ABC DEF Gxx
    args = [iter(iterable)] * n
    return itertools.izip_longest(fillvalue=fillvalue, *args)

, "" None , , . , :

def grouper2(iterable, n):
    iterable = iter(iterable)
    while True:
        tup = tuple(itertools.islice(iterable, 0, n))
        if tup:
            yield tup
        else:
            break

. , , , , , ...

"" , , - . , 2 , . , , !

---

, , , - "" islice, , - . - , StopIteration, , . , itertools.chain:

def grouper3(iterable, n):
    iterable = iter(iterable)
    while True:
        group = itertools.islice(iterable, n)
        item = next(group)  # raises StopIteration if the group doesn't yield anything
        yield itertools.chain((item,), group)

, , , . , , . list(grouper3(..., n)), "m" , 1 , n ( "m" - "" ). , . , itutools "" ( collections itertools):

def grouper4(iterable, n):
    iterable = iter(iterable)
    group = []
    while True:
        collections.deque(group, maxlen=0)  # consume all of the last group
        group = itertools.islice(iterable, n)
        item = next(group)  # raises StopIteration if the group doesn't yield anything
        group = itertools.chain((item,), group)
        yield group

, list(grouper4(..., n)) . , , next (, for ) .

Answer 3

Maybe something like this?

l = []
for ii, x in enumerate(records):
    data = {}
    for y in sObjectName.describe()['fields']
        data[y['name']] = x[y['name']]
    l.append(data)
    if not ii % 500:
        ls.append(adapter.insert_posts(collection, l))
        l = []