Is linux disk cache buffer making python cPickle more efficient than shelf?

Question

Is IO more efficient due to the Linux buffer buffer cache when storing commonly used python objects as separate cPickle files instead of storing all objects in one large shelf?

Is the disk cache buffer described differently in these two scenarios regarding performance?

There may be thousands of large files (usually around 100 MB, but sometimes 1 GB), but large RAM (for example, 64 GB).

Answer 1

, , , , . .

pickle/shelve , . , - , ( - ), , , , - .

test.py:

import cPickle
import shelve
import os

class PickleManager(object):
    def store(self,name,value):
        with open(name,'w') as f:
            cPickle.dump(value,f)
    def load(self,name):
        with open(name,'r') as f:
            return cPickle.load(f)

class ShelveManager(object):
    def __enter__(self):
        if os.path.exists(self.fname):
            self.shelf=shelve.open(self.fname)
        else:
            self.shelf=shelve.open(self.fname,'n')
        return self
    def __exit__(self,ext_type,exc_value,traceback):
        self.shelf.close()
    def __init__(self,fname):
        self.fname=fname
    def store(self,name,value):
        self.shelf[name]=value        
    def load(self,name):
        return self.shelf[name]

def write(manager):                
    for i in range(100):
        fname='/tmp/{i}.dat'.format(i=i)
        data='The sky is so blue'*100
        manager.store(fname,data)
def read(manager):        
    for i in range(100):
        fname='/tmp/{i}.dat'.format(i=i)        
        manager.load(fname)

PickleManager :

manager=PickleManager()
manager.load(...)
manager.store(...)

ShelveManager :

with ShelveManager('/tmp/shelve.dat') as manager:        
    manager.load(...)
    manager.store(...)

- :

python -mtimeit -s'import test' 'with test.ShelveManager("/tmp/shelve.dat") as s: test.read(s)'
python -mtimeit -s'import test' 'test.read(test.PickleManager())'
python -mtimeit -s'import test' 'with test.ShelveManager("/tmp/shelve.dat") as s: test.write(s)'
python -mtimeit -s'import test' 'test.write(test.PickleManager())'

, :

                  read (ms)     write (ms)
PickleManager     9.26          7.92 
ShelveManager     5.32          30.9 

, , ShelveManager , PickleManager .

. - Python, , , ..

, write read . , .