Pyinstaller.exe cannot find the _tiffile module. Downloading some compressed images will be very slow.

Question

When I run my code from Pyinstaller, the tiff reader works fine. After freezing with Pyinstaller, I get the following warning:

UserWarning: ImportError: No module named '_tifffile'. Loading of some compressed images will be very slow. Tifffile.c can be obtained at http://www.lfd.uci.edu/~gohlke 

And, of course, the tiff file, which took seconds to load into the numpy array, can now take minutes.

Here is a simplified form of my code to focus on the problem. If you download a tiff example, like this one , it should load quickly without any problems.

If you use C:\Python35\python.exe C:\Python35\Scripts\pyinstaller.exe --additional-hooks-dir=. --clean --win-private-assemblies tiffile_problems.py C:\Python35\python.exe C:\Python35\Scripts\pyinstaller.exe --additional-hooks-dir=. --clean --win-private-assemblies tiffile_problems.py , you should get a functional .exe with the above error message when it starts. When you try to load the same tiff, now it takes much longer.

tiffile_problems.py

 #!/usr/bin/env python3 import os import sys import traceback import numpy as np import matplotlib.pyplot as plt from PyQt4.QtGui import * from PyQt4.QtCore import * sys.path.append('..') from MBE_for_SO.util import fileloader, fileconverter class NotConvertedError(Exception): pass class FileAlreadyInProjectError(Exception): def __init__(self, filename): self.filename = filename class Widget(QWidget): def __init__(self, project, parent=None): super(Widget, self).__init__(parent) if not project: self.setup_ui() return def setup_ui(self): vbox = QVBoxLayout() ## Related to importing Raws self.setWindowTitle('Import Raw File') #vbox.addWidget(QLabel('Set the size all data are to be rescaled to')) grid = QGridLayout() vbox.addLayout(grid) vbox.addStretch() self.setLayout(vbox) self.resize(400, 220) self.listview = QListView() self.listview.setStyleSheet('QListView::item { height: 26px; }') self.listview.setSelectionMode(QAbstractItemView.NoSelection) vbox.addWidget(self.listview) hbox = QVBoxLayout() pb = QPushButton('New Video') pb.clicked.connect(self.new_video) hbox.addWidget(pb) vbox.addLayout(hbox) vbox.addStretch() self.setLayout(vbox) def convert_tif(self, filename): path = os.path.splitext(os.path.basename(filename))[0] + '.npy' #path = os.path.join(self.project.path, path) progress = QProgressDialog('Converting tif to npy...', 'Abort', 0, 100, self) progress.setAutoClose(True) progress.setMinimumDuration(0) progress.setValue(0) def callback(value): progress.setValue(int(value * 100)) QApplication.processEvents() try: fileconverter.tif2npy(filename, path, callback) print('Tifffile saved to wherever this script is') except: # qtutil.critical('Converting tiff to npy failed.') progress.close() return path def to_npy(self, filename): if filename.endswith('.raw'): print('No raws allowed') #filename = self.convert_raw(filename) elif filename.endswith('.tif'): filename = self.convert_tif(filename) else: raise fileloader.UnknownFileFormatError() return filename def import_file(self, filename): if not filename.endswith('.npy'): new_filename = self.to_npy(filename) if not new_filename: raise NotConvertedError() else: filename = new_filename return filename def import_files(self, filenames): for filename in filenames: try: filename = self.import_file(filename) except NotConvertedError: # qtutil.warning('Skipping file \'{}\' since not converted.'.format(filename)) print('Skipping file \'{}\' since not converted.'.format(filename)) except FileAlreadyInProjectError as e: # qtutil.warning('Skipping file \'{}\' since already in project.'.format(e.filename)) print('Skipping file \'{}\' since already in project.'.format(e.filename)) except: # qtutil.critical('Import of \'{}\' failed:\n'.format(filename) +\ # traceback.format_exc()) print('Import of \'{}\' failed:\n'.format(filename) + traceback.format_exc()) # else: # self.listview.model().appendRow(QStandardItem(filename)) def new_video(self): filenames = QFileDialog.getOpenFileNames( self, 'Load images', QSettings().value('last_load_data_path'), 'Video files (*.npy *.tif *.raw)') if not filenames: return QSettings().setValue('last_load_data_path', os.path.dirname(filenames[0])) self.import_files(filenames) class MyPlugin: def __init__(self, project): self.name = 'Import video files' self.widget = Widget(project) def run(self): pass if __name__ == '__main__': app = QApplication(sys.argv) app.aboutToQuit.connect(app.deleteLater) w = QMainWindow() w.setCentralWidget(Widget(None)) w.show() app.exec_() sys.exit() 

fileconverter.py

 #!/usr/bin/env python3 import os import numpy as np import tifffile as tiff class ConvertError(Exception): pass def tif2npy(filename_from, filename_to, progress_callback): with tiff.TiffFile(filename_from) as tif: w, h = tif[0].shape shape = len(tif), w, h np.save(filename_to, np.empty(shape, tif[0].dtype)) fp = np.load(filename_to, mmap_mode='r+') for i, page in enumerate(tif): progress_callback(i / float(shape[0]-1)) fp[i] = page.asarray() def raw2npy(filename_from, filename_to, dtype, width, height, num_channels, channel, progress_callback): fp = np.memmap(filename_from, dtype, 'r') frame_size = width * height * num_channels if len(fp) % frame_size: raise ConvertError() num_frames = len(fp) / frame_size fp = np.memmap(filename_from, dtype, 'r', shape=(num_frames, width, height, num_channels)) np.save(filename_to, np.empty((num_frames, width, height), dtype)) fp_to = np.load(filename_to, mmap_mode='r+') for i, frame in enumerate(fp): progress_callback(i / float(len(fp)-1)) fp_to[i] = frame[:,:,channel-1] 

fileloader.py

 #!/usr/bin/env python3 import numpy as np class UnknownFileFormatError(Exception): pass def load_npy(filename): frames = np.load(filename) # frames[np.isnan(frames)] = 0 return frames def load_file(filename): if filename.endswith('.npy'): frames = load_npy(filename) else: raise UnknownFileFormatError() return frames def load_reference_frame_npy(filename, offset): frames_mmap = np.load(filename, mmap_mode='c') if frames_mmap is None: return None frame = np.array(frames_mmap[offset]) frame[np.isnan(frame)] = 0 frame = frame.swapaxes(0, 1) if frame.ndim == 2: frame = frame[:, ::-1] elif frame.ndim == 3: frame = frame[:, ::-1, :] return frame def load_reference_frame(filename, offset=0): if filename.endswith('.npy'): frame = load_reference_frame_npy(filename, offset) else: raise UnknownFileFormatError() return frame 

Why? And how do I fix this? I found tifffile.py, tifffile.cpython-35.pyc, tifffile.c and put them all in the same directory as .exe. There is no effect. _tifffile.cp35-win_amd64.pyd is created by pyinstaller and placed in the same directory as .exe. I do not know what other options are available to me.

tifffile_problems.spec

 # -*- mode: python -*- block_cipher = None a = Analysis(['tiffile_problems.py'], pathex=['C:\\Users\\Cornelis\\PycharmProjects\\tester\\MBE_for_SO'], binaries=None, datas=None, hiddenimports=[], hookspath=[], runtime_hooks=[], excludes=[], win_no_prefer_redirects=False, win_private_assemblies=True, cipher=block_cipher) pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) exe = EXE(pyz, a.scripts, exclude_binaries=True, name='tiffile_problems', debug=False, strip=False, upx=True, console=True ) coll = COLLECT(exe, a.binaries, a.zipfiles, a.datas, strip=False, upx=True, name='tiffile_problems') 

tiffile.spec when using C:\Python35\python.exe C:\Python35\Scripts\pyinstaller.exe --additional-hooks-dir=. --clean --win-private-assemblies --onefile tiffile_problems.py C:\Python35\python.exe C:\Python35\Scripts\pyinstaller.exe --additional-hooks-dir=. --clean --win-private-assemblies --onefile tiffile_problems.py

 # -*- mode: python -*- block_cipher = None a = Analysis(['tiffile_problems.py'], pathex=['C:\\Users\\Cornelis\\PycharmProjects\\tester\\MBE_for_SO'], binaries=None, datas=None, hiddenimports=[], hookspath=['.'], runtime_hooks=[], excludes=[], win_no_prefer_redirects=False, win_private_assemblies=True, cipher=block_cipher) pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) exe = EXE(pyz, a.scripts, a.binaries, a.zipfiles, a.datas, name='tiffile_problems', debug=False, strip=False, upx=True, console=True ) 

Answer 1

I think the magic is correct about its weirdness with __package__ causing the problem here. I did not find the exact reason for the fix, but this seems to be fixed using the latest upgrade to pyinstaller . Check your version with:

→ pyinstaller --version 3.2.1

and update using

→ pip3 install --upgrade pyinstaller

The update was only made on January 15, 2017, so that would not help when you initially asked, but now it helps.

Answer 2

I really saw this with upwork while I was just browsing the net and decided to play.

kazemakase was on the right track at the very beginning, when you start normally, __package__ is None, and when its packed __package__ is set to tifffile and the first condition is fulfilled, and it becomes relative to the tifffile module.

 if __package__: from . import _tifffile else: import _tifffile 

I simply converted tifffile to a module manually, creating tifffile packages in sites, creating an empty __init__.py file in a new folder, placing tifffile.py, _tifffile.pyd from the site packages in the tifffile folder and changing the import statement, admittedly, in a simple skeleton .

 import tifffile.tifffile as tiff 

If this helps your entire project, I don’t know. And it should be noted that I used the wheel from http://www.lfd.uci.edu/~gohlke/pythonlibs/ for the initial installation in order to save the compilation step so that your mileage may differ. At first I did it on 2.7, but it seems to work fine on 3.5 of some of the tests I could do. And there was no need to put anything in .spec files when I tested it, originally created.

Answer 3

When checking the code, tifffile.py seems to be looking for a module named _tifffile , which is supposedly the expected name of the compiled C extension:

 try: if __package__: from . import _tifffile else: import _tifffile except ImportError: warnings.warn( "ImportError: No module named '_tifffile'. " "Loading of some compressed images will be very slow. " "Tifffile.c can be obtained at http://www.lfd.uci.edu/~gohlke/") 

tifffile.cpython-35.pyc is just the bytecode generated from tiffile.py . You do not need to worry about it.

The .c file alone will not help. It needs to be compiled to create a usable Python extension, which needs to be called something like _tiffile.cp35-win_amd64.pyd (may vary depending on your system and python version / installation), so you can use import _tifffile .

Compiling can be challenging if you have not already done so. If you think this is possible, the Python documentation can help you get started. You will need the same compiler and parameters with which your version of Python was compiled.

However, there may be a simpler solution. If your code works fine before freezing, you may have a compiled extension installed correctly on your system. Pyinstaller probably skips it because it can find tifffile.py and is satisfied. Find the correct .pyd file in your Python directories and see if you can modify the .spec file Pyinstaller created for your project, where you specify to include the .pyd file.