New to python, web development and therefore bulbs. I have a small application that accepts excel attachments, extracts data and writes it to oracle. My stack includes flask-sqlalchemy, cx_oracle. When I upload a small excel file about 10K lines in size, it works fine. When I drop really large files on it up to a million lines, it twitches, and I get:
sqlalchemy.exc.operationalerror: (cx_oracle.operationalerror) ora-03114: not connected to oracle
This is my view function:
@theapp.route('/upload', methods=('GET', 'POST'))
def upload():
start_time = time.time()
form = UploadForm()
if form.validate_on_submit():
file_name = secure_filename(form.upload.data.filename)
form.upload.data.save(os.path.join(theapp.config['UPLOAD_FOLDER'], file_name))
input_file = os.path.join(theapp.config['UPLOAD_FOLDER'], file_name)
wb = load_workbook(input_file, data_only = True, read_only=True, use_iterators=True)
sheet = wb.worksheets[0]
allrows = sheet.iter_rows()
headerobject = next(allrows)
headerlist = []
for c in headerobject:
if c.value is not None:
if isinstance(c.value, str) is False:
return bad_request("Column names have to be strings without spaces. Value in cell {} is not a string".format(c.coordinate))
if not c.value.strip():
return bad_request("Cell {} has line space(s). Delete the contents of the cell".format(c.coordinate))
headerlist.append(c.value.strip())
duplicate_elements = [k for k,v in Counter(headerlist).items() if v >1]
if duplicate_elements:
return bad_request("Column(s) {} are duplicates".format(duplicate_elements))
tableinfo = form.targettableinfo.data
tablecolumns = []
for col in tableinfo.fields:
tablecolumns.append(col.name)
if not set(headerlist) == set(tablecolumns):
foreign_columns = [x for x in headerlist if x not in tablecolumns]
if foreign_columns:
return bad_request("Column(s) {} in your excel file are not defined".format(foreign_columns))
unfulfilled_columns = [x for x in tablecolumns if x not in headerlist]
if unfulfilled_columns:
return bad_request("Column(s) {} are expected by the table but not found in your excel file.".format(unfulfilled_columns))
sortedtablecolumnsobject = sorted(tableinfo.fields, key=lambda x: headerlist.index(x.name))
output_data = []
for row in allrows:
row_dict = {}
for fobject,cobject in zip(sortedtablecolumnsobject,row):
if fobject.type.name == "Text":
if cobject.value == None:
row_dict[fobject.name] = cobject.value
else:
row_dict[fobject.name] = removeNonAscii(str(cobject.value))
elif fobject.type.name == "Number":
if dao.isnumber(cobject.value):
row_dict[fobject.name] = float(cobject.value)
elif cobject.value is None:
row_dict[fobject.name] = cobject.value
else:
return bad_request("Cell {} has value {}. A numeric value is expected".format(cobject.coordinate, cobject.value))
elif fobject.type.name == "Date":
if type(cobject.value) == datetime.datetime:
row_dict[fobject.name] = cobject.value
elif cobject.value is None:
row_dict[fobject.name] = cobject.value
else:
return bad_request("Cell {} does not contain a valid date".format(cobject.coordinate))
output_data.append(row_dict)
m = db.MetaData()
t = db.Table(tableinfo.table_name,m,autoload = True, autoload_with = db.engine)
db.engine.execute(t.insert(),output_data)
print("{} seconds" .format(time.time() - start_time))
return redirect(url_for('summary'))
return render_template('upload.html', title = 'Upload', form = form)
, . , . , , , . , , .
, ? SQLALCHEMY_POOL_RECYCLE SQLALCHEMY_POOL_TIMEOUT flask-sqlalchemy, . , / ?