How to download, archive and save multiple files using Javascript and make progress?

Question

I am creating a Chrome extension that needs to download multiple files (images and / or videos) from a website. These files can be huge, so I want to show the loading process to the user. After some research, I found that a possible solution at present might be:

• Download all files using XMLHttpRequests.
• When downloading, write all the files to a single archive using the JavaScript library (for example, JSZip.js, zip.js).
• Ask the user to save the zip in the SaveAs dialog box.

I am stuck in the aisle 2), how can I fix the downloaded files?

To understand, here is a sample code:

var fileURLs = ['http://www.test.com/img.jpg',...]; var zip = new JSZip(); var count = 0; for (var i = 0; i < fileURLs.length; i++){ var xhr = new XMLHttpRequest(); xhr.onprogress = calculateAndUpdateProgress; xhr.open('GET', fileURLs[i], true); xhr.responseType = "blob"; xhr.onreadystatechange = function () { if (xhr.readyState == 4) { var blob_url = URL.createObjectURL(response); // add downloaded file to zip: var fileName = fileURLs[count].substring(fileURLs[count].lastIndexOf('/')+1); zip.file(fileName, blob_url); // <- here one problem count++; if (count == fileURLs.length){ // all download are completed, create the zip var content = zip.generate(); // then trigger the download link: var zipName = 'download.zip'; var a = document.createElement('a'); a.href = "data:application/zip;base64," + content; a.download = zipName; a.click(); } } }; xhr.send(); } function calculateAndUpdateProgress(evt) { if (evt.lengthComputable) { // get download progress by performing some average // calculations with evt.loaded, evt.total and the number // of file to download / already downloaded ... // then update the GUI elements (eg. page-action icon and popup if showed) ... } } 

The top code creates a downloadable archive containing small corrupted files. There is also a problem with filename sync: the blob object does not contain the file name, so if, for example. fileURLs[0] takes longer to load than fileURLs[1] names become incorrect (inverted) ..

NOTE. I know that Chrome has a download API, but it is in the dev channel, so unfortunately this is not a solution now, and I would like to avoid using NPAPI for such a simple task.

Answer 1

I was reminded of this question .. since there is no answer yet, I am writing a possible solution if it can be useful to someone else:

• as said, the first problem is related to the transfer of blob url to jszip (it does not support blobs, but also does not cause any error to notify about it, and it successfully generates archive of damaged files): to fix this, simply go through the base64 data line instead of the object url blob;
• The second problem is the synchronization of file names: the easiest way is to download one file at a time, rather than using parallels xhr queries.

So, the modified top code could be:

 var fileURLs = ['http://www.test.com/img.jpg',...]; var zip = new JSZip(); var count = 0; downloadFile(fileURLs[count], onDownloadComplete); function downloadFile(url, onSuccess) { var xhr = new XMLHttpRequest(); xhr.onprogress = calculateAndUpdateProgress; xhr.open('GET', url, true); xhr.responseType = "blob"; xhr.onreadystatechange = function () { if (xhr.readyState == 4) { if (onSuccess) onSuccess(xhr.response); } function onDownloadComplete(blobData){ if (count < fileURLs.length) { blobToBase64(blobData, function(binaryData){ // add downloaded file to zip: var fileName = fileURLs[count].substring(fileURLs[count].lastIndexOf('/')+1); zip.file(fileName, binaryData, {base64: true}); if (count < fileURLs.length -1){ count++; downloadFile(fileURLs[count], onDownloadCompleted); } else { // all files have been downloaded, create the zip var content = zip.generate(); // then trigger the download link: var zipName = 'download.zip'; var a = document.createElement('a'); a.href = "data:application/zip;base64," + content; a.download = zipName; a.click(); } }); } } function blobToBase64(blob, callback) { var reader = new FileReader(); reader.onload = function() { var dataUrl = reader.result; var base64 = dataUrl.split(',')[1]; callback(base64); }; reader.readAsDataURL(blob); } function calculateAndUpdateProgress(evt) { if (evt.lengthComputable) { ... } } 

Last note, this solution works very well if you upload several and small files (less than 1 MB in size as the size for less than 10 files), in other cases JSZip will cause the browser tab to fail when the archive is generated, so it is better to use a dedicated stream for compression (WebWorker, e.g. zip.js).

If after that the archive was generated, the browser still continues to crash with large files and without reporting any errors, try launching the saveAs window without sending binary data, but passing the blob link ( a.href = URL.createObjectURL(zippedBlobData); where zippedBlobData is a blob object that refers to the generated archive data);

Answer 2

Based on the @guari code, I tested it locally and applied it to the response application, attaching the code to help others.

 import JSZip from "jszip"; import saveAs from "jszip/vendor/FileSaver.js"; // ....... // download button click event btnDownloadAudio = record =>{ let fileURLs = ['https://www.test.com/52f6c50.AMR', 'https://www.test.com/061940.AMR']; let count = 0; let zip = new JSZip(); const query = { record, fileURLs, count, zip }; this.downloadFile(query, this.onDownloadComplete); } downloadFile = (query, onSuccess) => { const { fileURLs, count, } = query; var xhr = new XMLHttpRequest(); xhr.onprogress = this.calculateAndUpdateProgress; xhr.open('GET', fileURLs[count], true); xhr.responseType = "blob"; xhr.onreadystatechange = function (e) { if (xhr.readyState == 4) { if (onSuccess) onSuccess(query, xhr.response); } } xhr.send(); } onDownloadComplete = (query, blobData) => { let { record, fileURLs, count, zip } = query; if (count < fileURLs.length) { const _this = this; const { audio_list, customer_user_id, } = record; this.blobToBase64(blobData, function(binaryData){ // add downloaded file to zip: var sourceFileName = fileURLs[count].substring(fileURLs[count].lastIndexOf('/')+1); // convert the source file name to the file name to display var displayFileName = audio_list[count].seq + sourceFileName.substring(sourceFileName.lastIndexOf('.')); zip.file(displayFileName, binaryData, {base64: true}); if (count < fileURLs.length -1){ count++; _this.downloadFile({ ...query, count }, _this.onDownloadComplete); } else { // all files have been downloaded, create the zip zip.generateAsync({type:"blob"}).then(function(content) { // see FileSaver.js saveAs(content, '${customer_user_id}.zip'); }); } }); } } blobToBase64 = (blob, callback) => { var reader = new FileReader(); reader.onload = function() { var dataUrl = reader.result; var base64 = dataUrl.split(',')[1]; callback(base64); }; reader.readAsDataURL(blob); } calculateAndUpdateProgress = (evt) => { if (evt.lengthComputable) { // console.log(evt); } }