All geek questions in one place

Google bucket upload does not work in production

My platform has a problem downloading to Google Cloud Storage, which I only experience in a production environment, and this is a real head cleaner. My platform uses NodeJS, and this file download is done on the server in the API service.

Language: Javascript, nodejs version 6.5

Used packages: @google-cloud/storage version 0.2.0,gm version 1.23.0

The service essentially takes an image in base 64, creates an image buffer, and then transfers that image to the Google Cloud Cloud. Easy peasy.

Unfortunately, he is not working on production, and I cannot understand why. Exactly the same code works in production, it just points to another bucket.

Some relevant codes at the top of the file

var gm = require('gm').subClass({ imageMagick: true });
var Promise = require('bluebird');
var Q = require('q');
var request = require('request').defaults({ encoding: null });
var gcs = require('@google-cloud/storage')({
      keyFilename: sails.config.gcloud.keyFileName,
      projectId: sails.config.gcloud.projectId
});
var bucket = gcs.bucket(sails.config.gcloud.buckets.cdn);

// Takes raw base64Image from client and converts it into:
// {
//      meta: data:image/jpeg;base64,
//      data: hugeStringOfImageDataInBase64,
//      dateType: image/jpeg
// }
function decodeImage(base64Image) {
    // Not using regex due to perf.
    var startImageData = base64Image.indexOf(',');
    var regex = /^data:(.+);base64$/;
    var meta = base64Image.slice(0, startImageData);
    var data = base64Image.slice(startImageData + 1);
    var dataType = meta.match(regex)[1];

    return { type: dataType, meta: meta, data: data };
}

Relevant code in which the buffer is created

createNewPictureAndUploadToCDNBase: function(imageInBase64, imageBuffer, cb) {

    return Picture.create()
        .then(function(emptyPicture) {
            var buffer;

            if (imageInBase64) {
                buffer = PictureService.bufferFromBase64(imageInBase64);
            } else {
                buffer = imageBuffer;
            }

            return PictureService.resizeAndUploadPictureToCDN(emptyPicture.id, buffer, function (err, results) {

                emptyPicture = _.merge(emptyPicture, results);

                return emptyPicture.save(function(err) {
                    cb(err, emptyPicture);
                });
            });

        }).catch(function(err) {
            return cb(err);
        });
},

bufferFromBase64: function (imageInBase64) {
    return new Buffer(decodeImage(imageInBase64).data, 'base64');
},

, Google Cloud Storage

resizeAndUploadPictureToCDN: function (pictureId, imageBuffer, cb) {

    var resizingOptions = {
        original: {
            quality: 80,
            size: {}
        },
        thumbnail: {
            quality: 75,
            size: {
                width: 60,
                height: 60
            }
        },
        medium: {
            quality: 65,
            size: {height: 250}
        }
    };

    var resizingPromises = _.mapValues(resizingOptions, function (options, resizingName) {

        return new Promise(function (resolve, reject) {
            var file = bucket.file(pictureId + '-' + resizingName);

            // NOTE: 'gm' is a call to GraphicMagick for node
            // More can be found here: https://github.com/aheckmann/gm

            gm(imageBuffer)
                .quality(options.quality)
                .resize(options.size.width || null, options.size.height || null)
                .stream()
                .pipe(file.createWriteStream())
                .on('finish', function(err, success) {

                    file.getMetadata(function(err, metadata) {
                        if (err) {
                            console.log("Error getting metadata from google cloud", err);
                            return reject(err);
                        }

                        resolve(metadata.mediaLink);
                    });

                }).on('error', function(err) {
                    console.log("Got an error uploading to Cloud Storage:", err);
                    reject(err);
                });
        });
    });

    Promise.props(resizingPromises)
        .then(function (results) {
            cb(null, results);
        }).error(cb);
},

, , . . , , , Google.

. , , . , , .

. , , , . , , .

console.log file.getMetaData()

Corresponding metadata:  
{
    kind: 'storage#object',
    id: 'cdn.texasca.com/5a3d938b84d560010012b0b3-thumbnail/1513984907989412',
    selfLink: 'https://www.googleapis.com/storage/v1/b/cdn.texasca.com/o/5a3d938b84d560010012b0b3-thumbnail',
    name: '5a3d938b84d560010012b0b3-thumbnail',
    bucket: 'cdn.texasca.com',
    generation: '1513984907989412',
    metageneration: '1',
    timeCreated: '2017-12-22T23:21:47.915Z',
    updated: '2017-12-22T23:21:47.915Z',
    storageClass: 'STANDARD',
    timeStorageClassUpdated: '2017-12-22T23:21:47.915Z',
    size: '0',
    md5Hash: '1B2M2Y8AsgTpgAmY7PhCfg==',
    mediaLink: 'https://www.googleapis.com/download/storage/v1/b/cdn.texasca.com/o/5a3d938b84d560010012b0b3-thumbnail?generation=1513984907989412&alt=media',
    crc32c: 'AAAAAA==',
    etag: 'CKSji6XhntgCEAE=' 
}

, size '0', 0. , google , , "13908" , mediaLink, , .

...

? ? - , ?

.

!

,

, , . - . , . , ( -, , 0).

, . dev0.cdn.texasca.com, - cdn.texasca.com. cdn.texasca.com dev ... BOOM! . , , .

"", . .

, . , , . !

+4
2

- GraphicMagick . , , GraphicMagick.

GraphicMagick.

        return new Promise(function (resolve, reject) {
            var file = bucket.file(pictureId + '-' + resizingName);

            var stream = file.createWriteStream();

            stream.on('error', function (err) {
                if (err) {
                    console.log("Got an error uploading to GBucket: ", err);
                    return reject(err);
                }
            });

            stream.on('finish', function() {

                console.log("Uploaded successfully!!");

                file.getMetadata(function(err, metadata) {
                    if (err) {
                        console.log("Error getting metadata from google cloud", err);
                        return reject(err);
                    }

                    console.log("File metadata: ", metadata);

                    return resolve(metadata.mediaLink);
                });     
            });

            stream.end(imageBuffer);
        });
0

Source: https://habr.com/ru/post/1691260/

More articles:


All Articles