Uploaded about 3K objects (files) on S3. There is an event trigger for each file that is loaded into this S3 bucket.
Lambda receives an event trigger for only about 300 objects. If I try again (return from S3 and return it back to S3), it will generate an event for another 400 objects, the rest of the events have not even reached the lambda.
What am I missing here, and how can I scale any number of created objects?
var async = require('async');
var aws = require('aws-sdk');
var s3 = new aws.S3();
var kinesis = new aws.Kinesis();
var sns = new aws.SNS();
var config = require('./config.js');
var logError = function(errormsg) {
sns.publish({
TopicArn: config.TopicArn,
Message: errormsg
}, function(err, data) {
if (err) {
console.log(errormsg);
}
});
};
exports.handler = function(event, context, callback) {
var readS3andSendtoKinesis = function(record, index, cb) {
var params = {
Bucket: record.s3.bucket.name,
Key: record.s3.object.key
};
console.log('Received File: ' + record.s3.object.key);
s3.getObject(params, function(err, data) {
if (!err) {
var kinesisParams = {
Data: data.Body.toString('utf8'),
PartitionKey: config.PartitionKey,
StreamName: config.StreamName
};
kinesis.putRecord(kinesisParams, function(err, data) {
if (err) {
logError(JSON.stringify(err, null, 2));
}
cb(null, 'done');
});
} else {
logError(JSON.stringify(err, null, 2));
cb(null, 'done');
}
});
};
async.eachOfLimit(event.Records, 1, readS3andSendtoKinesis, function(err) {
callback(null, 'Done');
});
};
Since everyone recommends looking at cloudwatch, general cloud information for related lambda,
