I need help in properly structuring the code to process some text files using S3 buckets and the Lambda function.
I want to use the Lambda function called by creating a new object in an S3 bucket to read a file and extract some data and write it to a file that fits in another S3 bucket.
So far I have a function that works great by copying a file from one S3 bucket to another, but I cannot figure out how to add a function to process the file and write the result to the final destination of S3.
Files are simple text files, and I need to extract data from each line in the file.
Below, if the Node.js code that I am currently using with an extra function added to process the file - see comments with? where I am looking for help.
var async = require('async');
var AWS = require('aws-sdk');
var util = require('util');
var s3 = new AWS.S3();
exports.handler = function(event, context) {
console.log("Reading options from event:\n", util.inspect(event, {depth: 5}));
var srcBucket = event.Records[0].s3.bucket.name;
var srcKey =
decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, " "));
var dstBucket = "inputBucket";
var dstKey = srcKey + ".txt";
if (srcBucket == dstBucket) {
console.error("Destination bucket must not match source bucket.");
return;
}
var typeMatch = srcKey.match(/\.([^.]*)$/);
if (!typeMatch) {
console.error('unable to infer file type for key ' + srcKey);
return;
}
var imageType = typeMatch[1];
if (imageType != "txt") {
console.log('skipping non-image ' + srcKey);
return;
}
async.waterfall([
function download(next) {
s3.getObject({
Bucket: srcBucket,
Key: srcKey
},
next);
},
function transform(response, next) {
var rl = require('readline').createInterface({
input: require('fs').createReadStream('file.in')
});
rl.on('line', function (line) {
console.log('Line from file:', line);
});
next;
}
function upload(response, next) {
s3.putObject({
Bucket: dstBucket,
Key: dstKey,
Body: response.Body,
ContentType: response.contentType
},
next);
}
], function (err) {
if (err) {
console.error(
'Unable to process ' + srcBucket + '/' + srcKey +
' and upload to ' + dstBucket + '/' + dstKey +
' due to an error: ' + err
);
} else {
console.log(
'Successfully processed ' + srcBucket + '/' + srcKey +
' and uploaded to ' + dstBucket + '/' + dstKey
);
}
context.done();
}
);
};
source
share