Recursively fetch all elements in a DynamoDB query using Node JS

Question

This is probably a JS / Async question rather than a specific DynamoDB question -

I want to get all the items in a table with a hash key in Amazon DynamoDB. The table also has a Range key.

I use the NodeJS library, which is a wrapper around the AWS DynamoDB REST API. - Node -DynamoDB

DynamoDB returns only 1 MB of results with each query. To get a reminder of the results, it includes lastEvaluatedKey . We can include this in another query to get another 1 MB of results and so on ...

It's hard for me to write a recursive async function that should hit the service sequentially until I can return all the results. (the table will never have more than 10 MB for my use case, no chance of a cursory request)

Some pseudo codes to illustrate:

ddb.query('products', primarykey, {}, function(err,result){ //check err if(result && result.lastEvaluatedKey){ //run the query again var tempSet = result.items; //temporarily store result.items so we can continue and fetch remaining items. } else{ var finalSet = result.items; //figure out how to merge with items that were fetched before. } }); 

Answer 1

 var getAll = function(primarykey, cb) { var finalSet = [], nextBatch = function(lek) { ddb.query('products', primarykey, { exclusiveStartKey: lek }, function(err, result) { if (err) return cb(err); if (result.items.length) finalSet.push.apply(finalSet, result.items); if (result.lastEvaluatedKey) nextBatch(result.lastEvaluatedKey); else cb(null, finalSet); }); }; nextBatch(); }; getAll(primarykey, function(err, all) { console.log(err, all); }); 

Answer 2

After a few cups of coffee, I wrote this recursive function. Hope this helps others. If you see a mistake, edit it or leave a comment.

  var DynamoDbItemFetcher = function(table,hash,maxItems,callback){ var self = this; self.table = table; self.startKey = null; self.hash = hash; self.maxItems = maxItems; self.items = []; self.callback = callback; self.getItems = function(){ var params = {}; if(self.startKey){ params.exclusiveStartKey = self.startKey; } ddb.query(self.table,self.hash,params,function(err1,result){ if(err1) return self.callback(err1, null); if(result){ self.items = self.items.concat(result.items); if(result.lastEvaluatedKey && result.lastEvaluatedKey.hash){ if(self.maxItems && self.items.length > self.maxItems){ self.callback(null,self.items); }else { self.startKey = result.lastEvaluatedKey;//reset start key self.getItems(callback);//recursive call... } }else{ //no more items..return whatever is in store. self.callback(null,self.items); } } else{ self.callback(null, null); } }); }; }; 

Answer 3

Here is an option using promises. I needed to get a list of table names, and not scan items from the table, but similar concepts apply.

 function getTableNames(key, prevTableNames) { return new Promise(function(resolve, reject) { let request = dynamodb.listTables({ ExclusiveStartTableName: key }, function(err, response) { if (err) { reject(err); } else { let tableNames = (prevTableNames || []).concat(response.TableNames); if (response.LastEvaluatedTableName) { getTableNames(response.LastEvaluatedTableName, tableNames) .then(resolve) .catch(reject); } else { resolve(tableNames) } } }); }); }