Handling DynamoDB BatchWrite Errors in Node.js

-- 2016-10-19 --

So, you’re building a Node.js app to do DynamoDB Batch Writes or better yet, you’re diving into AWS Lambda? You want to be a good programmer and handle errors gracefully, right? Well, you came to the right place.

First, how do errors look?
When all is well, you will get {}.
When any items are unprocessed by DynamoDB you will receive the following:

{
    "UnprocessedItems": {
        "Table_Name": [ // This will be your actual table name 😛
            {
                "PutRequest": {
                    "Item": {
                        "artist": {
                            "S": "The Beatles"
                        },
                        "Instruments": {
                            "SS": [
                                "Guitar", "Bass", "Drums"
                            ]
                        },
                        "song": {
                            "S": "Revolution 9"
                        },
                        "album": {
                            "S": "The White Album"
                        },
                        "track": {
                            "N": "2"
                        },
                        "classic": {
                            "BOOL": true
                        }
                    }
                }
            },
            {
                "PutRequest": {
                  "Item": {
                      "artist": {
                          "S": "The Grateful Dead"
                      },
                      "Instruments": {
                          "SS": [
                              "Guitar", "Bass", "Drums"
                          ]
                      },
                      "song": {
                          "S": "Touch of Grey"
                      },
                      "album": {
                          "S": "In the Dark"
                      },
                      "track": {
                          "N": "2"
                      },
                      "classic": {
                          "BOOL": true
                      }
                  }
                }
            },
        ]
    }
}

Now, this is really cool because the AWS SDK just perfectly returned all the code you need to do a new batch write. All you need to do now is write a callback and ideally put in some exponential backoff logic, like so:

let count = 1;
dynamoDB.batchWriteItem(params, processItemsCallback);
function processItemsCallback(err, data) {
  if (err) {
    console.log(JSON.stringify(err, null, 2));
  } else {
    console.log('Response Data: ', JSON.stringify(data));
    let itemsLost = data.UnprocessedItems;
    // Check if Object size is greater than 0 so we can process missed items if needed
    if (
      itemsLost.constructor === Object &&
      Object.keys(itemsLost).length === 0
    ) {
      console.log('Everything made it - hurray!');
    } else {
      console.log('Re-sending missed items');
      // Exponentially raise the backoff time
      count++;
      setTimeout(function() {
        let params = {};
        params.RequestItems = itemsLost;
        dynamoDB.batchWriteItem(params, processItemsCallback);
      }, 1000 * count);
    }
  }
}

-- Cameron Eckelberry