date validation, utf8 strings handled

README.md

@@ -1,9 +1,15 @@
 # gatsby-source-google-sheets
 
-## a WIP
-I wrote this for a 1-off solution, so it's not likely to be ready for every usecase.
+Why go through the hassle of setting up a complicated headless CMS when Google Sheets already has user permissions, revision history, and a powerful UI? 
 
-How to:
+This source plugin for [Gatsby JS](https://github.com/gatsbyjs/gatsby) will turn any Google Sheets worksheet into a GraphQL type for build-time consumption. 
+
+## heads-up! 
+Until [this issue in Gatsby](https://github.com/gatsbyjs/gatsby/issues/2727) is addressed, when this plugin fails, it will fail *silently*. Your build will still complete, but the data you were expecting to be available won't be there. 
+
+I'll add `retry` to this plugin eventually, but this is a limitation of Gatsby's build that likely affects all source plugins.
+
+# How to:
 
 ## Step 1: set up sheets/permissions
 
@@ -38,7 +44,15 @@ yarn add gatsby-source-google-sheets
 ```
 
 The plugin makes the following conversions before feeding Gatsby nodes:
-1. Numbers (as determined by isNan()) are converted to numbers
+1. Numbers are converted to numbers. Sheets formats numbers as comma-delineated strings, so to determine if something is a number, the plugin tests to see if the string (a) is non-empty and (b) is composed only of commas, decimals, and digits:
+```
+if (
+    "value".replace(/[,\.\d]/g, "").length === 0 
+      && "value" !== ""
+   ) { 
+    ...assume value is a number and handle accordingly
+}
+```
 2. "TRUE"/"FALSE" converted to boolean true/false
 3. empty cells ("" in sheets payload) converted to null
 4. Column names are converted to camelcase via lodash _.camelCase() (see note 2 in 'A few notes')

__test__/fetch-sheet.test.js

@@ -0,0 +1,48 @@
+"use strict";
+
+var _extends2 = require("babel-runtime/helpers/extends");
+
+var _extends3 = _interopRequireDefault(_extends2);
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+var _require = require("../fetch-sheet.js"),
+    cleanRows = _require.cleanRows;
+
+var _ = require("lodash");
+
+describe("cleaning rows from GSheets response", function () {
+  it("removes keys that don't correspond to column names", function () {
+    var badKeys = ["_xml", "app:edited", "save", "del", "_links"];
+    var row = badKeys.reduce(function (prev, curr) {
+      return (0, _extends3.default)({}, prev, {
+        [curr]: "true"
+      });
+    }, { validKey: "true" });
+    var cleaned = cleanRows([row])[0];
+    expect(Object.keys(cleaned)).toHaveLength(1);
+
+    expect(Object.keys(cleaned)[0]).toBe("validKey");
+    expect(cleaned["validKey"]).toBe("true");
+  });
+
+  it('converts "TRUE" and "FALSE" into actual booleans', function () {
+    var boolRow = { truthy: "TRUE", falsy: "FALSE" };
+    var cleaned = cleanRows([boolRow])[0];
+    expect(Object.keys(cleaned)).toEqual(["truthy", "falsy"]);
+    expect(cleaned["truthy"]).toBe(true);
+    expect(cleaned["falsy"]).toBe(false);
+  });
+
+  it("respects emoji", function () {
+    var TEST_EMOJI_STRING = "🔑🔑🔑🔑🔑🔑🔑🔑🔑🔑🔑🔑🔑";
+    var emojiRow = { emoji: TEST_EMOJI_STRING };
+    var cleaned = cleanRows([emojiRow])[0];
+    expect(cleaned.emoji).toEqual(TEST_EMOJI_STRING);
+  });
+  it("returns comma-delineated number strings as numbers", function () {
+    var numRow = { short: "1", long: "123,456,789", decimal: "0.5912", mixed: "123,456.789" };
+    var cleaned = cleanRows([numRow])[0];
+    expect(Object.values(cleaned)).toEqual([1, 123456789, 0.5912, 123456.789]);
+  });
+});

fetch-sheet.js

@@ -1,63 +1,116 @@
-'use strict';
+"use strict";
+
+exports.__esModule = true;
+exports.cleanRows = undefined;
+
+var _regenerator = require("babel-runtime/regenerator");
+
+var _regenerator2 = _interopRequireDefault(_regenerator);
+
+var _asyncToGenerator2 = require("babel-runtime/helpers/asyncToGenerator");
 
-var _asyncToGenerator2 = require('babel-runtime/helpers/asyncToGenerator');
 
 var _asyncToGenerator3 = _interopRequireDefault(_asyncToGenerator2);
 
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
-const GoogleSpreadsheet = require('google-spreadsheet');
-const _ = require('lodash');
+var GoogleSpreadsheet = require("google-spreadsheet");
+var _ = require("lodash");
+var isValid = require("date-fns/is_valid");
 
-const getSpreadsheet = (spreadsheetId, credentials) => new Promise((resolve, reject) => {
-    const doc = new GoogleSpreadsheet(spreadsheetId);
+var getSpreadsheet = function getSpreadsheet(spreadsheetId, credentials) {
+  return new Promise(function (resolve, reject) {
+    var doc = new GoogleSpreadsheet(spreadsheetId);
     doc.useServiceAccountAuth(credentials, function (err) {
-        if (err) reject(err);else resolve(doc);
+      if (err) reject(err);else resolve(doc);
     });
-});
+  });
+};
 
-const getWorksheetByTitle = (spreadsheet, worksheetTitle) => new Promise((resolve, reject) => spreadsheet.getInfo((e, s) => {
-    if (e) reject(e);
-    const targetSheet = s.worksheets.find(sheet => sheet.title === worksheetTitle);
-    if (!targetSheet) {
+var getWorksheetByTitle = function getWorksheetByTitle(spreadsheet, worksheetTitle) {
+  return new Promise(function (resolve, reject) {
+    return spreadsheet.getInfo(function (e, s) {
+      if (e) reject(e);
+      var targetSheet = s.worksheets.find(function (sheet) {
+        return sheet.title === worksheetTitle;
+      });
+      if (!targetSheet) {
         reject(`Found no worksheet with the title ${worksheetTitle}`);
-    }
-    resolve(targetSheet);
-}));
-
-const getRows = (worksheet, options = {}) => new Promise((resolve, reject) => worksheet.getRows(options, (err, rows) => {
-    if (err) reject(err);else {
-        resolve(rows.map(r =>
-        // google sheets mangles column names, so we use a dash-lowercase convention
-        // , and now we make that JS friendly w/ camelcase
-        _.mapKeys(
-        // system values we don't need
-        _.omit(r, ['_xml', 'app:edited', 'save', 'del', '_links']), (val, key) => _.camelCase(key))));
-    }
-}));
-
-const cleanRows = rows => rows.map(r => _.mapValues(r, val => {
-    if (val === '') return null;
-    // sheets apparently leaves commas in some #s depending on formatting
-    if (!isNaN(val.replace(/[\W]/g, '')) && val !== '') {
-        return Number(val.replace(/[\W]/g, ''));
-    }
-    if (val === 'TRUE') return true;
-    if (val === 'FALSE') return false;
-    return val;
-}));
-
-const fetchData = (() => {
-    var _ref = (0, _asyncToGenerator3.default)(function* (spreadsheetId, worksheetTitle, credentials) {
-        const spreadsheet = yield getSpreadsheet(spreadsheetId, credentials);
-        const worksheet = yield getWorksheetByTitle(spreadsheet, worksheetTitle);
-        const rows = yield getRows(worksheet);
-        return cleanRows(rows);
+      }
+      resolve(targetSheet);
+    });
+  });
+};
+
+var getRows = function getRows(worksheet) {
+  var options = arguments.length > 1 && arguments[1] !== undefined ? arguments[1] : {};
+  return new Promise(function (resolve, reject) {
+    return worksheet.getRows(options, function (err, rows) {
+      if (err) reject(err);else {
+        resolve(rows);
+      }
     });
+  });
+};
+
+var cleanRows = function cleanRows(rows) {
+  return rows.map(function (r) {
+    return _.chain(r).omit(["_xml", "app:edited", "save", "del", "_links"]).mapKeys(function (v, k) {
+      return _.camelCase(k);
+    }).mapValues(function (val, key) {
+      if (key === "date") {
+        if (val !== "" && isValid(new Date(val))) {
+          return unescape(encodeURIComponent(new Date(val)));
+        }
+      }
+      if (val === "") return null;
+      // sheets apparently leaves commas in some #s depending on formatting
+      if (val.replace(/[,\.\d]/g, "").length === 0 && val !== "") {
+        return Number(val.replace(/,/g, ""));
+      }
+      if (val === "TRUE") return true;
+      if (val === "FALSE") return false;
+      return unescape(encodeURIComponent(val));
+    }).value();
+  });
+};
+
+var fetchData = function () {
+  var _ref = (0, _asyncToGenerator3.default)( /*#__PURE__*/_regenerator2.default.mark(function _callee(spreadsheetId, worksheetTitle, credentials) {
+    var spreadsheet, worksheet, rows;
+    return _regenerator2.default.wrap(function _callee$(_context) {
+      while (1) {
+        switch (_context.prev = _context.next) {
+          case 0:
+            _context.next = 2;
+            return getSpreadsheet(spreadsheetId, credentials);
+
+          case 2:
+            spreadsheet = _context.sent;
+            _context.next = 5;
+            return getWorksheetByTitle(spreadsheet, worksheetTitle);
+
+          case 5:
+            worksheet = _context.sent;
+            _context.next = 8;
+            return getRows(worksheet);
+
+          case 8:
+            rows = _context.sent;
+            return _context.abrupt("return", cleanRows(rows));
+
+          case 10:
+          case "end":
+            return _context.stop();
+        }
+      }
+    }, _callee, undefined);
+  }));
 
-    return function fetchData(_x, _x2, _x3) {
-        return _ref.apply(this, arguments);
-    };
-})();
+  return function fetchData(_x2, _x3, _x4) {
+    return _ref.apply(this, arguments);
+  };
+}();
 
-module.exports = fetchData;
+exports.cleanRows = cleanRows;
+exports.default = fetchData;

gatsby-node.js

@@ -1,44 +1,64 @@
-'use strict';
+"use strict";
 
-var _asyncToGenerator2 = require('babel-runtime/helpers/asyncToGenerator');
+var _regenerator = require("babel-runtime/regenerator");
+
+var _regenerator2 = _interopRequireDefault(_regenerator);
+
+var _asyncToGenerator2 = require("babel-runtime/helpers/asyncToGenerator");
 
 var _asyncToGenerator3 = _interopRequireDefault(_asyncToGenerator2);
 
 function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
 
-const fetchSheet = require(`./fetch-sheet.js`);
-const uuidv5 = require('uuid/v5');
-const _ = require('lodash');
-const crypto = require('crypto');
-const seedConstant = '2972963f-2fcf-4567-9237-c09a2b436541';
-
-exports.sourceNodes = (() => {
-  var _ref = (0, _asyncToGenerator3.default)(function* ({ boundActionCreators, getNode, store, cache }, { spreadsheetId, worksheetTitle, credentials }) {
-    const { createNode } = boundActionCreators;
-
-    let rows = yield fetchSheet(spreadsheetId, worksheetTitle, credentials);
-
-    rows.forEach(function (r) {
-      /* console.log(
-          _.mapValues(r, (val, key) => ({
-              isNull: _.isNull(val),
-              isNumber: _.isFinite(val),
-              isString: _.isString(val)
-          }))
-      ); */
-      createNode(Object.assign(r, {
-        id: uuidv5(r.id, uuidv5('gsheet', seedConstant)),
-        parent: '__SOURCE__',
-        children: [],
-        internal: {
-          type: _.camelCase(`googleSheet ${worksheetTitle} row`),
-          contentDigest: crypto.createHash('md5').update(JSON.stringify(r)).digest('hex')
+var fetchSheet = require(`./fetch-sheet.js`);
+var uuidv5 = require("uuid/v5");
+var _ = require("lodash");
+var crypto = require("crypto");
+var seedConstant = "2972963f-2fcf-4567-9237-c09a2b436541";
+
+exports.sourceNodes = function () {
+  var _ref3 = (0, _asyncToGenerator3.default)( /*#__PURE__*/_regenerator2.default.mark(function _callee(_ref, _ref2) {
+    var boundActionCreators = _ref.boundActionCreators,
+        getNode = _ref.getNode,
+        store = _ref.store,
+        cache = _ref.cache;
+    var spreadsheetId = _ref2.spreadsheetId,
+        worksheetTitle = _ref2.worksheetTitle,
+        credentials = _ref2.credentials;
+    var createNode, rows;
+    return _regenerator2.default.wrap(function _callee$(_context) {
+      while (1) {
+        switch (_context.prev = _context.next) {
+          case 0:
+            createNode = boundActionCreators.createNode;
+            _context.next = 3;
+            return fetchSheet(spreadsheetId, worksheetTitle, credentials);
+
+          case 3:
+            rows = _context.sent;
+
+
+            rows.forEach(function (r) {
+              createNode(Object.assign(r, {
+                id: uuidv5(r.id, uuidv5("gsheet", seedConstant)),
+                parent: "__SOURCE__",
+                children: [],
+                internal: {
+                  type: _.camelCase(`googleSheet ${worksheetTitle} row`),
+                  contentDigest: crypto.createHash("md5").update(JSON.stringify(r)).digest("hex")
+                }
+              }));
+            });
+
+          case 5:
+          case "end":
+            return _context.stop();
         }
-      }));
-    });
-  });
+      }
+    }, _callee, undefined);
+  }));
 
   return function (_x, _x2) {
-    return _ref.apply(this, arguments);
+    return _ref3.apply(this, arguments);
   };
-})();
+}();

package.json

@@ -1,21 +1,21 @@
 {
     "name": "gatsby-source-google-sheets",
-    "version": "1.0.5",
-    "description": "A source plugin for Gatsby that allows reading data from Google Sheets.",
+    "version": "1.1.0",
+    "description":
+        "A source plugin for Gatsby that allows reading data from Google Sheets.",
     "main": "index.js",
     "scripts": {
         "build": "babel src --out-dir .",
         "watch": "babel -w src --out-dir .",
-        "prepublish": "cross-env NODE_ENV=production npm run build"
+        "prepublish": "npm run prettify && cross-env NODE_ENV=production npm run build",
+        "prettify": "prettier ./src/** --write",
+        "test": "jest"
     },
     "author": "Brandon Paquette <[email protected]>",
-    "keywords": [
-        "gatsby",
-        "gatsby-plugin",
-        "gatsby-source-plugin"
-    ],
+    "keywords": ["gatsby", "gatsby-plugin", "gatsby-source-plugin"],
     "license": "MIT",
     "dependencies": {
+        "date-fns": "^1.29.0",
         "flow-bin": "^0.56.0",
         "google-spreadsheet": "2.0.3",
         "lodash": "^4.17.4",
@@ -31,6 +31,12 @@
         "babel-preset-env": "^1.6.1",
         "babel-preset-flow": "^6.23.0",
         "babel-preset-react": "^6.24.1",
-        "cross-env": "^5.1.3"
+        "cross-env": "^5.1.3",
+        "jest": "^22.4.2",
+        "prettier": "1.10.2"
+    },
+    "prettier": {
+        "printWidth": 100,
+        "parser": "flow"
     }
 }