diff --git a/clean.js b/clean.js index 37427b9..2d7a2b9 100644 --- a/clean.js +++ b/clean.js @@ -8,12 +8,16 @@ module.exports.data = function (obj) { return null; } var key = obj._key; - + + // Cleanup some "problematic" keys that could result in too long keys for index. They are not needed. + if (key === "errors:404" || key === "ip:recent") + return null; + // clean up importer bugs delete obj.undefined; - if ((key.startsWith('chat:room:') && key.endsWith('uids') && !key.endsWith(':uids')) || (key.startsWith('uid:') && key.endsWith('sessionUUID:sessionId') && !key.endsWith(':sessionUUID:sessionId'))) { - return null; - } + // if ((key.startsWith('chat:room:') && key.endsWith('uids') && !key.endsWith(':uids')) || (key.startsWith('uid:') && key.endsWith('sessionUUID:sessionId') && !key.endsWith(':sessionUUID:sessionId'))) { + // return null; + // } // remove importer cache on live objects if (!key.startsWith('_imported')) { @@ -27,12 +31,24 @@ module.exports.data = function (obj) { return module.exports.value(obj); } +function isNumber(n) { + return !isNaN(parseFloat(n)) && isFinite(n); +} + +var dotRE = /\./g; + module.exports.value = function (obj) { + var key = obj._key; for (var k in obj) { if (!Object.prototype.hasOwnProperty.call(obj, k)) { continue; } var v = obj[k]; + // if there is a '.' in the field name it inserts subdocument in mongo, replace '.'s with \uff0E, + if (dotRE.test(k)) { + delete obj[k]; + k = k.replace(dotRE, '\uff0E'); + } if (!v || v === true) { continue; } @@ -46,6 +62,11 @@ module.exports.value = function (obj) { } continue; } + // Convert value to a real number (to allow mongo $inc operation to work after migration) + // Skipping some objects that should keep value as string && !key.endsWith(":members") + if (k !== "value" && isNumber(v)) { + obj[k] = Number(v); + } if (typeof v === 'string') { if (v.indexOf('\x00') !== -1) { obj[k] = v.replace(/\x00/g, 'x00');