Skip to content
This repository was archived by the owner on Jul 23, 2020. It is now read-only.

Fix mongodb migration problems #15

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Fix mongodb migration problems #15

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ba2ab06
Fix mongodb migration problems
henrikekblad Dec 22, 2019
6bb2146
Convert numeric object fields to numbers.
henrikekblad Dec 25, 2019
2da62c0
Tune conversion fields
henrikekblad Dec 25, 2019
8bc39a4
Final cleanup, keep value-fields as is
henrikekblad Dec 27, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 25 additions & 4 deletions clean.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,16 @@ module.exports.data = function (obj) {
return null;
}
var key = obj._key;


// Cleanup some "problematic" keys that could result in too long keys for index. They are not needed.
if (key === "errors:404" || key === "ip:recent")
return null;

// clean up importer bugs
delete obj.undefined;
if ((key.startsWith('chat:room:') && key.endsWith('uids') && !key.endsWith(':uids')) || (key.startsWith('uid:') && key.endsWith('sessionUUID:sessionId') && !key.endsWith(':sessionUUID:sessionId'))) {
return null;
}
// if ((key.startsWith('chat:room:') && key.endsWith('uids') && !key.endsWith(':uids')) || (key.startsWith('uid:') && key.endsWith('sessionUUID:sessionId') && !key.endsWith(':sessionUUID:sessionId'))) {
// return null;
// }

// remove importer cache on live objects
if (!key.startsWith('_imported')) {
Expand All @@ -27,12 +31,24 @@ module.exports.data = function (obj) {
return module.exports.value(obj);
}

function isNumber(n) {
return !isNaN(parseFloat(n)) && isFinite(n);
}

var dotRE = /\./g;

module.exports.value = function (obj) {
var key = obj._key;
for (var k in obj) {
if (!Object.prototype.hasOwnProperty.call(obj, k)) {
continue;
}
var v = obj[k];
// if there is a '.' in the field name it inserts subdocument in mongo, replace '.'s with \uff0E,
if (dotRE.test(k)) {
delete obj[k];
k = k.replace(dotRE, '\uff0E');
}
if (!v || v === true) {
continue;
}
Expand All @@ -46,6 +62,11 @@ module.exports.value = function (obj) {
}
continue;
}
// Convert value to a real number (to allow mongo $inc operation to work after migration)
// Skipping some objects that should keep value as string && !key.endsWith(":members")
if (k !== "value" && isNumber(v)) {
obj[k] = Number(v);
}
if (typeof v === 'string') {
if (v.indexOf('\x00') !== -1) {
obj[k] = v.replace(/\x00/g, 'x00');
Expand Down