From ba2ab06c51024265d68fc757ae200eaa92bc4385 Mon Sep 17 00:00:00 2001 From: Henrik Ekblad Date: Sun, 22 Dec 2019 20:46:15 +0100 Subject: [PATCH 1/4] Fix mongodb migration problems This patch fixes problems found when migrating old redis production forum. --- clean.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/clean.js b/clean.js index 37427b9..0ed3b3c 100644 --- a/clean.js +++ b/clean.js @@ -8,7 +8,16 @@ module.exports.data = function (obj) { return null; } var key = obj._key; - + // Check if object keys contains ".". Skip as mongodb won't be able to create them. + // Keys found when migrating production db "username:uid_old", "email:uid_old" and "email:confirmed". + // Hopefully we can live without them. + if (Object.keys(obj).find(k => k.includes("."))) { + return null; + } + // Cleanup some "problematic" keys that could result in too long keys for index. They are not needed. + if (key === "errors:404" || key === "ip:recent") + return null; + // clean up importer bugs delete obj.undefined; if ((key.startsWith('chat:room:') && key.endsWith('uids') && !key.endsWith(':uids')) || (key.startsWith('uid:') && key.endsWith('sessionUUID:sessionId') && !key.endsWith(':sessionUUID:sessionId'))) { From 6bb21467f6c769f11a28e60277f6242d7ff68533 Mon Sep 17 00:00:00 2001 From: Henrik Ekblad Date: Wed, 25 Dec 2019 14:00:59 +0100 Subject: [PATCH 2/4] Convert numeric object fields to numbers. --- clean.js | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/clean.js b/clean.js index 0ed3b3c..e47a500 100644 --- a/clean.js +++ b/clean.js @@ -8,12 +8,7 @@ module.exports.data = function (obj) { return null; } var key = obj._key; - // Check if object keys contains ".". Skip as mongodb won't be able to create them. - // Keys found when migrating production db "username:uid_old", "email:uid_old" and "email:confirmed". - // Hopefully we can live without them. - if (Object.keys(obj).find(k => k.includes("."))) { - return null; - } + // Cleanup some "problematic" keys that could result in too long keys for index. They are not needed. if (key === "errors:404" || key === "ip:recent") return null; @@ -36,12 +31,19 @@ module.exports.data = function (obj) { return module.exports.value(obj); } +function isNumber(n) { + return !isNaN(parseFloat(n)) && isFinite(n); +} + module.exports.value = function (obj) { for (var k in obj) { if (!Object.prototype.hasOwnProperty.call(obj, k)) { continue; } var v = obj[k]; + // if there is a '.' in the field name it inserts subdocument in mongo, replace '.'s with \uff0E + k = k.replace(/\./g, '\uff0E'); + if (!v || v === true) { continue; } @@ -55,6 +57,10 @@ module.exports.value = function (obj) { } continue; } + // Convert value to a real number (to allow mongo $inc operation to work after migration) + if (isNumber(v)) { + obj[k] = Number(v); + } if (typeof v === 'string') { if (v.indexOf('\x00') !== -1) { obj[k] = v.replace(/\x00/g, 'x00'); From 2da62c0ff7fc984cbff86c6241a1b268a04e719d Mon Sep 17 00:00:00 2001 From: Henrik Ekblad Date: Wed, 25 Dec 2019 17:43:45 +0100 Subject: [PATCH 3/4] Tune conversion fields --- clean.js | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/clean.js b/clean.js index e47a500..3c6fda2 100644 --- a/clean.js +++ b/clean.js @@ -35,15 +35,20 @@ function isNumber(n) { return !isNaN(parseFloat(n)) && isFinite(n); } +var dotRE = /\./g; + module.exports.value = function (obj) { + var key = obj._key; for (var k in obj) { if (!Object.prototype.hasOwnProperty.call(obj, k)) { continue; } var v = obj[k]; - // if there is a '.' in the field name it inserts subdocument in mongo, replace '.'s with \uff0E - k = k.replace(/\./g, '\uff0E'); - + // if there is a '.' in the field name it inserts subdocument in mongo, replace '.'s with \uff0E, + if (dotRE.test(k)) { + delete obj[k]; + k = k.replace(dotRE, '\uff0E'); + } if (!v || v === true) { continue; } @@ -58,7 +63,8 @@ module.exports.value = function (obj) { continue; } // Convert value to a real number (to allow mongo $inc operation to work after migration) - if (isNumber(v)) { + // Skipping some objects that should keep value as string + if (!key.startsWith("nodebbpostsearch:object") && isNumber(v)) { obj[k] = Number(v); } if (typeof v === 'string') { From 8bc39a470dbaf4e6e1d588874d69ee88d803e8e0 Mon Sep 17 00:00:00 2001 From: Henrik Ekblad Date: Fri, 27 Dec 2019 15:57:11 +0100 Subject: [PATCH 4/4] Final cleanup, keep value-fields as is --- clean.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/clean.js b/clean.js index 3c6fda2..2d7a2b9 100644 --- a/clean.js +++ b/clean.js @@ -15,9 +15,9 @@ module.exports.data = function (obj) { // clean up importer bugs delete obj.undefined; - if ((key.startsWith('chat:room:') && key.endsWith('uids') && !key.endsWith(':uids')) || (key.startsWith('uid:') && key.endsWith('sessionUUID:sessionId') && !key.endsWith(':sessionUUID:sessionId'))) { - return null; - } + // if ((key.startsWith('chat:room:') && key.endsWith('uids') && !key.endsWith(':uids')) || (key.startsWith('uid:') && key.endsWith('sessionUUID:sessionId') && !key.endsWith(':sessionUUID:sessionId'))) { + // return null; + // } // remove importer cache on live objects if (!key.startsWith('_imported')) { @@ -63,8 +63,8 @@ module.exports.value = function (obj) { continue; } // Convert value to a real number (to allow mongo $inc operation to work after migration) - // Skipping some objects that should keep value as string - if (!key.startsWith("nodebbpostsearch:object") && isNumber(v)) { + // Skipping some objects that should keep value as string && !key.endsWith(":members") + if (k !== "value" && isNumber(v)) { obj[k] = Number(v); } if (typeof v === 'string') {