diff --git a/.gitignore b/.gitignore index 96d434d..db42dd6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ /node_modules/ /log/*.* +config diff --git a/README.md b/README.md index de58485..774759c 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ npm install npm install -g pm2 -pm2 start main.js --name=cosmos_node_checker +pm2 start ./app.json pm2 stop cosmos_node_checker diff --git a/app.json b/app.json new file mode 100644 index 0000000..c490c67 --- /dev/null +++ b/app.json @@ -0,0 +1,9 @@ +{ + "apps": [{ + "name": "cosmos_node_checker", + "script": "main.js", + "watch": false, + "env": { + } + }] +} diff --git a/config b/config deleted file mode 100644 index 68c007c..0000000 --- a/config +++ /dev/null @@ -1,40 +0,0 @@ -# Project info -PROJECT_NAME=desmos -PROJECT_DAEMON_NAME=desmosd -PROJECT_CLIENT_NAME=desmoscli -PROJECT_DAEMON_PORT=26656 -PROJECT_DIAL_PORT=26657 -PROJECT_LCD_USE=false -PROJECT_LCD_PORT=1317 - -# RPC info -EXTERN_RPC_URL=http://rpc.morpheus.desmos.network:26657 -#EXTERN_EXPLORER=https://morpheus.desmos.network - -# Validator info -VALIDATOR_SERVER_IP=161.35.125.20 -VALIDATOR_NODE_ID=193d02a5bf28f2a1741b4ecb4c9be812f73d51cb -VALIDATOR_DAEMON_PORT=26656 -VALIDATOR_HASH=11361C0BEF8929F40DC5CB911230C335C7D1C594 - -# Server info -SERVER_TYPE=validator -SERVER_NAME=newyork -SERVER_ALERT_MEMORY=90 -SERVER_ALERT_CPU=90 -SERVER_ALERT_DISK=80 -SERVER_ALERT_PEER=10 - -SERVER_ALERT_MEMORY_WAIT=10 -SERVER_ALERT_CPU_WAIT=10 -SERVER_ALERT_DISK_WAIT=10 -SERVER_ALERT_PEER_WAIT=10 -SERVER_ALERT_VALIDATORCONNECT_WAIT=30 -SERVER_ALERT_LCD_WAIT=10 - -# Telegram info -TELEGRAM_BOT_TOKEN=1495145921:AAF7lbLUO1MbC476UXkM_0a3xnVoUQ6I_Io -TELEGRAM_BOT_ROOM=-1001361430882 - -# Manager info (sms) - unused -MANAGER_PHONNUM=+821092649104,+821092649104 \ No newline at end of file diff --git a/main.js b/main.js index 91df192..3ff8848 100644 --- a/main.js +++ b/main.js @@ -16,6 +16,8 @@ let checkedBlockHeight = 0 let missedBlockHeight = 0 let validatorConnectTryCnt = 0 let botStatusFlag = false +let executeCnt = 0 +let blockCheck = [] // block check height array const botJob = new CronJob(`*/10 * * * * *`, async function () { let mem = await server.getMemoryUsage() @@ -23,7 +25,6 @@ const botJob = new CronJob(`*/10 * * * * *`, async function () { let disk = await server.getDiskUsage() let peer = await server.getPeerCount() let blockHeight = await server.getBlockHeight() - let rpcHeight = await rpc.getRpcHeight() let checkDialPort = await server.checkDialPort() let checkLcdPort = false let checkValidatorConnect = false @@ -34,8 +35,8 @@ const botJob = new CronJob(`*/10 * * * * *`, async function () { cpu : cpu, disk : disk, peer : peer, - blockHeight : blockHeight, - rpcHeight : rpcHeight + blockHeight : blockHeight +// rpcHeight : rpcHeight }) // memory check if(mem > parseFloat(cfg.SERVER_ALERT_MEMORY)) { @@ -78,23 +79,44 @@ const botJob = new CronJob(`*/10 * * * * *`, async function () { } // block height check - if(blockHeight+5 < rpcHeight) { - let heightDiff = rpcHeight - blockHeight - - if(blockHeight > checkedBlockHeight){ - checkedBlockHeight = blockHeight - alert.sendMSG(`ALERT! Server height is lower than extern height. ${cfg.EXTERN_RPC_URL}\nDiff=${heightDiff.toLocaleString()}\nserver=${blockHeight.toLocaleString()}\nextern=${rpcHeight.toLocaleString()}`) - } -// logger.info(`blockHeightAlertCnt : ${blockHeightAlertCnt}`) - } else if (blockHeight > checkedBlockHeight){ - let heightDiff = blockHeight - rpcHeight - - if(checkedBlockHeight > blockHeight){ - checkedBlockHeight = blockHeight - alert.sendMSG(`ALERT! Extern height is lower than server height. ${cfg.EXTERN_RPC_URL}\nDiff=${heightDiff.toLocaleString()}\nserver=${blockHeight.toLocaleString()}\nextern=${rpcHeight.toLocaleString()}`) - } -// logger.info(`blockHeightAlertCnt : ${blockHeightAlertCnt}`) - } + blockCheck[executeCnt] = blockHeight + let heightDiff = blockCheck[executeCnt] - blockCheck[executeCnt-1] + +// logger.info(`executeCnt:${executeCnt}`) +// logger.info(`blockCheck.length:${blockCheck.length}`) + + if(blockCheck.length > 1){ //need history + if(heightDiff > cfg.SERVER_ALERT_BLOCK_ERROR_RANGE){ // server block height is abnormal + let rpcHeight = await rpc.getRpcHeight() + alert.sendMSG(`ALERT! Server height is abnormal.\n${cfg.EXTERN_RPC_URL}/status\nExtern=${rpcHeight.toLocaleString()}\nDiff=${heightDiff.toLocaleString()}\nCurrentblockheight=${blockCheck[executeCnt].toLocaleString()}\nPreblockheight=${blockCheck[executeCnt-1].toLocaleString()}`) + } else { + let rpcHeight = await rpc.getRpcHeight() + if(blockCheck[executeCnt] === blockCheck[executeCnt-1] === blockCheck[executeCnt-2] === blockCheck[executeCnt-3] === blockCheck[executeCnt-4]){ //chain is stop + alert.sendMSG(`ALERT! Maybe chain is down.\n${cfg.EXTERN_RPC_URL}/status\nExtern=${rpcHeight.toLocaleString()}\nDiff=${heightDiff.toLocaleString()}\nCurrentblockheight=${blockCheck[executeCnt].toLocaleString()}\nPreblockheight=${blockCheck[executeCnt-1].toLocaleString()}`) + }else{ + // normal +// logger.info(`Diff=${heightDiff.toLocaleString()}\nCurrentblockheight=${blockCheck[executeCnt].toLocaleString()}\nPreblockheight=${blockCheck[executeCnt-1].toLocaleString()}`) + } + } + } + +// if(blockHeight+5 < rpcHeight) { +// let heightDiff = rpcHeight - blockHeight +// +// if(blockHeight > checkedBlockHeight){ +// checkedBlockHeight = blockHeight +// alert.sendMSG(`ALERT! Server height is lower than extern height. ${cfg.EXTERN_RPC_URL}\nDiff=${heightDiff.toLocaleString()}\nserver=${blockHeight.toLocaleString()}\nextern=${rpcHeight.toLocaleString()}`) +// } +//// logger.info(`blockHeightAlertCnt : ${blockHeightAlertCnt}`) +// } else if (blockHeight > checkedBlockHeight){ +// let heightDiff = blockHeight - rpcHeight +// +// if(checkedBlockHeight > blockHeight){ +// checkedBlockHeight = blockHeight +// alert.sendMSG(`ALERT! Extern height is lower than server height. ${cfg.EXTERN_RPC_URL}\nDiff=${heightDiff.toLocaleString()}\nserver=${blockHeight.toLocaleString()}\nextern=${rpcHeight.toLocaleString()}`) +// } +//// logger.info(`blockHeightAlertCnt : ${blockHeightAlertCnt}`) +// } // validator connect check if(cfg.SERVER_TYPE == 'validator'){ @@ -148,15 +170,17 @@ const botJob = new CronJob(`*/10 * * * * *`, async function () { // console.log(`checkValidatorSign : ${checkValidatorSign}`) // // console.log('====================================\n\n') - + executeCnt = executeCnt < 5 ? executeCnt + 1 : 0 //execute count history limit 5 })//.start() const botStart = (() => { botJob.start() + botStatusFlag = true }) const botStop = (() => { botJob.stop() + botStatusFlag = false }) const botStatus = (() => { diff --git a/process/server.js b/process/server.js index 870a924..f9ae597 100644 --- a/process/server.js +++ b/process/server.js @@ -53,7 +53,8 @@ const getDeamonStatus = (async () => { // block height const getBlockHeight = (async () => { - let cmd = `${cfg.PROJECT_CLIENT_NAME} status | jq .sync_info.latest_block_height | tr -d '"'` + let cmd = `curl -s 'http://localhost:26657/status' | jq '.result.sync_info.latest_block_height' | tr -d '"'` + //let cmd = `${cfg.PROJECT_CLIENT_NAME} status | jq .sync_info.latest_block_height | tr -d '"'` let res = await exec(cmd) let blockHeight = parseInt(res.toString()) return blockHeight @@ -94,7 +95,7 @@ const checkValidatorConnect = (async () => { // validator sign check const checkValidatorSign = (async (latestHeight) => { //let cmd = `${cfg.PROJECT_CLIENT_NAME} query block ${latestHeight} --trust-node=true | jq .block.last_commit.precommits[].validator_address | grep ${cfg.VALIDATOR_HASH} | tr -d '"'` - let cmd = `${cfg.PROJECT_CLIENT_NAME} q block ${latestHeight} --trust-node=true | jq .block.last_commit.signatures[].validator_address | grep ${cfg.VALIDATOR_HASH} | wc -l` + let cmd = `${cfg.PROJECT_CLIENT_NAME} q block ${latestHeight} | jq .block.last_commit.signatures[].validator_address | grep ${cfg.VALIDATOR_HASH} | wc -l` let res = await exec(cmd) let count = parseInt(res.toString()) return count > 0 ? true : false diff --git a/process/telegram_bot.js b/process/telegram_bot.js index 5fc79d0..5dda100 100644 --- a/process/telegram_bot.js +++ b/process/telegram_bot.js @@ -32,8 +32,9 @@ bot.command('status', async (ctx) => { bot.command('server', (ctx) => { try{ let blockHeight = typeof variables.blockHeight === undefined ? undefined : variables.blockHeight.toLocaleString() - let rpcHeight = typeof variables.rpcHeight === undefined ? undefined : variables.rpcHeight.toLocaleString() - ctx.reply(`Memory : ${variables.mem}%\nCpu : ${variables.cpu}%\nDisk : ${variables.disk}%\nPeerCount : ${variables.peer}\nLatestHeight : ${blockHeight}\nRpcHeight : ${rpcHeight}\n`) +// let rpcHeight = typeof variables.rpcHeight === undefined ? undefined : variables.rpcHeight.toLocaleString() + ctx.reply(`Memory : ${variables.mem}%\nCpu : ${variables.cpu}%\nDisk : ${variables.disk}%\nPeerCount : ${variables.peer}\nLatestHeight : ${blockHeight}`) + //\nRpcHeight : ${rpcHeight}\n } catch (err){ ctx.reply(err) }