V0.9

.gitignore

@@ -1,2 +1,3 @@
 /node_modules/
 /log/*.*
+config

README.md

@@ -4,7 +4,7 @@ npm install
 
 npm install -g pm2
 
-pm2 start main.js --name=cosmos_node_checker
+pm2 start ./app.json
 
 pm2 stop cosmos_node_checker
 

app.json

@@ -0,0 +1,9 @@
+{
+	"apps": [{
+		"name": "cosmos_node_checker",
+		"script": "main.js",
+		"watch": false,
+		"env": {
+		}
+	}]
+}

main.js

@@ -16,14 +16,15 @@ let checkedBlockHeight = 0
 let missedBlockHeight = 0
 let validatorConnectTryCnt = 0
 let botStatusFlag = false
+let executeCnt = 0
+let blockCheck = [] // block check height array
 
 const botJob = new CronJob(`*/10 * * * * *`, async function () {
 	let mem = await server.getMemoryUsage()
 	let cpu = await server.getCpuUsage()
 	let disk = await server.getDiskUsage()
 	let peer = await server.getPeerCount()
 	let blockHeight = await server.getBlockHeight()
-	let rpcHeight = await rpc.getRpcHeight()
 	let checkDialPort = await server.checkDialPort()
 	let checkLcdPort = false
 	let checkValidatorConnect = false
@@ -34,8 +35,8 @@ const botJob = new CronJob(`*/10 * * * * *`, async function () {
 		cpu : cpu,
 		disk : disk,
 		peer : peer,
-		blockHeight : blockHeight,
-		rpcHeight : rpcHeight		
+		blockHeight : blockHeight
+//		rpcHeight : rpcHeight
 	})
 	// memory check
 	if(mem > parseFloat(cfg.SERVER_ALERT_MEMORY)) {
@@ -78,23 +79,44 @@ const botJob = new CronJob(`*/10 * * * * *`, async function () {
 	}
 
 	// block height check
-	if(blockHeight+5 < rpcHeight) {
-		let heightDiff = rpcHeight - blockHeight
-
-		if(blockHeight > checkedBlockHeight){
-			checkedBlockHeight = blockHeight 
-			alert.sendMSG(`ALERT! Server height is lower than extern height. ${cfg.EXTERN_RPC_URL}\nDiff=${heightDiff.toLocaleString()}\nserver=${blockHeight.toLocaleString()}\nextern=${rpcHeight.toLocaleString()}`)
-		} 
-//		logger.info(`blockHeightAlertCnt : ${blockHeightAlertCnt}`)
-	} else if (blockHeight > checkedBlockHeight){
-		let heightDiff = blockHeight - rpcHeight 
-
-		if(checkedBlockHeight > blockHeight){
-			checkedBlockHeight = blockHeight 
-			alert.sendMSG(`ALERT! Extern height is lower than server height. ${cfg.EXTERN_RPC_URL}\nDiff=${heightDiff.toLocaleString()}\nserver=${blockHeight.toLocaleString()}\nextern=${rpcHeight.toLocaleString()}`)
-		} 
-//		logger.info(`blockHeightAlertCnt : ${blockHeightAlertCnt}`)
-	}
+	blockCheck[executeCnt] = blockHeight
+	let heightDiff = blockCheck[executeCnt] - blockCheck[executeCnt-1]
+
+//	logger.info(`executeCnt:${executeCnt}`)
+//	logger.info(`blockCheck.length:${blockCheck.length}`)
+
+	if(blockCheck.length > 1){ //need history
+		if(heightDiff > cfg.SERVER_ALERT_BLOCK_ERROR_RANGE){ // server block height is abnormal
+			let rpcHeight = await rpc.getRpcHeight()
+			alert.sendMSG(`ALERT! Server height is abnormal.\n${cfg.EXTERN_RPC_URL}/status\nExtern=${rpcHeight.toLocaleString()}\nDiff=${heightDiff.toLocaleString()}\nCurrentblockheight=${blockCheck[executeCnt].toLocaleString()}\nPreblockheight=${blockCheck[executeCnt-1].toLocaleString()}`)
+		} else {
+			let rpcHeight = await rpc.getRpcHeight()
+			if(blockCheck[executeCnt] === blockCheck[executeCnt-1] === blockCheck[executeCnt-2] === blockCheck[executeCnt-3] === blockCheck[executeCnt-4]){ //chain is stop
+				alert.sendMSG(`ALERT! Maybe chain is down.\n${cfg.EXTERN_RPC_URL}/status\nExtern=${rpcHeight.toLocaleString()}\nDiff=${heightDiff.toLocaleString()}\nCurrentblockheight=${blockCheck[executeCnt].toLocaleString()}\nPreblockheight=${blockCheck[executeCnt-1].toLocaleString()}`)
+			}else{
+				// normal
+//				logger.info(`Diff=${heightDiff.toLocaleString()}\nCurrentblockheight=${blockCheck[executeCnt].toLocaleString()}\nPreblockheight=${blockCheck[executeCnt-1].toLocaleString()}`)
+			}
+		}
+	}	
+
+//	if(blockHeight+5 < rpcHeight) {
+//		let heightDiff = rpcHeight - blockHeight
+//		
+//		if(blockHeight > checkedBlockHeight){
+//			checkedBlockHeight = blockHeight 
+//			alert.sendMSG(`ALERT! Server height is lower than extern height. ${cfg.EXTERN_RPC_URL}\nDiff=${heightDiff.toLocaleString()}\nserver=${blockHeight.toLocaleString()}\nextern=${rpcHeight.toLocaleString()}`)
+//		} 
+////		logger.info(`blockHeightAlertCnt : ${blockHeightAlertCnt}`)
+//	} else if (blockHeight > checkedBlockHeight){
+//		let heightDiff = blockHeight - rpcHeight 
+//		
+//		if(checkedBlockHeight > blockHeight){
+//			checkedBlockHeight = blockHeight 
+//			alert.sendMSG(`ALERT! Extern height is lower than server height. ${cfg.EXTERN_RPC_URL}\nDiff=${heightDiff.toLocaleString()}\nserver=${blockHeight.toLocaleString()}\nextern=${rpcHeight.toLocaleString()}`)
+//		} 
+////		logger.info(`blockHeightAlertCnt : ${blockHeightAlertCnt}`)
+//	}
 
 	// validator connect check
 	if(cfg.SERVER_TYPE == 'validator'){
@@ -148,15 +170,17 @@ const botJob = new CronJob(`*/10 * * * * *`, async function () {
 	//	console.log(`checkValidatorSign : ${checkValidatorSign}`)
 	//	
 	//	console.log('====================================\n\n')
-		
+	executeCnt = executeCnt < 5 ? executeCnt + 1 : 0 //execute count history limit 5   
 })//.start()
 
 const botStart = (() => {
 	botJob.start()
+	botStatusFlag = true
 })
 
 const botStop = (() => {
 	botJob.stop()
+	botStatusFlag = false
 })
 
 const botStatus = (() => {

process/server.js

@@ -53,7 +53,8 @@ const getDeamonStatus = (async () => {
 
 // block height
 const getBlockHeight = (async () => {
-	let cmd = `${cfg.PROJECT_CLIENT_NAME} status | jq .sync_info.latest_block_height | tr -d '"'`
+	let cmd = `curl -s 'http://localhost:26657/status' | jq '.result.sync_info.latest_block_height' | tr -d '"'`
+	//let cmd = `${cfg.PROJECT_CLIENT_NAME} status | jq .sync_info.latest_block_height | tr -d '"'`
 	let res = await exec(cmd)
 	let blockHeight = parseInt(res.toString())
 	return blockHeight
@@ -94,7 +95,7 @@ const checkValidatorConnect = (async () => {
 // validator sign check
 const checkValidatorSign = (async (latestHeight) => {
 	//let cmd = `${cfg.PROJECT_CLIENT_NAME} query block ${latestHeight} --trust-node=true | jq .block.last_commit.precommits[].validator_address | grep ${cfg.VALIDATOR_HASH} | tr -d '"'`
-	let cmd = `${cfg.PROJECT_CLIENT_NAME} q block ${latestHeight} --trust-node=true | jq .block.last_commit.signatures[].validator_address | grep ${cfg.VALIDATOR_HASH} | wc -l`
+	let cmd = `${cfg.PROJECT_CLIENT_NAME} q block ${latestHeight} | jq .block.last_commit.signatures[].validator_address | grep ${cfg.VALIDATOR_HASH} | wc -l`
 	let res = await exec(cmd)
 	let count = parseInt(res.toString())	
 	return count > 0 ? true : false

process/telegram_bot.js

@@ -32,8 +32,9 @@ bot.command('status', async (ctx) => {
 bot.command('server', (ctx) => {
 	try{
 		let blockHeight = typeof variables.blockHeight === undefined ? undefined : variables.blockHeight.toLocaleString()
-		let rpcHeight = typeof variables.rpcHeight === undefined ? undefined : variables.rpcHeight.toLocaleString()
-		ctx.reply(`Memory : ${variables.mem}%\nCpu : ${variables.cpu}%\nDisk : ${variables.disk}%\nPeerCount : ${variables.peer}\nLatestHeight : ${blockHeight}\nRpcHeight : ${rpcHeight}\n`)
+//		let rpcHeight = typeof variables.rpcHeight === undefined ? undefined : variables.rpcHeight.toLocaleString()
+		ctx.reply(`Memory : ${variables.mem}%\nCpu : ${variables.cpu}%\nDisk : ${variables.disk}%\nPeerCount : ${variables.peer}\nLatestHeight : ${blockHeight}`)
+		//\nRpcHeight : ${rpcHeight}\n
 	} catch (err){
 		ctx.reply(err)
 	}