Node died

[yasiryagi]

• CPU: 8 core
• memory: 32 G
• swap: 2G
• ulimit -n: 1024
• Logs:
  panic: open /root/.rizon/data/cs.wal: too many open files
  goroutine 63021 [running]:
  github.com/tendermint/tendermint/libs/autofile.(*Group).readGroupInfo(0xc00b0cd760, 0x0, 0x0, 0x0, 0x0)
  /root/go/pkg/mod/github.com/tendermint/tendermint@v0.34.9/libs/autofile/group.go:365 +0x52b
  github.com/tendermint/tendermint/libs/autofile.(*Group).checkTotalSizeLimit(0xc00b0cd760)
  /root/go/pkg/mod/github.com/tendermint/tendermint@v0.34.9/libs/autofile/group.go:274 +0x69
  github.com/tendermint/tendermint/libs/autofile.(*Group).processTicks(0xc00b0cd760)
  /root/go/pkg/mod/github.com/tendermint/tendermint@v0.34.9/libs/autofile/group.go:245 +0x85
  created by github.com/tendermint/tendermint/libs/autofile.(*Group).OnStart
  /root/go/pkg/mod/github.com/tendermint/tendermint@v0.34.9/libs/autofile/group.go:140 +0x68

[yasiryagi]

Crashed again here is more logs

panic: can't get node 7D2B25CC55A25C384BF22429199430EF7132593FCE91BDEA3255555795FF9CCB: open /root/.rizon/data/application.db/483380.ldb: too many open files

goroutine 107 [running]:
github.com/cosmos/iavl.(*nodeDB).GetNode(0xc0001263c0, 0xc015743ea0, 0x20, 0x20, 0x0)
/root/go/pkg/mod/github.com/cosmos/iavl@v0.15.3/nodedb.go:87 +0x779
github.com/cosmos/iavl.(*Node).getRightNode(...)
/root/go/pkg/mod/github.com/cosmos/iavl@v0.15.3/node.go:428
github.com/cosmos/iavl.(*Node).get(0xc010169720, 0xc009638138, 0xc00bb1c960, 0x1d, 0x30, 0x1d, 0xffffffffffffffff, 0xc0005fcc30, 0xc0001263c0)
/root/go/pkg/mod/github.com/cosmos/iavl@v0.15.3/node.go:175 +0x2f6
github.com/cosmos/iavl.(*Node).get(0xc0101692c0, 0xc009638138, 0xc00bb1c960, 0x1d, 0x30, 0x1d, 0xffffffffffffffff, 0xc0005fcc30, 0xc0001263c0)
/root/go/pkg/mod/github.com/cosmos/iavl@v0.15.3/node.go:173 +0x1af
github.com/cosmos/iavl.(*Node).get(0xc010169040, 0xc009638138, 0xc00bb1c960, 0x1d, 0x30, 0x1d, 0xffffffffffffffff, 0xc0005fcc30, 0xc0001263c0)
/root/go/pkg/mod/github.com/cosmos/iavl@v0.15.3/node.go:173 +0x1af
github.com/cosmos/iavl.(*Node).get(0xc010168fa0, 0xc009638138, 0xc00bb1c960, 0x1d, 0x30, 0x1d, 0xffffffffffffffff, 0xc0005fcc30, 0xc0001263c0)
/root/go/pkg/mod/github.com/cosmos/iavl@v0.15.3/node.go:173 +0x1af
github.com/cosmos/iavl.(*Node).get(0xc010256aa0, 0xc009638138, 0xc00bb1c960, 0x1d, 0x30, 0x1d, 0x1, 0xc0005fcc30, 0xc0001263c0)
/root/go/pkg/mod/github.com/cosmos/iavl@v0.15.3/node.go:173 +0x1af
github.com/cosmos/iavl.(*Node).get(0xc005685040, 0xc009638138, 0xc00bb1c960, 0x1d, 0x30, 0x1d, 0x1, 0xc0005fcc30, 0xc0001263c0)
/root/go/pkg/mod/github.com/cosmos/iavl@v0.15.3/node.go:176 +0x273
github.com/cosmos/iavl.(*Node).get(0xc005684fa0, 0xc009638138, 0xc00bb1c960, 0x1d, 0x30, 0x1d, 0x1, 0xc0005fcc30, 0xc0001263c0)
/root/go/pkg/mod/github.com/cosmos/iavl@v0.15.3/node.go:176 +0x273
github.com/cosmos/iavl.(*Node).get(0xc005684f00, 0xc009638138, 0xc00bb1c960, 0x1d, 0x30, 0x1d, 0xffffffffffffffff, 0xc0005fcc30, 0xc0001263c0)
/root/go/pkg/mod/github.com/cosmos/iavl@v0.15.3/node.go:176 +0x273
github.com/cosmos/iavl.(*Node).get(0xc012bd4f00, 0xc009638138, 0xc00bb1c960, 0x1d, 0x30, 0x1d, 0x1, 0xc0005fcc30, 0xc0001263c0)
/root/go/pkg/mod/github.com/cosmos/iavl@v0.15.3/node.go:173 +0x1af
github.com/cosmos/iavl.(*Node).get(0xc0150e0780, 0xc009638138, 0xc00bb1c960, 0x1d, 0x30, 0x1d, 0x1, 0xc0005fcc30, 0xc0001263c0)
/root/go/pkg/mod/github.com/cosmos/iavl@v0.15.3/node.go:176 +0x273
github.com/cosmos/iavl.(*Node).get(0xc0150e06e0, 0xc009638138, 0xc00bb1c960, 0x1d, 0x30, 0x1d, 0x1, 0x40466b, 0xc0001263c0)
/root/go/pkg/mod/github.com/cosmos/iavl@v0.15.3/node.go:176 +0x273
github.com/cosmos/iavl.(*Node).get(0xc0150e0640, 0xc009638138, 0xc00bb1c960, 0x1d, 0x30, 0x1d, 0xffffffffffffffff, 0x20, 0xc0001263c0)
/root/go/pkg/mod/github.com/cosmos/iavl@v0.15.3/node.go:176 +0x273
github.com/cosmos/iavl.(*Node).get(0xc0150e1cc0, 0xc009638138, 0xc00bb1c960, 0x1d, 0x30, 0x1d, 0xffffffffffffffff, 0x28, 0x30)
/root/go/pkg/mod/github.com/cosmos/iavl@v0.15.3/node.go:173 +0x1af
github.com/cosmos/iavl.(*Node).get(0xc0150e6820, 0xc009638138, 0xc00bb1c960, 0x1d, 0x30, 0x30, 0x19225c0, 0xc001fe0201, 0xc011f12cf0)
/root/go/pkg/mod/github.com/cosmos/iavl@v0.15.3/node.go:173 +0x1af
github.com/cosmos/iavl.(*ImmutableTree).Get(0xc009638138, 0xc00bb1c960, 0x1d, 0x30, 0x0, 0x0, 0x0, 0x0)
/root/go/pkg/mod/github.com/cosmos/iavl@v0.15.3/immutable_tree.go:152 +0x5a
github.com/cosmos/cosmos-sdk/store/iavl.(*Store).Get(0xc000e8f080, 0xc00bb1c960, 0x1d, 0x30, 0x0, 0x0, 0x0)
/root/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.42.4/store/iavl/store.go:172 +0x166
github.com/cosmos/cosmos-sdk/store/cache.(*CommitKVStoreCache).Get(0xc000284750, 0xc00bb1c960, 0x1d, 0x30, 0x2c4b1c0, 0x0, 0x30)
/root/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.42.4/store/cache/cache.go:111 +0x11e
github.com/cosmos/cosmos-sdk/store/cachekv.(*Store).Get(0xc00fe43d10, 0xc00bb1c960, 0x1d, 0x30, 0x0, 0x0, 0x0)
/root/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.42.4/store/cachekv/store.go:64 +0x2b0
github.com/cosmos/cosmos-sdk/store/gaskv.(*Store).Get(0xc00ebb7020, 0xc00bb1c960, 0x1d, 0x30, 0x0, 0x0, 0x0)
/root/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.42.4/store/gaskv/store.go:42 +0x198
github.com/cosmos/cosmos-sdk/x/slashing/keeper.Keeper.GetValidatorMissedBlockBitArray(0x200c548, 0xc0001297d0, 0x203b638, 0xc000128330, 0x2036048, 0xc000295a20, 0x2029fe0, 0xc00115d440, 0x2026448, 0xc0000400f0, ...)
/root/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.42.4/x/slashing/keeper/signing_info.go:60 +0x11d
github.com/cosmos/cosmos-sdk/x/slashing/keeper.Keeper.HandleValidatorSignature(0x200c548, 0xc0001297d0, 0x203b638, 0xc000128330, 0x2036048, 0xc000295a20, 0x2029fe0, 0xc00115d440, 0x2026448, 0xc0000400f0, ...)
/root/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.42.4/x/slashing/keeper/infractions.go:36 +0x550
github.com/cosmos/cosmos-sdk/x/slashing.BeginBlocker(0x2026448, 0xc0000400f0, 0x203afc8, 0xc008c1d600, 0xb, 0x0, 0xc00e419658, 0x8, 0x1478b, 0x26fc8b9f, ...)
/root/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.42.4/x/slashing/abci.go:23 +0x247
github.com/cosmos/cosmos-sdk/x/slashing.AppModule.BeginBlock(...)
/root/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.42.4/x/slashing/module.go:164
github.com/cosmos/cosmos-sdk/types/module.(*Manager).BeginBlock(0xc000ebd810, 0x2026448, 0xc0000400f0, 0x203afc8, 0xc008c1d600, 0xb, 0x0, 0xc00e419658, 0x8, 0x1478b, ...)
/root/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.42.4/types/module/module.go:338 +0x1b8
github.com/rizon-world/rizon/app.(*RizonApp).BeginBlocker(...)
/root/rizon/app/app.go:461
github.com/cosmos/cosmos-sdk/baseapp.(*BaseApp).BeginBlock(0xc00059f860, 0xc0088ace80, 0x20, 0x20, 0xb, 0x0, 0xc00e419658, 0x8, 0x1478b, 0x26fc8b9f, ...)
/root/go/pkg/mod/github.com/cosmos/cosmos-sdk@v0.42.4/baseapp/abci.go:179 +0x638
github.com/tendermint/tendermint/abci/client.(*localClient).BeginBlockSync(0xc00115c780, 0xc0088ace80, 0x20, 0x20, 0xb, 0x0, 0xc00e419658, 0x8, 0x1478b, 0x26fc8b9f, ...)
/root/go/pkg/mod/github.com/tendermint/tendermint@v0.34.9/abci/client/local_client.go:274 +0xfa
github.com/tendermint/tendermint/proxy.(*appConnConsensus).BeginBlockSync(0xc0010310e0, 0xc0088ace80, 0x20, 0x20, 0xb, 0x0, 0xc00e419658, 0x8, 0x1478b, 0x26fc8b9f, ...)
/root/go/pkg/mod/github.com/tendermint/tendermint@v0.34.9/proxy/app_conn.go:81 +0x75
github.com/tendermint/tendermint/state.execBlockOnProxyApp(0x2026fe0, 0xc0000b3740, 0x20321f8, 0xc0010310e0, 0xc006169c20, 0x203b168, 0xc001030570, 0x1, 0xc00698ffa0, 0x20, ...)
/root/go/pkg/mod/github.com/tendermint/tendermint@v0.34.9/state/execution.go:307 +0x51b
github.com/tendermint/tendermint/state.(*BlockExecutor).ApplyBlock(0xc000e1b260, 0xb, 0x0, 0xc0013e7b80, 0x7, 0xc0013e7b88, 0x8, 0x1, 0x1478a, 0xc00698ffa0, ...)
/root/go/pkg/mod/github.com/tendermint/tendermint@v0.34.9/state/execution.go:140 +0x168
github.com/tendermint/tendermint/blockchain/v0.(*BlockchainReactor).poolRoutine(0xc000d81340, 0x0)
/root/go/pkg/mod/github.com/tendermint/tendermint@v0.34.9/blockchain/v0/reactor.go:398 +0x1033
created by github.com/tendermint/tendermint/blockchain/v0.(*BlockchainReactor).OnStart
/root/go/pkg/mod/github.com/tendermint/tendermint@v0.34.9/blockchain/v0/reactor.go:110 +0x8c

[YogaSakti]

the problem you are facing is too many open files

here's the solution