Report from customer:
I made many transactions with 20ms interval, but the network shut down again. so I restart whole network and I got the error from slave node. I attached the logs following :
=== log start ===
INFO [01-22|07:55:12.156] db-operation Write RootBlock height=4620 hash=48175b…f71946
INFO [01-22|07:55:12.156] shard:1 tipMinor=9215 hash=0x1bf8061b3ccfd05cd411f18e57072c6ce4a3dce19f4f2d0730e501c44f9dfa26 mete.root=0xdbcd9a42a7b78a68a05418db0499a91144c4bfd1431dc64643d6ff4fa6b20f6a rootBlock=4620 rootTip=4620
WARN [01-22|07:55:12.156] shard:1 miss trie block=9215 block.hash=0x1bf8061b3ccfd05cd411f18e57072c6ce4a3dce19f4f2d0730e501c44f9dfa26 currNumber=9215 currHash=0x1bf8061b3ccfd05cd411f18e57072c6ce4a3dce19f4f2d0730e501c44f9dfa26
INFO [01-22|07:55:12.377] reRunBlockWithState blockchain to past state number=8704 hash=0xc761a2c5db8fbf7524950c6158a9d44154bfda14ff3e3ad3555f78a60edbdbce
INFO [01-22|07:55:15.147] slave ping response request op=2
INFO [01-22|07:56:00.195] shard:1 commit trie number=8704 hash=0xc761a2c5db8fbf7524950c6158a9d44154bfda14ff3e3ad3555f78a60edbdbce root=0xe1d70098901cf2e24b6990bc7b6168aad1bd32ccd2964b41849763f47c5d8233
WARN [01-22|07:56:37.854] shard:1 miss trie reRun time=85.698 currentBlock=9215 currHash=0x1bf8061b3ccfd05cd411f18e57072c6ce4a3dce19f4f2d0730e501c44f9dfa26
WARN [01-22|07:56:37.854] reorg same chain curr=9215 curr.Hash=0x1bf8061b3ccfd05cd411f18e57072c6ce4a3dce19f4f2d0730e501c44f9dfa26 newBlock=9215 newBlock's hash=1bf806…9dfa26 newBlock=false
WARN [01-22|07:56:37.854] Rewinding blockchain target=9215
WARN [01-22|07:56:37.854] Rewinding blockchain-end target number=9215
INFO [01-22|07:56:37.855] shard:1 init from root block end=9215
INFO [01-22|07:56:37.855] shard:1 InitFromRootBlock=end
INFO [01-22|07:56:37.855] slave conn manager, add connect to slave add target=172.16.1.8:48000
INFO [01-22|07:56:37.856] slave conn manager, add connect to slave add target=172.16.1.8:48000
INFO [01-22|07:56:37.856] slave conn manager, add connect to slave add target=172.16.1.39:48000
INFO [01-22|07:56:37.857] slave ping response request op=2
INFO [01-22|07:56:37.857] slave conn manager, add connect to slave add target=172.16.1.39:48000
INFO [01-22|07:56:37.857] slave master info response master endpoint=172.16.1.17:38191
INFO [01-22|07:59:46.708] db-operation Write MinorBlock branch=1 height=9216 hash=0x247e8a40197fee5ea333abd69eb9cbb8344bce0356012b3f4c1ec0fc433f24fd len(tx)=0
fatal error: unexpected signal during runtime execution
[signal SIGSEGV: segmentation violation code=0x1 addr=0x38 pc=0x7fa1f48f9e4e]
runtime stack:
runtime.throw(0xe5793f, 0x2a)
/usr/local/go/src/runtime/panic.go:617 +0x72
runtime.sigpanic()
/usr/local/go/src/runtime/signal_unix.go:374 +0x4a9
goroutine 36 [syscall]:
runtime.cgocall(0xc1eab5, 0xc008c69c60, 0x7fa1d8238c70)
/usr/local/go/src/runtime/cgocall.go:128 +0x5b fp=0xc008c69c30 sp=0xc008c69bf8 pc=0x4151bb
github.com/tecbot/gorocksdb._Cfunc_rocksdb_writebatch_clear(0x0)
_cgo_gotypes.go:4312 +0x41 fp=0xc008c69c60 sp=0xc008c69c30 pc=0xa82aa1
github.com/tecbot/gorocksdb.(*WriteBatch).Clear.func1(0xc00e0d64e8)
/go/pkg/mod/github.com/tecbot/gorocksdb@v0.0.0-20181010114359-8752a9433481/write_batch.go:94 +0x5e fp=0xc008c69ca0 sp=0xc008c69c60 pc=0xa9b3be
github.com/tecbot/gorocksdb.(*WriteBatch).Clear(0xc00e0d64e8)
/go/pkg/mod/github.com/tecbot/gorocksdb@v0.0.0-20181010114359-8752a9433481/write_batch.go:94 +0x2b fp=0xc008c69cb8 sp=0xc008c69ca0 pc=0xa8ee6b
github.com/QuarkChain/goquarkchain/qkcdb.(*rdbBatch).Reset(0xc00d8fbee0)
/go/src/github.com/QuarkChain/goquarkchain/qkcdb/database.go:170 +0x2f fp=0xc008c69cd0 sp=0xc008c69cb8 pc=0xa9d2cf
github.com/ethereum/go-ethereum/trie.(*Database).commit(0xc0002ae1e0, 0x8a85952d68a3096e, 0xfb7ea0d8da8e4e86, 0x4e92eee4ebefcec1, 0xea6e8ba2589eaa17, 0xfdd0c0, 0xc00d8fbee0, 0x0, 0x0)
/go/pkg/mod/github.com/ethereum/go-ethereum@v1.8.20/trie/database.go:749 +0x298 fp=0xc008c69db0 sp=0xc008c69cd0 pc=0x6f5548
...
github.com/ethereum/go-ethereum/trie.(*Database).Commit(0xc0002ae1e0, 0x47ffdcbd38722293, 0xa042b6592f067cab, 0xde765f7a2e66c7db, 0x827d02b6cb5d0bf7, 0x0, 0x0, 0x0)
/go/pkg/mod/github.com/ethereum/go-ethereum@v1.8.20/trie/database.go:689 +0x52b fp=0xc008c6a8a8 sp=0xc008c6a670 pc=0x6f481b
github.com/QuarkChain/goquarkchain/core.(*MinorBlockChain).WriteBlockWithState(0xc0026ea000, 0xc002d8be60, 0xc00ed8d8e0, 0x2, 0x2, 0xc00b7a42c0, 0xc00ed8d870, 0x2, 0x2, 0x1, ...)
/go/src/github.com/QuarkChain/goquarkchain/core/minorblockchain.go:921 +0xbbd fp=0xc008c6ab70 sp=0xc008c6a8a8 pc=0xb2866d
github.com/QuarkChain/goquarkchain/core.(*MinorBlockChain).insertChain(0xc0026ea000, 0xc00ed8d2a0, 0x1, 0x1, 0x1, 0x0, 0xc005b37200, 0x42942f, 0x7fa1e55c3ce0, 0x0, ...)
/go/src/github.com/QuarkChain/goquarkchain/core/minorblockchain.go:1151 +0x9d4 fp=0xc008c6b170 sp=0xc008c6ab70 pc=0xb2a8e4
github.com/QuarkChain/goquarkchain/core.(*MinorBlockChain).InsertChainForDeposits(0xc0026ea000, 0xc00ed8d2a0, 0x1, 0x1, 0x3f2b015603ce4b00, 0xfd243f43fcc01e4c, 0x8163e3432ee33100, 0xca30ca728342d1e9, 0xf1c97c8f62b9495d, 0x4b19f0d1d1709210, ...)
/go/src/github.com/QuarkChain/goquarkchain/core/minorblockchain.go:1015 +0x9cf fp=0xc008c6b590 sp=0xc008c6b170 pc=0xb298bf
github.com/QuarkChain/goquarkchain/cluster/shard.(*ShardBackend).AddMinorBlock(0xc0002ae000, 0xc002d8be60, 0x0, 0x0)
/go/src/github.com/QuarkChain/goquarkchain/cluster/shard/api_backend.go:275 +0x264 fp=0xc008c6b8b0 sp=0xc008c6b590 pc=0xbd5964
github.com/QuarkChain/goquarkchain/cluster/shard.(*ShardBackend).NewMinorBlock(0xc0002ae000, 0x0, 0x0, 0xc002d8be60, 0x41, 0x81)
/go/src/github.com/QuarkChain/goquarkchain/cluster/shard/api_backend.go:262 +0xa8d fp=0xc008c6bc90 sp=0xc008c6b8b0 pc=0xbd4ded
github.com/QuarkChain/goquarkchain/cluster/shard.(*ShardBackend).InsertMinedBlock(0xc0002ae000, 0xfe56a0, 0xc002d8be60, 0x0, 0x0)
/go/src/github.com/QuarkChain/goquarkchain/cluster/shard/api_backend.go:409 +0xa5 fp=0xc008c6bcd8 sp=0xc008c6bc90 pc=0xbd7605
github.com/QuarkChain/goquarkchain/cluster/miner.(*Miner).mainLoop(0xc00010a400)
/go/src/github.com/QuarkChain/goquarkchain/cluster/miner/miner.go:130 +0x339 fp=0xc008c6bfd8 sp=0xc008c6bcd8 pc=0xb80829
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1337 +0x1 fp=0xc008c6bfe0 sp=0xc008c6bfd8 pc=0x46f2c1
created by github.com/QuarkChain/goquarkchain/cluster/miner.New
/go/src/github.com/QuarkChain/goquarkchain/cluster/miner/miner.go:59 +0x1c9
goroutine 1 [chan receive, 5 minutes]:
github.com/QuarkChain/goquarkchain/cluster/service.(*Node).Wait(0xc0000fc780)
/go/src/github.com/QuarkChain/goquarkchain/cluster/service/node.go:498 +0x77
main.cluster(0xc0000dd4a0, 0x18, 0xc000219c50)
/go/src/github.com/QuarkChain/goquarkchain/cmd/cluster/main.go:130 +0x144
gopkg.in/urfave/cli%2ev1.HandleAction(0xce2120, 0xe72840, 0xc0000dd4a0, 0xc0000dd4a0, 0xc000219f58)
/go/pkg/mod/gopkg.in/urfave/cli.v1@v1.20.0/app.go:490 +0xc8
gopkg.in/urfave/cli%2ev1.(*App).Run(0xc0000c5520, 0xc0000bc000, 0x5, 0x5, 0x0, 0x0)
/go/pkg/mod/gopkg.in/urfave/cli.v1@v1.20.0/app.go:264 +0x590
main.main()
/go/src/github.com/QuarkChain/goquarkchain/cmd/cluster/main.go:115 +0x55
goroutine 7 [syscall, 5 minutes]:
os/signal.signal_recv(0x0)
/usr/local/go/src/runtime/sigqueue.go:139 +0x9c
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.init.0
/usr/local/go/src/os/signal/signal_unix.go:29 +0x41
goroutine 8 [chan receive, 4 minutes]:
github.com/QuarkChain/goquarkchain/core.(*txSenderCacher).cache(0xc0000447d0)
/go/src/github.com/QuarkChain/goquarkchain/core/tx_cacher.go:62 +0x9b
created by github.com/QuarkChain/goquarkchain/core.newTxSenderCacher
/go/src/github.com/QuarkChain/goquarkchain/core/tx_cacher.go:54 +0x9f
goroutine 9 [chan receive, 4 minutes]:
github.com/QuarkChain/goquarkchain/core.(*txSenderCacher).cache(0xc0000447d0)
/go/src/github.com/QuarkChain/goquarkchain/core/tx_cacher.go:62 +0x9b
created by github.com/QuarkChain/goquarkchain/core.newTxSenderCacher
/go/src/github.com/QuarkChain/goquarkchain/core/tx_cacher.go:54 +0x9f
goroutine 34 [select]:
github.com/QuarkChain/goquarkchain/core.(*MinorBlockChain).update(0xc0026ea000)
/go/src/github.com/QuarkChain/goquarkchain/core/minorblockchain.go:1478 +0xfc
created by github.com/QuarkChain/goquarkchain/core.NewMinorBlockChain
/go/src/github.com/QuarkChain/goquarkchain/core/minorblockchain.go:236 +0xa33
goroutine 25 [IO wait, 3 minutes]:
internal/poll.runtime_pollWait(0x7fa1f4f2c2d8, 0x72, 0x0)
/usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc00010a018, 0x72, 0x0, 0x0, 0xe36df3)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc00010a000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:384 +0x1ba
net.(*netFD).accept(0xc00010a000, 0xc00db8e001, 0x0, 0x0)
/usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc0000b6108, 0xc000069ec0, 0xc000069ec8, 0x18)
/usr/local/go/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc0000b6108, 0xe72360, 0xc000001e00, 0xfe3200, 0xc00db8e0a0)
/usr/local/go/src/net/tcpsock.go:260 +0x48
google.golang.org/grpc.(*Server).Serve(0xc000001e00, 0xfd7400, 0xc0000b6108, 0x0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/server.go:561 +0x1e9
created by github.com/QuarkChain/goquarkchain/cluster/rpc.StartGRPCServer
/go/src/github.com/QuarkChain/goquarkchain/cluster/rpc/grpc_server.go:38 +0x161
goroutine 26 [chan receive, 5 minutes]:
github.com/QuarkChain/goquarkchain/cmd/utils.StartService.func1(0xc0000fc780)
/go/src/github.com/QuarkChain/goquarkchain/cmd/utils/cmd.go:43 +0xbf
created by github.com/QuarkChain/goquarkchain/cmd/utils.StartService
/go/src/github.com/QuarkChain/goquarkchain/cmd/utils/cmd.go:40 +0xad
goroutine 14 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00014a100, 0x1, 0x0, 0x0, 0x0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/controlbuf.go:317 +0x104
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000082900, 0x0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/controlbuf.go:435 +0x1b6
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc000506180)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:276 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:273 +0xfba
goroutine 15 [select, 5 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000506180)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:921 +0x1ed
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:282 +0xfdf
goroutine 16 [IO wait]:
internal/poll.runtime_pollWait(0x7fa1f4f2c208, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc0001c0198, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0001c0180, 0xc00034e000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc0001c0180, 0xc00034e000, 0x8000, 0x8000, 0x0, 0x800010601, 0x0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000010008, 0xc00034e000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:177 +0x69
bufio.(*Reader).Read(0xc0000828a0, 0xc000366038, 0x9, 0x9, 0xc0001aee00, 0x7fa1f4f836d0, 0x0)
/usr/local/go/src/bufio/bufio.go:223 +0x23e
io.ReadAtLeast(0xfc2580, 0xc0000828a0, 0xc000366038, 0x9, 0x9, 0x9, 0xbf821e1caa354c46, 0x45f1e84e85, 0x19067a0)
/usr/local/go/src/io/io.go:310 +0x88
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc000366038, 0x9, 0x9, 0xfc2580, 0xc0000828a0, 0x0, 0x0, 0xc0000bc0a0, 0x7070e0910100402)
/go/pkg/mod/golang.org/x/net@v0.0.0-20190404232315-eb5bcb51f2a3/http2/frame.go:237 +0x88
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000366000, 0xc027eb11a0, 0xc027eb11a0, 0x0, 0x0)
/go/pkg/mod/golang.org/x/net@v0.0.0-20190404232315-eb5bcb51f2a3/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000506180, 0xc000144600, 0xe723c0)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:429 +0x7c
google.golang.org/grpc.(*Server).serveStreams(0xc000001e00, 0xfe43a0, 0xc000506180)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/server.go:686 +0xdd
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000001e00, 0xfe43a0, 0xc000506180)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/server.go:648 +0x43
created by google.golang.org/grpc.(*Server).handleRawConn
/go/pkg/mod/google.golang.org/grpc@v1.19.1/server.go:647 +0x583
goroutine 107 [select]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00ad64a40)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/balancer_conn_wrappers.go:122 +0x110
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.19.1/balancer_conn_wrappers.go:113 +0x14f
goroutine 39 [select, 3 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00014ab40)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/balancer_conn_wrappers.go:122 +0x110
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.19.1/balancer_conn_wrappers.go:113 +0x14f
goroutine 35 [select, 4 minutes]:
github.com/QuarkChain/goquarkchain/cluster/sync.(*synchronizer).loop(0xc0000cd380)
/go/src/github.com/QuarkChain/goquarkchain/cluster/sync/sync.go:105 +0x19c
created by github.com/QuarkChain/goquarkchain/cluster/sync.NewSynchronizer
/go/src/github.com/QuarkChain/goquarkchain/cluster/sync/sync.go:138 +0xf7
goroutine 31 [chan receive, 4 minutes]:
github.com/QuarkChain/goquarkchain/cluster/sync.(*synchronizer).loop.func1(0xc0000cd380)
/go/src/github.com/QuarkChain/goquarkchain/cluster/sync/sync.go:82 +0x104
created by github.com/QuarkChain/goquarkchain/cluster/sync.(*synchronizer).loop
/go/src/github.com/QuarkChain/goquarkchain/cluster/sync/sync.go:80 +0x57
goroutine 54 [select, 3 minutes]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00007e080, 0x1, 0x0, 0x0, 0x0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/controlbuf.go:317 +0x104
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00e52e3c0, 0x0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/controlbuf.go:435 +0x1b6
google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc00ea4c000)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_client.go:330 +0x7b
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_client.go:328 +0xeb2
goroutine 33 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00ad65240, 0x1, 0x0, 0x0, 0x0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/controlbuf.go:317 +0x104
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0000831a0, 0x0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/controlbuf.go:435 +0x1b6
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc000507b00)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:276 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:273 +0xfba
goroutine 50 [select, 4 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000507b00)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:921 +0x1ed
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:282 +0xfdf
goroutine 51 [IO wait]:
internal/poll.runtime_pollWait(0x7fa1f4f2c138, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc00a70dd18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00a70dd00, 0xc00adfa000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc00a70dd00, 0xc00adfa000, 0x8000, 0x8000, 0x0, 0x8, 0x0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00ac97b48, 0xc00adfa000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:177 +0x69
bufio.(*Reader).Read(0xc000083140, 0xc000366738, 0x9, 0x9, 0x18, 0x18, 0xc00d974320)
/usr/local/go/src/bufio/bufio.go:223 +0x23e
io.ReadAtLeast(0xfc2580, 0xc000083140, 0xc000366738, 0x9, 0x9, 0x9, 0x46b6f0, 0x20, 0xc0006c2d98)
/usr/local/go/src/io/io.go:310 +0x88
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc000366738, 0x9, 0x9, 0xfc2580, 0xc000083140, 0x0, 0x0, 0xbf821e1caa344ce2, 0x45f1e73803)
/go/pkg/mod/golang.org/x/net@v0.0.0-20190404232315-eb5bcb51f2a3/http2/frame.go:237 +0x88
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000366700, 0xc027eb10e0, 0xc027eb10e0, 0x0, 0x0)
/go/pkg/mod/golang.org/x/net@v0.0.0-20190404232315-eb5bcb51f2a3/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000507b00, 0xc00adea870, 0xe723c0)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:429 +0x7c
google.golang.org/grpc.(*Server).serveStreams(0xc000001e00, 0xfe43a0, 0xc000507b00)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/server.go:686 +0xdd
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000001e00, 0xfe43a0, 0xc000507b00)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/server.go:648 +0x43
created by google.golang.org/grpc.(*Server).handleRawConn
/go/pkg/mod/google.golang.org/grpc@v1.19.1/server.go:647 +0x583
goroutine 37 [select, 3 minutes]:
github.com/QuarkChain/goquarkchain/core.(*TxPool).scheduleReorgLoop(0xc01fb712c0)
/go/src/github.com/QuarkChain/goquarkchain/core/tx_pool.go:813 +0x227
created by github.com/QuarkChain/goquarkchain/core.NewTxPool
/go/src/github.com/QuarkChain/goquarkchain/core/tx_pool.go:216 +0x5c3
goroutine 38 [select]:
github.com/QuarkChain/goquarkchain/core.(*TxPool).loop(0xc01fb712c0)
/go/src/github.com/QuarkChain/goquarkchain/core/tx_pool.go:244 +0x250
created by github.com/QuarkChain/goquarkchain/core.NewTxPool
/go/src/github.com/QuarkChain/goquarkchain/core/tx_pool.go:221 +0x651
goroutine 40 [chan receive, 3 minutes]:
google.golang.org/grpc.(*addrConn).resetTransport(0xc0001e6280)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/clientconn.go:1067 +0x9ec
created by google.golang.org/grpc.(*addrConn).connect
/go/pkg/mod/google.golang.org/grpc@v1.19.1/clientconn.go:684 +0xb6
goroutine 53 [IO wait, 3 minutes]:
internal/poll.runtime_pollWait(0x7fa1f4f2c068, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc001b9a298, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc001b9a280, 0xc00e668000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc001b9a280, 0xc00e668000, 0x8000, 0x8000, 0x0, 0x800010601, 0x0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00db8e000, 0xc00e668000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:177 +0x69
bufio.(*Reader).Read(0xc00e52e360, 0xc00e97c038, 0x9, 0x9, 0xc0001aea80, 0x7fa1f4f836d0, 0x0)
/usr/local/go/src/bufio/bufio.go:223 +0x23e
io.ReadAtLeast(0xfc2580, 0xc00e52e360, 0xc00e97c038, 0x9, 0x9, 0x9, 0x9a1375, 0xc00a8ac4cc, 0xc00006de38)
/usr/local/go/src/io/io.go:310 +0x88
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc00e97c038, 0x9, 0x9, 0xfc2580, 0xc00e52e360, 0x0, 0xc000000000, 0x19f9802abc, 0x19067a0)
/go/pkg/mod/golang.org/x/net@v0.0.0-20190404232315-eb5bcb51f2a3/http2/frame.go:237 +0x88
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00e97c000, 0xc00a8ac4c0, 0xc00a8ac4c0, 0x0, 0x0)
/go/pkg/mod/golang.org/x/net@v0.0.0-20190404232315-eb5bcb51f2a3/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00ea4c000)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_client.go:1223 +0x168
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_client.go:286 +0xd15
Report from customer:
I made many transactions with 20ms interval, but the network shut down again. so I restart whole network and I got the error from slave node. I attached the logs following :
=== log start ===
INFO [01-22|07:55:12.156] db-operation Write RootBlock height=4620 hash=48175b…f71946
INFO [01-22|07:55:12.156] shard:1 tipMinor=9215 hash=0x1bf8061b3ccfd05cd411f18e57072c6ce4a3dce19f4f2d0730e501c44f9dfa26 mete.root=0xdbcd9a42a7b78a68a05418db0499a91144c4bfd1431dc64643d6ff4fa6b20f6a rootBlock=4620 rootTip=4620
WARN [01-22|07:55:12.156] shard:1 miss trie block=9215 block.hash=0x1bf8061b3ccfd05cd411f18e57072c6ce4a3dce19f4f2d0730e501c44f9dfa26 currNumber=9215 currHash=0x1bf8061b3ccfd05cd411f18e57072c6ce4a3dce19f4f2d0730e501c44f9dfa26
INFO [01-22|07:55:12.377] reRunBlockWithState blockchain to past state number=8704 hash=0xc761a2c5db8fbf7524950c6158a9d44154bfda14ff3e3ad3555f78a60edbdbce
INFO [01-22|07:55:15.147] slave ping response request op=2
INFO [01-22|07:56:00.195] shard:1 commit trie number=8704 hash=0xc761a2c5db8fbf7524950c6158a9d44154bfda14ff3e3ad3555f78a60edbdbce root=0xe1d70098901cf2e24b6990bc7b6168aad1bd32ccd2964b41849763f47c5d8233
WARN [01-22|07:56:37.854] shard:1 miss trie reRun time=85.698 currentBlock=9215 currHash=0x1bf8061b3ccfd05cd411f18e57072c6ce4a3dce19f4f2d0730e501c44f9dfa26
WARN [01-22|07:56:37.854] reorg same chain curr=9215 curr.Hash=0x1bf8061b3ccfd05cd411f18e57072c6ce4a3dce19f4f2d0730e501c44f9dfa26 newBlock=9215 newBlock's hash=1bf806…9dfa26 newBlock=false
WARN [01-22|07:56:37.854] Rewinding blockchain target=9215
WARN [01-22|07:56:37.854] Rewinding blockchain-end target number=9215
INFO [01-22|07:56:37.855] shard:1 init from root block end=9215
INFO [01-22|07:56:37.855] shard:1 InitFromRootBlock=end
INFO [01-22|07:56:37.855] slave conn manager, add connect to slave add target=172.16.1.8:48000
INFO [01-22|07:56:37.856] slave conn manager, add connect to slave add target=172.16.1.8:48000
INFO [01-22|07:56:37.856] slave conn manager, add connect to slave add target=172.16.1.39:48000
INFO [01-22|07:56:37.857] slave ping response request op=2
INFO [01-22|07:56:37.857] slave conn manager, add connect to slave add target=172.16.1.39:48000
INFO [01-22|07:56:37.857] slave master info response master endpoint=172.16.1.17:38191
INFO [01-22|07:59:46.708] db-operation Write MinorBlock branch=1 height=9216 hash=0x247e8a40197fee5ea333abd69eb9cbb8344bce0356012b3f4c1ec0fc433f24fd len(tx)=0
fatal error: unexpected signal during runtime execution
[signal SIGSEGV: segmentation violation code=0x1 addr=0x38 pc=0x7fa1f48f9e4e]
runtime stack:
runtime.throw(0xe5793f, 0x2a)
/usr/local/go/src/runtime/panic.go:617 +0x72
runtime.sigpanic()
/usr/local/go/src/runtime/signal_unix.go:374 +0x4a9
goroutine 36 [syscall]:
runtime.cgocall(0xc1eab5, 0xc008c69c60, 0x7fa1d8238c70)
/usr/local/go/src/runtime/cgocall.go:128 +0x5b fp=0xc008c69c30 sp=0xc008c69bf8 pc=0x4151bb
github.com/tecbot/gorocksdb._Cfunc_rocksdb_writebatch_clear(0x0)
_cgo_gotypes.go:4312 +0x41 fp=0xc008c69c60 sp=0xc008c69c30 pc=0xa82aa1
github.com/tecbot/gorocksdb.(*WriteBatch).Clear.func1(0xc00e0d64e8)
/go/pkg/mod/github.com/tecbot/gorocksdb@v0.0.0-20181010114359-8752a9433481/write_batch.go:94 +0x5e fp=0xc008c69ca0 sp=0xc008c69c60 pc=0xa9b3be
github.com/tecbot/gorocksdb.(*WriteBatch).Clear(0xc00e0d64e8)
/go/pkg/mod/github.com/tecbot/gorocksdb@v0.0.0-20181010114359-8752a9433481/write_batch.go:94 +0x2b fp=0xc008c69cb8 sp=0xc008c69ca0 pc=0xa8ee6b
github.com/QuarkChain/goquarkchain/qkcdb.(*rdbBatch).Reset(0xc00d8fbee0)
/go/src/github.com/QuarkChain/goquarkchain/qkcdb/database.go:170 +0x2f fp=0xc008c69cd0 sp=0xc008c69cb8 pc=0xa9d2cf
github.com/ethereum/go-ethereum/trie.(*Database).commit(0xc0002ae1e0, 0x8a85952d68a3096e, 0xfb7ea0d8da8e4e86, 0x4e92eee4ebefcec1, 0xea6e8ba2589eaa17, 0xfdd0c0, 0xc00d8fbee0, 0x0, 0x0)
/go/pkg/mod/github.com/ethereum/go-ethereum@v1.8.20/trie/database.go:749 +0x298 fp=0xc008c69db0 sp=0xc008c69cd0 pc=0x6f5548
...
github.com/ethereum/go-ethereum/trie.(*Database).Commit(0xc0002ae1e0, 0x47ffdcbd38722293, 0xa042b6592f067cab, 0xde765f7a2e66c7db, 0x827d02b6cb5d0bf7, 0x0, 0x0, 0x0)
/go/pkg/mod/github.com/ethereum/go-ethereum@v1.8.20/trie/database.go:689 +0x52b fp=0xc008c6a8a8 sp=0xc008c6a670 pc=0x6f481b
github.com/QuarkChain/goquarkchain/core.(*MinorBlockChain).WriteBlockWithState(0xc0026ea000, 0xc002d8be60, 0xc00ed8d8e0, 0x2, 0x2, 0xc00b7a42c0, 0xc00ed8d870, 0x2, 0x2, 0x1, ...)
/go/src/github.com/QuarkChain/goquarkchain/core/minorblockchain.go:921 +0xbbd fp=0xc008c6ab70 sp=0xc008c6a8a8 pc=0xb2866d
github.com/QuarkChain/goquarkchain/core.(*MinorBlockChain).insertChain(0xc0026ea000, 0xc00ed8d2a0, 0x1, 0x1, 0x1, 0x0, 0xc005b37200, 0x42942f, 0x7fa1e55c3ce0, 0x0, ...)
/go/src/github.com/QuarkChain/goquarkchain/core/minorblockchain.go:1151 +0x9d4 fp=0xc008c6b170 sp=0xc008c6ab70 pc=0xb2a8e4
github.com/QuarkChain/goquarkchain/core.(*MinorBlockChain).InsertChainForDeposits(0xc0026ea000, 0xc00ed8d2a0, 0x1, 0x1, 0x3f2b015603ce4b00, 0xfd243f43fcc01e4c, 0x8163e3432ee33100, 0xca30ca728342d1e9, 0xf1c97c8f62b9495d, 0x4b19f0d1d1709210, ...)
/go/src/github.com/QuarkChain/goquarkchain/core/minorblockchain.go:1015 +0x9cf fp=0xc008c6b590 sp=0xc008c6b170 pc=0xb298bf
github.com/QuarkChain/goquarkchain/cluster/shard.(*ShardBackend).AddMinorBlock(0xc0002ae000, 0xc002d8be60, 0x0, 0x0)
/go/src/github.com/QuarkChain/goquarkchain/cluster/shard/api_backend.go:275 +0x264 fp=0xc008c6b8b0 sp=0xc008c6b590 pc=0xbd5964
github.com/QuarkChain/goquarkchain/cluster/shard.(*ShardBackend).NewMinorBlock(0xc0002ae000, 0x0, 0x0, 0xc002d8be60, 0x41, 0x81)
/go/src/github.com/QuarkChain/goquarkchain/cluster/shard/api_backend.go:262 +0xa8d fp=0xc008c6bc90 sp=0xc008c6b8b0 pc=0xbd4ded
github.com/QuarkChain/goquarkchain/cluster/shard.(*ShardBackend).InsertMinedBlock(0xc0002ae000, 0xfe56a0, 0xc002d8be60, 0x0, 0x0)
/go/src/github.com/QuarkChain/goquarkchain/cluster/shard/api_backend.go:409 +0xa5 fp=0xc008c6bcd8 sp=0xc008c6bc90 pc=0xbd7605
github.com/QuarkChain/goquarkchain/cluster/miner.(*Miner).mainLoop(0xc00010a400)
/go/src/github.com/QuarkChain/goquarkchain/cluster/miner/miner.go:130 +0x339 fp=0xc008c6bfd8 sp=0xc008c6bcd8 pc=0xb80829
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1337 +0x1 fp=0xc008c6bfe0 sp=0xc008c6bfd8 pc=0x46f2c1
created by github.com/QuarkChain/goquarkchain/cluster/miner.New
/go/src/github.com/QuarkChain/goquarkchain/cluster/miner/miner.go:59 +0x1c9
goroutine 1 [chan receive, 5 minutes]:
github.com/QuarkChain/goquarkchain/cluster/service.(*Node).Wait(0xc0000fc780)
/go/src/github.com/QuarkChain/goquarkchain/cluster/service/node.go:498 +0x77
main.cluster(0xc0000dd4a0, 0x18, 0xc000219c50)
/go/src/github.com/QuarkChain/goquarkchain/cmd/cluster/main.go:130 +0x144
gopkg.in/urfave/cli%2ev1.HandleAction(0xce2120, 0xe72840, 0xc0000dd4a0, 0xc0000dd4a0, 0xc000219f58)
/go/pkg/mod/gopkg.in/urfave/cli.v1@v1.20.0/app.go:490 +0xc8
gopkg.in/urfave/cli%2ev1.(*App).Run(0xc0000c5520, 0xc0000bc000, 0x5, 0x5, 0x0, 0x0)
/go/pkg/mod/gopkg.in/urfave/cli.v1@v1.20.0/app.go:264 +0x590
main.main()
/go/src/github.com/QuarkChain/goquarkchain/cmd/cluster/main.go:115 +0x55
goroutine 7 [syscall, 5 minutes]:
os/signal.signal_recv(0x0)
/usr/local/go/src/runtime/sigqueue.go:139 +0x9c
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:23 +0x22
created by os/signal.init.0
/usr/local/go/src/os/signal/signal_unix.go:29 +0x41
goroutine 8 [chan receive, 4 minutes]:
github.com/QuarkChain/goquarkchain/core.(*txSenderCacher).cache(0xc0000447d0)
/go/src/github.com/QuarkChain/goquarkchain/core/tx_cacher.go:62 +0x9b
created by github.com/QuarkChain/goquarkchain/core.newTxSenderCacher
/go/src/github.com/QuarkChain/goquarkchain/core/tx_cacher.go:54 +0x9f
goroutine 9 [chan receive, 4 minutes]:
github.com/QuarkChain/goquarkchain/core.(*txSenderCacher).cache(0xc0000447d0)
/go/src/github.com/QuarkChain/goquarkchain/core/tx_cacher.go:62 +0x9b
created by github.com/QuarkChain/goquarkchain/core.newTxSenderCacher
/go/src/github.com/QuarkChain/goquarkchain/core/tx_cacher.go:54 +0x9f
goroutine 34 [select]:
github.com/QuarkChain/goquarkchain/core.(*MinorBlockChain).update(0xc0026ea000)
/go/src/github.com/QuarkChain/goquarkchain/core/minorblockchain.go:1478 +0xfc
created by github.com/QuarkChain/goquarkchain/core.NewMinorBlockChain
/go/src/github.com/QuarkChain/goquarkchain/core/minorblockchain.go:236 +0xa33
goroutine 25 [IO wait, 3 minutes]:
internal/poll.runtime_pollWait(0x7fa1f4f2c2d8, 0x72, 0x0)
/usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc00010a018, 0x72, 0x0, 0x0, 0xe36df3)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Accept(0xc00010a000, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:384 +0x1ba
net.(*netFD).accept(0xc00010a000, 0xc00db8e001, 0x0, 0x0)
/usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc0000b6108, 0xc000069ec0, 0xc000069ec8, 0x18)
/usr/local/go/src/net/tcpsock_posix.go:139 +0x32
net.(*TCPListener).Accept(0xc0000b6108, 0xe72360, 0xc000001e00, 0xfe3200, 0xc00db8e0a0)
/usr/local/go/src/net/tcpsock.go:260 +0x48
google.golang.org/grpc.(*Server).Serve(0xc000001e00, 0xfd7400, 0xc0000b6108, 0x0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/server.go:561 +0x1e9
created by github.com/QuarkChain/goquarkchain/cluster/rpc.StartGRPCServer
/go/src/github.com/QuarkChain/goquarkchain/cluster/rpc/grpc_server.go:38 +0x161
goroutine 26 [chan receive, 5 minutes]:
github.com/QuarkChain/goquarkchain/cmd/utils.StartService.func1(0xc0000fc780)
/go/src/github.com/QuarkChain/goquarkchain/cmd/utils/cmd.go:43 +0xbf
created by github.com/QuarkChain/goquarkchain/cmd/utils.StartService
/go/src/github.com/QuarkChain/goquarkchain/cmd/utils/cmd.go:40 +0xad
goroutine 14 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00014a100, 0x1, 0x0, 0x0, 0x0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/controlbuf.go:317 +0x104
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc000082900, 0x0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/controlbuf.go:435 +0x1b6
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc000506180)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:276 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:273 +0xfba
goroutine 15 [select, 5 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000506180)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:921 +0x1ed
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:282 +0xfdf
goroutine 16 [IO wait]:
internal/poll.runtime_pollWait(0x7fa1f4f2c208, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc0001c0198, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0001c0180, 0xc00034e000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc0001c0180, 0xc00034e000, 0x8000, 0x8000, 0x0, 0x800010601, 0x0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc000010008, 0xc00034e000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:177 +0x69
bufio.(*Reader).Read(0xc0000828a0, 0xc000366038, 0x9, 0x9, 0xc0001aee00, 0x7fa1f4f836d0, 0x0)
/usr/local/go/src/bufio/bufio.go:223 +0x23e
io.ReadAtLeast(0xfc2580, 0xc0000828a0, 0xc000366038, 0x9, 0x9, 0x9, 0xbf821e1caa354c46, 0x45f1e84e85, 0x19067a0)
/usr/local/go/src/io/io.go:310 +0x88
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc000366038, 0x9, 0x9, 0xfc2580, 0xc0000828a0, 0x0, 0x0, 0xc0000bc0a0, 0x7070e0910100402)
/go/pkg/mod/golang.org/x/net@v0.0.0-20190404232315-eb5bcb51f2a3/http2/frame.go:237 +0x88
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000366000, 0xc027eb11a0, 0xc027eb11a0, 0x0, 0x0)
/go/pkg/mod/golang.org/x/net@v0.0.0-20190404232315-eb5bcb51f2a3/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000506180, 0xc000144600, 0xe723c0)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:429 +0x7c
google.golang.org/grpc.(*Server).serveStreams(0xc000001e00, 0xfe43a0, 0xc000506180)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/server.go:686 +0xdd
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000001e00, 0xfe43a0, 0xc000506180)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/server.go:648 +0x43
created by google.golang.org/grpc.(*Server).handleRawConn
/go/pkg/mod/google.golang.org/grpc@v1.19.1/server.go:647 +0x583
goroutine 107 [select]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00ad64a40)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/balancer_conn_wrappers.go:122 +0x110
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.19.1/balancer_conn_wrappers.go:113 +0x14f
goroutine 39 [select, 3 minutes]:
google.golang.org/grpc.(*ccBalancerWrapper).watcher(0xc00014ab40)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/balancer_conn_wrappers.go:122 +0x110
created by google.golang.org/grpc.newCCBalancerWrapper
/go/pkg/mod/google.golang.org/grpc@v1.19.1/balancer_conn_wrappers.go:113 +0x14f
goroutine 35 [select, 4 minutes]:
github.com/QuarkChain/goquarkchain/cluster/sync.(*synchronizer).loop(0xc0000cd380)
/go/src/github.com/QuarkChain/goquarkchain/cluster/sync/sync.go:105 +0x19c
created by github.com/QuarkChain/goquarkchain/cluster/sync.NewSynchronizer
/go/src/github.com/QuarkChain/goquarkchain/cluster/sync/sync.go:138 +0xf7
goroutine 31 [chan receive, 4 minutes]:
github.com/QuarkChain/goquarkchain/cluster/sync.(*synchronizer).loop.func1(0xc0000cd380)
/go/src/github.com/QuarkChain/goquarkchain/cluster/sync/sync.go:82 +0x104
created by github.com/QuarkChain/goquarkchain/cluster/sync.(*synchronizer).loop
/go/src/github.com/QuarkChain/goquarkchain/cluster/sync/sync.go:80 +0x57
goroutine 54 [select, 3 minutes]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00007e080, 0x1, 0x0, 0x0, 0x0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/controlbuf.go:317 +0x104
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc00e52e3c0, 0x0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/controlbuf.go:435 +0x1b6
google.golang.org/grpc/internal/transport.newHTTP2Client.func3(0xc00ea4c000)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_client.go:330 +0x7b
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_client.go:328 +0xeb2
goroutine 33 [select]:
google.golang.org/grpc/internal/transport.(*controlBuffer).get(0xc00ad65240, 0x1, 0x0, 0x0, 0x0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/controlbuf.go:317 +0x104
google.golang.org/grpc/internal/transport.(*loopyWriter).run(0xc0000831a0, 0x0, 0x0)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/controlbuf.go:435 +0x1b6
google.golang.org/grpc/internal/transport.newHTTP2Server.func2(0xc000507b00)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:276 +0xcb
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:273 +0xfba
goroutine 50 [select, 4 minutes]:
google.golang.org/grpc/internal/transport.(*http2Server).keepalive(0xc000507b00)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:921 +0x1ed
created by google.golang.org/grpc/internal/transport.newHTTP2Server
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:282 +0xfdf
goroutine 51 [IO wait]:
internal/poll.runtime_pollWait(0x7fa1f4f2c138, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc00a70dd18, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc00a70dd00, 0xc00adfa000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc00a70dd00, 0xc00adfa000, 0x8000, 0x8000, 0x0, 0x8, 0x0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00ac97b48, 0xc00adfa000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:177 +0x69
bufio.(*Reader).Read(0xc000083140, 0xc000366738, 0x9, 0x9, 0x18, 0x18, 0xc00d974320)
/usr/local/go/src/bufio/bufio.go:223 +0x23e
io.ReadAtLeast(0xfc2580, 0xc000083140, 0xc000366738, 0x9, 0x9, 0x9, 0x46b6f0, 0x20, 0xc0006c2d98)
/usr/local/go/src/io/io.go:310 +0x88
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc000366738, 0x9, 0x9, 0xfc2580, 0xc000083140, 0x0, 0x0, 0xbf821e1caa344ce2, 0x45f1e73803)
/go/pkg/mod/golang.org/x/net@v0.0.0-20190404232315-eb5bcb51f2a3/http2/frame.go:237 +0x88
golang.org/x/net/http2.(*Framer).ReadFrame(0xc000366700, 0xc027eb10e0, 0xc027eb10e0, 0x0, 0x0)
/go/pkg/mod/golang.org/x/net@v0.0.0-20190404232315-eb5bcb51f2a3/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Server).HandleStreams(0xc000507b00, 0xc00adea870, 0xe723c0)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_server.go:429 +0x7c
google.golang.org/grpc.(*Server).serveStreams(0xc000001e00, 0xfe43a0, 0xc000507b00)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/server.go:686 +0xdd
google.golang.org/grpc.(*Server).handleRawConn.func1(0xc000001e00, 0xfe43a0, 0xc000507b00)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/server.go:648 +0x43
created by google.golang.org/grpc.(*Server).handleRawConn
/go/pkg/mod/google.golang.org/grpc@v1.19.1/server.go:647 +0x583
goroutine 37 [select, 3 minutes]:
github.com/QuarkChain/goquarkchain/core.(*TxPool).scheduleReorgLoop(0xc01fb712c0)
/go/src/github.com/QuarkChain/goquarkchain/core/tx_pool.go:813 +0x227
created by github.com/QuarkChain/goquarkchain/core.NewTxPool
/go/src/github.com/QuarkChain/goquarkchain/core/tx_pool.go:216 +0x5c3
goroutine 38 [select]:
github.com/QuarkChain/goquarkchain/core.(*TxPool).loop(0xc01fb712c0)
/go/src/github.com/QuarkChain/goquarkchain/core/tx_pool.go:244 +0x250
created by github.com/QuarkChain/goquarkchain/core.NewTxPool
/go/src/github.com/QuarkChain/goquarkchain/core/tx_pool.go:221 +0x651
goroutine 40 [chan receive, 3 minutes]:
google.golang.org/grpc.(*addrConn).resetTransport(0xc0001e6280)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/clientconn.go:1067 +0x9ec
created by google.golang.org/grpc.(*addrConn).connect
/go/pkg/mod/google.golang.org/grpc@v1.19.1/clientconn.go:684 +0xb6
goroutine 53 [IO wait, 3 minutes]:
internal/poll.runtime_pollWait(0x7fa1f4f2c068, 0x72, 0xffffffffffffffff)
/usr/local/go/src/runtime/netpoll.go:182 +0x56
internal/poll.(*pollDesc).wait(0xc001b9a298, 0x72, 0x8000, 0x8000, 0xffffffffffffffff)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x9b
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc001b9a280, 0xc00e668000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:169 +0x19b
net.(*netFD).Read(0xc001b9a280, 0xc00e668000, 0x8000, 0x8000, 0x0, 0x800010601, 0x0)
/usr/local/go/src/net/fd_unix.go:202 +0x4f
net.(*conn).Read(0xc00db8e000, 0xc00e668000, 0x8000, 0x8000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:177 +0x69
bufio.(*Reader).Read(0xc00e52e360, 0xc00e97c038, 0x9, 0x9, 0xc0001aea80, 0x7fa1f4f836d0, 0x0)
/usr/local/go/src/bufio/bufio.go:223 +0x23e
io.ReadAtLeast(0xfc2580, 0xc00e52e360, 0xc00e97c038, 0x9, 0x9, 0x9, 0x9a1375, 0xc00a8ac4cc, 0xc00006de38)
/usr/local/go/src/io/io.go:310 +0x88
io.ReadFull(...)
/usr/local/go/src/io/io.go:329
golang.org/x/net/http2.readFrameHeader(0xc00e97c038, 0x9, 0x9, 0xfc2580, 0xc00e52e360, 0x0, 0xc000000000, 0x19f9802abc, 0x19067a0)
/go/pkg/mod/golang.org/x/net@v0.0.0-20190404232315-eb5bcb51f2a3/http2/frame.go:237 +0x88
golang.org/x/net/http2.(*Framer).ReadFrame(0xc00e97c000, 0xc00a8ac4c0, 0xc00a8ac4c0, 0x0, 0x0)
/go/pkg/mod/golang.org/x/net@v0.0.0-20190404232315-eb5bcb51f2a3/http2/frame.go:492 +0xa1
google.golang.org/grpc/internal/transport.(*http2Client).reader(0xc00ea4c000)
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_client.go:1223 +0x168
created by google.golang.org/grpc/internal/transport.newHTTP2Client
/go/pkg/mod/google.golang.org/grpc@v1.19.1/internal/transport/http2_client.go:286 +0xd15