goroutine profile: total 153 41 @ 0x465e1c 0x45e4be 0x4990bc 0x58cbba 0x58cc51 0x58e771 0x761d33 0x786725 0x942c4b 0x4dcf9a 0x4dd35b 0x94cae6 0x49e881 # 0x4990bb internal/poll.runtime_pollWait+0x3b /usr/local/go/src/runtime/netpoll.go:345 # 0x58cbb9 internal/poll.(*pollDesc).wait+0x79 /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 # 0x58cc50 internal/poll.(*pollDesc).waitRead+0x30 /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 # 0x58e770 internal/poll.(*FD).Read+0x410 /usr/local/go/src/internal/poll/fd_unix.go:164 # 0x761d32 net.(*netFD).Read+0x72 /usr/local/go/src/net/fd_posix.go:55 # 0x786724 net.(*conn).Read+0xa4 /usr/local/go/src/net/net.go:179 # 0x942c4a net/http.(*connReader).Read+0x22a /usr/local/go/src/net/http/server.go:789 # 0x4dcf99 bufio.(*Reader).fill+0x259 /usr/local/go/src/bufio/bufio.go:110 # 0x4dd35a bufio.(*Reader).Peek+0x15a /usr/local/go/src/bufio/bufio.go:148 # 0x94cae5 net/http.(*conn).serve+0x1d05 /usr/local/go/src/net/http/server.go:2074 18 @ 0x465e1c 0x4782a7 0x1066dc5 0x49e881 # 0x1066dc4 0chain.net/core/memorystore.(*MemoryDBChunkProcessor).Run+0xe4 /0chain/code/go/0chain.net/core/memorystore/worker.go:97 13 @ 0x465e1c 0x4782a7 0x107edf8 0x49e881 # 0x107edf7 0chain.net/chaincore/node.RequestEntityHandler.func1.1.1.1+0xb7 /0chain/code/go/0chain.net/chaincore/node/n2n_request.go:250 12 @ 0x465e1c 0x45e4be 0x4990bc 0x58cbba 0x58cc51 0x58e771 0x761d33 0x786725 0x974842 0x4dcf9a 0x4dd35b 0x9759b5 0x49e881 # 0x4990bb internal/poll.runtime_pollWait+0x3b /usr/local/go/src/runtime/netpoll.go:345 # 0x58cbb9 internal/poll.(*pollDesc).wait+0x79 /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 # 0x58cc50 internal/poll.(*pollDesc).waitRead+0x30 /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 # 0x58e770 internal/poll.(*FD).Read+0x410 /usr/local/go/src/internal/poll/fd_unix.go:164 # 0x761d32 net.(*netFD).Read+0x72 /usr/local/go/src/net/fd_posix.go:55 # 0x786724 net.(*conn).Read+0xa4 /usr/local/go/src/net/net.go:179 # 0x974841 net/http.(*persistConn).Read+0x1e1 /usr/local/go/src/net/http/transport.go:1977 # 0x4dcf99 bufio.(*Reader).fill+0x259 /usr/local/go/src/bufio/bufio.go:110 # 0x4dd35a bufio.(*Reader).Peek+0x15a /usr/local/go/src/bufio/bufio.go:148 # 0x9759b4 net/http.(*persistConn).readLoop+0x2f4 /usr/local/go/src/net/http/transport.go:2141 12 @ 0x465e1c 0x4782a7 0x97865a 0x49e881 # 0x978659 net/http.(*persistConn).writeLoop+0x179 /usr/local/go/src/net/http/transport.go:2444 4 @ 0x465e1c 0x42969e 0x4293f2 0xdd1711 0x49e881 # 0xdd1710 gopkg.in/natefinch/lumberjack%2ev2.(*Logger).millRun+0x30 /0chain/code/go/0chain.net/vendor/gopkg.in/natefinch/lumberjack.v2/lumberjack.go:379 4 @ 0x465e1c 0x4782a7 0xe490aa 0x49e881 # 0xe490a9 github.com/patrickmn/go-cache.(*janitor).Run+0xc9 /0chain/code/go/0chain.net/vendor/github.com/patrickmn/go-cache/cache.go:1079 3 @ 0x49e881 2 @ 0x465e1c 0x42969e 0x4293f2 0x150f267 0x1592d99 0x49e881 # 0x150f266 github.com/IBM/sarama.(*Broker).responseReceiver+0x66 /0chain/code/go/0chain.net/vendor/github.com/IBM/sarama/broker.go:1163 # 0x1592d98 github.com/IBM/sarama.withRecover+0x38 /0chain/code/go/0chain.net/vendor/github.com/IBM/sarama/utils.go:43 2 @ 0x465e1c 0x4782a7 0x10a2d4b 0x49e881 # 0x10a2d4a 0chain.net/chaincore/node.(*Pool).StatusMonitor+0x3ea /0chain/code/go/0chain.net/chaincore/node/worker.go:31 2 @ 0x465e1c 0x4782a7 0xf3aaf8 0x49e881 # 0xf3aaf7 0chain.net/core/datastore.(*EntityChunkBuilder).Run+0x1b7 /0chain/code/go/0chain.net/core/datastore/worker.go:108 1 @ 0x42fec9 0x49af69 0xe0f797 0x49e881 # 0x49af68 os/signal.signal_recv+0x28 /usr/local/go/src/runtime/sigqueue.go:152 # 0xe0f796 os/signal.loop+0x16 /usr/local/go/src/os/signal/signal_unix.go:23 1 @ 0x459bd1 0x498cdd 0x9edd43 0x9ed986 0x9e8ad8 0x9fa613 0x9fb565 0x94dbb3 0x950f7e 0x9530d7 0x94c895 0x49e881 # 0x498cdc runtime/pprof.runtime_goroutineProfileWithLabels+0x1c /usr/local/go/src/runtime/mprof.go:1079 # 0x9edd42 runtime/pprof.writeRuntimeProfile+0x182 /usr/local/go/src/runtime/pprof/pprof.go:774 # 0x9ed985 runtime/pprof.writeGoroutine+0x85 /usr/local/go/src/runtime/pprof/pprof.go:734 # 0x9e8ad7 runtime/pprof.(*Profile).WriteTo+0x97 /usr/local/go/src/runtime/pprof/pprof.go:369 # 0x9fa612 net/http/pprof.handler.ServeHTTP+0x4f2 /usr/local/go/src/net/http/pprof/pprof.go:267 # 0x9fb564 net/http/pprof.Index+0x124 /usr/local/go/src/net/http/pprof/pprof.go:384 # 0x94dbb2 net/http.HandlerFunc.ServeHTTP+0x32 /usr/local/go/src/net/http/server.go:2166 # 0x950f7d net/http.(*ServeMux).ServeHTTP+0x37d /usr/local/go/src/net/http/server.go:2683 # 0x9530d6 net/http.serverHandler.ServeHTTP+0x256 /usr/local/go/src/net/http/server.go:3137 # 0x94c894 net/http.(*conn).serve+0x1ab4 /usr/local/go/src/net/http/server.go:2039 1 @ 0x465e1c 0x42969e 0x4293d2 0x1045033 0x49e881 # 0x1045032 github.com/rcrowley/go-metrics.(*meterArbiter).tick+0x32 /0chain/code/go/0chain.net/vendor/github.com/rcrowley/go-metrics/meter.go:239 1 @ 0x465e1c 0x42969e 0x4293d2 0x10a6793 0x49e881 # 0x10a6792 0chain.net/chaincore/node.(*Node).MemoryUsage+0x52 /0chain/code/go/0chain.net/chaincore/node/worker.go:183 1 @ 0x465e1c 0x42969e 0x4293d2 0x14f9445 0x1592d99 0x49e881 # 0x14f9444 github.com/IBM/sarama.(*asyncProducer).retryHandler+0x84 /0chain/code/go/0chain.net/vendor/github.com/IBM/sarama/async_producer.go:1215 # 0x1592d98 github.com/IBM/sarama.withRecover+0x38 /0chain/code/go/0chain.net/vendor/github.com/IBM/sarama/utils.go:43 1 @ 0x465e1c 0x42969e 0x4293d2 0x15ff9ab 0x49e881 # 0x15ff9aa 0chain.net/smartcontract/dbs/event.(*EventDb).addEventsWorker+0x2ca /0chain/code/go/0chain.net/smartcontract/dbs/event/process.go:322 1 @ 0x465e1c 0x42969e 0x4293d2 0xe5985e 0x49e881 # 0xe5985d 0chain.net/core/common.SetupRootContext.func1+0x1d /0chain/code/go/0chain.net/core/common/context.go:26 1 @ 0x465e1c 0x42969e 0x4293f2 0x14f0c0a 0x1592d99 0x49e881 # 0x14f0c09 github.com/IBM/sarama.(*asyncProducer).dispatcher+0x129 /0chain/code/go/0chain.net/vendor/github.com/IBM/sarama/async_producer.go:393 # 0x1592d98 github.com/IBM/sarama.withRecover+0x38 /0chain/code/go/0chain.net/vendor/github.com/IBM/sarama/utils.go:43 1 @ 0x465e1c 0x42969e 0x4293f2 0x14f1b10 0x1592d99 0x49e881 # 0x14f1b0f github.com/IBM/sarama.(*topicProducer).dispatch+0x4f /0chain/code/go/0chain.net/vendor/github.com/IBM/sarama/async_producer.go:500 # 0x1592d98 github.com/IBM/sarama.withRecover+0x38 /0chain/code/go/0chain.net/vendor/github.com/IBM/sarama/utils.go:43 1 @ 0x465e1c 0x42969e 0x4293f2 0x14f2ef8 0x1592d99 0x49e881 # 0x14f2ef7 github.com/IBM/sarama.(*partitionProducer).dispatch+0x397 /0chain/code/go/0chain.net/vendor/github.com/IBM/sarama/async_producer.go:645 # 0x1592d98 github.com/IBM/sarama.withRecover+0x38 /0chain/code/go/0chain.net/vendor/github.com/IBM/sarama/utils.go:43 1 @ 0x465e1c 0x42969e 0x4293f2 0x14f5152 0x1592d99 0x49e881 # 0x14f5151 github.com/IBM/sarama.(*asyncProducer).newBrokerProducer.func2+0x71 /0chain/code/go/0chain.net/vendor/github.com/IBM/sarama/async_producer.go:854 # 0x1592d98 github.com/IBM/sarama.withRecover+0x38 /0chain/code/go/0chain.net/vendor/github.com/IBM/sarama/utils.go:43 1 @ 0x465e1c 0x42969e 0x4293f2 0x14f544a 0x1592d99 0x49e881 # 0x14f5449 github.com/IBM/sarama.(*asyncProducer).newBrokerProducer.func1+0x89 /0chain/code/go/0chain.net/vendor/github.com/IBM/sarama/async_producer.go:798 # 0x1592d98 github.com/IBM/sarama.withRecover+0x38 /0chain/code/go/0chain.net/vendor/github.com/IBM/sarama/utils.go:43 1 @ 0x465e1c 0x42969e 0x4293f2 0x15a2825 0x49e881 # 0x15a2824 0chain.net/smartcontract/dbs/queueProvider.(*KafkaProvider).createKafkaWriter.func1+0x64 /0chain/code/go/0chain.net/smartcontract/dbs/queueProvider/kafka.go:140 1 @ 0x465e1c 0x42969e 0x4293f2 0xe59c8c 0x49e881 # 0xe59c8b 0chain.net/core/common.HandleShutdown.func1+0x8b /0chain/code/go/0chain.net/core/common/context.go:53 1 @ 0x465e1c 0x45e4be 0x4990bc 0x58cbba 0x58cc51 0x58e771 0x761d33 0x786725 0x94243a 0x49e881 # 0x4990bb internal/poll.runtime_pollWait+0x3b /usr/local/go/src/runtime/netpoll.go:345 # 0x58cbb9 internal/poll.(*pollDesc).wait+0x79 /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 # 0x58cc50 internal/poll.(*pollDesc).waitRead+0x30 /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 # 0x58e770 internal/poll.(*FD).Read+0x410 /usr/local/go/src/internal/poll/fd_unix.go:164 # 0x761d32 net.(*netFD).Read+0x72 /usr/local/go/src/net/fd_posix.go:55 # 0x786724 net.(*conn).Read+0xa4 /usr/local/go/src/net/net.go:179 # 0x942439 net/http.(*connReader).backgroundRead+0x79 /usr/local/go/src/net/http/server.go:681 1 @ 0x465e1c 0x45e4be 0x4990bc 0x58cbba 0x58cc51 0x594265 0x764ee9 0x79811a 0x795f25 0x9538cb 0x9532dc 0x1cb869f 0x1cb7fab 0x465992 0x49e881 # 0x4990bb internal/poll.runtime_pollWait+0x3b /usr/local/go/src/runtime/netpoll.go:345 # 0x58cbb9 internal/poll.(*pollDesc).wait+0x79 /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 # 0x58cc50 internal/poll.(*pollDesc).waitRead+0x30 /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 # 0x594264 internal/poll.(*FD).Accept+0x524 /usr/local/go/src/internal/poll/fd_unix.go:611 # 0x764ee8 net.(*netFD).accept+0x68 /usr/local/go/src/net/fd_unix.go:172 # 0x798119 net.(*TCPListener).accept+0x59 /usr/local/go/src/net/tcpsock_posix.go:159 # 0x795f24 net.(*TCPListener).Accept+0x64 /usr/local/go/src/net/tcpsock.go:327 # 0x9538ca net/http.(*Server).Serve+0x50a /usr/local/go/src/net/http/server.go:3255 # 0x9532db net/http.(*Server).ListenAndServe+0x17b /usr/local/go/src/net/http/server.go:3184 # 0x1cb869e main.Listen+0x1e /0chain/code/go/0chain.net/sharder/sharder/sharder.go:287 # 0x1cb7faa main.main+0x308a /0chain/code/go/0chain.net/sharder/sharder/sharder.go:254 # 0x465991 runtime.main+0x251 /usr/local/go/src/runtime/proc.go:271 1 @ 0x465e1c 0x4782a7 0x14f5abf 0x1592d99 0x49e881 # 0x14f5abe github.com/IBM/sarama.(*brokerProducer).run+0x25e /0chain/code/go/0chain.net/vendor/github.com/IBM/sarama/async_producer.go:917 # 0x1592d98 github.com/IBM/sarama.withRecover+0x38 /0chain/code/go/0chain.net/vendor/github.com/IBM/sarama/utils.go:43 1 @ 0x465e1c 0x4782a7 0x151f9ae 0x1592d99 0x49e881 # 0x151f9ad github.com/IBM/sarama.(*client).backgroundMetadataUpdater+0x22d /0chain/code/go/0chain.net/vendor/github.com/IBM/sarama/client.go:957 # 0x1592d98 github.com/IBM/sarama.withRecover+0x38 /0chain/code/go/0chain.net/vendor/github.com/IBM/sarama/utils.go:43 1 @ 0x465e1c 0x4782a7 0x1602f17 0x49e881 # 0x1602f16 0chain.net/smartcontract/dbs/event.(*EventDb).managePartitionsWorker+0xd6 /0chain/code/go/0chain.net/smartcontract/dbs/event/process.go:536 1 @ 0x465e1c 0x4782a7 0x16037b7 0x49e881 # 0x16037b6 0chain.net/smartcontract/dbs/event.(*EventDb).managePermanentPartitionsWorker+0xd6 /0chain/code/go/0chain.net/smartcontract/dbs/event/process.go:559 1 @ 0x465e1c 0x4782a7 0x1b57447 0x49e881 # 0x1b57446 0chain.net/chaincore/chain.(*BlockFetcher).StartBlockFetchWorker+0x4e6 /0chain/code/go/0chain.net/chaincore/chain/block_fetcher.go:152 1 @ 0x465e1c 0x4782a7 0x1be3597 0x49e881 # 0x1be3596 0chain.net/chaincore/chain.(*Chain).StartLFBTicketWorker+0x556 /0chain/code/go/0chain.net/chaincore/chain/protocol_lfb_ticket.go:515 1 @ 0x465e1c 0x4782a7 0x1be6a52 0x49e881 # 0x1be6a51 0chain.net/chaincore/chain.(*Chain).StartLFMBWorker+0x1b1 /0chain/code/go/0chain.net/chaincore/chain/protocol_lfb_ticket.go:742 1 @ 0x465e1c 0x4782a7 0x1bfbbe5 0x49e881 # 0x1bfbbe4 0chain.net/chaincore/chain.(*Chain).SetupSC+0x124 /0chain/code/go/0chain.net/chaincore/chain/protocol_view_change_main.go:25 1 @ 0x465e1c 0x4782a7 0x1c1cf36 0x49e881 # 0x1c1cf35 0chain.net/chaincore/chain.(*Chain).StatusMonitor+0x175 /0chain/code/go/0chain.net/chaincore/chain/worker.go:47 1 @ 0x465e1c 0x4782a7 0x1c1e305 0x49e881 # 0x1c1e304 0chain.net/chaincore/chain.(*Chain).FinalizeRoundWorker+0x384 /0chain/code/go/0chain.net/chaincore/chain/worker.go:167 1 @ 0x465e1c 0x4782a7 0x1c1e7ed 0x49e881 # 0x1c1e7ec 0chain.net/chaincore/chain.(*Chain).FinalizeRoundWorker.func3+0x10c /0chain/code/go/0chain.net/chaincore/chain/worker.go:137 1 @ 0x465e1c 0x4782a7 0x1c1f190 0x49e881 # 0x1c1f18f 0chain.net/chaincore/chain.(*Chain).FinalizedBlockWorker+0xef /0chain/code/go/0chain.net/chaincore/chain/worker.go:215 1 @ 0x465e1c 0x4782a7 0x1c22e0a 0x49e881 # 0x1c22e09 0chain.net/chaincore/chain.(*Chain).PruneClientStateWorker+0x149 /0chain/code/go/0chain.net/chaincore/chain/worker.go:374 1 @ 0x465e1c 0x4782a7 0x1c2357f 0x49e881 # 0x1c2357e 0chain.net/chaincore/chain.(*Chain).SyncLFBStateWorker+0x2de /0chain/code/go/0chain.net/chaincore/chain/worker.go:439 1 @ 0x465e1c 0x4782a7 0x1c2667a 0x49e881 # 0x1c26679 0chain.net/chaincore/chain.(*Chain).PruneStorageWorker+0xf9 /0chain/code/go/0chain.net/chaincore/chain/worker.go:610 1 @ 0x465e1c 0x4782a7 0x1c26a95 0x49e881 # 0x1c26a94 0chain.net/chaincore/chain.(*Chain).SyncLFBTicketWorker+0x194 /0chain/code/go/0chain.net/chaincore/chain/worker.go:644 1 @ 0x465e1c 0x4782a7 0x1c7ad77 0x49e881 # 0x1c7ad76 0chain.net/sharder.(*Chain).TrackTransactionErrors+0x1b6 /0chain/code/go/0chain.net/sharder/worker.go:206 1 @ 0x465e1c 0x4782a7 0xe8d99c 0x49e881 # 0xe8d99b database/sql.(*DB).connectionCleaner+0xfb /usr/local/go/src/database/sql/sql.go:1089 1 @ 0x465e1c 0x4782a7 0xe8e95e 0x49e881 # 0xe8e95d database/sql.(*DB).connectionOpener+0xbd /usr/local/go/src/database/sql/sql.go:1246 1 @ 0x465e1c 0x49b825 0x1b7307c 0x49e881 # 0x49b824 time.Sleep+0x104 /usr/local/go/src/runtime/time.go:195 # 0x1b7307b 0chain.net/chaincore/chain.(*Chain).BlockWorker+0x2abb /0chain/code/go/0chain.net/chaincore/chain/entity.go:564 1 @ 0x465e1c 0x49b825 0x1c575fe 0x49e881 # 0x49b824 time.Sleep+0x104 /usr/local/go/src/runtime/time.go:195 # 0x1c575fd 0chain.net/sharder.(*Chain).HealthCheckWorker+0x111d /0chain/code/go/0chain.net/sharder/health_check.go:215 1 @ 0x465e1c 0x49b825 0x1c59b54 0x1c577d6 0x49e881 # 0x49b824 time.Sleep+0x104 /usr/local/go/src/runtime/time.go:195 # 0x1c59b53 0chain.net/sharder.(*Chain).waitForWork+0x19f3 /0chain/code/go/0chain.net/sharder/health_check.go:374 # 0x1c577d5 0chain.net/sharder.(*Chain).HealthCheckWorker+0x12f5 /0chain/code/go/0chain.net/sharder/health_check.go:264 1 @ 0x465e1c 0x49b825 0x1c7a049 0x49e881 # 0x49b824 time.Sleep+0x104 /usr/local/go/src/runtime/time.go:195 # 0x1c7a048 0chain.net/sharder.(*Chain).RegisterSharderKeepWorker+0x8e8 /0chain/code/go/0chain.net/sharder/worker.go:99 1 @ 0x465e1c 0x49b825 0x1c7a9f8 0x49e881 # 0x49b824 time.Sleep+0x104 /usr/local/go/src/runtime/time.go:195 # 0x1c7a9f7 0chain.net/sharder.(*Chain).SharderHealthCheck+0x797 /0chain/code/go/0chain.net/sharder/worker.go:193