/0chain/code/go/0chain.net/blobbercore/handler/protocol.go:163 github.com/0chain/blobber/code/go/0chain.net/core/common/handler.StartHealthCheck.func1 /0chain/code/go/0chain.net/core/common/handler/worker.go:36 2026-05-30T23:12:50.587Z ERROR storage/challenge_handler.go:118 Error verifying the challenge from BC {"challenge_id": "e91774a8f6fade7779a70bce6e6eb82ce1604f2d1aacb707fa228a1cb1966877", "error": "invalid_challenge: Invalid challenge id. Challenge not found in blockchain. consensus_failed: consensus failed on sharders"} github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.NewChallengeObj /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:118 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.challengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:38 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.ChallengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler_main.go:12 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.SetupContext.func1 /0chain/code/go/0chain.net/validatorcore/storage/context.go:16 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.ToJSONResponse.func2 /0chain/code/go/0chain.net/core/common/handler.go:117 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.RateLimit.RateLimit.func5 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 github.com/gorilla/handlers.(*cors).ServeHTTP /go/pkg/mod/github.com/gorilla/handlers@v1.5.1/cors.go:54 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-05-30T23:58:59.860Z ERROR handler/health.go:61 Failed to health check validator on the blockchain {"error": "[txn] too less sharders to confirm it: min_confirmation is 10%, but got 0/1 sharders\r\nhttps://test.zus.network/sharder02/v1/transaction/get/confirmation?hash=875aa59c6abc59cc6c789338ead68b8e1918f7c41b148f0840e3df46f6712183: [400] {\"error\":\"unexpected end of JSON input\"}\n"} github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.ValidatorHealthCheck /0chain/code/go/0chain.net/blobbercore/handler/health.go:61 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.SendHealthCheck /0chain/code/go/0chain.net/blobbercore/handler/protocol.go:163 github.com/0chain/blobber/code/go/0chain.net/core/common/handler.StartHealthCheck.func1 /0chain/code/go/0chain.net/core/common/handler/worker.go:36 2026-05-31T02:28:18.289Z ERROR handler/health.go:61 Failed to health check validator on the blockchain {"error": "[txn] too less sharders to confirm it: min_confirmation is 10%, but got 0/2 sharders\r\nhttps://test.zus.network/sharder02/v1/transaction/get/confirmation?hash=413168692c726f4a5f82647967cfa2e171cb8ebe534889e03a6dd53992a7d82b: [400] {\"error\":\"unexpected end of JSON input\"}\n\r\nhttps://test.zus.network/sharder01/v1/transaction/get/confirmation?hash=413168692c726f4a5f82647967cfa2e171cb8ebe534889e03a6dd53992a7d82b: [400] {\"error\":\"unexpected end of JSON input\"}\n"} github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.ValidatorHealthCheck /0chain/code/go/0chain.net/blobbercore/handler/health.go:61 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.SendHealthCheck /0chain/code/go/0chain.net/blobbercore/handler/protocol.go:163 github.com/0chain/blobber/code/go/0chain.net/core/common/handler.StartHealthCheck.func1 /0chain/code/go/0chain.net/core/common/handler/worker.go:36 2026-05-31T02:55:57.484Z ERROR storage/challenge_handler.go:47 Error getting allocation from chain {"allocation_id": "b143cc430ebf2209cd999ae2855d2aebc18ead48182d3f6aa9f1bcc8ecc92513", "error": "consensus_failed: consensus failed on sharders"} github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.challengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:47 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.ChallengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler_main.go:12 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.SetupContext.func1 /0chain/code/go/0chain.net/validatorcore/storage/context.go:16 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.ToJSONResponse.func2 /0chain/code/go/0chain.net/core/common/handler.go:117 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.RateLimit.RateLimit.func5 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 github.com/gorilla/handlers.(*cors).ServeHTTP /go/pkg/mod/github.com/gorilla/handlers@v1.5.1/cors.go:54 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-05-31T05:52:15.092Z ERROR storage/challenge_handler.go:118 Error verifying the challenge from BC {"challenge_id": "d292a2af48a91063e568c59f7237e0f7d01e4a04e5ef56d133494d5f40a05aae", "error": "invalid_challenge: Invalid challenge id. Challenge not found in blockchain. consensus_failed: consensus failed on sharders"} github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.NewChallengeObj /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:118 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.challengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:38 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.ChallengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler_main.go:12 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.SetupContext.func1 /0chain/code/go/0chain.net/validatorcore/storage/context.go:16 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.ToJSONResponse.func2 /0chain/code/go/0chain.net/core/common/handler.go:117 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.RateLimit.RateLimit.func5 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 github.com/gorilla/handlers.(*cors).ServeHTTP /go/pkg/mod/github.com/gorilla/handlers@v1.5.1/cors.go:54 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-05-31T09:35:03.629Z ERROR storage/challenge_handler.go:47 Error getting allocation from chain {"allocation_id": "b143cc430ebf2209cd999ae2855d2aebc18ead48182d3f6aa9f1bcc8ecc92513", "error": "consensus_failed: consensus failed on sharders"} github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.challengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:47 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.ChallengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler_main.go:12 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.SetupContext.func1 /0chain/code/go/0chain.net/validatorcore/storage/context.go:16 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.ToJSONResponse.func2 /0chain/code/go/0chain.net/core/common/handler.go:117 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.RateLimit.RateLimit.func5 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 github.com/gorilla/handlers.(*cors).ServeHTTP /go/pkg/mod/github.com/gorilla/handlers@v1.5.1/cors.go:54 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-05-31T09:35:05.289Z ERROR storage/challenge_handler.go:47 Error getting allocation from chain {"allocation_id": "b143cc430ebf2209cd999ae2855d2aebc18ead48182d3f6aa9f1bcc8ecc92513", "error": "consensus_failed: consensus failed on sharders"} github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.challengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:47 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.ChallengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler_main.go:12 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.SetupContext.func1 /0chain/code/go/0chain.net/validatorcore/storage/context.go:16 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.ToJSONResponse.func2 /0chain/code/go/0chain.net/core/common/handler.go:117 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.RateLimit.RateLimit.func5 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 github.com/gorilla/handlers.(*cors).ServeHTTP /go/pkg/mod/github.com/gorilla/handlers@v1.5.1/cors.go:54 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-05-31T15:12:58.139Z ERROR storage/challenge_handler.go:47 Error getting allocation from chain {"allocation_id": "b143cc430ebf2209cd999ae2855d2aebc18ead48182d3f6aa9f1bcc8ecc92513", "error": "consensus_failed: consensus failed on sharders"} github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.challengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:47 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.ChallengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler_main.go:12 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.SetupContext.func1 /0chain/code/go/0chain.net/validatorcore/storage/context.go:16 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.ToJSONResponse.func2 /0chain/code/go/0chain.net/core/common/handler.go:117 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.RateLimit.RateLimit.func5 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 github.com/gorilla/handlers.(*cors).ServeHTTP /go/pkg/mod/github.com/gorilla/handlers@v1.5.1/cors.go:54 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-05-31T17:40:52.910Z ERROR storage/challenge_handler.go:118 Error verifying the challenge from BC {"challenge_id": "36cb98ab742436cfd96974a60e5a348d8a811cda936a366bcaa00625492859b6", "error": "invalid_challenge: Invalid challenge id. Challenge not found in blockchain. internal_error: can't get challenge: error retriving Challenge node with ID 36cb98ab742436cfd96974a60e5a348d8a811cda936a366bcaa00625492859b6; error: record not found"} github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.NewChallengeObj /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:118 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.challengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:38 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.ChallengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler_main.go:12 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.SetupContext.func1 /0chain/code/go/0chain.net/validatorcore/storage/context.go:16 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.ToJSONResponse.func2 /0chain/code/go/0chain.net/core/common/handler.go:117 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.RateLimit.RateLimit.func5 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 github.com/gorilla/handlers.(*cors).ServeHTTP /go/pkg/mod/github.com/gorilla/handlers@v1.5.1/cors.go:54 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-05-31T17:42:53.983Z ERROR storage/challenge_handler.go:118 Error verifying the challenge from BC {"challenge_id": "d01d478b2e2ee54487e40be6c3290f5ce3ae1b28741bbb0baf04d928d52b5f39", "error": "invalid_challenge: Invalid challenge id. Challenge not found in blockchain. consensus_failed: consensus failed on sharders"} github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.NewChallengeObj /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:118 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.challengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:38 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.ChallengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler_main.go:12 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.SetupContext.func1 /0chain/code/go/0chain.net/validatorcore/storage/context.go:16 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.ToJSONResponse.func2 /0chain/code/go/0chain.net/core/common/handler.go:117 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.RateLimit.RateLimit.func5 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 github.com/gorilla/handlers.(*cors).ServeHTTP /go/pkg/mod/github.com/gorilla/handlers@v1.5.1/cors.go:54 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-05-31T17:42:54.192Z ERROR storage/challenge_handler.go:118 Error verifying the challenge from BC {"challenge_id": "2dcf472e79e1de4374fd7f99d8ecf8ad9d5196de9033d8bc3e6b844a551fed7b", "error": "invalid_challenge: Invalid challenge id. Challenge not found in blockchain. consensus_failed: consensus failed on sharders"} github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.NewChallengeObj /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:118 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.challengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:38 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.ChallengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler_main.go:12 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.SetupContext.func1 /0chain/code/go/0chain.net/validatorcore/storage/context.go:16 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.ToJSONResponse.func2 /0chain/code/go/0chain.net/core/common/handler.go:117 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.RateLimit.RateLimit.func5 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 github.com/gorilla/handlers.(*cors).ServeHTTP /go/pkg/mod/github.com/gorilla/handlers@v1.5.1/cors.go:54 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-05-31T17:42:55.571Z ERROR storage/challenge_handler.go:118 Error verifying the challenge from BC {"challenge_id": "6c2c0cdba80fe3030d0d3befe122b29bed213d048347c4698928e000356592f5", "error": "invalid_challenge: Invalid challenge id. Challenge not found in blockchain. consensus_failed: consensus failed on sharders"} github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.NewChallengeObj /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:118 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.challengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:38 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.ChallengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler_main.go:12 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.SetupContext.func1 /0chain/code/go/0chain.net/validatorcore/storage/context.go:16 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.ToJSONResponse.func2 /0chain/code/go/0chain.net/core/common/handler.go:117 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.RateLimit.RateLimit.func5 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 github.com/gorilla/handlers.(*cors).ServeHTTP /go/pkg/mod/github.com/gorilla/handlers@v1.5.1/cors.go:54 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-05-31T17:45:33.823Z ERROR storage/challenge_handler.go:118 Error verifying the challenge from BC {"challenge_id": "585b4d14be4f0e7725a374afcfbaf30b5e8d893d75cfcfe7050e3272fce7c845", "error": "invalid_challenge: Invalid challenge id. Challenge not found in blockchain. consensus_failed: consensus failed on sharders"} github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.NewChallengeObj /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:118 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.challengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:38 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.ChallengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler_main.go:12 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.SetupContext.func1 /0chain/code/go/0chain.net/validatorcore/storage/context.go:16 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.ToJSONResponse.func2 /0chain/code/go/0chain.net/core/common/handler.go:117 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.RateLimit.RateLimit.func5 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 github.com/gorilla/handlers.(*cors).ServeHTTP /go/pkg/mod/github.com/gorilla/handlers@v1.5.1/cors.go:54 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-05-31T20:48:44.839Z ERROR handler/health.go:61 Failed to health check validator on the blockchain {"error": "[txn] too less sharders to confirm it: min_confirmation is 10%, but got 0/2 sharders\r\nhttps://test.zus.network/sharder01/v1/transaction/get/confirmation?hash=bbb58e64d7c1757918a06375a4908f7e7a55725e759bc722d4f807fe685f227c: [400] {\"error\":\"unexpected end of JSON input\"}\n\r\nhttps://test.zus.network/sharder02/v1/transaction/get/confirmation?hash=bbb58e64d7c1757918a06375a4908f7e7a55725e759bc722d4f807fe685f227c: [400] {\"error\":\"unexpected end of JSON input\"}\n"} github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.ValidatorHealthCheck /0chain/code/go/0chain.net/blobbercore/handler/health.go:61 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.SendHealthCheck /0chain/code/go/0chain.net/blobbercore/handler/protocol.go:163 github.com/0chain/blobber/code/go/0chain.net/core/common/handler.StartHealthCheck.func1 /0chain/code/go/0chain.net/core/common/handler/worker.go:36 2026-06-01T00:07:51.154Z ERROR handler/health.go:61 Failed to health check validator on the blockchain {"error": "[txn] too less sharders to confirm it: min_confirmation is 10%, but got 0/2 sharders\r\nhttps://test.zus.network/sharder02/v1/transaction/get/confirmation?hash=62305aecf0945bc61c14e6d2891525800d980c1aaae7e4ea6d409f4740a6ff6f: [400] {\"error\":\"unexpected end of JSON input\"}\n\r\nhttps://test.zus.network/sharder01/v1/transaction/get/confirmation?hash=62305aecf0945bc61c14e6d2891525800d980c1aaae7e4ea6d409f4740a6ff6f: [400] {\"error\":\"unexpected end of JSON input\"}\n"} github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.ValidatorHealthCheck /0chain/code/go/0chain.net/blobbercore/handler/health.go:61 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.SendHealthCheck /0chain/code/go/0chain.net/blobbercore/handler/protocol.go:163 github.com/0chain/blobber/code/go/0chain.net/core/common/handler.StartHealthCheck.func1 /0chain/code/go/0chain.net/core/common/handler/worker.go:36 2026-06-01T01:47:43.625Z ERROR storage/challenge_handler.go:47 Error getting allocation from chain {"allocation_id": "b143cc430ebf2209cd999ae2855d2aebc18ead48182d3f6aa9f1bcc8ecc92513", "error": "consensus_failed: consensus failed on sharders"} github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.challengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:47 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.ChallengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler_main.go:12 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.SetupContext.func1 /0chain/code/go/0chain.net/validatorcore/storage/context.go:16 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.ToJSONResponse.func2 /0chain/code/go/0chain.net/core/common/handler.go:117 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.RateLimit.RateLimit.func5 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 github.com/gorilla/handlers.(*cors).ServeHTTP /go/pkg/mod/github.com/gorilla/handlers@v1.5.1/cors.go:54 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-01T01:48:08.150Z ERROR handler/health.go:61 Failed to health check validator on the blockchain {"error": "[txn] too less sharders to confirm it: min_confirmation is 10%, but got 0/1 sharders\r\nGet \"https://test.zus.network/sharder02/v1/transaction/get/confirmation?hash=b04fcf44b21dd494085e8f2112e3361c6cbf6a104ce6b9fc8bc20fa8466541b8\": dial tcp 198.18.0.82:7172: connect: connection refused"} github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.ValidatorHealthCheck /0chain/code/go/0chain.net/blobbercore/handler/health.go:61 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.SendHealthCheck /0chain/code/go/0chain.net/blobbercore/handler/protocol.go:163 github.com/0chain/blobber/code/go/0chain.net/core/common/handler.StartHealthCheck.func1 /0chain/code/go/0chain.net/core/common/handler/worker.go:36 2026-06-01T04:59:02.736Z ERROR storage/challenge_handler.go:118 Error verifying the challenge from BC {"challenge_id": "0f9176c9e7985d950877aaa71cd9242138c196bfdd9291ea2304477942af8542", "error": "invalid_challenge: Invalid challenge id. Challenge not found in blockchain. consensus_failed: consensus failed on sharders"} github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.NewChallengeObj /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:118 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.challengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:38 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.ChallengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler_main.go:12 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.SetupContext.func1 /0chain/code/go/0chain.net/validatorcore/storage/context.go:16 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.ToJSONResponse.func2 /0chain/code/go/0chain.net/core/common/handler.go:117 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.RateLimit.RateLimit.func5 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 github.com/gorilla/handlers.(*cors).ServeHTTP /go/pkg/mod/github.com/gorilla/handlers@v1.5.1/cors.go:54 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-01T12:37:51.192Z ERROR handler/health.go:61 Failed to health check validator on the blockchain {"error": "submit transaction failed: context deadline exceeded"} github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.ValidatorHealthCheck /0chain/code/go/0chain.net/blobbercore/handler/health.go:61 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.SendHealthCheck /0chain/code/go/0chain.net/blobbercore/handler/protocol.go:163 github.com/0chain/blobber/code/go/0chain.net/core/common/handler.StartHealthCheck.func1 /0chain/code/go/0chain.net/core/common/handler/worker.go:36 2026-06-01T15:09:43.674Z ERROR storage/challenge_handler.go:118 Error verifying the challenge from BC {"challenge_id": "ca4da0215abc915cff0ecfd9abdf7b8b2f6bd9578e799578edde8208bbee88a9", "error": "invalid_challenge: Invalid challenge id. Challenge not found in blockchain. internal_error: can't get challenge: error retriving Challenge node with ID ca4da0215abc915cff0ecfd9abdf7b8b2f6bd9578e799578edde8208bbee88a9; error: record not found"} github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.NewChallengeObj /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:118 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.challengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:38 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.ChallengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler_main.go:12 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.SetupContext.func1 /0chain/code/go/0chain.net/validatorcore/storage/context.go:16 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.ToJSONResponse.func2 /0chain/code/go/0chain.net/core/common/handler.go:117 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.RateLimit.RateLimit.func5 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 github.com/gorilla/handlers.(*cors).ServeHTTP /go/pkg/mod/github.com/gorilla/handlers@v1.5.1/cors.go:54 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039 2026-06-01T15:31:05.821Z ERROR storage/challenge_handler.go:118 Error verifying the challenge from BC {"challenge_id": "c78870864dd842d19dfb1b43a6d0ba3562394d4042346daf817bae4b844eb1a0", "error": "invalid_challenge: Invalid challenge id. Challenge not found in blockchain. internal_error: can't get challenge: failed to connect to `host=sharder-postgres-2 user=zchain_user database=events_db`: hostname resolving error (lookup sharder-postgres-2 on 127.0.0.11:53: dial udp 127.0.0.11:53: i/o timeout)"} github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.NewChallengeObj /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:118 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.challengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler.go:38 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.ChallengeHandler /0chain/code/go/0chain.net/validatorcore/storage/challenge_handler_main.go:12 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.SetupContext.func1 /0chain/code/go/0chain.net/validatorcore/storage/context.go:16 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.ToJSONResponse.func2 /0chain/code/go/0chain.net/core/common/handler.go:117 github.com/0chain/blobber/code/go/0chain.net/validatorcore/storage.SetupHandlers.RateLimit.RateLimit.func5 /0chain/code/go/0chain.net/core/common/rate_limiter.go:50 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseCors.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:35 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/0chain/blobber/code/go/0chain.net/blobbercore/handler.UseRecovery.func1 /0chain/code/go/0chain.net/blobbercore/handler/handler_middlewares.go:48 net/http.HandlerFunc.ServeHTTP /usr/local/go/src/net/http/server.go:2166 github.com/gorilla/mux.(*Router).ServeHTTP /go/pkg/mod/github.com/gorilla/mux@v1.8.1/mux.go:212 github.com/gorilla/handlers.(*cors).ServeHTTP /go/pkg/mod/github.com/gorilla/handlers@v1.5.1/cors.go:54 net/http.serverHandler.ServeHTTP /usr/local/go/src/net/http/server.go:3137 net/http.(*conn).serve /usr/local/go/src/net/http/server.go:2039