Related
I'm facing an issue with Rancher 2.5.15 where the main Rancher pod and eks-config-operator are constantly crashing and rebooting. This started around 2 weeks ago without any change in the deployed Rancher application, which we confirmed by restoring a snapshot 5 days prior to this issue appearing, with no change.
I assume the issue is due to a registered EKS cluster having issues when returning the cluster information, as it's failing on a function called BuildUpstreamClusterState, however it doesn't indicate which cluster it may be. Deleting the clusters from Rancher is not an option, as there are several EKS clusters added, and since some of them were provisioned from Rancher itself, deleting the clusters from the Rancher UI will also delete them from AWS.
The clusters themselves are working correctly, including the cluster where Rancher is running.
Full log of the panic event:
E0722 12:59:58.631866 7 runtime.go:78] Observed a panic: runtime.boundsError{x:0, y:0, signed:true, code:0x0} (runtime error: index out of range [0] with length 0)
goroutine 1523 [running]:
k8s.io/apimachinery/pkg/util/runtime.logPanic(0x3658a80, 0xc09820cba0)
/go/pkg/mod/k8s.io/apimachinery#v0.20.6/pkg/util/runtime/runtime.go:74 +0x95
k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0)
/go/pkg/mod/k8s.io/apimachinery#v0.20.6/pkg/util/runtime/runtime.go:48 +0x86
panic(0x3658a80, 0xc09820cba0)
/usr/local/go/src/runtime/panic.go:965 +0x1b9
github.com/rancher/eks-operator/controller.BuildUpstreamClusterState(0xc037b89ae0, 0x20, 0xc0631001c8, 0x14, 0xc0453237d0, 0xc0818104b8, 0x1, 0x1, 0xc0818104c8, 0xc062c43f00, ...)
/go/pkg/mod/github.com/rancher/eks-operator#v1.0.9/controller/eks-cluster-config-handler.go:865 +0x1931
github.com/rancher/rancher/pkg/controllers/management/clusterupstreamrefresher.BuildEKSUpstreamSpec(0x3f3d538, 0xc04cdf3050, 0xc04a2ad100, 0x30, 0xc0d89d79a0, 0x1)
/go/src/github.com/rancher/rancher/pkg/controllers/management/clusterupstreamrefresher/eks_upstream_spec.go:80 +0xc92
github.com/rancher/rancher/pkg/controllers/management/clusterupstreamrefresher.getComparableUpstreamSpec(0x3f3d538, 0xc04cdf3050, 0xc04a2ad100, 0x30, 0xc0d89d79a0, 0x1)
/go/src/github.com/rancher/rancher/pkg/controllers/management/clusterupstreamrefresher/cluster_upstream_refresher.go:263 +0x14c
github.com/rancher/rancher/pkg/controllers/management/clusterupstreamrefresher.(*clusterRefreshController).refreshClusterUpstreamSpec(0xc04cdb7740, 0xc04a2ad100, 0x38cc6bf, 0x3, 0xa303efde8, 0x59d1700, 0x1)
/go/src/github.com/rancher/rancher/pkg/controllers/management/clusterupstreamrefresher/cluster_upstream_refresher.go:147 +0xef
github.com/rancher/rancher/pkg/controllers/management/clusterupstreamrefresher.(*clusterRefreshController).onClusterChange(0xc04cdb7740, 0xc0065a5200, 0x7, 0xc04a2ad100, 0x37761a0, 0x383d380, 0x7ff83d307a98)
/go/src/github.com/rancher/rancher/pkg/controllers/management/clusterupstreamrefresher/cluster_upstream_refresher.go:75 +0x225
github.com/rancher/rancher/pkg/generated/controllers/management.cattle.io/v3.FromClusterHandlerToHandler.func1(0xc0065a5200, 0x7, 0x3ef5738, 0xc04a2ad100, 0xc04a2ad100, 0x7ff83d307a98, 0xc04a2ad100, 0x1)
/go/src/github.com/rancher/rancher/pkg/generated/controllers/management.cattle.io/v3/cluster.go:105 +0x6b
github.com/rancher/lasso/pkg/controller.SharedControllerHandlerFunc.OnChange(0xc04d858010, 0xc0065a5200, 0x7, 0x3ef5738, 0xc04a2ad100, 0x0, 0xc04a2ad100, 0x0, 0x0)
/go/pkg/mod/github.com/rancher/lasso#v0.0.0-20210408231703-9ddd9378d08d/pkg/controller/sharedcontroller.go:29 +0x4e
github.com/rancher/lasso/pkg/controller.(*SharedHandler).OnChange(0xc001b351a0, 0xc0065a5200, 0x7, 0x3ef5738, 0xc04a2ad100, 0xc022e42c01, 0x0)
/go/pkg/mod/github.com/rancher/lasso#v0.0.0-20210408231703-9ddd9378d08d/pkg/controller/sharedhandler.go:66 +0x123
github.com/rancher/lasso/pkg/controller.(*controller).syncHandler(0xc000dbd340, 0xc0065a5200, 0x7, 0xc032d3be58, 0x0)
/go/pkg/mod/github.com/rancher/lasso#v0.0.0-20210408231703-9ddd9378d08d/pkg/controller/controller.go:210 +0xd1
github.com/rancher/lasso/pkg/controller.(*controller).processSingleItem(0xc000dbd340, 0x3089ba0, 0xc04da70400, 0x0, 0x0)
/go/pkg/mod/github.com/rancher/lasso#v0.0.0-20210408231703-9ddd9378d08d/pkg/controller/controller.go:192 +0xe7
github.com/rancher/lasso/pkg/controller.(*controller).processNextWorkItem(0xc000dbd340, 0x203001)
/go/pkg/mod/github.com/rancher/lasso#v0.0.0-20210408231703-9ddd9378d08d/pkg/controller/controller.go:169 +0x54
github.com/rancher/lasso/pkg/controller.(*controller).runWorker(...)
/go/pkg/mod/github.com/rancher/lasso#v0.0.0-20210408231703-9ddd9378d08d/pkg/controller/controller.go:158
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc010473450)
/go/pkg/mod/k8s.io/apimachinery#v0.20.6/pkg/util/wait/wait.go:155 +0x5f
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc010473450, 0x3ed54c0, 0xc022aa9050, 0x1, 0xc0019563c0)
/go/pkg/mod/k8s.io/apimachinery#v0.20.6/pkg/util/wait/wait.go:156 +0x9b
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc010473450, 0x3b9aca00, 0x0, 0x1, 0xc0019563c0)
/go/pkg/mod/k8s.io/apimachinery#v0.20.6/pkg/util/wait/wait.go:133 +0x98
k8s.io/apimachinery/pkg/util/wait.Until(0xc010473450, 0x3b9aca00, 0xc0019563c0)
/go/pkg/mod/k8s.io/apimachinery#v0.20.6/pkg/util/wait/wait.go:90 +0x4d
created by github.com/rancher/lasso/pkg/controller.(*controller).run
/go/pkg/mod/github.com/rancher/lasso#v0.0.0-20210408231703-9ddd9378d08d/pkg/controller/controller.go:129 +0x33b
panic: runtime error: index out of range [0] with length 0 [recovered]
panic: runtime error: index out of range [0] with length 0
goroutine 1523 [running]:
k8s.io/apimachinery/pkg/util/runtime.HandleCrash(0x0, 0x0, 0x0)
/go/pkg/mod/k8s.io/apimachinery#v0.20.6/pkg/util/runtime/runtime.go:55 +0x109
panic(0x3658a80, 0xc09820cba0)
/usr/local/go/src/runtime/panic.go:965 +0x1b9
github.com/rancher/eks-operator/controller.BuildUpstreamClusterState(0xc037b89ae0, 0x20, 0xc0631001c8, 0x14, 0xc0453237d0, 0xc0818104b8, 0x1, 0x1, 0xc0818104c8, 0xc062c43f00, ...)
/go/pkg/mod/github.com/rancher/eks-operator#v1.0.9/controller/eks-cluster-config-handler.go:865 +0x1931
github.com/rancher/rancher/pkg/controllers/management/clusterupstreamrefresher.BuildEKSUpstreamSpec(0x3f3d538, 0xc04cdf3050, 0xc04a2ad100, 0x30, 0xc0d89d79a0, 0x1)
/go/src/github.com/rancher/rancher/pkg/controllers/management/clusterupstreamrefresher/eks_upstream_spec.go:80 +0xc92
github.com/rancher/rancher/pkg/controllers/management/clusterupstreamrefresher.getComparableUpstreamSpec(0x3f3d538, 0xc04cdf3050, 0xc04a2ad100, 0x30, 0xc0d89d79a0, 0x1)
/go/src/github.com/rancher/rancher/pkg/controllers/management/clusterupstreamrefresher/cluster_upstream_refresher.go:263 +0x14c
github.com/rancher/rancher/pkg/controllers/management/clusterupstreamrefresher.(*clusterRefreshController).refreshClusterUpstreamSpec(0xc04cdb7740, 0xc04a2ad100, 0x38cc6bf, 0x3, 0xa303efde8, 0x59d1700, 0x1)
/go/src/github.com/rancher/rancher/pkg/controllers/management/clusterupstreamrefresher/cluster_upstream_refresher.go:147 +0xef
github.com/rancher/rancher/pkg/controllers/management/clusterupstreamrefresher.(*clusterRefreshController).onClusterChange(0xc04cdb7740, 0xc0065a5200, 0x7, 0xc04a2ad100, 0x37761a0, 0x383d380, 0x7ff83d307a98)
/go/src/github.com/rancher/rancher/pkg/controllers/management/clusterupstreamrefresher/cluster_upstream_refresher.go:75 +0x225
github.com/rancher/rancher/pkg/generated/controllers/management.cattle.io/v3.FromClusterHandlerToHandler.func1(0xc0065a5200, 0x7, 0x3ef5738, 0xc04a2ad100, 0xc04a2ad100, 0x7ff83d307a98, 0xc04a2ad100, 0x1)
/go/src/github.com/rancher/rancher/pkg/generated/controllers/management.cattle.io/v3/cluster.go:105 +0x6b
github.com/rancher/lasso/pkg/controller.SharedControllerHandlerFunc.OnChange(0xc04d858010, 0xc0065a5200, 0x7, 0x3ef5738, 0xc04a2ad100, 0x0, 0xc04a2ad100, 0x0, 0x0)
/go/pkg/mod/github.com/rancher/lasso#v0.0.0-20210408231703-9ddd9378d08d/pkg/controller/sharedcontroller.go:29 +0x4e
github.com/rancher/lasso/pkg/controller.(*SharedHandler).OnChange(0xc001b351a0, 0xc0065a5200, 0x7, 0x3ef5738, 0xc04a2ad100, 0xc022e42c01, 0x0)
/go/pkg/mod/github.com/rancher/lasso#v0.0.0-20210408231703-9ddd9378d08d/pkg/controller/sharedhandler.go:66 +0x123
github.com/rancher/lasso/pkg/controller.(*controller).syncHandler(0xc000dbd340, 0xc0065a5200, 0x7, 0xc032d3be58, 0x0)
/go/pkg/mod/github.com/rancher/lasso#v0.0.0-20210408231703-9ddd9378d08d/pkg/controller/controller.go:210 +0xd1
github.com/rancher/lasso/pkg/controller.(*controller).processSingleItem(0xc000dbd340, 0x3089ba0, 0xc04da70400, 0x0, 0x0)
/go/pkg/mod/github.com/rancher/lasso#v0.0.0-20210408231703-9ddd9378d08d/pkg/controller/controller.go:192 +0xe7
github.com/rancher/lasso/pkg/controller.(*controller).processNextWorkItem(0xc000dbd340, 0x203001)
/go/pkg/mod/github.com/rancher/lasso#v0.0.0-20210408231703-9ddd9378d08d/pkg/controller/controller.go:169 +0x54
github.com/rancher/lasso/pkg/controller.(*controller).runWorker(...)
/go/pkg/mod/github.com/rancher/lasso#v0.0.0-20210408231703-9ddd9378d08d/pkg/controller/controller.go:158
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0xc010473450)
/go/pkg/mod/k8s.io/apimachinery#v0.20.6/pkg/util/wait/wait.go:155 +0x5f
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc010473450, 0x3ed54c0, 0xc022aa9050, 0x1, 0xc0019563c0)
/go/pkg/mod/k8s.io/apimachinery#v0.20.6/pkg/util/wait/wait.go:156 +0x9b
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0xc010473450, 0x3b9aca00, 0x0, 0x1, 0xc0019563c0)
/go/pkg/mod/k8s.io/apimachinery#v0.20.6/pkg/util/wait/wait.go:133 +0x98
k8s.io/apimachinery/pkg/util/wait.Until(0xc010473450, 0x3b9aca00, 0xc0019563c0)
/go/pkg/mod/k8s.io/apimachinery#v0.20.6/pkg/util/wait/wait.go:90 +0x4d
created by github.com/rancher/lasso/pkg/controller.(*controller).run
/go/pkg/mod/github.com/rancher/lasso#v0.0.0-20210408231703-9ddd9378d08d/pkg/controller/controller.go:129 +0x33b
1.9 GiB database size on disk.
60 GiB memory.
After start cockroach starts to use all memory and increase goroutines count, until it eats whole memory.
gossip client (0/3 cur/max conns)
gossip server (0/3 cur/max conns, infos 0/0 sent/received, bytes 0B/0B sent/received)
W200117 11:21:26.697551 195 storage/store.go:3926 [n1,s1] handle raft ready: 0.6s [processed=1]
W200117 11:21:26.839611 177 storage/store.go:3926 [n1,s1] handle raft ready: 0.6s [processed=1]
W200117 11:21:26.839933 206 storage/store.go:3926 [n1,s1] handle raft ready: 0.7s [processed=1]
W200117 11:21:26.840083 170 storage/store.go:3926 [n1,s1] handle raft ready: 0.6s [processed=1]
W200117 11:21:27.036814 190 storage/store.go:3926 [n1,s1] handle raft ready: 0.6s [processed=1]
W200117 11:21:29.764620 198 storage/store.go:3926 [n1,s1] handle raft ready: 0.7s [processed=1]
W200117 11:21:34.446099 167 storage/store.go:3926 [n1,s1] handle raft ready: 0.7s [processed=1]
W200117 11:21:34.563275 182 storage/store.go:3926 [n1,s1] handle raft ready: 1.0s [processed=1]
I200117 11:21:35.034645 276 server/status/runtime.go:465 [n1] runtime stats: 60 GiB RSS, 9012 goroutines, 52 GiB/3.6 GiB/59 GiB GO alloc/idle/total, 761 MiB/852 MiB CGO alloc/total, 11145.7 CGO/sec, 732.2/48.3 %(u/s)time, 0.1 %gc (1x), 411 KiB/571 KiB (r/w)net
W200117 11:21:36.774832 170 storage/store.go:3926 [n1,s1] handle raft ready: 0.7s [processed=1]
I200117 11:21:45.128897 276 server/status/runtime.go:465 [n1] runtime stats: 61 GiB RSS, 9111 goroutines, 53 GiB/4.0 GiB/60 GiB GO alloc/idle/total, 445 MiB/539 MiB CGO alloc/total, 23047.0 CGO/sec, 754.2/28.9 %(u/s)time, 0.0 %gc (0x), 362 KiB/455 KiB (r/w)net
fatal error: runtime: out of memory
After that there are only errors in log:
runtime stack:
runtime.throw(0x2b00eec, 0x16)
/usr/local/go/src/runtime/panic.go:616 +0x81
runtime.sysMap(0xd2d44b0000, 0x2db0000, 0x0, 0x4882bf8)
/usr/local/go/src/runtime/mem_linux.go:216 +0x20a
runtime.(*mheap).sysAlloc(0x48410a0, 0x2db0000, 0x20224af01)
/usr/local/go/src/runtime/malloc.go:470 +0xd4
runtime.(*mheap).grow(0x48410a0, 0x16d7, 0x0)
/usr/local/go/src/runtime/mheap.go:907 +0x60
runtime.(*mheap).allocSpanLocked(0x48410a0, 0x16d7, 0x4882c08, 0x2050e5e)
/usr/local/go/src/runtime/mheap.go:820 +0x301
runtime.(*mheap).alloc_m(0x48410a0, 0x16d7, 0x101, 0x0)
/usr/local/go/src/runtime/mheap.go:686 +0x118
runtime.(*mheap).alloc.func1()
/usr/local/go/src/runtime/mheap.go:753 +0x4d
runtime.(*mheap).alloc(0x48410a0, 0x16d7, 0xd00f000101, 0xc4204a4180)
/usr/local/go/src/runtime/mheap.go:752 +0x8a
runtime.largeAlloc(0x2dae000, 0xd00f970100, 0x0)
/usr/local/go/src/runtime/malloc.go:826 +0x94
runtime.mallocgc.func1()
/usr/local/go/src/runtime/malloc.go:721 +0x46
runtime.systemstack(0x7f1b00000000)
/usr/local/go/src/runtime/asm_amd64.s:409 +0x79
runtime.mstart()
/usr/local/go/src/runtime/proc.go:1175
goroutine 16380 [running]:
runtime.systemstack_switch()
/usr/local/go/src/runtime/asm_amd64.s:363 fp=0xc9995451b8 sp=0xc9995451b0 pc=0x70b970
runtime.mallocgc(0x2dae000, 0x0, 0x0, 0x0)
/usr/local/go/src/runtime/malloc.go:720 +0x8a2 fp=0xc999545258 sp=0xc9995451b8 pc=0x6c46e2
runtime.growslice(0x268d940, 0xd29e7c0000, 0xd146e4, 0xd146e4, 0x2dad295, 0x0, 0x0, 0x0)
/usr/local/go/src/runtime/slice.go:172 +0x21d fp=0xc9995452c0 sp=0xc999545258 pc=0x6f5c2d
github.com/cockroachdb/cockroach/pkg/roachpb.(*ScanResponse).combine(0xd064b26ee8, 0x302bdc0, 0xd0bc2462b0, 0x302bdc0, 0xd0bc2462b0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/api.go:259 +0x3fb fp=0xc9995453e0 sp=0xc9995452c0 pc=0xda0efb
github.com/cockroachdb/cockroach/pkg/roachpb.(*BatchResponse).Combine(0xcae57fb7a0, 0xc503e11da0, 0xc7b2972a40, 0x6, 0x8, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/roachpb/batch.go:360 +0x32e fp=0xc9995454c8 sp=0xc9995453e0 pc=0xe4d50e
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges.func1(0xc999545848, 0xc999545b28, 0xc999545a70, 0xc999545b20, 0xc99954578f, 0xc999545830, 0xc999545794)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:821 +0x273 fp=0xc999545680 sp=0xc9995454c8 pc=0x1767443
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).divideAndSendBatchToRanges(0xc42077a700, 0x305a3c0, 0xcb14dc0d20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc4de731900, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:1000 +0x797 fp=0xc999545a58 sp=0xc999545680 pc=0x174b557
github.com/cockroachdb/cockroach/pkg/kv.(*DistSender).Send(0xc42077a700, 0x305a3c0, 0xcb14dc0d20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc67b3dbd00, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/dist_sender.go:683 +0x4c7 fp=0xc999545d58 sp=0xc999545a58 pc=0x174a497
github.com/cockroachdb/cockroach/pkg/kv.(*txnLockGatekeeper).SendLocked(0xcb19e3abc8, 0x305a3c0, 0xcb14dc0d20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc67b3dbd00, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:234 +0xf5 fp=0xc999545e08 sp=0xc999545d58 pc=0x17588a5
github.com/cockroachdb/cockroach/pkg/kv.(*txnMetrics).SendLocked(0xcb19e3ab90, 0x305a3c0, 0xcb14dc0d20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc67b3dbd00, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_interceptor_metrics.go:58 +0x12d fp=0xc999545ed8 sp=0xc999545e08 pc=0x1761f7d
github.com/cockroachdb/cockroach/pkg/kv.(*txnSpanRefresher).sendLockedWithRefreshAttempts(0xcb19e3aaf8, 0x305a3c0, 0xcb14dc0d20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc67b3dbd00, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_interceptor_span_refresher.go:167 +0x98 fp=0xc999545fb8 sp=0xc999545ed8 pc=0x17645a8
github.com/cockroachdb/cockroach/pkg/kv.(*txnSpanRefresher).SendLocked(0xcb19e3aaf8, 0x305a3c0, 0xcb14dc0d20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc67b3dbd00, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_interceptor_span_refresher.go:105 +0x11e fp=0xc9995460a0 sp=0xc999545fb8 pc=0x176404e
github.com/cockroachdb/cockroach/pkg/kv.(*txnPipeliner).SendLocked(0xcb19e3aa78, 0x305a3c0, 0xcb14dc0d20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc67b3dbd00, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_interceptor_pipeliner.go:169 +0x165 fp=0xc9995461b8 sp=0xc9995460a0 pc=0x1762555
github.com/cockroachdb/cockroach/pkg/kv.(*txnIntentCollector).SendLocked(0xcb19e3aa38, 0x305a3c0, 0xcb14dc0d20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc67b3dbd00, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_interceptor_intent_collector.go:106 +0x45d fp=0xc999546338 sp=0xc9995461b8 pc=0x17605ad
github.com/cockroachdb/cockroach/pkg/kv.(*txnSeqNumAllocator).SendLocked(0xcb19e3ab78, 0x305a3c0, 0xcb14dc0d20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc67b3dbd00, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_interceptor_sequence_nums.go:62 +0x1f3 fp=0xc999546450 sp=0xc999546338 pc=0x1763e33
github.com/cockroachdb/cockroach/pkg/kv.(*txnHeartbeat).SendLocked(0xcb19e3a990, 0x305a3c0, 0xcb14dc0d20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc67b3dbd00, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_interceptor_heartbeat.go:230 +0x542 fp=0xc999546e00 sp=0xc999546450 pc=0x175df82
github.com/cockroachdb/cockroach/pkg/kv.(*TxnCoordSender).Send(0xcb19e3a800, 0x305a3c0, 0xcb14dc0d20, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc67b3dbd00, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/kv/txn_coord_sender.go:648 +0x456 fp=0xc999547160 sp=0xc999546e00 pc=0x175b136
github.com/cockroachdb/cockroach/pkg/internal/client.(*DB).sendUsingSender(0xc420778280, 0x305a3c0, 0xc9d8025380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/db.go:623 +0x135 fp=0xc999547228 sp=0xc999547160 pc=0xfdc3f5
github.com/cockroachdb/cockroach/pkg/internal/client.(*Txn).Send(0xc909d72990, 0x305a3c0, 0xc9d8025380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/internal/client/txn.go:805 +0x14c fp=0xc999547380 sp=0xc999547228 pc=0xfe64ac
github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*txnKVFetcher).fetch(0xc6b75efd50, 0x305a3c0, 0xc9d8025380, 0x0, 0xc9995479c0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/kvfetcher.go:327 +0x597 fp=0xc9995477f8 sp=0xc999547380 pc=0x15708a7
github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*txnKVFetcher).nextBatch(0xc6b75efd50, 0x305a3c0, 0xc9d8025380, 0x10000cc88b49400, 0x0, 0x15, 0xcc88b49420, 0x0, 0xc42b83d400, 0x2, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/kvfetcher.go:394 +0x6d fp=0xc999547878 sp=0xc9995477f8 pc=0x157112d
github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*RowFetcher).nextKV(0xcad73e5f18, 0x305a3c0, 0xc9d8025380, 0x3f, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/rowfetcher.go:479 +0x370 fp=0xc999547a20 sp=0xc999547878 pc=0x157a730
github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*RowFetcher).NextKey(0xcad73e5f18, 0x305a3c0, 0xc9d8025380, 0x0, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/rowfetcher.go:498 +0x8a fp=0xc999547b00 sp=0xc999547a20 pc=0x157a9da
github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*RowFetcher).StartScanFrom(0xcad73e5f18, 0x305a3c0, 0xc9d8025380, 0x30387c0, 0xc6b75efd50, 0x0, 0x1)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/rowfetcher.go:447 +0xd9 fp=0xc999547b40 sp=0xc999547b00 pc=0x157a359
github.com/cockroachdb/cockroach/pkg/sql/sqlbase.(*RowFetcher).StartScan(0xcad73e5f18, 0x305a3c0, 0xc9d8025380, 0xc909d72990, 0xc9f8754400, 0x3f, 0x40, 0x0, 0x0, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/sqlbase/rowfetcher.go:435 +0x208 fp=0xc999547c88 sp=0xc999547b40 pc=0x157a1c8
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*joinReader).readInput(0xcad73e5500, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/joinreader.go:440 +0x92c fp=0xc999547e38 sp=0xc999547c88 pc=0x17c8b3c
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*joinReader).Next(0xcad73e5500, 0x0, 0x0, 0x0, 0xcb15328e10)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/joinreader.go:356 +0x2df fp=0xc999547e90 sp=0xc999547e38 pc=0x17c81af
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.Run(0x305a3c0, 0xc9d8025380, 0x3063380, 0xcad73e5500, 0x3037440, 0xcb19e92e00)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/base.go:172 +0x35 fp=0xc999547ed0 sp=0xc999547e90 pc=0x1789045
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*ProcessorBase).Run(0xcad73e5500, 0x305a3c0, 0xc9d8025380, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/processors.go:731 +0x98 fp=0xc999547f10 sp=0xc999547ed0 pc=0x17d4b58
github.com/cockroachdb/cockroach/pkg/sql/distsqlrun.(*Flow).StartSync(0xcad736dc00, 0x305a3c0, 0xc9d8025380, 0x2c375e8, 0xcad72389a0, 0x3037140)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsqlrun/flow.go:581 +0x191 fp=0xc999547f60 sp=0xc999547f10 pc=0x17aaa41
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).Run(0xc4208e7700, 0xc4c10cb020, 0xc909d72360, 0xc999548898, 0xcb19e8a500, 0xc8e49839b0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:253 +0x886 fp=0xc999548790 sp=0xc999547f60 pc=0x1b119a6
github.com/cockroachdb/cockroach/pkg/sql.(*DistSQLPlanner).PlanAndRun(0xc4208e7700, 0x305a3c0, 0xc9d8024b10, 0xc8e49839b0, 0xc4c10cb020, 0xc909d72360, 0x304d340, 0xcad73ecdc0, 0xcb19e8a500)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/distsql_running.go:756 +0x24c fp=0xc999548a38 sp=0xc999548790 pc=0x1b14fac
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execWithDistSQLEngine(0xc8e4983500, 0x305a3c0, 0xc9d8024b10, 0xc8e4983918, 0x3, 0x7f1b1a3c03c0, 0xc909d72480, 0xcabe553601, 0xd, 0xd)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:971 +0x2d8 fp=0xc999548c10 sp=0xc999548a38 pc=0x1ac15d8
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).dispatchToExecutionEngine(0xc8e4983500, 0x305a3c0, 0xc9d8024b10, 0x305da40, 0xcafa482f80, 0xcabe553680, 0xd, 0xd, 0xc90a26f440, 0x119, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:818 +0xa7e fp=0xc999548dd8 sp=0xc999548c10 pc=0x1ac096e
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmtInOpenState(0xc8e4983500, 0x305a3c0, 0xc9d8024b10, 0x305da40, 0xcafa482f80, 0xcabe553680, 0xd, 0xd, 0xc90a26f440, 0x119, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:396 +0xa8f fp=0xc9995495b0 sp=0xc999548dd8 pc=0x1abc96f
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).execStmt(0xc8e4983500, 0x305a3c0, 0xc9d8024b10, 0x305da40, 0xcafa482f80, 0xcabe553680, 0xd, 0xd, 0xc90a26f440, 0x119, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor_exec.go:96 +0x341 fp=0xc999549748 sp=0xc9995495b0 pc=0x1abbac1
github.com/cockroachdb/cockroach/pkg/sql.(*connExecutor).run(0xc8e4983500, 0x305a300, 0xc9e1391740, 0xc4207bee58, 0x5400, 0x15000, 0xc4207beef0, 0xc4c0d0c790, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:1180 +0x1400 fp=0xc999549e88 sp=0xc999549748 pc=0x1ab3560
github.com/cockroachdb/cockroach/pkg/sql.(*Server).ServeConn(0xc42087e780, 0x305a300, 0xc9e1391740, 0xc8e4983500, 0x5400, 0x15000, 0xc4207beef0, 0xc4c0d0c790, 0x0, 0x0)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/conn_executor.go:391 +0xce fp=0xc999549ee8 sp=0xc999549e88 pc=0x1aaf1de
github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync.func1(0xc66dbb98c1, 0xc719da4640, 0x305a300, 0xc9e1391740, 0xc4c0d0c790, 0x304f900, 0xc719da4620, 0xc9c20116e0, 0xc426828a00, 0x0, ...)
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:520 +0x1e7 fp=0xc999549f78 sp=0xc999549ee8 pc=0x1f07747
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:2361 +0x1 fp=0xc999549f80 sp=0xc999549f78 pc=0x70e501
created by github.com/cockroachdb/cockroach/pkg/sql/pgwire.(*conn).processCommandsAsync
/go/src/github.com/cockroachdb/cockroach/pkg/sql/pgwire/conn.go:475 +0x17b
I've used this config max-sql-memory=.3 and cache=.40.
From the handle raft ready messages, it seems like your disk is underprovisioned on IOPS or slow to handle requests, which is causing KV operations to queue up in memory.
One workaround to physically limit the amount of memory Cockroach's SQL processors and storage-level caches use, using the flags --max-sql-memory and --cache respectively. This will not solve this problem entirely as raft operations will still queue up until the disk is correctly provisioned, but it will help.
The below command will set both SQL and cache memory limits to 25% of total system memory:
cockroach start --store=<dir> --max-sql-memory=0.25 --cache=0.25
More on memory limits: https://www.cockroachlabs.com/blog/memory-usage-cockroachdb/
we use github.com/gogo/protobuf, I get service crash at proto.Marshal(), this happens almost once a day,But I did not receive a memory alarm,This looks like a memory leak,has anyone encountered this situation?
fatal error: runtime: out of memory
runtime stack:
runtime.throw(0xe25648, 0x16)
/usr/local/go/src/runtime/panic.go:617 +0x72
runtime.sysMap(0xc5cc000000, 0xc490000000, 0x18671b8)
/usr/local/go/src/runtime/mem_linux.go:170 +0xc7
runtime.(*mheap).sysAlloc(0x184e180, 0xc48d3d8000, 0x184e190, 0x62469ec)
/usr/local/go/src/runtime/malloc.go:633 +0x1cd
runtime.(*mheap).grow(0x184e180, 0x62469ec, 0x0)
/usr/local/go/src/runtime/mheap.go:1222 +0x42
runtime.(*mheap).allocSpanLocked(0x184e180, 0x62469ec, 0x18671c8, 0x7f213328e6d8)
/usr/local/go/src/runtime/mheap.go:1150 +0x37f
runtime.(*mheap).alloc_m(0x184e180, 0x62469ec, 0x7f21bed20101, 0x184e190)
/usr/local/go/src/runtime/mheap.go:977 +0xc2
runtime.(*mheap).alloc.func1()
/usr/local/go/src/runtime/mheap.go:1048 +0x4c
runtime.(*mheap).alloc(0x184e180, 0x62469ec, 0xc000000101, 0x7f21bed2a0e0)
/usr/local/go/src/runtime/mheap.go:1047 +0x8a
runtime.largeAlloc(0xc48d3d8000, 0xffffffffffff0100, 0x7f21bed2a0e0)
/usr/local/go/src/runtime/malloc.go:1055 +0x99
runtime.mallocgc.func1()
/usr/local/go/src/runtime/malloc.go:950 +0x46
runtime.systemstack(0x0)
/usr/local/go/src/runtime/asm_amd64.s:351 +0x66
runtime.mstart()
/usr/local/go/src/runtime/proc.go:1153
goroutine 521107 [running]:
runtime.systemstack_switch()
/usr/local/go/src/runtime/asm_amd64.s:311 fp=0xc0982eae30 sp=0xc0982eae28 pc=0x45cdf0
runtime.mallocgc(0xc48d3d8000, 0x0, 0x1700, 0xc492070000)
/usr/local/go/src/runtime/malloc.go:949 +0x872 fp=0xc0982eaed0 sp=0xc0982eae30 pc=0x40e6d2
runtime.growslice(0xca6f00, 0xc492070000, 0x1c43, 0x2000, 0xc48d3d6e73, 0x0, 0x0, 0x0)
/usr/local/go/src/runtime/slice.go:175 +0x151 fp=0xc0982eaf38 sp=0xc0982eaed0 pc=0x4462a1
xxxx/vendor/github.com/gogo/protobuf/proto.(*Buffer).EncodeStringBytes(...)
xxxx/vendor/github.com/gogo/protobuf/proto.Marshal(0xf58840, 0xc45eda2a00, 0x2d, 0x1, 0xc1e72e7801, 0x1, 0xc4860713e0)
/opt/go/src/xxxx/vendor/github.com/gogo/protobuf/proto/encode.go:236 +0x92 fp=0xc0982eb9a8 sp=0xc0982eb960 pc=0x79dd12
xxxx/feature_server_user/get_feature.(*GetUserFeatureAction).Handler(0x1864ca8, 0xc46a746a00, 0x17, 0x200, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
an extremely weird problem happened recently. my program was running on a docker which has 8GB memory and 4GB was used when the panic happened. there were more than 4GB memory available, why could this happen? below is the panic stack output.
# ulimit -m
unlimited
# go version
1.6.2
i tried to run the code line getAllCombinationComplex repeatedly, and the memory-used grew to about 5GB but the panic did not happen again.
fatal error: runtime: out of memory
runtime stack:
runtime.throw(0xd28a90, 0x16)
/usr/local/go1.6.2/src/runtime/panic.go:547 +0x90
runtime.sysMap(0xf409e80000, 0x3ecc390000, 0x434a00, 0x1080918)
/usr/local/go1.6.2/src/runtime/mem_linux.go:206 +0x9b
runtime.(*mheap).sysAlloc(0x1066280, 0x3ecc390000, 0xdfc4d6d680)
/usr/local/go1.6.2/src/runtime/malloc.go:429 +0x191
runtime.(*mheap).grow(0x1066280, 0x1f661c8, 0x0)
/usr/local/go1.6.2/src/runtime/mheap.go:651 +0x63
runtime.(*mheap).allocSpanLocked(0x1066280, 0x1f661c4, 0x10677f0)
/usr/local/go1.6.2/src/runtime/mheap.go:553 +0x4f6
runtime.(*mheap).alloc_m(0x1066280, 0x1f661c4, 0xffffff0100000000, 0x7ff85dbfddd0)
/usr/local/go1.6.2/src/runtime/mheap.go:437 +0x119
runtime.(*mheap).alloc.func1()
/usr/local/go1.6.2/src/runtime/mheap.go:502 +0x41
runtime.systemstack(0x7ff85dbfdde8)
/usr/local/go1.6.2/src/runtime/asm_amd64.s:307 +0xab
runtime.(*mheap).alloc(0x1066280, 0x1f661c4, 0x10100000000, 0x41587c)
/usr/local/go1.6.2/src/runtime/mheap.go:503 +0x63
runtime.largeAlloc(0x3ecc386800, 0x7ff800000000, 0x6)
/usr/local/go1.6.2/src/runtime/malloc.go:766 +0xb3
runtime.mallocgc.func3()
/usr/local/go1.6.2/src/runtime/malloc.go:664 +0x33
runtime.systemstack(0xc820028000)
/usr/local/go1.6.2/src/runtime/asm_amd64.s:291 +0x79
runtime.mstart()
/usr/local/go1.6.2/src/runtime/proc.go:1051
goroutine 63798322149 [running]:
runtime.systemstack_switch()
/usr/local/go1.6.2/src/runtime/asm_amd64.s:245 fp=0xea27f3c798 sp=0xea27f3c790
runtime.mallocgc(0x3ecc386800, 0xa572e0, 0x0, 0xec6c1e8000)
/usr/local/go1.6.2/src/runtime/malloc.go:665 +0x9eb fp=0xea27f3c870 sp=0xea27f3c798
runtime.newarray(0xa572e0, 0x3ecc38680, 0x2)
/usr/local/go1.6.2/src/runtime/malloc.go:798 +0xc9 fp=0xea27f3c8b0 sp=0xea27f3c870
runtime.makeslice(0xa38840, 0x3ecc38680, 0x3ecc38680, 0x0, 0x0, 0x0)
/usr/local/go1.6.2/src/runtime/slice.go:32 +0x165 fp=0xea27f3c900 sp=0xea27f3c8b0
indexdb/db_template.XCludeList.getAllCombinationComplex(0xea4382c480, 0x8, 0x9, 0xe4ec1dc460, 0x4, 0x4, 0x0, 0x0, 0x0)
/home/scmbuild/workspaces_cluster/index/src/indexdb/db_template/db.go:145 +0x371 fp=0xea27f3ca88
sp=0xea27f3c900
indexdb/db_template.XCludeList.getAllCombinationComplex(0xea4382c480, 0x8, 0x9, 0xe4ec1dc438, 0x5, 0x5, 0x0, 0x0, 0x0)
/home/scmbuild/workspaces_cluster/index/src/indexdb/db_template/db.go:141 +0x2ee fp=0xea27f3cc10
sp=0xea27f3ca88
indexdb/db_template.XCludeList.getAllCombinationComplex(0xea4382c480, 0x8, 0x9, 0xe4ec1dc410, 0x6, 0x6, 0x0, 0x0, 0x0)
/home/scmbuild/workspaces_cluster/index/src/indexdb/db_template/db.go:141 +0x2ee fp=0xea27f3cd98
sp=0xea27f3cc10
indexdb/db_template.XCludeList.getAllCombinationComplex(0xea4382c480, 0x8, 0x9, 0xe4ec1dc3e8, 0x7, 0x7, 0x0, 0x0, 0x0)
/home/scmbuild/workspaces_cluster/index/src/indexdb/db_template/db.go:141 +0x2ee fp=0xea27f3cf20
sp=0xea27f3cd98
indexdb/db_template.XCludeList.getAllCombinationComplex(0xea4382c480, 0x8, 0x9, 0xe4ec1dc3c0, 0x8, 0x8, 0x0, 0x0, 0x0)
/home/scmbuild/workspaces_cluster/index/src/indexdb/db_template/db.go:141 +0x2ee fp=0xea27f3d0a8
sp=0xea27f3cf20
indexdb/db_template.XCludeList.GetAllCombinationString(0xea4382c480, 0x8, 0x9, 0x0, 0x0, 0x0)
/home/scmbuild/workspaces_cluster/index/src/indexdb/db_template/db.go:114 +0x530 fp=0xea27f3d398
sp=0xea27f3d0a8
indexdb/mem.MemDB.QueryCountersFullMatchByTags(0xead2bf0003, 0x38, 0xead2bf0043, 0x10, 0xea4382c480, 0x8, 0x9, 0x0, 0x0, 0x0, ...)
/home/scmbuild/workspaces_cluster/index/src/indexdb/mem/mem.go:391 +0x1f5 fp=0xea27f3d670 sp=0xe
a27f3d398
indexdb/mem.(*MemDB).QueryCountersFullMatchByTags(0x107e7b8, 0xead2bf0003, 0x38, 0xead2bf0043, 0x10, 0xea4382c480, 0x8, 0x9, 0x0, 0x0, ...)
<autogenerated>:10 +0x112 fp=0xea27f3d6d8 sp=0xea27f3d670
router.QCountersFullMatchByTags(0xd2745f5ef0)
/home/scmbuild/workspaces_cluster/index/src/router/http.go:505 +0x54a fp=0xea27f3d8d8 sp=0xea27f
3d6d8
github.com/gin-gonic/gin.(*Context).Next(0xd2745f5ef0)
/home/scmbuild/workspaces_cluster/index/deps/src/github.com/gin-gonic/gin/context.go:110 +0x7a f
p=0xea27f3d908 sp=0xea27f3d8d8
github.com/gin-gonic/gin.RecoveryWithWriter.func1(0xd2745f5ef0)
/home/scmbuild/workspaces_cluster/index/deps/src/github.com/gin-gonic/gin/recovery.go:45 +0x51 f
p=0xea27f3d930 sp=0xea27f3d908
github.com/gin-gonic/gin.(*Context).Next(0xd2745f5ef0)
/home/scmbuild/workspaces_cluster/index/deps/src/github.com/gin-gonic/gin/context.go:110 +0x7a f
p=0xea27f3d960 sp=0xea27f3d930
github.com/gin-gonic/gin.(*Engine).handleHTTPRequest(0xd4d0bd2360, 0xd2745f5ef0)
/home/scmbuild/workspaces_cluster/index/deps/src/github.com/gin-gonic/gin/gin.go:337 +0x2fd fp=0
xea27f3dae8 sp=0xea27f3d960
github.com/gin-gonic/gin.(*Engine).ServeHTTP(0xd4d0bd2360, 0x7ff7611f00b0, 0xdb6777f1e0, 0xd6e25a0700)
/home/scmbuild/workspaces_cluster/index/deps/src/github.com/gin-gonic/gin/gin.go:301 +0x197 fp=0
xea27f3db60 sp=0xea27f3dae8
net/http.serverHandler.ServeHTTP(0xd129858080, 0x7ff7611f00b0, 0xdb6777f1e0, 0xd6e25a0700)
/usr/local/go1.6.2/src/net/http/server.go:2081 +0x19e fp=0xea27f3dbc0 sp=0xea27f3db60
net/http.(*conn).serve(0xce774f1400)
/usr/local/go1.6.2/src/net/http/server.go:1472 +0xf2e fp=0xea27f3df88 sp=0xea27f3dbc0
runtime.goexit()
/usr/local/go1.6.2/src/runtime/asm_amd64.s:1998 +0x1 fp=0xea27f3df90 sp=0xea27f3df88
created by net/http.(*Server).Serve
/usr/local/go1.6.2/src/net/http/server.go:2137 +0x44e
Analyzing the stack trace, it looks you are attempting to allocate a far-too-large slice here: db_template.XCludeList.getAllCombinationComplex()
indexdb/db_template.XCludeList.getAllCombinationComplex(0xea4382c480, 0x8, 0x9, 0xe4ec1dc460, 0x4, 0x4, 0x0, 0x0, 0x0)
/home/scmbuild/workspaces_cluster/index/src/indexdb/db_template/db.go:145 +0x371 fp=0xea27f3ca88 sp=0xea27f3c900
This calls into runtime.makeslice():
runtime.makeslice(0xa38840, 0x3ecc38680, 0x3ecc38680, 0x0, 0x0, 0x0)
/usr/local/go1.6.2/src/runtime/slice.go:32 +0x165 fp=0xea27f3c900 sp=0xea27f3c8b0
The source code for runtime.makeslice() for Go 1.6 is here: slice.go:
func makeslice(t *slicetype, len64, cap64 int64) slice {
...
}
And it is called with params:
runtime.makeslice(0xa38840, 0x3ecc38680, 0x3ecc38680, 0x0, 0x0, 0x0)
The second value is the length, which is
0x3ecc38680 = 16857138816
You attempt to create a slice with more than 16*109 elements. If the element type of the slice would require the minimum of 1 byte (excluding the zero-sized types), this would be roughly 16 GB! And this is just a lower estimation. Obviously this operation cannot succeed with 8 GB of RAM available.
Also, please update your Go, 1.6.2 is almost 3 years old, not supported anymore (doesn't even receive security patches).
I'm using go v1.8 on an armv7 target and everything works until I import zeromq, and then not even this works:
package main
import (
zmq "github.com/pebbe/zmq4"
"fmt"
)
func main() {
fmt.Println(zmq.Version())
}
Running it on the target gives the following:
./zmq
fatal error: runtime: out of memory
runtime stack:
runtime.throw(0xc8d3d, 0x16)
/usr/lib/go/src/runtime/panic.go:596 +0x70 fp=0x7eb018a0 sp=0x7eb01894
runtime.sysMap(0x6d752000, 0x74d4000, 0x1, 0x146b60)
/usr/lib/go/src/runtime/mem_linux.go:227 +0xb0 fp=0x7eb018c4 sp=0x7eb018a0
runtime.(*mheap).mapBits(0x139d58, 0x74d26000)
/usr/lib/go/src/runtime/mbitmap.go:159 +0x94 fp=0x7eb018dc sp=0x7eb018c4
runtime.(*mheap).sysAlloc(0x139d58, 0x100000, 0x30414)
/usr/lib/go/src/runtime/malloc.go:428 +0x2dc fp=0x7eb0191c sp=0x7eb018dc
runtime.(*mheap).grow(0x139d58, 0x8, 0x0)
/usr/lib/go/src/runtime/mheap.go:774 +0xd0 fp=0x7eb0194c sp=0x7eb0191c
runtime.(*mheap).allocSpanLocked(0x139d58, 0x1, 0x0)
/usr/lib/go/src/runtime/mheap.go:678 +0x468 fp=0x7eb0196c sp=0x7eb0194c
runtime.(*mheap).alloc_m(0x139d58, 0x1, 0x10, 0x0, 0x0)
/usr/lib/go/src/runtime/mheap.go:562 +0xdc fp=0x7eb019a4 sp=0x7eb0196c
runtime.(*mheap).alloc.func1()
/usr/lib/go/src/runtime/mheap.go:627 +0x3c fp=0x7eb019c0 sp=0x7eb019a4
runtime.systemstack(0x7eb019d4)
/usr/lib/go/src/runtime/asm_arm.s:278 +0xa8 fp=0x7eb019c4 sp=0x7eb019c0
runtime.(*mheap).alloc(0x139d58, 0x1, 0x10, 0x100, 0x1)
/usr/lib/go/src/runtime/mheap.go:628 +0x60 fp=0x7eb019ec sp=0x7eb019c4
runtime.(*mcentral).grow(0x13ab98, 0x0)
/usr/lib/go/src/runtime/mcentral.go:212 +0x84 fp=0x7eb01a18 sp=0x7eb019ec
runtime.(*mcentral).cacheSpan(0x13ab98, 0x146b68)
/usr/lib/go/src/runtime/mcentral.go:93 +0x104 fp=0x7eb01a5c sp=0x7eb01a18
runtime.(*mcache).refill(0x649e5000, 0x10, 0x146b68)
/usr/lib/go/src/runtime/mcache.go:122 +0x7c fp=0x7eb01a70 sp=0x7eb01a5c
runtime.(*mcache).nextFree.func1()
/usr/lib/go/src/runtime/malloc.go:525 +0x24 fp=0x7eb01a80 sp=0x7eb01a70
runtime.systemstack(0x7eb01aa4)
/usr/lib/go/src/runtime/asm_arm.s:278 +0xa8 fp=0x7eb01a84 sp=0x7eb01a80
runtime.(*mcache).nextFree(0x649e5000, 0x649e5010, 0x7eb01acc, 0x1ebd4, 0x21c40)
/usr/lib/go/src/runtime/malloc.go:526 +0x9c fp=0x7eb01ab0 sp=0x7eb01a84
runtime.mallocgc(0xf0, 0xc2548, 0x1, 0x64a26001)
/usr/lib/go/src/runtime/malloc.go:678 +0x8c0 fp=0x7eb01b08 sp=0x7eb01ab0
runtime.newobject(0xc2548, 0x1384a0)
/usr/lib/go/src/runtime/malloc.go:807 +0x2c fp=0x7eb01b1c sp=0x7eb01b08
runtime.malg(0x8000, 0x2710)
/usr/lib/go/src/runtime/proc.go:2821 +0x1c fp=0x7eb01b38 sp=0x7eb01b1c
runtime.mpreinit(0x138730)
/usr/lib/go/src/runtime/os_linux.go:302 +0x1c fp=0x7eb01b44 sp=0x7eb01b38
runtime.mcommoninit(0x138730)
/usr/lib/go/src/runtime/proc.go:540 +0x94 fp=0x7eb01b5c sp=0x7eb01b44
runtime.schedinit()
/usr/lib/go/src/runtime/proc.go:476 +0x40 fp=0x7eb01b78 sp=0x7eb01b5c
runtime.rt0_go(0x7eb01d14, 0x76ee4000, 0x7eb01d14, 0x1, 0x5cd10, 0x7eb01c58, 0x76f8e000, 0xcef6f281, 0xc6a63e08, 0x128, ...)
/usr/lib/go/src/runtime/asm_arm.s:61 +0x84 fp=0x7eb01bb8 sp=0x7eb01b78
Tracing shows the offending mmap:
mmap2(0x74c54000, 1048576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x74c54000
mmap2(0x6d77e000, 122511360, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = -1 ENOMEM (Cannot allocate memory)
The first one is right, that's the chunk size, but the second is where it allocates the bitmap. It should be probably 16x smaller.
I tried debugging but it works under GDB. Maybe GDB does some virtualization?
gdbserver :1000 zmq
Process zmq created; pid = 817
Listening on port 1000
Remote debugging from host 10.0.0.36
4 2 1
Child exited with status 0
So far, all pure go programs I've tested work as well.
Why does importing zeromq cause it to fail? Is there a bug in the arm port of go?
Fixed by https://github.com/golang/go/commit/bb6309cd63b35a81a8527efaad58847a83039947
libzmq must contain some global constructors which adjust the process memory map before the go runtime starts.