Is there any possibility of a goroutine interrupt without panic? - go

Question

i'm setting up a service, provide http server and run goroutine to deal with some job, look code
once in a cycle, a sub job seems like interrupt， there is no logs after one func call
it didn't catch any panic err, and the defer seems not trigger because the mutex lock is not unlock
the log is not interrupted or lost
log of other job is complete
there is no restart or exit or oom kill on those time
this is for a CentOS 7.5, my service running in docker
go1.11
docker 18.09
this is an occasional bug, i add more log and open pprof, and try to reproduce this bug
main.go
func main() {
....
// this is a cycle job, with custom time intervals
router.Cycle(r)
....
endless.ListenAndServe(":"+conf.Conf.Port, r)
}
router/cycle.go
// this is a loop job, when job end, sleep custom time intervals and run again
// implemented by encapsulating a goroutine, and create a context
func Cycle(g *gin.Engine) {
cyclec := cli.InitCycle(g)
cyclec.AddFunc(time.Second, schedule.RunSomeDeal)
cyclec.Start()
}
///RunSomeDeal
func RunSomeDeal(c *gin.Context) error {
...
// deal some sub job
for i := 0; i < missionLen; i++ {
// this is once job, like cycle but only run once
// a new context is generated by passing the exist context and a goroutine executes the callback function
helpers.Job.Run(c, func(newCtx *gin.Context) error {
return DealMission(newCtx, someparams...)
})
}
return nil
}
// Job.Run
func (c *Job) Run(ctx *gin.Context, f func(ctx *gin.Context) error) {
e := &Entry{
Job: FuncJob(f),
}
if c.getJobContext != nil {
e.span = c.getJobContext(ctx)
}
go c.runWithRecovery(e)
}
func (c *Job) runWithRecovery(e *Entry) {
ctx := gin.CreateNewContext(c.gin)
...
defer func() {
if r := recover(); r != nil {
const size = 64 << 10
buf := make([]byte, size)
buf = buf[:runtime.Stack(buf, false)]
requestId, _ := ctx.Get("requestId")
handleName := ctx.CustomContext.HandlerName()
info, _ := json.Marshal(map[string]interface{}{
...some kv for log
})
log.Printf(...)
}
gin.RecycleContext(c.gin, ctx)
}()
if c.beforeRun != nil {
ok := c.beforeRun(ctx, e.span)
if !ok {
return
}
}
error := e.Job.Run(ctx)
...
if c.afterRun != nil {
c.afterRun(ctx)
}
}
// DealMission
func DealMission(c *gin.Context, params...) {
// lock something use sync.mutex
doSomeLock()
defer func() {
// ...not trigger
unlockErr := unlockxxxxx(...)
if unlockErr != nil {
panic("some error info")
}
} ()
base.DebugLog(...)
err := SomeOtherFunc(c, params...)
base.DebugLog(...)
}
// some other func
func SomeOtherFunc(ctx *gin.Context, params...) error {
err := CallOther()
base.DebugLog(...)
err := CallOther()
base.DebugLog(...)
// there is no logs after this call func, and Job.runWithRecovery not catch any panic error
err := CallOther()
// print log...
base.DebugLog(...)
}
in this sub job, the log stop at a certain line， and no panic，no error， and the defer seems not trigger because the mutex lock is not unlock
log for other job is well, and log of next cycle job is well too