golang goroutines, chanel and sync.WaitGroup - go

I use simple code, but I get deadlock all the time.
Please explain to the beginner what I am doing wrong.
package main
import (
"fmt"
"sync"
)
func main() {
var wg sync.WaitGroup
ok := make(chan int, 3)
for i := 0; i < 2; i++ {
wg.Add(1)
go func(i int, wg *sync.WaitGroup) {
for x := range ok {
fmt.Println(i, x)
if x >= 5 {
ok <- x - 1
}
}
wg.Done()
}(i, &wg)
}
ok <- 1
ok <- 3
ok <- 2
ok <- 5
ok <- 3
ok <- 9
wg.Wait()
}
fatal error: all goroutines are asleep - deadlock!

The background of the problem is that the range operator reads from the channel until the channel is closed. Hence the for-range loop keeps waiting for more input from the channel ok even though there are no more inputs to come at some point. At the same time wg.Wait() is waiting for the goroutines to finish. Hence a deadlock!
Either you'll have to close the ok channel at some point, where
there are no more incoming values to the channel ok.
Or you may use the worker pool just like,
package main
import (
"fmt"
"strconv"
"sync"
)
func worker(wg *sync.WaitGroup, ch chan string, i int) {
defer wg.Done()
ch <- "worker process value " + strconv.Itoa(i)
}
func workerMonitor(wg *sync.WaitGroup, ch chan string) {
wg.Wait()
close(ch)
}
func doWork(ch <-chan string, done chan<- bool) {
for i := range ch {
fmt.Println(i)
}
done <- true
}
func main() {
var wg sync.WaitGroup
ch := make(chan string)
var arr = [6]int{1, 3, 2, 5, 3, 9}
for i := 0; i < len(arr); i++ {
wg.Add(1)
if arr[i] >= 5 {
for j := arr[i]; j >= 5; j-- {
wg.Add(1)
go worker(&wg, ch, j-1)
}
}
go worker(&wg, ch, arr[i])
}
go workerMonitor(&wg, ch)
done := make(chan bool, 1)
go doWork(ch, done)
<-done
}
Play ground url

Related

All goroutines are asleep when reading from buffered channel

func writeToChan(wg *sync.WaitGroup, ch chan int, stop int) {
defer wg.Done()
for i := 0; i < stop; i++ {
ch <- i
}
}
func readToChan(wg *sync.WaitGroup, ch chan int) {
defer wg.Done()
for n := range ch {
fmt.Println(n)
}
}
func main() {
ch := make(chan int, 3)
wg := new(sync.WaitGroup)
wg.Add(2)
go writeToChan(wg, ch, 5)
go readToChan(wg, ch)
wg.Wait()
}
0
1
2
3
4
fatal error: all goroutines are asleep - deadlock!
I assume that the readToChan always reads continuously, and the writeToChan write to the channel and waits while the channel is read.
I don't know why the output showed deadlock while I added two 'wait' to the WaitGroup.
You need to close channel at the sender side.
By using
for n := range ch {
fmt.Println(n)
}
The loop will only stop when ch is closed
correct example:
package main
import (
"fmt"
"sync"
)
func writeToChan(wg *sync.WaitGroup, ch chan int, stop int) {
defer wg.Done()
for i := 0; i < stop; i++ {
ch <- i
}
close(ch)
}
func readToChan(wg *sync.WaitGroup, ch chan int) {
defer wg.Done()
for n := range ch {
fmt.Println(n)
}
}
func main() {
ch := make(chan int, 3)
wg := new(sync.WaitGroup)
wg.Add(2)
go writeToChan(wg, ch, 5)
go readToChan(wg, ch)
wg.Wait()
}
If close is not called on buffered channel, reader doesn't know when to stop reading.
Check this example with for and select calls(to handle multi channels).
https://go.dev/play/p/Lx5g9o4RsqW
package main
import (
"fmt"
"sync"
"time")
func writeToChan(wg *sync.WaitGroup, ch chan int, stop int, quit chan<- bool) {
defer func() {
wg.Done()
close(ch)
fmt.Println("write wg done")
}()
for i := 0; i < stop; i++ {
ch <- i
fmt.Println("write:", i)
}
fmt.Println("write done")
fmt.Println("sleeping for 5 sec")
time.Sleep(5 * time.Second)
quit <- true
close(quit)
}
func readToChan(wg *sync.WaitGroup, ch chan int, quit chan bool) {
defer func() {
wg.Done()
fmt.Println("read wg done")
}()
//using rang over
//for n := range ch {
// fmt.Println(n)
//}
//using Select if you have multiple channels.
for {
//fmt.Println("waiting for multiple channels")
select {
case n := <-ch:
fmt.Println("read:", n)
// if ok == false {
// fmt.Println("read done")
// //return
// }
case val := <-quit:
fmt.Println("received quit :", val)
return
// default:
// fmt.Println("default")
}
}
}
func main() {
ch := make(chan int, 5)
ch2 := make(chan bool)
wg := new(sync.WaitGroup)
wg.Add(2)
go writeToChan(wg, ch, 3, ch2)
go readToChan(wg, ch, ch2)
wg.Wait()
}
Output:
write: 0
write: 1
write: 2
write done
sleeping for 5 sec
read: 0
read: 1
read: 2
write wg done
received quit : true
read wg done
Program exited.

Expected output as well as deadlock in a worker pool

I'm learning Go concurrency and wrote the obligatory worker pool example, where there are N pieces of work and M workers (N > M). I'm running into a deadlock (all goroutines are asleep), which I can't figure out; however, I'm also getting the expected output before the deadlock occurs, which has me even more confused. Can someone please point out the things I'm doing wrong?
My code is this:
package main
import (
"fmt"
"sync"
)
// A simple worker pool implementation using channels and WaitGroups.
// Our workers simply read from a channel of integers from an input
// channel and write their squares to an output channel.
func addJobs(jobsCh chan<- int, wg *sync.WaitGroup) {
// 100 numbers to crunch (jobs)
for i := 1; i < 101; i++ {
jobsCh <- i
}
wg.Done()
}
func worker(jobsCh <-chan int, resultsCh chan<- int, wg2 *sync.WaitGroup) {
for num := range jobsCh {
resultsCh <- num * num
}
wg2.Done()
}
func addWorkers(jobsCh <-chan int, resultsCh chan<- int, wg *sync.WaitGroup) {
var wg2 sync.WaitGroup
// 10 workers
for i := 0; i < 10; i++ {
wg2.Add(1)
go worker(jobsCh, resultsCh, &wg2)
}
wg.Done()
}
func readResults(resultsCh <-chan int, wg *sync.WaitGroup) {
for sq := range resultsCh {
fmt.Printf("%v ", sq)
}
wg.Done()
}
func main() {
var wg sync.WaitGroup
jobsCh := make(chan int)
resultsCh := make(chan int)
wg.Add(1)
go addJobs(jobsCh, &wg)
wg.Add(1)
go addWorkers(jobsCh, resultsCh, &wg)
wg.Add(1)
go readResults(resultsCh, &wg)
wg.Wait()
}
This prints the squares of the numbers (in random order), as expected, but also runs into a deadlock. Please see this playground link. :(
Close jobsCh to cause workers to exit:
func addJobs(jobsCh chan<- int, wg *sync.WaitGroup) {
// 100 numbers to crunch (jobs)
for i := 1; i < 101; i++ {
jobsCh <- i
}
close(jobsCh) // <-- add this line
wg.Done()
}
After workers are done, close resultsCh to cause results loop to exit:
func addWorkers(jobsCh <-chan int, resultsCh chan<- int, wg *sync.WaitGroup) {
var wg2 sync.WaitGroup
// 10 workers
for i := 0; i < 10; i++ {
wg2.Add(1)
go worker(jobsCh, resultsCh, &wg2)
}
wg2.Wait() // <-- add this line
close(resultsCh) // and this line
wg.Done()
}

How to set a max number of goroutines to complete the work?

var wg sync.WaitGroup
wg.Add(len(work))
sem := make(chan struct{}, 10)
wgDone := make(chan bool)
for i < len(work)-1 {
go func() {
defer wg.Done()
sem <- struct{}{}
defer func() {
<-sem
}()
worker(work[i])
}()
i = i + 1
}
go func() {
wg.Wait()
close(wgDone)
}()
I only want 10 new goroutines at a time performing the work. This is my current solution, it blocks goroutines from continuing so there is only 10 at a time. How can I change this so it doesn't create an abundance of goroutines that are blocked waiting to work and instead only creates 10 that complete all the work?
Based on the use case one of these methods is useful:
Using max number of new goroutines and a channel as a queue (The Go playground):
package main
import (
"fmt"
"sync"
)
func main() {
const max = 10
queue := make(chan int, max)
wg := &sync.WaitGroup{}
for i := 0; i < max; i++ {
wg.Add(1)
go worker(wg, queue)
}
for i := 0; i < 100; i++ {
queue <- i
}
close(queue)
wg.Wait()
fmt.Println("Done")
}
func worker(wg *sync.WaitGroup, queue chan int) {
defer wg.Done()
for job := range queue {
fmt.Print(job, " ") // a job
}
}
Using a buffered channel as a semaphore to limits the new goroutines number to the max number (The Go playground):
package main
import (
"fmt"
"sync"
)
func main() {
const max = 10
semaphore := make(chan struct{}, max)
wg := &sync.WaitGroup{}
for i := 0; i < 1000; i++ {
semaphore <- struct{}{} // acquire
wg.Add(1)
go limited(i, wg, semaphore)
}
wg.Wait()
fmt.Println("Done")
}
func limited(i int, wg *sync.WaitGroup, semaphore chan struct{}) {
defer wg.Done()
fmt.Println("i =", i) // a job
<-semaphore // release
}
Using a buffered channel as a semaphore to limits the number of jobs to the max number - here number of goroutines are more than max number (The Go playground):
package main
import (
"fmt"
"sync"
)
func main() {
const max = 10
semaphore := make(chan struct{}, max)
wg := &sync.WaitGroup{}
for i := 0; i < 1000; i++ {
wg.Add(1)
go limited(i, wg, semaphore)
}
wg.Wait()
fmt.Println("Done")
}
func limited(i int, wg *sync.WaitGroup, semaphore chan struct{}) {
defer wg.Done()
semaphore <- struct{}{} // acquire
fmt.Println("i =", i) // a job
<-semaphore // release
}
So if you want only say 10 workers you should spawn 10 workers listening to a job Queue this can be a channel you can push the inputs to this channel and workers will pick it
Now it will only block jobs when the queue is full so you can decide the queue size based on your use case
package main
import (
"fmt"
"sync"
)
var jobQ chan int
var wg sync.WaitGroup
func main() {
jobQ = make(chan int, 100)
go func(){
wg.Add(1)
defer wg.Done()
//Spawn 10 workers
for i:=0;i<10;i++ {
fmt.Println("Spawn :", i)
wg.Add(1)
go worker(jobQ)
}
}()
for i := 0; i< 1000;i++ {
jobQ<- i
}
close(jobQ)
wg.Wait()
}
func worker(jobs chan int) {
defer wg.Done()
for job:=range jobs {
fmt.Println(job)
}
}
Now you can customize this and find other worker pool implementations; worker pools are used a lot and you will find different implementaions
Playground : https://play.golang.org/p/lzIMRUCvqR9

Use channel to limit the number of active go routines

I'm reading the "The Go Programming Language"
One way to limit the number of running go routines is to use a "counting semaphore".
The other way is Limiting number of go routines running
I am allowing 2 more go routines in the case. I'm getting deadlock error.
What causes the deadlock in my code?
package main
import (
"bytes"
//"context"
"fmt"
"runtime"
"strconv"
"sync"
"time"
)
func main() {
max := 2
var wg sync.WaitGroup
squares := make(chan int)
tokens := make(chan struct{}, max)
for i := 20; i >= 1; i-- {
tokens <- struct{}{}
wg.Add(1)
go func(n int) {
defer func() { <-tokens }()
defer wg.Done()
fmt.Println("run go routine ", getGID())
squares <- Square(n)
}(i)
}
go func() {
wg.Wait()
close(squares)
}()
for s := range squares {
fmt.Println("Get square: ", s)
}
}
func Square(num int) int {
time.Sleep(time.Second * time.Duration(num))
fmt.Println(num * num)
return num * num
}
func getGID() uint64 {
b := make([]byte, 64)
b = b[:runtime.Stack(b, false)]
b = bytes.TrimPrefix(b, []byte("goroutine "))
b = b[:bytes.IndexByte(b, ' ')]
n, _ := strconv.ParseUint(string(b), 10, 64)
return n
}
The goroutines block on sending to squares. Main is not receiving on squares because it blocks on starting the the goroutines.
Fix by moving the code that starts the goroutines to a goroutine. This allows main to continue executing to the receive on squares.
squares := make(chan int)
go func() {
max := 2
var wg sync.WaitGroup
tokens := make(chan struct{}, max)
for i := 20; i >= 1; i-- {
tokens <- struct{}{}
wg.Add(1)
go func(n int) {
defer func() { <-tokens }()
defer wg.Done()
fmt.Println("run go routine ", getGID())
squares <- Square(n)
}(i)
}
wg.Wait()
close(squares)
}()
fmt.Println("About to receive")
for s := range squares {
fmt.Println("Get square: ", s)
}
Run it on the playground.

Always getting deadlock with channels

I'm learning to work with Go channels, and I'm always getting deadlocks. What might be wrong with this code? Printer randomly stops working when array sizes are unequal; I guess it would help to somehow notify printer that receiver stopped working. Any ideas how to fix it? My code is pasted below.
package main
import (
"fmt"
"sync"
)
var wg = sync.WaitGroup{}
var wgs = sync.WaitGroup{}
var sg = make(chan int, 50)
var gp1 = make(chan int, 50)
var gp2 = make(chan int, 50)
func main(){
wgs.Add(2)
go Sender(0)
go Sender(11)
wg.Add(3)
go Receiver()
go Printer()
go Printer2()
wg.Wait()
}
func Sender(start int){
defer wgs.Done()
for i := start; i < 20; i++ {
sg <- i
}
}
func Receiver(){
defer wg.Done()
for i := 0; i < 20; i++{
nr := <- sg
if nr % 2 == 0{
gp1 <- nr
} else{
gp2 <- nr
}
}
}
func Printer(){
defer wg.Done()
var m [10]int
for i := 0; i < 10; i++ {
m[i] = <- gp1
}
wgs.Wait()
fmt.Println(m)
}
func Printer2(){
defer wg.Done()
var m [10]int
for i := 0; i < 10; i++ {
m[i] = <- gp2
}
wgs.Wait()
fmt.Println(m)
}
// Better to use this one
// func Receiver(senderChannel <-chan int, printerChannel1 chan<- int, printerChannel2 chan<- int, wg *sync.WaitGroup) {
The Sender generates (I think 28 messages) . Roughly half the first 20 of these go to one of gp1 and gp2. Printer and Printer2 then unload the messages
Trouble is, the way that Receiver splits the messages depends on if the number received is odd or even. But you aren't controlling for this. If one of the Printers has less than 10 items in it's queue it will hang
That's one potential problem
Your core problem is that everything in this is "dead reckoning": they expect to see a fixed number of messages, but this doesn't necessarily match up with reality. You should set up the channels so that they get closed once all of their data gets produced.
This probably means setting up an intermediate function to manage the sending:
func Sender(from, to int, c chan<- int) {
for i := from; i < to; i++ {
c <- i
}
}
func SendEverything(c chan<- int) {
var wg sync.WaitGroup
wg.Add(2)
go func() {
defer wg.Done()
Sender(0, 20, c)
}()
go func() {
defer wg.Done()
Sender(11, 20, c)
}()
wg.Wait()
close(c)
}
Make the dispatcher function work on everything in the channel:
func Receive(c <-chan int, odds, evens chan<- int) {
for n := range c {
if n%2 == 0 {
evens <- n
} else {
odds <- n
}
}
close(odds)
close(evens)
}
And then you can share a single print function:
func Printer(prefix string, c <-chan int) {
for n := range c {
fmt.Printf("%s: %d\n", prefix, n)
}
}
Finally, you have a main function that stitches it all together:
func main() {
var wg sync.WaitGroup
inputs := make(chan int)
odds := make(chan int)
evens := make(chan int)
wg.Add(4)
go func() {
defer wg.Done()
SendEverything(inputs)
}()
go func() {
defer wg.Done()
Receive(inputs, odds, evens)
}()
go func() {
defer wg.Done()
Printer("odd number", odds)
}()
go func() {
defer wg.Done()
Printer("even number", evens)
}()
wg.Wait()
}
The complete example is at https://play.golang.org/p/qTUqlt-uaWH.
Note that I've completely refrained from using any global variables, and either things have a hopefully self-explanatory very short name (i and n are simple integers, c is a channel) or are complete words (odds, evens). I've tended to keep sync.WaitGroup objects local to where they're created. Since everything is passed as parameters, I don't need two copies of the same function to act on different global variables, and if I chose to write test code for this, I could create my own local channels.

Resources