My task is to sync 2 goroutines so the output should look like that:
foobarfoobarfoobarfoobar
.The issue is that when I call them they come out completely randomized. This is my code:
package main
import (
"fmt"
"sync"
"time"
)
type ConcurrentPrinter struct {
sync.WaitGroup
sync.Mutex
}
func (cp *ConcurrentPrinter) printFoo(times int) {
cp.WaitGroup.Add(times)
go func() {
cp.Lock()
fmt.Print("foo")
cp.Unlock()
}()
}
func (cp *ConcurrentPrinter) printBar(times int) {
cp.WaitGroup.Add(times)
go func() {
cp.Lock()
fmt.Print("bar")
cp.Unlock()
}()
}
func main() {
times := 10
cp := &ConcurrentPrinter{}
for i := 0; i <= times; i++ {
cp.printFoo(i)
cp.printBar(i)
}
time.Sleep(10 * time.Millisecond)
}
As outlined in the comments, using goroutines may not be the best use case for what you are trying to achieve - and thus this may be an XY problem.
Having said that, if you want to ensure two independent goroutines interleave their work in an alternating sequence, you can implement a set of "ping-pong" mutexs:
var ping, pong sync.Mutex
pong.Lock() // ensure the 2nd goroutine waits & the 1st goes first
go func() {
for {
ping.Lock()
foo()
pong.Unlock()
}
}()
go func() {
for {
pong.Lock()
bar()
ping.Unlock()
}
}()
https://go.dev/play/p/VO2LoMJ8fek
Using channel:
func printFoo(i int, ch chan<- bool, wg *sync.WaitGroup) {
wg.Add(1)
go func() {
defer wg.Done()
fmt.Print("foo")
ch <- true
}()
}
func printBar(i int, ch chan<- bool, wg *sync.WaitGroup) {
wg.Add(1)
go func() {
defer wg.Done()
fmt.Print("bar")
ch <- true
}()
}
func main() {
times := 4
firstchan := make(chan bool)
secondchan := make(chan bool)
var wg sync.WaitGroup
for i := 0; i <= times; i++ {
printFoo(i, firstchan, &wg)
<-firstchan
printBar(i, secondchan, &wg)
<-secondchan
}
wg.Wait()
}
https://go.dev/play/p/MlZ9dHkUXGb
I'm trying to figure out why I have a dead lock with waitgroup.Wait()
package main
import (
"fmt"
"sync"
)
var wg sync.WaitGroup
func foo(c chan int, i int) {
defer wg.Done()
c <- i
}
func main() {
ch := make(chan int)
for i := 0; i < 10; i++ {
wg.Add(1)
go foo(ch, i)
}
wg.Wait()
close(ch)
for item := range ch {
fmt.Println(item)
}
}
When I run it like this, it prints fatal error: all goroutines are asleep - deadlock!
I tried to change ch to a buffered channel and that solved the problem. But I really want to know why is there a dead lock.
I've commented out the parts where your program's logic is not correct:
package main
import (
"fmt"
"sync"
)
var wg sync.WaitGroup
func foo(c chan int, i int) {
defer wg.Done()
c <- i
}
func main() {
ch := make(chan int) // unbuffered channel
for i := 0; i < 10; i++ {
wg.Add(1)
go foo(ch, i)
}
// wg.Wait is waiting for all goroutines to finish but that's
// only possible if the send to channel succeeds. In this case,
// it is not possible as your receiver "for item := range ch" is below
// this. Hence, a deadlock.
wg.Wait()
// Ideally, it should be the sender's duty to close the channel.
// And closing a channel before the receiver where the channel
// is unbuffered is not correct.
close(ch)
for item := range ch {
fmt.Println(item)
}
}
Corrected program:
package main
import (
"fmt"
"sync"
)
var wg sync.WaitGroup
func foo(c chan int, i int) {
defer wg.Done()
c <- i
}
func main() {
ch := make(chan int)
go func() {
for item := range ch {
fmt.Println(item)
}
}()
for i := 0; i < 10; i++ {
wg.Add(1)
go foo(ch, i)
}
wg.Wait()
close(ch)
}
I need your help to understand why my readFromWorker func lead to deadlock. When I comment out lines like below it works correctly (thus I know the problem is here).
The whole is here https://play.golang.org/p/-0mRDAeD2tr
I would really appreciate your help
func readFromWorker(inCh <-chan *data, wg *sync.WaitGroup) {
defer func() {
wg.Done()
}()
//stageIn1 := make(chan *data)
//stageOut1 := make(chan *data)
for v := range inCh {
fmt.Println("v", v)
//stageIn1 <- v
}
//go stage1(stageIn1, stageOut1)
//go stage2(stageOut1)
}
I've commented on the relevant parts where you were doing it wrong. Also, I'd recommend thinking of a better pattern.
Do remember that for range on channels doesn't stop looping unless close is called for the same channel it's looping on. Also, the rule of thumb of closing a channel is that the sender sending to the channel must also close it because sending to a closed channel causes panic.
Also, be very careful when using unbuffered and buffered channels. For unbuffered channels, the sender and receiver must be ready otherwise there would be a deadlock which happened in your case as well.
package main
import (
"fmt"
"sync"
)
type data struct {
id int
url string
field int
}
type job struct {
id int
url string
}
func sendToWorker(id int, inCh <-chan job, outCh chan<- *data, wg *sync.WaitGroup) {
// wg.Done() is itself a function call, no need to wrap it inside
// an anonymous function just to use defer.
defer wg.Done()
for v := range inCh {
// some pre process stuff and then pass to pipeline
outCh <- &data{id: v.id, url: v.url}
}
}
func readFromWorker(inCh <-chan *data, wg *sync.WaitGroup) {
// wg.Done() is itself a function call, no need to wrap it inside
// an anonymous function just to use defer.
defer wg.Done()
var (
stageIn1 = make(chan *data)
stageOut1 = make(chan *data)
)
// Spawn the goroutines so that there's no deadlock
// as the sender and receiver both should be ready
// when using unbuffered channels.
go stage1(stageIn1, stageOut1)
go stage2(stageOut1)
for v := range inCh {
fmt.Println("v", v)
stageIn1 <- v
}
close(stageIn1)
}
func stage1(in <-chan *data, out chan<- *data) {
for s := range in {
fmt.Println("stage1 = ", s)
out <- s
}
// Close the out channel
close(out)
}
func stage2(out <-chan *data) {
// Loop until close
for s := range out {
fmt.Println("stage2 = ", s)
}
}
func main() {
const chanBuffer = 1
var (
inputsCh = make(chan job, chanBuffer)
resultsCh = make(chan *data, chanBuffer)
wgInput sync.WaitGroup
wgResult sync.WaitGroup
)
for i := 1; i <= 4; i++ {
wgInput.Add(1)
go sendToWorker(i, inputsCh, resultsCh, &wgInput)
}
wgResult.Add(1)
go readFromWorker(resultsCh, &wgResult)
for j := 1; j <= 10; j++ {
inputsCh <- job{id: j, url: "google.com"}
}
close(inputsCh)
wgInput.Wait()
close(resultsCh)
wgResult.Wait()
}
I have the following program which serves as a proof of concept. I'm attempting to aggregate the results from chann, that is, too merge each instance of chann in to a common slice. Is this possible with my approach?
So my output for the following example would be a slice containing the following (in any order): []int{0,1,2}, thanks.
func DoStuff(i int, chann chan[]int, wg *sync.WaitGroup) {
defer wg.Done()
chann <-[]int{i}
}
func main() {
var wg sync.WaitGroup
chann := make(chan int[], 3)
defer close(chann)
for i := 0; i < count; 3 {
wg.Add(1)
go DoStuff(i, chann, &wg)
}
wg.Wait()
for {
select {
case result := <-chann:
fmt.Println(result)
os.Exit(1)
}
}
return nil
}
what you want to do is possible, but your program will not run because you are reading from the channel after wg.Wait(), so all goroutines will stop waiting to write, because you will never read from the channel.
You can read from the channel in a goroutine:
for i := 0; i < count; 3 {
wg.Add(1)
go DoStuff(i, chann, &wg)
}
}
go func() {
for data:=range chann {
// Process data
}
}()
wg.Wait()
// Close here, so the reading goroutine can terminate
close(chann)
I have been following a pattern of checking if there is anything in the channel before proceeding with work:
func consume(msg <-chan message) {
for {
if m, ok := <-msg; ok {
fmt.Println("More messages:", m)
} else {
break
}
}
}
that is based on this video. Here is my full code:
package main
import (
"fmt"
"strconv"
"strings"
"sync"
)
type message struct {
body string
code int
}
var markets []string = []string{"BTC", "ETH", "LTC"}
// produces messages into the chan
func produce(n int, market string, msg chan<- message, wg *sync.WaitGroup) {
// for i := 0; i < n; i++ {
var msgToSend = message{
body: strings.Join([]string{"market: ", market, ", #", strconv.Itoa(1)}, ""),
code: 1,
}
fmt.Println("Producing:", msgToSend)
msg <- msgToSend
// }
wg.Done()
}
func receive(msg <-chan message, wg *sync.WaitGroup) {
for {
if m, ok := <-msg; ok {
fmt.Println("Received:", m)
} else {
fmt.Println("Breaking from receiving")
break
}
}
wg.Done()
}
func main() {
wg := sync.WaitGroup{}
msgC := make(chan message, 100)
defer func() {
close(msgC)
}()
for ix, market := range markets {
wg.Add(1)
go produce(ix+1, market, msgC, &wg)
}
wg.Add(1)
go receive(msgC, &wg)
wg.Wait()
}
If you try to run it, we get the deadlock at the very end before we ever print a message that we are about to break. Which, tbh, makes sense, since the last time, when there is nothing else in the chan, we are trying to pull the value out, and so we get this error. But then this pattern isn't workable if m, ok := <- msg; ok. How do I make this code work & why do I get this deadlock error (presumably this pattern should work?).
Given that you do have multiple writers on a single channel, you have a bit of a challenge, because the easy way to do this in Go in general is to have a single writer on a single channel, and then have that single writer close the channel upon sending the last datum:
func produce(... args including channel) {
defer close(ch)
for stuff_to_produce {
ch <- item
}
}
This pattern has the nice property that no matter how you get out of produce, the channel gets closed, signalling the end of production.
You're not using this pattern—you deliver one channel to many goroutines, each of which can send one message—so you need to move the close (or, of course, use yet some other pattern). The simplest way to express the pattern you need is this:
func overall_produce(... args including channel ...) {
var pg sync.WaitGroup
defer close(ch)
for stuff_to_produce {
pg.Add(1)
go produceInParallel(ch, &pg) // add more args if appropriate
}
pg.Wait()
}
The pg counter accumulates active producers. Each must call pg.Done() to indicate that it is done using ch. The overall producer now waits for them all to be done, then it closes the channel on its way out.
(If you write the inner produceInParallel function as a closure, you don't need to pass ch and pg to it explicitly. You may also write overallProducer as a closure.)
Note that your single consumer's loop is probably best expressed using the for ... range construct:
func receive(msg <-chan message, wg *sync.WaitGroup) {
for m := range msg {
fmt.Println("Received:", m)
}
wg.Done()
}
(You mention an intent to add a select to the loop so that you can do some other computing if a message is not yet ready. If that code cannot be spun off into independent goroutines, you will in fact need the fancier m, ok := <-msg construct.)
Note also that the wg for receive—which may turn out to be unnecessary, depending on how you structure other things—is quite independent from the wait-group pg for the producers. While it's true that, as written, the consumer cannot be done until all the producers are done, we'd like to wait independently for the producers to be done, so that we can close the channel in the overall-producer wrapper.
Try this code, I have made few fixes that made it work:
package main
import (
"fmt"
"strconv"
"strings"
"sync"
)
type message struct {
body string
code int
}
var markets []string = []string{"BTC", "ETH", "LTC"}
// produces messages into the chan
func produce(n int, market string, msg chan<- message, wg *sync.WaitGroup) {
// for i := 0; i < n; i++ {
var msgToSend = message{
body: strings.Join([]string{"market: ", market, ", #", strconv.Itoa(1)}, ""),
code: 1,
}
fmt.Println("Producing:", msgToSend)
msg <- msgToSend
// }
}
func receive(msg <-chan message, wg *sync.WaitGroup) {
for {
if m, ok := <-msg; ok {
fmt.Println("Received:", m)
wg.Done()
}
}
}
func consume(msg <-chan message) {
for {
if m, ok := <-msg; ok {
fmt.Println("More messages:", m)
} else {
break
}
}
}
func main() {
wg := sync.WaitGroup{}
msgC := make(chan message, 100)
defer func() {
close(msgC)
}()
for ix, market := range markets {
wg.Add(1)
go produce(ix+1, market, msgC, &wg)
}
go receive(msgC, &wg)
wg.Wait()
fmt.Println("Breaking from receiving")
}
Only when main returns, you can close(msgC), but meanwhile receive is waiting for close signal, that's why DeadLock occurs. After producing messages, close the channel.
package main
import (
"fmt"
"strconv"
"strings"
"sync"
)
type message struct {
body string
code int
}
var markets []string = []string{"BTC", "ETH", "LTC"}
// produces messages into the chan
func produce(n int, market string, msg chan<- message, wg *sync.WaitGroup) {
// for i := 0; i < n; i++ {
var msgToSend = message{
body: strings.Join([]string{"market: ", market, ", #", strconv.Itoa(1)}, ""),
code: 1,
}
fmt.Println("Producing:", msgToSend)
msg <- msgToSend
// }
wg.Done()
}
func receive(msg <-chan message, wg *sync.WaitGroup) {
for {
if m, ok := <-msg; ok {
fmt.Println("Received:", m)
} else {
fmt.Println("Breaking from receiving")
break
}
}
wg.Done()
}
func main() {
wg := sync.WaitGroup{}
msgC := make(chan message, 100)
// defer func() {
// close(msgC)
// }()
for ix, market := range markets {
wg.Add(1)
go produce(ix+1, market, msgC, &wg)
}
wg.Wait() // wait for producer
close(msgC)
wg.Add(1)
go receive(msgC, &wg)
wg.Wait()
}