I have been following a pattern of checking if there is anything in the channel before proceeding with work:
func consume(msg <-chan message) {
for {
if m, ok := <-msg; ok {
fmt.Println("More messages:", m)
} else {
break
}
}
}
that is based on this video. Here is my full code:
package main
import (
"fmt"
"strconv"
"strings"
"sync"
)
type message struct {
body string
code int
}
var markets []string = []string{"BTC", "ETH", "LTC"}
// produces messages into the chan
func produce(n int, market string, msg chan<- message, wg *sync.WaitGroup) {
// for i := 0; i < n; i++ {
var msgToSend = message{
body: strings.Join([]string{"market: ", market, ", #", strconv.Itoa(1)}, ""),
code: 1,
}
fmt.Println("Producing:", msgToSend)
msg <- msgToSend
// }
wg.Done()
}
func receive(msg <-chan message, wg *sync.WaitGroup) {
for {
if m, ok := <-msg; ok {
fmt.Println("Received:", m)
} else {
fmt.Println("Breaking from receiving")
break
}
}
wg.Done()
}
func main() {
wg := sync.WaitGroup{}
msgC := make(chan message, 100)
defer func() {
close(msgC)
}()
for ix, market := range markets {
wg.Add(1)
go produce(ix+1, market, msgC, &wg)
}
wg.Add(1)
go receive(msgC, &wg)
wg.Wait()
}
If you try to run it, we get the deadlock at the very end before we ever print a message that we are about to break. Which, tbh, makes sense, since the last time, when there is nothing else in the chan, we are trying to pull the value out, and so we get this error. But then this pattern isn't workable if m, ok := <- msg; ok. How do I make this code work & why do I get this deadlock error (presumably this pattern should work?).
Given that you do have multiple writers on a single channel, you have a bit of a challenge, because the easy way to do this in Go in general is to have a single writer on a single channel, and then have that single writer close the channel upon sending the last datum:
func produce(... args including channel) {
defer close(ch)
for stuff_to_produce {
ch <- item
}
}
This pattern has the nice property that no matter how you get out of produce, the channel gets closed, signalling the end of production.
You're not using this pattern—you deliver one channel to many goroutines, each of which can send one message—so you need to move the close (or, of course, use yet some other pattern). The simplest way to express the pattern you need is this:
func overall_produce(... args including channel ...) {
var pg sync.WaitGroup
defer close(ch)
for stuff_to_produce {
pg.Add(1)
go produceInParallel(ch, &pg) // add more args if appropriate
}
pg.Wait()
}
The pg counter accumulates active producers. Each must call pg.Done() to indicate that it is done using ch. The overall producer now waits for them all to be done, then it closes the channel on its way out.
(If you write the inner produceInParallel function as a closure, you don't need to pass ch and pg to it explicitly. You may also write overallProducer as a closure.)
Note that your single consumer's loop is probably best expressed using the for ... range construct:
func receive(msg <-chan message, wg *sync.WaitGroup) {
for m := range msg {
fmt.Println("Received:", m)
}
wg.Done()
}
(You mention an intent to add a select to the loop so that you can do some other computing if a message is not yet ready. If that code cannot be spun off into independent goroutines, you will in fact need the fancier m, ok := <-msg construct.)
Note also that the wg for receive—which may turn out to be unnecessary, depending on how you structure other things—is quite independent from the wait-group pg for the producers. While it's true that, as written, the consumer cannot be done until all the producers are done, we'd like to wait independently for the producers to be done, so that we can close the channel in the overall-producer wrapper.
Try this code, I have made few fixes that made it work:
package main
import (
"fmt"
"strconv"
"strings"
"sync"
)
type message struct {
body string
code int
}
var markets []string = []string{"BTC", "ETH", "LTC"}
// produces messages into the chan
func produce(n int, market string, msg chan<- message, wg *sync.WaitGroup) {
// for i := 0; i < n; i++ {
var msgToSend = message{
body: strings.Join([]string{"market: ", market, ", #", strconv.Itoa(1)}, ""),
code: 1,
}
fmt.Println("Producing:", msgToSend)
msg <- msgToSend
// }
}
func receive(msg <-chan message, wg *sync.WaitGroup) {
for {
if m, ok := <-msg; ok {
fmt.Println("Received:", m)
wg.Done()
}
}
}
func consume(msg <-chan message) {
for {
if m, ok := <-msg; ok {
fmt.Println("More messages:", m)
} else {
break
}
}
}
func main() {
wg := sync.WaitGroup{}
msgC := make(chan message, 100)
defer func() {
close(msgC)
}()
for ix, market := range markets {
wg.Add(1)
go produce(ix+1, market, msgC, &wg)
}
go receive(msgC, &wg)
wg.Wait()
fmt.Println("Breaking from receiving")
}
Only when main returns, you can close(msgC), but meanwhile receive is waiting for close signal, that's why DeadLock occurs. After producing messages, close the channel.
package main
import (
"fmt"
"strconv"
"strings"
"sync"
)
type message struct {
body string
code int
}
var markets []string = []string{"BTC", "ETH", "LTC"}
// produces messages into the chan
func produce(n int, market string, msg chan<- message, wg *sync.WaitGroup) {
// for i := 0; i < n; i++ {
var msgToSend = message{
body: strings.Join([]string{"market: ", market, ", #", strconv.Itoa(1)}, ""),
code: 1,
}
fmt.Println("Producing:", msgToSend)
msg <- msgToSend
// }
wg.Done()
}
func receive(msg <-chan message, wg *sync.WaitGroup) {
for {
if m, ok := <-msg; ok {
fmt.Println("Received:", m)
} else {
fmt.Println("Breaking from receiving")
break
}
}
wg.Done()
}
func main() {
wg := sync.WaitGroup{}
msgC := make(chan message, 100)
// defer func() {
// close(msgC)
// }()
for ix, market := range markets {
wg.Add(1)
go produce(ix+1, market, msgC, &wg)
}
wg.Wait() // wait for producer
close(msgC)
wg.Add(1)
go receive(msgC, &wg)
wg.Wait()
}
Related
I need your help to understand why my readFromWorker func lead to deadlock. When I comment out lines like below it works correctly (thus I know the problem is here).
The whole is here https://play.golang.org/p/-0mRDAeD2tr
I would really appreciate your help
func readFromWorker(inCh <-chan *data, wg *sync.WaitGroup) {
defer func() {
wg.Done()
}()
//stageIn1 := make(chan *data)
//stageOut1 := make(chan *data)
for v := range inCh {
fmt.Println("v", v)
//stageIn1 <- v
}
//go stage1(stageIn1, stageOut1)
//go stage2(stageOut1)
}
I've commented on the relevant parts where you were doing it wrong. Also, I'd recommend thinking of a better pattern.
Do remember that for range on channels doesn't stop looping unless close is called for the same channel it's looping on. Also, the rule of thumb of closing a channel is that the sender sending to the channel must also close it because sending to a closed channel causes panic.
Also, be very careful when using unbuffered and buffered channels. For unbuffered channels, the sender and receiver must be ready otherwise there would be a deadlock which happened in your case as well.
package main
import (
"fmt"
"sync"
)
type data struct {
id int
url string
field int
}
type job struct {
id int
url string
}
func sendToWorker(id int, inCh <-chan job, outCh chan<- *data, wg *sync.WaitGroup) {
// wg.Done() is itself a function call, no need to wrap it inside
// an anonymous function just to use defer.
defer wg.Done()
for v := range inCh {
// some pre process stuff and then pass to pipeline
outCh <- &data{id: v.id, url: v.url}
}
}
func readFromWorker(inCh <-chan *data, wg *sync.WaitGroup) {
// wg.Done() is itself a function call, no need to wrap it inside
// an anonymous function just to use defer.
defer wg.Done()
var (
stageIn1 = make(chan *data)
stageOut1 = make(chan *data)
)
// Spawn the goroutines so that there's no deadlock
// as the sender and receiver both should be ready
// when using unbuffered channels.
go stage1(stageIn1, stageOut1)
go stage2(stageOut1)
for v := range inCh {
fmt.Println("v", v)
stageIn1 <- v
}
close(stageIn1)
}
func stage1(in <-chan *data, out chan<- *data) {
for s := range in {
fmt.Println("stage1 = ", s)
out <- s
}
// Close the out channel
close(out)
}
func stage2(out <-chan *data) {
// Loop until close
for s := range out {
fmt.Println("stage2 = ", s)
}
}
func main() {
const chanBuffer = 1
var (
inputsCh = make(chan job, chanBuffer)
resultsCh = make(chan *data, chanBuffer)
wgInput sync.WaitGroup
wgResult sync.WaitGroup
)
for i := 1; i <= 4; i++ {
wgInput.Add(1)
go sendToWorker(i, inputsCh, resultsCh, &wgInput)
}
wgResult.Add(1)
go readFromWorker(resultsCh, &wgResult)
for j := 1; j <= 10; j++ {
inputsCh <- job{id: j, url: "google.com"}
}
close(inputsCh)
wgInput.Wait()
close(resultsCh)
wgResult.Wait()
}
I need to fetch responses from multiple go routines and put them into an array. I know that channels could be used for this, however I am not sure how I can make sure that all go routines have finished processing the results. Thus I am using a waitgroup.
Code
func main() {
log.Info("Collecting ints")
var results []int32
for _, broker := range e.BrokersByBrokerID {
wg.Add(1)
go getInt32(&wg)
}
wg.Wait()
log.info("Collected")
}
func getInt32(wg *sync.WaitGroup) (int32, error) {
defer wg.Done()
// Just to show that this method may just return an error and no int32
err := broker.Open(config)
if err != nil && err != sarama.ErrAlreadyConnected {
return 0, fmt.Errorf("Cannot connect to broker '%v': %s", broker.ID(), err)
}
defer broker.Close()
return 1003, nil
}
My question
How can I put all the response int32 (which may return an error) into my int32 array, making sure that all go routines have finished their processing work and returned either the error or the int?
If you don't process the return values of the function launched as a goroutine, they are discarded. See What happens to return value from goroutine.
You may use a slice to collect the results, where each goroutine could receive the index to put the results to, or alternatively the address of the element. See Can I concurrently write different slice elements. Note that if you use this, the slice must be pre-allocated and only the element belonging to the goroutine may be written, you can't "touch" other elements and you can't append to the slice.
Or you may use a channel, on which the goroutines send values that include the index or ID of the item they processed, so the collecting goroutine can identify or order them. See How to collect values from N goroutines executed in a specific order?
If processing should stop on the first error encountered, see Close multiple goroutine if an error occurs in one in go
Here's an example how it could look like when using a channel. Note that no waitgroup is needed here, because we know that we expect as many values on the channel as many goroutines we launch.
type result struct {
task int32
data int32
err error
}
func main() {
tasks := []int32{1, 2, 3, 4}
ch := make(chan result)
for _, task := range tasks {
go calcTask(task, ch)
}
// Collect results:
results := make([]result, len(tasks))
for i := range results {
results[i] = <-ch
}
fmt.Printf("Results: %+v\n", results)
}
func calcTask(task int32, ch chan<- result) {
if task > 2 {
// Simulate failure
ch <- result{task: task, err: fmt.Errorf("task %v failed", task)}
return
}
// Simulate success
ch <- result{task: task, data: task * 2, err: nil}
}
Output (try ot on the Go Playground):
Results: [{task:4 data:0 err:0x40e130} {task:1 data:2 err:<nil>} {task:2 data:4 err:<nil>} {task:3 data:0 err:0x40e138}]
I also believe you have to use channel, it must be something like this:
package main
import (
"fmt"
"log"
"sync"
)
var (
BrokersByBrokerID = []int32{1, 2, 3}
)
type result struct {
data string
err string // you must use error type here
}
func main() {
var wg sync.WaitGroup
var results []result
ch := make(chan result)
for _, broker := range BrokersByBrokerID {
wg.Add(1)
go getInt32(ch, &wg, broker)
}
go func() {
for v := range ch {
results = append(results, v)
}
}()
wg.Wait()
close(ch)
log.Printf("collected %v", results)
}
func getInt32(ch chan result, wg *sync.WaitGroup, broker int32) {
defer wg.Done()
if broker == 1 {
ch <- result{err: fmt.Sprintf("error: gor broker 1")}
return
}
ch <- result{data: fmt.Sprintf("broker %d - ok", broker)}
}
Result will look like this:
2019/02/05 15:26:28 collected [{broker 3 - ok } {broker 2 - ok } { error: gor broker 1}]
package main
import (
"fmt"
"log"
"sync"
)
var (
BrokersByBrokerID = []int{1, 2, 3, 4}
)
type result struct {
data string
err string // you must use error type here
}
func main() {
var wg sync.WaitGroup
var results []int
ch := make(chan int)
done := make(chan bool)
for _, broker := range BrokersByBrokerID {
wg.Add(1)
go func(i int) {
defer wg.Done()
ch <- i
if i == 4 {
done <- true
}
}(broker)
}
L:
for {
select {
case v := <-ch:
results = append(results, v)
if len(results) == 4 {
//<-done
close(ch)
break L
}
case _ = <-done:
break
}
}
fmt.Println("STOPPED")
//<-done
wg.Wait()
log.Printf("collected %v", results)
}
Thank cn007b and Edenshaw. My answer is based on their answers.
As Edenshaw commented, need another sync.Waitgroup for goroutine which getting results from channel, or you may get an incomplete array.
package main
import (
"fmt"
"sync"
"encoding/json"
)
type Resp struct {
id int
}
func main() {
var wg sync.WaitGroup
chanRes := make(chan interface{}, 3)
for i := 0; i < 3; i++ {
wg.Add(1)
resp := &Resp{}
go func(i int, resp *Resp) {
defer wg.Done()
resp.id = i
chanRes <- resp
}(i, resp)
}
res := make([]interface{}, 0)
var wg2 sync.WaitGroup
wg2.Add(1)
go func() {
defer wg2.Done()
for v := range chanRes {
res = append(res, v.(*Resp).id)
}
}()
wg.Wait()
close(chanRes)
wg2.Wait()
resStr, _ := json.Marshal(res)
fmt.Println(string(resStr))
}
package main
import (
"fmt"
"log"
"sync"
"time"
)
var (
BrokersByBrokerID = []int{1, 2, 3, 4}
)
type result struct {
data string
err string // you must use error type here
}
func main() {
var wg sync.WaitGroup.
var results []int
ch := make(chan int)
done := make(chan bool)
for _, broker := range BrokersByBrokerID {
wg.Add(1)
go func(i int) {
defer wg.Done()
ch <- i
if i == 4 {
done <- true
}
}(broker)
}
for v := range ch {
results = append(results, v)
if len(results) == 4 {
close(ch)
}
}
fmt.Println("STOPPED")
<-done
wg.Wait()
log.Printf("collected %v", results)
}
</pre>
This code is for a fairly simple demo for my programming language class. I'm trying to display a few different techniques that Go allows, like interfaces and concurrency, but I can't seem to get the WaitGroups to work right, so it keeps deadlocking on me at the end. My biggest question is: how do I get the WaitGroups to synchronize correctly and not deadlock the system when the goroutines stop? I'm most likely missing something obvious.
package main
import (
"bufio"
"fmt"
"os"
"sync"
)
func Reader(wg *sync.WaitGroup, message chan string, done chan bool){
defer wg.Done()
reader := bufio.NewReader(os.Stdin)
for {
msg, _ := reader.ReadString('\n')
if msg == "exit\n" {
<-done
return
} else {
message <- msg
}
}
}
func main() {
message := make(chan string)
done := make(chan bool)
wg := &sync.WaitGroup{}
wg.Add(1)
go Reader(wg, message, done)
wg.Add(1)
go func(){
defer wg.Done()
for {
select {
case <-done:
return
case msg := <-message:
fmt.Println("main: "+msg)
}
}
}()
wg.Wait()
close(message)
close(done)
}
Your break statement in main breaks the select, not the for loop. Use return or a label instead:
go func() {
defer wg.Done()
for {
select {
case <-done:
return // don't break here without label
case msg := <-message:
fmt.Println("main: " + msg)
}
}
}()
In addition, both functions try to receive from done. Reader should close the channel instead to signal completion:
func Reader(wg *sync.WaitGroup, message chan string, done chan bool) {
defer wg.Done()
defer close(done) // close channel to signal completion
reader := bufio.NewReader(os.Stdin)
for {
msg, _ := reader.ReadString('\n')
if msg == "exit\n" {
return
} else {
message <- msg
}
}
}
Don't close the channel in main. Channels should always be closed on the sender's side.
Once you have done all that you should recognize that message and done are redunant. The whole program can be simplified to:
package main
import (
"bufio"
"fmt"
"os"
)
func Reader(message chan string) {
defer close(message)
reader := bufio.NewReader(os.Stdin)
for {
msg, _ := reader.ReadString('\n')
if msg == "exit\n" {
return
} else {
message <- msg
}
}
}
func main() {
message := make(chan string)
go Reader(message)
for msg := range message {
fmt.Println("main: " + msg)
}
}
I started learning go recently and I am stuck on a problem.
I have a simple go routine which either returns or pushes value to a channel.
And my main fn delegates work to this routine till it meets condition or data is exhausted.
This code seem to deadlock on "found" channel. What am I doing wrong?
There are multiple workers
Item can be found in more than one worker at the same time
Once item is found, all workers should be stopped.
.
func workerRoutine(data Data, found chan bool, wg *sync.WaitGroup){
defer (*wg).Done()
// data processing
// return on false
// multiple routines can set this at the same time
found <-true
}
func main {
// ....
found:=make(chan bool)
var wg sync.WaitGroup
itemFound:=false
Loop:
for i:=0; i<limit; i++ {
select {
case <-found:
itemFound = true
break Loop
default:
if(some_check) {
wg.Add(1)
go workerRoutine(mdata,found,&wg)
}
}
}
wg.Wait()
// use itemFound
}
One possible solution is to avoid select statement and use separate goroutine for receiver (or sender, or both).
Example:
package main
import "sync"
func worker(res chan bool, wg *sync.WaitGroup) {
res <- true
wg.Done()
}
func receiver(res chan bool, wg *sync.WaitGroup) {
for range res {
}
wg.Done()
}
func main() {
var wg, wg2 sync.WaitGroup
wg.Add(1)
wg2.Add(10)
found := make(chan bool)
go receiver(found, &wg)
for i := 0; i < 10; i++ {
go worker(found, &wg2)
}
wg2.Wait()
close(found)
wg.Done()
}
So I have seen a lot of ways of implementing one consumer and many producers in Go - the classic fanIn function from the Concurrency in Go talk.
What I want is a fanOut function. It takes as a parameter a channel it reads a value from and returns a slice of channels that it writes copies of this value to.
Is there a correct/recommended way of implementing this?
You pretty much described the best way to do it but here is a small sample of code that does it.
Go playground: https://play.golang.org/p/jwdtDXVHJk
package main
import (
"fmt"
"time"
)
func producer(iters int) <-chan int {
c := make(chan int)
go func() {
for i := 0; i < iters; i++ {
c <- i
time.Sleep(1 * time.Second)
}
close(c)
}()
return c
}
func consumer(cin <-chan int) {
for i := range cin {
fmt.Println(i)
}
}
func fanOut(ch <-chan int, size, lag int) []chan int {
cs := make([]chan int, size)
for i, _ := range cs {
// The size of the channels buffer controls how far behind the recievers
// of the fanOut channels can lag the other channels.
cs[i] = make(chan int, lag)
}
go func() {
for i := range ch {
for _, c := range cs {
c <- i
}
}
for _, c := range cs {
// close all our fanOut channels when the input channel is exhausted.
close(c)
}
}()
return cs
}
func fanOutUnbuffered(ch <-chan int, size int) []chan int {
cs := make([]chan int, size)
for i, _ := range cs {
// The size of the channels buffer controls how far behind the recievers
// of the fanOut channels can lag the other channels.
cs[i] = make(chan int)
}
go func() {
for i := range ch {
for _, c := range cs {
c <- i
}
}
for _, c := range cs {
// close all our fanOut channels when the input channel is exhausted.
close(c)
}
}()
return cs
}
func main() {
c := producer(10)
chans := fanOutUnbuffered(c, 3)
go consumer(chans[0])
go consumer(chans[1])
consumer(chans[2])
}
The important part to note is how we close the output channels once the input channel has been exhausted. Also if one of the output channels blocks on the send it will hold up the send on the other output channels. We control the amount of lag by setting the buffer size of the channels.
This solution below is a bit contrived, but it works for me:
package main
import (
"fmt"
"time"
"crypto/rand"
"encoding/binary"
)
func handleNewChannels(arrchangen chan [](chan uint32),
intchangen chan (chan uint32)) {
currarr := []chan uint32{}
arrchangen <- currarr
for {
newchan := <-intchangen
currarr = append(currarr, newchan)
arrchangen <- currarr
}
}
func sendToChannels(arrchangen chan [](chan uint32)) {
tick := time.Tick(1 * time.Second)
currarr := <-arrchangen
for {
select {
case <-tick:
sent := false
var n uint32
binary.Read(rand.Reader, binary.LittleEndian, &n)
for i := 0 ; i < len(currarr) ; i++ {
currarr[i] <- n
sent = true
}
if sent {
fmt.Println("Sent generated ", n)
}
case newarr := <-arrchangen:
currarr = newarr
}
}
}
func handleChannel(tchan chan uint32) {
for {
val := <-tchan
fmt.Println("Got the value ", val)
}
}
func createChannels(intchangen chan (chan uint32)) {
othertick := time.Tick(5 * time.Second)
for {
<-othertick
fmt.Println("Creating new channel! ")
newchan := make(chan uint32)
intchangen <- newchan
go handleChannel(newchan)
}
}
func main() {
arrchangen := make(chan [](chan uint32))
intchangen := make(chan (chan uint32))
go handleNewChannels(arrchangen, intchangen)
go sendToChannels(arrchangen)
createChannels(intchangen)
}
First, see related question What is the neatest idiom for producer/consumer in Go? and One thread showing interest in another thread (consumer / producer). Also, take a look to producer-consumer problem. About concurrency see how to achieve concurrency In Google Go.
We can handle multiple consumers without making the copy of channel data for each consumer.
Go playground: https://play.golang.org/p/yOKindnqiZv
package main
import (
"fmt"
"sync"
)
type data struct {
msg string
consumers int
}
func main() {
ch := make(chan *data) // both block or non-block are ok
var wg sync.WaitGroup
consumerCount := 3 // specify no. of consumers
producer := func() {
obj := &data {
msg: "hello everyone!",
consumers: consumerCount,
}
ch <- obj
}
consumer := func(idx int) {
defer wg.Done()
obj := <-ch
fmt.Printf("consumer %d received data %v\n", idx, obj)
obj.consumers--
if obj.consumers > 0 {
ch <- obj // forward to others
} else {
fmt.Printf("last receiver: %d\n", idx)
}
}
go producer()
for i:=1; i<=consumerCount; i++ {
wg.Add(1)
go consumer(i)
}
wg.Wait()
}