I was trying to debug this code but am stuck here. I wanted to access ch1, ch2 but found printed nothing.
package main
import (
"fmt"
)
type degen struct {
i, j string
}
func (x degen) CVIO(ch1, ch2 chan string, quit chan int, m, n string) {
for {
select {
case ch1 <- m:
fmt.Println(x.i)
case ch2 <- n:
fmt.Println("ok")
case <-quit:
fmt.Println("quit")
return
}
}
}
func main() {
ch1 := make(chan string)
ch2 := make(chan string)
quit := make(chan int)
x := degen{"goosebump", "ok"}
go x.CVIO(ch1, ch2, quit, "goosebump", "ok")
}
Desired:
It should print the channel data as to be produced.
Its not really clear what you expect your code to do:
main() ends without waiting for the go routine to exit (its quite possible it the loop will not run at all).
in the select the sends will not proceed because there is no receiver (spec - "if the capacity is zero or absent, the channel is unbuffered and communication succeeds only when both a sender and receiver are ready.").
Nothing is sent to the quit channel.
I suspect that the following (playground) might do what you were expecting.
package main
import (
"fmt"
"sync"
)
type degen struct {
i, j string
}
func (x degen) CVIO(ch1, ch2 chan string, quit chan int, m, n string) {
for {
select {
case ch1 <- m:
fmt.Println(x.i)
case ch2 <- n:
fmt.Println("ok")
case <-quit:
fmt.Println("quit")
return
}
}
}
func main() {
ch1 := make(chan string)
ch2 := make(chan string)
quit := make(chan int)
x := degen{"goosebump", "ok"}
var wg sync.WaitGroup
wg.Add(1)
go func() {
x.CVIO(ch1, ch2, quit, "goosebump", "ok")
wg.Done()
}()
<-ch1 // Receive from CH1 (allowing "ch1 <- m" in go routine to proceed)
<-ch2 // Receive from CH2 (allowing "ch2 <- n" in go routine to proceed)
quit <- 1
wg.Wait() // Wait for CVIO to end (which it should do due to above send)
}
Related
The problem is that both the goOne and goTwo functions are sending values to the channels ch1 and ch2 respectively, but there is no corresponding receiver for these values in the main function. This means that the channels are blocked and the program is unable to proceed. As a result, the select statement in the main function is unable to read from the channels, so it always executes the default case.
package main
import (
"fmt"
"sync"
)
func main() {
var wg sync.WaitGroup
ch1 := make(chan string)
ch2 := make(chan string)
wg.Add(2)
go goOne(&wg, ch1)
go goTwo(&wg, ch2)
select {
case <-ch1:
fmt.Println(<-ch1)
close(ch1)
case <-ch2:
fmt.Println(<-ch2)
close(ch2)
default:
fmt.Println("Default Case")
}
wg.Wait()
}
func goTwo(wg *sync.WaitGroup, ch2 chan string) {
ch2 <- "Channel 2"
wg.Done()
}
func goOne(wg *sync.WaitGroup, ch1 chan string) {
ch1 <- "Channel 1"
wg.Done()
}
Output:
Default Case
fatal error: all goroutines are asleep - deadlock!
goroutine 1 \[semacquire\]:
sync.runtime_Semacquire(0xc000108270?)
/usr/local/go/src/runtime/sema.go:62 +0x25
sync.(\*WaitGroup).Wait(0x4b9778?)
/usr/local/go/src/sync/waitgroup.go:139 +0x52
main.main()
/home/nidhey/Documents/Go_Learning/goroutines/select.go:29 +0x2af
goroutine 6 \[chan send\]:
main.goOne(0x0?, 0x0?)
/home/nidhey/Documents/Go_Learning/goroutines/select.go:39 +0x28
created by main.main
/home/nidhey/Documents/Go_Learning/goroutines/select.go:14 +0xc5
goroutine 7 \[chan send\]:
main.goTwo(0x0?, 0x0?)
/home/nidhey/Documents/Go_Learning/goroutines/select.go:33 +0x28
created by main.main
/home/nidhey/Documents/Go_Learning/goroutines/select.go:15 +0x119\```
I'm looking for a different pattern such as select to handle the case when the channels are blocked.
To fix the issue, I've added a <-ch1 or <-ch2 in the main function after wg.Wait() to receive the values sent to the channels and unblock them
It's not entirely clear what you want to do. If you want to wait for both goroutines to complete their work and get the result of their work into the channel, you don't need a weight group (because it won't be reached).
You can do something like this.
package main
import (
"fmt"
"time"
)
func main() {
ch1 := make(chan string)
ch2 := make(chan string)
go goOne(ch1)
go goTwo(ch2)
for {
select {
case v := <-ch1:
fmt.Println("Done ch1:", v)
ch1 = nil
case v := <-ch2:
fmt.Println("Done ch2:", v)
ch2 = nil
case <-time.After(time.Second):
fmt.Println("I didn't get result so lets skip it!")
ch1, ch2 = nil, nil
}
if ch1 == nil && ch2 == nil {
break
}
}
}
func goTwo(ch chan string) {
ch <- "Channel 2"
}
func goOne(_ chan string) {
//ch1 <- "Channel 1"
}
UPD:
Imagine if we are having two api end points, API1 & API2 which are returning same data but are hosted on different regions. So what I want to do, I need to make API calls for both apis in two different function ie goroutines and as soon as any one api sends us response back, I want to process the data received. So for that Im check whcih api is fetching data first using select block.
package main
import (
"context"
"fmt"
"math/rand"
"time"
)
func main() {
regions := []string{
"Europe",
"America",
"Asia",
}
// Just for different results for each run
rand.Seed(time.Now().UnixNano())
rand.Shuffle(len(regions), func(i, j int) { regions[i], regions[j] = regions[j], regions[i] })
output := make(chan string)
ctx, cancel := context.WithTimeout(context.Background(), 2 * time.Second)
for i, region := range regions {
go func(ctx context.Context, region string, output chan <- string, i int) {
// Do call with context
// If context will be cancelled just ignore it here
timeout := time.Duration(i)*time.Second
fmt.Printf("Call %s (with timeout %s)\n", region, timeout)
time.Sleep(timeout) // Simulate request timeout
select {
case <-ctx.Done():
fmt.Println("Cancel by context:", region)
case output <- fmt.Sprintf("Answer from `%s`", region):
fmt.Println("Done:", region)
}
}(ctx, region, output, i)
}
select {
case v := <-output:
cancel() // Cancel all requests in progress (if possible)
// Do not close output chan to avoid panics: When the channel is no longer used, it will be garbage collected.
fmt.Println("Result:", v)
case <-ctx.Done():
fmt.Println("Timeout by context done")
}
fmt.Println("There is we already have result or timeout, but wait a little bit to check all is okay")
time.Sleep(5*time.Second)
}
Firstly you have a race condition in that your channel publishing goroutines will probably not have been started by the time you enter the select statement, and it will immediately fall through to the default.
But assuming you resolve this (e.g. with another form of semaphore) you're on to the next issue - your select statement will either get chan1 or chan2 message, then wait at the bottom of the method, but since it is no longer in the select statement, one of your messages won't have arrived and you'll be waiting forever.
You'll need either a loop (twice in this case) or a receiver for both channels running in their own goroutines to pick up both messages and complete the waitgroup.
But in any case - as others have queried - what are you trying to achieve?
You can try something like iterating over a single channel (assuming both channels return the same type of data) and then count in the main method how many tasks are completed. Then close the channel once all the tasks are done. Example:
package main
import (
"fmt"
)
func main() {
ch := make(chan string)
go goOne(ch)
go goTwo(ch)
doneCount := 0
for v := range ch {
fmt.Println(v)
doneCount++
if doneCount == 2 {
close(ch)
}
}
}
func goTwo(ch chan string) {
ch <- "Channel 2"
}
func goOne(ch chan string) {
ch <- "Channel 1"
}
In the code hereunder, I don't understand why the "Worker" methods seem to exit instead of pulling values from the input channel "in" and processing them.
I had assumed they would only return after having consumed all input from the input channel "in" and processing them
package main
import (
"fmt"
"sync"
)
type ParallelCallback func(chan int, chan Result, int, *sync.WaitGroup)
type Result struct {
i int
val int
}
func Worker(in chan int, out chan Result, id int, wg *sync.WaitGroup) {
for item := range in {
item *= item // returns the square of the input value
fmt.Printf("=> %d: %d\n", id, item)
out <- Result{item, id}
}
wg.Done()
fmt.Printf("%d exiting ", id)
}
func Run_parallel(n_workers int, in chan int, out chan Result, Worker ParallelCallback) {
wg := sync.WaitGroup{}
for id := 0; id < n_workers; id++ {
fmt.Printf("Starting : %d\n", id)
wg.Add(1)
go Worker(in, out, id, &wg)
}
wg.Wait() // wait for all workers to complete their tasks
close(out) // close the output channel when all tasks are completed
}
const (
NW = 4
)
func main() {
in := make(chan int)
out := make(chan Result)
go func() {
for i := 0; i < 100; i++ {
in <- i
}
close(in)
}()
Run_parallel(NW, in, out, Worker)
for item := range out {
fmt.Printf("From out : %d: %d", item.i, item.val)
}
}
The output is
Starting : 0
Starting : 1
Starting : 2
Starting : 3
=> 3: 0
=> 0: 1
=> 1: 4
=> 2: 9
fatal error: all goroutines are asleep - deadlock!
fatal error: all goroutines are asleep - deadlock!
The full error shows where each goroutine is "stuck". If you run this in the playground, it will even show you the line number. That made it easy for me to diagnose.
Your Run_parallel runs in the main groutine, so before main can read from out, Run_parallel must return. Before Run_parallel can return, it must wg.Wait(). But before the workers call wg.Done(), they must write to out. That's what causes a deadlock.
One solution is simple: just run Run_parallel concurrently in its own Goroutine.
go Run_parallel(NW, in, out, Worker)
Now, main ranges over out, waiting on outs closure to signal completion. Run_parallel waits for the workers with wg.Wait(), and the workers will range over in. All the work will get done, and the program won't end until it's all done. (https://go.dev/play/p/oMrgH2U09tQ)
Solution :
Run_parallel has to run in it’s own goroutine:
package main
import (
"fmt"
"sync"
)
type ParallelCallback func(chan int, chan Result, int, *sync.WaitGroup)
type Result struct {
id int
val int
}
func Worker(in chan int, out chan Result, id int, wg *sync.WaitGroup) {
defer wg.Done()
for item := range in {
item *= 2 // returns the double of the input value (Bogus handling of data)
out <- Result{id, item}
}
}
func Run_parallel(n_workers int, in chan int, out chan Result, Worker ParallelCallback) {
wg := sync.WaitGroup{}
for id := 0; id < n_workers; id++ {
wg.Add(1)
go Worker(in, out, id, &wg)
}
wg.Wait() // wait for all workers to complete their tasks
close(out) // close the output channel when all tasks are completed
}
const (
NW = 8
)
func main() {
in := make(chan int)
out := make(chan Result)
go func() {
for i := 0; i < 10; i++ {
in <- i
}
close(in)
}()
go Run_parallel(NW, in, out, Worker)
for item := range out {
fmt.Printf("From out [%d]: %d\n", item.id, item.val)
}
println("- - - All done - - -")
}
Alternative formulation of the solution:
In that alternative formulation , it is not necessary to start Run_parallel as a goroutine (it triggers its own goroutine).
I prefer that second solution, because it automates the fact that Run_parallel() has to run parallel to the main function. Also, for the same reason it's safer, less error-prone (no need to remember to run Run_parallel with the go keyword).
package main
import (
"fmt"
"sync"
)
type ParallelCallback func(chan int, chan Result, int, *sync.WaitGroup)
type Result struct {
id int
val int
}
func Worker(in chan int, out chan Result, id int, wg *sync.WaitGroup) {
defer wg.Done()
for item := range in {
item *= 2 // returns the double of the input value (Bogus handling of data)
out <- Result{id, item}
}
}
func Run_parallel(n_workers int, in chan int, out chan Result, Worker ParallelCallback) {
go func() {
wg := sync.WaitGroup{}
defer close(out) // close the output channel when all tasks are completed
for id := 0; id < n_workers; id++ {
wg.Add(1)
go Worker(in, out, id, &wg)
}
wg.Wait() // wait for all workers to complete their tasks *and* trigger the -differed- close(out)
}()
}
const (
NW = 8
)
func main() {
in := make(chan int)
out := make(chan Result)
go func() {
defer close(in)
for i := 0; i < 10; i++ {
in <- i
}
}()
Run_parallel(NW, in, out, Worker)
for item := range out {
fmt.Printf("From out [%d]: %d\n", item.id, item.val)
}
println("- - - All done - - -")
}
I am prototyping a series of go routines for a pipeline that each perform a transformation. The routines are terminating before all the data has passed through.
I have checked Donavan and Kernighan book and Googled for solutions.
Here is my code:
package main
import (
"fmt"
"sync"
)
func main() {
a1 := []string{"apple", "apricot"}
chan1 := make(chan string)
chan2 := make(chan string)
chan3 := make(chan string)
var wg sync.WaitGroup
go Pipe1(chan2, chan1, &wg)
go Pipe2(chan3, chan2, &wg)
go Pipe3(chan3, &wg)
func (data []string) {
defer wg.Done()
for _, s := range data {
wg.Add(1)
chan1 <- s
}
go func() {
wg.Wait()
close(chan1)
}()
}(a1)
}
func Pipe1(out chan<- string, in <-chan string, wg *sync.WaitGroup) {
defer wg.Done()
for s := range in {
wg.Add(1)
out <- s + "s are"
}
}
func Pipe2(out chan<- string, in <-chan string, wg *sync.WaitGroup) {
defer wg.Done()
for s := range in {
wg.Add(1)
out <- s + " good for you"
}
}
func Pipe3(in <-chan string, wg *sync.WaitGroup) {
defer wg.Done()
for s := range in {
wg.Add(1)
fmt.Println(s)
}
}
My expected output is:
apples are good for you
apricots are good for you
The results of running main are inconsistent. Sometimes I get both lines. Sometimes I just get the apples. Sometimes nothing is output.
As Adrian already pointed out, your WaitGroup.Add and WaitGroup.Done calls are mismatched. However, in cases like this the "I am done" signal is typically given by closing the output channel. WaitGroups are only necessary if work is shared between several goroutines (i.e. several goroutines consume the same channel), which isn't the case here.
package main
import (
"fmt"
)
func main() {
a1 := []string{"apple", "apricot"}
chan1 := make(chan string)
chan2 := make(chan string)
chan3 := make(chan string)
go func() {
for _, s := range a1 {
chan1 <- s
}
close(chan1)
}()
go Pipe1(chan2, chan1)
go Pipe2(chan3, chan2)
// This range loop terminates when chan3 is closed, which Pipe2 does after
// chan2 is closed, which Pipe1 does after chan1 is closed, which the
// anonymous goroutine above does after it sent all values.
for s := range chan3 {
fmt.Println(s)
}
}
func Pipe1(out chan<- string, in <-chan string) {
for s := range in {
out <- s + "s are"
}
close(out) // let caller know that we're done
}
func Pipe2(out chan<- string, in <-chan string) {
for s := range in {
out <- s + " good for you"
}
close(out) // let caller know that we're done
}
Try it on the playground: https://play.golang.org/p/d2J4APjs_lL
You're calling wg.Wait in a goroutine, so main is allowed to return (and therefore your program exits) before the other routines have finished. This would cause the behavior your see, but taking out of a goroutine alone isn't enough.
You're also misusing the WaitGroup in general; your Add and Done calls don't relate to one another, and you don't have as many Dones as you have Adds, so the WaitGroup will never finish. If you're calling Add in a loop, then every loop iteration must also result in a Done call; as you have it now, you defer wg.Done() before each of your loops, then call Add inside the loop, resulting in one Done and many Adds. This code would need to be significantly revised to work as intended.
I have made a simple code example to understand the usage of pipeline, here it is.
package main
import (
"fmt"
"sync"
"time"
)
func main() {
ch1 := make(chan int, 10) // Use buffered channel so as to avoid clogging
ch2 := make(chan string, 10)
var wg sync.WaitGroup
for i := 0; i < 3; i++ {
wg.Add(1)
go func1(i, ch1, &wg)
go func2(ch1, ch2)
}
wg.Wait()
close(ch1)
for val := range ch2 {
fmt.Println(val)
}
}
func func1(seconds int, ch chan<- int, wg *sync.WaitGroup) {
defer wg.Done()
time.Sleep(time.Duration(seconds) * time.Second)
fmt.Println(seconds)
ch <- seconds
}
func func2(ch1 chan int, ch2 chan string) {
for range ch1 {
ch2 <- "hello"
}
close(ch2)
}
Now, the problem is I don't get consistent output ( I understand it's some concurrency issue, which I haven't fully understood ).
Output
> go run pipeline-loop.go
0
1
2
hello
hello
> go run pipeline-loop.go
0
1
2
hello
hello
hello
> go run pipeline-loop.go
0
1
2
hello
hello
> go run pipeline-loop.go
0
1
2
hello
hello
> go run pipeline-loop.go
0
1
2
hello
hello
panic: close of closed channel
goroutine 6 [running]:
main.func2(0xc00006c000, 0xc000056180)
/home/projects/go-tuts/pipeline-loop.go:36 +0x72
created by main.main
/home/projects/go-tuts/pipeline-loop.go:16 +0x10f
exit status 2
Another guy changed the code ( and it was working ) and put func2 outside the loop but I want func2 for each iteration of the func1.
Problem
So, I want to understand as to where should the WaitGroup and close(ch) be used ?
Thanks.
Temporarya
( A golang noobie )
Update
Based on a user's answer, I changed the code, now I get output as expected ( but not the solution to this question ) but still there's a deadlock.
https://play.golang.org/p/O_rp_FLvNh8
package main
import (
"fmt"
"sync"
"time"
)
func main() {
ch1 := make(chan int, 10) // Use buffered channel so as to avoid clogging
ch2 := make(chan string, 10)
var wg1 sync.WaitGroup
// var wg2 sync.WaitGroup
for i := 0; i < 3; i++ {
wg1.Add(1)
go func1(i, ch1)
go func2(ch1, ch2, &wg1)
}
for val := range ch2 {
fmt.Println(val)
}
wg1.Wait()
close(ch1)
close(ch2)
}
// func func1(seconds int, ch chan<- int, wg *sync.WaitGroup) {
func func1(seconds int, ch chan<- int) {
// defer wg.Done()
time.Sleep(time.Duration(seconds) * time.Second)
fmt.Println(seconds)
ch <- seconds
}
func func2(ch1 chan int, ch2 chan string, wg *sync.WaitGroup) {
defer wg.Done()
for range ch1 {
ch2 <- "hello"
}
}
There are multiple problems in your code.
In the loop, you are spawning multiple (3) goroutines that runs func2, and in func2, you send data to ch2 and call close(ch2). This is a problem. It can happen that when one goroutine is seding data to ch2, the other has closed that channel, which causes:
panic: close of closed channel
goroutine 6 [running]:
main.func2(0xc00006c000, 0xc000056180)
/home/projects/go-tuts/pipeline-loop.go:36 +0x72
created by main.main
/home/projects/go-tuts/pipeline-loop.go:16 +0x10f
exit status 2
In general, you don't need close a chanel multiple times - you only need to shut them once they are all finished. You need another WaitGroup for this; you need to pass both functions a WaitGroup.
Further reading: https://blog.golang.org/pipelines
UPDATE:
Personally I use a pattern for "works" that produce data into a same channel and the channel needs to be closed after all works are done:
for something {
wg.Add(1)
go func(i int) {
work(ch)
wg.Done()
}
}
go func() {
wg.Wait()
close()
}()
I think it is a good idea that keeps the API clean from WorkGroup as WorkGroup is about how you sync the work instead of how the work is done.
I have changed your code into this pattern: https://play.golang.org/p/vdCNsxWhgyQ
I suspect you only want one channel to read from ch1 and write to ch2. There is not much point in creating 3 go-routines to do the same thing (and you also end up closing the same channel mutliple time which causes a panic as leaf bebop pointed out)
func main() {
ch1 := make(chan int, 10) // Use buffered channel so as to avoid clogging
ch2 := make(chan string, 10)
var wg sync.WaitGroup
for i := 0; i < 3; i++ {
wg.Add(1)
go func1(i, ch1, &wg)
}
go func2(ch1, ch2)
wg.Wait()
close(ch1)
for val := range ch2 {
fmt.Println(val)
}
}
I am trying to figure out whether I can reopen a channel after closing it.
Test case:
I have a channel with some things in it
I want to range over them therefore I need to close the channel beforehand
I want to put more stuff in the channel and iterate through it once more
go func() {
queue <- "1"
queue <- "2"
close(queue)
}()
for i := range queue {
go func(i string) {
fmt.Println("From queue: ", i)
}(i)
}
go func() {
open(queue)
queue <- "3"
queue <- "4"
close(queue)
}()
for i := range queue {
go func(i string) {
fmt.Println("From queue: ", i)
}(i)
}
Of course open does not exist. How can I implement something like what I need in Go?
I don't want to use the Sleep function
I want to range over them therefore I need to close the channel beforehand
Nope, no need to close the channel. It'll resume iteration when another item is pushed thru the channel.
The below code accepts console input and pushes it to the channel:
main.go
package main
import (
"log"
"bufio"
"os"
"fmt"
)
func process(c chan string) {
for s := range c {
log.Println("processed", s)
}
}
func main() {
c := make(chan string, 10)
go process(c)
// get from console and process
reader := bufio.NewReader(os.Stdin)
fmt.Println("INPUT STUFF. TYPE #quit TO EXIT.")
for {
input, _, _ := reader.ReadLine()
if string(input) == "#quit" {
break
}
c <- string(input)
}
log.Println("BYE!")
}
output
INPUT STUFF. TYPE #quit TO EXIT.
hello
2018/10/23 10:43:52 processed hello
world
2018/10/23 10:43:54 processed world
#quit
2018/10/23 10:43:57 BYE!
The below sample uses Sleep() and is runnable as a Go Playground snippet
package main
import (
"log"
"time"
)
func process(c chan string) {
for s := range c {
log.Println("processed", s)
}
}
func main() {
c := make(chan string, 10)
go process(c)
// push some data
c <- "barry allen"
c <- "iris west"
time.Sleep(time.Second * 2)
// push more data
c <- "joe west"
c <- "caitlin snow"
time.Sleep(time.Second * 3)
}
output
2009/11/10 23:00:00 processed barry allen
2009/11/10 23:00:00 processed iris west
2009/11/10 23:00:02 processed joe west
2009/11/10 23:00:02 processed caitlin snow
Hope this helps. Cheers,
You cannot reopen a closed channel but you can send a channel on a channel, perhaps this is what you're looking for?
package main
import (
"fmt"
"time"
)
func main() {
queue := make(chan chan int)
defer close(queue)
go func() { // reader
for {
ch := <-queue
for i := range ch {
fmt.Println(i)
}
fmt.Println("Done with this channel")
}
}()
go func() { // writer-1
ch := make(chan int)
defer close(ch)
queue <- ch
ch <- 4
ch <- 2
}()
go func() { // writer-2
ch := make(chan int)
defer close(ch)
queue <- ch
ch <- 4
ch <- 20
}()
time.Sleep(time.Second)
}
Tho you cannot reopen the channel, it is possible to create a new one and assign to the variable. The previously closed channel will be garbage collected. In this case I reused the queue variable, but you can also create a new queue2 variable and pass it to the goroutine.
package main
import (
"context"
"fmt"
"time"
)
func main() {
ctx := context.Background()
ctx, cancel := context.WithCancel(ctx)
queue := make(chan int)
go printFormat(nil, queue)
queue <- 1
queue <- 2
close(queue)
// fake some actions in between
time.Sleep(2 * time.Second)
queue = make(chan int)
go printFormat(cancel, queue)
queue <- 3
queue <- 4
close(queue)
<-ctx.Done()
}
func printFormat(c context.CancelFunc, q chan int) {
for i := range q {
fmt.Printf("Data %d \n", i)
}
if c != nil {
c()
}
}