Deadlock in Housie Program. Producer-Consumer Pattern - go

I am trying to implement a housie game where a goroutine produces numbers, 3 other goroutines check if these are in their tokens and inform the producer if all their numbers were produced. I have implemented it in golang in the following way. This results in a deadlock. Any idea why this is happening? This is a "homework problem", I am just implementing it in go to learn go better.
package main
import (
"fmt"
"math/rand"
)
type PersonID int
func contains(s []int, e int) bool {
for _, a := range s {
if a == e {
return true
}
}
return false
}
func Person(called_number chan int, claim_prize chan PersonID, received chan bool, coupon []int, person_id PersonID) {
numFound := 0
for i := 0; i < len(coupon); i++ {
current_number := <-called_number
found := contains(coupon, current_number)
if found {
numFound++
}
if numFound == len(coupon) {
claim_prize <- person_id
} else {
received <- true
}
}
}
func main() {
var called_number chan int
var claim_prize chan PersonID
var received chan bool
tokens := make([][]int, 3)
for i := 0; i < 3; i++ {
tokens[i] = make([]int, 12)
for j := 0; j < 12; j++ {
num := rand.Intn(100) + 1
found := contains(tokens[i], num)
for found {
num = rand.Intn(100) + 1
found = contains(tokens[i], num)
}
tokens[i][j] = num
}
}
go Person(called_number, claim_prize, received, tokens[0], 0)
go Person(called_number, claim_prize, received, tokens[1], 1)
go Person(called_number, claim_prize, received, tokens[2], 2)
claimants := make([]PersonID, 0)
prev_called := make(map[int]bool)
for i := 0; i < 100; i++ {
if len(claimants) == 3 {
break
}
num := rand.Intn(100) + 1
_, ok := prev_called[num]
for ok {
num = rand.Intn(100) + 1
_, ok = prev_called[num]
}
prev_called[num] = true
called_number <- num
for j := 0; j < 3; j++ {
select {
case _ = <-received:
continue
case pid := <-claim_prize:
claimants = append(claimants, pid)
}
}
}
fmt.Println(claimants)
}
EDIT:
The exact problem is that that the producer needs to send the number to each of the consumers. When a consumer receives all the numbers in it's token, it can claim the prize. Based on what #OneOfOne said, I have made some changes to the program. The changes are that now there is a separate channels for each of the consumers and I am closing it after it claims a prize. Below is the new program, it still deadlocks.
package main
import (
"fmt"
"math/rand"
)
func contains(s []int, e int) bool {
for _, a := range s {
if a == e {
return true
}
}
return false
}
func Person(called_number chan int, claim_prize chan int, received chan bool, coupon []int, person_id int) {
numFound := 0
for current_number := range called_number {
if contains(coupon, current_number) {
numFound++
}
if numFound == len(coupon) {
fmt.Println(person_id)
claim_prize <- person_id
} else {
received <- true
}
}
}
func main() {
var (
called_number1 = make(chan int, 1)
called_number2 = make(chan int, 1)
called_number3 = make(chan int, 1)
claim_prize = make(chan int, 1)
received = make(chan bool, 1)
)
tokens := make([][]int, 3)
for i := 0; i < 3; i++ {
tokens[i] = make([]int, 12)
for j := 0; j < 12; j++ {
num := rand.Intn(100) + 1
found := contains(tokens[i], num)
for found {
num = rand.Intn(100) + 1
found = contains(tokens[i], num)
}
tokens[i][j] = num
}
}
go Person(called_number1, claim_prize, received, tokens[0], 0)
go Person(called_number2, claim_prize, received, tokens[1], 1)
go Person(called_number3, claim_prize, received, tokens[2], 2)
claimants := make([]int, 0)
prev_called := make(map[int]bool)
for i := 0; i < 100; i++ {
if len(claimants) == 3 {
break
}
num := rand.Intn(100) + 1
_, ok := prev_called[num]
for ok {
num = rand.Intn(100) + 1
_, ok = prev_called[num]
}
prev_called[num] = true
if !contains(claimants, 0) {
called_number1 <- num
}
if !contains(claimants, 1) {
called_number2 <- num
}
if !contains(claimants, 2) {
called_number3 <- num
}
for j := 0; j < 3; j++ {
select {
case _ = <-received:
continue
case pid := <-claim_prize:
if pid == 0 { close(called_number1) }
if pid == 1 { close(called_number2) }
if pid == 2 { close(called_number3) }
claimants = append(claimants, pid)
}
}
}
fmt.Println(claimants)
}
EDIT2: This still deadlocked because I was not reducing the number of channels to wait for even after the goroutines were completed. Did that and everything works.

Few problems:
You're using a nil channel, so it just blocks forever, for some reason it's not panicing.
Also, your second inner loop will block indefinitely because it's waiting to read but nothing is being sent.
After your Person's loop ends, called_number <- num will block forever.
//edit
Kinda-working-code : http://play.golang.org/p/3At5nuJTuk
But the logic is flawed, you will have to rethink it.

Related

How to collect values from a channel into a slice in Go?

Suppose I have a helper function helper(n int) which returns a slice of integers of variable length. I would like to run helper(n) in parallel for various values of n and collect the output in one big slice. My first attempt at this is the following:
package main
import (
"fmt"
"golang.org/x/sync/errgroup"
)
func main() {
out := make([]int, 0)
ch := make(chan int)
go func() {
for i := range ch {
out = append(out, i)
}
}()
g := new(errgroup.Group)
for n := 2; n <= 3; n++ {
n := n
g.Go(func() error {
for _, i := range helper(n) {
ch <- i
}
return nil
})
}
if err := g.Wait(); err != nil {
panic(err)
}
close(ch)
// time.Sleep(time.Second)
fmt.Println(out) // should have the same elements as [0 1 0 1 2]
}
func helper(n int) []int {
out := make([]int, 0)
for i := 0; i < n; i++ {
out = append(out, i)
}
return out
}
However, if I run this example I do not get all 5 expected values, instead I get
[0 1 0 1]
(If I uncomment the time.Sleep I do get all five values, [0 1 2 0 1], but this is not an acceptable solution).
It seems that the problem with this is that out is being updated in a goroutine, but the main function returns before it is done updating.
One thing that would work is using a buffered channel of size 5:
func main() {
ch := make(chan int, 5)
g := new(errgroup.Group)
for n := 2; n <= 3; n++ {
n := n
g.Go(func() error {
for _, i := range helper(n) {
ch <- i
}
return nil
})
}
if err := g.Wait(); err != nil {
panic(err)
}
close(ch)
out := make([]int, 0)
for i := range ch {
out = append(out, i)
}
fmt.Println(out) // should have the same elements as [0 1 0 1 2]
}
However, although in this simplified example I know what the size of the output should be, in my actual application this is not known a priori. Essentially what I would like is an 'infinite' buffer such that sending to the channel never blocks, or a more idiomatic way to achieve the same thing; I've read https://blog.golang.org/pipelines but wasn't able to find a close match to my use case. Any ideas?
In this version of the code, the execution is blocked until ch is closed.
ch is always closed at the end of a routine that is responsible to push into ch. Because the program pushes to ch in a routine, it is not needed to use a buffered channel.
package main
import (
"fmt"
"golang.org/x/sync/errgroup"
)
func main() {
ch := make(chan int)
go func() {
g := new(errgroup.Group)
for n := 2; n <= 3; n++ {
n := n
g.Go(func() error {
for _, i := range helper(n) {
ch <- i
}
return nil
})
}
if err := g.Wait(); err != nil {
panic(err)
}
close(ch)
}()
out := make([]int, 0)
for i := range ch {
out = append(out, i)
}
fmt.Println(out) // should have the same elements as [0 1 0 1 2]
}
func helper(n int) []int {
out := make([]int, 0)
for i := 0; i < n; i++ {
out = append(out, i)
}
return out
}
Here is the fixed version of the first code, it is convoluted but demonstrates the usage of sync.WaitGroup.
package main
import (
"fmt"
"sync"
"golang.org/x/sync/errgroup"
)
func main() {
out := make([]int, 0)
ch := make(chan int)
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
for i := range ch {
out = append(out, i)
}
}()
g := new(errgroup.Group)
for n := 2; n <= 3; n++ {
n := n
g.Go(func() error {
for _, i := range helper(n) {
ch <- i
}
return nil
})
}
if err := g.Wait(); err != nil {
panic(err)
}
close(ch)
wg.Wait()
// time.Sleep(time.Second)
fmt.Println(out) // should have the same elements as [0 1 0 1 2]
}
func helper(n int) []int {
out := make([]int, 0)
for i := 0; i < n; i++ {
out = append(out, i)
}
return out
}

Why having a default clause in a goroutine's select makes it slower?

Referring to the following benchmarking test codes:
func BenchmarkRuneCountNoDefault(b *testing.B) {
b.StopTimer()
var strings []string
numStrings := 10
for n := 0; n < numStrings; n++{
s := RandStringBytesMaskImprSrc(10)
strings = append(strings, s)
}
jobs := make(chan string)
results := make (chan int)
for i := 0; i < runtime.NumCPU(); i++{
go RuneCountNoDefault(jobs, results)
}
b.StartTimer()
for n := 0; n < b.N; n++ {
go func(){
for n := 0; n < numStrings; n++{
<-results
}
return
}()
for n := 0; n < numStrings; n++{
jobs <- strings[n]
}
}
close(jobs)
}
func RuneCountNoDefault(jobs chan string, results chan int){
for{
select{
case j, ok := <-jobs:
if ok{
results <- utf8.RuneCountInString(j)
} else {
return
}
}
}
}
func BenchmarkRuneCountWithDefault(b *testing.B) {
b.StopTimer()
var strings []string
numStrings := 10
for n := 0; n < numStrings; n++{
s := RandStringBytesMaskImprSrc(10)
strings = append(strings, s)
}
jobs := make(chan string)
results := make (chan int)
for i := 0; i < runtime.NumCPU(); i++{
go RuneCountWithDefault(jobs, results)
}
b.StartTimer()
for n := 0; n < b.N; n++ {
go func(){
for n := 0; n < numStrings; n++{
<-results
}
return
}()
for n := 0; n < numStrings; n++{
jobs <- strings[n]
}
}
close(jobs)
}
func RuneCountWithDefault(jobs chan string, results chan int){
for{
select{
case j, ok := <-jobs:
if ok{
results <- utf8.RuneCountInString(j)
} else {
return
}
default: //DIFFERENCE
}
}
}
//https://stackoverflow.com/questions/22892120/how-to-generate-a-random-string-of-a-fixed-length-in-golang
const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
const (
letterIdxBits = 6 // 6 bits to represent a letter index
letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits
)
var src = rand.NewSource(time.Now().UnixNano())
func RandStringBytesMaskImprSrc(n int) string {
b := make([]byte, n)
// A src.Int63() generates 63 random bits, enough for letterIdxMax characters!
for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
if remain == 0 {
cache, remain = src.Int63(), letterIdxMax
}
if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
b[i] = letterBytes[idx]
i--
}
cache >>= letterIdxBits
remain--
}
return string(b)
}
When I benchmarked both the functions where one function, RuneCountNoDefault has no default clause in the select and the other, RuneCountWithDefault has a default clause, I'm getting the following benchmark:
BenchmarkRuneCountNoDefault-4 200000 8910 ns/op
BenchmarkRuneCountWithDefault-4 5 277798660 ns/op
Checking the cpuprofile generated by the tests, I noticed that the function with the default clause spends a lot of time in the following channel operations:
Why having a default clause in the goroutine's select makes it slower?
I'm using Go version 1.10 for windows/amd64
The Go Programming Language
Specification
Select statements
If one or more of the communications can proceed, a single one that
can proceed is chosen via a uniform pseudo-random selection.
Otherwise, if there is a default case, that case is chosen. If there
is no default case, the "select" statement blocks until at least one
of the communications can proceed.
Modifying your benchmark to count the number of proceed and default cases taken:
$ go test default_test.go -bench=.
goos: linux
goarch: amd64
BenchmarkRuneCountNoDefault-4 300000 4108 ns/op
BenchmarkRuneCountWithDefault-4 10 209890782 ns/op
--- BENCH: BenchmarkRuneCountWithDefault-4
default_test.go:90: proceeds 114
default_test.go:91: defaults 128343308
$
While other cases were unable to proceed, the default case was taken 128343308 times in 209422470, (209890782 - 114*4108), nanoseconds or 1.63 nanoseconds per default case. If you do something small a large number of times, it adds up.
default_test.go:
package main
import (
"math/rand"
"runtime"
"sync/atomic"
"testing"
"time"
"unicode/utf8"
)
func BenchmarkRuneCountNoDefault(b *testing.B) {
b.StopTimer()
var strings []string
numStrings := 10
for n := 0; n < numStrings; n++ {
s := RandStringBytesMaskImprSrc(10)
strings = append(strings, s)
}
jobs := make(chan string)
results := make(chan int)
for i := 0; i < runtime.NumCPU(); i++ {
go RuneCountNoDefault(jobs, results)
}
b.StartTimer()
for n := 0; n < b.N; n++ {
go func() {
for n := 0; n < numStrings; n++ {
<-results
}
return
}()
for n := 0; n < numStrings; n++ {
jobs <- strings[n]
}
}
close(jobs)
}
func RuneCountNoDefault(jobs chan string, results chan int) {
for {
select {
case j, ok := <-jobs:
if ok {
results <- utf8.RuneCountInString(j)
} else {
return
}
}
}
}
var proceeds ,defaults uint64
func BenchmarkRuneCountWithDefault(b *testing.B) {
b.StopTimer()
var strings []string
numStrings := 10
for n := 0; n < numStrings; n++ {
s := RandStringBytesMaskImprSrc(10)
strings = append(strings, s)
}
jobs := make(chan string)
results := make(chan int)
for i := 0; i < runtime.NumCPU(); i++ {
go RuneCountWithDefault(jobs, results)
}
b.StartTimer()
for n := 0; n < b.N; n++ {
go func() {
for n := 0; n < numStrings; n++ {
<-results
}
return
}()
for n := 0; n < numStrings; n++ {
jobs <- strings[n]
}
}
close(jobs)
b.Log("proceeds", atomic.LoadUint64(&proceeds))
b.Log("defaults", atomic.LoadUint64(&defaults))
}
func RuneCountWithDefault(jobs chan string, results chan int) {
for {
select {
case j, ok := <-jobs:
atomic.AddUint64(&proceeds, 1)
if ok {
results <- utf8.RuneCountInString(j)
} else {
return
}
default: //DIFFERENCE
atomic.AddUint64(&defaults, 1)
}
}
}
//https://stackoverflow.com/questions/22892120/how-to-generate-a-random-string-of-a-fixed-length-in-golang
const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
const (
letterIdxBits = 6 // 6 bits to represent a letter index
letterIdxMask = 1<<letterIdxBits - 1 // All 1-bits, as many as letterIdxBits
letterIdxMax = 63 / letterIdxBits // # of letter indices fitting in 63 bits
)
var src = rand.NewSource(time.Now().UnixNano())
func RandStringBytesMaskImprSrc(n int) string {
b := make([]byte, n)
// A src.Int63() generates 63 random bits, enough for letterIdxMax characters!
for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; {
if remain == 0 {
cache, remain = src.Int63(), letterIdxMax
}
if idx := int(cache & letterIdxMask); idx < len(letterBytes) {
b[i] = letterBytes[idx]
i--
}
cache >>= letterIdxBits
remain--
}
return string(b)
}
Playground: https://play.golang.org/p/DLnAY0hovQG

golang: strange issue with coroutines and channels

I wrote a test code, but do not understand why I get this result.
My sub() should update or return counter, based on the channel value
send 1 = counter++
send 0 = return counter
I start 10 go routines con().
They should simply send many 1 to channel (this increase counter)
I wait 1 sec and send 0 to channel. What value should I get?
I think first, I get a "random" value,
but i get 100000 (ok 10x 10000 is faster than 1 sec)
Now I change
for i:=0; i < 10; i++ {
to
for i:=0; i < 10000; i++ {
and now my returned value is 1
Why!?
Now uncomment fmt.Println(counter) in main().
As you see counter works and has this "random" number
package main
import (
"fmt"
"time"
)
var ch chan int = make(chan int)
var counter int
func main() {
go sub()
for i:=0; i < 10; i++ { //change to 10000
go con()
}
time.Sleep(1000 * time.Millisecond)
ch <- 0
fmt.Println(<- ch)
//fmt.Println(counter) //uncomment this
}
func sub() {
for c := range ch {
if c == 0 { ch <- counter }
if c == 1 { counter++ }
}
}
func con() {
for i := 0; i < 10000; i++ {
ch <- 1
}
}
with 2 channels, this work:
package main
import (
"fmt"
"time"
)
var ch chan int = make(chan int)
var ch2 chan int = make(chan int)
var counter int
func main() {
go sub()
for i:=0; i < 10000; i++ { //change to 10000
go con()
}
time.Sleep(1000 * time.Millisecond)
ch2 <- 0
fmt.Println(<- ch2)
//fmt.Println(counter) //uncomment this
}
func sub() {
for ;; {
select {
case <- ch:
counter++
case <- ch2:
ch2 <- counter
}
}
}
func con() {
for i := 0; i < 10000; i++ {
ch <- 1
}
}

Writing a nested iterator of depth d

How to realize a nested iterator that takes a depth argument. A simple iterator would be when depth = 1. it is a simple iterator which runs like a simple for loop.
func Iter () chan int {
ch := make(chan int);
go func () {
for i := 1; i < 60; i++ {
ch <- i
}
close(ch)
} ();
return ch
}
Output is 1,2,3...59
For depth = 2 Output would be "1,1" "1,2" ... "1,59" "2,1" ... "59,59"
For depth = 3 Output would be "1,1,1" ... "59,59,59"
I want to avoid a nested for loop. What is the solution here ?
I don't know if it is possible to avoid nested loops, but one solution is to use a pipeline of channels. For example:
const ITER_N = 60
// ----------------
func _goFunc1(out chan string) {
for i := 1; i < ITER_N; i++ {
out <- fmt.Sprintf("%d", i)
}
close(out)
}
func _goFuncN(in chan string, out chan string) {
for j := range in {
for i := 1; i < ITER_N; i++ {
out <- fmt.Sprintf("%s,%d", j, i)
}
}
close(out)
}
// ----------------
// create the pipeline
func IterDepth(d int) chan string {
c1 := make(chan string)
go _goFunc1(c1)
var c2 chan string
for ; d > 1; d-- {
c2 = make(chan string)
go _goFuncN(c1, c2)
c1 = c2
}
return c1
}
You can test it with:
func main() {
c := IterDepth(2)
for i := range c {
fmt.Println(i)
}
}
I usually implement iterators using closures. Multiple dimensions don't make the problem much harder. Here's one example of how to do this:
package main
import "fmt"
func iter(min, max, depth int) func() ([]int, bool) {
s := make([]int, depth)
for i := range s {
s[i] = min
}
s[0] = min - 1
return func() ([]int, bool) {
s[0]++
for i := 0; i < depth-1; i++ {
if s[i] >= max {
s[i] = min
s[i+1]++
}
}
if s[depth-1] >= max {
return nil, false
}
return s, true
}
}
func main() {
// Three dimensions, ranging between [1,4)
i := iter(1, 4, 3)
for s, ok := i(); ok; s, ok = i() {
fmt.Println(s)
}
}
Try it out on the Playground.
It'd be a simple change for example to give arguments as a single int slice instead, so that you could have per-dimension limits, if such a thing were necessary.

Deadlock in go function channel

Why is there a deadlock even tho I just pass one and get one output from the channel?
package main
import "fmt"
import "math/cmplx"
func max(a []complex128, base int, ans chan float64, index chan int) {
fmt.Printf("called for %d,%d\n",len(a),base)
maxi_i := 0
maxi := cmplx.Abs(a[maxi_i]);
for i:=1 ; i< len(a) ; i++ {
if cmplx.Abs(a[i]) > maxi {
maxi_i = i
maxi = cmplx.Abs(a[i])
}
}
fmt.Printf("called for %d,%d and found %f %d\n",len(a),base,maxi,base+maxi_i)
ans <- maxi
index <- base+maxi_i
}
func main() {
ans := make([]complex128,128)
numberOfSlices := 4
incr := len(ans)/numberOfSlices
tmp_val := make([]chan float64,numberOfSlices)
tmp_index := make([]chan int,numberOfSlices)
for i,j := 0 , 0; i < len(ans); j++{
fmt.Printf("From %d to %d - %d\n",i,i+incr,len(ans))
go max(ans[i:i+incr],i,tmp_val[j],tmp_index[j])
i = i+ incr
}
//After Here is it stops deadlock
maximumFreq := <- tmp_index[0]
maximumMax := <- tmp_val[0]
for i := 1; i < numberOfSlices; i++ {
tmpI := <- tmp_index[i]
tmpV := <- tmp_val[i]
if(tmpV > maximumMax ) {
maximumMax = tmpV
maximumFreq = tmpI
}
}
fmt.Printf("Max freq = %d",maximumFreq)
}
For those reading this question and perhaps wondering why his code failed here's an explanation.
When he constructed his slice of channels like so:
tmp_val := make([]chan float64,numberOfSlices)
He made slice of channels where every index was to the channels zero value. A channels zero value is nil since channels are reference types and a nil channel blocks on send forever and since there is never anything in a nil channel it will also block on recieve forever. Thus you get a deadlock.
When footy changes his code to construct each channel individually using
tmp_val[i] = make(chan float64)
in a loop he constructs non-nil channels and everything is good.
I was wrong in making of the chan. Should have done
numberOfSlices := 4
incr := len(ans)/numberOfSlices
var tmp_val [4]chan float64
var tmp_index [4]chan int
for i := range tmp_val {
tmp_val[i] = make(chan float64)
tmp_index[i] = make(chan int)
}
for i,j := 0 , 0; i < len(ans); j++{
fmt.Printf("From %d to %d [j:%d] - %d\n",i,i+incr,j,len(ans))
go maximumFunc(ans[i:i+incr],i,tmp_val[j],tmp_index[j])
i = i+ incr
}

Resources