Synchronization for several goroutines using channels - go

I need to start a number of workers with single task queue and single result queue. Each worker should be started in different goroutine. And I need to wait till all workers will be finished and task queue will be empty before exiting from program.
I have prepare small example for goroutine synchronization.
The main idea was that we count tasks in queue and waiting for all workers to finish jobs. But current implementation sometime miss values.
Why this happends and how to solve the problem?
The sample code:
import (
"fmt"
"os"
"os/signal"
"strconv"
)
const num_workers = 5
type workerChannel chan uint64
// Make channel for tasks
var workCh workerChannel
// Make channel for task counter
var cntChannel chan int
// Task counter
var tskCnt int64
// Worker function
func InitWorker(input workerChannel, result chan string, num int) {
for {
select {
case inp := <-input:
getTask()
result <- ("Worker " + strconv.Itoa(num) + ":" + strconv.FormatUint(inp, 10))
}
}
}
// Function to manage task counter
// should be in uniq goroutine
func taskCounter(inp chan int) {
for {
val := <-inp
tskCnt += int64(val)
}
}
// Put pask to the queue
func putTask(val uint64) {
func() {
fmt.Println("Put ", val)
cntChannel <- int(1)
workCh <- val
}()
}
// Get task from queue
func getTask() {
func() {
cntChannel <- int(-1)
}()
}
func main() {
// Init service channels
abort := make(chan os.Signal)
done := make(chan bool)
// init queue for results
result := make(chan string)
// init task queue
workCh = make(workerChannel)
// start some workers
for i := uint(0); i < num_workers; i++ {
go InitWorker(workCh, result, int(i))
}
// init counter for synchro
cntChannel = make(chan int)
go taskCounter(cntChannel)
// goroutine that put some tasks into queue
go func() {
for i := uint(0); i < 21; i++ {
putTask(uint64(i))
}
// wait for processing all tasks and close application
for len(cntChannel) != 0 {}
for tskCnt != 0 {}
for len(workCh) != 0 {}
for len(result) != 0 {}
// send signal for close
done <- true
}()
signal.Notify(abort, os.Interrupt)
for {
select {
case <-abort:
fmt.Println("Aborted.")
os.Exit(0)
// print results
case res := <-result:
fmt.Println(res)
case <-done:
fmt.Println("Done")
os.Exit(0)
}
}
}

Use sync.WaitGroup to wait for goroutines to complete. Close channels to cause loops reading on channels to exit.
package main
import (
"fmt"
"sync"
)
type workerChannel chan uint64
const num_workers = 5
func main() {
results := make(chan string)
workCh := make(workerChannel)
// Start workers
var wg sync.WaitGroup
wg.Add(num_workers)
for i := 0; i < num_workers; i++ {
go func(num int) {
defer wg.Done()
// Loop processing work until workCh is closed
for w := range workCh {
results <- fmt.Sprintf("worker %d, task %d", num, w)
}
}(i)
}
// Close result channel when workers are done
go func() {
wg.Wait()
close(results)
}()
// Send work to be done
go func() {
for i := 0; i < 21; i++ {
workCh <- uint64(i)
}
// Closing the channel causes workers to break out of loop
close(workCh)
}()
// Process results. Loop exits when result channel is closed.
for r := range results {
fmt.Println(r)
}
}
https://play.golang.org/p/ZifpzsP6fNv

I suggest using close(chan) for this kind of tasks.
WaitGroup version.
package main
import (
"log"
"sync"
)
func worker(in chan int, wg *sync.WaitGroup) {
defer wg.Done()
for i := range in {
log.Println(i)
}
}
func main() {
in := make(chan int)
lc := 25
maxValue := 30
wg := sync.WaitGroup{}
wg.Add(lc)
for i := 0; i < lc; i++ {
go worker(in, &wg)
}
for c := 0; c <= maxValue; c++ {
in <- c
}
close(in)
wg.Wait()
}
Channel version
package main
import (
"log"
"os"
)
func worker(in chan int, end chan struct{}) {
defer func() { end <- struct{}{} }()
for i := range in {
log.Println(i)
}
}
func main() {
in := make(chan int)
lc := 25
maxValue := 30
end := make(chan struct{})
var fin int
go func() {
for {
<-end
fin++
log.Println(`fin`, fin)
if fin == lc {
break
}
}
close(end)
os.Exit(0)
}()
for i := 0; i < lc; i++ {
go worker(in, end)
}
for c := 0; c <= maxValue; c++ {
in <- c
}
close(in)
<-make(chan struct{})
}

Related

Execute multiple independent jobs continuously

I have a set of jobs which are independent of each other. Hence each of these jobs can be run concurrently using goroutines. Note that once a single job completes, it should wait for few seconds and start again (applies to all the jobs) and this goes on in a loop until the Go API service stops. Also note that all these jobs execute the same goroutine (makes a REST call). What would be the best pattern to implement this in Go. Please note that I would also want to wait for currently executing jobs to complete before my service shuts down.
If I got you right, you are looking for something likes this
This is a service with a consumer pool to execute jobs concurrently. When a job is done, it will repeat again after a interval until you stop the service.
type job struct {
id int
result chan error
}
func newJob(id int) job {
return job{
id: id,
result: make(chan error, 1),
}
}
type service struct {
pending chan job
consumerLimit int
repeatInterval time.Duration
isClosed chan struct{}
shutdown chan chan error
}
func newService(repeatInterval time.Duration, consumerLimit int, pendingChannelSize int) *service {
s := &service{
pending: make(chan job, pendingChannelSize),
consumerLimit: consumerLimit,
repeatInterval: repeatInterval,
isClosed: make(chan struct{}, consumerLimit),
shutdown: make(chan chan error),
}
for i := 0; i < s.consumerLimit; i++ {
go s.consumer()
}
return s
}
func (s *service) do(ctx context.Context, job job) error {
select {
case <-ctx.Done():
return ctx.Err()
case s.pending <- job:
return <-job.result
case <-s.isClosed:
return errors.New("service has been shut down")
}
}
func (s *service) consumer() {
for {
select {
case j := <-s.pending:
//Simulate working process
time.Sleep(time.Duration(rand.Intn(200)) + 200)
j.result <- nil
fmt.Println(fmt.Sprintf("job %v is done", j.id))
go func() {
//Repeat after a time
time.Sleep(s.repeatInterval)
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
if err := s.do(ctx, newJob(j.id)); err != nil {
fmt.Println(fmt.Errorf("failed to send job to repeat: %v", err))
}
}()
case result := <-s.shutdown:
result <- nil
return
}
}
}
func (s *service) close() error {
result := make(chan error, 1)
for i := 0; i < s.consumerLimit; i++ {
s.shutdown <- result
}
close(s.isClosed)
return <-result
}
func main() {
interrupt := make(chan os.Signal, 1)
signal.Notify(interrupt, os.Interrupt)
service := newService(time.Second, 5, 1000)
//Assign jobs
for i := 1; i < 10; i++ {
go func(i int) {
if err := service.do(context.Background(), newJob(i)); err != nil {
fmt.Println(fmt.Errorf("failed to send job: %v", err))
}
}(i)
}
select {
case <-interrupt:
switch err := service.close(); err {
case nil:
fmt.Println("service has been shutdown successfully")
default:
fmt.Println(fmt.Errorf("failed to graceful shut down service: %w", err))
}
return
}
}
If I understand correctly, you are looking for something like this.
This code will run the workers in a loop, the workers run parallel as a group until you exit the program sending an end signal, but wait for the current loop to finihish before exiting.
func main() {
srv := server{
workers: 5,
}
srv.Run()
}
// inspired by: https://goinbigdata.com/golang-wait-for-all-goroutines-to-finish/#:~:text=A%20WaitGroup%20allows%20to%20wait,until%20all%20goroutines%20have%20finished.
func work(wg *sync.WaitGroup, i int) {
defer wg.Done()
rand.Seed(time.Now().UnixNano())
n := rand.Intn(10)
fmt.Printf("Worker %v: Started\n", i)
time.Sleep(time.Duration(n) * time.Second)
fmt.Printf("Worker %v: Finished\n", i)
}
type server struct {
running bool
workers int
}
func (srv *server) Run() {
done := make(chan bool, 1) // this channel
signalCh := make(chan os.Signal, 1) // this channel will get a signal on system call
signal.Notify(signalCh, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-signalCh
srv.running = false
done <- true
}()
srv.running = true
for srv.running {
var wg sync.WaitGroup
for i := 0; i < srv.workers; i++ {
wg.Add(1)
go work(&wg, i)
}
wg.Wait()
}
<-done
}
You want to implement worker pools. Here is the simple way to create pools of worker. You can customize worker method and jobs type according to your requirement.
package main
import (
"fmt"
"sync"
)
type Jobs struct {
ID string
// or anything you want to add
}
func main() {
jobs := make(Jobs)
var wg sync.WaitGroup
numWorker := 16
for i := 0; i < numWorker; i++ {
wg.Add(1)
go func() {
worker(jobs)
wg.Done()
}()
}
tasks := []Jobs{}
// inset your task here
for _, i := range tasks {
jobs <- i
}
close(jobs)
wg.Wait()
}
func worker(jobs chan Jobs) {
for job := range jobs {
// do whatever you want to do
doSomething(job)
}
}
func doSomething(job Jobs) {
fmt.Println(job)
}

Stuck in infinite loop in for select in Golang

The code given below is the sample code for my use case. I want to read data from ch1 and ch2 but got stuck into infinite loop.
package main
import "fmt"
func main() {
ch1, ch2 := func() (<-chan int, <-chan int) {
ch_1 := make(chan int)
ch_2 := make(chan int)
go worker_1(ch_1, ch_2)
go worker_2(ch_1, ch_2)
return ch_1, ch_2
}()
// trying to read this way but it is not working
for {
select {
case a := <-ch1:
fmt.Println("from ch1", a)
case a := <-ch2:
fmt.Println("from ch2", a)
default:
fmt.Println("done")
}
}
}
func worker_1(ch1, ch2 chan int) {
for i := 0; i < 100; i++ {
if i%2 == 0 {
ch1 <- i
} else {
ch2 <- i
}
}
}
func worker_2(ch1, ch2 chan int) {
for i := 101; i < 200; i++ {
if i%2 == 0 {
ch1 <- i
} else {
ch2 <- i
}
}
}
Here is one solution:
package main
import (
"fmt"
"sync"
)
func main() {
// Create channels
ch1, ch2 := make(chan int), make(chan int)
// Create workers waitgroup with a counter of 2
wgWorkers := sync.WaitGroup{}
wgWorkers.Add(2)
// Run workers
go worker(&wgWorkers, ch1, ch2, 0, 100) // Worker 1
go worker(&wgWorkers, ch1, ch2, 101, 200) // Worker 2
// Create readers waitgroup with a counter of 2
wgReader := sync.WaitGroup{}
wgReader.Add(2)
// Run readers
go reader(&wgReader, ch1, 1) // Reader 1
go reader(&wgReader, ch2, 2) // Reader 2
// Wait for workers to finish
wgWorkers.Wait()
// Close workers channels
close(ch1) // Makes reader 1 exit after processing the last element in the channel
close(ch2) // Makes reader 2 exit after processing the last element in the channel
// Wait for both readers to finish processing before exiting the program
wgReader.Wait()
}
// Worker function definition
func worker(wg *sync.WaitGroup, ch1, ch2 chan<- int, from, to int) {
// Decrement worker waitgroup counter by one when function returns
defer wg.Done()
for i := from; i < to; i++ {
if i%2 == 0 {
ch1 <- i
} else {
ch2 <- i
}
}
}
// Reader function definition
func reader(wg *sync.WaitGroup, ch <-chan int, chNum int) {
// Decrement reader waitgroup counter by one when function returns
defer wg.Done()
// Here we iterate on the channel fed by worker 1 or worker 2.
// for-range on a channel exits when the channel is closed.
for i := range ch {
fmt.Printf("from ch%d: %d\n", chNum, i)
}
}
Explainations are in the code comments.
Close the channels when the workers are done. Break out of the receive loop after both channels are closed.
package main
import (
"fmt"
"sync"
)
func main() {
ch1, ch2 := func() (<-chan int, <-chan int) {
ch_1 := make(chan int)
ch_2 := make(chan int)
var wg sync.WaitGroup
wg.Add(2)
go worker_1(&wg, ch_1, ch_2)
go worker_2(&wg, ch_1, ch_2)
// Close channels after goroutiens complete.
go func() {
wg.Wait()
close(ch_1)
close(ch_2)
}()
return ch_1, ch_2
}()
// While we still have open channels ...
for ch1 != nil || ch2 != nil {
select {
case a, ok := <-ch1:
if ok {
fmt.Println("from ch1", a)
} else {
// note that channel is closed.
ch1 = nil
}
case a, ok := <-ch2:
if ok {
fmt.Println("from ch2", a)
} else {
// note that channel is closed.
ch2 = nil
}
}
}
}
func worker_1(wg *sync.WaitGroup, ch1, ch2 chan int) {
defer wg.Done()
for i := 0; i < 100; i++ {
if i%2 == 0 {
ch1 <- i
} else {
ch2 <- i
}
}
}
func worker_2(wg *sync.WaitGroup, ch1, ch2 chan int) {
defer wg.Done()
for i := 101; i < 200; i++ {
if i%2 == 0 {
ch1 <- i
} else {
ch2 <- i
}
}
}

waitgroup.Wait() causing dead lock

I'm trying to figure out why I have a dead lock with waitgroup.Wait()
package main
import (
"fmt"
"sync"
)
var wg sync.WaitGroup
func foo(c chan int, i int) {
defer wg.Done()
c <- i
}
func main() {
ch := make(chan int)
for i := 0; i < 10; i++ {
wg.Add(1)
go foo(ch, i)
}
wg.Wait()
close(ch)
for item := range ch {
fmt.Println(item)
}
}
When I run it like this, it prints fatal error: all goroutines are asleep - deadlock!
I tried to change ch to a buffered channel and that solved the problem. But I really want to know why is there a dead lock.
I've commented out the parts where your program's logic is not correct:
package main
import (
"fmt"
"sync"
)
var wg sync.WaitGroup
func foo(c chan int, i int) {
defer wg.Done()
c <- i
}
func main() {
ch := make(chan int) // unbuffered channel
for i := 0; i < 10; i++ {
wg.Add(1)
go foo(ch, i)
}
// wg.Wait is waiting for all goroutines to finish but that's
// only possible if the send to channel succeeds. In this case,
// it is not possible as your receiver "for item := range ch" is below
// this. Hence, a deadlock.
wg.Wait()
// Ideally, it should be the sender's duty to close the channel.
// And closing a channel before the receiver where the channel
// is unbuffered is not correct.
close(ch)
for item := range ch {
fmt.Println(item)
}
}
Corrected program:
package main
import (
"fmt"
"sync"
)
var wg sync.WaitGroup
func foo(c chan int, i int) {
defer wg.Done()
c <- i
}
func main() {
ch := make(chan int)
go func() {
for item := range ch {
fmt.Println(item)
}
}()
for i := 0; i < 10; i++ {
wg.Add(1)
go foo(ch, i)
}
wg.Wait()
close(ch)
}

How to write a better two channel select

In the following code there are two channels A & B that contain work, in the real code they are different structures, the workers need to drain both channels before quitting. The workers need the information coming in from both channels. The two select statements work but it's very clumsy. If I add default: to make them non-blocking then the code fails to drain the channels. Is there a better way of writing the selects?
Right now if channel A has no work then channel B does not get serviced either. Another problem to solve, but not my main concern.
playground for testing following code:
package main
import (
"fmt"
"time"
)
const (
fillCount = 10 // number of elements in each input channel
numWorkers = 3 // number of consumers.
)
func Wait() {
time.Sleep(2000 * time.Millisecond)
}
func fillChannel(work chan string, name string) {
for i := 0; i < fillCount; i++ {
work <- fmt.Sprintf("%s%d", name, i)
}
close(work) // we're finished
}
func doWork(id int, ch1 chan string, ch2 chan string, done chan bool) {
fmt.Println("Running worker", id)
defer fmt.Println("Ending worker", id)
for ch1Open, ch2Open := true, true; ch1Open && ch2Open; {
cnt1 := len(ch1)
cnt2 := len(ch2)
if ch1Open {
select {
case str, more := <-ch1:
if more {
fmt.Printf("%d: ch1(%d) %s\n", id, cnt1, str)
} else {
fmt.Printf("%d: ch1 closed\n", id)
ch1Open = false
}
}
}
if ch2Open {
select {
case str, more := <-ch2:
if more {
fmt.Printf("%d: ch2(%d) %s\n", id, cnt2, str)
} else {
fmt.Printf("%d: ch2 closed\n", id)
ch2Open = false
}
}
}
}
done <- true
}
func main() {
a := make(chan string, 2) // a small channel
b := make(chan string, 5) // a bigger channel
// generate work
go fillChannel(a, "A")
go fillChannel(b, "B")
// launch the consumers
done := make(chan bool)
for i := 0; i < numWorkers; i++ {
go doWork(i, a, b, done)
}
// wait for the goroutines to finish.
for i := 0; i < numWorkers; i++ {
<-done
}
fmt.Println("All workers done.")
Wait() // without this the defered prints from the workers doesn't flush
}
Select on both channels in a loop. When a channel is closed, set the channel variable to nil to make receive on that channel not ready. Break out of the loop when both channels are nil.
http://play.golang.org/p/9gRY1yKqJ9
package main
import (
"fmt"
"time"
)
const (
fillCount = 10 // number of elements in each input channel
numWorkers = 3 // number of consumers.
)
func fillChannel(work chan string, name string) {
for i := 0; i < fillCount; i++ {
work <- fmt.Sprintf("%s%d", name, i)
}
close(work) // we're finished
}
func doWork(id int, ch1 chan string, ch2 chan string, done chan bool) {
fmt.Println("Running worker", id)
for ch1 != nil || ch2 != nil {
select {
case str, ok := <-ch1:
if ok {
fmt.Printf("%d: ch1(%d) %s\n", id, len(ch1), str)
} else {
ch1 = nil
fmt.Printf("%d: ch1 closed\n", id)
}
case str, ok := <-ch2:
if ok {
fmt.Printf("%d: ch2(%d) %s\n", id, len(ch2), str)
} else {
ch2 = nil
fmt.Printf("%d: ch2 closed\n", id)
}
}
}
fmt.Println("Ending worker", id)
done <- true
}
func main() {
a := make(chan string, 2) // a small channel
b := make(chan string, 5) // a bigger channel
// generate work
go fillChannel(a, "A")
go fillChannel(b, "B")
// launch the consumers
done := make(chan bool)
for i := 0; i < numWorkers; i++ {
go doWork(i, a, b, done)
}
// wait for the goroutines to finish.
for i := 0; i < numWorkers; i++ {
<-done
}
fmt.Println("All workers done.")
}

Go: One producer many consumers

So I have seen a lot of ways of implementing one consumer and many producers in Go - the classic fanIn function from the Concurrency in Go talk.
What I want is a fanOut function. It takes as a parameter a channel it reads a value from and returns a slice of channels that it writes copies of this value to.
Is there a correct/recommended way of implementing this?
You pretty much described the best way to do it but here is a small sample of code that does it.
Go playground: https://play.golang.org/p/jwdtDXVHJk
package main
import (
"fmt"
"time"
)
func producer(iters int) <-chan int {
c := make(chan int)
go func() {
for i := 0; i < iters; i++ {
c <- i
time.Sleep(1 * time.Second)
}
close(c)
}()
return c
}
func consumer(cin <-chan int) {
for i := range cin {
fmt.Println(i)
}
}
func fanOut(ch <-chan int, size, lag int) []chan int {
cs := make([]chan int, size)
for i, _ := range cs {
// The size of the channels buffer controls how far behind the recievers
// of the fanOut channels can lag the other channels.
cs[i] = make(chan int, lag)
}
go func() {
for i := range ch {
for _, c := range cs {
c <- i
}
}
for _, c := range cs {
// close all our fanOut channels when the input channel is exhausted.
close(c)
}
}()
return cs
}
func fanOutUnbuffered(ch <-chan int, size int) []chan int {
cs := make([]chan int, size)
for i, _ := range cs {
// The size of the channels buffer controls how far behind the recievers
// of the fanOut channels can lag the other channels.
cs[i] = make(chan int)
}
go func() {
for i := range ch {
for _, c := range cs {
c <- i
}
}
for _, c := range cs {
// close all our fanOut channels when the input channel is exhausted.
close(c)
}
}()
return cs
}
func main() {
c := producer(10)
chans := fanOutUnbuffered(c, 3)
go consumer(chans[0])
go consumer(chans[1])
consumer(chans[2])
}
The important part to note is how we close the output channels once the input channel has been exhausted. Also if one of the output channels blocks on the send it will hold up the send on the other output channels. We control the amount of lag by setting the buffer size of the channels.
This solution below is a bit contrived, but it works for me:
package main
import (
"fmt"
"time"
"crypto/rand"
"encoding/binary"
)
func handleNewChannels(arrchangen chan [](chan uint32),
intchangen chan (chan uint32)) {
currarr := []chan uint32{}
arrchangen <- currarr
for {
newchan := <-intchangen
currarr = append(currarr, newchan)
arrchangen <- currarr
}
}
func sendToChannels(arrchangen chan [](chan uint32)) {
tick := time.Tick(1 * time.Second)
currarr := <-arrchangen
for {
select {
case <-tick:
sent := false
var n uint32
binary.Read(rand.Reader, binary.LittleEndian, &n)
for i := 0 ; i < len(currarr) ; i++ {
currarr[i] <- n
sent = true
}
if sent {
fmt.Println("Sent generated ", n)
}
case newarr := <-arrchangen:
currarr = newarr
}
}
}
func handleChannel(tchan chan uint32) {
for {
val := <-tchan
fmt.Println("Got the value ", val)
}
}
func createChannels(intchangen chan (chan uint32)) {
othertick := time.Tick(5 * time.Second)
for {
<-othertick
fmt.Println("Creating new channel! ")
newchan := make(chan uint32)
intchangen <- newchan
go handleChannel(newchan)
}
}
func main() {
arrchangen := make(chan [](chan uint32))
intchangen := make(chan (chan uint32))
go handleNewChannels(arrchangen, intchangen)
go sendToChannels(arrchangen)
createChannels(intchangen)
}
First, see related question What is the neatest idiom for producer/consumer in Go? and One thread showing interest in another thread (consumer / producer). Also, take a look to producer-consumer problem. About concurrency see how to achieve concurrency In Google Go.
We can handle multiple consumers without making the copy of channel data for each consumer.
Go playground: https://play.golang.org/p/yOKindnqiZv
package main
import (
"fmt"
"sync"
)
type data struct {
msg string
consumers int
}
func main() {
ch := make(chan *data) // both block or non-block are ok
var wg sync.WaitGroup
consumerCount := 3 // specify no. of consumers
producer := func() {
obj := &data {
msg: "hello everyone!",
consumers: consumerCount,
}
ch <- obj
}
consumer := func(idx int) {
defer wg.Done()
obj := <-ch
fmt.Printf("consumer %d received data %v\n", idx, obj)
obj.consumers--
if obj.consumers > 0 {
ch <- obj // forward to others
} else {
fmt.Printf("last receiver: %d\n", idx)
}
}
go producer()
for i:=1; i<=consumerCount; i++ {
wg.Add(1)
go consumer(i)
}
wg.Wait()
}

Resources