How to use multiple sync.WaitGroup in a single program - go

In my Go program I start multiple worker groups for every department.
I want to wait for workers from each department to complete before exiting the program
I cannot use a single WaitGroups because in the actual scenario I may have to end any particular department and need to wait only on that.
This is simplified version of code, but it panics with a message
panic: runtime error: invalid memory address or nil pointer dereference
package main
import (
"fmt"
"sync"
"time"
)
var wgMap map[string]*sync.WaitGroup
func deptWorker(dName string, id int) {
defer wgMap[dName].Done()
fmt.Printf("Department %s : Worker %d starting\n", dName, id)
time.Sleep(time.Second)
fmt.Printf("Department %s : Worker %d done\n", dName, id)
}
func department(dName string) {
var wg sync.WaitGroup
for i := 1; i <= 3; i++ {
wg.Add(1)
go deptWorker(dName, i)
}
wgMap[dName] = &wg
}
func main() {
go department("medical")
go department("electronics")
wgMap["medical"].Wait()
wgMap["electronics"].Wait()
}

Two fix nil panic you simply need to use
var wgMap = map[string]*sync.WaitGroup{}
It will initialize the map. However, in my view, it's better here to create a new abstraction, let's name it 'WaitMap'.
It can be implemented in this way:
package main
import (
"fmt"
"sync"
"time"
)
type WaitMapObject struct {
wg map[string]int
mu sync.Mutex
cond sync.Cond
}
func WaitMap() *WaitMapObject {
m := &WaitMapObject{}
m.wg = make(map[string]int)
m.cond.L = &m.mu
return m
}
func (m *WaitMapObject) Wait(name string) {
m.mu.Lock()
for m.wg[name] != 0 {
m.cond.Wait()
}
m.mu.Unlock()
}
func (m *WaitMapObject) Done(name string) {
m.mu.Lock()
no := m.wg[name] - 1
if no < 0 {
panic("")
}
m.wg[name] = no
m.mu.Unlock()
m.cond.Broadcast()
}
func (m *WaitMapObject) Add(name string, no int) {
m.mu.Lock()
m.wg[name] = m.wg[name] + no
m.mu.Unlock()
}
func deptWorker(dName string, id int, wm *WaitMapObject) {
defer wm.Done(dName)
fmt.Printf("Department %s : Worker %d starting\n", dName, id)
time.Sleep(time.Second)
fmt.Printf("Department %s : Worker %d done\n", dName, id)
}
func department(dName string, wm *WaitMapObject) {
for i := 1; i <= 3; i++ {
wm.Add(dName,1)
go deptWorker(dName, i, wm)
}
wm.Done(dName)
}
func main() {
wm := WaitMap()
wm.Add("mediacal",1)
go department("medical", wm)
wm.Add("electronics",1)
go department("electronics", wm)
wm.Wait("medical")
wm.Wait("electronics")
}

Related

How to know the number of specific goroutine?

I'm making a goroutine worker pool that constantly increases and decreases according to the situation.
But never falldown under specific count.
To do this, I want to know the number of specific goroutines. Not use global variable.
package main
import (
"fmt"
"time"
)
func Adaptive_Worker_Pool(value_input chan int) {
kill_sig := make(chan bool)
make_sig := make(chan bool)
for i := 0; i < 5; i++ {
go Do(kill_sig, value_input)
}
go Make_Routine(make_sig, kill_sig, value_input)
go Judge(kill_sig, make_sig, value_input)
}
func Make_Routine(make_sig chan bool, kill_sig chan bool, value_input chan int) {
for {
<-make_sig
go Do(kill_sig, value_input)
}
}
func Do(kill_sig chan bool, value_input chan int) {
outer:
for {
select {
case value := <-value_input:
fmt.Println(value)
case <-kill_sig:
break outer
}
}
}
func Judge(make_sig chan bool, kill_sig chan bool, value_input chan int) {
for {
time.Sleep(time.Millisecond * 500)
count_value_in_channel := len(value_input)
if count_value_in_channel > 5 {
make_sig <- true
} else {
if { // if Count(Do( )) > 5 { continue }
continue // else {kill_sig <- true}
} else { // like this
kill_sig <- true
}
}
}
}
func main() {
value_input := make(chan int, 10)
Adaptive_Worker_Pool(value_input)
a := 0
for {
value_input <- a
a++
}
}
Some value input to a value_input channel and five goroutines that receive and output the value are created by default.
However, if the number of variables in the value_input channel is 5 or more, Do( ) goroutine will made.
I want to make the Judge( ) function decide whether to increment or decrement the Do( ) goroutine, If number of Do( ) goroutine.
How can I do it?
I think you should define a struct like:
type MyChannel struct {
value int
index int
}
and in for loop
value_input := make(chan MyChannel, 10)
a := 0
for {
value_input <- MyChannel{
index: a,
value: a,
}
a++
}
and in some check, you can check the index of MyChannel

How to implement efficient locks on Map of Map

I'm trying to modify values of a map of maps.
Following code shows fatal error: concurrent map read and map write unless I lock whole Map (using defer in method Map.Add in line 17).
My question is, how can I implement it correctly to effectively make use of goroutines concurrency and do not turn my asynchronous code to some synchronous code?
package main
import (
"fmt"
"sync"
"time"
)
type Map struct {
mutex sync.Mutex
item map[int]*Item
}
func (m *Map) Add(key int) {
m.mutex.Lock()
m.item[key] = &Item{value: make(map[int]int)}
defer m.mutex.Unlock()
for j := 0; j < 100; j++ {
go m.item[key].Inc(j)
}
}
type Item struct {
mutex sync.Mutex
value map[int]int
}
func (m *Item) Inc(key int) {
m.mutex.Lock()
m.value[key]++
m.mutex.Unlock()
}
func (m *Item) Value(key int) int {
m.mutex.Lock()
defer m.mutex.Unlock()
return m.value[key]
}
func main() {
m := Map{item: make(map[int]*Item)}
for i := 0; i < 100; i++ {
go m.Add(i)
}
time.Sleep(time.Second)
fmt.Println(m.item[1])
}

How can serial some goroutings to be finish

I have a Go Project sample, trying to simulate something like baking 1000 pizza concurrency but the oven just has 10 parts to put the pizza. I developed like this, but goroutines are deadlocked. would anyone help
package main
import (
"fmt"
"sync"
"time"
)
type Material struct {
material int
mutex sync.Mutex
}
var (
mOven sync.Mutex
inOven int
)
func main() {
rawMaterial := Material{material: 10000}
var waitGroup sync.WaitGroup
for i := 0; i < 1000; i++ {
waitGroup.Add(1)
go perparePizza(&waitGroup, &rawMaterial, i)
}
waitGroup.Wait()
fmt.Println("finished with remained material:", rawMaterial.material)
}
func perparePizza(wg *sync.WaitGroup, m *Material, num int) {
defer wg.Done()
fmt.Println("Preparing Pizza:", num)
m.mutex.Lock()
m.material--
m.mutex.Unlock()
// time.Sleep(time.Second * 2)
var isCooking chan bool
for {
ovenManeger(num, isCooking)
select {
case <-isCooking:
putInOven(num)
default:
fmt.Println("waiting for accepting cook!", num)
time.Sleep(time.Second)
}
}
fmt.Printf("Pizza %d is ready \n", num)
}
func ovenManeger(num int, couldPleaseCook chan bool) {
if inOven < 10 {
mOven.Lock()
inOven = inOven + 1
couldPleaseCook <- true
mOven.Unlock()
}
}
func putInOven(num int) {
fmt.Println("putInOven", num)
time.Sleep(time.Second * 30)
mOven.Lock()
inOven = inOven - 1
mOven.Unlock()
}
error raise when 10 oven part filled with deadlocking
Preparing Pizza: 9
Preparing Pizza: 0
Preparing Pizza: 1
Preparing Pizza: 2
Preparing Pizza: 3
Preparing Pizza: 4
Preparing Pizza: 5
Preparing Pizza: 6
Preparing Pizza: 7
Preparing Pizza: 8
fatal error: all goroutines are asleep - deadlock!
I changed Like this, but I think there would be a better solution too.
package main
import (
"fmt"
"sync"
"time"
)
type Material struct {
material int
mutex sync.RWMutex
}
var readValue = make(chan int)
var writeValue = make(chan int)
func set(newValue int) {
writeValue <- newValue
}
func read() int {
return <-readValue
}
func backing() {
var value int
var internalCounter int
for {
select {
case newValue := <-writeValue:
internalCounter = internalCounter + 1
value = newValue
putInOven(value)
case readValue <- value:
internalCounter = internalCounter - 1
}
}
}
func main() {
rawMaterial := Material{material: 10000}
var waitGroup sync.WaitGroup
for i := 1; i <= 100; i++ {
waitGroup.Add(1)
go perparePizza(&waitGroup, &rawMaterial, i)
}
for i := 0; i < 10; i++ {
go backing()
}
waitGroup.Wait()
fmt.Println("finished with remained material:", rawMaterial.material)
}
func perparePizza(wg *sync.WaitGroup, m *Material, num int) {
defer wg.Done()
// fmt.Println("Preparing Pizza:", num)
m.mutex.Lock()
m.material--
m.mutex.Unlock()
// time.Sleep(time.Second * 2)
set(num)
fmt.Printf("Pizza %d is ready \n", num)
}
func putInOven(num int) {
// fmt.Println("putInOven", num)
time.Sleep(time.Second * 3)
}

Mutex write locking of a channel value

I have a channel of thousands of IDs that need to be processed in parallel inside goroutines. How could I implement a lock so that goroutines cannot process the same id at the same time, should it be repeated in the channel?
package main
import (
"fmt"
"sync"
"strconv"
"time"
)
var wg sync.WaitGroup
func main() {
var data []string
for d := 0; d < 30; d++ {
data = append(data, "id1")
data = append(data, "id2")
data = append(data, "id3")
}
chanData := createChan(data)
for i := 0; i < 10; i++ {
wg.Add(1)
process(chanData, i)
}
wg.Wait()
}
func createChan(data []string) <-chan string {
var out = make(chan string)
go func() {
for _, val := range data {
out <- val
}
close(out)
}()
return out
}
func process(ids <-chan string, i int) {
go func() {
defer wg.Done()
for id := range ids {
fmt.Println(id + " (goroutine " + strconv.Itoa(i) + ")")
time.Sleep(1 * time.Second)
}
}()
}
--edit:
All values need to be processed in any order, but "id1, "id2" & "id3" need to block so they cannot be processed by more than one goroutine at the same time.
The simplest solution here is to not send the duplicate values at all, and then no synchronization is required.
func createChan(data []string) <-chan string {
seen := make(map[string]bool)
var out = make(chan string)
go func() {
for _, val := range data {
if seen[val] {
continue
}
seen[val] = true
out <- val
}
close(out)
}()
return out
}
I've found a solution. Someone has written a package (github.com/EagleChen/mapmutex) to do exactly what I needed:
package main
import (
"fmt"
"github.com/EagleChen/mapmutex"
"strconv"
"sync"
"time"
)
var wg sync.WaitGroup
var mutex *mapmutex.Mutex
func main() {
mutex = mapmutex.NewMapMutex()
var data []string
for d := 0; d < 30; d++ {
data = append(data, "id1")
data = append(data, "id2")
data = append(data, "id3")
}
chanData := createChan(data)
for i := 0; i < 10; i++ {
wg.Add(1)
process(chanData, i)
}
wg.Wait()
}
func createChan(data []string) <-chan string {
var out = make(chan string)
go func() {
for _, val := range data {
out <- val
}
close(out)
}()
return out
}
func process(ids <-chan string, i int) {
go func() {
defer wg.Done()
for id := range ids {
if mutex.TryLock(id) {
fmt.Println(id + " (goroutine " + strconv.Itoa(i) + ")")
time.Sleep(1 * time.Second)
mutex.Unlock(id)
}
}
}()
}
Your problem as stated is difficult by definition and my first choice would be to re-architect the application to avoid it, but if that's not an option:
First, I assume that if a given ID is repeated you still want it processed twice, but not in parallel (if that's not the case and the 2nd instance must be ignored, it becomes even more difficult, because you have to remember every ID you have processed forever, so you don't run the task over it twice).
To achieve your goal, you must keep track of every ID that is being acted upon in a goroutine - a go map is your best option here (note that its size will grow up to as many goroutines as you spin in parallel!). The map itself must be protected by a lock, as it is modified from multiple goroutines.
Another simplification that I'd take is that it is OK for an ID removed from the channel to be added back to it if found to be currently processed by another gorotuine. Then, we need map[string]bool as the tracking device, plus a sync.Mutex to guard it. For simplicity, I assume the map, mutex and the channel are global variables; but that may not be convenient for you - arrange access to those as you see fit (arguments to the goroutine, closure, etc.).
import "sync"
var idmap map[string]bool
var mtx sync.Mutex
var queue chan string
func process_one_id(id string) {
busy := false
mtx.Lock()
if idmap[id] {
busy = true
} else {
idmap[id] = true
}
mtx.Unlock()
if busy { // put the ID back on the queue and exit
queue <- id
return
}
// ensure the 'busy' mark is cleared at the end:
defer func() { mtx.Lock(); delete(idmap, id); mtx.Unlock() }()
// do your processing here
// ....
}

How to handle the buffered channel properly in Golang?

I have a channel which stores received data, I want to process it when one of following conditions is met:
1, the channel reaches its capacity.
2, the timer is fired since last process.
I saw the post
Golang - How to know a buffered channel is full
Update:
I inspired from that post and OneOfOne's advice, here is the play :
package main
import (
"fmt"
"math/rand"
"time"
)
var c chan int
var timer *time.Timer
const (
capacity = 5
timerDration = 3
)
func main() {
c = make(chan int, capacity)
timer = time.NewTimer(time.Second * timerDration)
go checkTimer()
go sendRecords("A")
go sendRecords("B")
go sendRecords("C")
time.Sleep(time.Second * 20)
}
func sendRecords(name string) {
for i := 0; i < 20; i++ {
fmt.Println(name+" sending record....", i)
sendOneRecord(i)
interval := time.Duration(rand.Intn(500))
time.Sleep(time.Millisecond * interval)
}
}
func sendOneRecord(record int) {
select {
case c <- record:
default:
fmt.Println("channel is full !!!")
process()
c <- record
timer.Reset(time.Second * timerDration)
}
}
func checkTimer() {
for {
select {
case <-timer.C:
fmt.Println("3s timer ----------")
process()
timer.Reset(time.Second * timerDration)
}
}
}
func process() {
for i := 0; i < capacity; i++ {
fmt.Println("process......", <-c)
}
}
This seems to work fine, but I have a concern, I want to block the channel writing from other goroutine when process() is called, is the code above capable to do so? Or should I add a mutex at the beginning of the process method?
Any elegant solution?
As was mentioned by #OneOfOne, select is really the only way to check if a channel is full.
If you are using the channel to effect batch processing, you could always create an unbuffered channel and have a goroutine pull items and append to a slice.
When the slice reaches a specific size, process the items.
Here's an example on play
package main
import (
"fmt"
"sync"
"time"
)
const BATCH_SIZE = 10
func batchProcessor(ch <-chan int) {
batch := make([]int, 0, BATCH_SIZE)
for i := range ch {
batch = append(batch, i)
if len(batch) == BATCH_SIZE {
fmt.Println("Process batch:", batch)
time.Sleep(time.Second)
batch = batch[:0] // trim back to zero size
}
}
fmt.Println("Process last batch:", batch)
}
func main() {
var wg sync.WaitGroup
ch := make(chan int)
wg.Add(1)
go func() {
batchProcessor(ch)
wg.Done()
}()
fmt.Println("Submitting tasks")
for i := 0; i < 55; i++ {
ch <- i
}
close(ch)
wg.Wait()
}
No, select is the only way to do it:
func (t *T) Send(v *Val) {
select {
case t.ch <- v:
default:
// handle v directly
}
}

Resources