I'm trying to modify values of a map of maps.
Following code shows fatal error: concurrent map read and map write unless I lock whole Map (using defer in method Map.Add in line 17).
My question is, how can I implement it correctly to effectively make use of goroutines concurrency and do not turn my asynchronous code to some synchronous code?
package main
import (
"fmt"
"sync"
"time"
)
type Map struct {
mutex sync.Mutex
item map[int]*Item
}
func (m *Map) Add(key int) {
m.mutex.Lock()
m.item[key] = &Item{value: make(map[int]int)}
defer m.mutex.Unlock()
for j := 0; j < 100; j++ {
go m.item[key].Inc(j)
}
}
type Item struct {
mutex sync.Mutex
value map[int]int
}
func (m *Item) Inc(key int) {
m.mutex.Lock()
m.value[key]++
m.mutex.Unlock()
}
func (m *Item) Value(key int) int {
m.mutex.Lock()
defer m.mutex.Unlock()
return m.value[key]
}
func main() {
m := Map{item: make(map[int]*Item)}
for i := 0; i < 100; i++ {
go m.Add(i)
}
time.Sleep(time.Second)
fmt.Println(m.item[1])
}
Related
It says: (no value) used as value, but I'm passing loop values from a slice to it!
package main
import "fmt"
func greet(n string) {
fmt.Printf("Hi, %v\n", n)
}
func cycle(n []string, f func(string)) {
for i := 0; i < len(n); i++ {
fmt.Println(f(n[i]))
}
}
func main() {
cycle([]string{"John", "Marie"}, greet)
}
Code snippet on Go Playground
I found the solution: I should have called the function directly, not inside Println().
package main
import "fmt"
func greet(n string) {
fmt.Printf("Hi, %v\n", n)
}
func cycle(n []string, f func(string)) {
for i := 0; i < len(n); i++ {
f(n[i])
}
}
func main() {
cycle([]string{"John", "Marie"}, greet)
}
In my Go program I start multiple worker groups for every department.
I want to wait for workers from each department to complete before exiting the program
I cannot use a single WaitGroups because in the actual scenario I may have to end any particular department and need to wait only on that.
This is simplified version of code, but it panics with a message
panic: runtime error: invalid memory address or nil pointer dereference
package main
import (
"fmt"
"sync"
"time"
)
var wgMap map[string]*sync.WaitGroup
func deptWorker(dName string, id int) {
defer wgMap[dName].Done()
fmt.Printf("Department %s : Worker %d starting\n", dName, id)
time.Sleep(time.Second)
fmt.Printf("Department %s : Worker %d done\n", dName, id)
}
func department(dName string) {
var wg sync.WaitGroup
for i := 1; i <= 3; i++ {
wg.Add(1)
go deptWorker(dName, i)
}
wgMap[dName] = &wg
}
func main() {
go department("medical")
go department("electronics")
wgMap["medical"].Wait()
wgMap["electronics"].Wait()
}
Two fix nil panic you simply need to use
var wgMap = map[string]*sync.WaitGroup{}
It will initialize the map. However, in my view, it's better here to create a new abstraction, let's name it 'WaitMap'.
It can be implemented in this way:
package main
import (
"fmt"
"sync"
"time"
)
type WaitMapObject struct {
wg map[string]int
mu sync.Mutex
cond sync.Cond
}
func WaitMap() *WaitMapObject {
m := &WaitMapObject{}
m.wg = make(map[string]int)
m.cond.L = &m.mu
return m
}
func (m *WaitMapObject) Wait(name string) {
m.mu.Lock()
for m.wg[name] != 0 {
m.cond.Wait()
}
m.mu.Unlock()
}
func (m *WaitMapObject) Done(name string) {
m.mu.Lock()
no := m.wg[name] - 1
if no < 0 {
panic("")
}
m.wg[name] = no
m.mu.Unlock()
m.cond.Broadcast()
}
func (m *WaitMapObject) Add(name string, no int) {
m.mu.Lock()
m.wg[name] = m.wg[name] + no
m.mu.Unlock()
}
func deptWorker(dName string, id int, wm *WaitMapObject) {
defer wm.Done(dName)
fmt.Printf("Department %s : Worker %d starting\n", dName, id)
time.Sleep(time.Second)
fmt.Printf("Department %s : Worker %d done\n", dName, id)
}
func department(dName string, wm *WaitMapObject) {
for i := 1; i <= 3; i++ {
wm.Add(dName,1)
go deptWorker(dName, i, wm)
}
wm.Done(dName)
}
func main() {
wm := WaitMap()
wm.Add("mediacal",1)
go department("medical", wm)
wm.Add("electronics",1)
go department("electronics", wm)
wm.Wait("medical")
wm.Wait("electronics")
}
Functions WithMutex and WithoutMutex are giving different results.
WithoutMutex implementation is losing values even though I have Waitgroup set up.
What could be wrong?
Do not run on Playground
P.S. I am on Windows 10 and Go 1.8.1
package main
import (
"fmt"
"sync"
)
var p = fmt.Println
type MuType struct {
list []int
*sync.RWMutex
}
var muData *MuType
var data *NonMuType
type NonMuType struct {
list []int
}
func (data *MuType) add(i int, wg *sync.WaitGroup) {
data.Lock()
defer data.Unlock()
data.list = append(data.list, i)
wg.Done()
}
func (data *MuType) read() []int {
data.RLock()
defer data.RUnlock()
return data.list
}
func (nonmu *NonMuType) add(i int, wg *sync.WaitGroup) {
nonmu.list = append(nonmu.list, i)
wg.Done()
}
func (nonmu *NonMuType) read() []int {
return nonmu.list
}
func WithoutMutex() {
nonmu := &NonMuType{}
nonmu.list = make([]int, 0)
var wg = sync.WaitGroup{}
for i := 0; i < 10; i++ {
wg.Add(1)
go nonmu.add(i, &wg)
}
wg.Wait()
data = nonmu
p(data.read())
}
func WithMutex() {
mtx := &sync.RWMutex{}
withMu := &MuType{list: make([]int, 0)}
withMu.RWMutex = mtx
var wg = sync.WaitGroup{}
for i := 0; i < 10; i++ {
wg.Add(1)
go withMu.add(i, &wg)
}
wg.Wait()
muData = withMu
p(muData.read())
}
func stressTestWOMU(max int) {
p("Without Mutex")
for ii := 0; ii < max; ii++ {
WithoutMutex()
}
}
func stressTest(max int) {
p("With Mutex")
for ii := 0; ii < max; ii++ {
WithMutex()
}
}
func main() {
stressTestWOMU(20)
stressTest(20)
}
Slices are not safe for concurrent writes, so I am in no way surprised that WithoutMutex does not appear to be consistent at all, and has dropped items.
The WithMutex version consistently has 10 items, but in jumbled orders. This is also to be expected, since the mutex protects it so that only one can append at a time. There is no guarantee as to which goroutine will run in which order though, so it is a race to see which of the rapidly spawned goroutines will get to append first.
The waitgroup does not do anything to control access or enforce ordering. It merely provides a signal at the end that everything is done.
I have a channel which stores received data, I want to process it when one of following conditions is met:
1, the channel reaches its capacity.
2, the timer is fired since last process.
I saw the post
Golang - How to know a buffered channel is full
Update:
I inspired from that post and OneOfOne's advice, here is the play :
package main
import (
"fmt"
"math/rand"
"time"
)
var c chan int
var timer *time.Timer
const (
capacity = 5
timerDration = 3
)
func main() {
c = make(chan int, capacity)
timer = time.NewTimer(time.Second * timerDration)
go checkTimer()
go sendRecords("A")
go sendRecords("B")
go sendRecords("C")
time.Sleep(time.Second * 20)
}
func sendRecords(name string) {
for i := 0; i < 20; i++ {
fmt.Println(name+" sending record....", i)
sendOneRecord(i)
interval := time.Duration(rand.Intn(500))
time.Sleep(time.Millisecond * interval)
}
}
func sendOneRecord(record int) {
select {
case c <- record:
default:
fmt.Println("channel is full !!!")
process()
c <- record
timer.Reset(time.Second * timerDration)
}
}
func checkTimer() {
for {
select {
case <-timer.C:
fmt.Println("3s timer ----------")
process()
timer.Reset(time.Second * timerDration)
}
}
}
func process() {
for i := 0; i < capacity; i++ {
fmt.Println("process......", <-c)
}
}
This seems to work fine, but I have a concern, I want to block the channel writing from other goroutine when process() is called, is the code above capable to do so? Or should I add a mutex at the beginning of the process method?
Any elegant solution?
As was mentioned by #OneOfOne, select is really the only way to check if a channel is full.
If you are using the channel to effect batch processing, you could always create an unbuffered channel and have a goroutine pull items and append to a slice.
When the slice reaches a specific size, process the items.
Here's an example on play
package main
import (
"fmt"
"sync"
"time"
)
const BATCH_SIZE = 10
func batchProcessor(ch <-chan int) {
batch := make([]int, 0, BATCH_SIZE)
for i := range ch {
batch = append(batch, i)
if len(batch) == BATCH_SIZE {
fmt.Println("Process batch:", batch)
time.Sleep(time.Second)
batch = batch[:0] // trim back to zero size
}
}
fmt.Println("Process last batch:", batch)
}
func main() {
var wg sync.WaitGroup
ch := make(chan int)
wg.Add(1)
go func() {
batchProcessor(ch)
wg.Done()
}()
fmt.Println("Submitting tasks")
for i := 0; i < 55; i++ {
ch <- i
}
close(ch)
wg.Wait()
}
No, select is the only way to do it:
func (t *T) Send(v *Val) {
select {
case t.ch <- v:
default:
// handle v directly
}
}
I've been trying some things out in Go and I've hit a problem that I can't figure out.
package main
import "fmt"
import "strconv"
func writeHello(i int, ) {
fmt.Printf("hello, world "+strconv.Itoa(i)+"\n")
}
type SliceStruct struct {
data [][]int;
}
func (s SliceStruct) New() {
s.data=make([][]int,10);
}
func (s SliceStruct) AllocateSlice(i int) {
s.data[i]=make([]int,10);
}
func (s SliceStruct) setData(i int, j int, data int) {
s.data[i][j] = data;
}
func (s SliceStruct) getData(i int, j int) int {
return s.data[i][j]
}
func useSliceStruct(){
sliceStruct := SliceStruct{};
sliceStruct.New();
for i := 0; i < 10; i++ {
sliceStruct.AllocateSlice(i);
for j:=0; j<10; j++ {
sliceStruct.setData(i,j,i);
writeHello(sliceStruct.getData(i,j));
}
}
}
func dontUseSliceStruct(){
data:=make([][]int,10);
for i := 0; i < 10; i++ {
data[i]=make([]int,10);
for j:=0; j<10; j++ {
data[i][j] = i;
writeHello(data[i][j]);
}
}
}
func main() {
dontUseSliceStruct();
useSliceStruct();
}
When it gets to the function useSliceStruct, the code fails at the first call to AllocateSlice() with an index out of range error.
As far as I can tell the code for the two methods does idential things. So what am I missing?
DOH, just worked it out.
I wasn't using a reference to the struct in the function declarations.
func (s SliceStruct)
Should have been
func (s *SliceStruct)