I have a Go Project sample, trying to simulate something like baking 1000 pizza concurrency but the oven just has 10 parts to put the pizza. I developed like this, but goroutines are deadlocked. would anyone help
package main
import (
"fmt"
"sync"
"time"
)
type Material struct {
material int
mutex sync.Mutex
}
var (
mOven sync.Mutex
inOven int
)
func main() {
rawMaterial := Material{material: 10000}
var waitGroup sync.WaitGroup
for i := 0; i < 1000; i++ {
waitGroup.Add(1)
go perparePizza(&waitGroup, &rawMaterial, i)
}
waitGroup.Wait()
fmt.Println("finished with remained material:", rawMaterial.material)
}
func perparePizza(wg *sync.WaitGroup, m *Material, num int) {
defer wg.Done()
fmt.Println("Preparing Pizza:", num)
m.mutex.Lock()
m.material--
m.mutex.Unlock()
// time.Sleep(time.Second * 2)
var isCooking chan bool
for {
ovenManeger(num, isCooking)
select {
case <-isCooking:
putInOven(num)
default:
fmt.Println("waiting for accepting cook!", num)
time.Sleep(time.Second)
}
}
fmt.Printf("Pizza %d is ready \n", num)
}
func ovenManeger(num int, couldPleaseCook chan bool) {
if inOven < 10 {
mOven.Lock()
inOven = inOven + 1
couldPleaseCook <- true
mOven.Unlock()
}
}
func putInOven(num int) {
fmt.Println("putInOven", num)
time.Sleep(time.Second * 30)
mOven.Lock()
inOven = inOven - 1
mOven.Unlock()
}
error raise when 10 oven part filled with deadlocking
Preparing Pizza: 9
Preparing Pizza: 0
Preparing Pizza: 1
Preparing Pizza: 2
Preparing Pizza: 3
Preparing Pizza: 4
Preparing Pizza: 5
Preparing Pizza: 6
Preparing Pizza: 7
Preparing Pizza: 8
fatal error: all goroutines are asleep - deadlock!
I changed Like this, but I think there would be a better solution too.
package main
import (
"fmt"
"sync"
"time"
)
type Material struct {
material int
mutex sync.RWMutex
}
var readValue = make(chan int)
var writeValue = make(chan int)
func set(newValue int) {
writeValue <- newValue
}
func read() int {
return <-readValue
}
func backing() {
var value int
var internalCounter int
for {
select {
case newValue := <-writeValue:
internalCounter = internalCounter + 1
value = newValue
putInOven(value)
case readValue <- value:
internalCounter = internalCounter - 1
}
}
}
func main() {
rawMaterial := Material{material: 10000}
var waitGroup sync.WaitGroup
for i := 1; i <= 100; i++ {
waitGroup.Add(1)
go perparePizza(&waitGroup, &rawMaterial, i)
}
for i := 0; i < 10; i++ {
go backing()
}
waitGroup.Wait()
fmt.Println("finished with remained material:", rawMaterial.material)
}
func perparePizza(wg *sync.WaitGroup, m *Material, num int) {
defer wg.Done()
// fmt.Println("Preparing Pizza:", num)
m.mutex.Lock()
m.material--
m.mutex.Unlock()
// time.Sleep(time.Second * 2)
set(num)
fmt.Printf("Pizza %d is ready \n", num)
}
func putInOven(num int) {
// fmt.Println("putInOven", num)
time.Sleep(time.Second * 3)
}
Related
I'm making a goroutine worker pool that constantly increases and decreases according to the situation.
But never falldown under specific count.
To do this, I want to know the number of specific goroutines. Not use global variable.
package main
import (
"fmt"
"time"
)
func Adaptive_Worker_Pool(value_input chan int) {
kill_sig := make(chan bool)
make_sig := make(chan bool)
for i := 0; i < 5; i++ {
go Do(kill_sig, value_input)
}
go Make_Routine(make_sig, kill_sig, value_input)
go Judge(kill_sig, make_sig, value_input)
}
func Make_Routine(make_sig chan bool, kill_sig chan bool, value_input chan int) {
for {
<-make_sig
go Do(kill_sig, value_input)
}
}
func Do(kill_sig chan bool, value_input chan int) {
outer:
for {
select {
case value := <-value_input:
fmt.Println(value)
case <-kill_sig:
break outer
}
}
}
func Judge(make_sig chan bool, kill_sig chan bool, value_input chan int) {
for {
time.Sleep(time.Millisecond * 500)
count_value_in_channel := len(value_input)
if count_value_in_channel > 5 {
make_sig <- true
} else {
if { // if Count(Do( )) > 5 { continue }
continue // else {kill_sig <- true}
} else { // like this
kill_sig <- true
}
}
}
}
func main() {
value_input := make(chan int, 10)
Adaptive_Worker_Pool(value_input)
a := 0
for {
value_input <- a
a++
}
}
Some value input to a value_input channel and five goroutines that receive and output the value are created by default.
However, if the number of variables in the value_input channel is 5 or more, Do( ) goroutine will made.
I want to make the Judge( ) function decide whether to increment or decrement the Do( ) goroutine, If number of Do( ) goroutine.
How can I do it?
I think you should define a struct like:
type MyChannel struct {
value int
index int
}
and in for loop
value_input := make(chan MyChannel, 10)
a := 0
for {
value_input <- MyChannel{
index: a,
value: a,
}
a++
}
and in some check, you can check the index of MyChannel
I have a use case where I want to have a pool of N integers (0 - N-1) shared by N workers (N <= 100), each claiming an integer from the pool, "working" (for this example sleeping for a random duration), and returning them to the pool, and starting the process again. Each thread can take an arbitrary amount of time to return the key. I've quickly thrown together the following 2 solutions, and would like to know if there's a "best" or "safest" one, and if I'm missing a better approach. For the moment, these workers will never stop unless the application is killed, and we will have a fixed number of workers for the life of the application.
Single Buffered Channel
type Worker struct {
ID int
KeyIndex int
KeyChan chan int
}
func (w *Worker) GetKey() {
w.KeyIndex = <- w.KeyChan
}
func (w *Worker) ReturnKey() {
w.KeyChan <- w.KeyIndex
}
func (w *Worker) Work() {
for {
w.GetKey()
rand.Seed(time.Now().UnixNano())
n := rand.Intn(10)
time.Sleep(time.Duration(n) * time.Second)
w.ReturnKey()
}
}
func main() {
numWorkers := 5
c := make(chan int, numWorkers)
for i := 0; i < numWorkers; i++ {
c <- i
}
workers := make([]*Worker, numWorkers)
for i := range workers {
workers[i] = &Worker{
ID: i,
KeyChan: c,
}
}
for _, w := range workers {
go w.Work()
}
ch := make(chan byte, 1)
<-ch
}
Broker w/ Array + Mutex
type KeyBrokerMutex struct {
mu sync.Mutex
keys []bool
}
func (kb *KeyBrokerMutex) GetKey() int {
kb.mu.Lock()
defer kb.mu.Unlock()
for i, k := range kb.keys {
if k {
kb.keys[i] = false
return i
}
}
return -1
}
func (kb *KeyBrokerMutex) ReturnKey(index int) {
kb.mu.Lock()
defer kb.mu.Unlock()
kb.keys[index] = true
}
type Worker struct {
ID int
KeyIndex int
KeyBroker *KeyBrokerMutex
}
func (w *Worker) GetKeyBrokerMutex() {
w.KeyIndex = w.KeyPool.GetKey()
}
func (w *Worker) ReturnKeyBrokerMutex() {
w.KeyPool.ReturnKey(w.KeyIndex)
w.KeyIndex = -1
}
func (w *Worker) WorkMutex() {
for {
w.GetKeyBrokerMutex()
rand.Seed(time.Now().UnixNano())
n := rand.Intn(10)
time.Sleep(time.Duration(n) * time.Second)
w.ReturnKeyBrokerMutex()
}
}
func main() {
numWorkers := 5
keyBroker := KeyBrokerMutex{keys: make([]bool, numWorkers)}
for i := range keyBroker.keys {
keyBroker.keys[i] = true
}
workers := make([]*Worker, numWorkers)
for i := range workers {
workers[i] = &Worker{
ID: i,
KeyBroker: &keyBroker,
}
}
for _, w := range workers {
go w.WorkMutex()
}
ch := make(chan byte, 1)
<-ch
}
I also have a broker approach using 2 separate channels for getting and returning keys, however I don't think that offers any benefits over the above solutions.
I like the simplicity of the single channel approach, but is there any downside to having multiple consumers and producers to a single buffered channel?
I am implementing a set of codes that prints Lamport logical time upon the completion of sending messages to servers and broadcasting to nodes. My program runs fine before I implemented the codes Lamport logical time. Upon printing closing server..., the program breaks and shows deadlock. May I know if anyone can help me spot my mistake?
import (
"fmt"
"math/rand"
"time"
)
const num_nodes int = 3
const num_messages int = 2
// some arbitary large number
const buffer_channel = 10000
type Server struct {
serverChannel chan Message
nodeArray []Node
timestamp int
}
type Node struct {
nodeId int
nodeChannel chan Message
server Server
closeChannel chan int
readyChannel chan int
timestamp int
}
type Message struct {
senderId int
messageId int
timestamp int
}
func max(x int, y int) int {
if x > y {
return x
}
return y
}
func broadcast(m Message, s Server) {
for _, n := range s.nodeArray {
if n.nodeId != m.senderId {
broadcastMessage := Message{
m.senderId,
m.messageId,
s.timestamp,
}
go s.broadcastMessage(n.nodeChannel, broadcastMessage)
}
}
}
func (s Server) broadcastMessage(nodeChannel chan Message, broadcastMessage Message) {
fmt.Printf("[Server] is sending Message %d.%d to [Node %d]\n", broadcastMessage.senderId, broadcastMessage.messageId, broadcastMessage.senderId)
nodeChannel <- broadcastMessage
}
func (s Server) listen(messagesBufferChannel chan Message) {
numCompletedNodes := 0
for {
nodeMessage := <-s.serverChannel
s.timestamp = max(s.timestamp, nodeMessage.timestamp) + 1
nodeMessage.timestamp = s.timestamp
fmt.Printf("TS: %d -- [Server] has received Message %d.%d from [Node %d]\n", s.timestamp, nodeMessage.senderId, nodeMessage.messageId, nodeMessage.senderId)
messagesBufferChannel <- nodeMessage
s.timestamp += 1
broadcast(nodeMessage, s)
if nodeMessage.messageId == num_messages-1 {
numCompletedNodes += 1
if numCompletedNodes == num_nodes {
fmt.Println("Server finish broadcasting all messages. Stopping Server...")
return
}
}
numMilliSeconds := rand.Intn(1000) + 2000
time.Sleep(time.Duration(numMilliSeconds) * time.Millisecond)
}
}
func (n Node) preSendMessage() {
for i := 1; i <= num_messages; i++ {
numMilliSeconds := rand.Intn(1000) + 2000
time.Sleep(time.Duration(numMilliSeconds) * time.Millisecond)
n.readyChannel <- i
}
}
func (n Node) listenSendMessages(messagesBufferChannel chan Message) {
for {
select {
case receivedMessage := <-n.nodeChannel:
n.timestamp = max(n.timestamp, receivedMessage.timestamp) + 1
receivedMessage.timestamp = n.timestamp
fmt.Printf("TS: %d -- [Node %d] has received Message %d.%d from [Server]\n", n.timestamp, n.nodeId, receivedMessage.senderId, receivedMessage.messageId)
messagesBufferChannel <- receivedMessage
case nodeMessageId := <-n.readyChannel:
n.timestamp += 1
fmt.Printf("TS: %d -- [Node %d] is sending Message %d.%d to [Server]\n", n.timestamp, n.nodeId, n.nodeId, nodeMessageId)
nodeMessage := Message{
n.nodeId,
nodeMessageId,
n.timestamp,
}
n.server.serverChannel <- nodeMessage
case <-n.closeChannel:
fmt.Printf("Stopping [node %d]\n", n.nodeId)
return
default:
}
}
}
func main() {
fmt.Println("Start of Program...")
server := Server{
serverChannel: make(chan Message),
nodeArray: []Node{},
timestamp: 0,
}
for i := 1; i <= num_nodes; i++ {
newNode := Node{
nodeId: i,
nodeChannel: make(chan Message),
server: server,
readyChannel: make(chan int),
closeChannel: make(chan int),
timestamp: 0,
}
server.nodeArray = append(server.nodeArray, newNode)
}
var messagesBufferChannel chan Message = make(chan Message, buffer_channel)
for _, n := range server.nodeArray {
go n.preSendMessage()
go n.listenSendMessages(messagesBufferChannel)
}
server.listen(messagesBufferChannel)
time.Sleep(time.Second)
for _, n := range server.nodeArray {
n.closeChannel <- 1
}
time.Sleep(time.Second)
close(messagesBufferChannel)
}
In my Go program I start multiple worker groups for every department.
I want to wait for workers from each department to complete before exiting the program
I cannot use a single WaitGroups because in the actual scenario I may have to end any particular department and need to wait only on that.
This is simplified version of code, but it panics with a message
panic: runtime error: invalid memory address or nil pointer dereference
package main
import (
"fmt"
"sync"
"time"
)
var wgMap map[string]*sync.WaitGroup
func deptWorker(dName string, id int) {
defer wgMap[dName].Done()
fmt.Printf("Department %s : Worker %d starting\n", dName, id)
time.Sleep(time.Second)
fmt.Printf("Department %s : Worker %d done\n", dName, id)
}
func department(dName string) {
var wg sync.WaitGroup
for i := 1; i <= 3; i++ {
wg.Add(1)
go deptWorker(dName, i)
}
wgMap[dName] = &wg
}
func main() {
go department("medical")
go department("electronics")
wgMap["medical"].Wait()
wgMap["electronics"].Wait()
}
Two fix nil panic you simply need to use
var wgMap = map[string]*sync.WaitGroup{}
It will initialize the map. However, in my view, it's better here to create a new abstraction, let's name it 'WaitMap'.
It can be implemented in this way:
package main
import (
"fmt"
"sync"
"time"
)
type WaitMapObject struct {
wg map[string]int
mu sync.Mutex
cond sync.Cond
}
func WaitMap() *WaitMapObject {
m := &WaitMapObject{}
m.wg = make(map[string]int)
m.cond.L = &m.mu
return m
}
func (m *WaitMapObject) Wait(name string) {
m.mu.Lock()
for m.wg[name] != 0 {
m.cond.Wait()
}
m.mu.Unlock()
}
func (m *WaitMapObject) Done(name string) {
m.mu.Lock()
no := m.wg[name] - 1
if no < 0 {
panic("")
}
m.wg[name] = no
m.mu.Unlock()
m.cond.Broadcast()
}
func (m *WaitMapObject) Add(name string, no int) {
m.mu.Lock()
m.wg[name] = m.wg[name] + no
m.mu.Unlock()
}
func deptWorker(dName string, id int, wm *WaitMapObject) {
defer wm.Done(dName)
fmt.Printf("Department %s : Worker %d starting\n", dName, id)
time.Sleep(time.Second)
fmt.Printf("Department %s : Worker %d done\n", dName, id)
}
func department(dName string, wm *WaitMapObject) {
for i := 1; i <= 3; i++ {
wm.Add(dName,1)
go deptWorker(dName, i, wm)
}
wm.Done(dName)
}
func main() {
wm := WaitMap()
wm.Add("mediacal",1)
go department("medical", wm)
wm.Add("electronics",1)
go department("electronics", wm)
wm.Wait("medical")
wm.Wait("electronics")
}
I have a channel which stores received data, I want to process it when one of following conditions is met:
1, the channel reaches its capacity.
2, the timer is fired since last process.
I saw the post
Golang - How to know a buffered channel is full
Update:
I inspired from that post and OneOfOne's advice, here is the play :
package main
import (
"fmt"
"math/rand"
"time"
)
var c chan int
var timer *time.Timer
const (
capacity = 5
timerDration = 3
)
func main() {
c = make(chan int, capacity)
timer = time.NewTimer(time.Second * timerDration)
go checkTimer()
go sendRecords("A")
go sendRecords("B")
go sendRecords("C")
time.Sleep(time.Second * 20)
}
func sendRecords(name string) {
for i := 0; i < 20; i++ {
fmt.Println(name+" sending record....", i)
sendOneRecord(i)
interval := time.Duration(rand.Intn(500))
time.Sleep(time.Millisecond * interval)
}
}
func sendOneRecord(record int) {
select {
case c <- record:
default:
fmt.Println("channel is full !!!")
process()
c <- record
timer.Reset(time.Second * timerDration)
}
}
func checkTimer() {
for {
select {
case <-timer.C:
fmt.Println("3s timer ----------")
process()
timer.Reset(time.Second * timerDration)
}
}
}
func process() {
for i := 0; i < capacity; i++ {
fmt.Println("process......", <-c)
}
}
This seems to work fine, but I have a concern, I want to block the channel writing from other goroutine when process() is called, is the code above capable to do so? Or should I add a mutex at the beginning of the process method?
Any elegant solution?
As was mentioned by #OneOfOne, select is really the only way to check if a channel is full.
If you are using the channel to effect batch processing, you could always create an unbuffered channel and have a goroutine pull items and append to a slice.
When the slice reaches a specific size, process the items.
Here's an example on play
package main
import (
"fmt"
"sync"
"time"
)
const BATCH_SIZE = 10
func batchProcessor(ch <-chan int) {
batch := make([]int, 0, BATCH_SIZE)
for i := range ch {
batch = append(batch, i)
if len(batch) == BATCH_SIZE {
fmt.Println("Process batch:", batch)
time.Sleep(time.Second)
batch = batch[:0] // trim back to zero size
}
}
fmt.Println("Process last batch:", batch)
}
func main() {
var wg sync.WaitGroup
ch := make(chan int)
wg.Add(1)
go func() {
batchProcessor(ch)
wg.Done()
}()
fmt.Println("Submitting tasks")
for i := 0; i < 55; i++ {
ch <- i
}
close(ch)
wg.Wait()
}
No, select is the only way to do it:
func (t *T) Send(v *Val) {
select {
case t.ch <- v:
default:
// handle v directly
}
}