Familiarizing myself with concurrency, so started writing a simple ping cli with concurrent calls (let's ignore that I'm not really measuring pings).
Problem is, I can't close the channel properly for the range loop while also waiting for all the goroutines to finish.
If I want to concurrently call the ping function, I can't synchronize it with waitgroups, because I will reach the wg.Wait() line before all the goroutines finish.
Is there a way to keep the ping calls concurrent and close the channel after they're done, so the range loop can exit?
func main() {
domain := flag.String("domain", "google.com", "the domain u want to ping")
flag.Parse()
sum := 0
ch := make(chan int)
go start_pings(*domain, ch)
for elapsed := range ch {
fmt.Println("Part time: " + strconv.Itoa(elapsed))
sum += elapsed
}
avg := sum / 3
fmt.Println("Average: " + strconv.Itoa(avg))
}
func start_pings(domain string, ch chan int) {
var wg sync.WaitGroup
for i := 0; i < 3; i++ {
wg.Add(1)
go ping(domain, ch, wg)
}
wg.Wait()
close(ch)
}
func ping(domain string, ch chan int, wg sync.WaitGroup) {
url := "http://" + domain
start := time.Now()
fmt.Println("Start pinging " + url + "...")
resp, err := http.Get(url)
elapsed := time.Now().Sub(start)
if err != nil {
fmt.Println(err)
return
}
defer resp.Body.Close()
ch <- int(elapsed)
wg.Done()
}
You must not copy a sync.WaitGroup! It's doc explicitly states:
A WaitGroup must not be copied after first use.
Pass a pointer to it: wg *sync.WaitGroup. And call wg.Done() deferred! You have other return statements which will cause wg.Done() to be skipped!
func start_pings(domain string, ch chan int) {
var wg sync.WaitGroup
for i := 0; i < 3; i++ {
wg.Add(1)
go ping(domain, ch, &wg)
}
wg.Wait()
close(ch)
}
func ping(domain string, ch chan int, wg *sync.WaitGroup) {
defer wg.Done()
url := "http://" + domain
start := time.Now()
fmt.Println("Start pinging " + url + "...")
resp, err := http.Get(url)
elapsed := time.Since(start)
if err != nil {
fmt.Println(err)
return
}
defer resp.Body.Close()
ch <- int(elapsed)
}
Today's IDE's (and golinter) warns about such obvious misuses. To avoid such mistakes, declare wg to be a pointer in the first place:
func start_pings(domain string, ch chan int) {
wg := &sync.WaitGroup{} // Pointer!
for i := 0; i < 3; i++ {
wg.Add(1)
go ping(domain, ch, wg)
}
wg.Wait()
close(ch)
}
This leaves less room for errors and misuses.
Related
func writeToChan(wg *sync.WaitGroup, ch chan int, stop int) {
defer wg.Done()
for i := 0; i < stop; i++ {
ch <- i
}
}
func readToChan(wg *sync.WaitGroup, ch chan int) {
defer wg.Done()
for n := range ch {
fmt.Println(n)
}
}
func main() {
ch := make(chan int, 3)
wg := new(sync.WaitGroup)
wg.Add(2)
go writeToChan(wg, ch, 5)
go readToChan(wg, ch)
wg.Wait()
}
0
1
2
3
4
fatal error: all goroutines are asleep - deadlock!
I assume that the readToChan always reads continuously, and the writeToChan write to the channel and waits while the channel is read.
I don't know why the output showed deadlock while I added two 'wait' to the WaitGroup.
You need to close channel at the sender side.
By using
for n := range ch {
fmt.Println(n)
}
The loop will only stop when ch is closed
correct example:
package main
import (
"fmt"
"sync"
)
func writeToChan(wg *sync.WaitGroup, ch chan int, stop int) {
defer wg.Done()
for i := 0; i < stop; i++ {
ch <- i
}
close(ch)
}
func readToChan(wg *sync.WaitGroup, ch chan int) {
defer wg.Done()
for n := range ch {
fmt.Println(n)
}
}
func main() {
ch := make(chan int, 3)
wg := new(sync.WaitGroup)
wg.Add(2)
go writeToChan(wg, ch, 5)
go readToChan(wg, ch)
wg.Wait()
}
If close is not called on buffered channel, reader doesn't know when to stop reading.
Check this example with for and select calls(to handle multi channels).
https://go.dev/play/p/Lx5g9o4RsqW
package main
import (
"fmt"
"sync"
"time")
func writeToChan(wg *sync.WaitGroup, ch chan int, stop int, quit chan<- bool) {
defer func() {
wg.Done()
close(ch)
fmt.Println("write wg done")
}()
for i := 0; i < stop; i++ {
ch <- i
fmt.Println("write:", i)
}
fmt.Println("write done")
fmt.Println("sleeping for 5 sec")
time.Sleep(5 * time.Second)
quit <- true
close(quit)
}
func readToChan(wg *sync.WaitGroup, ch chan int, quit chan bool) {
defer func() {
wg.Done()
fmt.Println("read wg done")
}()
//using rang over
//for n := range ch {
// fmt.Println(n)
//}
//using Select if you have multiple channels.
for {
//fmt.Println("waiting for multiple channels")
select {
case n := <-ch:
fmt.Println("read:", n)
// if ok == false {
// fmt.Println("read done")
// //return
// }
case val := <-quit:
fmt.Println("received quit :", val)
return
// default:
// fmt.Println("default")
}
}
}
func main() {
ch := make(chan int, 5)
ch2 := make(chan bool)
wg := new(sync.WaitGroup)
wg.Add(2)
go writeToChan(wg, ch, 3, ch2)
go readToChan(wg, ch, ch2)
wg.Wait()
}
Output:
write: 0
write: 1
write: 2
write done
sleeping for 5 sec
read: 0
read: 1
read: 2
write wg done
received quit : true
read wg done
Program exited.
I am encountering for the below code fatal error: all goroutines are asleep - deadlock!
Am I right in using a buffered channel? I would appreciate it if you can give me pointers. I am unfortunately at the end of my wits.
func main() {
valueChannel := make(chan int, 2)
defer close(valueChannel)
var wg sync.WaitGroup
for i := 0; i < 10; i++ {
wg.Add(1)
go doNothing(&wg, valueChannel)
}
for {
v, ok := <- valueChannel
if !ok {
break
}
fmt.Println(v)
}
wg.Wait()
}
func doNothing(wg *sync.WaitGroup, numChan chan int) {
defer wg.Done()
time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond)
numChan <- 12
}
The main goroutine blocks on <- valueChannel after receiving all values. Close the channel to unblock the main goroutine.
func main() {
valueChannel := make(chan int, 2)
var wg sync.WaitGroup
for i := 0; i < 10; i++ {
wg.Add(1)
go doNothing(&wg, valueChannel)
}
// Close channel after goroutines complete.
go func() {
wg.Wait()
close(valueChannel)
}()
// Receive values until channel is closed.
// The for / range loop here does the same
// thing as the for loop in the question.
for v := range valueChannel {
fmt.Println(v)
}
}
Run the example on the playground.
The code above works independent of the number of values sent by the goroutines.
If the main() function can determine the number of values sent by the goroutines, then receive that number of values from main():
func main() {
const n = 10
valueChannel := make(chan int, 2)
for i := 0; i < n; i++ {
go doNothing(valueChannel)
}
// Each call to doNothing sends one value. Receive
// one value for each call to doNothing.
for i := 0; i < n; i++ {
fmt.Println(<-valueChannel)
}
}
func doNothing(numChan chan int) {
time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond)
numChan <- 12
}
Run the example on the playground.
The main problem is on the for loop of channel receiving.
The comma ok idiom is slightly different on channels, ok indicates whether the received value was sent on the channel (true) or is a zero value returned because the channel is closed and empty (false).
In this case the channel is waiting a data to be sent and since it's already finished sending the value ten times : Deadlock.
So apart of the design of the code if I just need to do the less change possible here it is:
func main() {
valueChannel := make(chan int, 2)
defer close(valueChannel)
var wg sync.WaitGroup
for i := 0; i < 10; i++ {
wg.Add(1)
go doNothing(&wg, valueChannel)
}
for i := 0; i < 10; i++ {
v := <- valueChannel
fmt.Println(v)
}
wg.Wait()
}
func doNothing(wg *sync.WaitGroup, numChan chan int) {
defer wg.Done()
time.Sleep(time.Duration(rand.Intn(1000)) * time.Millisecond)
numChan <- 12
}
Closed. This question is not reproducible or was caused by typos. It is not currently accepting answers.
This question was caused by a typo or a problem that can no longer be reproduced. While similar questions may be on-topic here, this one was resolved in a way less likely to help future readers.
Closed 1 year ago.
Improve this question
I have written a code, tried to use concurrency but it's not helping to run any faster. How can I improve that?
package main
import (
"bufio"
"fmt"
"os"
"strings"
"sync"
)
var wg sync.WaitGroup
func checkerr(e error) {
if e != nil {
fmt.Println(e)
}
}
func readFile() {
file, err := os.Open("data.txt")
checkerr(err)
fres, err := os.Create("resdef.txt")
checkerr(err)
defer file.Close()
defer fres.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
wg.Add(1)
go func() {
words := strings.Fields(scanner.Text())
shellsort(words)
writeToFile(fres, words)
wg.Done()
}()
wg.Wait()
}
}
func shellsort(words []string) {
for inc := len(words) / 2; inc > 0; inc = (inc + 1) * 5 / 11 {
for i := inc; i < len(words); i++ {
j, temp := i, words[i]
for ; j >= inc && strings.ToLower(words[j-inc]) > strings.ToLower(temp); j -= inc {
words[j] = words[j-inc]
}
words[j] = temp
}
}
}
func writeToFile(f *os.File, words []string) {
datawriter := bufio.NewWriter(f)
for _, s := range words {
datawriter.WriteString(s + " ")
}
datawriter.WriteString("\n")
datawriter.Flush()
}
func main() {
readFile()
}
Everything works well except that it take the same time to do everything as without concurrency.
You must place wg.Wait() after the for loop:
for condition {
wg.Add(1)
go func() {
// a concurrent job here
wg.Done()
}()
}
wg.Wait()
Note: the work itself should have a concurrent nature.
Here is my tested solution - read from the input file sequentially then do n concurrent tasks and finally write to the output file sequentially in order, try this:
package main
import (
"bufio"
"fmt"
"log"
"os"
"runtime"
"sort"
"strings"
"sync"
)
type sortQueue struct {
index int
data []string
}
func main() {
n := runtime.NumCPU()
a := make(chan sortQueue, n)
b := make(chan sortQueue, n)
var wg sync.WaitGroup
for i := 0; i < n; i++ {
wg.Add(1)
go parSort(a, b, &wg)
}
go func() {
file, err := os.Open("data.txt")
if err != nil {
log.Fatal(err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
i := 0
for scanner.Scan() {
a <- sortQueue{index: i, data: strings.Fields(scanner.Text())}
i++
}
close(a)
err = scanner.Err()
if err != nil {
log.Fatal(err)
}
}()
fres, err := os.Create("resdef.txt")
if err != nil {
log.Fatal(err)
}
defer fres.Close()
go func() {
wg.Wait()
close(b)
}()
writeToFile(fres, b, n)
}
func writeToFile(f *os.File, b chan sortQueue, n int) {
m := make(map[int][]string, n)
order := 0
for v := range b {
m[v.index] = v.data
var slice []string
exist := true
for exist {
slice, exist = m[order]
if exist {
delete(m, order)
order++
s := strings.Join(slice, " ")
fmt.Println(s)
_, err := f.WriteString(s + "\n")
if err != nil {
log.Fatal(err)
}
}
}
}
}
func parSort(a, b chan sortQueue, wg *sync.WaitGroup) {
defer wg.Done()
for q := range a {
sort.Slice(q.data, func(i, j int) bool { return q.data[i] < q.data[j] })
b <- q
}
}
data.txt file:
1 2 0 3
a 1 b d 0 c
aa cc bb
Output:
0 1 2 3
0 1 a b c d
aa bb cc
You're not parallelizing anything, because for every call to wg.Add(1) you have matching call to wg.Wait(). It's one-to-one: You spawn a Go routine, and then immediately block the main Go routine waiting for the newly spawned routine to finish.
The point of a WaitGroup is to wait for many things to finish, with a single call to wg.Wait() when all the Go routines have been spawned.
However, in addition to fixing your call to wg.Wait, you need to control concurrent access to your scanner. One approach to this might be to use a channel for your scanner to emit lines of text to waiting Go routines:
lines := make(chan string)
go func() {
for line := range lines {
go func(line string) {
words := strings.Fields(line)
shellsort(words)
writeToFile(fres, words)
}(line)
}
}()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
lines <- scanner.Text()
}
close(lines)
Note that this may lead to garbled output in your file, as you have many concurrent Go routines all writing their results at the same time. You can control output through a second channel:
lines := make(chan string)
out := make(chan []string)
go func() {
for line := range lines {
go func(line string) {
words := strings.Fields(line)
shellsort(words)
out <- words
}(line)
}
}()
go func() {
for words := range out {
writeToFile(fres, words)
}
}()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
lines <- scanner.Text()
}
close(lines)
close(out)
At this point, you can refactor into a "reader", a "processor" and a "writer", which form a pipeline that communicates via channels.
The reader and writer use a single go routine to prevent concurrent access to a resource, while the processor spawns many go routines (currently unbounded) to "fan out" the work across many processors:
package main
import (
"bufio"
"os"
"strings"
)
func main() {
lines := reader()
out := processor(lines)
writer(out)
}
func reader() chan<- string {
lines := make(chan string)
file, err := os.Open("data.txt")
checkerr(err)
go func() {
scanner := bufio.NewScanner(file)
for scanner.Scan() {
lines <- scanner.Text()
}
close(lines)
}()
return lines
}
func processor(lines chan<- string) chan []string {
out := make(chan []string)
go func() {
for line := range lines {
go func(line string) {
words := strings.Fields(line)
shellsort(words)
out <- words
}(line)
}
close(out)
}()
return out
}
func writer(out chan<- []string) {
fres, err := os.Create("resdef.txt")
checkerr(err)
for words := range out {
writeToFile(fres, words)
}
}
As other answers have said, by waiting on the WaitGroup each loop iteration, you're limiting your concurrency to 1 (no concurrency). There are a number of ways to solve this, but what's correct depends entirely on what is taking time, and that hasn't been shown in the question. Concurrency doesn't magically make things faster; it just lets things happen at the same time, which only makes things faster if things that take a lot of time can happen concurrently.
Presumably, in your code, the thing that takes a long time is the sort. If that is the case, you could do something like this:
results := make(chan []string)
for scanner.Scan() {
wg.Add(1)
go func(line string) {
words := strings.Fields(line)
shellsort(words)
result <- words
}(scanner.Text())
}
go func() {
wg.Wait()
close(results)
}()
for words := range results {
writeToFile(fres, words)
}
This moves the Wait to where it should be, and avoids concurrent use of the scanner and writer. This should be faster than serial processing, if the sort is taking a significant amount of processing time.
I'm trying to learn channels in Go, so this is a contrived example of having a channel where there are multiple writers, but only one reader. This is a very basic example, but I would like to take it further, by imagining its a http server, where there is a goroutine created for each new request, and each goroutine makes a write on a channel.
func Start2(){
var wg sync.WaitGroup
wg.Add(2)
c := make(chan int)
go func(){
defer wg.Done()
i := 0
for {
i+=2
c<-i
time.Sleep(time.Duration(rand.Intn(5)) * time.Second)
if i > 10 {
break
}
}
}()
go func(){
defer wg.Done()
i := 1
for {
i+=2
c<-i
time.Sleep(time.Duration(rand.Intn(5)) * time.Second)
if i > 10 {
break
}
}
}()
for a := range c {
log.Println(a)
}
wg.Wait()
}
Why does this deadlock?
How could I organise the logic of reading from the channel, so that the code doesn't deadlock when there are no writers on the channel? (For instance, in the end goal example, if there isn't currently a request being made to the http server)?
Unbuffered channel needs at least two goroutines to operate and by exiting one of then. the Go runtime is smart enough to detect the deadlock, so you have two options here:
Since you want to use the channel in the http server - there is no deadlock here:
package main
import (
"fmt"
"log"
"net/http"
"time"
)
func main() {
http.HandleFunc("/", home)
go func() {
err := http.ListenAndServe(":8080", nil)
if err != nil {
log.Fatal(err)
}
}()
count := 0
for a := range c {
count++
fmt.Println(count, a)
}
}
var c = make(chan time.Time)
func home(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, "Hi")
c <- time.Now()
}
Or close the channel to force the main goroutine to exit too - just for your current example code - try it:
package main
import (
"log"
"math/rand"
"sync"
"time"
)
func main() {
wg := &sync.WaitGroup{}
wg.Add(2)
c := make(chan int)
go routine(0, c, wg)
go routine(1, c, wg)
go func() {
wg.Wait()
close(c)
}()
for a := range c {
log.Println(a)
}
}
func routine(i int, c chan int, wg *sync.WaitGroup) {
defer wg.Done()
for {
i += 2
c <- i
time.Sleep(time.Duration(rand.Intn(5)) * time.Second)
if i > 10 {
break
}
}
}
You are not closing your channel, you need to close it when you are done.
You can create a goroutine for reading from your channel and close it after the wait group.
Example:
var wg sync.WaitGroup
wg.Add(2)
c := make(chan int)
go func() {
defer wg.Done()
i := 0
for {
i += 2
c <- i
time.Sleep(time.Duration(rand.Intn(5)) * time.Second)
if i > 10 {
return
}
}
}()
go func() {
defer wg.Done()
i := 1
for {
i += 2
c <- i
time.Sleep(time.Duration(rand.Intn(5)) * time.Second)
if i > 10 {
break
}
}
}()
//read channel in a goroutine
go func() {
for a := range c {
log.Println(a)
}
}()
wg.Wait()
//close channel when done
close(c)
I'm reading the "The Go Programming Language"
One way to limit the number of running go routines is to use a "counting semaphore".
The other way is Limiting number of go routines running
I am allowing 2 more go routines in the case. I'm getting deadlock error.
What causes the deadlock in my code?
package main
import (
"bytes"
//"context"
"fmt"
"runtime"
"strconv"
"sync"
"time"
)
func main() {
max := 2
var wg sync.WaitGroup
squares := make(chan int)
tokens := make(chan struct{}, max)
for i := 20; i >= 1; i-- {
tokens <- struct{}{}
wg.Add(1)
go func(n int) {
defer func() { <-tokens }()
defer wg.Done()
fmt.Println("run go routine ", getGID())
squares <- Square(n)
}(i)
}
go func() {
wg.Wait()
close(squares)
}()
for s := range squares {
fmt.Println("Get square: ", s)
}
}
func Square(num int) int {
time.Sleep(time.Second * time.Duration(num))
fmt.Println(num * num)
return num * num
}
func getGID() uint64 {
b := make([]byte, 64)
b = b[:runtime.Stack(b, false)]
b = bytes.TrimPrefix(b, []byte("goroutine "))
b = b[:bytes.IndexByte(b, ' ')]
n, _ := strconv.ParseUint(string(b), 10, 64)
return n
}
The goroutines block on sending to squares. Main is not receiving on squares because it blocks on starting the the goroutines.
Fix by moving the code that starts the goroutines to a goroutine. This allows main to continue executing to the receive on squares.
squares := make(chan int)
go func() {
max := 2
var wg sync.WaitGroup
tokens := make(chan struct{}, max)
for i := 20; i >= 1; i-- {
tokens <- struct{}{}
wg.Add(1)
go func(n int) {
defer func() { <-tokens }()
defer wg.Done()
fmt.Println("run go routine ", getGID())
squares <- Square(n)
}(i)
}
wg.Wait()
close(squares)
}()
fmt.Println("About to receive")
for s := range squares {
fmt.Println("Get square: ", s)
}
Run it on the playground.