Program goes into deadlock using waitgroup - go

I'm writing a program that reads a list of order numbers in a file called orders.csv and compares it with the other csv files that are present in the folder.
The problem is that it goes into deadlock even using waitgroup and I don't know why.
For some reason stackoverflow says that my post is mostly code, so I have to add this line, because the whole code is necessary if someone wants to help me debug this problem I'm having.
package main
import (
"bufio"
"fmt"
"log"
"os"
"path/filepath"
"strings"
"sync"
)
type Files struct {
filenames []string
}
type Orders struct {
ID []string
}
var ordersFilename string = "orders.csv"
func main() {
var (
ordersFile *os.File
files Files
orders Orders
err error
)
mu := new(sync.Mutex)
wg := &sync.WaitGroup{}
wg.Add(1)
if ordersFile, err = os.Open(ordersFilename); err != nil {
log.Fatalln("Could not open file: " + ordersFilename)
}
orders = getOrderIDs(ordersFile)
files.filenames = getCSVsFromCurrentDir()
var filenamesSize = len(files.filenames)
var ch = make(chan map[string][]string, filenamesSize)
var done = make(chan bool)
for i, filename := range files.filenames {
go func(currentFilename string, ch chan<- map[string][]string, i int, orders Orders, wg *sync.WaitGroup, filenamesSize *int, mu *sync.Mutex, done chan<- bool) {
wg.Add(1)
defer wg.Done()
checkFile(currentFilename, orders, ch)
mu.Lock()
*filenamesSize--
mu.Unlock()
if i == *filenamesSize {
done <- true
close(done)
}
}(filename, ch, i, orders, wg, &filenamesSize, mu, done)
}
select {
case str := <-ch:
fmt.Printf("%+v\n", str)
case <-done:
wg.Done()
break
}
wg.Wait()
close(ch)
}
// getCSVsFromCurrentDir returns a string slice
// with the filenames of csv files inside the
// current directory that are not "orders.csv"
func getCSVsFromCurrentDir() []string {
var filenames []string
err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
if path != "." && strings.HasSuffix(path, ".csv") && path != ordersFilename {
filenames = append(filenames, path)
}
return nil
})
if err != nil {
log.Fatalln("Could not read file names in current dir")
}
return filenames
}
// getOrderIDs returns an Orders struct filled
// with order IDs retrieved from the file
func getOrderIDs(file *os.File) Orders {
var (
orders Orders
err error
fileContent string
)
reader := bufio.NewReader(file)
if fileContent, err = readLine(reader); err != nil {
log.Fatalln("Could not read file: " + ordersFilename)
}
for err == nil {
orders.ID = append(orders.ID, fileContent)
fileContent, err = readLine(reader)
}
return orders
}
func checkFile(filename string, orders Orders, ch chan<- map[string][]string) {
var (
err error
file *os.File
fileContent string
orderFilesMap map[string][]string
counter int
)
orderFilesMap = make(map[string][]string)
if file, err = os.Open(filename); err != nil {
log.Fatalln("Could not read file: " + filename)
}
reader := bufio.NewReader(file)
if fileContent, err = readLine(reader); err != nil {
log.Fatalln("Could not read file: " + filename)
}
for err == nil {
if containedInSlice(fileContent, orders.ID) && !containedInSlice(fileContent, orderFilesMap[filename]) {
orderFilesMap[filename] = append(orderFilesMap[filename], fileContent)
// fmt.Println("Found: ", fileContent, " in ", filename)
} else {
// fmt.Printf("Could not find: '%s' in '%s'\n", fileContent, filename)
}
counter++
fileContent, err = readLine(reader)
}
ch <- orderFilesMap
}
// containedInSlice returns true or false
// based on whether the string is contained
// in the slice
func containedInSlice(str string, slice []string) bool {
for _, ID := range slice {
if ID == str {
return true
}
}
return false
}
// readLine returns a line from the passed reader
func readLine(r *bufio.Reader) (string, error) {
var (
isPrefix bool = true
err error = nil
line, ln []byte
)
for isPrefix && err == nil {
line, isPrefix, err = r.ReadLine()
ln = append(ln, line...)
}
return string(ln), err
}

The first issue is the wg.Add always must be outside of the goroutine(s) it stands for. If it isn't, the
wg.Wait call might be called before the goutine(s) have actually started running (and called wg.Add) and therefore will "think"
that there is nothing to wait for.
The second issue with the code is that there are multiple ways it waits for the routines to be done. There is
the WaitGroup and there is the done channel. Use only one of them. Which one depends also on how the results of the
goroutines are used. Here we come to the next problem.
The third issue is with gathering the results. Currently the code only prints / uses a single result from the goroutines.
Put a for { ... } loop around the select and use return to break out of the loop if the done channel is closed.
(Note that you don't need to send anything on the done channel, closing it is enough.)
Improved Version 0.0.1
So here the first version (including some other "code cleanup") with a done channel used for closing and the WaitGroup removed:
func main() {
ordersFile, err := os.Open(ordersFilename)
if err != nil {
log.Fatalln("Could not open file: " + ordersFilename)
}
orders := getOrderIDs(ordersFile)
files := Files{
filenames: getCSVsFromCurrentDir(),
}
var (
mu = new(sync.Mutex)
filenamesSize = len(files.filenames)
ch = make(chan map[string][]string, filenamesSize)
done = make(chan bool)
)
for i, filename := range files.filenames {
go func(currentFilename string, ch chan<- map[string][]string, i int, orders Orders, filenamesSize *int, mu *sync.Mutex, done chan<- bool) {
checkFile(currentFilename, orders, ch)
mu.Lock()
*filenamesSize--
mu.Unlock()
// TODO: This also accesses filenamesSize, so it also needs to be protected with the mutex:
if i == *filenamesSize {
done <- true
close(done)
}
}(filename, ch, i, orders, &filenamesSize, mu, done)
}
// Note: closing a channel is not really needed, so you can omit this:
defer close(ch)
for {
select {
case str := <-ch:
fmt.Printf("%+v\n", str)
case <-done:
return
}
}
}
Improved Version 0.0.2
In your case we have some advantage however. We know exactly how many goroutines we started and therefore also how
many results we expect. (Of course if each goroutine returns a result which currently this code does.) That gives
us another option as we can collect the results with another for loop having the same amount of iterations:
func main() {
ordersFile, err := os.Open(ordersFilename)
if err != nil {
log.Fatalln("Could not open file: " + ordersFilename)
}
orders := getOrderIDs(ordersFile)
files := Files{
filenames: getCSVsFromCurrentDir(),
}
var (
// Note: a buffered channel helps speed things up. The size does not need to match the size of the items that will
// be passed through the channel. A fixed, small size is perfect here.
ch = make(chan map[string][]string, 5)
)
for _, filename := range files.filenames {
go func(filename string) {
// orders and channel are not variables of the loop and can be used without copying
checkFile(filename, orders, ch)
}(filename)
}
for range files.filenames {
str := <-ch
fmt.Printf("%+v\n", str)
}
}
A lot simpler, isn't it? Hope that helps!

There is a lot wrong with this code.
You're using the WaitGroup wrong. Add has to be called in the main goroutine, else there is a chance that Wait is called before all Add calls complete.
There's an extraneous Add(1) call right after initializing the WaitGroup that isn't matched by a Done() call, so Wait will never return (assuming the point above is fixed).
You're using both a WaitGroup and a done channel to signal completion. This is redundant at best.
You're reading filenamesSize while not holding the lock (in the if i == *filenamesSize statement). This is a race condition.
The i == *filenamesSize condition makes no sense in the first place. Goroutines execute in an arbitrary order, so you can't be sure that the goroutine with i == 0 is the last one to decrement filenamesSize
This can all be simplified by getting rid of most if the synchronization primitives and simply closing the ch channel when all goroutines are done:
func main() {
ch := make(chan map[string][]string)
var wg WaitGroup
for _, filename := range getCSVsFromCurrentDir() {
filename := filename // capture loop var
wg.Add(1)
go func() {
checkFile(filename, orders, ch)
wg.Done()
}()
}
go func() {
wg.Wait() // after all goroutines are done...
close(ch) // let range loop below exit
}()
for str := range ch {
// ...
}
}

not an answer, but some comments that does not fit the comment box.
In this part of the code
func main() {
var (
ordersFile *os.File
files Files
orders Orders
err error
)
mu := new(sync.Mutex)
wg := &sync.WaitGroup{}
wg.Add(1)
The last statement is a call to wg.Add that appears dangling. By that i mean we can hardly understand what will trigger the required wg.Done counter part. This is a mistake to call for wg.Add without a wg.Done, this is prone to errors to not write them in such way we can not immediately find them in pair.
In that part of the code, it is clearly wrong
go func(currentFilename string, ch chan<- map[string][]string, i int, orders Orders, wg *sync.WaitGroup, filenamesSize *int, mu *sync.Mutex, done chan<- bool) {
wg.Add(1)
defer wg.Done()
Consider that by the time the routine is executed, and that you added 1 to the waitgroup, the parent routine continues to execute. See this example: https://play.golang.org/p/N9Chaqkv4bd
The main routine does not wait for the waitgroup because it does not have time to increment.
There is more to say but i find it hard to understand the purpose of your code so i am not sure how to help you further without basically rewrite it.

Related

All Goroutines Are Asleep (The Go Programming Language)

I'm working through The Go Programming Language and learning about goroutines, and came across the following issue. In this example, the following function is meant to take a channel of files and process each of them:
func makeThumbnails5(filenames <-chan string) int64 {
sizes := make(chan int64)
var wg sync.WaitGroup
for f := range filenames {
wg.Add(1)
// worker
go func(f string) {
defer wg.Done()
thumb, err := thumbnail.ImageFile(f)
if err != nil {
log.Println(err)
return
}
info, _ := os.Stat(thumb)
sizes <- info.Size()
}(f)
}
// closer
go func() {
wg.Wait()
close(sizes)
}()
var total int64
for size := range sizes {
total += size
}
wg.Wait()
return total
}
I've tried to use this function the following way:
func main() {
thumbnails := os.Args[1:] /* Get a list of all the images from the CLI */
ch := make(chan string, len(thumbnails))
for _, val := range thumbnails {
ch <- val
}
makeThumbnails5(ch)
}
However, when I run this program, I get the following error:
fatal error: all goroutines are asleep - deadlock!
It doesn't appear that the closer goroutine is running. Could someone help me understand what is going wrong here, and what I can do to run this function correctly?
As I commented it deadlocks because the filenames chan is never closed and thus the for f := range filenames loop never completes. However, just closing the input chan means that all goroutines launched in the loop would get stuck at the line sizes <- info.Size() until the loop ends. Not a problem in this case but if the input can be huge it could be (then you'd probably want to limit the number of concurrent workers too). So it makes sense to have the main loop in a goroutine too so that the for size := range sizes loop can start consuming. Following should work:
func makeThumbnails5(filenames <-chan string) int64 {
sizes := make(chan int64)
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
for f := range filenames {
wg.Add(1)
// worker
go func(f string) {
defer wg.Done()
thumb, err := thumbnail.ImageFile(f)
if err != nil {
log.Println(err)
return
}
info, _ := os.Stat(thumb)
sizes <- info.Size()
}(f)
}
}()
// closer
go func() {
wg.Wait()
close(sizes)
}()
var total int64
for size := range sizes {
total += size
}
return total
}
The implementation of the main has a similar problem that if the input is huge you're essentially load it all into memory (buffered chan) before passing it on to be processed. Perhaps something like following is better
func main() {
ch := make(chan string)
go func(thumbnails []string) {
defer close(ch)
for _, val := range thumbnails {
ch <- val
}
}(os.Args[1:])
makeThumbnails5(ch)
}

Ensure two commands are running before starting a third one

I have three commands to run, but I'd like to make sure the two first are running before running the third one.
Currently, it does run A and B then C.
I run A and B in goroutines
I communicate their name through chan if there's no stderr
the main functions pushes the names received through chan into a slice
once the slice contains all names of module A and B it starts C
Some context
I'm in the process of learning goroutines and chan as a hobbyist. It's not clear to me how to output exec.Command("foo", "bar").Run() in a reliable way while it's running. It's not clear either how to handle errors received by each process through chan.
The reason why I need A and B to run before C is because A and B are graphql microservices, C needs them to run in order to get their schemas through HTTP and start doing some graphql federation (f.k.a. graphql stitching)
Inconsistencies
With my current approach, I will know if A and B are running only if they print something I guess.
I don't like that each subsequent stdout will hit an if statement, just to know if the process is running.
My error handling is not as clean as I'd like it to be.
Question
How could I have a more reliable way to ensure that A and B are running, event if they don't print anything and that they did not throw errors?
package main
import (
"bufio"
"fmt"
"log"
"os/exec"
"reflect"
"sort"
"strings"
"sync"
)
var wg sync.WaitGroup
var modulesToRun = []string{"micro-post", "micro-hello"}
func main() {
// Send multiple values to chan
// https://stackoverflow.com/a/50857250/9077800
c := make(chan func() (string, error))
go runModule([]string{"go", "run", "micro-post"}, c) // PROCESS A
go runModule([]string{"go", "run", "micro-hello"}, c) // PROCESS B
modulesRunning := []string{}
for {
msg, err := (<-c)()
if err != nil {
log.Fatalln(err)
}
if strings.HasPrefix(msg, "micro-") && err == nil {
modulesRunning = append(modulesRunning, msg)
if CompareUnorderedSlices(modulesToRun, modulesRunning) {
go runModule([]string{"go", "run", "micro-federation"}, c) // PROCESS C
}
}
}
}
func runModule(commandArgs []string, o chan func() (string, error)) {
cmd := exec.Command(commandArgs[0], commandArgs[1], commandArgs[2]+"/main.go")
// Less verbose solution to stream output with io?
// var stdBuffer bytes.Buffer
// mw := io.MultiWriter(os.Stdout, &stdBuffer)
// cmd.Stdout = mw
// cmd.Stderr = mw
c := make(chan struct{})
wg.Add(1)
// Stream command output
// https://stackoverflow.com/a/38870609/9077800
go func(cmd *exec.Cmd, c chan struct{}) {
defer wg.Done()
stdout, err := cmd.StdoutPipe()
if err != nil {
close(o)
panic(err)
}
stderr, err := cmd.StderrPipe()
if err != nil {
close(o)
panic(err)
}
<-c
outScanner := bufio.NewScanner(stdout)
for outScanner.Scan() {
m := outScanner.Text()
fmt.Println(commandArgs[2]+":", m)
o <- (func() (string, error) { return commandArgs[2], nil })
}
errScanner := bufio.NewScanner(stderr)
for errScanner.Scan() {
m := errScanner.Text()
fmt.Println(commandArgs[2]+":", m)
o <- (func() (string, error) { return "bad", nil })
}
}(cmd, c)
c <- struct{}{}
cmd.Start()
wg.Wait()
close(o)
}
// CompareUnorderedSlices orders slices before comparing them
func CompareUnorderedSlices(a, b []string) bool {
if len(a) != len(b) {
return false
}
sort.Strings(a)
sort.Strings(b)
return reflect.DeepEqual(a, b)
}
About process management
Starting the process is the action of calling the binary path with its arguments.
It will fail if the bin path is not found, or some malformed arguments syntax is provided.
As a consequence you might start a process with success, but receive an exit error because somehow its execution fails.
Those details are important to figure out if you need only to startup the process to consider the operation as successful or dig further its state and/or output.
In your code it appears you wait for the first line of stderr to be printed to consider it as started, without any consideration to the content being printed.
It resemble more to a kind of sleeping time to ensure the process has initialized.
Consider that starting the binary happens much faster in comparison to the execution of its bootstrap sequence.
About the code, your exit rules are unclear. What is keeping main from exiting ?
In the current code it will exit before C is executed when A and B has started (not anylising other cases)
Your implementation of job concurrency in main is not standard. It is missing the loop to collect results, quit and close(chan).
The chan signature is awkward, i would rather use a struct {Module string, Err error}
The runModule function is buggy. It might close(o) while another routine might attempt to write it. If starts fails, you are not returning any error signal.
A somewhat solution might look like this, consider it as being opinniated and depending the binary run other strategies can/should be implemented to detect error over the standard FDs.
package main
import (
"bufio"
"fmt"
"log"
"os"
"os/exec"
"strings"
"sync"
"time"
)
type cmd struct {
Module string
Cmd string
Args []string
Err error
}
func main() {
torun := []cmd{
cmd{
Module: "A",
Cmd: "ping",
Args: []string{"8.8.8.8"},
},
cmd{
Module: "B",
Cmd: "ping",
// Args: []string{"8.8.8.8.9"},
Args: []string{"8.8.8.8"},
},
}
var wg sync.WaitGroup // use a waitgroup to ensure all concurrent jobs are done
wg.Add(len(torun))
out := make(chan cmd) // a channel to output cmd status
go func() {
wg.Wait() //wait for the group to finish
close(out) // then close the signal channel
}()
// start the commands
for _, c := range torun {
// go runCmd(c, out, &wg)
go runCmdAndWaitForSomeOutput(c, out, &wg)
}
// loop over the chan to collect errors
// it ends when wg.Wait unfreeze and closes out
for c := range out {
if c.Err != nil {
log.Fatalf("%v %v has failed with %v", c.Cmd, c.Args, c.Err)
}
}
// here all commands started you can proceed further to run the last command
fmt.Println("all done")
os.Exit(0)
}
func runCmd(o cmd, out chan cmd, wg *sync.WaitGroup) {
defer wg.Done()
cmd := exec.Command(o.Cmd, o.Args...)
if err := cmd.Start(); err != nil {
o.Err = err // save err
out <- o // signal completion error
return // return to unfreeze the waitgroup wg
}
go cmd.Wait() // dont wait for command completion,
// consider its done once the program started with success.
// out <- o // useless as main look ups only for error
}
func runCmdAndWaitForSomeOutput(o cmd, out chan cmd, wg *sync.WaitGroup) {
defer wg.Done()
cmd := exec.Command(o.Cmd, o.Args...)
stdout, err := cmd.StdoutPipe()
if err != nil {
o.Err = err // save err
out <- o // signal completion
return // return to unfreeze the waitgroup wg
}
stderr, err := cmd.StderrPipe()
if err != nil {
o.Err = err
out <- o
return
}
if err := cmd.Start(); err != nil {
o.Err = err
out <- o
return
}
go cmd.Wait() // dont wait for command completion
// build a concurrent fd's scanner
outScan := make(chan error) // to signal errors detected on the fd
var wg2 sync.WaitGroup
wg2.Add(2) // the number of fds being watched
go func() {
defer wg2.Done()
sc := bufio.NewScanner(stdout)
for sc.Scan() {
line := sc.Text()
if strings.Contains(line, "icmp_seq") { // the OK marker
return // quit asap to unfreeze wg2
} else if strings.Contains(line, "not known") { // the nOK marker, if any...
outScan <- fmt.Errorf("%v", line)
return // quit to unfreeze wg2
}
}
}()
go func() {
defer wg2.Done()
sc := bufio.NewScanner(stderr)
for sc.Scan() {
line := sc.Text()
if strings.Contains(line, "icmp_seq") { // the OK marker
return // quit asap to unfreeze wg2
} else if strings.Contains(line, "not known") { // the nOK marker, if any...
outScan <- fmt.Errorf("%v", line) // signal error
return // quit to unfreeze wg2
}
}
}()
go func() {
wg2.Wait() // consider that if the program does not output anything,
// or never prints ok/nok, this will block forever
close(outScan) // close the chan so the next loop is finite
}()
// - simple timeout less loop
// for err := range outScan {
// if err != nil {
// o.Err = err // save the execution error
// out <- o // signal the cmd
// return // qui to unfreeze the wait group wg
// }
// }
// - more complex version with timeout
timeout := time.After(time.Second * 3)
for {
select {
case err, ok := <-outScan:
if !ok { // if !ok, outScan is closed and we should quit the loop
return
}
if err != nil {
o.Err = err // save the execution error
out <- o // signal the cmd
return // quit to unfreeze the wait group wg
}
case <-timeout:
o.Err = fmt.Errorf("timed out...%v", timeout) // save the execution error
out <- o // signal the cmd
return // quit to unfreeze the wait group wg
}
}
// exit and unfreeze the wait group wg
}

How to assign the values to struct while go routines are running?

I'm using goroutines in my project and I want to to assign the values to the struct fields but I don't know that how I will assign the values get by using mongodb quires to the struct fields I'm showing my struct and the query too.
type AppLoadNew struct{
StripeTestKey string `json:"stripe_test_key" bson:"stripe_test_key,omitempty"`
Locations []Locations `json:"location" bson:"location,omitempty"`
}
type Locations struct{
Id int `json:"_id" bson:"_id"`
Location string `json:"location" bson:"location"`
}
func GoRoutine(){
values := AppLoadNew{}
go func() {
data, err := GetStripeTestKey(bson.M{"is_default": true})
if err == nil {
values.StripeTestKey := data.TestStripePublishKey
}
}()
go func() {
location, err := GetFormLocation(bson.M{"is_default": true})
if err == nil {
values.Locations := location
}
}()
fmt.Println(values) // Here it will nothing
// empty
}
Can you please help me that I will assign all the values to the AppLoadNew struct.
In Go no value is safe for concurrent read and write (from multiple goroutines). You must synchronize access.
Reading and writing variables from multiple goroutines can be protected using sync.Mutex or sync.RWMutex, but in your case there is something else involved: you should wait for the 2 launched goroutines to complete. For that, the go-to solution is sync.WaitGroup.
And since the 2 goroutines write 2 different fields of a struct (which act as 2 distinct variables), they don't have to be synchronized to each other (see more on this here: Can I concurrently write different slice elements). Which means using a sync.WaitGroup is sufficient.
This is how you can make it safe and correct:
func GoRoutine() {
values := AppLoadNew{}
wg := &sync.WaitGroup{}
wg.Add(1)
go func() {
defer wg.Done()
data, err := GetStripeTestKey(bson.M{"is_default": true})
if err == nil {
values.StripeTestKey = data.StripeTestKey
}
}()
wg.Add(1)
go func() {
defer wg.Done()
location, err := GetFormLocation(bson.M{"is_default": true})
if err == nil {
values.Locations = location
}
}()
wg.Wait()
fmt.Println(values)
}
See a (slightly modified) working example on the Go Playground.
See related / similar questions:
Reading values from a different thread
golang struct concurrent read and write without Lock is also running ok?
How to make a variable thread-safe
You can use sync package with WaitGroup, here is an example:
package main
import (
"fmt"
"sync"
"time"
)
type Foo struct {
One string
Two string
}
func main() {
f := Foo{}
var wg sync.WaitGroup
wg.Add(1)
go func() {
defer wg.Done()
// Perform long calculations
<-time.After(time.Second * 1)
f.One = "foo"
}()
wg.Add(1)
go func() {
defer wg.Done()
// Perform long calculations
<-time.After(time.Second * 2)
f.Two = "bar"
}()
fmt.Printf("Before %+v\n", f)
wg.Wait()
fmt.Printf("After %+v\n", f)
}
The output:
Before {One: Two:}
After {One:foo Two:bar}

What's the idiomatic solution to embarassingly parallel tasks in Go?

I'm currently staring at a beefed up version of the following code:
func embarrassing(data []string) []string {
resultChan := make(chan string)
var waitGroup sync.WaitGroup
for _, item := range data {
waitGroup.Add(1)
go func(item string) {
defer waitGroup.Done()
resultChan <- doWork(item)
}(item)
}
go func() {
waitGroup.Wait()
close(resultChan)
}()
var results []string
for result := range resultChan {
results = append(results, result)
}
return results
}
This is just blowing my mind. All this is doing can be expressed in other languages as
results = parallelMap(data, doWork)
Even if it can't be done quite this easily in Go, isn't there still a better way than the above?
If you need all the results, you don't need the channel (and the extra goroutine to close it) to communicate the results, you can write directly into the results slice:
func cleaner(data []string) []string {
results := make([]string, len(data))
wg := &sync.WaitGroup{}
wg.Add(len(data))
for i, item := range data {
go func(i int, item string) {
defer wg.Done()
results[i] = doWork(item)
}(i, item)
}
wg.Wait()
return results
}
This is possible because slice elements act as distinct variables, and thus can be written individually without synchronization. For details, see Can I concurrently write different slice elements. You also get the results in the same order as your input for free.
Anoter variation: if doWork() would not return the result but get the address where the result should be "placed", and additionally the sync.WaitGroup to signal completion, that doWork() function could be executed "directly" as a new goroutine.
We can create a reusable wrapper for doWork():
func doWork2(item string, result *string, wg *sync.WaitGroup) {
defer wg.Done()
*result = doWork(item)
}
If you have the processing logic in such format, this is how it can be executed concurrently:
func cleanest(data []string) []string {
results := make([]string, len(data))
wg := &sync.WaitGroup{}
wg.Add(len(data))
for i, item := range data {
go doWork2(item, &results[i], wg)
}
wg.Wait()
return results
}
Yet another variation could be to pass a channel to doWork() on which it is supposed to deliver the result. This solution doesn't even require a sync.Waitgroup, as we know how many elements we want to receive from the channel:
func cleanest2(data []string) []string {
ch := make(chan string)
for _, item := range data {
go doWork3(item, ch)
}
results := make([]string, len(data))
for i := range results {
results[i] = <-ch
}
return results
}
func doWork3(item string, res chan<- string) {
res <- "done:" + item
}
"Weakness" of this last solution is that it may collect the result "out-of-order" (which may or may not be a problem). This approach can be improved to retain order by letting doWork() receive and return the index of the item. For details and examples, see How to collect values from N goroutines executed in a specific order?
You can also use reflection to achieve something similar.
In this example it distribute the handler function over 4 goroutines and returns the results in a new instance of the given source slice type.
package main
import (
"fmt"
"reflect"
"strings"
"sync"
)
func parralelMap(some interface{}, handle interface{}) interface{} {
rSlice := reflect.ValueOf(some)
rFn := reflect.ValueOf(handle)
dChan := make(chan reflect.Value, 4)
rChan := make(chan []reflect.Value, 4)
var waitGroup sync.WaitGroup
for i := 0; i < 4; i++ {
waitGroup.Add(1)
go func() {
defer waitGroup.Done()
for v := range dChan {
rChan <- rFn.Call([]reflect.Value{v})
}
}()
}
nSlice := reflect.MakeSlice(rSlice.Type(), rSlice.Len(), rSlice.Cap())
for i := 0; i < rSlice.Len(); i++ {
dChan <- rSlice.Index(i)
}
close(dChan)
go func() {
waitGroup.Wait()
close(rChan)
}()
i := 0
for v := range rChan {
nSlice.Index(i).Set(v[0])
i++
}
return nSlice.Interface()
}
func main() {
fmt.Println(
parralelMap([]string{"what", "ever"}, strings.ToUpper),
)
}
Test here https://play.golang.org/p/iUPHqswx8iS

Golang: Processing 5 huge files concurrently

I have 5 huge (4 million rows each) logfiles that I process in Perl currently and I thought I may try to implement the same in Go and its concurrent features. So, being very inexperienced in Go, I was thinking of doing as below. Any comments on the approach will be greatly appreciated.
Some rough pseudocode:
var wg1 sync.WaitGroup
var wg2 sync.WaitGroup
func processRow (r Row) {
wg2.Add(1)
defer wg2.Done()
res = <process r>
return res
}
func processFile(f File) {
wg1.Add(1)
open(newfile File)
defer wg1.Done()
line = <row from f>
result = go processRow(line)
newFile.Println(result) // Write new processed line to newFile
wg2.Wait()
newFile.Close()
}
func main() {
for each f logfile {
go processFile(f)
}
wg1.Wait()
}
So, idea is that I process these 5 files concurrently and then all rows of each file will in turn also be processed concurrently.
Will that work?
You should definitely use channels to manage your processed rows. Alternatively you could also write another goroutine to handle your output.
var numGoWriters = 10
func processRow(r Row, ch chan<- string) {
res := process(r)
ch <- res
}
func writeRow(f File, ch <-chan string) {
w := bufio.NewWriter(f)
for s := range ch {
_, err := w.WriteString(s + "\n")
}
func processFile(f File) {
outFile, err := os.Create("/path/to/file.out")
if err != nil {
// handle it
}
defer outFile.Close()
var wg sync.WaitGroup
ch := make(chan string, 10) // play with this number for performance
defer close(ch) // once we're done processing rows, we close the channel
// so our worker threads exit
fScanner := bufio.NewScanner(f)
for fScanner.Scan() {
wg.Add(1)
go func() {
processRow(fScanner.Text(), ch)
wg.Done()
}()
}
for i := 0; i < numGoWriters; i++ {
go writeRow(outFile, ch)
}
wg.Wait()
}
Here we have processRow doing all the processing (I assumed to string), writeRow doing all the out I/O, and processFile tying each file together. Then all main has to do is hand off the files, spawn the goroutines, et voila.
func main() {
var wg sync.WaitGroup
filenames := [...]string{"here", "are", "some", "log", "paths"}
for fname := range filenames {
inFile, err := os.Open(fname)
if err != nil {
// handle it
}
defer inFile.Close()
wg.Add(1)
go processFile(inFile)
}
wg.Wait()

Resources