i have written the following code in order to run until someone exit the program manually.
it does is
----- check if the exists every 1 second
----- if available then read the file and print the file content line by line
for this i have first call a function from the main
and then i call a waitgroup and call a function again from there to do the aforementioned tasks.
please check if i have written the source code correctly as im a newbi on GO
plus this only runs once and stop... i want to it keep alive and see if the file exsists
please help me
package main
import (
"encoding/csv"
"fmt"
"io"
"log"
"os"
"sync"
"time"
)
func main() {
mainfunction()
}
//------------------------------------------------------------------
func mainfunction() {
var wg sync.WaitGroup
wg.Add(1)
go filecheck(&wg)
wg.Wait()
fmt.Printf("Program finished \n")
}
func filecheck(wg *sync.WaitGroup) {
for range time.Tick(time.Second * 1) {
fmt.Println("Foo")
var wgi sync.WaitGroup
wgi.Add(1)
oldName := "test.csv"
newName := "testi.csv"
if _, err := os.Stat(oldName); os.IsNotExist(err) {
fmt.Printf("Path does not exsist \n")
} else {
os.Rename(oldName, newName)
if err != nil {
log.Fatal(err)
}
looping(newName, &wgi)
}
fmt.Printf("Test complete \n")
wgi.Wait()
wg.Done()
time.Sleep(time.Second * 5)
}
}
func looping(newName string, wgi *sync.WaitGroup) {
file, _ := os.Open(newName)
r := csv.NewReader(file)
for {
record, err := r.Read()
if err == io.EOF {
break
}
if err != nil {
log.Fatal(err)
}
var Date = record[0]
var Agent = record[1]
var Srcip = record[2]
var Level = record[3]
fmt.Printf("Data: %s Agent: %s Srcip: %s Level: %s\n", Date, Agent, Srcip, Level)
}
fmt.Printf("Test complete 2 \n")
wgi.Done()
fmt.Printf("for ended")
}
The short answer is that you have this in the loop:
wg.Done()
Which makes the main goroutine proceed to exit as soon as the file is read once.
The longer answer is that you're not using wait groups correctly here, IMHO. For example there's absolutely no point in passing a WaitGroup into looping.
It's not clear what your code is trying to accomplish - you certainly don't need any goroutines to just perform the task you've specified - it can all be gone with no concurrency and thus simpler code.
Related
I'm writing a program that reads a list of order numbers in a file called orders.csv and compares it with the other csv files that are present in the folder.
The problem is that it goes into deadlock even using waitgroup and I don't know why.
For some reason stackoverflow says that my post is mostly code, so I have to add this line, because the whole code is necessary if someone wants to help me debug this problem I'm having.
package main
import (
"bufio"
"fmt"
"log"
"os"
"path/filepath"
"strings"
"sync"
)
type Files struct {
filenames []string
}
type Orders struct {
ID []string
}
var ordersFilename string = "orders.csv"
func main() {
var (
ordersFile *os.File
files Files
orders Orders
err error
)
mu := new(sync.Mutex)
wg := &sync.WaitGroup{}
wg.Add(1)
if ordersFile, err = os.Open(ordersFilename); err != nil {
log.Fatalln("Could not open file: " + ordersFilename)
}
orders = getOrderIDs(ordersFile)
files.filenames = getCSVsFromCurrentDir()
var filenamesSize = len(files.filenames)
var ch = make(chan map[string][]string, filenamesSize)
var done = make(chan bool)
for i, filename := range files.filenames {
go func(currentFilename string, ch chan<- map[string][]string, i int, orders Orders, wg *sync.WaitGroup, filenamesSize *int, mu *sync.Mutex, done chan<- bool) {
wg.Add(1)
defer wg.Done()
checkFile(currentFilename, orders, ch)
mu.Lock()
*filenamesSize--
mu.Unlock()
if i == *filenamesSize {
done <- true
close(done)
}
}(filename, ch, i, orders, wg, &filenamesSize, mu, done)
}
select {
case str := <-ch:
fmt.Printf("%+v\n", str)
case <-done:
wg.Done()
break
}
wg.Wait()
close(ch)
}
// getCSVsFromCurrentDir returns a string slice
// with the filenames of csv files inside the
// current directory that are not "orders.csv"
func getCSVsFromCurrentDir() []string {
var filenames []string
err := filepath.Walk(".", func(path string, info os.FileInfo, err error) error {
if path != "." && strings.HasSuffix(path, ".csv") && path != ordersFilename {
filenames = append(filenames, path)
}
return nil
})
if err != nil {
log.Fatalln("Could not read file names in current dir")
}
return filenames
}
// getOrderIDs returns an Orders struct filled
// with order IDs retrieved from the file
func getOrderIDs(file *os.File) Orders {
var (
orders Orders
err error
fileContent string
)
reader := bufio.NewReader(file)
if fileContent, err = readLine(reader); err != nil {
log.Fatalln("Could not read file: " + ordersFilename)
}
for err == nil {
orders.ID = append(orders.ID, fileContent)
fileContent, err = readLine(reader)
}
return orders
}
func checkFile(filename string, orders Orders, ch chan<- map[string][]string) {
var (
err error
file *os.File
fileContent string
orderFilesMap map[string][]string
counter int
)
orderFilesMap = make(map[string][]string)
if file, err = os.Open(filename); err != nil {
log.Fatalln("Could not read file: " + filename)
}
reader := bufio.NewReader(file)
if fileContent, err = readLine(reader); err != nil {
log.Fatalln("Could not read file: " + filename)
}
for err == nil {
if containedInSlice(fileContent, orders.ID) && !containedInSlice(fileContent, orderFilesMap[filename]) {
orderFilesMap[filename] = append(orderFilesMap[filename], fileContent)
// fmt.Println("Found: ", fileContent, " in ", filename)
} else {
// fmt.Printf("Could not find: '%s' in '%s'\n", fileContent, filename)
}
counter++
fileContent, err = readLine(reader)
}
ch <- orderFilesMap
}
// containedInSlice returns true or false
// based on whether the string is contained
// in the slice
func containedInSlice(str string, slice []string) bool {
for _, ID := range slice {
if ID == str {
return true
}
}
return false
}
// readLine returns a line from the passed reader
func readLine(r *bufio.Reader) (string, error) {
var (
isPrefix bool = true
err error = nil
line, ln []byte
)
for isPrefix && err == nil {
line, isPrefix, err = r.ReadLine()
ln = append(ln, line...)
}
return string(ln), err
}
The first issue is the wg.Add always must be outside of the goroutine(s) it stands for. If it isn't, the
wg.Wait call might be called before the goutine(s) have actually started running (and called wg.Add) and therefore will "think"
that there is nothing to wait for.
The second issue with the code is that there are multiple ways it waits for the routines to be done. There is
the WaitGroup and there is the done channel. Use only one of them. Which one depends also on how the results of the
goroutines are used. Here we come to the next problem.
The third issue is with gathering the results. Currently the code only prints / uses a single result from the goroutines.
Put a for { ... } loop around the select and use return to break out of the loop if the done channel is closed.
(Note that you don't need to send anything on the done channel, closing it is enough.)
Improved Version 0.0.1
So here the first version (including some other "code cleanup") with a done channel used for closing and the WaitGroup removed:
func main() {
ordersFile, err := os.Open(ordersFilename)
if err != nil {
log.Fatalln("Could not open file: " + ordersFilename)
}
orders := getOrderIDs(ordersFile)
files := Files{
filenames: getCSVsFromCurrentDir(),
}
var (
mu = new(sync.Mutex)
filenamesSize = len(files.filenames)
ch = make(chan map[string][]string, filenamesSize)
done = make(chan bool)
)
for i, filename := range files.filenames {
go func(currentFilename string, ch chan<- map[string][]string, i int, orders Orders, filenamesSize *int, mu *sync.Mutex, done chan<- bool) {
checkFile(currentFilename, orders, ch)
mu.Lock()
*filenamesSize--
mu.Unlock()
// TODO: This also accesses filenamesSize, so it also needs to be protected with the mutex:
if i == *filenamesSize {
done <- true
close(done)
}
}(filename, ch, i, orders, &filenamesSize, mu, done)
}
// Note: closing a channel is not really needed, so you can omit this:
defer close(ch)
for {
select {
case str := <-ch:
fmt.Printf("%+v\n", str)
case <-done:
return
}
}
}
Improved Version 0.0.2
In your case we have some advantage however. We know exactly how many goroutines we started and therefore also how
many results we expect. (Of course if each goroutine returns a result which currently this code does.) That gives
us another option as we can collect the results with another for loop having the same amount of iterations:
func main() {
ordersFile, err := os.Open(ordersFilename)
if err != nil {
log.Fatalln("Could not open file: " + ordersFilename)
}
orders := getOrderIDs(ordersFile)
files := Files{
filenames: getCSVsFromCurrentDir(),
}
var (
// Note: a buffered channel helps speed things up. The size does not need to match the size of the items that will
// be passed through the channel. A fixed, small size is perfect here.
ch = make(chan map[string][]string, 5)
)
for _, filename := range files.filenames {
go func(filename string) {
// orders and channel are not variables of the loop and can be used without copying
checkFile(filename, orders, ch)
}(filename)
}
for range files.filenames {
str := <-ch
fmt.Printf("%+v\n", str)
}
}
A lot simpler, isn't it? Hope that helps!
There is a lot wrong with this code.
You're using the WaitGroup wrong. Add has to be called in the main goroutine, else there is a chance that Wait is called before all Add calls complete.
There's an extraneous Add(1) call right after initializing the WaitGroup that isn't matched by a Done() call, so Wait will never return (assuming the point above is fixed).
You're using both a WaitGroup and a done channel to signal completion. This is redundant at best.
You're reading filenamesSize while not holding the lock (in the if i == *filenamesSize statement). This is a race condition.
The i == *filenamesSize condition makes no sense in the first place. Goroutines execute in an arbitrary order, so you can't be sure that the goroutine with i == 0 is the last one to decrement filenamesSize
This can all be simplified by getting rid of most if the synchronization primitives and simply closing the ch channel when all goroutines are done:
func main() {
ch := make(chan map[string][]string)
var wg WaitGroup
for _, filename := range getCSVsFromCurrentDir() {
filename := filename // capture loop var
wg.Add(1)
go func() {
checkFile(filename, orders, ch)
wg.Done()
}()
}
go func() {
wg.Wait() // after all goroutines are done...
close(ch) // let range loop below exit
}()
for str := range ch {
// ...
}
}
not an answer, but some comments that does not fit the comment box.
In this part of the code
func main() {
var (
ordersFile *os.File
files Files
orders Orders
err error
)
mu := new(sync.Mutex)
wg := &sync.WaitGroup{}
wg.Add(1)
The last statement is a call to wg.Add that appears dangling. By that i mean we can hardly understand what will trigger the required wg.Done counter part. This is a mistake to call for wg.Add without a wg.Done, this is prone to errors to not write them in such way we can not immediately find them in pair.
In that part of the code, it is clearly wrong
go func(currentFilename string, ch chan<- map[string][]string, i int, orders Orders, wg *sync.WaitGroup, filenamesSize *int, mu *sync.Mutex, done chan<- bool) {
wg.Add(1)
defer wg.Done()
Consider that by the time the routine is executed, and that you added 1 to the waitgroup, the parent routine continues to execute. See this example: https://play.golang.org/p/N9Chaqkv4bd
The main routine does not wait for the waitgroup because it does not have time to increment.
There is more to say but i find it hard to understand the purpose of your code so i am not sure how to help you further without basically rewrite it.
I have this simple script to attempt to traverse the file system and read files line-by-line to match lines on a regex:
package main
import (
"bufio"
"fmt"
"io/ioutil"
"log"
"os"
"regexp"
"sync"
)
type FileWithLine struct{
Line int
Path string
}
var set = map[string]FileWithLine{}
var rgx = regexp.MustCompile("ErrId\\s*:\\s*\"[[:alnum:]]+\"");
func traverseDir(d string, wg *sync.WaitGroup){
fmt.Println("traversing dir:", d)
if d == ".git"{
return
}
wg.Add(1)
go func(wg *sync.WaitGroup){
defer wg.Done()
files, err := ioutil.ReadDir(d)
if err != nil {
log.Fatal(err)
}
for _, f := range files {
fmt.Println("we see file:", f.Name())
if f.IsDir() {
traverseDir(f.Name(), wg)
return
}
file, err := os.Open(f.Name())
if err != nil {
log.Fatalf("failed opening file: %s", err)
}
scanner := bufio.NewScanner(file)
scanner.Split(bufio.ScanLines)
for scanner.Scan() {
var line = scanner.Text()
if rgx.MatchString(line) {
fmt.Println("line matches:", line);
}
}
file.Close()
}
}(wg)
}
func main() {
var wg sync.WaitGroup
traverseDir(".", &wg)
fmt.Println("Main: Waiting for workers to finish")
wg.Wait()
fmt.Println("Main: Completed")
}
the problem is that it's exiting before it reads all the files, I get this output:
traversing dir: .
Main: Waiting for workers to finish
we see file: .git
traversing dir: .git
Main: Completed
but there are more files in the current directory, than just the .git folder. It just so happens that the .git folder is the first item in the current working dir and it exits right after that. Anyone know why my program is exciting early?
It is stopping processing because of these lines:
if f.IsDir() {
traverseDir(f.Name(), wg)
return
}
When it sees a directory, it goes in it and immediately returns, without processing the remaining files in the current directory. And when the first seen directory is ".git", since you handle it as an exception, the nested traverseDir also returns.
I would like to understand why this case deadlock and why it's not in the other case.
If I close the channel inside the goroutine, it works fine, but if I close it after the WaitGroup.Wait() it cause a deadlock.
package main
import (
"fmt"
"io/ioutil"
"os"
"sync"
)
var (
wg = sync.WaitGroup{}
links = make(chan string)
)
func rec_readdir(depth int, path string) {
files, _ := ioutil.ReadDir(path)
for _, f := range files {
if symlink, err := os.Readlink(path + "/" + f.Name()); err == nil {
links <- path + "/" + symlink
}
rec_readdir(depth+1, path+"/"+f.Name())
}
if depth == 0 {
wg.Done()
// close(links) // if close here ok
}
}
func main() {
wg.Add(1)
go rec_readdir(0, ".")
for slink := range links {
fmt.Println(slink)
}
wg.Wait()
close(links) // if close here deadlock
}
https://play.golang.org/p/Ntl_zsV5nwO
for slink := range links will continue looping until the channel is closed. So you obviously can't close after that loop. When you do, you get a deadlock, as you have observed.
I recently started learning go and I am really impressed with all the features. I been playing with go routines and term-ui and facing some trouble. I am trying to exit this code from console after I run it but it just doesn't respond. If I run it without go-routine it does respond to my q key press event.
Any help is appreciated.
My code
package main
import (
"fmt"
"github.com/gizak/termui"
"time"
"strconv"
)
func getData(ch chan string) {
i := 0
for {
ch <- strconv.Itoa(i)
i++
time.Sleep(time.Second)
if i == 20 {
break
}
}
}
func Display(ch chan string) {
err := termui.Init()
if err != nil {
panic(err)
}
defer termui.Close()
termui.Handle("/sys/kbd/q", func(termui.Event) {
fmt.Println("q captured")
termui.Close()
termui.StopLoop()
})
for elem := range ch {
par := termui.NewPar(elem)
par.Height = 5
par.Width = 37
par.Y = 4
par.BorderLabel = "term ui example with chan"
par.BorderFg = termui.ColorYellow
termui.Render(par)
}
}
func main() {
ch := make(chan string)
go getData(ch)
Display(ch)
}
This is possibly the answer you are looking for. First off, you aren't using termui correctly. You need to call it's Loop function to start the Event loop so that it can actually start listening for the q key. Loop is called last because it essentially takes control of the main goroutine from then on until StopLoop is called and it quits.
In order to stop the goroutines, it is common to have a "stop" channel. Usually it is a chan struct{} to save memory because you don't ever have to put anything in it. Wherever you want the goroutine to possibly stop and shutoff (or do something else perhaps), you use a select statement with the channels you are using. This select is ordered, so it will take from them in order unless they block, in which case it tries the next one, so the stop channel usually goes first. The stop channel normally blocks, but to get it to take this path, simply close()ing it will cause this path to be chosen in the select. So we close() it in the q keyboard handler.
package main
import (
"fmt"
"github.com/gizak/termui"
"strconv"
"time"
)
func getData(ch chan string, stop chan struct{}) {
i := 0
for {
select {
case <-stop:
break
case ch <- strconv.Itoa(i):
}
i++
time.Sleep(time.Second)
if i == 20 {
break
}
}
}
func Display(ch chan string, stop chan struct{}) {
for {
var elem string
select {
case <-stop:
break
case elem = <-ch:
}
par := termui.NewPar(elem)
par.Height = 5
par.Width = 37
par.Y = 4
par.BorderLabel = "term ui example with chan"
par.BorderFg = termui.ColorYellow
termui.Render(par)
}
}
func main() {
ch := make(chan string)
stop := make(chan struct{})
err := termui.Init()
if err != nil {
panic(err)
}
defer termui.Close()
termui.Handle("/sys/kbd/q", func(termui.Event) {
fmt.Println("q captured")
close(stop)
termui.StopLoop()
})
go getData(ch, stop)
go Display(ch, stop)
termui.Loop()
}
Code below works fine with hard coded JSON data however doesn't work when I read JSON data from a file. I'm getting fatal error: all goroutines are asleep - deadlock error when using sync.WaitGroup.
WORKING EXAMPLE WITH HARD-CODED JSON DATA:
package main
import (
"bytes"
"fmt"
"os/exec"
"time"
)
func connect(host string) {
cmd := exec.Command("ssh", host, "uptime")
var out bytes.Buffer
cmd.Stdout = &out
err := cmd.Run()
if err != nil {
fmt.Println(err)
}
fmt.Printf("%s: %q\n", host, out.String())
time.Sleep(time.Second * 2)
fmt.Printf("%s: DONE\n", host)
}
func listener(c chan string) {
for {
host := <-c
go connect(host)
}
}
func main() {
hosts := [2]string{"user1#111.79.154.111", "user2#111.79.190.222"}
var c chan string = make(chan string)
go listener(c)
for i := 0; i < len(hosts); i++ {
c <- hosts[i]
}
var input string
fmt.Scanln(&input)
}
OUTPUT:
user#user-VirtualBox:~/go$ go run channel.go
user1#111.79.154.111: " 09:46:40 up 86 days, 18:16, 0 users, load average: 5"
user2#111.79.190.222: " 09:46:40 up 86 days, 17:27, 1 user, load average: 9"
user1#111.79.154.111: DONE
user2#111.79.190.222: DONE
NOT WORKING - EXAMPLE WITH READING JSON DATA FILE:
package main
import (
"bytes"
"fmt"
"os/exec"
"time"
"encoding/json"
"os"
"sync"
)
func connect(host string) {
cmd := exec.Command("ssh", host, "uptime")
var out bytes.Buffer
cmd.Stdout = &out
err := cmd.Run()
if err != nil {
fmt.Println(err)
}
fmt.Printf("%s: %q\n", host, out.String())
time.Sleep(time.Second * 2)
fmt.Printf("%s: DONE\n", host)
}
func listener(c chan string) {
for {
host := <-c
go connect(host)
}
}
type Content struct {
Username string `json:"username"`
Ip string `json:"ip"`
}
func main() {
var wg sync.WaitGroup
var source []Content
var hosts []string
data := json.NewDecoder(os.Stdin)
data.Decode(&source)
for _, value := range source {
hosts = append(hosts, value.Username + "#" + value.Ip)
}
var c chan string = make(chan string)
go listener(c)
for i := 0; i < len(hosts); i++ {
wg.Add(1)
c <- hosts[i]
defer wg.Done()
}
var input string
fmt.Scanln(&input)
wg.Wait()
}
OUTPUT
user#user-VirtualBox:~/go$ go run deploy.go < hosts.txt
user1#111.79.154.111: " 09:46:40 up 86 days, 18:16, 0 users, load average: 5"
user2#111.79.190.222: " 09:46:40 up 86 days, 17:27, 1 user, load average: 9"
user1#111.79.154.111 : DONE
user2#111.79.190.222: DONE
fatal error: all goroutines are asleep - deadlock!
goroutine 1 [semacquire]:
sync.runtime_Semacquire(0xc210000068)
/usr/lib/go/src/pkg/runtime/sema.goc:199 +0x30
sync.(*WaitGroup).Wait(0xc210047020)
/usr/lib/go/src/pkg/sync/waitgroup.go:127 +0x14b
main.main()
/home/user/go/deploy.go:64 +0x45a
goroutine 3 [chan receive]:
main.listener(0xc210038060)
/home/user/go/deploy.go:28 +0x30
created by main.main
/home/user/go/deploy.go:53 +0x30b
exit status 2
user#user-VirtualBox:~/go$
HOSTS.TXT
[
{
"username":"user1",
"ip":"111.79.154.111"
},
{
"username":"user2",
"ip":"111.79.190.222"
}
]
Go program ends when the main function ends.
From the language specification
Program execution begins by initializing the main package and then invoking the function main. When that function invocation returns, the program exits. It does not wait for other (non-main) goroutines to complete.
Therefore, you need to wait for your goroutines to finish. The common solution for this is to use sync.WaitGroup object.
The simplest possible code to synchronize goroutine:
package main
import "fmt"
import "sync"
var wg sync.WaitGroup // 1
func routine() {
defer wg.Done() // 3
fmt.Println("routine finished")
}
func main() {
wg.Add(1) // 2
go routine() // *
wg.Wait() // 4
fmt.Println("main finished")
}
And for synchronizing multiple goroutines
package main
import "fmt"
import "sync"
var wg sync.WaitGroup // 1
func routine(i int) {
defer wg.Done() // 3
fmt.Printf("routine %v finished\n", i)
}
func main() {
for i := 0; i < 10; i++ {
wg.Add(1) // 2
go routine(i) // *
}
wg.Wait() // 4
fmt.Println("main finished")
}
WaitGroup usage in order of execution.
Declaration of global variable. Making it global is the easiest way to make it visible to all functions and methods.
Increasing the counter. This must be done in main goroutine because there is no guarantee that newly started goroutine will execute before 4 due to memory model guarantees.
Decreasing the counter. This must be done at the exit of goroutine. Using deferred call, we make sure that it will be called whenever function ends no matter but no matter how it ends.
Waiting for the counter to reach 0. This must be done in main goroutine to prevent program exit.
* The actual parameters are evaluated before starting new gouroutine. Thus it is needed to evaluate them explicitly before wg.Add(1) so the possibly panicking code would not leave increased counter.
Use
param := f(x)
wg.Add(1)
go g(param)
instead of
wg.Add(1)
go g(f(x))
Thanks for the very nice and detailed explanation Grzegorz Żur.
One thing that I want to point it out that typically the func that needs to be threaded wont be in main(), so we would have something like this:
package main
import (
"bufio"
"fmt"
"io"
"io/ioutil"
"math/rand"
"os"
"reflect"
"regexp"
"strings"
"sync"
"time"
)
var wg sync.WaitGroup // VERY IMP to declare this globally, other wise one //would hit "fatal error: all goroutines are asleep - deadlock!"
func doSomething(arg1 arg1Type) {
// cured cancer
}
func main() {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
randTime := r.Intn(10)
wg.Add(1)
go doSomething(randTime)
wg.Wait()
fmt.Println("Waiting for all threads to finish")
}
The thing that I want to point it out is that global declaration of wg is very crucial for all threads to finish before main()
try this code snippest
package main
import (
"bytes"
"fmt"
"os/exec"
"time"
"sync"
)
func connect(host string, wg *sync.WaitGroup) {
defer wg.Done()
cmd := exec.Command("ssh", host, "uptime")
var out bytes.Buffer
cmd.Stdout = &out
err := cmd.Run()
if err != nil {
fmt.Println(err)
}
fmt.Printf("%s: %q\n", host, out.String())
time.Sleep(time.Second * 2)
fmt.Printf("%s: DONE\n", host)
}
func listener(c chan string，wg *sync.WaitGroup) {
for {
host，ok := <-c
// check channel is closed or not
if !ok{
break
}
go connect(host)
}
}
func main() {
var wg sync.WaitGroup
hosts := [2]string{"user1#111.79.154.111", "user2#111.79.190.222"}
var c chan string = make(chan string)
go listener(c)
for i := 0; i < len(hosts); i++ {
wg.Add(1)
c <- hosts[i]
}
close(c)
var input string
fmt.Scanln(&input)
wg.Wait()
}