I am trying to achieve some sort of multi-thread processing over here.
func (m *Map) Parse(mapData Node) error {
wg := &sync.WaitGroup{}
for _, node := range mapData.child {
wg.Add(1)
go parseChild(node, m, wg)
}
wg.Wait()
close(errors)
return nil
}
func parseChild(node Node, m *Map, wg *sync.WaitGroup) {
defer wg.Done()
var nodeType uint8
if err := binary.Read(node.data, binary.LittleEndian, &nodeType); err != nil {
errors <- err
}
if nodeType == OTBMNodeTowns {
for _, town := range node.child {
var nodeType uint8
if err := binary.Read(town.data, binary.LittleEndian, &nodeType); err != nil {
errors <- err
return
}
if nodeType != OTBMNodeTown {
errors <- fmt.Errorf("Parsing map towns: expected %v got %v", OTBMNodeTown, nodeType)
return
}
currentTown := Town{}
if err := binary.Read(town.data, binary.LittleEndian, ¤tTown.ID); err != nil {
errors <- err
return
} else if currentTown.Name, err = town.ReadString(); err != nil {
errors <- err
return
} else if currentTown.TemplePosition, err = town.ReadPosition(); err != nil {
errors <- err
return
}
m.Towns = append(m.Towns, currentTown)
errors <- fmt.Errorf("This should be called: %v, nodeType)
return
}
}
}
But my goroutine never sends anything to the errors channel. Seems to be that the main thread is not waiting for the goroutines to even finish
I have no idea what I am missing here. Im waiting for all routines to finish using wg.Wait but doesnt seem to be working as I think it should
And yes. the slice is populated with atleast 3 results. This is the errrors channel
var (
errors = make(chan error, 0)
)
func init() {
go errChannel()
}
func errChannel() {
for {
select {
case err := <-errors:
log.Println(err)
}
}
}
Related
I have two functions that I cannot change (see first() and second() below). They are returning some data and errors (the output data is different, but in the examples below I use (string, error) for simplicity)
I would like to run them in separate goroutines - my approach:
package main
import (
"fmt"
"os"
)
func first(name string) (string, error) {
if name == "" {
return "", fmt.Errorf("empty name is not allowed")
}
fmt.Println("processing first")
return fmt.Sprintf("First hello %s", name), nil
}
func second(name string) (string, error) {
if name == "" {
return "", fmt.Errorf("empty name is not allowed")
}
fmt.Println("processing second")
return fmt.Sprintf("Second hello %s", name), nil
}
func main() {
firstCh := make(chan string)
secondCh := make(chan string)
go func() {
defer close(firstCh)
res, err := first("one")
if err != nil {
fmt.Printf("Failed to run first: %v\n", err)
}
firstCh <- res
}()
go func() {
defer close(secondCh)
res, err := second("two")
if err != nil {
fmt.Printf("Failed to run second: %v\n", err)
}
secondCh <- res
}()
resultsOne := <-firstCh
resultsTwo := <-secondCh
// It's important for my app to do error checking and stop if errors exist.
if resultsOne == "" || resultsTwo == "" {
fmt.Println("There was an ERROR")
os.Exit(1)
}
fmt.Println("ONE:", resultsOne)
fmt.Println("TWO:", resultsTwo)
}
I believe one caveat is that resultsOne := <- firstCh blocks until first goroutine finishes, but I don't care too much about this.
Can you please confirm that my approach is good? What other approaches would be better in my situation?
The example looks mostly good. A couple improvements are:
declaring your channels as buffered
firstCh := make(chan string, 1)
secondCh := make(chan string, 1)
With unbuffered channels, send operations block (until someone receives). If your goroutine #2 is much faster than the first, it will have to wait until the first finishes as well, since you receive in sequence:
resultsOne := <-firstCh // waiting on this one first
resultsTwo := <-secondCh // sender blocked because the main thread hasn't reached this point
use "golang.org/x/sync/errgroup".Group. The program will feel "less native" but it dispenses you from managing channels by hand — which trades, in a non-contrived setting, for sync'ing writes on the results:
func main() {
var (
resultsOne string
resultsTwo string
)
g := errgroup.Group{}
g.Go(func() error {
res, err := first("one")
if err != nil {
return err
}
resultsOne = res
return nil
})
g.Go(func() error {
res, err := second("two")
if err != nil {
return err
}
resultsTwo = res
return nil
})
err := g.Wait()
// ... handle err
I read this and this and this but none of them solving my issue..
I'm trying to read 2 files async, so I wrote the below:
//readlines.go
package main
import (
"bufio"
"os"
)
// readLines reads a whole file into memory
// and returns a slice of its lines.
func readLines(path string) ([]string, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
var lines []string
scanner := bufio.NewScanner(file)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}
return lines, scanner.Err()
}
And calling it as:
package main
import (
"fmt"
"os"
"github.com/gocarina/gocsv"
)
func (s *stocks) Read() {
fmt.Println("Reading")
stockFile, err := os.OpenFile("current_invenory.csv", os.O_RDWR|os.O_CREATE, os.ModePerm)
if err != nil {
panic(err)
}
defer stockFile.Close()
stocks := []systemStock{}
if err := gocsv.UnmarshalFile(stockFile, &stocks); err != nil { // Load stocks from file
panic(err)
}
*s = stocks
}
package main
import (
"fmt"
"os"
"github.com/gocarina/gocsv"
)
func (t *transactions) Read() {
fmt.Println("Reading")
trxFile, err := os.OpenFile("current_transactions.csv", os.O_RDWR|os.O_CREATE, os.ModePerm)
if err != nil {
panic(err)
}
defer trxFile.Close()
trx := []systemTransactions{}
if err := gocsv.UnmarshalFile(trxFile, &trx); err != nil { // Load stocks from file
panic(err)
}
*t = trx
}
The above working very fine with:
stock := stocks{}
trx := transactions{}
stock.Read()
trx.Read()
for _, s := range stock {
fmt.Println("Hello", s.Code)
}
But give the error fatal error: all goroutines are asleep - deadlock! when I tried to read them as:
cs, ct := readData()
for _, s := range cs {
fmt.Println("Hello", s.Code)
}
for _, t := range ct {
fmt.Println("Hello trx of ", t.Code)
}
Using
import "sync"
//func readData(cs chan stocks, ct chan transactions) (stocks, transactions) {
func readData() (stocks, transactions) {
var wg sync.WaitGroup
defer wg.Done()
stock := stocks{}
trx := transactions{}
wg.Add(1)
go stock.Read()
wg.Add(1)
go trx.Read()
wg.Wait()
return stock, trx
}
So the error is related for something wrong I made (or do not understand) in the last block~
To run the Read methods for stocks and transactions concurrently, these methods need to have a way of signaling when they are finished executing. This can be done in a lot of ways, but here are two which require the least modifications to your code.
Solution 1
Use the sync.WaitGroup package. With this package, the Read methods should execute wg.Done() statement when they are done with executing. It should look something like this:
func (s *stocks) Read(wg *sync.WaitGroup) {
defer wg.Done()
fmt.Println("Reading")
stockFile, err := os.OpenFile("current_invenory.csv", os.O_RDWR|os.O_CREATE, os.ModePerm)
if err != nil {
panic(err)
}
defer stockFile.Close()
stocks := []systemStock{}
if err := gocsv.UnmarshalFile(stockFile, &stocks); err != nil { // Load stocks from file
panic(err)
}
*s = stocks
}
func (t *transactions) Read(wg *sync.WaitGroup) {
defer wg.Done()
fmt.Println("Reading")
trxFile, err := os.OpenFile("current_transactions.csv", os.O_RDWR|os.O_CREATE, os.ModePerm)
if err != nil {
panic(err)
}
defer trxFile.Close()
trx := []systemTransactions{}
if err := gocsv.UnmarshalFile(trxFile, &trx); err != nil { // Load stocks from file
panic(err)
}
*t = trx
}
func readData() (stocks, transactions) {
var wg sync.WaitGroup
wg.Add(2)
stock := stocks{}
trx := transactions{}
go stock.Read(&wg)
go trx.Read(&wg)
wg.Wait()
return stock, trx
}
Solution 2
This approach uses the golang.org/x/sync/errgroup package. In this case, you do not need to handle the synchronization and signaling yourself, but functions that are added with errgroup.Go method need to have a strict func() error signature. Your code should look like this:
func (s *stocks) Read() error {
fmt.Println("Reading")
stockFile, err := os.OpenFile("current_invenory.csv", os.O_RDWR|os.O_CREATE, os.ModePerm)
if err != nil {
return err
}
defer stockFile.Close()
stocks := []systemStock{}
if err := gocsv.UnmarshalFile(stockFile, &stocks); err != nil { // Load stocks from file
return err
}
*s = stocks
return nil
}
func (t *transactions) Read() error {
fmt.Println("Reading")
trxFile, err := os.OpenFile("current_transactions.csv", os.O_RDWR|os.O_CREATE, os.ModePerm)
if err != nil {
return err
}
defer trxFile.Close()
trx := []systemTransactions{}
if err := gocsv.UnmarshalFile(trxFile, &trx); err != nil { // Load stocks from file
return err
}
*t = trx
return nil
}
func readData() (stocks, transactions) {
g, _ := errgroup.WithContext(context.Background())
stock := stocks{}
trx := transactions{}
g.Go(stock.Read)
g.Go(trx.Read)
if err:= g.Wait(); err != nil {
panic(err)
}
return stock, trx
}
Solution 3
You’re (correctly) adding 1 to the wait group when you start reading from each CSV, bringing the wait group’s internal counter to 2, but wg.Wait() will wait until that counter goes down to zero and you don’t have any calls to wg.Done() to do that. I recommend changing go stock.Read() to:
go func() {
defer wg Done()
stock.Read()
}()
So, the full working code be:
func readData() (stocks, transactions) {
var wg sync.WaitGroup
stock := stocks{}
trx := transactions{}
wg.Add(1)
go func() {
defer wg.Done()
stock.Read()
}()
wg.Add(1)
go func() {
defer wg.Done()
trx.Read()
}()
wg.Wait()
return stock, trx
}
I've created function which needs to run in goroutine, the code is working(this is just simple sample to illustrate the issue)
go rze(ftFilePath, 2)
func rze(ftDataPath,duration time.Duration) error {
}
I want to do something like this
errs := make(chan error, 1)
err := go rze(ftFilePath, 2)
if err != nil{
r <- Result{result, errs}
}
but not sure how to it, most of the examples show
how you do it when you using func
https://tour.golang.org/concurrency/5
You cannot use the return value of a function that is executed with the go keyword. Use an anonymous function instead:
errs := make(chan error, 1)
go func() {
errs <- rze(ftFilePath, 2)
}()
// later:
if err := <-errs; err != nil {
// handle error
}
You can use golang errgroup pkg.
var eg errgroup.Group
eg.Go(func() error {
return rze(ftFilePath, 2)
})
if err := g.Wait(); err != nil {
r <- Result{result, errs}
}
You can handle error from go routines using a separate channel for errors. Create a separate channel specifically for errors. Each child go routine must pass corresponding errors into this channel.
This is one of the ways how it can be done:
func main() {
type err error
items := []string{"1", "2", "3"}
ch := make(chan err, len(items))
for _, f := range items {
go func(f string) {
var e err
e = testFunc(f)
ch <- e
}(f)
}
for range items {
e := <-ch
fmt.Println("Error: ", e)
if e != nil {
// DO Something
}
}
}
func testFunc(item string) error {
if item == "1" {
return errors.New("err")
}
return nil
}
The following snippet validates a phone number and write the details to CSV.
func Parse(phone Input, output *PhoneNumber) error {
var n PhoneNumber
num, _ := phonenumbers.Parse(phone.Number, phone.Prefix)
n.PhoneNumber = phonenumbers.Format(num, phonenumbers.E164)
n.CountryCode = num.GetCountryCode()
n.PhoneType = phonenumbers.GetNumberType(num)
n.NetworkName, _ = phonenumbers.GetCarrierForNumber(num, "EN")
n.Region = phonenumbers.GetRegionCodeForNumber(num)
*output = n
return nil
}
func createFile(path string) {
// detect if file exists
var _, err = os.Stat(path)
// create file if not exists
if os.IsNotExist(err) {
var file, err = os.Create(path)
if err != nil {
return
}
defer file.Close()
}
}
func worker(ctx context.Context, dst chan string, src chan []string) {
for {
select {
case dataArray, ok := <-src: // you must check for readable state of the channel.
if !ok {
return
}
go processNumber(dataArray[0])
case <-ctx.Done(): // if the context is cancelled, quit.
return
}
}
}
func processNumber(number string) {
num, e := phonenumbers.Parse(number, "")
if e != nil {
return
}
region := phonenumbers.GetRegionCodeForNumber(num)
carrier, _ := phonenumbers.GetCarrierForNumber(num, "EN")
path := "sample_all.csv"
createFile(path)
var csvFile, _ = os.OpenFile(path, os.O_APPEND|os.O_WRONLY, os.ModeAppend)
csvwriter := csv.NewWriter(csvFile)
_ = csvwriter.Write([]string{phonenumbers.Format(num, phonenumbers.E164), fmt.Sprintf("%v", num.GetCountryCode()), fmt.Sprintf("%v", phonenumbers.GetNumberType(num)), carrier, region})
defer csvFile.Close()
csvwriter.Flush()
}
func ParseFile(phone Input, output *PhoneNumber) error {
// create a context
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// that cancels at ctrl+C
go onSignal(os.Interrupt, cancel)
numberOfWorkers := 2
start := time.Now()
csvfile, err := os.Open(phone.File)
if err != nil {
log.Fatal(err)
}
defer csvfile.Close()
reader := csv.NewReader(csvfile)
// create the pair of input/output channels for the controller=>workers com.
src := make(chan []string)
out := make(chan string)
// use a waitgroup to manage synchronization
var wg sync.WaitGroup
// declare the workers
for i := 0; i < numberOfWorkers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
worker(ctx, out, src)
}()
}
// read the csv and write it to src
go func() {
for {
record, err := reader.Read()
if err == io.EOF {
break
} else if err != nil {
log.Fatal(err)
}
src <- record // you might select on ctx.Done().
}
close(src) // close src to signal workers that no more job are incoming.
}()
// wait for worker group to finish and close out
go func() {
wg.Wait() // wait for writers to quit.
close(out) // when you close(out) it breaks the below loop.
}()
// drain the output
for res := range out {
fmt.Println(res)
}
fmt.Printf("\n%2fs", time.Since(start).Seconds())
return nil
}
In processNumber function, if I skip writing to CSV, the process of verifying number completes 6 seconds but writing one record at a time on CSV stretch the time consumption to 15s.
How can I optimize the code?
Can I chunk the records and write them in chunks instead of writing one row at a time?
Do work directly in worker goroutine instead of firing off goroutine per task.
Open file output file once. Flush output file once.
func worker(ctx context.Context, dst chan []string, src chan []string) {
for {
select {
case dataArray, ok := <-src: // you must check for readable state of the channel.
if !ok {
return
}
dst <- processNumber(dataArray[0])
case <-ctx.Done(): // if the context is cancelled, quit.
return
}
}
}
func processNumber(number string) []string {
num, e := phonenumbers.Parse(number, "")
if e != nil {
return
}
region := phonenumbers.GetRegionCodeForNumber(num)
carrier, _ := phonenumbers.GetCarrierForNumber(num, "EN")
return []string{phonenumbers.Format(num, phonenumbers.E164), fmt.Sprintf("%v", num.GetCountryCode()), fmt.Sprintf("%v", phonenumbers.GetNumberType(num)), carrier, region}
}
func ParseFile(phone Input, output *PhoneNumber) error {
// create a context
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// that cancels at ctrl+C
go onSignal(os.Interrupt, cancel)
numberOfWorkers := 2
start := time.Now()
csvfile, err := os.Open(phone.File)
if err != nil {
log.Fatal(err)
}
defer csvfile.Close()
reader := csv.NewReader(csvfile)
// create the pair of input/output channels for the controller=>workers com.
src := make(chan []string)
out := make(chan string)
// use a waitgroup to manage synchronization
var wg sync.WaitGroup
// declare the workers
for i := 0; i < numberOfWorkers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
worker(ctx, out, src)
}()
}
// read the csv and write it to src
go func() {
for {
record, err := reader.Read()
if err == io.EOF {
break
} else if err != nil {
log.Fatal(err)
}
src <- record // you might select on ctx.Done().
}
close(src) // close src to signal workers that no more job are incoming.
}()
// wait for worker group to finish and close out
go func() {
wg.Wait() // wait for writers to quit.
close(out) // when you close(out) it breaks the below loop.
}()
path := "sample_all.csv"
file, err := os.Create(path)
if err != nil {
return err
}
defer file.Close()
csvwriter := csv.NewWriter(csvFile)
// drain the output
for res := range out {
csvwriter.Write(res)
}
csvwriter.Flush()
fmt.Printf("\n%2fs", time.Since(start).Seconds())
return nil
}
I want to return an error from a function if it panics (in Go):
func getReport(filename string) (rep report, err error) {
rep.data = make(map[string]float64)
defer func() {
if r := recover(); r != nil {
fmt.Println("Recovered in f", r)
err, _ = r.(error)
return nil, err
}
}()
panic("Report format not recognized.")
// rest of the getReport function, which can try to out-of-bound-access a slice
...
}
I appear to have misunderstood the very concept of panic and defer. Can anybody enlighten me?
In a deferred function you can alter the returned parameters, but you can't return a new set. So a simple change to what you have will make it work.
There is another problem with what you wrote, namely that the you've paniced with a string but are expecting an error in your type assertion.
Here is a fix for both of those (Play)
defer func() {
if r := recover(); r != nil {
fmt.Println("Recovered in f", r)
// find out exactly what the error was and set err
switch x := r.(type) {
case string:
err = errors.New(x)
case error:
err = x
default:
err = errors.New("Unknown panic")
}
// invalidate rep
rep = nil
// return the modified err and rep
}
}()
have a look at this
package main
import "fmt"
func iWillPanic() {
panic("ops, panic")
}
func runner() (rtnValue string) {
rtnValue := ""
defer func() {
if r := recover(); r != nil {
// and your logs or something here, log nothing with panic is not a good idea
rtnValue = "don't panic" // modify the return value, and it will return
}
}()
iWillPanic()
return rtnValue
}
func main() {
fmt.Println("Return Value:", runner())
}
func TestReturnFromPanic(t *testing.T) {
fn := func(filename string) (rep string, err error) {
defer func() {
if r := recover(); r != nil {
err = fmt.Errorf("panic in getReport %s", r)
}
}()
return filename[100:], nil
}
t.Log(fn(``))
}
The named return parameter err is the trick.
https://play.golang.org/p/jpaCa9j2iAf