I am newbie in Golang - While trying to rewrite the following to a single thread implementation.
.....
run := func(handler func(chan<- modify), threads int) <-chan modify {
result := make(chan modify)
go func() {
var wg sync.WaitGroup
for i := 0; i < threads; i++ {
wg.Add(1)
go func() {
defer wg.Done()
handler(result)
}()
}
wg.Wait()
close(result)
}()
return result
}
modifyAgg := run(func(result chan<- modify) {
aggre := run(func(result chan<- modify) {
u.addAgg(slices, result) // returns result channel
}, u.threads.GrpTxns)
....
In the above code the variable addAgg is of type chan<- modify. The following is not - get error while ranging over the variable aggre "cannot range over addAgg(type func())"
aggre := func() {
result:= make(chan modify)
defer close(result)
u.addAgg(slices, result) // returns result channel
}
How to change the second implementation to mimic the first one? Thank you !
I was able to implement this in single thread...
aggre := func() <-chan modify{
result:= make(chan modify, 50) // make it non blocking
u.addAgg(slices, result)
defer close(result)
return result
}()
Related
I am having issue while using waitgroup with the buffered channel. The problem is waitgroup closes before channel is read completely, which make my channel is half read and break in between.
func main() {
var wg sync.WaitGroup
var err error
start := time.Now()
students := make([]studentDetails, 0)
studentCh := make(chan studentDetail, 10000)
errorCh := make(chan error, 1)
wg.Add(1)
go s.getDetailStudents(rCtx, studentCh , errorCh, &wg, s.Link, false)
go func(ch chan studentDetail, e chan error) {
LOOP:
for {
select {
case p, ok := <-ch:
if ok {
L.Printf("Links %s: [%s]\n", p.title, p.link)
students = append(students, p)
} else {
L.Print("Closed channel")
break LOOP
}
case err = <-e:
if err != nil {
break
}
}
}
}(studentCh, errorCh)
wg.Wait()
close(studentCh)
close(errorCh)
L.Warnln("closed: all wait-groups completed!")
L.Warnf("total items fetched: %d", len(students))
elapsed := time.Since(start)
L.Warnf("operation took %s", elapsed)
}
The problem is this function is recursive. I mean some http call to fetch students and then make more calls depending on condition.
func (s Student) getDetailStudents(rCtx context.Context, content chan<- studentDetail, errorCh chan<- error, wg *sync.WaitGroup, url string, subSection bool) {
util.MustNotNil(rCtx)
L := logger.GetLogger(rCtx)
defer func() {
L.Println("Closing all waitgroup!")
wg.Done()
}()
wc := getWC()
httpClient := wc.Registry.MustHTTPClient()
res, err := httpClient.Get(url)
if err != nil {
L.Fatal(err)
}
defer res.Body.Close()
if res.StatusCode != 200 {
L.Errorf("status code error: %d %s", res.StatusCode, res.Status)
errorCh <- errors.New("service_status_code")
return
}
// parse response and return error if found some through errorCh as done above.
// decide page subSection based on response if it is more.
if !subSection {
wg.Add(1)
go s.getDetailStudents(rCtx, content, errorCh, wg, link, true)
// L.Warnf("total pages found %d", pageSub.Length()+1)
}
// Find students from response list and parse each Student
students := s.parseStudentItemList(rCtx, item)
for _, student := range students {
content <- student
}
L.Warnf("Calling HTTP Service for %q with total %d record", url, elementsSub.Length())
}
Variables are changed to avoid original code base.
The problem is students are read randomly as soon as Waitgroup complete. I am expecting to hold the execution until all students are read, In case of error it should break as soon error encounter.
You need to know when the receiving goroutine completes. The WaitGroup does that for the generating goroutine. So, you can use two waitgroups:
wg.Add(1)
go s.getDetailStudents(rCtx, studentCh , errorCh, &wg, s.Link, false)
wgReader.Add(1)
go func(ch chan studentDetail, e chan error) {
defer wgReader.Done()
...
}
wg.Wait()
close(studentCh)
close(errorCh)
wgReader.Wait() // Wait for the readers to complete
Since you are using buffered channels you can retrieve the remaining values after closing the channel. You will also need a mechanism to prevent your main function from exiting too early while the reader is still doing work ,as #Burak Serdar has advised.
I restructured the code to give a working example but it should get the point across.
package main
import (
"context"
"log"
"sync"
"time"
)
type studentDetails struct {
title string
link string
}
func main() {
var wg sync.WaitGroup
var err error
students := make([]studentDetails, 0)
studentCh := make(chan studentDetails, 10000)
errorCh := make(chan error, 1)
start := time.Now()
wg.Add(1)
go getDetailStudents(context.TODO(), studentCh, errorCh, &wg, "http://example.com", false)
LOOP:
for {
select {
case p, ok := <-studentCh:
if ok {
log.Printf("Links %s: [%s]\n", p.title, p.link)
students = append(students, p)
} else {
log.Println("Draining student channel")
for p := range studentCh {
log.Printf("Links %s: [%s]\n", p.title, p.link)
students = append(students, p)
}
break LOOP
}
case err = <-errorCh:
if err != nil {
break LOOP
}
case <-wrapWait(&wg):
close(studentCh)
}
}
close(errorCh)
elapsed := time.Since(start)
log.Printf("operation took %s", elapsed)
}
func getDetailStudents(rCtx context.Context, content chan<- studentDetails, errorCh chan<- error, wg *sync.WaitGroup, url string, subSection bool) {
defer func() {
log.Println("Closing")
wg.Done()
}()
if !subSection {
wg.Add(1)
go getDetailStudents(rCtx, content, errorCh, wg, url, true)
// L.Warnf("total pages found %d", pageSub.Length()+1)
}
content <- studentDetails{
title: "title",
link: "link",
}
}
// helper function to allow using WaitGroup in a select
func wrapWait(wg *sync.WaitGroup) <-chan struct{} {
out := make(chan struct{})
go func() {
wg.Wait()
out <- struct{}{}
}()
return out
}
wg.Add(1)
go func(){
defer wg.Done()
// I do not think that you need a recursive function.
// this function overcomplicated.
s.getDetailStudents(rCtx, studentCh , errorCh, &wg, s.Link, false)
}(...)
wg.Add(1)
go func(ch chan studentDetail, e chan error) {
defer wg.Done()
...
}(...)
wg.Wait()
close(studentCh)
close(errorCh)
This should solve the problem. s.getDetailStudents function must be simplified. Making it recursive does not have any benefit.
I'm new to Golang, I would like to understand more about goroutine. I'll leave two examples and I wanted the opinion of which of the two is more performative and why?
func doRequest(method string, url string, body io.Reader) (*http.Response, error) {
request, _ := http.NewRequest(method, url, body)
response, err := c.httpClient.Do(request)
request.Close = true
c.httpClient.CloseIdleConnections()
return response, err
}
first:
func test() {
var wg *sync.WaitGroup = new(sync.WaitGroup)
qtd := 5
wg.Add(qtd)
for i := 0; i < qtd; i++ {
go func(wg *sync.WaitGroup) {
defer wg.Done()
doRequest(http.MethodGet, "http://test.com", nil)
}(wg)
}
wg.Wait()
}
Second:
func test() {
var wg *sync.WaitGroup = new(sync.WaitGroup)
wg.Add(1)
go func(wg *sync.WaitGroup) {
defer wg.Done()
for i := 0; i < 5; i++ {
doRequest(http.MethodGet, "http://test.com", nil)
}
}(wg)
wg.Wait()
}
Is there a better way than these two?
If not, which of the two is more performant?
The go keyword before a function will create a goroutine.
5 gouroutines, 1 doRequest() in per goroutine
1 gouroutine, 5 doRequest() in a goroutine
In the 1st case, 5 goroutines will run concurrently.
Check out concurrency in: A Tour of Go
My task is to sync 2 goroutines so the output should look like that:
foobarfoobarfoobarfoobar
.The issue is that when I call them they come out completely randomized. This is my code:
package main
import (
"fmt"
"sync"
"time"
)
type ConcurrentPrinter struct {
sync.WaitGroup
sync.Mutex
}
func (cp *ConcurrentPrinter) printFoo(times int) {
cp.WaitGroup.Add(times)
go func() {
cp.Lock()
fmt.Print("foo")
cp.Unlock()
}()
}
func (cp *ConcurrentPrinter) printBar(times int) {
cp.WaitGroup.Add(times)
go func() {
cp.Lock()
fmt.Print("bar")
cp.Unlock()
}()
}
func main() {
times := 10
cp := &ConcurrentPrinter{}
for i := 0; i <= times; i++ {
cp.printFoo(i)
cp.printBar(i)
}
time.Sleep(10 * time.Millisecond)
}
As outlined in the comments, using goroutines may not be the best use case for what you are trying to achieve - and thus this may be an XY problem.
Having said that, if you want to ensure two independent goroutines interleave their work in an alternating sequence, you can implement a set of "ping-pong" mutexs:
var ping, pong sync.Mutex
pong.Lock() // ensure the 2nd goroutine waits & the 1st goes first
go func() {
for {
ping.Lock()
foo()
pong.Unlock()
}
}()
go func() {
for {
pong.Lock()
bar()
ping.Unlock()
}
}()
https://go.dev/play/p/VO2LoMJ8fek
Using channel:
func printFoo(i int, ch chan<- bool, wg *sync.WaitGroup) {
wg.Add(1)
go func() {
defer wg.Done()
fmt.Print("foo")
ch <- true
}()
}
func printBar(i int, ch chan<- bool, wg *sync.WaitGroup) {
wg.Add(1)
go func() {
defer wg.Done()
fmt.Print("bar")
ch <- true
}()
}
func main() {
times := 4
firstchan := make(chan bool)
secondchan := make(chan bool)
var wg sync.WaitGroup
for i := 0; i <= times; i++ {
printFoo(i, firstchan, &wg)
<-firstchan
printBar(i, secondchan, &wg)
<-secondchan
}
wg.Wait()
}
https://go.dev/play/p/MlZ9dHkUXGb
I have the following program which serves as a proof of concept. I'm attempting to aggregate the results from chann, that is, too merge each instance of chann in to a common slice. Is this possible with my approach?
So my output for the following example would be a slice containing the following (in any order): []int{0,1,2}, thanks.
func DoStuff(i int, chann chan[]int, wg *sync.WaitGroup) {
defer wg.Done()
chann <-[]int{i}
}
func main() {
var wg sync.WaitGroup
chann := make(chan int[], 3)
defer close(chann)
for i := 0; i < count; 3 {
wg.Add(1)
go DoStuff(i, chann, &wg)
}
wg.Wait()
for {
select {
case result := <-chann:
fmt.Println(result)
os.Exit(1)
}
}
return nil
}
what you want to do is possible, but your program will not run because you are reading from the channel after wg.Wait(), so all goroutines will stop waiting to write, because you will never read from the channel.
You can read from the channel in a goroutine:
for i := 0; i < count; 3 {
wg.Add(1)
go DoStuff(i, chann, &wg)
}
}
go func() {
for data:=range chann {
// Process data
}
}()
wg.Wait()
// Close here, so the reading goroutine can terminate
close(chann)
I'm trying to follow along the bounded goroutine example that is posted at http://blog.golang.org/pipelines/bounded.go. The problem that I'm having is that if there are more workers spun up then the amount of work to do, the extra workers never get cancelled. Everything else seems to work, the values get computed and logged, but when I close the groups channel, the workers just hang at the range statement.
I guess what I don't understand (in both my code and the example code) is how do the workers know when there is no more work to do and that they should exit?
Update
A working (i.e. non-working) example is posted at http://play.golang.org/p/T7zBCYLECp. It shows the deadlock on the workers since they are all asleep and there is no work to do. What I'm confused about is that I think the example code would have the same problem.
Here is the code that I'm currently using:
// Creates a pool of workers to do a bunch of computations
func computeAll() error {
done := make(chan struct{})
defer close(done)
groups, errc := findGroups(done)
// start a fixed number of goroutines to schedule with
const numComputers = 20
c := make(chan result)
var wg sync.WaitGroup
wg.Add(numComputers)
for i := 0; i < numComputers; i++ {
go func() {
compute(done, groups, c)
wg.Done()
}()
}
go func() {
wg.Wait()
close(c)
}()
// log the results of the computation
for r := range c { // log the results }
if err := <-errc; err != nil {
return err
}
return nil
}
Here is the code that fills up the channel with data:
// Retrieves the groups of data the must be computed
func findGroups(done <-chan struct{}) (<-chan model, <-chan error) {
groups := make(chan model)
errc := make(chan error, 1)
go func() {
// close the groups channel after find returns
defer close(groups)
group, err := //... code to get the group ...
if err == nil {
// add the group to the channel
select {
case groups <- group:
}
}
}()
return groups, errc
}
And here is the code that reads the channel to do the computations.
// Computes the results for the groups of data
func compute(done <-chan struct{}, groups <-chan model, c chan<- result) {
for group := range groups {
value := compute(group)
select {
case c <- result{value}:
case <-done:
return
}
}
}
Because you're trying to read from errc and it's empty unless there's an error.
//edit
computeAll() will always block on <- errc if there are no errors, another approach is to use something like:
func computeAll() (err error) {
.........
select {
case err = <-errc:
default: //don't block
}
return
}
Try to close the errc as OneOfOne says
go func() {
wg.Wait()
close(c)
close(errc)
}()
// log the results of the computation
for r := range c { // log the results }
if err := range errc {
if err != nil {
return err
}
}