I've written this code for education purposes, now I need to write some test for it.
First, I need to test method Worker, but I don't understand how can I do it properly?
I'm totally new to test and to Go generally.
package worker
import (
"context"
"fmt"
)
type Queue interface {
TakeMessage() (<-chan string, error)
}
type Download interface {
Download(url string) error
}
type Worker struct {
queue Queue
download Download
}
func NewWorker(queue Queue, download Download) *Worker {
newWorker := Worker{}
newWorker.queue = queue
newWorker.download = download
return &newWorker
}
func (w *Worker) Worker(ctx context.Context) error {
msgs, err := w.queue.TakeMessage()
if err != nil {
return fmt.Errorf("error while consume queue: %w", err)
}
for {
select {
case <-ctx.Done():
return nil
case msg := <-msgs:
fmt.Println(msg)
w.download.Download(msg)
}
}
}
I write some lines of testing code and I suppossed to check if Worker return nil when ctx.Done() is passed to channel Now this test doesn't work, it's stuck maybe because of infinite cycle in worker method. Then I need to check if method takes messages from queue and pass it to Download method.
package worker
import (
"context"
"errors"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
type MockQueue struct {
mock.Mock
}
type MockDownloader struct {
mock.Mock
}
func (m *MockQueue) TakeMessage() (<-chan string, error) {
strCh := make(chan string)
strCh <- "some_url/some.txt"
return strCh, nil
}
func (d *MockDownloader) Download(url string) error {
if url == "some_url/some.txt" {
return nil
} else {
return errors.New(url)
}
}
func TestWorkerCloseContext(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
resultCh := make(chan error)
newQueue := &MockQueue{}
newDownload := &MockDownloader{}
newWorker := Worker{newQueue, newDownload}
go func() {
resultCh <- newWorker.Worker(ctx)
}()
cancel()
assert.Nil(t, <-resultCh)
}
func TestWorkerMessageReceive(t \*testing.T) {
ctx, cancel := context.WithCancel(context.Background())
resultCh := make(chan error)
newQueue := &MockQueue{}
newDownload := &MockDownloader{}
newWorker := Worker{newQueue, newDownload}
go func() {
resultCh \<- newWorker.Worker(ctx)
}()
//some code here
}
Go comes with a pretty extensive built-in testing library. See https://pkg.go.dev/testing
I would recommend looking at the below tutorials:
https://go.dev/blog/cover
https://blog.alexellis.io/golang-writing-unit-tests/
https://dev.to/quii/learn-go-by-writing-tests-structs-methods-interfaces--table-driven-tests-1p01
Related
I'm somewhat new to go and am reworking code that I found somewhere else to fit my needs. Because of that, I don't totally understand what is happening here, although I get the general idea.
I'm running a few websocket clients using go routines, but I'm getting an unexpected error that causes the program to crash. My program seems to close one too many threads (excuse me if this is the wrong terminology) when there is an error reading a message from the websocket (check the conn.ReadMessage() func in the readHandler func). Any ideas on how would I work around this issue? I would really appreciate anyone taking the time to look through it. Thanks in advance!
package main
import (
"context"
"fmt"
"os"
"time"
"os/signal"
"syscall"
"sync"
"net/url"
"github.com/gorilla/websocket"
"strconv"
"encoding/json"
"log"
"bytes"
"compress/gzip"
"io/ioutil"
)
// Structs
type Ping struct {
Ping int64 `json:"ping"`
}
type Pong struct {
Pong int64 `json:"pong"`
}
type SubParams struct {
Sub string `json:"sub"`
ID string `json:"id"`
}
func InitSub(subType string, pair string, i int) []byte {
var idInt string = "id" + strconv.Itoa(i)
subStr := "market." + pair + "." + subType
sub := &SubParams{
Sub: subStr,
ID: idInt,
}
out, err := json.MarshalIndent(sub, "", " ")
if err != nil {
log.Println(err);
}
//log.Println(string(out))
return out
}
// main func
func main() {
var server string = "api.huobi.pro"
pairs := []string{"btcusdt", "ethusdt", "ltcusdt"}
comms := make(chan os.Signal, 1)
signal.Notify(comms, os.Interrupt, syscall.SIGTERM)
ctx := context.Background()
ctx, cancel := context.WithCancel(ctx)
var wg sync.WaitGroup
for x, pair := range pairs {
wg.Add(1)
go control(server, "ws", pair, ctx, &wg, x+1)
}
<-comms
cancel()
wg.Wait()
}
func control(server string, path string, pair string, ctx context.Context, wg *sync.WaitGroup, i int) {
fmt.Printf("Started control for %s\n", server)
url := url.URL {
Scheme: "wss",
Host: server,
Path: path,
}
fmt.Println(url.String())
conn, _, err := websocket.DefaultDialer.Dial(url.String(), nil)
if err != nil {
panic(err)
}
subscribe(conn, pair, i)
defer conn.Close()
var localwg sync.WaitGroup
localwg.Add(1)
go readHandler(ctx, conn, &localwg, server)
<- ctx.Done()
localwg.Wait()
wg.Done()
return
}
func readHandler(ctx context.Context, conn *websocket.Conn, wg *sync.WaitGroup, server string) {
for {
select {
case <- ctx.Done():
wg.Done()
return
default:
_, p, err := conn.ReadMessage()
if err != nil {
wg.Done()
fmt.Println(err)
}
r, err := gzip.NewReader(bytes.NewReader(p))
if(err == nil) {
result, err := ioutil.ReadAll(r)
if(err != nil) {
fmt.Println(err)
}
d := string(result)
fmt.Println(d)
var ping Ping
json.Unmarshal([]byte(d), &ping)
if (ping.Ping > 0) {
str := Pong{Pong: ping.Ping}
msg, err := json.Marshal(str)
if (err == nil) {
fmt.Println(string(msg))
conn.WriteMessage(websocket.TextMessage, []byte(msg))
}
}
}
}
}
}
func subscribe(conn *websocket.Conn, pair string, id int) {
sub := string(InitSub("trade.detail", pair, id))
err := conn.WriteMessage(websocket.TextMessage, []byte(sub))
if err != nil {
panic(err)
}
}
Break out of the readHandler loop when the connection fails:
_, p, err := conn.ReadMessage()
if err != nil {
wg.Done()
fmt.Println(err)
return // <--- add this line
}
Without the return, the function spins in a tight loop reading errors until the panic.
Use defer wg.Done() at the beginning of the goroutine to ensure that Done is called exactly once.
func readHandler(ctx context.Context, conn *websocket.Conn, wg *sync.WaitGroup, server string) {
defer wg.Done()
for {
select {
case <-ctx.Done():
return
default:
_, p, err := conn.ReadMessage()
if err != nil {
fmt.Println(err)
return
}
...
Update the control function also.
Because the caller does not execute any code concurrently with readHander, there's no value in running readHandler is a goroutine. Remove all references to wait groups from readHandler and call the function directly: change go readHandler(ctx, conn, &localwg, server) to readHandler(ctx, conn, server).
There are more issues, but this should move you further along.
I have a goroutine inside a loop and the way I am handling the error is that I add it to a channel and after all the goroutines are finished, I check if there was an error and I return accordingly.
The issue with this is that I want to return an error as soon as I get it so that I don't spend time waiting for all the goroutines to finish as it would be inefficient.
I tried adding the select statement but it doesn't work and I can't add the select statement inside the goroutines since I want to exit the for loop and the try function too.
How can I do this?
Here is the code:
package main
import (
"sync"
"runtime"
"fmt"
"errors"
)
func try() (bool, error) {
wg := new(sync.WaitGroup)
s := []int{0,1,2,3,4,5}
ec := make(chan error)
for i, val := range s {
/*
select {
case err, ok := <-ec:
if ok {
println("error 1", err.Error())
return false, err
}
default:
}
*/
wg.Add(1)
i := i
val := val
go func() {
err := func(i int, val int, wg *sync.WaitGroup) error {
defer wg.Done()
if i == 3 {
return errors.New("one error")
} else {
return nil
}
}(i, val, wg)
if err != nil {
ec <- err
return
}
}()
}
wg.Wait()
select {
case err, ok := <-ec:
if ok {
println("error 2", err.Error())
return false, err
}
default:
}
return true, nil
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
b, e := try()
if e != nil {
fmt.Println(e.Error(), b)
}
}
This is the go playground link
With wg.Wait() before your select statement, you are effectively waiting for all goroutines to return.
The issue with this is that I want to return an error as soon as I get it
I assume that with this you mean stopping running goroutines as soon as any one of them returns an error.
In this case, you could use context.Context to manage cancellation, but even better is an errgroup.Group, which nicely combines context functionality and synchronization:
Package errgroup provides synchronization, error propagation, and Context cancelation for groups of goroutines working on subtasks of a common task.
In particular Group.Go:
The first call to return a non-nil error cancels the group; its error will be returned by Wait.
import (
"sync"
"runtime"
"fmt"
"errors"
"golang.org/x/sync/errgroup"
)
func try() (bool, error) {
errg := new(errgroup.Group)
s := []int{0,1,2,3,4,5}
for i, val := range s {
i := i
val := val
errg.Go(func() error {
return func(i int, val int) error {
if i == 3 {
return errors.New("one error")
} else {
return nil
}
}(i, val)
})
}
if err := errg.Wait(); err != nil {
// handle error
}
return true, nil
}
https://play.golang.org/p/lSIIFJqXf0W
I have found tomb to be useful for this. Below is a stripped-down non-working example that shows the gist, without handling things like variable encapsulation in the loop. It should give you the idea, but I'm happy to clarify on any points.
package main
import (
"fmt"
"gopkg.in/tomb.v2"
"sync"
)
func main() {
ts := tomb.Tomb{}
s := []int{0,1,2,3,4,5}
for i, v := range s {
ts.Go(func() error {
// do some work here or return an error, make sure to watch the dying chan, if it closes,
//then one of the other go-routines failed.
select {
case <- ts.Dying():
return nil
case err := <- waitingForWork():
if err != nil {
return err
}
return nil
}
})
}
// If an error appears here, one of the go-routines must have failed
err := ts.Wait()
if err != nil {
fmt.Println(err)
}
}
For a personal project, I'm spawning an external process which hosts a REST API.
I only want to pass control back to the main thread once the external process has been initialized, which can be known by reading Stdout.
So I have created an io.Writer implementation which closes a channel once a certain criteria has been met.
type channelIOWriter struct {
InitializedChannel chan bool
isInitialized bool
buffer string
}
func newChannelIOWriter() *channelIOWriter {
retVal := new(channelIOWriter)
retVal.InitializedChannel = make(chan bool)
return retVal
}
func (writer *channelIOWriter) Write(data []byte) (int, error) {
for _, b := range data {
if b == '\n' {
writer.buffer = *bytes.NewBuffer(make([]byte, 0))
continue
}
writer.buffer.WriteByte(b)
if strings.HasPrefix(writer.buffer.String(), "ChromeDriver was started successfully.") {
if !writer.isInitialized {
close(writer.InitializedChannel)
}
writer.isInitialized = true
}
}
return len(data), nil
}
Next, I have the main function which, in a seperate goroutine, spawns the external process.
A wait is performed on the channel that's suposed to be closed by the io.Writer implementation.
func main() {
writer := newChannelIOWriter()
go func() {
cmd := exec.Command("chromedriver", "--port=9009")
cmd.Stdout = writer
cmd.Start()
}()
<-writer.InitializedChannel
fmt.Println("Control is passed back to the MAIN thread.")
}
According to the comments on the question, it seems to be better to use a context.
Anyone who can explain on how to use that?
I stripped out all unnecessary detail and demo how context with deadline works
package main
import (
"context"
"fmt"
"os/exec"
"time"
)
type example struct {
cancel context.CancelFunc
}
func (e *example) Write(data []byte) (int, error) {
defer e.cancel()
return len(data), nil
}
func main() {
deadline := time.Now().Add(5 * time.Second)
ctx, cancel := context.WithDeadline(context.Background(), deadline)
writer := &example{
cancel: cancel,
}
go func() {
cmd := exec.Command("ls", ".")
cmd.Stdout = writer
_ = cmd.Start()
}()
<-ctx.Done()
fmt.Println("Control is passed back to the MAIN thread.", ctx.Err())
}
I am playing around with Golang and I created this little app to make several concurrent api calls using goroutines.
While the app works, after the calls complete, the app gets stuck, which makes sense because it cannot exit the range c loop because the channel is not closed.
I am not sure where to better close the channel in this pattern.
package main
import "fmt"
import "net/http"
func main() {
links := []string{
"https://github.com/fabpot",
"https://github.com/andrew",
"https://github.com/taylorotwell",
"https://github.com/egoist",
"https://github.com/HugoGiraudel",
}
checkUrls(links)
}
func checkUrls(urls []string) {
c := make(chan string)
for _, link := range urls {
go checkUrl(link, c)
}
for msg := range c {
fmt.Println(msg)
}
close(c) //this won't get hit
}
func checkUrl(url string, c chan string) {
_, err := http.Get(url)
if err != nil {
c <- "We could not reach:" + url
} else {
c <- "Success reaching the website:" + url
}
}
You close a channel when there are no more values to send, so in this case it's when all checkUrl goroutines have completed.
var wg sync.WaitGroup
func checkUrls(urls []string) {
c := make(chan string)
for _, link := range urls {
wg.Add(1)
go checkUrl(link, c)
}
go func() {
wg.Wait()
close(c)
}()
for msg := range c {
fmt.Println(msg)
}
}
func checkUrl(url string, c chan string) {
defer wg.Done()
_, err := http.Get(url)
if err != nil {
c <- "We could not reach:" + url
} else {
c <- "Success reaching the website:" + url
}
}
(Note that the error from http.Get is only going to reflect connection and protocol errors. It is not going to contain http server errors if you're expecting those too, which you must be seeing how you're checking for paths and not just hosts.)
When writing programs in Go using channels and goroutines always think about who (which function) owns a channel. I prefer the practice of letting the function who owns a channel close it. If i were to write this i would do as shown below.
Note: A better way to handle situations like this is the Fan-out, fan-in concurrency pattern. refer(https://blog.golang.org/pipelines)Go Concurrency Patterns
package main
import "fmt"
import "net/http"
import "sync"
func main() {
links := []string{
"https://github.com/fabpot",
"https://github.com/andrew",
"https://github.com/taylorotwell",
"https://github.com/egoist",
"https://github.com/HugoGiraudel",
}
processURLS(links)
fmt.Println("End of Main")
}
func processURLS(links []string) {
resultsChan := checkUrls(links)
for msg := range resultsChan {
fmt.Println(msg)
}
}
func checkUrls(urls []string) chan string {
outChan := make(chan string)
go func(urls []string) {
defer close(outChan)
var wg sync.WaitGroup
for _, url := range urls {
wg.Add(1)
go checkUrl(&wg, url, outChan)
}
wg.Wait()
}(urls)
return outChan
}
func checkUrl(wg *sync.WaitGroup, url string, c chan string) {
defer wg.Done()
_, err := http.Get(url)
if err != nil {
c <- "We could not reach:" + url
} else {
c <- "Success reaching the website:" + url
}
}
I got a problem using sync.WaitGroup and select together. If you take a look at following http request pool you will notice that if an error occurs it will never be reported as wg.Done() will block and there is no read from the channel anymore.
package pool
import (
"fmt"
"log"
"net/http"
"sync"
)
var (
MaxPoolQueue = 100
MaxPoolWorker = 10
)
type Pool struct {
wg *sync.WaitGroup
queue chan *http.Request
errors chan error
}
func NewPool() *Pool {
return &Pool{
wg: &sync.WaitGroup{},
queue: make(chan *http.Request, MaxPoolQueue),
errors: make(chan error),
}
}
func (p *Pool) Add(r *http.Request) {
p.wg.Add(1)
p.queue <- r
}
func (p *Pool) Run() error {
for i := 0; i < MaxPoolWorker; i++ {
go p.doWork()
}
select {
case err := <-p.errors:
return err
default:
p.wg.Wait()
}
return nil
}
func (p *Pool) doWork() {
for r := range p.queue {
fmt.Printf("Request to %s\n", r.Host)
p.wg.Done()
_, err := http.DefaultClient.Do(r)
if err != nil {
log.Fatal(err)
p.errors <- err
} else {
fmt.Printf("no error\n")
}
}
}
Source can be found here
How can I still use WaitGroup but also get errors from go routines?
Just got the answer my self as I wrote the question and as I think it is an interesting case I would like to share it with you.
The trick to use sync.WaitGroup and chan together is that we wrap:
select {
case err := <-p.errors:
return err
default:
p.wg.Done()
}
Together in a for loop:
for {
select {
case err := <-p.errors:
return err
default:
p.wg.Done()
}
}
In this case select will always check for errors and wait if nothing happens :)
It looks a bit like the fail-fast mechanism enabled by the Tomb library (Tomb V2 GoDoc):
The tomb package handles clean goroutine tracking and termination.
If any of the tracked goroutines returns a non-nil error, or the Kill or Killf method is called by any goroutine in the system (tracked or not), the tomb Err is set, Alive is set to false, and the Dying channel is closed to flag that all tracked goroutines are supposed to willingly terminate as soon as possible.
Once all tracked goroutines terminate, the Dead channel is closed, and Wait unblocks and returns the first non-nil error presented to the tomb via a result or an explicit Kill or Killf method call, or nil if there were no errors.
You can see an example in this playground:
(extract)
// start runs all the given functions concurrently
// until either they all complete or one returns an
// error, in which case it returns that error.
//
// The functions are passed a channel which will be closed
// when the function should stop.
func start(funcs []func(stop <-chan struct{}) error) error {
var tomb tomb.Tomb
var wg sync.WaitGroup
allDone := make(chan struct{})
// Start all the functions.
for _, f := range funcs {
f := f
wg.Add(1)
go func() {
defer wg.Done()
if err := f(tomb.Dying()); err != nil {
tomb.Kill(err)
}
}()
}
// Start a goroutine to wait for them all to finish.
go func() {
wg.Wait()
close(allDone)
}()
// Wait for them all to finish, or one to fail
select {
case <-allDone:
case <-tomb.Dying():
}
tomb.Done()
return tomb.Err()
}
A simpler implementation would be like below. (Check in play.golang: https://play.golang.org/p/TYxxsDRt5Wu)
package main
import "fmt"
import "sync"
import "time"
type Error struct {
message string
}
func (e Error) Error() string {
return e.message
}
func main() {
var wg sync.WaitGroup
waitGroupLength := 8
errChannel := make(chan error, 1)
// Setup waitgroup to match the number of go routines we'll launch off
wg.Add(waitGroupLength)
finished := make(chan bool, 1) // this along with wg.Wait() are why the error handling works and doesn't deadlock
for i := 0; i < waitGroupLength; i++ {
go func(i int) {
fmt.Printf("Go routine %d executed\n", i+1)
time.Sleep(time.Duration(waitGroupLength - i))
time.Sleep(0) // only here so the time import is needed
if i%4 == 1 {
errChannel <- Error{fmt.Sprintf("Errored on routine %d", i+1)}
}
// Mark the wait group as Done so it does not hang
wg.Done()
}(i)
}
go func() {
wg.Wait()
close(finished)
}()
L:
for {
select {
case <-finished:
break L // this will break from loop
case err := <-errChannel:
if err != nil {
fmt.Println("error ", err)
// handle your error
}
}
}
fmt.Println("Executed all go routines")
}