Could not `chan` pass through RPC in Go? - go

For the following code, get Message via RPC, why its Done field is nil?
Could not chan pass through RPC?
type Message struct {
Text string
Done chan bool
}
type (
GetArgs struct {
Key string
}
GetReply struct {
Message *Message
}
)
// Client
func connect() *rpc.Client {
client, err := rpc.Dial("tcp", ":1234")
if err != nil {
log.Fatal("dialing:", err)
}
return client
}
func get(key string) *Message {
client := connect()
args := GetArgs{key}
reply := GetReply{}
err := client.Call("KV.Get", &args, &reply)
if err != nil {
log.Fatal("error:", err)
}
client.Close()
return reply.Message
}
// Server
type KV struct {
mu sync.Mutex
messages map[string]*Message
}
func server() {
kv := new(KV)
kv.messages = map[string]*Message{}
done := make(chan bool)
kv.messages["Jan"] = &Message{Text: "hello", Done: done}
rpcs := rpc.NewServer()
rpcs.Register(kv)
l, err := net.Listen("tcp", ":1234")
if err != nil {
log.Fatal("error:", err)
}
go func() {
for {
conn, err := l.Accept()
if err == nil {
go rpcs.ServeConn(conn)
} else {
break
}
}
l.Close()
}()
}
func (kv *KV) Get(args *GetArgs, reply *GetReply) error {
kv.mu.Lock()
defer kv.mu.Unlock()
val, ok := kv.messages[args.Key]
if ok {
reply.Message = val
}
return nil
}
func main() {
server()
fmt.Printf("get() -> %v\n", get("Jan"))
// Output:
// get() -> &{hello <nil>}
}

No. RPC arguments can only be those that are serializable by encoding/gob, and chans are not one of them
Functions and channels will not be sent in a gob. Attempting to encode such a value at the top level will fail. A struct field of chan or func type is treated exactly like an unexported field and is ignored.

Related

golang, goroutines race condition in test

I need to subscribe to a topic(topic is a channel) before publishing to a topic, but when creating a thread I need to run go Func to keep listening to channels to process messages (for example from publish or subscribe a new subscribe )
the test works (but not every time), sometimes when I run the test it ends up posting a message on the channel (topic) before I'm listening to the topic (channel)
i have this test:
func Test_useCase_publish(t *testing.T) {
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
t.Parallel()
tt.fields.storage = &RepositoryMock{
GetTopicFunc: func(ctx context.Context, topicName vos.TopicName) (entities.Topic, error) {
return tt.fields.topic, nil
},
}
useCase := New(tt.fields.storage)
subscribed := make(chan struct{})
go func() {
tt.fields.topic.Activate()
ch, _, err := useCase.Subscribe(tt.args.ctx, tt.args.message.TopicName)
require.NoError(t, err)
close(subscribed)
msg, ok := <-ch
if ok {
fmt.Println("msg", msg)
assert.Equal(t, tt.want, msg)
}
}()
<-subscribed
err := useCase.Publish(tt.args.ctx, tt.args.message)
assert.ErrorIs(t, err, tt.wantErr)
})
}
}
topic :
func (t Topic) Activate() {
go t.listenForSubscriptions()
go t.listenForMessages()
go t.listenForKills()
}
func (t *Topic) listenForSubscriptions() {
for newSubCh := range t.newSubCh {
t.Subscribers.Store(newSubCh.GetID(), newSubCh)
}
}
func (t *Topic) listenForKills() {
for subscriberID := range t.killSubCh {
t.Subscribers.Delete(subscriberID)
}
}
func (t *Topic) listenForMessages() {
for msg := range t.newMessageCh {
m := msg
t.Subscribers.Range(func(key, value interface{}) bool {
if key == nil || value == nil {
return false
}
if subscriber, ok := value.(Subscriber); ok {
subscriber.ReceiveMessage(m)
}
return true
})
}
func (t Topic) Dispatch(message vos.Message) {
t.newMessageCh <- message
}
func (t *Topic) listenForMessages() {
for msg := range t.newMessageCh {
m := msg
t.Subscribers.Range(func(key, value interface{}) bool {
if key == nil || value == nil {
return false
}
if subscriber, ok := value.(Subscriber); ok {
subscriber.ReceiveMessage(m)
}
return true
})
}
}
subscribe:
func (u useCase) Subscribe(ctx context.Context, topicName vos.TopicName) (chan vos.Message, vos.SubscriberID, error) {
if err := topicName.Validate(); err != nil {
return nil, "", err
}
topic, err := u.storage.GetTopic(ctx, topicName)
if err != nil {
if !errors.Is(err, entities.ErrTopicNotFound) {
return nil, "", err
}
topic, err = u.createTopic(ctx, topicName)
if err != nil {
return nil, "", err
}
subscriber := entities.NewSubscriber(topic)
subscriptionCh, id := subscriber.Subscribe()
return subscriptionCh, id, nil
}
subscriber := entities.NewSubscriber(topic)
subscriptionCh, id := subscriber.Subscribe()
return subscriptionCh, id, nil
}
func (s Subscriber) Subscribe() (chan vos.Message, vos.SubscriberID) {
s.topic.addSubscriber(s)
return s.subscriptionCh, s.GetID()
}
func (s Subscriber) ReceiveMessage(msg vos.Message) {
s.subscriptionCh <- msg
}
publisher :
func (u useCase) Publish(ctx context.Context, message vos.Message) error {
if err := message.Validate(); err != nil {
return err
}
topic, err := u.storage.GetTopic(ctx, message.TopicName)
if err != nil {
return err
}
topic.Dispatch(message)
return nil
}
when I call subscribe (I send a message to a subscribe to channel and add a subscribe to my thread) when I post a message to a topic I send a message to topic channel
Some points are missing from the code you show, such as the code for .Subscribe() and .Publish(), or how the channels are instanciated (are they buffered/unbuffered ?).
One point can be said, though :
from the looks of (t *Topic) listenForSubscriptions() : this subscribing method does not send any signal to the subscriber that it has been registered.
So my guess is : your useCase.Subscribe(...) call has the information that the created channel has been written on newSubCH, but it hasn't got the inforamtion that t.Subcribers.Store(...) has completed.
So, depending on how the goroutines are scheduled, the message sending in your test function can occur before the channel has actually been registered.
To fix this, you add something that will send a signal back to the caller. One possible way :
type subscribeReq struct{
ch chan Message
done chan struct{}
}
// turn Topic.newSubCh into a chan *subscribeReq
func (t *Topic) listenForSubscriptions() {
for req := range t.newSubCh {
t.Subscribers.Store(newSubCh.GetID(), req.ch)
close(req.done)
}
}
Another point : your test function does not check if the goroutine spun with your go func(){ ... }() call completes at all, so your unit test process may also exit before the goroutine has had the chance to execute fmt.Println(msg).
A common way to check this is to use a sync.WaitGroup :
t.Run(tt.name, func(t *testing.T) {
...
useCase := New(tt.fields.storage)
subscribed := make(chan struct{})
wg := &sync.WaitGroup{} // create a *sync.WaitGroup
wg.Add(1) // increment by 1 (you start only 1 goroutine)
go func() {
defer wg.Done() // have the goroutine call wg.Done() when returning
...
}()
// send message, check that no error occurs
wg.Wait() // block here until the goroutine has completed
})

Why this websocket server does not broadcast to connected client?

Below code is a websocket broadcast server that read from a specific connection then broadcasting it to connected clients.
But this server does not broadcast despite there is no error and warnings.
Why this server does not broadcast?
In this code self.KabucomConn is origin socket so read from this socket, then broadcast to client which stored in Hub.RClients.
When new connection established, passing connection object with register channel to Hub, then Hub adds a client to RClient that stored client object.
package main
import (
"log"
"net/http"
"net/url"
"github.com/gorilla/websocket"
)
var upgrader = websocket.Upgrader{
CheckOrigin: func(r *http.Request) bool {
return true
},
}
type Client struct {
hub *Hub
Conn *websocket.Conn
Send chan []byte
}
func (self *Client) writepump() {
for {
select {
case message := <-self.Send:
w, err := self.Conn.NextWriter(websocket.TextMessage)
if err != nil {
log.Print("writepump: nextwriter error")
}
w.Write(message)
w.Close()
}
}
}
type Hub struct {
RClients map[*Client]bool
KabucomConn *websocket.Conn
register chan *Client
unregister chan *Client
kabucomchan chan []byte
url url.URL
}
func (self *Hub) KabucomRun() {
for {
_, b, err := self.KabucomConn.ReadMessage() // read data from origin data connection
log.Println("read message")
if err != nil {
log.Println(err)
self.KabucomConn.Close()
for i := 1; i <= 5; i++ { //retry re-connect up to 5 times
self.KabucomConn, _, err = websocket.DefaultDialer.Dial(self.url.String(), nil)
if i >= 5 && err != nil {
log.Fatal(err)
} else if err != nil {
log.Println(err, "try", i)
continue
} else {
break
}
}
log.Println("conti")
continue
}
log.Println(b)
self.kabucomchan <- b
}
}
func (self *Hub) Run() {
defer func() {
for c, _ := range self.RClients {
close(c.Send)
c.Conn.Close()
}
}()
for {
select {
case message := <-self.kabucomchan:
log.Println("kabucomchan")
log.Println(message)
for c, _ := range self.RClients {
c.Send <- message
}
case c := <-self.register:
log.Println("reg")
self.RClients[c] = true
case c := <-self.unregister:
log.Println("unreg")
delete(self.RClients, c)
close(c.Send)
}
}
}
func newHub() *Hub {
u := url.URL{Scheme: "ws", Host: "192.168.1.8:20063", Path: "/ws"}
var conn *websocket.Conn
for i := 1; i <= 5; i++ {
d, _, err := websocket.DefaultDialer.Dial(u.String(), nil)
if err != nil && i < 5 {
log.Println(err)
continue
} else if i >= 5 {
log.Println("Hub: Kabucom connection error")
}
conn = d
break
}
return &Hub{RClients: make(map[*Client]bool), register: make(chan *Client), KabucomConn: conn}
}
func handler(w http.ResponseWriter, r *http.Request, hub *Hub) {
conn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
return
}
client := &Client{Conn: conn, hub: hub, Send: make(chan []byte, 256)}
go client.writepump()
hub.register <- client
}
func main() {
hub := newHub()
go hub.Run()
go hub.KabucomRun()
http.HandleFunc("/ws", func(w http.ResponseWriter, r *http.Request) {
handler(w, r, hub)
})
log.Println(":20021/ws")
http.ListenAndServe(":20021", nil)
}
Because you are not initializing Hub.kabucomchan.
func newHub() *Hub {
//...
return &Hub{RClients: make(map[*Client]bool), register: make(chan *Client), KabucomConn: conn, /* kabucomchan ??? */}
}
Send and receive operations on channels assume that both sender c<- and receiver <-c hold a reference to the same channel, but when the channel is nil this reference doesn't exist, and the send and receive just block forever.
Properly initialize the channel in the Hub constructor:
return &Hub{
RClients: make(map[*Client]bool),
register: make(chan *Client),
KabucomConn: conn,
kabucomchan: make(chan []byte, /* buffered? */), // <--- !
}

How to explain the go routine as callback function in this example code

I am confusing the following code, I didn't see the 1st go routine has any returns in TimelineItemStream function, and 2nd go routine call the unsub. In Subscriber interface define the unsub callbackup function.
the code is come from git clone https://github.com/nicolasparada/nakama.git
// TimelineItem model
type TimelineItem struct {
ID string `json:"id"`
UserID string `json:"-"`
PostID string `json:"-"`
Post *Post `json:"post,omitempty"`
}
// Subscriber interface.
type Subscriber interface {
Sub(topic string, cb func(data []byte)) (unsub func() error, err error)
}
func (s *Service) TimelineItemStream(ctx context.Context) (<-chan TimelineItem, error) {
uid, ok := ctx.Value(KeyAuthUserID).(string)
if !ok {
return nil, ErrUnauthenticated
}
tt := make(chan TimelineItem)
unsub, err := s.pubsub.Sub(timelineTopic(uid), func(data []byte) {
go func(r io.Reader) {
var ti TimelineItem
err := gob.NewDecoder(r).Decode(&ti)
if err != nil {
log.Printf("could not gob decode timeline item: %v\n", err)
return
}
tt <- ti
}(bytes.NewReader(data))
})
if err != nil {
return nil, fmt.Errorf("could not subscribe to timeline: %w", err)
}
go func() {
<-ctx.Done()
if err := unsub(); err != nil {
log.Printf("could not unsubcribe from timeline: %v\n", err)
// don't return
}
close(tt)
}()
return tt, nil
}
In the block of call
unsub, err := s.pubsub.Sub(timelineTopic(uid), func(data []byte) {
go func(r io.Reader) {
var ti TimelineItem
err := gob.NewDecoder(r).Decode(&ti)
if err != nil {
log.Printf("could not gob decode timeline item: %v\n", err)
return
}
tt <- ti
}(bytes.NewReader(data))
})
s.pubsub.Sub returns a callback function, but in this block go routine has not any return , how to explain the block of code

Pattern for fetching multiple fields in parallel

I need to fetch multiple fields in parallel for my system from external services (in this example, simulated by Name(), Age() and CanDrive() methods).
The fetchUser() method does what I want, but it seems too verbose specially if you consider I could have 10+ fields. Are there better ways I can implement this?
playground: https://play.golang.org/p/90sNq1GmrD8
Code (same as in playground):
package main
import (
"fmt"
"sync"
)
type User struct {
Name string
Age int
CanDrive *bool
}
func Name() (string, error) {
return "foobar", nil
}
func Age() (int, error) {
return 25, nil
}
func CanDrive() (bool, error) {
return true, nil
}
func fetchUser() (*User, error) {
var wg sync.WaitGroup
errs := make(chan error)
user := &User{}
wg.Add(1)
go func() {
var err error
defer wg.Done()
user.Name, err = Name()
errs <- err
}()
wg.Add(1)
go func() {
var err error
defer wg.Done()
user.Age, err = Age()
errs <- err
}()
wg.Add(1)
go func() {
defer wg.Done()
canDrive, err := CanDrive()
if err == nil {
user.CanDrive = &canDrive
}
errs <- err
}()
// wait until all go-routines are completed successfully
// if that's the case, close the errs channel
go func() {
wg.Wait()
close(errs)
}()
// keep waiting for errors (or for the error channel to be closed
// if all calls succeed)
for err := range errs {
if err != nil {
return nil, err
}
}
return user, nil
}
func main() {
user, _ := fetchUser()
fmt.Println(user)
}
Without knowing more of the specifics of your scenario, my only suggestion would be to separate out the Go routine error handling into another package.
Fortunately, a package already exists that does the same thing, named errgroup. Below is an implementation of your original code using the errgroup package:
package main
import (
"context"
"fmt"
"golang.org/x/sync/errgroup"
)
type User struct {
Name string
Age int
CanDrive *bool
}
func Name() (string, error) {
return "foobar", nil
}
func Age() (int, error) {
return 25, nil
}
func CanDrive() (bool, error) {
return true, nil
}
func fetchUser(ctx context.Context) (*User, error) {
group, ctx := errgroup.WithContext(ctx)
user := &User{}
group.Go(func() (err error) {
user.Name, err = Name()
return
})
group.Go(func() (err error) {
user.Age, err = Age()
return
})
group.Go(func() error {
canDrive, err := CanDrive()
if err == nil {
user.CanDrive = &canDrive
}
return err
})
if err := group.Wait(); err != nil {
return nil, err
}
return user, nil
}
func main() {
user, err := fetchUser(context.Background())
fmt.Println(user, err)
}

Golang Gorilla Websocket stops receiving information at 120 seconds

I'm currently trying to connect to the CEX.IO bitcoin exchange's websocket, but have been having issues not only with CEX.IO but with others too. All of my connections drop around the 120-second mark which makes me think there is some TTL problem going on. The Process() goroutine in the main package ends up just hanging and waiting for data from the readLoop which just stops receiving data. I've included some read-only API keys in the code so you can test if you'd like.
package main
import (
"fmt"
"bitbucket.org/tradedefender/cryptocurrency/exchange-connector/cexio"
"github.com/shopspring/decimal"
"encoding/json"
"time"
)
type OrderBook struct {
Asks []Ask
Bids []Bid
}
type Ask struct {
Rate decimal.Decimal
Amount decimal.Decimal
}
type Bid struct {
Rate decimal.Decimal
Amount decimal.Decimal
}
func main() {
cexioConn := new(cexio.Connection)
err := cexioConn.Connect()
if err != nil {
fmt.Errorf("error: %s", err.Error())
}
err = cexioConn.Authenticate("TLwYkktLf7Im6nqSKt6UO1IrU", "9ImOJcR7Qj3LMIyPCzky0D7WE")
if err != nil {
fmt.Errorf("error: %s", err.Error())
}
readChannel := make(chan cexio.IntraAppMessage, 25)
go cexioConn.ReadLoop(readChannel)
processor := Processor{
WatchPairs: [][2]string{
[2]string{
"BTC", "USD",
},
},
conn: cexioConn,
}
go processor.Process(readChannel)
// LOL
for {
continue
}
}
type Processor struct {
WatchPairs [][2]string
conn *cexio.Connection
}
func (p *Processor) Process(ch <-chan cexio.IntraAppMessage) {
p.conn.SubscribeToOrderBook(p.WatchPairs[0])
pingTimer := time.Now().Unix()
for {
fmt.Printf("(%v)\n", time.Now().Unix())
if (time.Now().Unix() - pingTimer) >= 10 {
fmt.Println("sending ping")
p.conn.SendPing()
pingTimer = time.Now().Unix()
}
readMsg := <- ch
output, _ := json.Marshal(readMsg.SocketMessage)
fmt.Println(string(output))
if readMsg.SocketMessage.Event == "ping" {
fmt.Println("sending pong")
p.conn.SendPong()
pingTimer = time.Now().Unix()
}
}
}
Below is the connector to the cexio websocket. Here is a link to their API: https://cex.io/websocket-api
package cexio
import (
"github.com/gorilla/websocket"
//"github.com/shopspring/decimal"
"github.com/satori/go.uuid"
"encoding/hex"
"encoding/json"
"crypto/hmac"
"crypto/sha256"
"bytes"
"strconv"
"time"
"fmt"
)
const Url = "wss://ws.cex.io/ws/"
type Connection struct {
conn *websocket.Conn
}
type IntraAppMessage struct {
SocketMessage GenericMessage
ProgramMessage ProgramMessage
}
type GenericMessage struct {
Event string `json:"e"`
Data interface{} `json:"data"`
Auth AuthData `json:"auth,omitempty"`
Ok string `json:"ok,omitempty"`
Oid string `json:"oid,omitempty"`
Time int64 `json:"time,omitempty"`
}
type ProgramMessage struct {
Error string
}
type AuthData struct {
Key string `json:"key"`
Signature string `json:"signature"`
Timestamp int64 `json:"timestamp"`
}
type OrderBookSubscribeData struct {
Pair [2]string `json:"pair"`
Subscribe bool `json:"subscribe"`
Depth int `json:"depth"`
}
func (c *Connection) SendPong() error {
pongMsg := GenericMessage{
Event: "pong",
}
err := c.conn.WriteJSON(pongMsg)
if err != nil {
return nil
}
deadline := time.Now().Add(15*time.Second)
err = c.conn.WriteControl(websocket.PongMessage, nil, deadline)
if err != nil {
return err
}
return nil
}
func (c *Connection) SendPing() error {
pingMsg := GenericMessage{
Event: "get-balance",
Oid: uuid.NewV4().String(),
}
err := c.conn.WriteJSON(pingMsg)
if err != nil {
return err
}
deadline := time.Now().Add(15*time.Second)
err = c.conn.WriteControl(websocket.PingMessage, nil, deadline)
if err != nil {
return err
}
return nil
}
func (c *Connection) Connect() error {
dialer := *websocket.DefaultDialer
wsConn, _, err := dialer.Dial(Url, nil)
if err != nil {
return err
}
c.conn = wsConn
//c.conn.SetPingHandler(c.HandlePing)
for {
_, msgBytes, err := c.conn.ReadMessage()
if err != nil {
c.Disconnect()
return err
}
fmt.Println(string(msgBytes))
var m GenericMessage
err = json.Unmarshal(msgBytes, &m)
if err != nil {
c.Disconnect()
return err
}
if m.Event != "connected" {
c.Disconnect()
return err
} else {
break
}
}
return nil
}
func (c *Connection) Disconnect() error {
return c.conn.Close()
}
func (c *Connection) ReadLoop(ch chan<- IntraAppMessage) {
for {
fmt.Println("starting new read")
_, msgBytes, err := c.conn.ReadMessage()
if err != nil {
ch <- IntraAppMessage{
ProgramMessage: ProgramMessage{
Error: err.Error(),
},
}
continue
}
var m GenericMessage
err = json.Unmarshal(msgBytes, &m)
if err != nil {
ch <- IntraAppMessage{
ProgramMessage: ProgramMessage{
Error: err.Error(),
},
}
continue
}
ch <- IntraAppMessage{
SocketMessage: m,
}
}
}
func CreateSignature(timestamp int64, key, secret string) string {
secretBytes := []byte(secret)
h := hmac.New(sha256.New, secretBytes)
var buffer bytes.Buffer
buffer.WriteString(strconv.FormatInt(timestamp, 10))
buffer.WriteString(key)
h.Write(buffer.Bytes())
return hex.EncodeToString(h.Sum(nil))
}
func (c *Connection) Authenticate(key, secret string) error {
timestamp := time.Now().Unix()
signature := CreateSignature(timestamp, key, secret)
var authMsg GenericMessage
authMsg.Event = "auth"
authMsg.Auth = AuthData{
Key: key,
Signature: signature,
Timestamp: timestamp,
}
err := c.conn.WriteJSON(authMsg)
if err != nil {
return err
}
for {
_, msgBytes, err := c.conn.ReadMessage()
if err != nil {
c.Disconnect()
return err
}
fmt.Println(string(msgBytes))
var m GenericMessage
err = json.Unmarshal(msgBytes, &m)
if err != nil {
c.Disconnect()
return err
}
if m.Event != "auth" && m.Ok != "ok" {
c.Disconnect()
return err
} else {
break
}
}
return nil
}
func (c *Connection) SubscribeToOrderBook(pair [2]string) error {
sendMsg := GenericMessage{
Event: "order-book-subscribe",
Data: OrderBookSubscribeData{
Pair: pair,
Subscribe: true,
Depth: 0,
},
Oid: uuid.NewV4().String(),
}
err := c.conn.WriteJSON(sendMsg)
if err != nil {
return err
}
return nil
}
func (c *Connection) GetBalance() error {
sendMsg := GenericMessage{
Event: "get-balance",
Oid: uuid.NewV4().String(),
}
err := c.conn.WriteJSON(sendMsg)
if err != nil {
return err
}
return nil
}
Solution was to remove the
for {
continue
}
at the end of the main function

Resources