In the GetList function the client sends a message and the StreamReply returns RowValue and boolean value from a loop one by one. How can I implement it with gRPC stream?
test.proto
service GetList {
rpc Listdata (StreamRequest) returns (stream StreamReply){}
}
message StreamRequest{
string StreamName= 1;
bool boolean= 3;
}
message StreamReply{
string RowValue=1;
bool boolean = 2;
}
The problem lies in my go function. I don't understand how to simply stream values using stream.Send()
server.go
func (s *server) GetList(in *pb.StreamRequest, stream pb.GetList_ListdataServer) error {
strings:= "random strings"
count := 5
for i := 0; i < count; i++ {
if err := stream.Send(strings, true); err != nil {
return err
}
}
return nil
}
Related
I'm making an http request in golang to an external api. It gives a general response of {"error":[]string, "result":changing interface{}}. depending on the function that is making the request, the Result field changes. Since I know the structure of the Result field for each function I run, I want to be able to change the value of Result before unmarshalling to json. I've tried to do this with the following code:
func GetAssets(output *Resp, resultType interface{}) error {
return publicRequest("/Assets", output, resultType)
}
func publicRequest(endPoint string, output *Resp, resultType interface{}) error {
url := Rest_url + Pub_rest_url + endPoint //"https://api.kraken.com/0/public/Assets in this case
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
output.Result = resultType
return json.NewDecoder(resp.Body).Decode(&output)
}
Here is how it's being ran in main
type Resp struct {
Error []string `json:"error"`
Result interface{} `json:"result"`
}
type AssetInfo struct {
Aclass string `json:"aclass"`
Altname string `json:"altname"`
Decimals int `json:"decimals"`
Display int `json:"display_decimals"`
}
func main() {
var result map[string]AssetInfo
jsonData := Resp{}
rest_api_client.GetAssets(&jsonData, result)
fmt.Println(jsonData)
}
The issue is that it doesn't unmarshal correctly. A map is created for each asset, but the data contained inside of each asset is also being stored inside of a map. I'm not sure if I explained this well, but here is the current response after unmarshalling to understand what I mean.
Here is the data type of Resp.Result: map[string]interface {}
{[] map[1INCH:map[aclass:currency altname:1INCH decimals:10 display_decimals:5] AAVE:map[aclass:currency altname:AAVE decimals:10 display_decimals:5] ACA:map[aclass:currency altname:ACA decimals:10 display_decimals:5] ADA:map[aclass:currency altname:ADA decimals:8 display_decimals:6]...}
The response type I'm looking for is map[string]AssetInfo. Hopefully it could be unmarshalled like this:
{[] map[1INCH:{currency 1INCH 10 5} AAVE:{currency AAVE 10 5} ACA:{currency ACA 10 5} ADA:{currency ADA 8 6} ADA.S:{currency ADA.S 8 6}...}
Any help? I'd rather keep the Resp struct as generic as possible and just change the value of the Result field (if this is even possible to do correctly) since I plan to have multiple functions that call different endpoints of the api, and they'll all have the same underlying response type of the Resp struct with different Result types
You can view a working example in the following repo:
https://github.com/alessiosavi/GoArbitrage/blob/e107af466852b1ed30c2413eb4401595f7412b4f/markets/kraken/kraken.go
Basically, I've defined the following structure:
type Tickers struct {
Error []interface{} `json:"error"`
Result map[string]Ticker `json:"result"`
}
type Ticker struct {
Aclass string `json:"aclass"`
Altname string `json:"altname"`
Decimals int `json:"decimals"`
DisplayDecimals int `json:"display_decimals"`
}
Than I execute the request in the following way:
const KRAKEN_TICKERS_URL string = `https://api.kraken.com/0/public/Assets`
type Kraken struct {
PairsNames []string `json:"pairs_name"`
Pairs map[string]datastructure.KrakenPair `json:"pairs"`
OrderBook map[string]datastructure.KrakenOrderBook `json:"orderbook"`
MakerFee float64 `json:"maker_fee"`
TakerFees float64 `json:"taker_fee"`
// FeePercent is delegated to save if the fee is in percent or in coin
FeePercent bool `json:"fee_percent"`
Tickers []string
}
// Init is delegated to initialize the maps for the kraken
func (k *Kraken) Init() {
k.Pairs = make(map[string]datastructure.KrakenPair)
k.OrderBook = make(map[string]datastructure.KrakenOrderBook)
k.SetFees()
}
// SetFees is delegated to initialize the fee type/amount for the given market
func (k *Kraken) SetFees() {
k.MakerFee = 0.16
k.TakerFees = 0.26
k.FeePercent = true
}
func (k *Kraken) GetTickers() error {
res := datastructure.Tickers{}
var err error
var request req.Request
var data []byte
var tickers []string
resp := request.SendRequest(KRAKEN_TICKERS_URL, "GET", nil, nil, false, 10*time.Second)
if resp.Error != nil {
zap.S().Debugw("Error during http request. Err: " + resp.Error.Error())
return resp.Error
}
if resp.StatusCode != 200 {
zap.S().Warnw("Received a non 200 status code: " + strconv.Itoa(resp.StatusCode))
return errors.New("NON_200_STATUS_CODE")
}
data = resp.Body
if err = json.Unmarshal(data, &res); err != nil {
zap.S().Warn("ERROR! :" + err.Error())
return err
}
zap.S().Infof("Data: %v", res.Result)
tickers = make([]string, len(res.Result))
i := 0
for key := range res.Result {
tickers[i] = res.Result[key].Altname
i++
}
k.Tickers = tickers
return nil
}
There is a function that is running in goroutine:
func (c *controlUC) WebhookPool() {
for {
if len(c.webhookPool) == 0 {
continue
}
for i := 0; i < len(c.webhookPool); i++ {
if !c.webhookPool[i].LastSentTime.IsZero() && time.Now().Before(c.webhookPool[i].LastSentTime.Add(GetDelayBySentCount(c.webhookPool[i].SendCount))) {
continue
}
var headers = make(map[string]string)
headers["Content-type"] = "application/json"
_, statusCode, err := c.fhttpClient.Request("POST", c.webhookPool[i].Path, c.webhookPool[i].Body, nil, headers)
if err != nil {
c.logger.Error(err)
return
}
if statusCode != 200 {
if c.webhookPool[i].SendCount >= 2 {
c.webhookPool = append(c.webhookPool[:i], c.webhookPool[i+1:]...)
i--
continue
}
c.webhookPool[i].SendCount++
} else {
c.webhookPool = append(c.webhookPool[:i], c.webhookPool[i+1:]...)
i--
continue
}
c.webhookPool[i].LastSentTime = time.Now()
}
}
}
// webhookPool []models.WebhookPoolElem
type WebhookPoolElem struct {
SendCount int
LastSentTime time.Time
Path string
Body []byte
}
The webhookPoolElem element is added to c.webhookpool, after which a request is sent to the server (the path is taken from WebhookPoolElem.path). If the server returned a non - 200 200, then I need to send the request again, after X seconds (taken from GetDelayBySentCount(), depending on SendCount returns different times). The number of attempts is limited (c.webhookpool[i].SendCount >= 2)
But maybe this function needs to be done through channels? If so, how?
Lets say controlUC receiver has a field webhookPool chan WebhookPoolElem and init as webhookPool: make(chan WebhookPoolElem, n) with n as buffer.
You can receive elements and more or less replace c.webhookPool[i] to elem. Rewrite like this:
func (c *controlUC) WebhookPool() {
for {
elem, open := <-c.webhookPool
if !open {
return
}
if !elem.LastSentTime.IsZero() && time.Now().Before(elem.LastSentTime.Add(GetDelayBySentCount(elem.SendCount))) {
continue
}
// I omit http request
if statusCode != 200 {
if elem.SendCount >= 2 {
// drop message from channel, no need to do anything
continue
}
elem.SendCount++
elem.LastSentTime = time.Now()
c.webhookPool <- elem // enqueue again
}
}
I suggest buffered channel so the last send c.webhookPool <- elem does not block, but it's best if you place the send in a select so if the send can not proceed regardless of the buffer, the goroutine doesn't block:
select {
case c.webhookPool <- elem:
// success
default:
// can not send
}
I am trying to create a grpc server streaming endpoint.
Here is my protobuf file
syntax = "proto3";
option go_package = "mirror_streampb";
option java_package = "com.mirror_stream";
option java_outer_classname = "StreamIdsProto";
option java_multiple_files = true;
package mirror_stream;
service StreamIDs {
rpc ListIDs(ListIDsRequest) returns (stream ListIDsResponse) {}
}
message ListIDsRequest {
int32 num = 1;
}
message ListIDsResponse {
int32 num = 1;
int32 id = 2;
}
And here is my golang implementation for that method. Which only returns some random numbers.
func (s *Server) ListIDs(req *streampb.ListIDsRequest, stream streampb.StreamIDs_ListIDsServer) error {
for i := int32(0); i < req.Num; i++ {
resp := &streampb.ListIDsResponse{
Num: i,
Id: int32(rand.Intn(10000000)),
}
if err := stream.Send(resp); err != nil {
return err
}
}
return nil
}
So when I try to call that method, I get this error Failed while making call: code:unknown message:grpc: client streaming protocol violation: get <nil>, want <EOF>
I am not sure why it is coming from and where it is coming from.
Can anyone help me figure this out?
I'm using grpc in my go project. Below is code:
example.proto:
syntax = "proto3";
message Example {
string message = 1;
google.protobuf.Any details = 2;
}
main.go
func logMessage (m string, d interface{}) {
message := & example.message{
message: m,
details: ??
}
log(&message)
}
But I'm not sure how to deal with the details(interface{}) field. I know I can use any type for interface, but not sure how to use it here. Anyone can help? Thanks
Since protobuf/ptypes is deprecated, it worth using anypb.UnmarshalTo
import (
"google.golang.org/protobuf/types/known/anypb"
"github.com/golang/protobuf/ptypes/any"
)
func Unmarshal(data *any.Any) (*YourMessage, err) {
var m YourMessage
err := anypb.UnmarshalTo(data, &m, proto.UnmarshalOptions{})
return &m,err
}
The protobuf/ptypes package has utilities to convert to/from arbitrary proto messages to any:
MarshalAny:
func MarshalAny(m proto.Message) (*anypb.Any, error)
MarshalAny marshals the given message m into an anypb.Any message.
UnmarshalAny:
func UnmarshalAny(any *anypb.Any, m proto.Message) error
UnmarshalAny unmarshals the encoded value contained in the anypb.Any message into the provided message m. It returns an error if the target message does not match the type in the Any message or if an unmarshal error occurs.
In your example, you would use something along the lines of:
func logMessage (m string, d proto.Message) {
details, err := ptypes.MarshalAny(d)
if err != nil {
panic(err)
}
message := & example.message{
message: m,
details: details
}
log(&message)
}
func pbany(v interface{}) (*anypb.Any, error) {
pv, ok := v.(proto.Message)
if !ok {
return &anypb.Any{}, fmt.Errorf("%v is not proto.Message", pv)
}
return anypb.New(pv)
use anypb.New api, In your code, pass d to pbany function
func interfaceToAny(v interface{}) (*anypb.Any, error) {
bytes, err := json.Marshal(v)
if err != nil {
println("error json.Marshal interfaceToAny")
return nil, err
}
m := api.Bytes{B: bytes}
return anypb.New(&m)
}
message Bytes {
bytes b = 1;
}
I was wondering if someone can explain this syntax to me. In the google maps go api, they have
type Client struct {
httpClient *http.Client
apiKey string
baseURL string
clientID string
signature []byte
requestsPerSecond int
rateLimiter chan int
}
// NewClient constructs a new Client which can make requests to the Google Maps WebService APIs.
func NewClient(options ...ClientOption) (*Client, error) {
c := &Client{requestsPerSecond: defaultRequestsPerSecond}
WithHTTPClient(&http.Client{})(c) //???????????
for _, option := range options {
err := option(c)
if err != nil {
return nil, err
}
}
if c.apiKey == "" && (c.clientID == "" || len(c.signature) == 0) {
return nil, errors.New("maps: API Key or Maps for Work credentials missing")
}
// Implement a bursty rate limiter.
// Allow up to 1 second worth of requests to be made at once.
c.rateLimiter = make(chan int, c.requestsPerSecond)
// Prefill rateLimiter with 1 seconds worth of requests.
for i := 0; i < c.requestsPerSecond; i++ {
c.rateLimiter <- 1
}
go func() {
// Wait a second for pre-filled quota to drain
time.Sleep(time.Second)
// Then, refill rateLimiter continuously
for _ = range time.Tick(time.Second / time.Duration(c.requestsPerSecond)) {
c.rateLimiter <- 1
}
}()
return c, nil
}
// WithHTTPClient configures a Maps API client with a http.Client to make requests over.
func WithHTTPClient(c *http.Client) ClientOption {
return func(client *Client) error {
if _, ok := c.Transport.(*transport); !ok {
t := c.Transport
if t != nil {
c.Transport = &transport{Base: t}
} else {
c.Transport = &transport{Base: http.DefaultTransport}
}
}
client.httpClient = c
return nil
}
}
And this is the line I don't understand in NewClient
WithHTTPClient(&http.Client{})(c)
Why are there two ()()?
I see that WithHTTPClient takes in a *http.Client which that line does, but then it also passes in a pointer to the client struct declared above it?
WithHTTPClient returns a function, ie:
func WithHTTPClient(c *http.Client) ClientOption {
return func(client *Client) error {
....
return nil
}
}
WithHTTPClient(&http.Client{})(c) is just calling that function with c (a pointer to a Client) as parameter. It could be written as:
f := WithHTTPClient(&http.Client{})
f(c)