I have a Go GRPC server-side streaming function:
func (server *Server) GetClients(req *iam.GetClientsRequest, client iam.IAM_GetClientsServer) error {
ctx := client.(interface{ Context() context.Context }).Context()
userID, err := getUserIDStream(client)
if err != nil {
return err
}
clients, err := server.db.QueryByUserID(ctx, userID)
if err != nil {
return grpc.Errorf(codes.Internal, apiutils.ServerError)
}
for _, value := range clients {
converted, err := server.fromInternalClient(value)
if err != nil {
return err
}
if err := client.Send(converted); err != nil {
return err
}
}
return nil
}
and I'm testing it like this:
It("GetClients - Send fails - Error", func() {
handler := createHandler(db)
lis := bufconn.Listen(bufSize)
server := grpc.NewServer()
iam.RegisterIAMServer(server, NewServer(handler))
go func() {
if err := server.Serve(lis); err != nil {
log.Fatalf("Server exited with error: %v", err)
}
}()
defer lis.Close()
defer server.GracefulStop()
conn, err := grpc.DialContext(context.Background(), "bufnet",
grpc.WithContextDialer(createBufDialier(lis)), grpc.WithInsecure())
Expect(err).ShouldNot(HaveOccurred())
defer conn.Close()
client := iam.NewIAMClient(conn)
cclient, _ := client.GetClients(addAccessToken(context.Background()), new(iam.GetClientsRequest))
resp, err := cclient.Recv()
Expect(resp).Should(BeNil())
Expect(err).Should(HaveOccurred())
Expect(err.Error()).Should(Equal(message))
})
My issue is that I'm not sure how to induce a failure on Send so I can test the response. Since I'm using an actual test server and client, I can't just mock out the object and I'd prefer not to go that route anyway. Is there a way I can do this?
Originally, I was trying to force Send to fail by setting bufSize to an artificially low value. However, this wasn't producing an error so I decided to try modifying the maxSendMessageSize on the server:
opts := []grpc.ServerOption{}
if sendFails {
opts = append(opts, grpc.MaxSendMsgSize(10))
}
lis := bufconn.Listen(bufSize)
server := grpc.NewServer(opts...)
And this worked in producing the error.
Related
I'm trying to make a music app that sends file through tcp protocol using go and microservice architecture. Now I'm creating a player service that should:
Get user token and get claims from it
Check is user exists using claims and user_service microservice
Get song from redis
Check is song exists using music_service
Read file by chunks and send it to client using tcp
Redis data looks like this:
{
"user_id": [{
"song_id": "<song_id>"
}]
}
But I faced with a small problem. My music files stored in a flac format and when I receive it on the client, my player doesn't play it. I don't really know what can be the problem. So here's my code:
SERVER
service_setup.go
//this function is called in main function
func setService() {
ln, err := net.Listen("tcp", config.TCPAddress)
if err != nil {
panic("couldn't start tcp server")
}
defer ln.Close()
for {
conn, err := ln.Accept()
if err != nil {
logger.ErrorLog(fmt.Sprintf("Error: couldn't accept connection. Details: %v", err))
return
}
service.DownloadSong(conn)
}
}
downloader_service.go
func DownloadSong(conn net.Conn) {
token, err := bufio.NewReader(conn).ReadString('\n')
if err != nil {
logger.ErrorLog(fmt.Sprintf("Error: couldn't get token. Details: %v", token))
conn.Close()
return
}
claims, err := jwt_funcs.DecodeJwt(token)
if err != nil {
conn.Close()
return
}
songs, err := redis_repo.Get(claims.Id)
if err != nil {
conn.Close()
return
}
for _, song := range songs {
download(song, conn)
}
}
func download(song models.SongsModel, conn net.Conn) {
filePath, err := filepath.Abs(fmt.Sprintf("./songs/%s.flac", song.SongId))
if err != nil {
logger.ErrorLog(fmt.Sprintf("Errror: couldn't create filepath. Details: %v", err))
conn.Close()
return
}
file, err := os.Open(filePath)
defer file.Close()
if err != nil {
logger.ErrorLog(fmt.Sprintf("Errror: couldn't open file. Details: %v", err))
conn.Close()
return
}
read(file, conn)
}
func read(file *os.File, conn net.Conn) {
reader := bufio.NewReader(file)
buf := make([]byte, 15)
defer conn.Close()
for {
_, err := reader.Read(buf)
if err != nil && err == io.EOF {
logger.InfoLog(fmt.Sprintf("Details: %v", err))
fmt.Println()
return
}
conn.Write(buf)
}
}
CLIENT
main.go
func main() {
conn, _ := net.Dial("tcp", "127.0.0.1:6060")
var glMessage []byte
text := "eyJhbGciOiJFUzI1NiIsInR5cCI6IkpXVCJ9.eyJpZCI6IjYzYzlhNmE1OWI3ZmQyNTQ2ZjA4ZWEyYSIsInVzZXJuYW1lIjoiMTIiLCJleHAiOjE2NzQyMTE5ODl9.aarSDhrFF1df3i2pIRyjNxTfSHKObqLU3kHJiPreredIhLNCzs7z7jMgRHQIcLaIvCOECN7bX0OaSvKdW7VKsQ\n"
fmt.Fprint(conn, text)
reader := bufio.NewReader(conn)
b := make([]byte, 15)
c := 0
for i, _ := reader.Read(b); int(i) != 0; i, _ = reader.Read(b) {
c += i
glMessage = append(glMessage, b...)
}
os.WriteFile("./test.flac", glMessage, 0644)
}
If you know what can be the problem, please tell me. I'd really appreciate it!
It looks like you're trying to send the music file over the network in 15 byte chunks, which is likely not enough to play the song on the client side.
You can try increasing the chunk size, for example, to 8192 bytes. To do this, replace buf := make([]byte, 15) with buf := make([]byte, 8192).
Also, it's better to write the received data directly to the file rather than storing it in memory. You can do this by creating a file and using os.Create to write the received data to it:
file, err := os.Create("./test.flac")
if err != nil {
fmt.Println("Error: couldn't create file")
return
}
defer file.Close()
for {
i, err := reader.Read(buf)
if err != nil && err == io.EOF {
break
}
file.Write(buf[:i])
}
I believe that this can solve the issue.
I have a weird issue with this code:
func PrepareFileUpload(filePath, url string) (*http.Request, error) {
pr, pw := io.Pipe()
mpw := multipart.NewWriter(pw)
go func() {
defer pw.Close()
part, err := mpw.CreateFormFile("file", filepath.Base(filePath))
if err != nil {
return
}
file, err := os.Open(filePath)
if err != nil {
return
}
defer file.Close()
if _, err = io.Copy(part, file); err != nil {
return
}
err = mpw.Close()
if err != nil {
return
}
}()
req, err := http.NewRequest("POST", url, pr)
req.Header.Set("Content-Type", mpw.FormDataContentType())
return req, err
}
which I use like this:
filePath := "foo.bar"
s := []byte("Test file")
ioutil.WriteFile(filePath, s, 0644)
values := url.Values{}
values.Set("folderid", "123456")
values.Set("filename", filepath.Base(filePath))
values.Set("nopartial", "1")
u := url.URL{
Scheme: "https",
Host: "eapi.pcloud.com",
Path: "/uploadfile",
RawQuery: values.Encode(),
}
req, err := PrepareFileUpload(filePath, u.String())
if err != nil {
log.Fatal(err)
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", "ACCESS_TOKEN"))
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
retData, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(retData))
For some reason, when used with the pCloud API, this hangs when running http.DefaultClient.Do(req). I have tried creating my own test server in Go, and there are no issues there, so I'm thinking it's some issues with the communication with the Go client and the pCloud server, but I can't figure out what it is (I've tried forcing HTTP/1.1, but no dice).
When uploading files without io.Pipe and with bytes.Buffer instead, everything is OK, but that doesn't work with large files (OOM).
The only warning I get when enabling verbose HTTP debugging is:
2022/04/21 10:43:29 http2: Transport failed to get client conn for eapi.pcloud.com:443: http2: no cached connection was available
This doesn't happen when I force HTTP/1.1, but the connection still hangs, so I'm not sure how relevant this error is.
Does anyone have any idea what could be the cause and how to fix it? Any help is appreciated.
https://cloud.google.com/translate/docs/samples/translate-text-with-model?hl=zh-cn#translate_text_with_model-go
I'm using the example in the link.When I turn off the proxy.It seems to take 30s to time out.How do I set the timeout duration?And is there an example?
Replace 'context.Background()' with 'context.WithTimeout()' seems doesn't work.
func main() {
fmt.Println("start..")
t := "The Go Gopher is cute"
now := time.Now()
r, err := translateTextWithModel("zh-CN", t, "nmt")
fmt.Println(time.Now().Sub(now).Milliseconds(), "ms")
fmt.Println(t, "-->", r)
fmt.Println("err:", err)
fmt.Println("end..")
}
func translateTextWithModel(targetLanguage, text, model string) (string, error) {
// targetLanguage := "ja"
// text := "The Go Gopher is cute"
// model := "nmt"
ctx, cancel := context.WithTimeout(context.Background(), time.Second*3)
defer cancel()
lang, err := language.Parse(targetLanguage)
if err != nil {
return "", fmt.Errorf("language.Parse: %v", err)
}
client, err := translate.NewClient(ctx)
if err != nil {
return "", fmt.Errorf("translate.NewClient: %v", err)
}
defer client.Close()
resp, err := client.Translate(ctx, []string{text}, lang, &translate.Options{
Model: model, // Either "nmt" or "base".
})
if err != nil {
return "", fmt.Errorf("Translate: %v", err)
}
if len(resp) == 0 {
return "", nil
}
return resp[0].Text, nil
}
When turning off the proxy,sometimes like.
As I see the library provides possibility to change settings by providing options into constructor function translate.NewClient(ctx context.Context, opts ...option.ClientOption).
You can try to change http.Client timeout by creating new one:
// timeout that you need to set
httpClientTimeout := time.Minute * 5
// create http client with custom timeout
httpClient := &http.Client{
Timeout: httpClientTimeout,
}
// inject new http client into translate client
client, err := translate.NewClient(ctx, option.WithHTTPClient(httpClient))
if err != nil {
return "", fmt.Errorf("translate.NewClient: %v", err)
}
defer client.Close()
Hope it helpful.
Source documentation
I wrote a connection closure function. It sends a closing frame and expects the same in response.
func TryCloseNormally(wsConn *websocket.Conn) error {
closeNormalClosure := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")
defer wsConn.Close()
if err := wsConn.WriteControl(websocket.CloseMessage, closeNormalClosure, time.Now().Add(time.Second)); err != nil {
return err
}
if err := wsConn.SetReadDeadline(time.Now().Add(time.Second)); err != nil {
return err
}
_, _, err := wsConn.ReadMessage()
if websocket.IsCloseError(err, websocket.CloseNormalClosure) {
return nil
} else {
return errors.New("Websocket doesn't send a close frame in response")
}
}
I wrote a test for this function.
func TestTryCloseNormally(t *testing.T) {
done := make(chan struct{})
exit := make(chan struct{})
ctx := context.Background()
ln, err := net.Listen("tcp", "localhost:")
require.Nil(t, err)
handler := HandlerFunc(func(conn *websocket.Conn) {
for {
_, _, err := conn.ReadMessage()
if err != nil {
require.True(t, websocket.IsCloseError(err, websocket.CloseNormalClosure), err.Error())
return
}
}
})
s, err := makeServer(ctx, handler)
require.Nil(t, err)
go func() {
require.Nil(t, s.Run(ctx, exit, ln))
close(done)
}()
wsConn, _, err := websocket.DefaultDialer.Dial(addr+strconv.Itoa(ln.Addr().(*net.TCPAddr).Port), nil) //nolint:bodyclose
require.Nil(t, err)
require.Nil(t, wsConn.WriteMessage(websocket.BinaryMessage, []byte{'o', 'k'}))
require.Nil(t, TryCloseNormally(wsConn))
close(exit)
<-done
}
To my surprise, it works correctly. Readmessage() reads the closing frame. But in the test, I don't write anything.
Is this happening at the gorilla/websocket level?
Did I write the function correctly? Maybe reading the response frame also happens at the gorilla level.
The function is mostly correct.
Websocket endpoints echo close messages unless the endpoint has already send a close message on its own. See Closing Handshake in the Websocket RFC for more details.
In the normal close scenario, an application should expect to receive a close message after sending a close message.
To handle the case where the peer sent a data message before the sending the close message, read and discard data messages until an error is returned.
func TryCloseNormally(wsConn *websocket.Conn) error {
defer wsConn.Close()
closeNormalClosure := websocket.FormatCloseMessage(websocket.CloseNormalClosure, "")
if err := wsConn.WriteControl(websocket.CloseMessage, closeNormalClosure, time.Now().Add(time.Second)); err != nil {
return err
}
if err := wsConn.SetReadDeadline(time.Now().Add(time.Second)); err != nil {
return err
}
for {
_, _, err := wsConn.ReadMessage()
if websocket.IsCloseError(err, websocket.CloseNormalClosure) {
return nil
}
if err != nil {
return err
}
}
}
I am able to zip a file using logic similar to the zip writer seen here.
This results in an array of bytes ([]byte) being created within the bytes.Buffer object that is returned. I would just like to know if there is there any way I can upload this 'zipped' array of bytes to an API endpoint that expects a 'multipart/form-data' request body (without having to save it locally).
Supplementary information:
I have code that utilizes this when compressing a folder. I am able to successfully execute an HTTP POST request with the zip file to the endpoint with this logic.
However, this unfortunately saves zipped files in a user's local file system. I would like to try to avoid this :)
You can create multipart writer and write []byte zipped data into field with field name you like and file name like below.
func addZipFileToReq(zipped []byte) (*http.Request, error){
body := bytes.NewBuffer(nil)
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile(`fileField`, `filename`)
if err != nil {
return nil, err
}
_, err = part.Write(zipped)
if err != nil {
return nil, err
}
err = writer.Close()
if err != nil {
return nil, err
}
r, err := http.NewRequest(http.MethodPost, "https://example.com", body)
if err != nil {
return nil, err
}
r.Header.Set("Content-Type", writer.FormDataContentType())
return r, nil
}
If you want to stream-upload the zip, you should be able to do so with io.Pipe. The following is an incomplete and untested example to demonstrate the general idea. To make it work you'll need to modify it and potentially fix whatever bugs you encounter.
func UploadReader(r io.Reader) error {
req, err := http.NewRequest("POST", "<UPLOAD_URL>", r)
if err != nil {
return err
}
// TODO set necessary headers (content type, auth, etc)
res, err := http.DefaultClient.Do(req)
if err != nil {
return err
} else if res.StatusCode != 200 {
return errors.New("not ok")
}
return nil
}
func ZipDir(dir string, w io.Writer) error {
zw := zip.NewWriter(w)
defer zw.Close()
return filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
if err != nil {
return err
}
if !fi.Mode().IsRegular() {
return nil
}
header, err := zip.FileInfoHeader(fi)
if err != nil {
return err
}
header.Name = path
header.Method = zip.Deflate
w, err := zw.CreateHeader(header)
if err != nil {
return err
}
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
if _, err := io.Copy(w, f); err != nil {
return err
}
return nil
})
}
func UploadDir(dir string) error {
r, w := io.Pipe()
ch := make(chan error)
wg := sync.WaitGroup{}
wg.Add(1)
go func() {
defer wg.Done()
defer w.Close()
if err := ZipDir(dir, w); err != nil {
ch <- err
}
}()
wg.Add(1)
go func() {
defer wg.Done()
defer r.Close()
if err := UploadReader(r); err != nil {
ch <- err
}
}()
go func() {
wg.Wait()
close(ch)
}()
return <-ch
}