I'm trying to use GRPC Client side stream by Image processing, I'm also newbie in GRPC stream, Here I will be creating the image in small chunks and send into the Server, Chunks are created but cannot able to send it. Finally I'm getting EOF error.
Here I attached my sample code any one can guide me thanks.
Example:
func (c *ClientGRPC) UploadFile(ctx context.Context) (stats stats.Stats, err error) {
var (
writing = true
buf []byte
n int
status *pb.UploadStatus
)
cwd, _ := os.Getwd()
templatePath := filepath.Join(cwd, "/unnamed.png")
file, err := os.Open(templatePath)
if err != nil {
err = errors.Wrapf(err,
"failed to open file %s",
file)
return
}
defer file.Close()
stream, err := c.client.Upload(ctx)
if err != nil {
err = errors.Wrapf(err,
"failed to create upload stream for file %s",
file)
return
}
defer stream.CloseSend()
buf = make([]byte, c.chunkSize)
for writing {
n, err = file.Read(buf)
if err != nil {
if err == io.EOF {
writing = false
err = nil
continue
}
err = errors.Wrapf(err,
"errored while copying from file to buf")
return
}
err = stream.Send(&pb.Chunk{
Content: buf[:n],
})
if err != nil {
err = errors.Wrapf(err,
"failed to send chunk via stream") //`Here, I'm getting EOF error`.
return
}
}
status, err = stream.CloseAndRecv()
if err != nil {
err = errors.Wrapf(err,
"failed to receive upstream status response")
return
}
if status.Code != pb.UploadStatusCode_Ok {
err = errors.Errorf(
"upload failed - msg: %s",
status.Message)
return
}
return
}
Output:
client=====> failed to send chunk via stream: EOF
If you are using grpc underneath, then run your program with the environment variable GRPC_GO_LOG_VERBOSITY_LEVEL=99 GRPC_GO_LOG_SEVERITY_LEVEL=info to get logs from grpc to debug deeper (i.e. it is a connection level problem, or stream level problem).
Related
I am newbie in kafka, i try build a service send mail with attach files.
Execution flow:
Kafka will receive a message to send mail
function get file will download file from url , scale image, and save file
when send mail i will get files from folder and attach to form
Issues:
when i send mail with large files many times , kafka retry many times, i will receive many mail
kafka error: "kafka server: The provided member is not known in the current generation"
I listened MaxProcessingTime , but i try to test a mail with large file, it still work fine
Kafka info : 1 broker , 3 consumer
func (s *customerMailService) SendPODMail() error { filePaths, err := DownloadFiles(podURLs, orderInfo.OrderCode)
if err != nil{
countRetry := 0
for countRetry <= NUM_OF_RETRY{
filePaths, err = DownloadFiles(podURLs, orderInfo.OrderCode)
if err == nil{
break
}
countRetry++
}
}
err = s.sendMailService.Send(ctx, orderInfo.CustomerEmail, tmsPod, content,filePaths)}
function download file :
func DownloadFiles(files []string, orderCode string) ([]string, error) {
var filePaths []string
err := os.Mkdir(tempDir, 0750)
if err != nil && !os.IsExist(err) {
return nil, err
}
tempDirPath := tempDir + "/" + orderCode
err = os.Mkdir(tempDirPath, 0750)
if err != nil && !os.IsExist(err) {
return nil, err
}
for _, fileUrl := range files {
fileUrlParsed, err := url.ParseRequestURI(fileUrl)
if err != nil {
logrus.WithError(err).Infof("Pod url is invalid %s", orderCode)
return nil, err
}
extFile := filepath.Ext(fileUrlParsed.Path)
dir, err := os.MkdirTemp(tempDirPath, "tempDir")
if err != nil {
return nil, err
}
f, err := os.CreateTemp(dir, "tmpfile-*"+extFile)
if err != nil {
return nil, err
}
defer f.Close()
response, err := http.Get(fileUrl)
if err != nil {
return nil, err
}
defer response.Body.Close()
contentTypes := response.Header["Content-Type"]
isTypeAllow := false
for _, contentType := range contentTypes {
if contentType == "image/png" || contentType == "image/jpeg" {
isTypeAllow = true
}
}
if !isTypeAllow {
logrus.WithError(err).Infof("Pod image type is invalid %s", orderCode)
return nil, errors.New("Pod image type is invalid")
}
decodedImg, err := imaging.Decode(response.Body)
if err != nil {
return nil, err
}
resizedImg := imaging.Resize(decodedImg, 1024, 0, imaging.Lanczos)
imaging.Save(resizedImg, f.Name())
filePaths = append(filePaths, f.Name())
}
return filePaths, nil}
function send mail
func (s *tikiMailService) SendFile(ctx context.Context, receiver string, templateCode string, data interface{}, filePaths []string) error {
path := "/v1/emails"
fullPath := fmt.Sprintf("%s%s", s.host, path)
formValue := &bytes.Buffer{}
writer := multipart.NewWriter(formValue)
_ = writer.WriteField("template", templateCode)
_ = writer.WriteField("to", receiver)
if data != nil {
b, err := json.Marshal(data)
if err != nil {
return errors.Wrapf(err, "Cannot marshal mail data to json with object %+v", data)
}
_ = writer.WriteField("params", string(b))
}
for _, filePath := range filePaths {
part, err := writer.CreateFormFile(filePath, filepath.Base(filePath))
if err != nil {
return err
}
pipeReader, pipeWriter := io.Pipe()
go func() {
defer pipeWriter.Close()
file, err := os.Open(filePath)
if err != nil {
return
}
defer file.Close()
io.Copy(pipeWriter, file)
}()
io.Copy(part, pipeReader)
}
err := writer.Close()
if err != nil {
return err
}
request, err := http.NewRequest("POST", fullPath, formValue)
if err != nil {
return err
}
request.Header.Set("Content-Type", writer.FormDataContentType())
resp, err := s.doer.Do(request)
if err != nil {
return errors.Wrap(err, "Cannot send request to send email")
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return errors.New(fmt.Sprintf("Send email with code %s error: status code %d, response %s",
templateCode, resp.StatusCode, string(b)))
} else {
logrus.Infof("Send email with attachment ,code %s success with response %s , box-code", templateCode, string(b),filePaths)
}
return nil
}
Thank
My team found my problem when I redeploy k8s pods, which lead to conflict leader partition causing rebalance. It will try to process the remaining messages in buffer of pods again.
Solution: I don't fetch many messages saved in buffer , I just get a message and process it by config :
ChannelBufferSize = 0
Example conflict leader parition:
consumer A and B startup in the same time
consumer A registers itself as leader, and owns the topic with all partitions
consumer B registers itself as leader, and then begins to rebalance and owns all partitions
consumer A rebalance and obtains all partitions, but can not consume because the memberId is old and need a new one
consumer B rebalance again and owns the topic with all partitions, but it's already obtained by consumer A
My two cents: in case of very big attachments, the consumer takes quite a lot of time to read the file and to send it as an attachment.
This increases the amount of time between two poll() calls. If that time is greater than max.poll.interval.ms, the consumer is thought to be failed and the partition offset is not committed. As a result, the message is processed again and eventually, if by chance the execution time stays below the poll interval, the offset is committed. The effect is a multiple email send.
Try increasing the max.poll.interval.ms on the consumer side.
I'm trying to figure out how to implement copying files from remote and get the data []byte from the buffer.
I have succeeded in doing the implementation with the upload by referring to this guide: https://chuacw.ath.cx/development/b/chuacw/archive/2019/02/04/how-the-scp-protocol-works.aspx
Inside the go func there's the implementation of the upload process of the SCP but I have no idea how to change it.
Any advice ?
func download(con *ssh.Client, buf bytes.Buffer, path string,) ([]byte,error) {
//https://chuacw.ath.cx/development/b/chuacw/archive/2019/02/04/how-the-scp-protocol-works.aspx
session, err := con.NewSession()
if err != nil {
return nil,err
}
buf.WriteString("sudo scp -f " + path + "\n")
stdin, err := session.StdinPipe()
if err != nil {
return nil,err
}
go func() {
defer stdin.Close()
fmt.Fprint(stdin, "C0660 "+strconv.Itoa(len(content))+" file\n")
stdin.Write(content)
fmt.Fprint(stdin, "\x00")
}()
output, err := session.CombinedOutput("sudo scp -f " + path)
buf.Write(output)
if err != nil {
return nil,&DeployError{
Err: err,
Output: buf.String(),
}
}
session.Close()
session, err = con.NewSession()
if err != nil {
return nil,err
}
defer session.Close()
return output,nil
}
The sink side is significantly more difficult than the source side. Made an example which should get you close to what you want. Note that I have not tested this code, that the error handling is sub optimal and it only supports 1/4th the protocol messages SCP may use. So you will still need to do some work to get it perfect.
With all that said, this is what I came up with:
func download(con *ssh.Client, path string) ([]byte, error) {
//https://chuacw.ath.cx/development/b/chuacw/archive/2019/02/04/how-the-scp-protocol-works.aspx
session, err := con.NewSession()
if err != nil {
return nil, err
}
defer session.Close()
// Local -> remote
stdin, err := session.StdinPipe()
if err != nil {
return nil, err
}
defer stdin.Close()
// Request a file, note that directories will require different handling
_, err = stdin.Write([]byte("sudo scp -f " + path + "\n"))
if err != nil {
return nil, err
}
// Remote -> local
stdout, err := session.StdoutPipe()
if err != nil {
return nil, err
}
// Make a buffer for the protocol messages
const megabyte = 1 << 20
b := make([]byte, megabyte)
// Offset into the buffer
off := 0
var filesize int64
// SCP may send multiple protocol messages, so keep reading
for {
n, err := stdout.Read(b[off:])
if err != nil {
return nil, err
}
nl := bytes.Index(b[:off+n], []byte("\n"))
// If there is no newline in the buffer, we need to read more
if nl == -1 {
off = off + n
continue
}
// We read a full message, reset the offset
off = 0
// if we did get a new line. We have the full protocol message
msg := string(b[:nl])
// Send back 0, which means OK, the SCP source will not send the next message otherwise
_, err = stdin.Write([]byte("0\n"))
if err != nil {
return nil, err
}
// First char is the mode (C=file, D=dir, E=End of dir, T=Time metadata)
mode := msg[0]
if mode != 'C' {
// Ignore other messags for now.
continue
}
// File message = Cmmmm <length> <filename>
msgParts := strings.Split(msg, " ")
if len(msgParts) > 1 {
// Parse the second part <length> as an base 10 integer
filesize, err = strconv.ParseInt(msgParts[1], 10, 64)
if err != nil {
return nil, err
}
}
// The file message will be followed with binary data containing the file
break
}
// Wrap the stdout reader in a limit reader so we will not read more than the filesize
fileReader := io.LimitReader(stdout, filesize)
// Seed the bytes buffer with the existing byte slice, saves additional allocation if file <= 1mb
buf := bytes.NewBuffer(b)
// Copy the file into the bytes buffer
_, err = io.Copy(buf, fileReader)
return buf.Bytes(), err
}
I am facing an issue where in, whenever I try to upload a file to a remote sftp server, I get an error saying "failed to send packet header: EOF". This occurs when I try to perform the uploading step from my own hosted EC2 instance. While locally, everything works fine.
Sftp client is initiated as follow.
// Connect to server
var authMethods []ssh.AuthMethod
// Use password authentication if password provided
if pass != "" {
authMethods = append(authMethods, ssh.Password(pass))
}
config := ssh.ClientConfig{
User: user,
Auth: authMethods,
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
}
conn, err := ssh.Dial("tcp", addr, &config)
if err != nil {
return nil, tearDown, errors.Wrap(err, fmt.Sprintf("failed to connect to %s", addr))
}
tearDown = func() {
_ = conn.Close()
}
// Create new SFTP client
sc, err := sftp.NewClient(conn)
if err != nil {
return nil, tearDown, errors.Wrap(err, "Unable to start SFTP subsystem")
}
tearDown = func() {
fmt.Println("defer is called. closing connection now .... ")
_ = conn.Close()
_ = sc.Close()
}
return sc, tearDown, nil
And instance of sc is attached to a struct and passed around the codebase
Function invoked while uploading file is as follow.
file, err := s.sc.OpenFile(remoteFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC)
defer func() {
if file == nil {
return
}
cErr := file.Close()
if cErr != nil {
fmt.Println(fmt.Sprintf("error while closing file %v", cErr))
}
}()
if err != nil {
fmt.Println(fmt.Sprintf("error while opening file %v", err))
return err
}
_, err = file.Write(data)
if err != nil {
fmt.Println(fmt.Sprintf("error while writing to file %v", err))
return err
}
return nil
Can someone guide me as in where is the error coming from?
I have written an API http server in Go using Gorilla Mux. It works well. One of the endpoints is for uploading files and saving them to an NFS share mounted to the server pod. The client is a Swift 5 app using Alamofire.
For smaller files we just use copy to write them from the request body. For larger files, we use a buffered stream reader and writer to perform the write, as we had issues with time outs and drops when just using copy.
However, when the write is happening, the server stops responding to all new requests. How can I change or optimize this code so that the server continues to respond as expected? See code here:
func uploadFile(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
repoBase := "./repo/gkp-directory/"
folderName := vars["uploadFolder"]
fileName := vars["uploadFile"]
fileSize := r.ContentLength
//
// Check if we have a package. They can be large and require special handling
//
if folderName == "pkgs" {
defer r.Body.Close()
//
// If the content is more than 10MB write to temp cache then move it
// in a seperate goroutine to the repo storage
//
if r.ContentLength > 10000000 {
buf := make([]byte, 10000000)
tempFile, err := os.Create(repoBase + folderName + "/" + fileName)
if err != nil {
log.Println("ERROR: Failed to create file.")
log.Println(err.Error())
return
}
defer tempFile.Close()
for {
n, err := r.Body.Read(buf)
if err != nil && err != io.EOF {
log.Println("ERROR: Error creating file on NFS")
log.Println(err.Error())
return
}
if n == 0 {
break
}
if _, err := tempFile.Write(buf[:n]); err != nil {
log.Println("ERROR: Error streaming to NFS")
log.Println(err.Error())
return
}
}
tempFile.Close()
r.Body.Close()
} else {
//
// If the package is smaller than 10MB we should be safe to write it directly to
// the NFS backend with no buffer
//
outputFile, err := os.Create(repoBase + folderName + "/" + fileName)
if err != nil {
log.Println("ERROR: Failed to create file.")
log.Println(err.Error())
return
}
defer outputFile.Close()
written, err := io.Copy(outputFile, r.Body)
if err != nil {
log.Println("ERROR: Failed to create file.")
log.Println(err.Error())
return
}
if written == fileSize {
outputFile.Close()
r.Body.Close()
}
}
} else {
//
// Otherwise file is not a package. This means it is just a small text file we
// should safetly be able to write this to NFS with no issues or buffer
//
outputFile, err := os.Create(repoBase + folderName + fileName)
if err != nil {
log.Println("ERROR: Failed to create file.")
log.Println(err.Error())
return
}
defer outputFile.Close()
written, err := io.Copy(outputFile, r.Body)
if err != nil {
log.Println("ERROR: Failed to create file.")
log.Println(err.Error())
return
}
if written == fileSize {
outputFile.Close()
r.Body.Close()
}
}
}
Is there any way how to serve files in Go with GRPC, like in gin-gonic's variant:
router.Static("/static", "/var/www")
You can't do it exactly like that.
But you can use the proto bytes type and put the file bytes in that field.
Also (as pointed out in the comments) with large files you should use streaming instead of a unary call. (most GRPC implementation have a limit of 4MB per message).
Proto example:
syntax = "proto3";
message Response {
bytes fileChunk = 1;
}
message Request {
string fileName = 1;
}
service TestService {
rpc Download(Request) returns (stream Response);
}
Server implementation example:
func (srv *Server) Download(req *pbgo.Request, responseStream pbgo.TestService_DownloadServer) error {
bufferSize := 64 *1024 //64KiB, tweak this as desired
file, err := os.Open(req.GetFileName())
if err != nil {
fmt.Println(err)
return err
}
defer file.Close()
buff := make([]byte, bufferSize)
for {
bytesRead, err := file.Read(buff)
if err != nil {
if err != io.EOF {
fmt.Println(err)
}
break
}
resp := &pbgo.Response{
FileChunk: buff[:bytesRead],
}
err = responseStream.Send(resp)
if err != nil {
log.Println("error while sending chunk:", err)
return err
}
}
return nil
}
Client would call it like this:
conn, err := grpc.Dial("localhost:9090", grpc.WithInsecure())
if err != nil {
log.Fatal("client could connect to grpc service:", err)
}
c := pbgo.NewTestServiceClient(conn)
fileStreamResponse, err := c.Download(context.TODO(), &pbgo.Request{
FileName: "test.txt",
})
if err != nil {
log.Println("error downloading:", err)
return
}
for {
chunkResponse, err := fileStreamResponse.Recv()
if err == io.EOF {
log.Println("received all chunks")
break
}
if err != nil {
log.Println("err receiving chunk:", err)
break
}
log.Printf("got new chunk with data: %s \n", chunkResponse.FileChunk)
}
If you need to be able to serve arbitrary files, you would need to handle which files you allow serving (say someone requests the file /etc/passwd or something).
Not sure what exactly is the use case here.