Gmail API shows empty body when getting message - go

When I send a request to get the email body, the Gmail API returns everything but the body data on the payload object.
Things I've tried so far
The "Watch" method is already implemented and working fine
As you can see from the screenshot, the response shows the "snipped", which means that the message get is working, but the body data and the "raw" field is still empty.
I am using the history id correctly (saving the current one to use for subsequent requests)
upgrade all the dependencies to the latest stable version
Am I missing anything?
func GetEmail(srv *gmail.Service, historyId uint64) (string, string) {
hist := getHistory(srv, historyId)
for _, h := range hist.History {
for _, m := range h.MessagesAdded {
id := m.Message.Id
mailContent, err := srv.Users.Messages.Get("me", id).Format("full").Do()
if err != nil {
log.Println("error when getting mail content: ", err)
}
if mailContent != nil {
if mailContent.Payload != nil {
payload := mailContent.Payload.Body
data, err := b64.RawURLEncoding.DecodeString(payload.Data)
if err != nil {
log.Println("error b64 decoding: ", err)
}
body := string(data)
if len(body) > 0 {
subject := getSubject(mailContent)
log.Println("subject ", subject)
return body, subject
}
}
}
}
}
return "No email to process, something's wrong - GetEmail func", ""
}

If you want the RAW message data then you need to use Format("RAW")
func GetEmail(srv *gmail.Service, messageId string) {
gmailMessageResposne, err := srv.Users.Messages.Get("me", messageId).Format("RAW").Do()
if err != nil {
log.Println("error when getting mail content: ", err)
}
if gmailMessageResposne != nil {
decodedData, err := base64.RawURLEncoding.DecodeString(gmailMessageResposne.Raw)
if err != nil {
log.Println("error b64 decoding: ", err)
}
fmt.Printf("- %s\n", decodedData)
}
}
Good question that was fun 😁
How to read a gmail email body with Go?

Related

kafka retry many times when i download large file

I am newbie in kafka, i try build a service send mail with attach files.
Execution flow:
Kafka will receive a message to send mail
function get file will download file from url , scale image, and save file
when send mail i will get files from folder and attach to form
Issues:
when i send mail with large files many times , kafka retry many times, i will receive many mail
kafka error: "kafka server: The provided member is not known in the current generation"
I listened MaxProcessingTime , but i try to test a mail with large file, it still work fine
Kafka info : 1 broker , 3 consumer
func (s *customerMailService) SendPODMail() error { filePaths, err := DownloadFiles(podURLs, orderInfo.OrderCode)
if err != nil{
countRetry := 0
for countRetry <= NUM_OF_RETRY{
filePaths, err = DownloadFiles(podURLs, orderInfo.OrderCode)
if err == nil{
break
}
countRetry++
}
}
err = s.sendMailService.Send(ctx, orderInfo.CustomerEmail, tmsPod, content,filePaths)}
function download file :
func DownloadFiles(files []string, orderCode string) ([]string, error) {
var filePaths []string
err := os.Mkdir(tempDir, 0750)
if err != nil && !os.IsExist(err) {
return nil, err
}
tempDirPath := tempDir + "/" + orderCode
err = os.Mkdir(tempDirPath, 0750)
if err != nil && !os.IsExist(err) {
return nil, err
}
for _, fileUrl := range files {
fileUrlParsed, err := url.ParseRequestURI(fileUrl)
if err != nil {
logrus.WithError(err).Infof("Pod url is invalid %s", orderCode)
return nil, err
}
extFile := filepath.Ext(fileUrlParsed.Path)
dir, err := os.MkdirTemp(tempDirPath, "tempDir")
if err != nil {
return nil, err
}
f, err := os.CreateTemp(dir, "tmpfile-*"+extFile)
if err != nil {
return nil, err
}
defer f.Close()
response, err := http.Get(fileUrl)
if err != nil {
return nil, err
}
defer response.Body.Close()
contentTypes := response.Header["Content-Type"]
isTypeAllow := false
for _, contentType := range contentTypes {
if contentType == "image/png" || contentType == "image/jpeg" {
isTypeAllow = true
}
}
if !isTypeAllow {
logrus.WithError(err).Infof("Pod image type is invalid %s", orderCode)
return nil, errors.New("Pod image type is invalid")
}
decodedImg, err := imaging.Decode(response.Body)
if err != nil {
return nil, err
}
resizedImg := imaging.Resize(decodedImg, 1024, 0, imaging.Lanczos)
imaging.Save(resizedImg, f.Name())
filePaths = append(filePaths, f.Name())
}
return filePaths, nil}
function send mail
func (s *tikiMailService) SendFile(ctx context.Context, receiver string, templateCode string, data interface{}, filePaths []string) error {
path := "/v1/emails"
fullPath := fmt.Sprintf("%s%s", s.host, path)
formValue := &bytes.Buffer{}
writer := multipart.NewWriter(formValue)
_ = writer.WriteField("template", templateCode)
_ = writer.WriteField("to", receiver)
if data != nil {
b, err := json.Marshal(data)
if err != nil {
return errors.Wrapf(err, "Cannot marshal mail data to json with object %+v", data)
}
_ = writer.WriteField("params", string(b))
}
for _, filePath := range filePaths {
part, err := writer.CreateFormFile(filePath, filepath.Base(filePath))
if err != nil {
return err
}
pipeReader, pipeWriter := io.Pipe()
go func() {
defer pipeWriter.Close()
file, err := os.Open(filePath)
if err != nil {
return
}
defer file.Close()
io.Copy(pipeWriter, file)
}()
io.Copy(part, pipeReader)
}
err := writer.Close()
if err != nil {
return err
}
request, err := http.NewRequest("POST", fullPath, formValue)
if err != nil {
return err
}
request.Header.Set("Content-Type", writer.FormDataContentType())
resp, err := s.doer.Do(request)
if err != nil {
return errors.Wrap(err, "Cannot send request to send email")
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
if resp.StatusCode != http.StatusOK {
return errors.New(fmt.Sprintf("Send email with code %s error: status code %d, response %s",
templateCode, resp.StatusCode, string(b)))
} else {
logrus.Infof("Send email with attachment ,code %s success with response %s , box-code", templateCode, string(b),filePaths)
}
return nil
}
Thank
My team found my problem when I redeploy k8s pods, which lead to conflict leader partition causing rebalance. It will try to process the remaining messages in buffer of pods again.
Solution: I don't fetch many messages saved in buffer , I just get a message and process it by config :
ChannelBufferSize = 0
Example conflict leader parition:
consumer A and B startup in the same time
consumer A registers itself as leader, and owns the topic with all partitions
consumer B registers itself as leader, and then begins to rebalance and owns all partitions
consumer A rebalance and obtains all partitions, but can not consume because the memberId is old and need a new one
consumer B rebalance again and owns the topic with all partitions, but it's already obtained by consumer A
My two cents: in case of very big attachments, the consumer takes quite a lot of time to read the file and to send it as an attachment.
This increases the amount of time between two poll() calls. If that time is greater than max.poll.interval.ms, the consumer is thought to be failed and the partition offset is not committed. As a result, the message is processed again and eventually, if by chance the execution time stays below the poll interval, the offset is committed. The effect is a multiple email send.
Try increasing the max.poll.interval.ms on the consumer side.

How to upload image or file as backend

I don't know how to upload image or file in Go. Here I share my code
this is my repository, what i must change or add more code?
func (db *reportConnection) CreateReport(report entity.Report) entity.Report {
db.connection.Save(&report)
db.connection.Preload("User").Find(&report)
return report
}
this is my service, what i must change or add more code?
func (service *reportService) Create(r dto.ReportCreateDTO) entity.Report {
report := entity.Report{}
err := smapping.FillStruct(&report, smapping.MapFields(&r))
if err != nil {
log.Fatalf("failed map %v: ", err)
}
res := service.reportRepo.CreateReport(report)
return res
}
this is my controller, what i must change or add more code?
func (c *reportController) Create(ctx *gin.Context) {
var createReport dto.ReportCreateDTO
err := ctx.ShouldBind(&createReport)
if err != nil {
response := response.BuildErrorResponse("Failed to process!", err.Error(), response.EmptyObj{})
ctx.AbortWithStatusJSON(http.StatusBadRequest, response)
} else {
authHeader := ctx.GetHeader("Authorization")
userID := c.GetUserIDByToken(authHeader)
convertUserID, err := strconv.ParseUint(userID, 10, 64)
if err == nil {
createReport.UserID = convertUserID
}
result := c.reportService.Create(createReport)
response := response.BuildResponse(true, "OK!", result)
ctx.JSON(http.StatusOK, response)
}
}
```
i think i need to set a header, but not sure how

json.Marshal for http post request with echo

I have two golang servers running on localhost.
They are using different ports.
I want to create a post request on one that sends a JSON object to the other one.
I am using the echo framework (if this matters)
The error I am getting is when I try to marshal the object for the post object:
2-valued json.Marshal(data) (value of type ([]byte, error)) where single value is expected
server 1:
type SendEmail struct {
SenderName string `json:"senderName,omitempty" bson:"senderName,omitempty" validate:"required,min=3,max=128"`
SenderEmail string `json:"senderEmail" bson:"senderEmail" validate:"required,min=10,max=128"`
Subject string `json:"subject" bson:"subject" validate:"required,min=10,max=128"`
RecipientName string `json:"recipientName" bson:"recipientName" validate:"required,min=3,max=128"`
RecipientEmail string `json:"recipientEmail" bson:"recipientEmail" validate:"required,min=10,max=128"`
PlainTextContent string `json:"plainTextContent" bson:"plainTextContent" validate:"required,min=10,max=512"`
}
func resetPassword(c echo.Context) error {
email := c.Param("email")
if email == "" {
return c.String(http.StatusNotFound, "You have not supplied a valid email")
}
data := SendEmail{
RecipientEmail: email,
RecipientName: email,
SenderEmail: “test#test”,
SenderName: “name”,
Subject: "Reset Password",
PlainTextContent: "Here is your code to reset your password, if you did not request this email then please ignore.",
}
// error here
req, err := http.NewRequest("POST", "127.0.0.1:8081/", json.Marshal(data))
if err != nil {
fmt.Println(err)
}
defer req.Body.Close()
return c.JSON(http.StatusOK, email)
}
server 2:
e.GET("/", defaultRoute)
func defaultRoute(c echo.Context) (err error) {
u := SendEmail{}
if err = c.Bind(u); err != nil {
return
}
return c.JSON(http.StatusOK, u)
}
It's always nice to meet a Gopher. A few things you might want to know, Go supports multi-value returns in that a function can return more than one value.
byteInfo, err := json.Marshal(data) // has two values returned
// check if there was an error returned first
if err != nil{
// handle your error here
}
Now the line below in your code
// error here
req, err := http.NewRequest("POST", "127.0.0.1:8081/", json.Marshal(data))
Will become this
// error here
req, err := http.NewRequest("POST", "127.0.0.1:8081/", bytes.NewBuffer(byteInfo))
And you can continue with the rest of your code. Happy Coding!
json.Marshal returns []byte and error which means you're passing 4 values to http.NewRequest.
You should call json.Marshal first and then use the result for http.NewRequest.
body, err := json.Marshal(data)
if err != nil {
// deal with error
}
req, err := http.NewRequest("POST", "127.0.0.1:8081/", body)

How to properly read errors from golang oauth2

token, err := googleOauthConfig.Exchange(context.Background(), code)
if err != nil {
fmt.Fprintf(w, "Err: %+v", err)
}
The output of the fprintf is:
Err: oauth2: cannot fetch token: 401 Unauthorized
Response: {"error":"code_already_used","error_description":"code_already_used"}
I want to check if "error" = "code_already_used". For the life of me, I can't sort out how.
How do I check/return/read "error" or "error_description" of err?
I've looked at the oauth2 code and it's a bit above me.
// retrieveToken takes a *Config and uses that to retrieve an *internal.Token.
// This token is then mapped from *internal.Token into an *oauth2.Token which is returned along
// with an error..
func retrieveToken(ctx context.Context, c *Config, v url.Values) (*Token, error) {
tk, err := internal.RetrieveToken(ctx, c.ClientID, c.ClientSecret, c.Endpoint.TokenURL, v)
if err != nil {
if rErr, ok := err.(*internal.RetrieveError); ok {
return nil, (*RetrieveError)(rErr)
}
return nil, err
}
return tokenFromInternal(tk), nil
}
How guess I'm trying to see the (*RetrieveError) part. Right?
THANK YOU!
The expression:
(*RetrieveError)(rErr)
converts therErr's type from *internal.RetrieveError to *RetrieveError. And since RetrieveError is declared in the oauth2 package, you can type assert the error you receive to *oauth2.RetrieveError to get the details. The details are contained in that type's Body field as a slice of bytes.
Since a slice of bytes is not the best format to be inspected and in your case it seems like the bytes contain a json object you can make your life easier by predefining a type into which you can unmarshal those details.
That is:
type ErrorDetails struct {
Error string `json:"error"`
ErrorDescription string `json:"error_description"`
}
token, err := googleOauthConfig.Exchange(context.Background(), code)
if err != nil {
fmt.Fprintf(w, "Err: %+v", err)
if rErr, ok := err.(*oauth2.RetrieveError); ok {
details := new(ErrorDetails)
if err := json.Unmarshal(rErr.Body, details); err != nil {
panic(err)
}
fmt.Println(details.Error, details.ErrorDescription)
}
}
Can do like this.
arr := strings.Split(err.Error(), "\n")
str := strings.Replace(arr[1], "Response: ", "", 1)
var details ErrorDetails
var json = jsoniter.ConfigCompatibleWithStandardLibrary
err := json.Unmarshal([]byte(str), &details)
if err == nil {
beego.Debug(details.Error)
beego.Debug(details.ErrorDescription)
}

Can't download attachment from multipart/signed type when using gmail api for Go

I'm trying to develop a simple utility for download a whole list of attachments by a specific label in Gmail.
I'm using the official gmail api for Go and I just started with the sample written here then I wrote my own code to archive my goal:
fmt.Println("Labels:")
for _, label := range labelList.Labels {
fmt.Printf("\n- %s id: %s\n", label.Name, label.Id)
if label.Id == "Label_N" {
messageList, _ := srv.Users.Messages.List(user).LabelIds(label.Id).Do()
if err != nil {
log.Fatalf("Error: %v", err)
}
for _, msg := range messageList.Messages {
// look for message
message, err := srv.Users.Messages.Get(user, msg.Id).Do()
if err != nil {
log.Fatalf("Unable to retrieve message: %v", err)
}
mailReceivedDate := time.Unix(0, message.InternalDate*int64(time.Millisecond)).Format(time.RFC3339Nano)
fmt.Println("______________________________________________")
fmt.Printf("Content-Type: %s\n", message.Payload.MimeType)
fmt.Printf("From: %s\n", message.Payload.Headers[7].Value)
if len(message.Payload.Parts) > 0 {
for _, p := range message.Payload.Parts {
fmt.Printf("Filename: %s\n", p.Filename)
fmt.Printf("Attachment ID: %s\n", p.Body.AttachmentId)
if isPdfAttachment(p.MimeType) {
attach, err := srv.Users.Messages.Attachments.Get(user, message.Id, p.Body.AttachmentId).Do()
decoded, _ := base64.URLEncoding.DecodeString(attach.Data)
if err != nil {
log.Fatalf("Unable to retrieve attachment: %v", err)
}
p.Filename = mailReceivedDate + "_" + p.Filename
fmt.Printf("File to write: %s - Type: %s\n", p.Filename, p.MimeType)
err = ioutil.WriteFile(p.Filename, decoded, 0644)
if err != nil {
log.Fatalf("Unable to save attachment: %v", err)
}
}
}
}
}
}
}
The problem is that I don't receive any AttachmentId when message MimeType is "multipart/signed" (the email contains 1 pdf file).
What should I do to for download attachments from this kind of email?
Can someone help me?

Resources