I have a function which uploads images to remote storage via PUT request
func sendFileToStorage(path string) error {
logger.Get().Info(nil, "Got new file: " + path)
request, err := newfileUploadRequest(path)
if err != nil {
return err
}
client := &http.Client{}
resp, err := client.Do(request)
if err != nil {
return err
}
defer resp.Body.Close()
fmt.Println(resp.StatusCode)
return nil
}
func newfileUploadRequest(path string) (*http.Request, error) {
file, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
url := config.Get().ExternalStorage.Url
req, err := http.NewRequest("PUT", url, bytes.NewBuffer(file))
if err != nil {
return nil, err
}
req.SetBasicAuth(config.Get().ExternalStorage.Login, config.Get().ExternalStorage.Password)
return req, err
}
but sometimes the file uploads not completely. I mean the file can be cut. It happens when the image is big (more than 7mb).
When I try to upload the image with postman I don't have this problem. What I am doing wrong?
Related
I'm wrote a function that posts multipart form data to an endpoint but doesn't seem to be working properly. Here's the code
func Upload(filePath string) error {
client := &http.Client{
Timeout: time.Second * 10,
}
// New multipart writer.
body := &bytes.Buffer{}
file, err := os.Open(filePath)
if err != nil {
return err
}
writer := multipart.NewWriter(body)
fw, err := writer.CreateFormFile("file", filepath.Base(filePath))
if err != nil {
return err
}
_, err = io.Copy(fw, bufio.NewReader(file))
if err != nil {
return err
}
// Close multipart writer.
writer.Close()
req, err := http.NewRequest("POST", "http://localhost:5050/upload", bytes.NewReader(body.Bytes()))
if err != nil {
return err
}
req.Header.Set("Content-Type", writer.FormDataContentType())
rsp, _ := client.Do(req)
if rsp == nil {
return fmt.Errorf("failed to upload")
}
if rsp.StatusCode != http.StatusOK {
fmt.Printf("Request failed with response code: %d", rsp.StatusCode)
}
fmt.Println("rsp: ", rsp.StatusCode)
return nil
}
The handler on the API receiving the POST is this:
func (m *MetadataService) uploadHandler(res http.ResponseWriter, req *http.Request) {
file, handler, err := req.FormFile("file")
if err != nil {
panic(err) //dont do this
}
defer file.Close()
fmt.Println("req: ", req)
// Create a buffer to store the header of the file in
fileHeader := make([]byte, 512)
// Copy the headers into the FileHeader buffer
if _, err := file.Read(fileHeader); err != nil {
panic(err) //dont do this
}
// set position back to start.
if _, err := file.Seek(0, 0); err != nil {
panic(err) //dont do this
}
// copy example
f, err := os.OpenFile(handler.Filename, os.O_WRONLY|os.O_CREATE, 0666)
if err != nil {
panic(err) //please dont
}
defer f.Close()
buf := bytes.NewBuffer(nil)
if _, err := io.Copy(buf, file); err != nil {
panic(err)
}
fmt.Println("filename:", handler.Filename)
fmt.Println("file.(Sizer).Size():", file.(Sizer).Size())
fmt.Println("contentType:", http.DetectContentType(fileHeader))
io.WriteString(res, id)
}
The code panics when reading the fileHeader thought. The content type when sent is "application/octet-stream" when I'd expect it to be "audio/mpeg" since i'm uploading a song. Not sure what's wrong with the aforementioned Upload function. Need help! Thank you!
I'm new to golang and I'm trying to write a function that uploads a file with a post request to API server. I try Post API in Postman, it is OK but in my code I have some error like this image
This is my golang code:
func (c *Client) PostUploadFile(endpoint string, params map[string]string) []byte {
url := "/examples/image/text.txt"
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
// Open the file
file, err := os.Open(url)
if err != nil {
// return nil, err
}
// Close the file later
defer file.Close()
part, err := writer.CreateFormFile("file", filepath.Base(url))
_, err = io.Copy(part, file)
if err != nil {
fmt.Println(err)
// return nil, err
}
for key, val := range params {
_ = writer.WriteField(key, val)
}
err = writer.Close()
if err != nil {
// return nil, err
}
fmt.Println("Data request:")
fmt.Println(body)
fmt.Println("Endpoint:")
fmt.Println(c.BaseUrl + endpoint)
req, requestErr := http.NewRequest("POST", c.BaseUrl+endpoint, body)
if requestErr != nil {
log.Fatalln(requestErr)
}
req.Header.Add("auth_token", c.AuthToken)
req.Header.Add("accept", "application/json")
// req.Header.Add("Content-Type", "application/json")
req.Header.Add("Content-Type", writer.FormDataContentType())
client := &http.Client{}
fmt.Println("Response:")
resp, err := client.Do(req)
fmt.Println(resp)
if err != nil {
log.Println(err)
return []byte(``)
}
return c.parseBody(resp)
}
and this is a param formdata in body:
fmt.Printf("%+v\n", c.UploadImage(map[string]string{
"file": "/examples/image/text.txt",
"wfs_id": "30578",
"id": "59284",
"element_id": "119726",
}))
I'm trying get the response from an API that uses JSON Web token, I need use the header : {
Authorization: "Bearer token"
}
But I would like to keep the timeout of the http.Client that I'm using. How could I do it?
var myClient = &http.Client{Timeout: 10 * time.Second}
func getJson(url string, target interface{}) error {
r, err := myClient.Get(url)
if err != nil {
return err
}
defer r.Body.Close()
return json.NewDecoder(r.Body).Decode(target)
}
net/http.Request has a Header field that you can directly edit, but this means you can't use the shortcut client.Get method. Something more like:
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return err
}
req.Header = map[string][]string{
"Authorization": {fmt.Sprintf("Bearer %s", jwt)},
}
r, err := myClient.Do(req)
...
You could do something like this,
func getJson(url string, target interface{}) error {
req, err := http.NewRequest(http.MethodGet, url, nil)
if nil != err {
return err
}
r, err := myClient.Do(req)
if err != nil {
return err
}
defer r.Body.Close()
return json.NewDecoder(r.Body).Decode(target)
}
Alternatively , you could also use context to control the request timeout
func getJsonWithContext(url string, target interface{}) error {
req, err := http.NewRequest(http.MethodGet, url, nil)
if nil != err {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second*1)
defer cancel()
reqWithContext := req.WithContext(ctx)
r, err := myClient.Do(reqWithContext)
if err != nil {
return err
}
defer r.Body.Close()
return json.NewDecoder(r.Body).Decode(target)
}
I'm using http to call RPC with code below
func (c *CallClient) Wallet(method string, req, rep interface{}) error {
client := &http.Client{}
data, _ := EncodeClientRequest(method, req)
reqest, _ := http.NewRequest("POST", c.endpoint, bytes.NewBuffer(data))
resp, err := client.Do(reqest)
if err != nil {
return err
}
defer resp.Body.Close()
io.Copy(ioutil.Discard, resp.Body)
return DecodeClientResponse(resp.Body, rep)
}
with EncodeClientRquest && DecodeClientResponse
// EncodeClientRequest encodes parameters for a JSON-RPC client request.
func EncodeClientRequest(method string, args interface{}) ([]byte, error) {
c := &clientRequest{
Version: "2.0",
Method: method,
Params: [1]interface{}{args},
Id: uint64(rand.Int63()),
}
return json.Marshal(c)
}
// DecodeClientResponse decodes the response body of a client request into
// the interface reply.
func DecodeClientResponse(r io.Reader, reply interface{}) error {
var c clientResponse
if err := json.NewDecoder(r).Decode(&c); err != nil {
return err
}
if c.Error != nil {
return fmt.Errorf("%v", c.Error)
}
if c.Result == nil {
return errors.New("result is null")
}
return json.Unmarshal(*c.Result, reply)
}
And I got error EOF.
This line:
io.Copy(ioutil.Discard, resp.Body)
reads the whole resp.Body, leaving the reader with no more bytes to be read. Therefore any successive calls to resp.Body.Read will return EOF and the json.Decoder.Decode method does use the io.Reader.Read method when decoding the given reader's content, so...
And since resp.Body is an io.ReadCloser, which is an interface that does not support "rewinding", and you want to read the body content more than once (ioutil.Discard and json.Decode), you'll have to read the body into a variable that you can re-read afterwards. It's up to you how you do that, a slice of bytes, or bytes.Reader, or something else.
Example using bytes.Reader:
func (c *CallClient) Wallet(method string, req, rep interface{}) error {
client := &http.Client{}
data, err := EncodeClientRequest(method, req)
if err != nil {
return err
}
reqest, err := http.NewRequest("POST", c.endpoint, bytes.NewBuffer(data))
if err != nil {
return err
}
resp, err := client.Do(reqest)
if err != nil {
return err
}
defer resp.Body.Close()
// get a reader that can be "rewound"
buf := bytes.NewBuffer(nil)
if _, err := io.Copy(buf, resp.Body); err != nil {
return err
}
br := bytes.NewReader(buf.Bytes())
if _, err := io.Copy(ioutil.Discard, br); err != nil {
return err
}
// rewind
if _, err := br.Seek(0, 0); err != nil {
return err
}
return DecodeClientResponse(br, rep)
}
I have a below code in Go:
func (api *ApiResource) create(request *restful.Request, response *restful.Response) {
account := &DefaultAccount
err := request.ReadEntity(account)
if err != nil {
response.WriteErrorString(http.StatusInternalServerError, err.Error())
return
}
tmpl := data_transformer.ParseTemplate("xml/accAdd.xml")
payload := data_transformer.RenderTemplate(tmpl, account)
resp, err := http.Post(url, "application/xml", payload)
if err != nil {
response.WriteErrorString(http.StatusInternalServerError, err.Error())
return
}
// Body closes when either at the end of the function or at return
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
aResp := new(AResp)
err = xml.Unmarshal(body, aResp)
if err != nil {
fmt.Printf("error: %v", err)
return
}
response.WriteHeader(http.StatusCreated)
response.WriteEntity(aResp)
}
func (api *ApiResource) updateLimit(request *restful.Request, response *restful.Response) {
account := &DefaultLimit
err := request.ReadEntity(account)
if err != nil {
response.WriteErrorString(http.StatusInternalServerError, err.Error())
return
}
tmpl := data_transformer.ParseTemplate("xml/addLimit.xml")
payload := data_transformer.RenderTemplate(tmpl, account)
resp, err := http.Post(url, "application/xml", payload)
if err != nil {
response.WriteErrorString(http.StatusInternalServerError, err.Error())
return
}
// Body closes when either at the end of the function or at return
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
aResp := new(AResp)
err = xml.Unmarshal(body, aResp)
if err != nil {
fmt.Printf("error: %v", err)
return
}
response.WriteHeader(http.StatusCreated)
response.WriteEntity(aResp)
}
I want to be able to remove duplicate codes in an elegant way Go style.
If I do remove them and put them in separate function, would all defer and net/http package work as expected for different calls and xml file loads?
What is a good Go type solutions for this?
You can separate almost the whole function and just pass to it the "account" and the xml file to read.
Also it's a better practice to use xml.Decoder instead of xml.Unmarshal when reading from streams.
func updateEntity(response *restful.Response, fn string, iface interface{}) {
tmpl := data_transformer.ParseTemplate("xml/" + fn)
payload := data_transformer.RenderTemplate(tmpl, iface)
resp, err := http.Post(url, "application/xml", payload)
if err != nil {
response.WriteErrorString(http.StatusInternalServerError, err.Error())
return
}
defer resp.Body.Close()
dec := xml.NewDecoder(resp.Body)
body, err := ioutil.ReadAll(resp.Body)
aResp := &AResp{}
err = dec.Decode(aResp)
if err != nil {
fmt.Printf("error: %v", err)
return
}
response.WriteHeader(http.StatusCreated)
response.WriteEntity(aResp)
}
func (*ApiResource) create(request *restful.Request, response *restful.Response) {
account := &DefaultAccount
err := request.ReadEntity(account)
if err != nil {
response.WriteErrorString(http.StatusInternalServerError, err.Error())
return
}
updateEntity(response, "accAdd.xml", account)
}
func (*ApiResource) updateLimit(request *restful.Request, response *restful.Response) {
account := &DefaultLimit
err := request.ReadEntity(account)
if err != nil {
response.WriteErrorString(http.StatusInternalServerError, err.Error())
return
}
updateEntity(response, "addLimit.xml", account)
}