I am trying to stream a file from one http endpoint to another, and avoid storing large files on disk. I thought I had working with this code, but it is creating empty files:
// out, err := os.Create(key)
resp, err := http.Get("http://source_url.com/_content/" + key)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
// now stream the file straight to the endpoint using put
req, err := http.NewRequest("PUT", "http://dest_url.com/_content/"+key, resp.Body)
if err != nil {
log.Fatal(err)
}
req.Header.Set("Content-Type", "application/octet-stream")
client := &http.Client{}
res, err := client.Do(req)
if err != nil {
log.Fatal(err)
}
fmt.Println(key, res.ContentLength, res.Status)
Related
Think I might be missing something obvious here. I'm attempting to grab the file from a client request hitting my server and forwarding that to an external API for processing by creating a new multipart request and copying the file over. In this case, the API is looking for a FormFile under the "files" key. The receiving API keeps telling me the file has invalid mime type application/octet-stream
API Call Documentation
func forwardFile(r *http.Request) (string, error) {
file, fileHandler, err := r.FormFile("image")
if err != nil {
return "", err
}
defer file.Close()
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile("files", fileHandler.Filename)
if err != nil {
return "", err
}
if _, err := io.Copy(part, file); err != nil {
return "", err
}
writer.Close()
req, _ := http.NewRequest("POST", newUploadUrl, body)
req.Header.Add("Content-Type", writer.FormDataContentType())
client := &http.Client{}
response, err := client.Do(req)
}
Thank you for your time.
I guess you need to change your content type to multipart/form-data
Solved it by creating a MIMEHeader and populating the disposition and content type myself, see below:
partHeader := textproto.MIMEHeader{}
disposition := fmt.Sprintf("form-data; name=\"files\"; filename=\"%s\"", fileHandler.Filename)
partHeader.Add("Content-Disposition", disposition)
partHeader.Add("Content-Type", "image/png")
part, err := writer.CreatePart(partHeader)
if _, err := io.Copy(part, file); err != nil {
log.Print("Error copying")
return "", err
}
I'm wrote a function that posts multipart form data to an endpoint but doesn't seem to be working properly. Here's the code
func Upload(filePath string) error {
client := &http.Client{
Timeout: time.Second * 10,
}
// New multipart writer.
body := &bytes.Buffer{}
file, err := os.Open(filePath)
if err != nil {
return err
}
writer := multipart.NewWriter(body)
fw, err := writer.CreateFormFile("file", filepath.Base(filePath))
if err != nil {
return err
}
_, err = io.Copy(fw, bufio.NewReader(file))
if err != nil {
return err
}
// Close multipart writer.
writer.Close()
req, err := http.NewRequest("POST", "http://localhost:5050/upload", bytes.NewReader(body.Bytes()))
if err != nil {
return err
}
req.Header.Set("Content-Type", writer.FormDataContentType())
rsp, _ := client.Do(req)
if rsp == nil {
return fmt.Errorf("failed to upload")
}
if rsp.StatusCode != http.StatusOK {
fmt.Printf("Request failed with response code: %d", rsp.StatusCode)
}
fmt.Println("rsp: ", rsp.StatusCode)
return nil
}
The handler on the API receiving the POST is this:
func (m *MetadataService) uploadHandler(res http.ResponseWriter, req *http.Request) {
file, handler, err := req.FormFile("file")
if err != nil {
panic(err) //dont do this
}
defer file.Close()
fmt.Println("req: ", req)
// Create a buffer to store the header of the file in
fileHeader := make([]byte, 512)
// Copy the headers into the FileHeader buffer
if _, err := file.Read(fileHeader); err != nil {
panic(err) //dont do this
}
// set position back to start.
if _, err := file.Seek(0, 0); err != nil {
panic(err) //dont do this
}
// copy example
f, err := os.OpenFile(handler.Filename, os.O_WRONLY|os.O_CREATE, 0666)
if err != nil {
panic(err) //please dont
}
defer f.Close()
buf := bytes.NewBuffer(nil)
if _, err := io.Copy(buf, file); err != nil {
panic(err)
}
fmt.Println("filename:", handler.Filename)
fmt.Println("file.(Sizer).Size():", file.(Sizer).Size())
fmt.Println("contentType:", http.DetectContentType(fileHeader))
io.WriteString(res, id)
}
The code panics when reading the fileHeader thought. The content type when sent is "application/octet-stream" when I'd expect it to be "audio/mpeg" since i'm uploading a song. Not sure what's wrong with the aforementioned Upload function. Need help! Thank you!
I have a weird issue with this code:
func PrepareFileUpload(filePath, url string) (*http.Request, error) {
pr, pw := io.Pipe()
mpw := multipart.NewWriter(pw)
go func() {
defer pw.Close()
part, err := mpw.CreateFormFile("file", filepath.Base(filePath))
if err != nil {
return
}
file, err := os.Open(filePath)
if err != nil {
return
}
defer file.Close()
if _, err = io.Copy(part, file); err != nil {
return
}
err = mpw.Close()
if err != nil {
return
}
}()
req, err := http.NewRequest("POST", url, pr)
req.Header.Set("Content-Type", mpw.FormDataContentType())
return req, err
}
which I use like this:
filePath := "foo.bar"
s := []byte("Test file")
ioutil.WriteFile(filePath, s, 0644)
values := url.Values{}
values.Set("folderid", "123456")
values.Set("filename", filepath.Base(filePath))
values.Set("nopartial", "1")
u := url.URL{
Scheme: "https",
Host: "eapi.pcloud.com",
Path: "/uploadfile",
RawQuery: values.Encode(),
}
req, err := PrepareFileUpload(filePath, u.String())
if err != nil {
log.Fatal(err)
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", "ACCESS_TOKEN"))
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
retData, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(retData))
For some reason, when used with the pCloud API, this hangs when running http.DefaultClient.Do(req). I have tried creating my own test server in Go, and there are no issues there, so I'm thinking it's some issues with the communication with the Go client and the pCloud server, but I can't figure out what it is (I've tried forcing HTTP/1.1, but no dice).
When uploading files without io.Pipe and with bytes.Buffer instead, everything is OK, but that doesn't work with large files (OOM).
The only warning I get when enabling verbose HTTP debugging is:
2022/04/21 10:43:29 http2: Transport failed to get client conn for eapi.pcloud.com:443: http2: no cached connection was available
This doesn't happen when I force HTTP/1.1, but the connection still hangs, so I'm not sure how relevant this error is.
Does anyone have any idea what could be the cause and how to fix it? Any help is appreciated.
I have a function which uploads images to remote storage via PUT request
func sendFileToStorage(path string) error {
logger.Get().Info(nil, "Got new file: " + path)
request, err := newfileUploadRequest(path)
if err != nil {
return err
}
client := &http.Client{}
resp, err := client.Do(request)
if err != nil {
return err
}
defer resp.Body.Close()
fmt.Println(resp.StatusCode)
return nil
}
func newfileUploadRequest(path string) (*http.Request, error) {
file, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
url := config.Get().ExternalStorage.Url
req, err := http.NewRequest("PUT", url, bytes.NewBuffer(file))
if err != nil {
return nil, err
}
req.SetBasicAuth(config.Get().ExternalStorage.Login, config.Get().ExternalStorage.Password)
return req, err
}
but sometimes the file uploads not completely. I mean the file can be cut. It happens when the image is big (more than 7mb).
When I try to upload the image with postman I don't have this problem. What I am doing wrong?
is there anyone can help me to converting this cURL command to Go?
curl -X PUT -H 'Content-Type: image/jpg' \
-H "Content-Length: 132093" \
-T "/Users/ikmal/Downloads/catcute.jpg" \
"https://hootsuite-video.s3.amazonaws.com/production/18395606_a9245dd7-73d6-4392-af4a-1cd9bf359cfb.jpg?AWSAccessKeyId=AKIAIM7ASX2JTE3ZFAAA&Expires=1543304067&Signature=n9sZkQ%2BF1DGuiYHqixrrvmoxIXQ%3D"
I already try generator like:
1. https://mholt.github.io/curl-to-go/
it created Go code like this:
// Generated by curl-to-Go: https://mholt.github.io/curl-to-go
req, err := http.NewRequest("PUT", "https://hootsuite-video.s3.amazonaws.com/production/18395606_a9245dd7-73d6-4392-af4a-1cd9bf359cfb.jpg?AWSAccessKeyId=AKIAIM7ASX2JTE3ZFAAA&Expires=1543304067&Signature=n9sZkQ%2BF1DGuiYHqixrrvmoxIXQ%3D", nil)
if err != nil {
// handle err
}
req.Header.Set("Content-Type", "image/jpg")
req.Header.Set("Content-Length", "132093")
resp, err := http.DefaultClient.Do(req)
if err != nil {
// handle err
}
defer resp.Body.Close()
2. https://curl.trillworks.com/
it created Go code like this:
package main
import (
"fmt"
"io/ioutil"
"log"
"net/http"
)
func main() {
client := &http.Client{}
req, err := http.NewRequest("PUT", "https://hootsuite-video.s3.amazonaws.com/production/18395606_a9245dd7-73d6-4392-af4a-1cd9bf359cfb.jpg?AWSAccessKeyId=AKIAIM7ASX2JTE3ZFAAA&Expires=1543304067&Signature=n9sZkQ%2BF1DGuiYHqixrrvmoxIXQ%3D", nil)
if err != nil {
log.Fatal(err)
}
req.Header.Set("Content-Type", "image/jpg")
req.Header.Set("Content-Length", "132093")
resp, err := client.Do(req)
if err != nil {
log.Fatal(err)
}
bodyText, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
fmt.Printf("%s\n", bodyText)
}
But neither gives me proper code because when I run those on project, it returning some error, because it doesn't generate this part:
-T "/Users/ikmal/Downloads/catcute.jpg"
and generate it as "nil" instead.
UPDATE :
What I have done for this problem is adding this code:
file, err := os.Open("/Users/ikmal/Downloads/catcute.jpg")
if err != nil {
panic(err)
}
defer file.Close()
so I put the variable "file" into body request, here is my final code:
func main(){
file, err := os.Open("/Users/ikmal/Downloads/catcute.jpg")
if err != nil {
panic(err)
}
defer file.Close()
client := &http.Client{}
req, err := http.NewRequest("PUT", "https://hootsuite-video.s3.amazonaws.com/production/18395606_a9245dd7-73d6-4392-af4a-1cd9bf359cfb.jpg?AWSAccessKeyId=AKIAIM7ASX2JTE3ZFAAA&Expires=1543304067&Signature=n9sZkQ%2BF1DGuiYHqixrrvmoxIXQ%3D", file)
if err != nil {
log.Fatal(err)
}
req.Header.Set("Content-Type", "image/jpg")
req.Header.Set("Content-Length", "132093")
resp, err := client.Do(req)
if err != nil {
log.Fatal(err)
}
bodyText, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
fmt.Printf("%s\n", bodyText)
}
then when I build and run this program, it returning response like this:
<?xml version="1.0" encoding="UTF-8"?>
<Error>
<Code>NotImplemented</Code>
<Message>A header you provided implies functionality that is not implemented</Message>
<Header>Transfer-Encoding</Header>
<RequestId>FBD2CEAF71EA4DEA</RequestId>
<HostId>K6hDrHIJr5YtoIBn2d64bfuLBgs6F17gKQV9jrTJ31X987A5gshhqtnKDs3lW2uSliBJwk1pri4=</HostId>
</Error>
It seems the error came from headers problem, but I'm sure I type the header correctly. Is there something wrong that I missed?
You must using multipart to upload content of image.
f, err := os.Open("./Users/ikmal/Downloads/catcute.jpg")
if err != nil{
log.Fatal(err)
}
defer f.Close()
var buf = new(bytes.Buffer)
writer := multipart.NewWriter(buf)
part, _ := writer.CreateFormFile("image", "dont care about name")
io.Copy(part, f)
writer.Close()
req, _ := http.NewRequest("POST", "https://hootsuite-video.s3.amazonaws.com/production/18395606_a9245dd7-73d6-4392-af4a-1cd9bf359cfb.jpg?AWSAccessKeyId=AKIAIM7ASX2JTE3ZFAAA&Expires=1543304067&Signature=n9sZkQ%2BF1DGuiYHqixrrvmoxIXQ%3D", buf)
req.Header.Set("Content-Type", writer.FormDataContentType())
client := &http.Client{}
res, _ := client.Do(req)
defer res.Body.Close()
b, _ := ioutil.ReadAll(res.Body)
fmt.Println(string(b))