How to duplicate a cURL command with file as body - go

is there anyone can help me to converting this cURL command to Go?
curl -X PUT -H 'Content-Type: image/jpg' \
-H "Content-Length: 132093" \
-T "/Users/ikmal/Downloads/catcute.jpg" \
"https://hootsuite-video.s3.amazonaws.com/production/18395606_a9245dd7-73d6-4392-af4a-1cd9bf359cfb.jpg?AWSAccessKeyId=AKIAIM7ASX2JTE3ZFAAA&Expires=1543304067&Signature=n9sZkQ%2BF1DGuiYHqixrrvmoxIXQ%3D"
I already try generator like:
1. https://mholt.github.io/curl-to-go/
it created Go code like this:
// Generated by curl-to-Go: https://mholt.github.io/curl-to-go
req, err := http.NewRequest("PUT", "https://hootsuite-video.s3.amazonaws.com/production/18395606_a9245dd7-73d6-4392-af4a-1cd9bf359cfb.jpg?AWSAccessKeyId=AKIAIM7ASX2JTE3ZFAAA&Expires=1543304067&Signature=n9sZkQ%2BF1DGuiYHqixrrvmoxIXQ%3D", nil)
if err != nil {
// handle err
}
req.Header.Set("Content-Type", "image/jpg")
req.Header.Set("Content-Length", "132093")
resp, err := http.DefaultClient.Do(req)
if err != nil {
// handle err
}
defer resp.Body.Close()
2. https://curl.trillworks.com/
it created Go code like this:
package main
import (
"fmt"
"io/ioutil"
"log"
"net/http"
)
func main() {
client := &http.Client{}
req, err := http.NewRequest("PUT", "https://hootsuite-video.s3.amazonaws.com/production/18395606_a9245dd7-73d6-4392-af4a-1cd9bf359cfb.jpg?AWSAccessKeyId=AKIAIM7ASX2JTE3ZFAAA&Expires=1543304067&Signature=n9sZkQ%2BF1DGuiYHqixrrvmoxIXQ%3D", nil)
if err != nil {
log.Fatal(err)
}
req.Header.Set("Content-Type", "image/jpg")
req.Header.Set("Content-Length", "132093")
resp, err := client.Do(req)
if err != nil {
log.Fatal(err)
}
bodyText, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
fmt.Printf("%s\n", bodyText)
}
But neither gives me proper code because when I run those on project, it returning some error, because it doesn't generate this part:
-T "/Users/ikmal/Downloads/catcute.jpg"
and generate it as "nil" instead.
UPDATE :
What I have done for this problem is adding this code:
file, err := os.Open("/Users/ikmal/Downloads/catcute.jpg")
if err != nil {
panic(err)
}
defer file.Close()
so I put the variable "file" into body request, here is my final code:
func main(){
file, err := os.Open("/Users/ikmal/Downloads/catcute.jpg")
if err != nil {
panic(err)
}
defer file.Close()
client := &http.Client{}
req, err := http.NewRequest("PUT", "https://hootsuite-video.s3.amazonaws.com/production/18395606_a9245dd7-73d6-4392-af4a-1cd9bf359cfb.jpg?AWSAccessKeyId=AKIAIM7ASX2JTE3ZFAAA&Expires=1543304067&Signature=n9sZkQ%2BF1DGuiYHqixrrvmoxIXQ%3D", file)
if err != nil {
log.Fatal(err)
}
req.Header.Set("Content-Type", "image/jpg")
req.Header.Set("Content-Length", "132093")
resp, err := client.Do(req)
if err != nil {
log.Fatal(err)
}
bodyText, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
fmt.Printf("%s\n", bodyText)
}
then when I build and run this program, it returning response like this:
<?xml version="1.0" encoding="UTF-8"?>
<Error>
<Code>NotImplemented</Code>
<Message>A header you provided implies functionality that is not implemented</Message>
<Header>Transfer-Encoding</Header>
<RequestId>FBD2CEAF71EA4DEA</RequestId>
<HostId>K6hDrHIJr5YtoIBn2d64bfuLBgs6F17gKQV9jrTJ31X987A5gshhqtnKDs3lW2uSliBJwk1pri4=</HostId>
</Error>
It seems the error came from headers problem, but I'm sure I type the header correctly. Is there something wrong that I missed?

You must using multipart to upload content of image.
f, err := os.Open("./Users/ikmal/Downloads/catcute.jpg")
if err != nil{
log.Fatal(err)
}
defer f.Close()
var buf = new(bytes.Buffer)
writer := multipart.NewWriter(buf)
part, _ := writer.CreateFormFile("image", "dont care about name")
io.Copy(part, f)
writer.Close()
req, _ := http.NewRequest("POST", "https://hootsuite-video.s3.amazonaws.com/production/18395606_a9245dd7-73d6-4392-af4a-1cd9bf359cfb.jpg?AWSAccessKeyId=AKIAIM7ASX2JTE3ZFAAA&Expires=1543304067&Signature=n9sZkQ%2BF1DGuiYHqixrrvmoxIXQ%3D", buf)
req.Header.Set("Content-Type", writer.FormDataContentType())
client := &http.Client{}
res, _ := client.Do(req)
defer res.Body.Close()
b, _ := ioutil.ReadAll(res.Body)
fmt.Println(string(b))

Related

Post multi-part form data for audio file

I'm wrote a function that posts multipart form data to an endpoint but doesn't seem to be working properly. Here's the code
func Upload(filePath string) error {
client := &http.Client{
Timeout: time.Second * 10,
}
// New multipart writer.
body := &bytes.Buffer{}
file, err := os.Open(filePath)
if err != nil {
return err
}
writer := multipart.NewWriter(body)
fw, err := writer.CreateFormFile("file", filepath.Base(filePath))
if err != nil {
return err
}
_, err = io.Copy(fw, bufio.NewReader(file))
if err != nil {
return err
}
// Close multipart writer.
writer.Close()
req, err := http.NewRequest("POST", "http://localhost:5050/upload", bytes.NewReader(body.Bytes()))
if err != nil {
return err
}
req.Header.Set("Content-Type", writer.FormDataContentType())
rsp, _ := client.Do(req)
if rsp == nil {
return fmt.Errorf("failed to upload")
}
if rsp.StatusCode != http.StatusOK {
fmt.Printf("Request failed with response code: %d", rsp.StatusCode)
}
fmt.Println("rsp: ", rsp.StatusCode)
return nil
}
The handler on the API receiving the POST is this:
func (m *MetadataService) uploadHandler(res http.ResponseWriter, req *http.Request) {
file, handler, err := req.FormFile("file")
if err != nil {
panic(err) //dont do this
}
defer file.Close()
fmt.Println("req: ", req)
// Create a buffer to store the header of the file in
fileHeader := make([]byte, 512)
// Copy the headers into the FileHeader buffer
if _, err := file.Read(fileHeader); err != nil {
panic(err) //dont do this
}
// set position back to start.
if _, err := file.Seek(0, 0); err != nil {
panic(err) //dont do this
}
// copy example
f, err := os.OpenFile(handler.Filename, os.O_WRONLY|os.O_CREATE, 0666)
if err != nil {
panic(err) //please dont
}
defer f.Close()
buf := bytes.NewBuffer(nil)
if _, err := io.Copy(buf, file); err != nil {
panic(err)
}
fmt.Println("filename:", handler.Filename)
fmt.Println("file.(Sizer).Size():", file.(Sizer).Size())
fmt.Println("contentType:", http.DetectContentType(fileHeader))
io.WriteString(res, id)
}
The code panics when reading the fileHeader thought. The content type when sent is "application/octet-stream" when I'd expect it to be "audio/mpeg" since i'm uploading a song. Not sure what's wrong with the aforementioned Upload function. Need help! Thank you!

Connection hangs when sending large file using io.Pipe to pCloud

I have a weird issue with this code:
func PrepareFileUpload(filePath, url string) (*http.Request, error) {
pr, pw := io.Pipe()
mpw := multipart.NewWriter(pw)
go func() {
defer pw.Close()
part, err := mpw.CreateFormFile("file", filepath.Base(filePath))
if err != nil {
return
}
file, err := os.Open(filePath)
if err != nil {
return
}
defer file.Close()
if _, err = io.Copy(part, file); err != nil {
return
}
err = mpw.Close()
if err != nil {
return
}
}()
req, err := http.NewRequest("POST", url, pr)
req.Header.Set("Content-Type", mpw.FormDataContentType())
return req, err
}
which I use like this:
filePath := "foo.bar"
s := []byte("Test file")
ioutil.WriteFile(filePath, s, 0644)
values := url.Values{}
values.Set("folderid", "123456")
values.Set("filename", filepath.Base(filePath))
values.Set("nopartial", "1")
u := url.URL{
Scheme: "https",
Host: "eapi.pcloud.com",
Path: "/uploadfile",
RawQuery: values.Encode(),
}
req, err := PrepareFileUpload(filePath, u.String())
if err != nil {
log.Fatal(err)
}
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", "ACCESS_TOKEN"))
resp, err := http.DefaultClient.Do(req)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
retData, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(retData))
For some reason, when used with the pCloud API, this hangs when running http.DefaultClient.Do(req). I have tried creating my own test server in Go, and there are no issues there, so I'm thinking it's some issues with the communication with the Go client and the pCloud server, but I can't figure out what it is (I've tried forcing HTTP/1.1, but no dice).
When uploading files without io.Pipe and with bytes.Buffer instead, everything is OK, but that doesn't work with large files (OOM).
The only warning I get when enabling verbose HTTP debugging is:
2022/04/21 10:43:29 http2: Transport failed to get client conn for eapi.pcloud.com:443: http2: no cached connection was available
This doesn't happen when I force HTTP/1.1, but the connection still hangs, so I'm not sure how relevant this error is.
Does anyone have any idea what could be the cause and how to fix it? Any help is appreciated.

Unable to set POST body in a http request

Is this not the right way to set POST request body?
data := url.Values{}
data.Set("url", "https://www.google.com/")
client := http.Client{}
r, err := http.NewRequest(http.MethodPost, apiURL, strings.NewReader(data.Encode()))
The code below when executed suggests that no url param was sent in the POST request.
package main
import (
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
)
func doAPICall() {
// curl -XPOST -d 'url=https://www.google.com/' 'https://cleanuri.com/api/v1/shorten'
apiURL := "https://cleanuri.com/api/v1/shorten"
data := url.Values{}
data.Set("url", "https://www.google.com/")
client := http.Client{}
r, err := http.NewRequest(http.MethodPost, apiURL, strings.NewReader(data.Encode()))
if err != nil {
panic(err)
}
resp, err := client.Do(r)
if err != nil {
panic(err)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
fmt.Println(string(body))
}
func main() {
doAPICall()
}
Output:-
$ go run .
{"error":"API Error: URL is empty"}
$
The server expects a valid value for the Content-Type request header.
r, err := http.NewRequest(http.MethodPost, apiURL, strings.NewReader(data.Encode()))
if err != nil {
panic(err)
}
r.Header.Set("Content-Type", "application/x-www-form-urlencoded") // <-- add this line
resp, err := client.Do(r)
The server also supports JSON request bodies:
r, err := http.NewRequest(http.MethodPost, apiURL, strings.NewReader(`{"url": "https://www.google.com/"}`))
if err != nil {
panic(err)
}
r.Header.Set("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(r)
You can do something like this
package main
import (
"fmt"
"io/ioutil"
"net/http"
"strings"
)
func main() {
url := "https://cleanuri.com/api/v1/shorten"
payload := strings.NewReader("url=https://www.google.com/")
req, err := http.NewRequest("POST", url, payload)
if err != nil {
fmt.Println(err)
return
}
req.Header.Add("content-type", "application/x-www-form-urlencoded")
req.Header.Add("cache-control", "no-cache")
res, err := http.DefaultClient.Do(req)
if err != nil {
fmt.Println(err)
return
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
fmt.Println(res)
fmt.Println(string(body))
}

Implementing multipart file upload with extra params

I am trying to replicate the following command:
curl -X POST --header 'Content-Type: multipart/form-data' --header 'Accept: text/html; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/HTML/1.7.0"' -F wikitext=%27%27%27Mahikari%27%27%27%20is%20a%20%5B%5BJapan%5D%5Dese%20%5B%5Bnew%20religious%20movement%5D%5D -F body_only=true -F 'https://en.wikipedia.org/api/rest_v1/transform/wikitext/to/html'
The file is passed as a url quoted parameter to curl.
The content of the original file is given as (with no trailing returns):
'''Mahikari''' is a [[Japan]]ese [[new religious movement]]
The only parameter I added, for now, is body_only=true
The expected and correct answer is:
<p id="mwAQ"><b id="mwAg">Mahikari</b> is a <a rel="mw:WikiLink" href="./Japan" title="Japan" id="mwAw">Japanese</a> <a rel="mw:WikiLink" href="./New_religious_movement" title="New religious movement" id="mwBA">new religious movement</a></p>
The code below is not returning anything (not even an error!):
package main
import (
"bytes"
"fmt"
"io"
// "io/ioutil"
"log"
"mime/multipart"
"net/http"
"os"
"path/filepath"
)
// Creates a new file upload http request with optional extra params
func newfileUploadRequest(uri string, params map[string]string, paramName, path string) (*http.Request, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
// fileContents, err := ioutil.ReadAll(file)
// if err != nil {
// return nil, err
// }
fi, err := file.Stat()
if err != nil {
return nil, err
}
body := new(bytes.Buffer)
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile(paramName, fi.Name())
if err != nil {
return nil, err
}
// part.Write(fileContents)
io.Copy(part, file)
for key, val := range params {
_ = writer.WriteField(key, val)
}
err = writer.Close()
if err != nil {
return nil, err
}
request, err := http.NewRequest("POST", uri, body)
request.Header.Add("Content-Type", writer.FormDataContentType())
request.Header.Add("Accept", "text/html; charset=utf-8; profile=\"https://www.mediawiki.org/wiki/Specs/HTML/1.7.0\"")
return request, err
}
func transformWikitextToHtml(path string) {
extraParams := map[string]string{
"body_only": "true",
}
request, err := newfileUploadRequest("https://en.wikipedia.org/api/rest_v1/transform/wikitext/to/html", extraParams, "file", path)
if err != nil {
log.Fatal(err)
}
client := &http.Client{}
resp, err := client.Do(request)
if err != nil {
log.Fatal(err)
} else {
var bodyContent []byte
fmt.Println(resp.StatusCode)
fmt.Println(resp.Header)
resp.Body.Read(bodyContent)
resp.Body.Close()
fmt.Println(bodyContent)
}
}
func main() {
transformWikitextToHtml("/tmp/2239217")
}
I set up the headers according to the documentation and what is expected. I tried a few things, as reading the file at once (commented out), but that didnt help. What am I missing?
In your CURL request, you are sending wikitext as a field (-F wikitext=...).
However, in your code you are sending it as a file part.
If you send that as a field it will work as you expect.
Just include the file contents as an additional extra field in your code:
func transformWikitextToHtml(path string) {
fileBytes, err := ioutil.ReadFile(path)
if err != nil {
log.Fatal(err)
}
extraParams := map[string]string{
"body_only": "true",
"wikitext": string(fileBytes),
}
// rest of the code should be as you posted
}
Then of course, remove the parts of newfileUploadRequest that work with the path and file param name, which are not needed any more.
Also, when writing the response body, you had a small bug and it was not printing anything even once the code was fixed, so please replace that part with:
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(bodyBytes))
Full working code:
package main
import (
"bytes"
"fmt"
"log"
"mime/multipart"
"net/http"
"io/ioutil"
)
// Creates a new file upload http request with optional extra params
func newfileUploadRequest(uri string, params map[string]string) (*http.Request, error) {
body := new(bytes.Buffer)
writer := multipart.NewWriter(body)
for key, val := range params {
err := writer.WriteField(key, val)
if err != nil {
log.Fatal(err)
}
}
err := writer.Close()
if err != nil {
return nil, err
}
request, err := http.NewRequest("POST", uri, body)
request.Header.Add("Content-Type", writer.FormDataContentType())
request.Header.Add("Accept", "text/html; charset=utf-8; profile=\"https://www.mediawiki.org/wiki/Specs/HTML/1.7.0\"")
return request, err
}
func transformWikitextToHtml(path string) {
fileBytes, err := ioutil.ReadFile(path)
if err != nil {
log.Fatal(err)
}
extraParams := map[string]string{
"body_only": "true",
"wikitext": string(fileBytes),
}
request, err := newfileUploadRequest("https://en.wikipedia.org/api/rest_v1/transform/wikitext/to/html", extraParams)
if err != nil {
log.Fatal(err)
}
client := &http.Client{}
resp, err := client.Do(request)
if err != nil {
log.Fatal(err)
} else {
fmt.Println(resp.StatusCode)
fmt.Println(resp.Header)
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(bodyBytes))
}
}
func main() {
transformWikitextToHtml("/tmp/2239217")
}

Stream file from one http endpoint to another

I am trying to stream a file from one http endpoint to another, and avoid storing large files on disk. I thought I had working with this code, but it is creating empty files:
// out, err := os.Create(key)
resp, err := http.Get("http://source_url.com/_content/" + key)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
// now stream the file straight to the endpoint using put
req, err := http.NewRequest("PUT", "http://dest_url.com/_content/"+key, resp.Body)
if err != nil {
log.Fatal(err)
}
req.Header.Set("Content-Type", "application/octet-stream")
client := &http.Client{}
res, err := client.Do(req)
if err != nil {
log.Fatal(err)
}
fmt.Println(key, res.ContentLength, res.Status)

Resources