Upload a file with POST request golang - go

I'm new to golang and I'm trying to write a function that uploads a file with a post request to telegram for a bot I'm writing.
I've tried with this code but the error I'm getting from telegram is
Bad Request: there is no photo in the request.
I've searched on the net for how to do that, but none of what I found helped me through the problem.
func SendPostRequest (url string, filename string) []byte {
file, err := os.Open(filename)
if err != nil {
log.Fatal(err)
}
defer file.Close()
response, err := http.Post(url, "binary/octet-stream", file)
if err != nil {
log.Fatal(err)
}
defer response.Body.Close()
content, err := ioutil.ReadAll(response.Body)
if err != nil {
log.Fatal(err)
}
return content
}
The function I'm calling the SendPostRequest from is
func (e Engine) SendPhoto (filename string, chatId int64) APIResponse {
var url = fmt.Sprintf("%ssendPhoto?chat_id=%d", e.baseUrl, chatId)
var content []byte = SendPostRequest(url, filename)
var response APIResponse
json.Unmarshal(content, &response)
return response
}
EDIT:
The link to the Telegram bot api I'm using in the code is https://core.telegram.org/bots/api
And the api method is https://core.telegram.org/bots/api#sendphoto

After some digging I figured it out with this
import (
"bytes"
"io"
"mime/multipart"
"net/http"
"path/filepath"
)
// content is a struct which contains a file's name, its type and its data.
type content struct {
fname string
ftype string
fdata []byte
}
func sendPostRequest(url string, files ...content) ([]byte, error) {
var (
buf = new(bytes.Buffer)
w = multipart.NewWriter(buf)
)
for _, f := range files {
part, err := w.CreateFormFile(f.ftype, filepath.Base(f.fname))
if err != nil {
return []byte{}, err
}
_, err = part.Write(f.fdata)
if err != nil {
return []byte{}, err
}
}
err := w.Close()
if err != nil {
return []byte{}, err
}
req, err := http.NewRequest("POST", url, buf)
if err != nil {
return []byte{}, err
}
req.Header.Add("Content-Type", w.FormDataContentType())
client := &http.Client{}
res, err := client.Do(req)
if err != nil {
return []byte{}, err
}
defer res.Body.Close()
cnt, err := io.ReadAll(res.Body)
if err != nil {
return []byte{}, err
}
return cnt, nil
}

Related

Having issues with multipart.NewWriter using io.pipe

I'm having issues with requests being sent randomly empty. It doesn't always happen, but sometimes out of the blue, it will not send any of the multipart fields. I thought it might have to do with the upload server, so I created a local upload server to print out the request that's being sent, and it comes out empty.
I have added error checks everywhere, but no errors are being returned.
I tried to run the code with -race, but no race condition has been reported.
Edit: Update the code to use CloseWithError()
package main
import (
"fmt"
"io"
"log"
"mime/multipart"
"net/http"
"os"
)
var (
upload_url string = "https://upload.imagekit.io/api/v1/files/upload"
file_name string = "favicon-516140983.ico"
api_secret_key string = "PRIVATE_KEY"
)
func UploadMultipartFile(client *http.Client, uri, key, path string) (*http.Response, error) {
body, writer := io.Pipe()
req, err := http.NewRequest(http.MethodPost, uri, body)
if err != nil {
log.Println(err)
return nil, err
}
mwriter := multipart.NewWriter(writer)
req.Header.Add("Content-Type", mwriter.FormDataContentType())
req.SetBasicAuth(api_secret_key, "")
go func() {
var err error
defer func() {
if err != nil {
writer.CloseWithError(err)
} else {
writer.Close()
}
}()
var file *os.File
file, err = os.Open(path)
if err != nil {
return
}
defer file.Close()
if err = mwriter.WriteField("fileName", file_name); err != nil {
return
}
var w io.Writer
w, err = mwriter.CreateFormFile("file", path)
if err != nil {
return
}
var written int64
if written, err = io.Copy(w, file); err != nil {
err = fmt.Errorf("error copying %s (%d bytes written): %v", path, written, err)
return
}
if err = mwriter.Close(); err != nil {
return
}
}()
resp, err := client.Do(req)
if err != nil {
return nil, err
}
return resp, nil
}
func main() {
path, _ := os.Getwd()
path += "/" + file_name
client := &http.Client{}
resp, err := UploadMultipartFile(client, upload_url, "file", path)
if err != nil {
log.Println(err)
} else {
fmt.Println(resp.StatusCode)
fmt.Println(resp.Header)
_, err := io.Copy(os.Stdout, resp.Body)
if err != nil {
log.Fatal(err)
}
resp.Body.Close()
}
}

Want to add a FormFile in unit test Golang

I want to test a httpRequest with a json body and a test file.
I don't know how to add the created test file to the request beside body json.
body := strings.NewReader(URLTest.RequestBody)
request, err := http.NewRequest(URLTest.MethodType, "localhost:"+string(listeningPort)+URLTest.URL, body)
if err != nil {
t.Fatalf("HTTP NOT WORKING")
}
fileBuffer := new(bytes.Buffer)
mpWriter := multipart.NewWriter(fileBuffer)
fileWriter, err := mpWriter.CreateFormFile("file", "testfile.pdf")
if err != nil {
t.Fatalf(err.Error())
}
file, err := os.Open("testfile.pdf")
if err != nil {
t.Fatalf(err.Error())
}
defer file.Close()
_, err = io.Copy(fileWriter, file)
if err != nil {
t.Fatalf(err.Error())
}
rec := httptest.NewRecorder()
UploadFiles(rec, request, nil)
response := rec.Result()
if response.StatusCode != URLTest.ExpectedStatusCode {
t.Errorf(URLTest.URL + " status mismatch")
}
responseBody, err := ioutil.ReadAll(response.Body)
defer response.Body.Close()
if err != nil {
t.Errorf(URLTest.URL + " cant read response")
} else {
if strings.TrimSpace(string(responseBody)) != URLTest.ExpectedResponseBody {
t.Errorf(URLTest.URL + " response mismatch - have: " + string(responseBody) + " want: " + URLTest.ExpectedResponseBody)
}
}
}
Can I add file as a value like request.FormFile.Add(...) or something?
Regarding your question about how to send a file in an HTTP request with Go, here's some sample code.
And you will need the mime/multipart package to build the form.
package main
import (
"bytes"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"net/http/httputil"
"os"
"strings"
)
func main() {
var client *http.Client
var remoteURL string
{
//setup a mocked http client.
ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
b, err := httputil.DumpRequest(r, true)
if err != nil {
panic(err)
}
fmt.Printf("%s", b)
}))
defer ts.Close()
client = ts.Client()
remoteURL = ts.URL
}
//prepare the reader instances to encode
values := map[string]io.Reader{
"file": mustOpen("main.go"), // lets assume its this file
"other": strings.NewReader("hello world!"),
}
err := Upload(client, remoteURL, values)
if err != nil {
panic(err)
}
}
func Upload(client *http.Client, url string, values map[string]io.Reader) (err error) {
// Prepare a form that you will submit to that URL.
var b bytes.Buffer
w := multipart.NewWriter(&b)
for key, r := range values {
var fw io.Writer
if x, ok := r.(io.Closer); ok {
defer x.Close()
}
// Add an image file
if x, ok := r.(*os.File); ok {
if fw, err = w.CreateFormFile(key, x.Name()); err != nil {
return
}
} else {
// Add other fields
if fw, err = w.CreateFormField(key); err != nil {
return
}
}
if _, err = io.Copy(fw, r); err != nil {
return err
}
}
// Don't forget to close the multipart writer.
// If you don't close it, your request will be missing the terminating boundary.
w.Close()
// Now that you have a form, you can submit it to your handler.
req, err := http.NewRequest("POST", url, &b)
if err != nil {
return
}
// Don't forget to set the content type, this will contain the boundary.
req.Header.Set("Content-Type", w.FormDataContentType())
// Submit the request
res, err := client.Do(req)
if err != nil {
return
}
// Check the response
if res.StatusCode != http.StatusOK {
err = fmt.Errorf("bad status: %s", res.Status)
}
return
}
Hope you can use this in your unit test

Implementing multipart file upload with extra params

I am trying to replicate the following command:
curl -X POST --header 'Content-Type: multipart/form-data' --header 'Accept: text/html; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/HTML/1.7.0"' -F wikitext=%27%27%27Mahikari%27%27%27%20is%20a%20%5B%5BJapan%5D%5Dese%20%5B%5Bnew%20religious%20movement%5D%5D -F body_only=true -F 'https://en.wikipedia.org/api/rest_v1/transform/wikitext/to/html'
The file is passed as a url quoted parameter to curl.
The content of the original file is given as (with no trailing returns):
'''Mahikari''' is a [[Japan]]ese [[new religious movement]]
The only parameter I added, for now, is body_only=true
The expected and correct answer is:
<p id="mwAQ"><b id="mwAg">Mahikari</b> is a <a rel="mw:WikiLink" href="./Japan" title="Japan" id="mwAw">Japanese</a> <a rel="mw:WikiLink" href="./New_religious_movement" title="New religious movement" id="mwBA">new religious movement</a></p>
The code below is not returning anything (not even an error!):
package main
import (
"bytes"
"fmt"
"io"
// "io/ioutil"
"log"
"mime/multipart"
"net/http"
"os"
"path/filepath"
)
// Creates a new file upload http request with optional extra params
func newfileUploadRequest(uri string, params map[string]string, paramName, path string) (*http.Request, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
// fileContents, err := ioutil.ReadAll(file)
// if err != nil {
// return nil, err
// }
fi, err := file.Stat()
if err != nil {
return nil, err
}
body := new(bytes.Buffer)
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile(paramName, fi.Name())
if err != nil {
return nil, err
}
// part.Write(fileContents)
io.Copy(part, file)
for key, val := range params {
_ = writer.WriteField(key, val)
}
err = writer.Close()
if err != nil {
return nil, err
}
request, err := http.NewRequest("POST", uri, body)
request.Header.Add("Content-Type", writer.FormDataContentType())
request.Header.Add("Accept", "text/html; charset=utf-8; profile=\"https://www.mediawiki.org/wiki/Specs/HTML/1.7.0\"")
return request, err
}
func transformWikitextToHtml(path string) {
extraParams := map[string]string{
"body_only": "true",
}
request, err := newfileUploadRequest("https://en.wikipedia.org/api/rest_v1/transform/wikitext/to/html", extraParams, "file", path)
if err != nil {
log.Fatal(err)
}
client := &http.Client{}
resp, err := client.Do(request)
if err != nil {
log.Fatal(err)
} else {
var bodyContent []byte
fmt.Println(resp.StatusCode)
fmt.Println(resp.Header)
resp.Body.Read(bodyContent)
resp.Body.Close()
fmt.Println(bodyContent)
}
}
func main() {
transformWikitextToHtml("/tmp/2239217")
}
I set up the headers according to the documentation and what is expected. I tried a few things, as reading the file at once (commented out), but that didnt help. What am I missing?
In your CURL request, you are sending wikitext as a field (-F wikitext=...).
However, in your code you are sending it as a file part.
If you send that as a field it will work as you expect.
Just include the file contents as an additional extra field in your code:
func transformWikitextToHtml(path string) {
fileBytes, err := ioutil.ReadFile(path)
if err != nil {
log.Fatal(err)
}
extraParams := map[string]string{
"body_only": "true",
"wikitext": string(fileBytes),
}
// rest of the code should be as you posted
}
Then of course, remove the parts of newfileUploadRequest that work with the path and file param name, which are not needed any more.
Also, when writing the response body, you had a small bug and it was not printing anything even once the code was fixed, so please replace that part with:
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(bodyBytes))
Full working code:
package main
import (
"bytes"
"fmt"
"log"
"mime/multipart"
"net/http"
"io/ioutil"
)
// Creates a new file upload http request with optional extra params
func newfileUploadRequest(uri string, params map[string]string) (*http.Request, error) {
body := new(bytes.Buffer)
writer := multipart.NewWriter(body)
for key, val := range params {
err := writer.WriteField(key, val)
if err != nil {
log.Fatal(err)
}
}
err := writer.Close()
if err != nil {
return nil, err
}
request, err := http.NewRequest("POST", uri, body)
request.Header.Add("Content-Type", writer.FormDataContentType())
request.Header.Add("Accept", "text/html; charset=utf-8; profile=\"https://www.mediawiki.org/wiki/Specs/HTML/1.7.0\"")
return request, err
}
func transformWikitextToHtml(path string) {
fileBytes, err := ioutil.ReadFile(path)
if err != nil {
log.Fatal(err)
}
extraParams := map[string]string{
"body_only": "true",
"wikitext": string(fileBytes),
}
request, err := newfileUploadRequest("https://en.wikipedia.org/api/rest_v1/transform/wikitext/to/html", extraParams)
if err != nil {
log.Fatal(err)
}
client := &http.Client{}
resp, err := client.Do(request)
if err != nil {
log.Fatal(err)
} else {
fmt.Println(resp.StatusCode)
fmt.Println(resp.Header)
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(bodyBytes))
}
}
func main() {
transformWikitextToHtml("/tmp/2239217")
}

How to refactor semantic duplication

I have defined two funcs that do slightly different things but are syntactically the same.
Functions in question send POST requests to an api.
The duplication occurs in constructing the request, adding headers, etc.
How can I refactor the code to remove said duplication.
package main
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"net/http/httputil"
)
type token struct {
Token string
}
type config struct {
Foo string
}
func main() {
token, err := getAuthToken()
if err != nil {
log.Fatal(err)
}
config, err := getConfig("foo", token)
if err != nil {
log.Fatal(err)
}
_ = config
}
func getAuthToken() (string, error) {
endpoint := "foo"
body := struct {
UserName string `json:"username"`
Password string `json:"password"`
}{
UserName: "foo",
Password: "bar",
}
jsnBytes, err := json.Marshal(body)
if err != nil {
return "", err
}
req, err := http.NewRequest("POST", endpoint, bytes.NewReader(jsnBytes))
if err != nil {
return "", fmt.Errorf("Unable to create request. %v", err)
}
req.Header.Add("Content-Type", "application/json")
dump, err := httputil.DumpRequest(req, true)
if err != nil {
return "", fmt.Errorf("Could not dump request. ", err)
}
log.Println("Request: ", string(dump))
client := http.Client{}
log.Println("Initiating http request")
resp, err := client.Do(req)
if err != nil {
return "", fmt.Errorf("HTTP Error: %v", err)
}
defer resp.Body.Close()
bytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", fmt.Errorf("Error reading response body: %v", err)
}
var token token
err = json.Unmarshal(bytes, &token)
if err != nil {
return "", fmt.Errorf("Could not unamrshal json. ", err)
}
return token.Token, nil
}
func getConfig(id string, token string) (*config, error) {
endpoint := "foo"
body := struct {
ID string `json:"id"`
}{
ID: id,
}
jsnBytes, err := json.Marshal(body)
if err != nil {
return nil, err
}
req, err := http.NewRequest("POST", endpoint, bytes.NewReader(jsnBytes))
if err != nil {
return nil, fmt.Errorf("Unable to create request. %v", err)
}
req.Header.Add("Authorization", "Bearer "+token)
req.Header.Add("Content-Type", "application/json")
dump, err := httputil.DumpRequest(req, true)
if err != nil {
return nil, fmt.Errorf("Could not dump request. ", err)
}
log.Println("Request: ", string(dump))
client := http.Client{}
log.Println("Initiating http request")
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("HTTP Error: %v", err)
}
defer resp.Body.Close()
bytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("Error reading response body: %v", err)
}
var config config
err = json.Unmarshal(bytes, &config)
if err != nil {
return nil, fmt.Errorf("Could not unamrshal json. ", err)
}
return &config, nil
}
I would say the essence of sending the request is that you are sending a body to an endpoint and parsing a result. The headers are then optional options that you can add to the request along the way. With this in mind I would make a single common function for sending the request with this signature:
type option func(*http.Request)
func sendRequest(endpoint string, body interface{}, result interface{}, options ...option) error {
Note this is using functional options which Dave Cheney did an excellent description of here:
https://dave.cheney.net/2014/10/17/functional-options-for-friendly-apis
The complete code then becomes:
https://play.golang.org/p/GV6FeipIybA
package main
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"net/http/httputil"
)
type token struct {
Token string
}
type config struct {
Foo string
}
func main() {
token, err := getAuthToken()
if err != nil {
log.Fatal(err)
}
config, err := getConfig("foo", token)
if err != nil {
log.Fatal(err)
}
_ = config
}
func getAuthToken() (string, error) {
endpoint := "foo"
body := struct {
UserName string `json:"username"`
Password string `json:"password"`
}{
UserName: "foo",
Password: "bar",
}
var token token
err := sendRequest(endpoint, body, &token)
if err != nil {
return "", err
}
return token.Token, nil
}
func getConfig(id string, token string) (*config, error) {
endpoint := "foo"
body := struct {
ID string `json:"id"`
}{
ID: id,
}
var config config
err := sendRequest(endpoint, body, &config, header("Content-Type", "application/json"))
if err != nil {
return nil, err
}
return &config, nil
}
type option func(*http.Request)
func header(key, value string) func(*http.Request) {
return func(req *http.Request) {
req.Header.Add(key, value)
}
}
func sendRequest(endpoint string, body interface{}, result interface{}, options ...option) error {
jsnBytes, err := json.Marshal(body)
if err != nil {
return err
}
req, err := http.NewRequest("POST", endpoint, bytes.NewReader(jsnBytes))
if err != nil {
return fmt.Errorf("Unable to create request. %v", err)
}
req.Header.Add("Content-Type", "application/json")
for _, option := range options {
option(req)
}
dump, err := httputil.DumpRequest(req, true)
if err != nil {
return fmt.Errorf("Could not dump request. ", err)
}
log.Println("Request: ", string(dump))
client := http.Client{}
log.Println("Initiating http request")
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("HTTP Error: %v", err)
}
defer resp.Body.Close()
bytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("Error reading response body: %v", err)
}
err = json.Unmarshal(bytes, result)
if err != nil {
return fmt.Errorf("Could not unamrshal json. ", err)
}
return nil
}
The way I would do this is to extract the two parts that are common to both request executions: 1) create a request and 2) execute the request.
Gist with new code using HTTP Bin as an example
Creating the request includes setting up the endpoint, headers and marshaling the request body to JSON. In your case, you're also dumping the request to the log, that can also go in there. This is how it would look like:
func buildRequest(endpoint string, body interface{}, extraHeaders map[string]string) (*http.Request, error) {
jsnBytes, err := json.Marshal(body)
if err != nil {
return nil, err
}
req, err := http.NewRequest("POST", endpoint, bytes.NewReader(jsnBytes))
if err != nil {
return nil, err
}
req.Header.Add("Content-Type", "application/json")
for name, value := range extraHeaders {
req.Header.Add(name, value)
}
dump, err := httputil.DumpRequest(req, true)
if err != nil {
return nil, err
}
log.Println("Request: ", string(dump))
return req, nil
}
If you have no extra headers, you can pass nil as the third argument here.
The second part to extract is actually executing the request and unmarshalling the data. This is how the executeRequest would look like:
func executeRequest(req *http.Request, responseBody interface{}) error {
client := http.Client{}
log.Println("Initiating http request")
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
bytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
log.Printf("Response is: %s\n", string(bytes))
err = json.Unmarshal(bytes, &responseBody)
return err
}

Requesting multiple URLs in Go

I have the following Go program: https://play.golang.org/p/-TUtJ7DIhi
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strconv"
)
func main() {
body, err := get("https://hacker-news.firebaseio.com/v0/topstories.json")
if err != nil {
panic(err)
}
var ids [500]int
if err = json.Unmarshal(body, &ids); err != nil {
panic(err)
}
var contents []byte
for _, value := range ids[0:10] {
body, err := get("https://hacker-news.firebaseio.com/v0/item/" + strconv.Itoa(value) + ".json")
if err != nil {
fmt.Println(err)
} else {
contents = append(contents, body...)
}
}
fmt.Println(contents)
}
func get(url string) ([]byte, error) {
res, err := http.Get(url)
if err != nil {
return nil, err
}
body, err := ioutil.ReadAll(res.Body)
res.Body.Close()
return body, err
}
When run it throws EOF json errors on the iterative get requests, but when I hit the URLs individually they do not appear to be malformed.
What am I missing?
It looks like there's something wrong with their server, and it's closing connections without sending a Connection: close header. The client therefore tries to reuse the connection per the HTTP/1.1 specification.
You can work around this by creating your own request, and setting Close = true, or using a custom Transport with DisableKeepAlives = true
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Close = true
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}

Resources