Golang multipart file form request - go

Question

I'm writing an API client against Mapbox, uploading a batch of svg images to a custom map. The api they provide for this is documented with an example cUrl call that works fine:
curl -F images=#include/mapbox/sprites_dark/aubergine_selected.svg "https://api.mapbox.com/styles/v1/<my_company>/<my_style_id>/sprite?access_token=$MAPBOX_API_KEY" --trace-ascii /dev/stdout
When attemting to do the same from golang I quickly came across that the multiform library is very limited, and wrote some code to make the request as similar to the cUrl request mentioned above.
func createMultipartFormData(fileMap map[string]string) (bytes.Buffer, *multipart.Writer) {
var b bytes.Buffer
var err error
w := multipart.NewWriter(&b)
var fw io.Writer
for fileName, filePath := range fileMap {
h := make(textproto.MIMEHeader)
h.Set("Content-Disposition",
fmt.Sprintf(`form-data; name="%s"; filename="%s"`, "images", fileName))
h.Set("Content-Type", "image/svg+xml")
if fw, err = w.CreatePart(h); err != nil {
fmt.Printf("Error creating form File %v, %v", fileName, err)
continue
}
fileContents, err := ioutil.ReadFile(filePath)
fileContents = bytes.ReplaceAll(fileContents, []byte("\n"), []byte("."))
blockSize := 64
remainder := len(fileContents) % blockSize
iterations := (len(fileContents) - remainder) / blockSize
newBytes := []byte{}
for i := 0; i < iterations; i++ {
start := i * blockSize
end := i*blockSize + blockSize
newBytes = append(newBytes, fileContents[start:end]...)
newBytes = append(newBytes, []byte("\n")...)
}
if remainder > 0 {
newBytes = append(newBytes, fileContents[iterations*blockSize:]...)
newBytes = append(newBytes, []byte("\n")...)
}
if err != nil {
fmt.Printf("Error reading svg file: %v: %v", filePath, err)
continue
}
_, err = fw.Write(newBytes)
if err != nil {
log.Debugf("Could not write file to multipart: %v, %v", fileName, err)
continue
}
}
w.Close()
return b, w
}
Along with setting the headers in the actual request:
bytes, formWriter := createMultipartFormData(filesMap)
req, err := http.NewRequest("Post", fmt.Sprintf("https://api.mapbox.com/styles/v1/%v/%v/sprite?access_token=%v", "my_company", styleID, os.Getenv("MAPBOX_API_KEY")), &bytes)
if err != nil {
return err
}
req.Header.Set("User-Agent", "curl/7.64.1")
req.Header.Set("Accept", "*/*")
req.Header.Set("Content-Length", fmt.Sprintf("%v", len(bytes.Bytes())))
req.Header.Set("Content-Type", formWriter.FormDataContentType())
byts, _ := httputil.DumpRequest(req, true)
fmt.Println(string(byts))
res, err := http.DefaultClient.Do(req)
Even want as far to limit the line length and replicate the encoding used by cUrl but so far no luck. Does anyone with experience know why this works from cUrl but not golang?

Answer 1

Well, I admit that all the parts of the "puzzle" to solve your task can be found on the 'net in abundance, there are two problems with this:
They quite often miss certain interesting details.
Sometimes, they give outright incorrect advice.
So, here's a working solution.
package main
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"mime"
"mime/multipart"
"net/http"
"net/textproto"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
)
func main() {
const (
dst = "https://api.mapbox.com/styles/v1/AcmeInc/Style_001/sprite"
fname = "path/to/a/sprite/image.svg"
token = "an_invalid_token"
)
err := post(dst, fname, token)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func post(dst, fname, token string) error {
u, err := url.Parse(dst)
if err != nil {
return fmt.Errorf("failed to parse destination url: %w", err)
}
form, err := makeRequestBody(fname)
if err != nil {
return fmt.Errorf("failed to prepare request body: %w", err)
}
q := u.Query()
q.Set("access_token", token)
u.RawQuery = q.Encode()
hdr := make(http.Header)
hdr.Set("Content-Type", form.contentType)
req := http.Request{
Method: "POST",
URL: u,
Header: hdr,
Body: ioutil.NopCloser(form.body),
ContentLength: int64(form.contentLen),
}
resp, err := http.DefaultClient.Do(&req)
if err != nil {
return fmt.Errorf("failed to perform http request: %w", err)
}
defer resp.Body.Close()
_, _ = io.Copy(os.Stdout, resp.Body)
return nil
}
type form struct {
body *bytes.Buffer
contentType string
contentLen int
}
func makeRequestBody(fname string) (form, error) {
ct, err := getImageContentType(fname)
if err != nil {
return form{}, fmt.Errorf(
`failed to get content type for image file "%s": %w`,
fname, err)
}
fd, err := os.Open(fname)
if err != nil {
return form{}, fmt.Errorf("failed to open file to upload: %w", err)
}
defer fd.Close()
stat, err := fd.Stat()
if err != nil {
return form{}, fmt.Errorf("failed to query file info: %w", err)
}
hdr := make(textproto.MIMEHeader)
cd := mime.FormatMediaType("form-data", map[string]string{
"name": "images",
"filename": fname,
})
hdr.Set("Content-Disposition", cd)
hdr.Set("Contnt-Type", ct)
hdr.Set("Content-Length", strconv.FormatInt(stat.Size(), 10))
var buf bytes.Buffer
mw := multipart.NewWriter(&buf)
part, err := mw.CreatePart(hdr)
if err != nil {
return form{}, fmt.Errorf("failed to create new form part: %w", err)
}
n, err := io.Copy(part, fd)
if err != nil {
return form{}, fmt.Errorf("failed to write form part: %w", err)
}
if int64(n) != stat.Size() {
return form{}, fmt.Errorf("file size changed while writing: %s", fd.Name())
}
err = mw.Close()
if err != nil {
return form{}, fmt.Errorf("failed to prepare form: %w", err)
}
return form{
body: &buf,
contentType: mw.FormDataContentType(),
contentLen: buf.Len(),
}, nil
}
var imageContentTypes = map[string]string{
"png": "image/png",
"jpg": "image/jpeg",
"jpeg": "image/jpeg",
"svg": "image/svg+xml",
}
func getImageContentType(fname string) (string, error) {
ext := filepath.Ext(fname)
if ext == "" {
return "", fmt.Errorf("file name has no extension: %s", fname)
}
ext = strings.ToLower(ext[1:])
ct, found := imageContentTypes[ext]
if !found {
return "", fmt.Errorf("unknown file name extension: %s", ext)
}
return ct, nil
}
Some random notes on implementation to help you understands the concepts:
To construct the request's payload (body), we use a bytes.Buffer instance.
It has a nice property in that a pointer to it (*bytes.Buffer) implements both io.Writer and io.Reader and hence can be easily composed with other parts of the Go stdlib which deal with I/O.
When preparing a multipart form for sending, we do not slurp the whole file's contents into memory but instead "pipe" them right into the "mutipart form writer".
We have a lookup table which maps the extension of a file name to submit to its MIME type; I have no idea whether this is needed by the API or not; if it's not really required, the part of the code which prepares a form's field containing a file could be simplified a lot, but cURL send it, so do we.

Answer 2

Just to be curious, What is this for?
fileContents = bytes.ReplaceAll(fileContents, []byte("\n"), []byte("."))
blockSize := 64
remainder := len(fileContents) % blockSize
iterations := (len(fileContents) - remainder) / blockSize
newBytes := []byte{}
for i := 0; i < iterations; i++ {
start := i * blockSize
end := i*blockSize + blockSize
newBytes = append(newBytes, fileContents[start:end]...)
newBytes = append(newBytes, []byte("\n")...)
}
if remainder > 0 {
newBytes = append(newBytes, fileContents[iterations*blockSize:]...)
newBytes = append(newBytes, []byte("\n")...)
}
if err != nil {
fmt.Printf("Error reading svg file: %v: %v", filePath, err)
continue
}
Reading a entire file into memory is rarely a good idea (ioutil.ReadFile).
As #muffin-top says, how about those three lines of code?
for fileName, filePath := range fileMap {
// h := ...
fw, _ := w.CreatePart(h) // TODO: handle error
f, _ := os.Open(filePath) // TODO: handle error
io.Copy(fw, f) // TODO: handle error
f.Close() // TODO: handle error
}