Unable to set POST body in a http request - go

Is this not the right way to set POST request body?
data := url.Values{}
data.Set("url", "https://www.google.com/")
client := http.Client{}
r, err := http.NewRequest(http.MethodPost, apiURL, strings.NewReader(data.Encode()))
The code below when executed suggests that no url param was sent in the POST request.
package main
import (
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
)
func doAPICall() {
// curl -XPOST -d 'url=https://www.google.com/' 'https://cleanuri.com/api/v1/shorten'
apiURL := "https://cleanuri.com/api/v1/shorten"
data := url.Values{}
data.Set("url", "https://www.google.com/")
client := http.Client{}
r, err := http.NewRequest(http.MethodPost, apiURL, strings.NewReader(data.Encode()))
if err != nil {
panic(err)
}
resp, err := client.Do(r)
if err != nil {
panic(err)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
fmt.Println(string(body))
}
func main() {
doAPICall()
}
Output:-
$ go run .
{"error":"API Error: URL is empty"}
$

The server expects a valid value for the Content-Type request header.
r, err := http.NewRequest(http.MethodPost, apiURL, strings.NewReader(data.Encode()))
if err != nil {
panic(err)
}
r.Header.Set("Content-Type", "application/x-www-form-urlencoded") // <-- add this line
resp, err := client.Do(r)
The server also supports JSON request bodies:
r, err := http.NewRequest(http.MethodPost, apiURL, strings.NewReader(`{"url": "https://www.google.com/"}`))
if err != nil {
panic(err)
}
r.Header.Set("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(r)

You can do something like this
package main
import (
"fmt"
"io/ioutil"
"net/http"
"strings"
)
func main() {
url := "https://cleanuri.com/api/v1/shorten"
payload := strings.NewReader("url=https://www.google.com/")
req, err := http.NewRequest("POST", url, payload)
if err != nil {
fmt.Println(err)
return
}
req.Header.Add("content-type", "application/x-www-form-urlencoded")
req.Header.Add("cache-control", "no-cache")
res, err := http.DefaultClient.Do(req)
if err != nil {
fmt.Println(err)
return
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
fmt.Println(res)
fmt.Println(string(body))
}

Related

https POST not working as expected in golang , but works fine in Python

I am trying to implement a python code from the JIRA REST API examples:
https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-jql/#api-rest-api-3-jql-parse-post
My python code (which works as expected):
import requests
from requests.auth import HTTPBasicAuth
import json
url = "https://my-url.com/rest/api/2/search"
auth = HTTPBasicAuth("user1", "pwd1")
headers = {
"Accept": "application/json",
"Content-Type": "application/json"
}
payload = json.dumps( {
"jql": "my-query-string"
}
response = requests.request("POST", url, data=payload, headers=headers, auth=auth, verify=False)
print(json.dumps(json.loads(response.text), sort_keys=True, indent=4, separators=(",", ": ")))
I'm trying to transform this to a golang code as below:
package main
import (
"io/ioutil"
"fmt"
"log"
"time"
"net/http"
"net/url"
}
func main() {
timeout := time.Duration(500 * time.Second)
client := http.Client{
Timeout: timeout,
}
req, err := http.NewRequest("POST", "https://my-url.com/rest/api/2/search", nil)
if err != nil {
log.Fatalln(err)
}
req.SetBasicAuth("user1", "pwd1")
req.Header.Set("Content-Type", "application/json")
q := url.Values{}
q.Add("jql", "my-query-string")
req.URL.RawQuery = q.Encode()
fmt.Println(req.URL.String())
resp, err := client.Do(req)
if err != nil {
log.Fatalln(err)
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatalln(err)
}
log.Println(string(data))
The code builds with no issues. When I run the go code, I get this error:
2021/04/17 19:36:31 {"errorMessages":["No content to map to Object due to end of input"]}
I have 2 questions :
a. How can I fix the above error ?
b. I also want to include concurrency in the same code, i.e the same POST request will actually be executed for 5 different query strings (concurrently) and fetch the results, how can i achieve that ?
For POST requests you need to send the data as json. Note that in Go setting a request's Content-Type header does not automagically convert whatever you give it to the specified type.
An example sending json.
package main
import (
"strings"
"net/http"
"io/ioutil"
"fmt"
)
func main() {
body := strings.NewReader(`{"jql": "project = HSP"}`)
req, err := http.NewRequest("POST", "https://your-domain.atlassian.com/rest/api/2/search", body)
if err != nil {
panic(err)
}
req.SetBasicAuth("email#example.com", "<api_token>")
req.Header.Set("Accept", "application/json")
req.Header.Set("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
out, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
fmt.Println(string(out))
}
If you want to use query parameters you should use the endpoint with the GET method.
package main
import (
"net/http"
"net/url"
"io/ioutil"
"fmt"
)
func main() {
query := url.Values{"jql": {"project = HSP"}}
req, err := http.NewRequest("GET", "https://your-domain.atlassian.com/rest/api/2/search?" + query.Encode(), nil)
if err != nil {
panic(err)
}
req.SetBasicAuth("email#example.com", "<api_token>")
req.Header.Set("Accept", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
out, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
fmt.Println(string(out))
}

gzip request body auto decompress

Doesn't Go automatically decompress gzip request body if Content-Encoding: gzip header is set?
e.g.
package main
import (
"compress/gzip"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
)
func main() {
requestBody, err := json.Marshal(map[string]string{
"accountReferenceId": "abcd",
"transactionType": "Deposit",
"amount": "100.00",
})
if err != nil {
log.Fatal(err)
}
r, w := io.Pipe()
go func() {
err := compressJSON(w, requestBody)
w.CloseWithError(err)
}()
client := &http.Client{}
req, err := http.NewRequest("POST", "http://localhost:7070/transactions", r)
if err != nil {
log.Fatal("error making request", err)
}
req.Header.Set("Content-Encoding", "gzip")
//req.Header.Set("Accept-Encoding", "gzip")
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
log.Fatal("unable to get response", err)
}
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal("unable to read response body", err)
}
fmt.Println(string(bodyBytes))
}
func compressJSON(w io.Writer, i interface{}) error {
gz := gzip.NewWriter(w)
if err := json.NewEncoder(gz).Encode(i); err != nil {
return err
}
return gz.Close()
}
When the server receives this request, the body content is still compressed. How can I make it auto decompress?

Implementing multipart file upload with extra params

I am trying to replicate the following command:
curl -X POST --header 'Content-Type: multipart/form-data' --header 'Accept: text/html; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/HTML/1.7.0"' -F wikitext=%27%27%27Mahikari%27%27%27%20is%20a%20%5B%5BJapan%5D%5Dese%20%5B%5Bnew%20religious%20movement%5D%5D -F body_only=true -F 'https://en.wikipedia.org/api/rest_v1/transform/wikitext/to/html'
The file is passed as a url quoted parameter to curl.
The content of the original file is given as (with no trailing returns):
'''Mahikari''' is a [[Japan]]ese [[new religious movement]]
The only parameter I added, for now, is body_only=true
The expected and correct answer is:
<p id="mwAQ"><b id="mwAg">Mahikari</b> is a <a rel="mw:WikiLink" href="./Japan" title="Japan" id="mwAw">Japanese</a> <a rel="mw:WikiLink" href="./New_religious_movement" title="New religious movement" id="mwBA">new religious movement</a></p>
The code below is not returning anything (not even an error!):
package main
import (
"bytes"
"fmt"
"io"
// "io/ioutil"
"log"
"mime/multipart"
"net/http"
"os"
"path/filepath"
)
// Creates a new file upload http request with optional extra params
func newfileUploadRequest(uri string, params map[string]string, paramName, path string) (*http.Request, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
// fileContents, err := ioutil.ReadAll(file)
// if err != nil {
// return nil, err
// }
fi, err := file.Stat()
if err != nil {
return nil, err
}
body := new(bytes.Buffer)
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile(paramName, fi.Name())
if err != nil {
return nil, err
}
// part.Write(fileContents)
io.Copy(part, file)
for key, val := range params {
_ = writer.WriteField(key, val)
}
err = writer.Close()
if err != nil {
return nil, err
}
request, err := http.NewRequest("POST", uri, body)
request.Header.Add("Content-Type", writer.FormDataContentType())
request.Header.Add("Accept", "text/html; charset=utf-8; profile=\"https://www.mediawiki.org/wiki/Specs/HTML/1.7.0\"")
return request, err
}
func transformWikitextToHtml(path string) {
extraParams := map[string]string{
"body_only": "true",
}
request, err := newfileUploadRequest("https://en.wikipedia.org/api/rest_v1/transform/wikitext/to/html", extraParams, "file", path)
if err != nil {
log.Fatal(err)
}
client := &http.Client{}
resp, err := client.Do(request)
if err != nil {
log.Fatal(err)
} else {
var bodyContent []byte
fmt.Println(resp.StatusCode)
fmt.Println(resp.Header)
resp.Body.Read(bodyContent)
resp.Body.Close()
fmt.Println(bodyContent)
}
}
func main() {
transformWikitextToHtml("/tmp/2239217")
}
I set up the headers according to the documentation and what is expected. I tried a few things, as reading the file at once (commented out), but that didnt help. What am I missing?
In your CURL request, you are sending wikitext as a field (-F wikitext=...).
However, in your code you are sending it as a file part.
If you send that as a field it will work as you expect.
Just include the file contents as an additional extra field in your code:
func transformWikitextToHtml(path string) {
fileBytes, err := ioutil.ReadFile(path)
if err != nil {
log.Fatal(err)
}
extraParams := map[string]string{
"body_only": "true",
"wikitext": string(fileBytes),
}
// rest of the code should be as you posted
}
Then of course, remove the parts of newfileUploadRequest that work with the path and file param name, which are not needed any more.
Also, when writing the response body, you had a small bug and it was not printing anything even once the code was fixed, so please replace that part with:
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(bodyBytes))
Full working code:
package main
import (
"bytes"
"fmt"
"log"
"mime/multipart"
"net/http"
"io/ioutil"
)
// Creates a new file upload http request with optional extra params
func newfileUploadRequest(uri string, params map[string]string) (*http.Request, error) {
body := new(bytes.Buffer)
writer := multipart.NewWriter(body)
for key, val := range params {
err := writer.WriteField(key, val)
if err != nil {
log.Fatal(err)
}
}
err := writer.Close()
if err != nil {
return nil, err
}
request, err := http.NewRequest("POST", uri, body)
request.Header.Add("Content-Type", writer.FormDataContentType())
request.Header.Add("Accept", "text/html; charset=utf-8; profile=\"https://www.mediawiki.org/wiki/Specs/HTML/1.7.0\"")
return request, err
}
func transformWikitextToHtml(path string) {
fileBytes, err := ioutil.ReadFile(path)
if err != nil {
log.Fatal(err)
}
extraParams := map[string]string{
"body_only": "true",
"wikitext": string(fileBytes),
}
request, err := newfileUploadRequest("https://en.wikipedia.org/api/rest_v1/transform/wikitext/to/html", extraParams)
if err != nil {
log.Fatal(err)
}
client := &http.Client{}
resp, err := client.Do(request)
if err != nil {
log.Fatal(err)
} else {
fmt.Println(resp.StatusCode)
fmt.Println(resp.Header)
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(bodyBytes))
}
}
func main() {
transformWikitextToHtml("/tmp/2239217")
}

How can I set HTTP Post entity like Java's method HttpPost.setEntity

I'm a new golang programmer. In java it's very easy to set with method HTTP.setEntity(). but in golang, I have test servel way to set it, but our server still missing receive entity data.
Here is code:
func bathPostDefects(){
url := "http://127.0.0.1/edit"
var jsonStr = []byte(`{"key":"abc","id":"110175653","resolve":2,"online_time":"2016-7-22","priority":1,"comment":"something.."}`)
req, err := http.NewRequest("POST",url,bytes.NewBuffer(jsonStr))
fmt.Println("ContentLength: ",len(jsonStr))
req.Header.Set("Content-Type","application/json")
req.Header.Set("Content-Length",string(len(jsonStr)))
client := &http.Client{}
resp,err := client.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
fmt.Println("response Status:", resp.Status)
fmt.Println("response Headers:", resp.Header)
body, _ := ioutil.ReadAll(resp.Body)
fmt.Println("response Body:", string(body))
}
problem find it, it is cause by our servlet has read the form values, not the request body, code update following:
func bathPostDefects(){
v := url.Values{}
v.Set("key", "abc")
v.Add("id", "110175653")
v.Add("resolve", "2")
v.Add("online_time", "2016-7-22")
v.Add("priority", "1")
v.Add("comment", "something..")
fmt.Println(v.Get("id"))
fmt.Println(v.Get("comment"))
resp, err := http.PostForm("http://127.0.0.1/edit",v)
if err != nil {
panic(err)
}
defer resp.Body.Close()
fmt.Println("response Status:", resp.Status)
fmt.Println("response Headers:", resp.Header)
body, _ := ioutil.ReadAll(resp.Body)
fmt.Println("response Body:", string(body))
}
thank you all you guys.
I changed a bit code to use NewBufferString, and tested it together with server that prints request's body:
package main
import (
"bytes"
"fmt"
"io/ioutil"
"log"
"net/http"
"time"
)
func bathPostDefects() {
url := "http://127.0.0.1:4141/"
var jsonStr = `{"key":"abc","id":"110175653","resolve":2,"online_time":"2016-7-22","priority":1,"comment":"something.."}`
req, err := http.NewRequest("POST", url, bytes.NewBufferString(jsonStr))
fmt.Println("ContentLength: ", len(jsonStr))
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Content-Length", string(len(jsonStr)))
client := &http.Client{}
_, err = client.Do(req)
if err != nil {
panic(err)
}
}
func server() {
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
body, _ := ioutil.ReadAll(r.Body)
fmt.Println("Body: ", string(body))
})
log.Fatal(http.ListenAndServe(":4141", nil))
}
func main() {
go server()
time.Sleep(time.Second)
bathPostDefects()
}

Post request with data in go

I'm trying to access an API like this:
package main
import (
"bytes"
"fmt"
"io/ioutil"
"net/http"
"net/url"
)
func main() {
apiUrl := "https://example.com/api/"
data := url.Values{}
data.Set("api_token", "MY_KEY")
data.Add("action", "list_projects")
req, _ := http.NewRequest("POST", apiUrl, bytes.NewBufferString(data.Encode()))
client := &http.Client{}
resp, err := client.Do(req)
defer resp.Body.Close()
if err == nil {
body, _ := ioutil.ReadAll(resp.Body)
fmt.Println(resp.Status)
fmt.Println(string(body))
}
}
But the response from an API tells me there was no data in POST request.
If I do it like this with curl, it works:
$ curl -X POST "https://example.com/api/" -d "api_token=MY_KEY" -d "action=list_projects"
You may want to use this form of request
resp, err := http.PostForm("http://example.com/form",
url.Values{"key": {"Value"}, "id": {"123"}})
or use the right mime type :
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
and encode data
strings.NewReader(data.Encode())
It's better if you test err != nil and return if necessary. This code may not work cause the request failed.
defer resp.Body.Close()
instead use this pattern:
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
fmt.Println(err)
return
}
defer resp.Body.Close()
body, _ := ioutil.ReadAll(resp.Body)
fmt.Println(resp.Status)
fmt.Println(string(body))
So you can see in the console if the request failed or not

Resources