Doesn't Go automatically decompress gzip request body if Content-Encoding: gzip header is set?
e.g.
package main
import (
"compress/gzip"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
)
func main() {
requestBody, err := json.Marshal(map[string]string{
"accountReferenceId": "abcd",
"transactionType": "Deposit",
"amount": "100.00",
})
if err != nil {
log.Fatal(err)
}
r, w := io.Pipe()
go func() {
err := compressJSON(w, requestBody)
w.CloseWithError(err)
}()
client := &http.Client{}
req, err := http.NewRequest("POST", "http://localhost:7070/transactions", r)
if err != nil {
log.Fatal("error making request", err)
}
req.Header.Set("Content-Encoding", "gzip")
//req.Header.Set("Accept-Encoding", "gzip")
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
log.Fatal("unable to get response", err)
}
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal("unable to read response body", err)
}
fmt.Println(string(bodyBytes))
}
func compressJSON(w io.Writer, i interface{}) error {
gz := gzip.NewWriter(w)
if err := json.NewEncoder(gz).Encode(i); err != nil {
return err
}
return gz.Close()
}
When the server receives this request, the body content is still compressed. How can I make it auto decompress?
Related
I am trying to get to grips with testing using the httptest.NewServer and I am hitting a roadblock.
In my code I am making a GET request to an external API and I want to write a test for this using httptest.NewServer.
Here is my code making the request (main.go):
package main
import (
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
)
type HTTPClient interface {
Do(req *http.Request) (*http.Response, error)
}
type NewRequest interface {
NewRequest(method string, url string, body io.Reader) (*http.Request, error)
}
var (
Client HTTPClient
)
func init() {
Client = &http.Client{}
}
func main() {
url := "https://httpbin.org/get"
GetData(url)
}
func GetData(url string) (*http.Response, error) {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
log.Fatalln(err)
return nil, err
}
resp, err := Client.Do(req)
if err != nil {
log.Fatalln(err)
return nil, err
}
defer resp.Body.Close()
responseBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
return nil, err
}
fmt.Println(resp.Status)
fmt.Println(string(responseBody))
return resp, nil
}
When I run this it works fine.
Here is my test file:
package main
import (
"fmt"
"io/ioutil"
"log"
"net/http"
"net/http/httptest"
"testing"
)
func TestYourHTTPGet(t *testing.T){
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, `response from the mock server goes here`)
}))
defer ts.Close()
mockServerURL := ts.URL
resp, err := GetData(mockServerURL)
if err != nil {
fmt.Println("Error 1: ", err)
}
defer resp.Body.Close()
responseBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal("Error 2: ", err)
}
fmt.Println(resp.Status)
fmt.Println(string(responseBody))
}
When I run go test I receive the error: http: read on closed response body. If I remove defer resp.Body.Close() from main.go the test passes correctly.
I am not sure why this is happening and was hoping that someone could explain what is going on here?
As #Cerise Limón says you call resp.Body.Close() twice and then try to read closed body. To fix yor code you can remove body processing from GetData function and do it outside GetData or return the body and do not read it in test.
main.go:
package main
import (
"fmt"
"io/ioutil"
"log"
"net/http"
)
var Client = &http.Client{}
func main() {
url := "https://httpbin.org/get"
status, data, err := GetData(url)
if err != nil {
log.Fatalln(err)
}
fmt.Println(status)
fmt.Println(string(data))
}
func GetData(url string) (status string, body []byte, err error) {
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return
}
resp, err := Client.Do(req)
if err != nil {
return
}
defer resp.Body.Close()
body, err = ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatalln(err)
}
return resp.Status, body, nil
}
main_test.go:
package main
import (
"fmt"
"net/http"
"net/http/httptest"
"testing"
)
func TestYourHTTPGet(t *testing.T){
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, `response from the mock server goes here`)
}))
defer ts.Close()
mockServerURL := ts.URL
status, data, err := GetData(mockServerURL)
if err != nil {
fmt.Println("Error 1: ", err)
}
fmt.Println(status)
fmt.Println(string(data))
}
Your GetData()'s return is a pointer. You run GetData() in main.go, when retun, it will close the resp.body. And if you read it again, it cause http: read on closed response body
So if you want read the body again, you should not return *http.Response, you should clone the resp.body to return
For example I need content from page https://aliexpress.ru/item/4001275226820.html
In Postman and in browser I get html content of page
But when I try use GO i can`t get content
package main
import (
"fmt"
"io/ioutil"
"net/http"
)
func main() {
resp, err := http.Get("https://aliexpress.ru/item/4001275226820.html")
defer resp.Body.Close()
if err != nil {panic(err)}
html, err1 := ioutil.ReadAll(resp.Body)
if err1 != nil {panic(err1)}
fmt.Printf("%s\n", html)
}
But get panic "stopped after 10 redirects"
what am I doing wrong?
The below code gets me a 200 response. You might be able to simplify it, but should be enough to get you started:
package main
import (
"fmt"
"net/http"
)
func cookies() ([]*http.Cookie, error) {
req, err := http.NewRequest(
"HEAD", "https://login.aliexpress.ru/sync_cookie_write.htm", nil,
)
if err != nil {
return nil, err
}
val := req.URL.Query()
val.Set("acs_random_token", "1")
req.URL.RawQuery = val.Encode()
res, err := new(http.Transport).RoundTrip(req)
if err != nil {
return nil, err
}
return res.Cookies(), nil
}
func main() {
req, err := http.NewRequest(
"HEAD", "https://aliexpress.ru/item/4001275226820.html", nil,
)
if err != nil {
panic(err)
}
cooks, err := cookies()
if err != nil {
panic(err)
}
for _, cook := range cooks {
if cook.Name == "xman_f" {
req.AddCookie(cook)
}
}
res, err := new(http.Transport).RoundTrip(req)
if err != nil {
panic(err)
}
fmt.Printf("%+v\n", res)
}
I am trying to implement a python code from the JIRA REST API examples:
https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-jql/#api-rest-api-3-jql-parse-post
My python code (which works as expected):
import requests
from requests.auth import HTTPBasicAuth
import json
url = "https://my-url.com/rest/api/2/search"
auth = HTTPBasicAuth("user1", "pwd1")
headers = {
"Accept": "application/json",
"Content-Type": "application/json"
}
payload = json.dumps( {
"jql": "my-query-string"
}
response = requests.request("POST", url, data=payload, headers=headers, auth=auth, verify=False)
print(json.dumps(json.loads(response.text), sort_keys=True, indent=4, separators=(",", ": ")))
I'm trying to transform this to a golang code as below:
package main
import (
"io/ioutil"
"fmt"
"log"
"time"
"net/http"
"net/url"
}
func main() {
timeout := time.Duration(500 * time.Second)
client := http.Client{
Timeout: timeout,
}
req, err := http.NewRequest("POST", "https://my-url.com/rest/api/2/search", nil)
if err != nil {
log.Fatalln(err)
}
req.SetBasicAuth("user1", "pwd1")
req.Header.Set("Content-Type", "application/json")
q := url.Values{}
q.Add("jql", "my-query-string")
req.URL.RawQuery = q.Encode()
fmt.Println(req.URL.String())
resp, err := client.Do(req)
if err != nil {
log.Fatalln(err)
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatalln(err)
}
log.Println(string(data))
The code builds with no issues. When I run the go code, I get this error:
2021/04/17 19:36:31 {"errorMessages":["No content to map to Object due to end of input"]}
I have 2 questions :
a. How can I fix the above error ?
b. I also want to include concurrency in the same code, i.e the same POST request will actually be executed for 5 different query strings (concurrently) and fetch the results, how can i achieve that ?
For POST requests you need to send the data as json. Note that in Go setting a request's Content-Type header does not automagically convert whatever you give it to the specified type.
An example sending json.
package main
import (
"strings"
"net/http"
"io/ioutil"
"fmt"
)
func main() {
body := strings.NewReader(`{"jql": "project = HSP"}`)
req, err := http.NewRequest("POST", "https://your-domain.atlassian.com/rest/api/2/search", body)
if err != nil {
panic(err)
}
req.SetBasicAuth("email#example.com", "<api_token>")
req.Header.Set("Accept", "application/json")
req.Header.Set("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
out, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
fmt.Println(string(out))
}
If you want to use query parameters you should use the endpoint with the GET method.
package main
import (
"net/http"
"net/url"
"io/ioutil"
"fmt"
)
func main() {
query := url.Values{"jql": {"project = HSP"}}
req, err := http.NewRequest("GET", "https://your-domain.atlassian.com/rest/api/2/search?" + query.Encode(), nil)
if err != nil {
panic(err)
}
req.SetBasicAuth("email#example.com", "<api_token>")
req.Header.Set("Accept", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
out, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
fmt.Println(string(out))
}
Is this not the right way to set POST request body?
data := url.Values{}
data.Set("url", "https://www.google.com/")
client := http.Client{}
r, err := http.NewRequest(http.MethodPost, apiURL, strings.NewReader(data.Encode()))
The code below when executed suggests that no url param was sent in the POST request.
package main
import (
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
)
func doAPICall() {
// curl -XPOST -d 'url=https://www.google.com/' 'https://cleanuri.com/api/v1/shorten'
apiURL := "https://cleanuri.com/api/v1/shorten"
data := url.Values{}
data.Set("url", "https://www.google.com/")
client := http.Client{}
r, err := http.NewRequest(http.MethodPost, apiURL, strings.NewReader(data.Encode()))
if err != nil {
panic(err)
}
resp, err := client.Do(r)
if err != nil {
panic(err)
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
fmt.Println(string(body))
}
func main() {
doAPICall()
}
Output:-
$ go run .
{"error":"API Error: URL is empty"}
$
The server expects a valid value for the Content-Type request header.
r, err := http.NewRequest(http.MethodPost, apiURL, strings.NewReader(data.Encode()))
if err != nil {
panic(err)
}
r.Header.Set("Content-Type", "application/x-www-form-urlencoded") // <-- add this line
resp, err := client.Do(r)
The server also supports JSON request bodies:
r, err := http.NewRequest(http.MethodPost, apiURL, strings.NewReader(`{"url": "https://www.google.com/"}`))
if err != nil {
panic(err)
}
r.Header.Set("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(r)
You can do something like this
package main
import (
"fmt"
"io/ioutil"
"net/http"
"strings"
)
func main() {
url := "https://cleanuri.com/api/v1/shorten"
payload := strings.NewReader("url=https://www.google.com/")
req, err := http.NewRequest("POST", url, payload)
if err != nil {
fmt.Println(err)
return
}
req.Header.Add("content-type", "application/x-www-form-urlencoded")
req.Header.Add("cache-control", "no-cache")
res, err := http.DefaultClient.Do(req)
if err != nil {
fmt.Println(err)
return
}
defer res.Body.Close()
body, _ := ioutil.ReadAll(res.Body)
fmt.Println(res)
fmt.Println(string(body))
}
I am trying to replicate the following command:
curl -X POST --header 'Content-Type: multipart/form-data' --header 'Accept: text/html; charset=utf-8; profile="https://www.mediawiki.org/wiki/Specs/HTML/1.7.0"' -F wikitext=%27%27%27Mahikari%27%27%27%20is%20a%20%5B%5BJapan%5D%5Dese%20%5B%5Bnew%20religious%20movement%5D%5D -F body_only=true -F 'https://en.wikipedia.org/api/rest_v1/transform/wikitext/to/html'
The file is passed as a url quoted parameter to curl.
The content of the original file is given as (with no trailing returns):
'''Mahikari''' is a [[Japan]]ese [[new religious movement]]
The only parameter I added, for now, is body_only=true
The expected and correct answer is:
<p id="mwAQ"><b id="mwAg">Mahikari</b> is a <a rel="mw:WikiLink" href="./Japan" title="Japan" id="mwAw">Japanese</a> <a rel="mw:WikiLink" href="./New_religious_movement" title="New religious movement" id="mwBA">new religious movement</a></p>
The code below is not returning anything (not even an error!):
package main
import (
"bytes"
"fmt"
"io"
// "io/ioutil"
"log"
"mime/multipart"
"net/http"
"os"
"path/filepath"
)
// Creates a new file upload http request with optional extra params
func newfileUploadRequest(uri string, params map[string]string, paramName, path string) (*http.Request, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
// fileContents, err := ioutil.ReadAll(file)
// if err != nil {
// return nil, err
// }
fi, err := file.Stat()
if err != nil {
return nil, err
}
body := new(bytes.Buffer)
writer := multipart.NewWriter(body)
part, err := writer.CreateFormFile(paramName, fi.Name())
if err != nil {
return nil, err
}
// part.Write(fileContents)
io.Copy(part, file)
for key, val := range params {
_ = writer.WriteField(key, val)
}
err = writer.Close()
if err != nil {
return nil, err
}
request, err := http.NewRequest("POST", uri, body)
request.Header.Add("Content-Type", writer.FormDataContentType())
request.Header.Add("Accept", "text/html; charset=utf-8; profile=\"https://www.mediawiki.org/wiki/Specs/HTML/1.7.0\"")
return request, err
}
func transformWikitextToHtml(path string) {
extraParams := map[string]string{
"body_only": "true",
}
request, err := newfileUploadRequest("https://en.wikipedia.org/api/rest_v1/transform/wikitext/to/html", extraParams, "file", path)
if err != nil {
log.Fatal(err)
}
client := &http.Client{}
resp, err := client.Do(request)
if err != nil {
log.Fatal(err)
} else {
var bodyContent []byte
fmt.Println(resp.StatusCode)
fmt.Println(resp.Header)
resp.Body.Read(bodyContent)
resp.Body.Close()
fmt.Println(bodyContent)
}
}
func main() {
transformWikitextToHtml("/tmp/2239217")
}
I set up the headers according to the documentation and what is expected. I tried a few things, as reading the file at once (commented out), but that didnt help. What am I missing?
In your CURL request, you are sending wikitext as a field (-F wikitext=...).
However, in your code you are sending it as a file part.
If you send that as a field it will work as you expect.
Just include the file contents as an additional extra field in your code:
func transformWikitextToHtml(path string) {
fileBytes, err := ioutil.ReadFile(path)
if err != nil {
log.Fatal(err)
}
extraParams := map[string]string{
"body_only": "true",
"wikitext": string(fileBytes),
}
// rest of the code should be as you posted
}
Then of course, remove the parts of newfileUploadRequest that work with the path and file param name, which are not needed any more.
Also, when writing the response body, you had a small bug and it was not printing anything even once the code was fixed, so please replace that part with:
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(bodyBytes))
Full working code:
package main
import (
"bytes"
"fmt"
"log"
"mime/multipart"
"net/http"
"io/ioutil"
)
// Creates a new file upload http request with optional extra params
func newfileUploadRequest(uri string, params map[string]string) (*http.Request, error) {
body := new(bytes.Buffer)
writer := multipart.NewWriter(body)
for key, val := range params {
err := writer.WriteField(key, val)
if err != nil {
log.Fatal(err)
}
}
err := writer.Close()
if err != nil {
return nil, err
}
request, err := http.NewRequest("POST", uri, body)
request.Header.Add("Content-Type", writer.FormDataContentType())
request.Header.Add("Accept", "text/html; charset=utf-8; profile=\"https://www.mediawiki.org/wiki/Specs/HTML/1.7.0\"")
return request, err
}
func transformWikitextToHtml(path string) {
fileBytes, err := ioutil.ReadFile(path)
if err != nil {
log.Fatal(err)
}
extraParams := map[string]string{
"body_only": "true",
"wikitext": string(fileBytes),
}
request, err := newfileUploadRequest("https://en.wikipedia.org/api/rest_v1/transform/wikitext/to/html", extraParams)
if err != nil {
log.Fatal(err)
}
client := &http.Client{}
resp, err := client.Do(request)
if err != nil {
log.Fatal(err)
} else {
fmt.Println(resp.StatusCode)
fmt.Println(resp.Header)
bodyBytes, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(bodyBytes))
}
}
func main() {
transformWikitextToHtml("/tmp/2239217")
}