I'm have a function to call a Twitter API. If the input contains a keyword with a non-ascii character (q=éxito) the API responds with 401:
https://api.twitter.com/1.1/search/tweets.json?q=éxito&count=100&result_type=recent&include_entities=true
but with all-ascii characters in the URL, responds w/OK:
https://api.twitter.com/1.1/search/tweets.json?q=teampixel&count=100&result_type=recent&include_entities=true
func GetJson(url string, target interface{}) error {
e := godotenv.Load()
if e != nil {
fmt.Print(e)
}
println(url)
config := oauth1.NewConfig(os.Getenv("API_KEY"), os.Getenv("API_SECRET_KEY"))
token := oauth1.NewToken(os.Getenv("ACCESS_TOKEN"), os.Getenv("ACCESS_TOKEN_SECRET"))
// httpClient will automatically authorize http.Request's
httpClient := config.Client(oauth1.NoContext, token)
resp, e := httpClient.Get(url)
const errorDelay = 30
if e != nil {
fmt.Println("Connection Issue")
time.Sleep(errorDelay * time.Second)
return GetJson(url, target)
}
defer resp.Body.Close()
if resp.StatusCode == 429 {
fmt.Println("\nThrotteling")
time.Sleep(errorDelay * time.Second)
return GetJson(url, target)
}
if resp.StatusCode == 404 {
fmt.Println("Could not find", url)
return errors.New("not found")
}
fmt.Printf("Results: %v\n", resp.StatusCode)
return json.NewDecoder(resp.Body).Decode(target)
}
Using url.Parse():
package main
import (
"fmt"
"log"
"net/url"
)
func main() {
_url := "https://abbreviated-path?q=%C3%A9xito&count=..."
u, err := url.Parse(_url)
if err != nil {
log.Fatal(err)
}
fmt.Println(u.String())
}
appears to solve the issue:
https://abbreviated-path?q=%C3%A9xito&count=...
Related
Hello Everyone, I'm new to Go.
I'm creating a reverse proxy server using Go.
My Server has websocket. I finally get it connected.
Now I want to change websocket message body.
Sorry If my code is weird to you. Forgive me, I'm new to Go 😢
I'm wraping resp.body to NewReadWriteBody() in which contains wrapper for Read, Write and Closer. And I'm modifying message body inside it.
Here is how I'm doing with it:
package rever
// https://blog.joshsoftware.com/2021/05/25/simple-and-powerful-reverseproxy-in-go/
// https://github.com/golang/go/blob/master/src/net/http/httputil/reverseproxy.go
import (
"bytes"
"crypto/tls"
"fmt"
"io"
"io/ioutil"
"log"
"net/http"
"net/http/httputil"
"net/url"
"strconv"
)
type ReadWriteBody struct {
originBody io.ReadWriteCloser
}
func NewReadWriteBody(body io.ReadCloser) *ReadWriteBody {
b := &ReadWriteBody{}
rw, ok := body.(io.ReadWriteCloser)
if !ok {
log.Println("29: error while casting body to ReadWriteCloser")
}
b.originBody = rw
return b
}
func (b *ReadWriteBody) Read(p []byte) (n int, err error) {
buf := make([]byte, len(p))
n, err = b.originBody.Read(buf)
if err != nil {
log.Println("43: ", err.Error())
return n, err
}
buf = bytes.ReplaceAll(buf, []byte("mm.remote"), []byte("mm.local"))
copy(p[:], buf)
return len(p), nil
}
func (b *ReadWriteBody) Write(p []byte) (n int, err error) {
buf := make([]byte, len(p))
n, err = b.originBody.Write(buf)
if err != nil {
log.Println(err.Error())
return n, err
}
buf = bytes.ReplaceAll(buf, []byte("mm.local"), []byte("mm.remote"))
copy(p[:], buf)
return len(p), nil
}
func (b *ReadWriteBody) Close() error {
return b.originBody.Close()
}
type transport struct {
http.RoundTripper
}
func (t *transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
resp, err = t.RoundTripper.RoundTrip(req)
if err != nil {
log.Println("99: ", err.Error())
return nil, err
}
if resp.StatusCode == http.StatusSwitchingProtocols {
resp.Body = NewReadWriteBody(resp.Body)
return resp, nil
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Println("114: ", err.Error())
return nil, err
}
err = resp.Body.Close()
if err != nil {
log.Println("119", err.Error())
return nil, err
}
b = bytes.ReplaceAll(b, []byte("mm.remote"), []byte("mm.local"))
body := ioutil.NopCloser(bytes.NewReader(b))
resp.Body = body
resp.ContentLength = int64(len(b))
resp.Header.Set("Content-Length", strconv.Itoa(len(b)))
return resp, nil
}
var _ http.RoundTripper = &transport{}
// NewProxy takes target host and creates a reverse proxy
func NewProxy(targetHost string) (*httputil.ReverseProxy, error) {
url, err := url.Parse(targetHost)
if err != nil {
log.Println("141: ", err.Error())
return nil, err
}
proxy := httputil.NewSingleHostReverseProxy(url)
originalDirector := proxy.Director
proxy.Director = func(req *http.Request) {
originalDirector(req)
modifyRequest(req)
}
proxy.ErrorHandler = errorHandler()
dt := http.DefaultTransport.(*http.Transport).Clone()
dt.TLSClientConfig = &tls.Config{}
dt.ForceAttemptHTTP2 = false
proxy.Transport = &transport{dt}
return proxy, nil
}
func modifyRequest(req *http.Request) {
req.Host = "mm.remote"
req.Header.Set("Accept-Encoding", "identity")
}
func errorHandler() func(http.ResponseWriter, *http.Request, error) {
return func(w http.ResponseWriter, req *http.Request, err error) {
// fmt.Printf("Got error while modifying response: %v \n", err)
}
}
// ProxyRequestHandler handles the http request using proxy
func ProxyRequestHandler(proxy *httputil.ReverseProxy) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
proxy.ServeHTTP(w, r)
}
}
func Main() {
// initialize a reverse proxy and pass the actual backend server url here
proxy, err := NewProxy("https://mm.remote")
if err != nil {
log.Println(err.Error())
panic(err)
}
// handle all requests to your server using the proxy
http.HandleFunc("/", ProxyRequestHandler(proxy))
fmt.Println("Server started")
log.Fatal(http.ListenAndServe(":8008", nil))
}
I'm trying to develop a Terraform provider but I have a problem of the first request body. Here is the code:
type Body struct {
id string
}
func resourceServerCreate(d *schema.ResourceData, m interface{}) error {
key := d.Get("key").(string)
token := d.Get("token").(string)
workspace_name := d.Get("workspace_name").(string)
board_name := d.Get("board_name").(string)
resp, err := http.Post("https://api.trello.com/1/organizations?key="+key+"&token="+token+"&displayName="+workspace_name,"application/json",nil)
if err != nil {
log.Fatalln(err)
}
defer resp.Body.Close()
//lettura body.
body := new(Body)
json.NewDecoder(resp.Body).Decode(body)
log.Println("[ORCA MADONNA] il log funzia "+body.id)
d.Set("board_id",body.id)
resp1, err1 := http.Post("https://api.trello.com/1/boards?key="+key+"&token="+token+"&idOrganization="+body.id+"&=&name="+board_name,"application/json",nil)
if err1 != nil {
log.Fatalln(resp1)
}
defer resp1.Body.Close()
d.SetId(board_name)
return resourceServerRead(d, m)
}
In the log is empty, but the second call have it and work fine. How is it possible?
Go doesn't force you to check error responses, therefore it's easy to make silly mistakes. Had you checked the return value from Decode(), you would have immediately discovered a problem.
err := json.NewDecoder(resp.Body).Decode(body)
if err != nil {
log.Fatal("Decode error: ", err)
}
Decode error: json: Unmarshal(non-pointer main.Body)
So your most immediate fix is to use & to pass a pointer to Decode():
json.NewDecoder(resp.Body).Decode(&body)
Also of note, some programming editors will highlight this mistake for you:
Here's a working demonstration, including a corrected Body structure as described at json.Marshal(struct) returns “{}”:
package main
import (
"bytes"
"encoding/json"
"fmt"
"log"
"net/http"
"time"
)
type JSON = map[string]interface{}
type JSONArray = []interface{}
func ErrFatal(err error, msg string) {
if err != nil {
log.Fatal(msg+": ", err)
}
}
func handleTestRequest(w http.ResponseWriter, req *http.Request) {
w.Write(([]byte)("{\"id\":\"yourid\"}"))
}
func launchTestServer() {
http.HandleFunc("/", handleTestRequest)
go http.ListenAndServe(":8080", nil)
time.Sleep(1 * time.Second) // allow server to get started
}
// Medium: "Don’t use Go’s default HTTP client (in production)"
var restClient = &http.Client{
Timeout: time.Second * 10,
}
func DoREST(method, url string, headers, payload JSON) *http.Response {
requestPayload, err := json.Marshal(payload)
ErrFatal(err, "json.Marshal(payload")
request, err := http.NewRequest(method, url, bytes.NewBuffer(requestPayload))
ErrFatal(err, "NewRequest "+method+" "+url)
for k, v := range headers {
request.Header.Add(k, v.(string))
}
response, err := restClient.Do(request)
ErrFatal(err, "DoRest client.Do")
return response
}
type Body struct {
Id string `json:"id"`
}
func clientDemo() {
response := DoREST("POST", "http://localhost:8080", JSON{}, JSON{})
defer response.Body.Close()
var body Body
err := json.NewDecoder(response.Body).Decode(&body)
ErrFatal(err, "Decode")
fmt.Printf("Body: %#v\n", body)
}
func main() {
launchTestServer()
for i := 0; i < 5; i++ {
clientDemo()
}
}
I want to simulate a re-try option with http like:
first two http attempts with error (using some faulty urls)
the third with success (with valid url)
This is a bit tricky any idea how to do it? I try with loop on the doSomething method with different url but it doesn't make the point,
which is for example, retry at least 3 times until you get http 200, (success) any idea how could I simulate it?
maybe run in loop on following...
www.stackoverflow.com2
www.stackoverflow.com1
www.stackoverflow.com
https://play.golang.org/p/dblPh1T0XBu
package main
import (
`fmt`
`log`
"net/http"
`time`
`github.com/cenkalti/backoff/v4`
)
func main() {
b := backoff.NewExponentialBackOff()
b.MaxElapsedTime = 3 * time.Second
retryable := func() error {
val, err := doSomething("https://www.google.com1")
if err != nil {
return err
}
fmt.Println(val)
return nil
}
notify := func(err error, t time.Duration) {
log.Printf("error: %v happened at time: %v", err, t)
}
err := backoff.RetryNotify(retryable, b, notify)
if err != nil {
fmt.Errorf("error after retrying: %v", err)
}
}
func doSomething(url string) (int, error) {
res, e := http.Get(url)
if e != nil {
fmt.Println("error occurred: ", e)
return 500, e
}
return res.StatusCode, nil
}
The idea on the comment below is part of the problem, I need to use the http calls
https://play.golang.org/p/FTR7J2r-QB7
package main
import (
`fmt`
`log`
`time`
`github.com/cenkalti/backoff/v4`
)
func main() {
b := backoff.NewExponentialBackOff()
b.MaxElapsedTime = 3 * time.Second
retrybuilder := func (count int) func() error {
return func() error {
var succeed bool
count -= 1
if count == 0 {
succeed = true
}
val, err := doSomething(succeed)
if err != nil {
fmt.Println("response: ", val)
}
return err
}
}
notify := func(err error, t time.Duration) {
log.Printf("error: %v happened at time: %v", err, t)
}
err := backoff.RetryNotify(retrybuilder(3), b, notify)
if err != nil {
fmt.Printf("error after retrying: %v", err)
}
}
func doSomething(succeed bool) (int, error) {
if !succeed {
return 500, fmt.Errorf("E_SIMULATED: sim error")
}
return 200, nil
}
I want to test a httpRequest with a json body and a test file.
I don't know how to add the created test file to the request beside body json.
body := strings.NewReader(URLTest.RequestBody)
request, err := http.NewRequest(URLTest.MethodType, "localhost:"+string(listeningPort)+URLTest.URL, body)
if err != nil {
t.Fatalf("HTTP NOT WORKING")
}
fileBuffer := new(bytes.Buffer)
mpWriter := multipart.NewWriter(fileBuffer)
fileWriter, err := mpWriter.CreateFormFile("file", "testfile.pdf")
if err != nil {
t.Fatalf(err.Error())
}
file, err := os.Open("testfile.pdf")
if err != nil {
t.Fatalf(err.Error())
}
defer file.Close()
_, err = io.Copy(fileWriter, file)
if err != nil {
t.Fatalf(err.Error())
}
rec := httptest.NewRecorder()
UploadFiles(rec, request, nil)
response := rec.Result()
if response.StatusCode != URLTest.ExpectedStatusCode {
t.Errorf(URLTest.URL + " status mismatch")
}
responseBody, err := ioutil.ReadAll(response.Body)
defer response.Body.Close()
if err != nil {
t.Errorf(URLTest.URL + " cant read response")
} else {
if strings.TrimSpace(string(responseBody)) != URLTest.ExpectedResponseBody {
t.Errorf(URLTest.URL + " response mismatch - have: " + string(responseBody) + " want: " + URLTest.ExpectedResponseBody)
}
}
}
Can I add file as a value like request.FormFile.Add(...) or something?
Regarding your question about how to send a file in an HTTP request with Go, here's some sample code.
And you will need the mime/multipart package to build the form.
package main
import (
"bytes"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"net/http/httputil"
"os"
"strings"
)
func main() {
var client *http.Client
var remoteURL string
{
//setup a mocked http client.
ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
b, err := httputil.DumpRequest(r, true)
if err != nil {
panic(err)
}
fmt.Printf("%s", b)
}))
defer ts.Close()
client = ts.Client()
remoteURL = ts.URL
}
//prepare the reader instances to encode
values := map[string]io.Reader{
"file": mustOpen("main.go"), // lets assume its this file
"other": strings.NewReader("hello world!"),
}
err := Upload(client, remoteURL, values)
if err != nil {
panic(err)
}
}
func Upload(client *http.Client, url string, values map[string]io.Reader) (err error) {
// Prepare a form that you will submit to that URL.
var b bytes.Buffer
w := multipart.NewWriter(&b)
for key, r := range values {
var fw io.Writer
if x, ok := r.(io.Closer); ok {
defer x.Close()
}
// Add an image file
if x, ok := r.(*os.File); ok {
if fw, err = w.CreateFormFile(key, x.Name()); err != nil {
return
}
} else {
// Add other fields
if fw, err = w.CreateFormField(key); err != nil {
return
}
}
if _, err = io.Copy(fw, r); err != nil {
return err
}
}
// Don't forget to close the multipart writer.
// If you don't close it, your request will be missing the terminating boundary.
w.Close()
// Now that you have a form, you can submit it to your handler.
req, err := http.NewRequest("POST", url, &b)
if err != nil {
return
}
// Don't forget to set the content type, this will contain the boundary.
req.Header.Set("Content-Type", w.FormDataContentType())
// Submit the request
res, err := client.Do(req)
if err != nil {
return
}
// Check the response
if res.StatusCode != http.StatusOK {
err = fmt.Errorf("bad status: %s", res.Status)
}
return
}
Hope you can use this in your unit test
package main
import (
"net/http"
"net/http/httputil"
"net/url"
)
func main() {
target := &url.URL{Scheme: "http", Host: "www.google.com"}
proxy := httputil.NewSingleHostReverseProxy(target)
http.Handle("/google", proxy)
http.ListenAndServe(":8099", nil)
}
Reverse Proxy is works. How can I get the response body?
now httputil/reverseproxy, support than, see source
type ReverseProxy struct {
...
// ModifyResponse is an optional function that
// modifies the Response from the backend
// If it returns an error, the proxy returns a StatusBadGateway error.
ModifyResponse func(*http.Response) error
}
func rewriteBody(resp *http.Response) (err error) {
b, err := ioutil.ReadAll(resp.Body) //Read html
if err != nil {
return err
}
err = resp.Body.Close()
if err != nil {
return err
}
b = bytes.Replace(b, []byte("server"), []byte("schmerver"), -1) // replace html
body := ioutil.NopCloser(bytes.NewReader(b))
resp.Body = body
resp.ContentLength = int64(len(b))
resp.Header.Set("Content-Length", strconv.Itoa(len(b)))
return nil
}
// ...
target, _ := url.Parse("http://example.com")
proxy := httputil.NewSingleHostReverseProxy(target)
proxy.ModifyResponse = rewriteBody
httputil.ReverseProxy has a Transport field. You can use it to modify the response. For example:
type transport struct {
http.RoundTripper
}
func (t *transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
resp, err = t.RoundTripper.RoundTrip(req)
if err != nil {
return nil, err
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
err = resp.Body.Close()
if err != nil {
return nil, err
}
b = bytes.Replace(b, []byte("server"), []byte("schmerver"), -1)
body := ioutil.NopCloser(bytes.NewReader(b))
resp.Body = body
resp.ContentLength = int64(len(b))
resp.Header.Set("Content-Length", strconv.Itoa(len(b)))
return resp, nil
}
// ...
proxy := httputil.NewSingleHostReverseProxy(target)
proxy.Transport = &transport{http.DefaultTransport}
Playground example of the whole thing: http://play.golang.org/p/b0S5CbCMrI.
I don't know best solution. But you can do something like this:
package main
import (
"fmt"
"net/http"
"net/http/httputil"
"net/url"
)
func main() {
target := &url.URL{Scheme: "http", Host: "www.google.com"}
proxy := httputil.NewSingleHostReverseProxy(target)
http.Handle("/google", CustomHandler(proxy))
http.ListenAndServe(":8099", nil)
}
func CustomHandler(h http.Handler) http.HandlerFunc {
return func(res http.ResponseWriter, req *http.Request) {
h.ServeHTTP(NewCustomWriter(res), req)
}
}
type customWriter struct {
http.ResponseWriter
}
func NewCustomWriter(w http.ResponseWriter) *customWriter {
return &customWriter{w}
}
func (c *customWriter) Header() http.Header {
return c.ResponseWriter.Header()
}
func (c *customWriter) Write(data []byte) (int, error) {
fmt.Println(string(data)) //get response here
return c.ResponseWriter.Write(data)
}
func (c *customWriter) WriteHeader(i int) {
c.ResponseWriter.WriteHeader(i)
}
From source code httptest.ResponseRecorder is use for get the response from the handler
func TestModifyResponseClosesBody(t *testing.T) {
req, _ := http.NewRequest("GET", "http://foo.tld/", nil)
req.RemoteAddr = "1.2.3.4:56789"
closeCheck := new(checkCloser)
logBuf := new(bytes.Buffer)
outErr := errors.New("ModifyResponse error")
rp := &ReverseProxy{
Director: func(req *http.Request) {},
Transport: &staticTransport{&http.Response{
StatusCode: 200,
Body: closeCheck,
}},
ErrorLog: log.New(logBuf, "", 0),
ModifyResponse: func(*http.Response) error {
return outErr
},
}
rec := httptest.NewRecorder()
rp.ServeHTTP(rec, req)
res := rec.Result()
if g, e := res.StatusCode, http.StatusBadGateway; g != e {
t.Errorf("got res.StatusCode %d; expected %d", g, e)
}
if !closeCheck.closed {
t.Errorf("body should have been closed")
}
if g, e := logBuf.String(), outErr.Error(); !strings.Contains(g, e) {
t.Errorf("ErrorLog %q does not contain %q", g, e)
}
}