I have created some Go functions that make HTTP GET calls to services that are out there on the internet and parse the results.
I am now working on writing test-cases for these functions.
In my test cases, I'm using the go package httptest to simulate calls to these external services. Below is my code. Error checking is purposefully removed for brevity. Here is the go-playground.
package main
import (
"fmt"
"io"
"context"
"net/http"
"net/http/httptest"
)
func handlerResponse() http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte(`{"A":"B"}`))
})
}
func buildMyRequest(ctx context.Context, url string) *http.Request {
request, _ := http.NewRequestWithContext(ctx, "GET", url, nil)
return request
}
func myPrint(response *http.Response) {
b := make([]byte, 60000)
for {
_, err := response.Body.Read(b)
if err == io.EOF {
break
}
}
fmt.Println(string(b))
}
func main() {
srv := httptest.NewServer(handlerResponse())
client := http.Client{}
myResponse1, _ := client.Do(buildMyRequest(context.Background(), srv.URL))
fmt.Println("myResponse1:")
myPrint(myResponse1)
myResponse2, _ := client.Do(buildMyRequest(context.Background(), srv.URL))
fmt.Println("myResponse2:")
myPrint(myResponse2)
}
This is the output it produces:
myResponse1:
{"A":"B"}
myResponse2:
{"A":"B"}
As you can see, I have created some dummy HTTP response data {"A":"B"} and when you send an HTTP request to srv.URL, it actually hits an ephemeral HTTP server which responds with the dummy data. Cool!
When you send the second HTTP request to srv.URL, it again responds with the same dummy data. But this is where my problem arises. I want the ephemeral HTTP server to return some different data the second time {"C":"D"} and third time {"E":"F"} it receives a request.
How can I change the first line of the main() function so that the server responds with my desired data on subsequent HTTP calls?
you could use a hack like follows ( playground : here)
package main
import (
"fmt"
"io"
"context"
"net/http"
"net/http/httptest"
"sync"
)
type responseWriter struct{
resp map[int]string
count int
lock *sync.Mutex
}
func NewResponseWriter()*responseWriter{
r := new(responseWriter)
r.lock = new(sync.Mutex)
r.resp = map[int]string{
0: `{"E":"F"}`,
1: `{"A":"B"}`,
2: `{"C":"D"}`,
}
r.count = 0
return r
}
func (r *responseWriter)GetResp()string{
r.lock.Lock()
defer r.lock.Unlock()
r.count ++
return r.resp[r.count%3]
}
func handlerResponse(rr *responseWriter) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
w.Write([]byte(rr.GetResp()))
})
}
func buildMyRequest(ctx context.Context, url string) *http.Request {
request, _ := http.NewRequestWithContext(ctx, "GET", url, nil)
return request
}
func myPrint(response *http.Response) {
b := make([]byte, 60000)
for {
_, err := response.Body.Read(b)
if err == io.EOF {
break
}
}
fmt.Println(string(b))
}
func main() {
rr := NewResponseWriter()
srv := httptest.NewServer(handlerResponse(rr))
client := http.Client{}
myResponse1, err := client.Do(buildMyRequest(context.Background(), srv.URL))
if err != nil{
fmt.Println(err)
return
}
defer myResponse1.Body.Close()
fmt.Println("myResponse1:")
myPrint(myResponse1)
myResponse2, err := client.Do(buildMyRequest(context.Background(), srv.URL))
if err != nil{
fmt.Println(err)
return
}
defer myResponse2.Body.Close()
fmt.Println("myResponse2:")
myPrint(myResponse2)
}
Related
I’m working on proxy server with gin and ServeHTTP.
Actually GET and OPTIONS request works well. But when I trying multiples POST request I get EOF error one in two request.
I’ve test to make repeat request without proxy service and its work well, so there is something not working in my code.
Edit :
I have test POST proxy with https://ptsv2.com/ and all request response return 200 status code.
// main.go
package main
import (
"fmt"
"../pkg/api"
)
func main() {
fmt.Println("Starting server")
api.InitServer()
}
// routes.go
package api
import (
"github.com/gin-gonic/gin"
)
const serviceUrl = "http://localhost:8732"
func InitServer() {
router := gin.Default()
defineRoutes(router)
router.Run()
}
func defineRoutes(router *gin.Engine) {
router.GET("/ping", Ping)
router.POST("/*any", Proxy(serviceUrl))
}
// controller.go
package api
import (
"bytes"
"fmt"
"io/ioutil"
"net/http"
"net/http/httputil"
"net/url"
"strconv"
"github.com/gin-gonic/gin"
)
type transport struct {
http.RoundTripper
}
func (t *transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
resp, err = t.RoundTripper.RoundTrip(req)
if err != nil {
// EOF ERROR HERE
return nil, err
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
err = resp.Body.Close()
if err != nil {
return nil, err
}
b = bytes.Replace(b, []byte("server"), []byte("schmerver"), -1)
body := ioutil.NopCloser(bytes.NewReader(b))
response := resp
response.Body = body
response.ContentLength = int64(len(b))
response.Header.Set("Content-Length", strconv.Itoa(len(b)))
response.Header.Set("Access-Control-Allow-Origin", "*")
return response, nil
}
func Proxy(targetUrl string) gin.HandlerFunc {
fn := func(c *gin.Context) {
remote, err := url.Parse(targetUrl)
if err != nil {
panic(err)
}
proxy := httputil.NewSingleHostReverseProxy(remote)
proxy.Director = func(req *http.Request) {
req.Header = c.Request.Header
req.Host = remote.Host
req.URL.Scheme = remote.Scheme
req.URL.Host = remote.Host
req.Close = true // True / False same result
}
proxy.Transport = &transport{http.DefaultTransport}
proxy.ServeHTTP(c.Writer, c.Request)
}
return fn
}
conclusion
your code have no bug. it works. maybe your network setting is wrong.
explain
I download your code and test it with a local backend server. It works.
appendix
backend server code
package main
import (
"fmt"
"github.com/gin-gonic/gin"
)
func main() {
fmt.Println("Starting server")
router := gin.Default()
router.POST("/*any", func(c *gin.Context) {
c.JSON(200, gin.H{
"mes": "hello",
})
})
router.Run(":8888")
}
I have rest-ful interface written in golang. I need to do the authentication on each endpoint. once authentication is done I have to forward it back to a tcp server.
I created a tcp client and any value that is coming from the channel is to be send to tcp server. the channel is populated from the http request body.
The issue is that once I issue curl command the client is stuck with no response; so obviously I am doing something wrong not sure what is wrong. does anyone have any insights on what my problem might be?
package main
import (
"bufio"
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"log"
"net"
"net/http"
"net/http/httputil"
"net/url"
"os"
"strconv"
auth "github.com/abbot/go-http-auth"
)
type Configuration struct {
Server string
Port int64
UserName string
Pwd string
Realm string
ProxyPort int64
Edeserver string
}
var (
Config *Configuration
logp = flag.Bool("log", false, "enable logging")
)
func ReadConfiguration() {
file, _ := os.Open("Config.json")
decoder := json.NewDecoder(file)
Config = &Configuration{}
err := decoder.Decode(&Config)
if err != nil {
fmt.Println("error:", err)
}
}
func Secret(user, realm string) string {
if user == Config.UserName {
// password is "hello"
return Config.Pwd
}
return ""
}
func reverseProxyTows(w http.ResponseWriter, authenticatedRequest *auth.AuthenticatedRequest) {
req := &authenticatedRequest.Request
if *logp {
log.Println(" Authenticated Username ", authenticatedRequest.Username)
log.Println(" Authenticated URL ", req.URL.RequestURI())
}
destinationURL := fmt.Sprintf("http://%s:%d", Config.Server, Config.Port)
u, err := url.Parse(destinationURL)
if err != nil {
log.Fatal(err)
}
if *logp {
log.Println("reverse_proxy", u)
}
reverseProxy := httputil.NewSingleHostReverseProxy(u)
reverseProxy.ServeHTTP(w, req)
}
func openConnectionTotcp(edechannel chan string) {
conn, _ := net.Dial("tcp", Config.Edeserver)
text := <-edechannel
fmt.Fprintf(conn, text+"\n")
message, _ := bufio.NewReader(conn).ReadString('\n')
fmt.Print("Message from server: " + message)
}
func main() {
ReadConfiguration()
flag.Parse()
c := make(chan string)
go openConnectionTotcp(c)
fmt.Printf("Started proxy to destination server %v:%d and is listening on %d ", Config.Server, Config.Port, Config.ProxyPort)
authenticator := auth.NewBasicAuthenticator(Config.Realm, Secret)
http.HandleFunc("/", authenticator.Wrap(reverseProxyTows))
http.HandleFunc("/tyrion/1", authenticator.Wrap(func(w http.ResponseWriter, authenticatedRequest *auth.AuthenticatedRequest) {
req := &authenticatedRequest.Request
bodyBytes, err2 := ioutil.ReadAll(req.Body)
if err2 != nil {
log.Fatal(err2)
}
bodyString := string(bodyBytes)
c <- bodyString
fmt.Fprintf(w, "success")
}))
http.ListenAndServe(":"+strconv.FormatInt(Config.ProxyPort, 10), nil)
}
Your code execution blocks at c <- bodyString because nothing appears to be reading from that unbuffered channel. That line will pause execution until another routine reads from the channel.
I would like to ask if we can create 'middleware' functions for Go http client? Example I want to add a log function, so every sent request will be logged, or add setAuthToken so the token will be added to each request's header.
You can use the Transport parameter in HTTP client to that effect, with a composition pattern, using the fact that:
http.Client.Transport defines the function that will handle all HTTP requests;
http.Client.Transport has interface type http.RoundTripper, and can thus be replaced with your own implementation;
For example:
package main
import (
"fmt"
"net/http"
)
// This type implements the http.RoundTripper interface
type LoggingRoundTripper struct {
Proxied http.RoundTripper
}
func (lrt LoggingRoundTripper) RoundTrip(req *http.Request) (res *http.Response, e error) {
// Do "before sending requests" actions here.
fmt.Printf("Sending request to %v\n", req.URL)
// Send the request, get the response (or the error)
res, e = lrt.Proxied.RoundTrip(req)
// Handle the result.
if (e != nil) {
fmt.Printf("Error: %v", e)
} else {
fmt.Printf("Received %v response\n", res.Status)
}
return
}
func main() {
httpClient := &http.Client{
Transport: LoggingRoundTripper{http.DefaultTransport},
}
httpClient.Get("https://example.com/")
}
Feel free to alter names as you wish, I did not think on them for very long.
I worked on a project that had similar requirement so I built a middleware pipeline library that allows setting multiple middleware to the http client. You can check it out here.
Using the library, you would solve this in the following way
type LoggingMiddleware struct{}
func (s LoggingMiddleware) Intercept(pipeline pipeline.Pipeline, req *http.Request) (*http.Response, error) {
body, _ := httputil.DumpRequest(req, true)
log.Println(fmt.Sprintf("%s", string(body)))
/*
If you want to perform an action based on the response, do the following
resp, err = pipeline.Next
// perform some action
return resp, err
*/
return pipeline.Next(req)
}
transport := pipeline.NewCustomTransport(&LoggingMiddleware{})
client := &http.Client{Transport: transport}
resp, err := client.Get("https://example.com")
if err != nil {
// handle err
}
fmt.Println(resp.Status)
I wrote a small tutorial/library to do just that https://github.com/HereMobilityDevelopers/mediary
Here is some basic usage example:
client := mediary.Init().AddInterceptors(dumpInterceptor).Build()
client.Get("https://golang.org")
func dumpInterceptor(req *http.Request, handler mediary.Handler) (*http.Response, error) {
if bytes, err := httputil.DumpRequestOut(req, true); err == nil {
fmt.Printf("%s", bytes)
//GET / HTTP/1.1
//Host: golang.org
//User-Agent: Go-http-client/1.1
//Accept-Encoding: gzip
}
return handler(req)
}
There is also an explanation here https://github.com/HereMobilityDevelopers/mediary/wiki/Reasoning
Good idea! Here is a simple implementation of HTTP service middleware in Go.
Usually a simple http service framework is to register a bunch of routes, and then call different logics to process them according to the routes.
But in fact, there may be some unified processing involving almost all routes, such as logs, permissions, and so on.
So it is a good idea to engage in intermediate preprocessing at this time.
Define a middleware unit:
package main
import (
"net/http"
)
// AdaptorHandle middleware func type
type AdaptorHandle func(w http.ResponseWriter, r *http.Request) (next bool, err error)
// MiddleWareAdaptor router middlewares mapped by url
type MiddleWareAdaptor struct {
URLs map[string][]AdaptorHandle
}
// MakeMiddleWareAdaptor make a middleware adaptor
func MakeMiddleWareAdaptor() *MiddleWareAdaptor {
mwa := &MiddleWareAdaptor{
URLs: make(map[string][]AdaptorHandle),
}
return mwa
}
// Regist regist a adaptor
func (mw *MiddleWareAdaptor) Regist(url string, Adaptor ...AdaptorHandle) {
for _, adp := range Adaptor {
mw.URLs[url] = append(mw.URLs[url], adp)
// mw.URLs[url] = adp
}
}
// Exec exec middleware adaptor funcs...
func (mw *MiddleWareAdaptor) Exec(url string, w http.ResponseWriter, r *http.Request) (bool, error) {
if adps, ok := mw.URLs[url]; ok {
for _, adp := range adps {
if next, err := adp(w, r); !next || (err != nil) {
return next, err
}
}
}
return true, nil
}
Then wrap the route processing function with a middleware entry:
func middlewareHandler(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// before call handler
start := time.Now()
do, _ := mwa.Exec(r.URL.Path, w, r) // exec middleware
// call next handler
if do {
log.Println("middleware done. next...")
next.ServeHTTP(w, r)
} else {
log.Println("middleware done.break...")
}
// after call handle
log.Printf("Comleted %s in %v", r.URL.Path, time.Since(start))
})
}
mux.Handle("/", middlewareHandler(&uPlusRouterHandler{}))
type uPlusRouterHandler struct {
}
func (rh *uPlusRouterHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
...
}
Finally, register the middleware you need:
mwa = MakeMiddleWareAdaptor() // init middleware
mwa.Regist("/", testMWAfunc, testMWAfunc2) // regist middleware
...
func testMWAfunc(w http.ResponseWriter, r *http.Request) (bool, error) {
log.Println("I am Alice Middleware...")
log.Printf("Started %s %s", r.Method, r.URL.Path)
return true, nil
}
func testMWAfunc2(w http.ResponseWriter, r *http.Request) (bool, error) {
log.Println("I am Ben Middleware...")
return false, nil // return false,break follow-up actions.
}
This can be achieved using closure functions. It's probably more clear with an example:
package main
import (
"fmt"
"net/http"
)
func main() {
http.HandleFunc("/hello", logged(hello))
http.ListenAndServe(":3000", nil)
}
func logged(f func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
fmt.Println("logging something")
f(w, r)
fmt.Println("finished handling request")
}
}
func hello(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, "<h1>Hello!</h1>")
}
credit goes to: http://www.calhoun.io/5-useful-ways-to-use-closures-in-go/
I've a reverse proxy like this:
Iam using RoundTrip but this proxy server don't work correctly.
How to correctly read and modify response?
and somebody create proxy server via NewSingleHostReverseProxy.
Please Help.
package main
import (
"bytes"
"fmt"
"io/ioutil"
"net/http"
"net/http/httputil"
"net/url"
)
type transport struct {
http.RoundTripper
}
func (t *transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
resp, err = t.RoundTripper.RoundTrip(req)
if err != nil {
return nil, err
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
err = resp.Body.Close()
if err != nil {
return nil, err
}
b = bytes.Replace(b, []byte("Google"), []byte("GOOGLE"), -1)
body := ioutil.NopCloser(bytes.NewReader(b))
resp.Body = body
return resp, nil
}
func sameHost(handler http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
r.Host = r.URL.Host
handler.ServeHTTP(w, r)
})
}
func main() {
u, _ := url.Parse("http://habrahabr.ru")
reverseProxy := httputil.NewSingleHostReverseProxy(u)
reverseProxy.Transport = &transport{http.DefaultTransport}
// wrap that proxy with our sameHost function
singleHosted := sameHost(reverseProxy)
http.ListenAndServe(":3000", singleHosted)
}
When you are going to http:// for most good sites (for example your habrahabr.ru) there is a redirect to https://, so request to http will return something like 301 Moved Permanently and you will not find content that you seek for. Also, after correct to https, make sure that site does not use javascript to load content, you can easily check this by curl:
curl localhost:3000
Also use some logging to determine what's wrong.
I have a problem launching multiple go routines over 90000 to do http requests via POST to the same server, both the server and the client are locally, some of the requests are executed successfully and some of them giving me this response read: connection reset by peer
Please notice I am posting a small amount of data in the provided example
but actually I am sending a huge amount of data using protobuf.
This is the server https://play.golang.org/p/r1-rYNuAos
package main
import (
"net/http"
"log"
"encoding/json"
)
var port string
type problem struct{
}
func main() {
p := &problem{}
p.server(":9090")
}
func (self *problem)server(port string) {
s := &http.Server{
Addr: port,
Handler: self,
}
log.Println("Server started")
// Should be last line as it is a blocking.
log.Fatal(s.ListenAndServe())
}
func (self *problem) ServeHTTP(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusCreated)
if err := json.NewEncoder(w).Encode(map[string]interface{}{"created": true}); err != nil {
log.Fatal(err.Error())
}
}
And this is the client https://play.golang.org/p/Xx5qQoqrYm
package main
import (
"net/http"
"bytes"
"io/ioutil"
"log"
"fmt"
)
type problem struct{
}
func main() {
p := &problem{}
p.client(":9090")
}
func (self *problem)client(port string) {
var k int
for k=0;k<90000;k++ {
go func(){
nativeRequest, err := http.NewRequest(
"POST",
"http://127.0.0.1" + port + "/",
bytes.NewBuffer([]byte(`testing`)),
)
nativeRequest.Close = true
if err != nil {
fmt.Println(err.Error())
}
client := &http.Transport{
}
nativeResponse, err := client.RoundTrip(nativeRequest)
if err != nil {
log.Println(err.Error())
}
if nativeResponse != nil {
defer nativeResponse.Body.Close()
if err != nil {
fmt.Println(err.Error())
}
body, err := ioutil.ReadAll(nativeResponse.Body)
if err != nil {
fmt.Println(err.Error())
}
fmt.Println(string(body))
}
}()
}
}