Related
I'm new in golang and programming at all, so I've a problem with this function that supposed gets it all values from a table, but just shows me one. Thanks you all for your knowledge :)
func GetAll(w http.ResponseWriter, r *http.Request) {
results := map[string]interface{}{}
c, _ := connection.GetDB()
c.Table("products").Order("id_producto asc").Find(&results)
fmt.Print(results)
jsonString, _ := json.Marshal(results)
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, string(jsonString))
}
Try to create a DTO struct and create a list that contains item of type that struct. Then pass that list to the query. Like this:
type Product struct {
ProductID string
}
func GetAll(w http.ResponseWriter, r *http.Request) {
products := []*Product{}
c, _ := connection.GetDB()
c.Order("id_producto asc").Find(&products).
jsonString, _ := json.Marshal(products)
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, string(jsonString))
}
You can code like this:
func GetAll(w http.ResponseWriter, r *http.Request) {
// New a slice to get the results,
// map[string]interface{}{} will only scan one item of results
results := []map[string]interface{}{}
c, _ := connection.GetDB()
c.Table("products").Order("id_producto asc").Find(&results)
fmt.Print(results)
jsonString, _ := json.Marshal(results)
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, string(jsonString))
}
I am trying to customize request pipeline through middleware pattern, the code as follow:
func helloHandler(w http.ResponseWriter, r *http.Request) {
fmt.Println("Hello, middleware!")
}
func middleware1(next http.HandlerFunc) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
fmt.Println("[START] middleware1")
ctx := r.Context()
ctx = context.WithValue(ctx, middleware1Key, middleware1Value)
r = r.WithContext(ctx)
next(w, r)
fmt.Println("[END] middleware1")
ctx = r.Context()
if val, ok := ctx.Value(middleware2Key).(string); ok {
fmt.Printf("Value from middleware2 %s \n", val)
}
}
}
func middleware2(next http.HandlerFunc) func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
fmt.Println("[START] middleware2")
ctx := r.Context()
if val, ok := ctx.Value(middleware1Key).(string); ok {
fmt.Printf("Value from middleware1 %s \n", val)
}
ctx = context.WithValue(ctx, middleware2Key, middleware2Value)
r = r.WithContext(ctx)
next(w, r)
fmt.Println("[END] middleware2")
}
}
func main() {
mux := http.NewServeMux()
middlewares := newMws(middleware1, middleware2)
mux.HandleFunc("/hello", middlewares.then(helloHandler))
if err := http.ListenAndServe(":8080", mux); err != nil {
panic(err)
}
}
and the output is :
[START] middleware1
[START] middleware2
Value from middleware1 middleware1Value
Hello, middleware!
[END] middleware2
[END] middleware1
According to the output, the value could pass from parent to the child , while, if the child add something to the context, it is invisible to the parent
How can I propagate value from child middleware to parent?
What you're doing is creating a new pointer to the modified http.Request via WithContext method. So if you're passing it to next middleware in the chain everything works as expected since you're passing this new pointer as an argument.
If you want to modify the request and make it visible for those who hold the pointer to it, you need to dereference the pointer and set the modified value.
So in your 'child' middleware instead of:
r = r.WithContext(ctx)
Just do the following:
*r = *r.WithContext(ctx)
Good exercise to understand pointers in Go but you SHOULD NOT do similar operations in your production code. The docs are clear about it. See https://pkg.go.dev/net/http#Handler.
Another possible solution (without messing with the request itself) is to pass a map inside a context and read/write from/to map instead. So in your first middleware:
ctx := r.Context()
m := make(map[string]string)
m[middleware1Key] = middleware1Value
ctx = context.WithValue(ctx, dummyStoreKey, m)
r = r.WithContext(ctx)
...
if val, ok := m[middleware2Key]; ok {
fmt.Printf("Value from middleware2 %s \n", val)
}
And in the second one:
ctx := r.Context()
if store, ok := ctx.Value(dummyStoreKey).(map[string]string); ok {
if val, ok := store[middleware1Key]; ok {
fmt.Printf("Value from middleware1 %s \n", val)
}
store[middleware2Key] = middleware2Value
}
You could add a AddStoreMiddleware as the first one in the pipeline and then use it in each successor if needed. Remember, maps in Go are not concurrently safe so in some subtle cases you should serialize access.
This code has two parts. One is to set and clear the session and the second part is login and logout.
What it does?
In the second part, If an email and password are found in the database and the match is true then it set the session and move to the about() function which has an about file. If the logout is called then it clears the session and redirects to the home page.
What it should do?
The problem is that even if I am logged out and the session is cleared, I can still visit an about page. I don't want to be allowed to visit an about page if I am not logged in.
First part
var cookieHandler = securecookie.New(
securecookie.GenerateRandomKey(64),
securecookie.GenerateRandomKey(32),
)
func setSession(email, password string, res http.ResponseWriter) {
value := map[string]string{ "email": email, "password": password}
encoded, err := cookieHandler.Encode("session", value)
if err == nil {
cookie := &http.Cookie{ Name: "session", Value: encoded, Path: "/"}
http.SetCookie(res, cookie)
}
}
func clearSession(res http.ResponseWriter) {
cookie := &http.Cookie{ Name: "session", Value: "", Path: "/", MaxAge: -1}
http.SetCookie(res, cookie)
}
Second part
func about(res http.ResponseWriter, req *http.Request) {
if err := tmpl.ExecuteTemplate(res, "about.html", nil); err != nil {
log.Fatal("template didn't execute", nil)
}
}
func loginAuth(res http.ResponseWriter, req *http.Request) {
email := req.FormValue("email")
password := req.FormValue("password")
match := database.Findaccount(email, password)
if match == true {
setSession(email, password, res)
about(res, req)
fmt.Println("You're logged in")
} else {
tmpl.ExecuteTemplate(res, "login.html", nil)
fmt.Println("Enter the correct email or password")
}
}
func logout(res http.ResponseWriter, req *http.Request) {
clearSession(res)
http.Redirect(res, req, "/", 302)
}
Few things you don't want to do, in general:
Don't use cookie encoder directly. Use a cookie session store.
Don't call an handler within an handler, prefer a redirect. This should prevent writing twice the headers/body on the response.
Don't pass the user/password in the cookie, even encoded, in 2021 we may even prevent sending that through the form at all (you might consider sending only a hash and re hash the hash on the server side to figure out if things are good to go).
Few things you always want to do:
Write tests.
Make use of middlewares.
Always provide small reproducible examples.
That being said, I have written a lookalike code with some stubs (mostly for db), I removed template support (i was not in the mood to write HTML) and more importantly I wrote tests !!
To the question How to clear the session :
Delete the values from the store, write the store
To the question and only visit an about page after login?:
Wrap that handler with a middleware that verifies login data attached to the user cookie store
-- main.go --
package main
import (
"crypto/sha256"
"encoding/gob"
"fmt"
"log"
"net/http"
"github.com/gorilla/mux"
"github.com/gorilla/securecookie"
"github.com/gorilla/sessions"
)
// Note: Don't store your key in your source code. Pass it via an
// environmental variable, or flag (or both), and don't accidentally commit it
// alongside your code. Ensure your key is sufficiently random - i.e. use Go's
// crypto/rand or securecookie.GenerateRandomKey(32) and persist the result.
var store = sessions.NewCookieStore(
securecookie.GenerateRandomKey(32),
)
//emulate db package
func dbLookupUser(user, pwd string) bool {
return user == "user" && pwd == "pwd"
}
func dbLookupHash(h string) bool {
return h == hash("user", "pwd")
}
func hash(s ...interface{}) string {
hr := sha256.New()
fmt.Fprint(hr, s...)
return fmt.Sprintf("%x", hr.Sum(nil))
}
// hashKey is a typed key for the session map store to prevent unintented overwrites.
type hashKey string
func init() {
gob.Register(hashKey(""))
}
func loginAuth(res http.ResponseWriter, req *http.Request) {
email := req.FormValue("email")
password := req.FormValue("password")
match := dbLookupUser(email, password)
if match == true {
session, _ := store.Get(req, "session-name")
session.Values["hash"] = hash(email, password)
// Save it before we write to the response/return from the handler.
err := session.Save(req, res)
if err == nil {
// about(res, req) // don't!
// the about handler might want to setup its own http response headers
// That would conflict with what we did here.
// prefer a redirect
http.Redirect(res, req, "/about", http.StatusFound)
return
}
} else {
fmt.Fprintf(res, "try again") // use a templatee instead!
// tmpl.ExecuteTemplate(res, "login.html", nil)
}
}
func logout(res http.ResponseWriter, req *http.Request) {
session, _ := store.Get(req, "session-name")
delete(session.Values, hashKey("hash"))
_ = session.Save(req, res)
http.Redirect(res, req, "/", 302)
}
func about(res http.ResponseWriter, req *http.Request) {
fmt.Fprintf(res, "welcome to about page")
}
func requireLogin(next http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
session, _ := store.Get(r, "session-name")
var h string
if x, ok := session.Values[hashKey("hash")]; ok {
h = x.(string)
}
var match bool
if h != "" {
match = dbLookupHash(h)
}
if !match {
// Write an error and stop the handler chain
http.Error(w, "Forbidden", http.StatusForbidden)
return
}
next(w, r)
}
}
func main() {
r := mux.NewRouter()
r.HandleFunc("/", loginAuth)
r.HandleFunc("/logout", logout)
r.HandleFunc("/about", requireLogin(about))
log.Fatal(http.ListenAndServe("localhost:8080", r))
}
-- main_test.go --
package main
import (
"net/http"
"net/http/httptest"
"net/url"
"testing"
)
func TestLogin(t *testing.T) {
req := httptest.NewRequest("POST", "http://example.com/foo", nil)
form := url.Values{}
form.Set("email", "user")
form.Set("password", "pwd")
req.Form = form
w := httptest.NewRecorder()
loginAuth(w, req)
resp := w.Result()
// body, _ := io.ReadAll(resp.Body)
if wanted := http.StatusFound; resp.StatusCode != wanted {
t.Fatalf("invalid response code, got=%v wanted=%v", resp.StatusCode, wanted)
}
// implement more check
}
func TestLoginFailure(t *testing.T) {
req := httptest.NewRequest("POST", "http://example.com/foo", nil)
form := url.Values{}
form.Set("email", "!user")
form.Set("password", "!pwd")
req.Form = form
w := httptest.NewRecorder()
loginAuth(w, req)
resp := w.Result()
// body, _ := io.ReadAll(resp.Body)
if wanted := http.StatusOK; resp.StatusCode != wanted {
t.Fatalf("invalid response code, got=%v wanted=%v", resp.StatusCode, wanted)
}
// implement more check
}
func TestAboutNotLogged(t *testing.T) {
req := httptest.NewRequest("POST", "http://example.com/foo", nil)
w := httptest.NewRecorder()
requireLogin(about)(w, req)
resp := w.Result()
// body, _ := io.ReadAll(resp.Body)
if wanted := http.StatusForbidden; resp.StatusCode != wanted {
t.Fatalf("invalid response code, got=%v wanted=%v", resp.StatusCode, wanted)
}
// implement more check
}
func TestAboutLogged(t *testing.T) {
req := httptest.NewRequest("POST", "http://example.com/foo", nil)
w := httptest.NewRecorder()
session, _ := store.Get(req, "session-name")
session.Values[hashKey("hash")] = hash("user", "pwd")
err := session.Save(req, w)
if err != nil {
t.Fatal(err)
}
hdr := w.Header()
req.Header.Add("Cookie", hdr["Set-Cookie"][0])
w = httptest.NewRecorder()
requireLogin(about)(w, req)
resp := w.Result()
// body, _ := io.ReadAll(resp.Body)
if wanted := http.StatusOK; resp.StatusCode != wanted {
t.Fatalf("invalid response code, got=%v wanted=%v", resp.StatusCode, wanted)
}
// implement more check
}
I have the following code
package main
import (
"bytes"
"fmt"
"github.com/gorilla/mux"
"log"
"net/http"
"time"
"io"
httprouter "github.com/fasthttp/router"
"github.com/valyala/fasthttp"
)
func main() {
router := mux.NewRouter().StrictSlash(true)
/*router := NewRouter()*/
router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
_, _ = fmt.Fprintf(w, "Hello!!!")
})
router.HandleFunc("/{name}", func(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
prepare(w, r, vars["name"])
}).Methods("POST")
log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", 8080), router))
}
//using fast http
func _() {
router := httprouter.New()
router.GET("/", func(w *fasthttp.RequestCtx) {
_, _ = fmt.Fprintf(w, "Hello!!!")
})
router.POST("/:name", func(w *fasthttp.RequestCtx) {
prepareRequest(w, w.UserValue("name").(string))
})
log.Fatal(fasthttp.ListenAndServe(fmt.Sprintf(":%d", 8080), router.Handler))
}
//func prepare(w *fasthttp.RequestCtx, name string)
func prepare(w http.ResponseWriter, r *http.Request, name string) {
//other part of the code and call to goroutine
var urls []string
//lets say all the url loaded, call the go routine func and wait for channel to respond and then proceed with the response of all url
results := callUrls(urls) //there are 10 urls atleast to call simultaneously for each request everytime
process(w, results)
}
type Response struct {
status int
url string
body string
}
func callUrls(urls []string) []*Response {
ch := make(chan *Response, len(urls))
for _, url := range urls {
go func(url string) {
//http post on url,
//base on status code of url call, add to status code
//some thing like
req, err := http.NewRequest("POST", url, bytes.NewBuffer(somePostData))
req.Header.Set("Content-Type", "application/json")
req.Close = true
client := &http.Client{
Timeout: time.Duration(time.Duration(100) * time.Millisecond),
}
response, err := client.Do(req)
//Using fast http client
/*req := fasthttp.AcquireRequest()
req.SetRequestURI(url)
req.Header.Set("Content-Type", "application/json")
req.Header.SetMethod("POST")
req.SetBody(somePostData)
response := fasthttp.AcquireResponse()
client := &fasthttp.Client{
ReadTimeout: time.Duration(time.Duration(100) * time.Millisecond),
}
err := client.Do(req, response)*/
if err != nil {
//do other thing with the response received
_, _ = io.Copy(ioutil.Discard, response.Body)
_ = response.Body.Close()
} else {
//success response
_, _ = io.Copy(ioutil.Discard, response.Body)
_ = response.Body.Close()
body, _:= ioutil.ReadAll(response.Body)
strBody := string(body)
strBody = strings.Replace(strBody, "\r", "", -1)
strBody = strings.Replace(strBody, "\n", "", -1)
}
// return to channel accordingly
ch <- &Response{200, "url", "response body"}
}(url)
}
var results []*Response
for {
select {
case r := <-ch:
results = append(results, r)
if len(results) == len(urls) {
//Done
close(ch)
return results
}
}
}
}
//func process(w *fasthttp.RequestCtx,results []*Response){
func process(w http.ResponseWriter, results []*Response){
fmt.Println("response", "response body")
}
After serving few request on multi core CPU (there are around 4000-6000 req coming per sec) I get too many files open error and response time and CPU goes beyond limit. (Could CPU be be high because I convert byte to string a few times to replace few character? Any suggestion?)
I have seen other question referring to closing req/res body and/or setting sysctl or ulimit to higher values, I did follow those but I always end up with the error.
Config on the server:
/etc/sysctl.conf net.ipv4.tcp_tw_recycle = 1
open files (-n) 65535
I need the code to respond in millisec but it take upto 50sec when cpu is high.
Have tried both net/http and fast http but with no improvement. My Node.js request npm does everything perfectly on the same server. What will be best way to handle those connection or change in the code needed for improvement.
You can use the following library:
Requests: A Go library for reduce the headache when making HTTP requests (20k/s req)
https://github.com/alessiosavi/Requests
It's developed for solve theto many open files dealing with parallel requests.
The idea is to allocate a list of request, than send them with a configurable "parallel" factor that allow to run only "N" request at time.
Initialize the requests (you have already a set of urls)
// This array will contains the list of request
var reqs []requests.Request
// N is the number of request to run in parallel, in order to avoid "TO MANY OPEN FILES. N have to be lower than ulimit threshold"
var N int = 12
// Create the list of request
for i := 0; i < 1000; i++ {
// In this case, we init 1000 request with same URL,METHOD,BODY,HEADERS
req, err := requests.InitRequest("https://127.0.0.1:5000", "GET", nil, nil, true)
if err != nil {
// Request is not compliant, and will not be add to the list
log.Println("Skipping request [", i, "]. Error: ", err)
} else {
// If no error occurs, we can append the request created to the list of request that we need to send
reqs = append(reqs, *req)
}
}
At this point, we have a list that contains the requests that have to be sent.
Let's send them in parallel!
// This array will contains the response from the givens request
var response []datastructure.Response
// send the request using N request to send in parallel
response = requests.ParallelRequest(reqs, N)
// Print the response
for i := range response {
// Dump is a method that print every information related to the response
log.Println("Request [", i, "] -> ", response[i].Dump())
// Or use the data present in the response
log.Println("Headers: ", response[i].Headers)
log.Println("Status code: ", response[i].StatusCode)
log.Println("Time elapsed: ", response[i].Time)
log.Println("Error: ", response[i].Error)
log.Println("Body: ", string(response[i].Body))
}
You can find example usage into the example folder of the repository.
SPOILER:
I'm the author of this little library
I have go web server running on port and handling post request which internally calls different url to fetch response using goroutine and proceed.
I have divided the whole flow to different method. Draft of the code.
package main
import (
"bytes"
"fmt"
"github.com/gorilla/mux"
"log"
"net/http"
"time"
)
var status_codes string
func main() {
router := mux.NewRouter().StrictSlash(true)
/*router := NewRouter()*/
router.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
_, _ = fmt.Fprintf(w, "Hello!!!")
})
router.HandleFunc("/{name}", func(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
prepare(w, r, vars["name"])
}).Methods("POST")
log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", 8080), router))
}
func prepare(w http.ResponseWriter, r *http.Request, name string) {
//initializing for the current request, need to maintain this variable for each request coming
status_codes = ""
//other part of the code and call to goroutine
var urls []string
//lets say all the url loaded, call the go routine func and wait for channel to respond and then proceed with the response of all url
results := callUrls(urls)
process(w, results)
}
type Response struct {
status int
url string
body string
}
func callUrls(urls []string) []*Response {
ch := make(chan *Response, len(urls))
for _, url := range urls {
go func(url string) {
//http post on url,
//base on status code of url call, add to status code
//some thing like
req, err := http.NewRequest("POST", url, bytes.NewBuffer(somePostData))
req.Header.Set("Content-Type", "application/json")
req.Close = true
client := &http.Client{
Timeout: time.Duration(time.Duration(100) * time.Second),
}
response, err := client.Do(req)
if err != nil {
status_codes += "200,"
//do other thing with the response received
} else {
status_codes += "500,"
}
// return to channel accordingly
ch <- &Response{200, "url", "response body"}
}(url)
}
var results []*Response
for {
select {
case r := <-ch:
results = append(results, r)
if len(results) == len(urls) {
//Done
close(ch)
return results
}
}
}
}
func process(w http.ResponseWriter, results []*Response){
//read those status code received from all urls call for the given request
fmt.Println("status", status_codes)
//Now the above line keep getting status code from other request as well
//for eg. if I have called 5 urls then it should have
//200,500,204,404,200,
//but instead it is
//200,500,204,404,200,204,404,200,204,404,200, and some more keep growing with time
}
The above code does:
Variable declare globally, Initialized in prepare function.
append value in go routine callUrls function
read those variable in process function
Now should I pass those variable declared globally to each function call to make them local as it won't be shared then?(I would hate to do this.)
Or is there any other approach to achieve the same thing without adding more argument to function being called.
As I will have few other string and int value as well that will be used across the program and in go routine function as well.
What will be the correct way of making them thread safe and only 5 codes for each request coming on port simultaneously.
Don't use global variables, be explicit instead and use function arguments. Moreover, you have a race condition on status_codes because it is accessed by multiple goroutines without any mutex lock.
Take a look at my fix below.
func prepare(w http.ResponseWriter, r *http.Request, name string) {
var urls []string
//status_codes is populated by callUris(), so let it return the slice with values
results, status_codes := callUrls(urls)
//process() needs status_codes in order to work, so pass the variable explicitely
process(w, results, status_codes)
}
type Response struct {
status int
url string
body string
}
func callUrls(urls []string) []*Response {
ch := make(chan *Response, len(urls))
//In order to avoid race condition, let's use a channel
statusChan := make(chan string, len(urls))
for _, url := range urls {
go func(url string) {
//http post on url,
//base on status code of url call, add to status code
//some thing like
req, err := http.NewRequest("POST", url, bytes.NewBuffer(somePostData))
req.Header.Set("Content-Type", "application/json")
req.Close = true
client := &http.Client{
Timeout: time.Duration(time.Duration(100) * time.Second),
}
response, err := client.Do(req)
if err != nil {
statusChan <- "200"
//do other thing with the response received
} else {
statusChan <- "500"
}
// return to channel accordingly
ch <- &Response{200, "url", "response body"}
}(url)
}
var results []*Response
var status_codes []string
for !doneRes || !doneStatus { //continue until both slices are filled with values
select {
case r := <-ch:
results = append(results, r)
if len(results) == len(urls) {
//Done
close(ch) //Not really needed here
doneRes = true //we are done with results, set the corresponding flag
}
case status := <-statusChan:
status_codes = append(status_codes, status)
if len(status_codes) == len(urls) {
//Done
close(statusChan) //Not really needed here
doneStatus = true //we are done with statusChan, set the corresponding flag
}
}
}
return results, status_codes
}
func process(w http.ResponseWriter, results []*Response, status_codes []string) {
fmt.Println("status", status_codes)
}