Go url parameters mapping - go

Is there a native way for inplace url parameters in native Go?
For Example, if I have a URL: http://localhost:8080/blob/123/test I want to use this URL as /blob/{id}/test.
This is not a question about finding go libraries. I am starting with the basic question, does go itself provide a basic facility to do this natively.

There is no built in simple way to do this, however, it is not hard to do.
This is how I do it, without adding a particular library. It is placed in a function so that you can invoke a simple getCode() function within your request handler.
Basically you just split the r.URL.Path into parts, and then analyse the parts.
// Extract a code from a URL. Return the default code if code
// is missing or code is not a valid number.
func getCode(r *http.Request, defaultCode int) (int, string) {
p := strings.Split(r.URL.Path, "/")
if len(p) == 1 {
return defaultCode, p[0]
} else if len(p) > 1 {
code, err := strconv.Atoi(p[0])
if err == nil {
return code, p[1]
} else {
return defaultCode, p[1]
}
} else {
return defaultCode, ""
}
}

Well, without external libraries you can't, but may I recommend two excellent ones:
httprouter - https://github.com/julienschmidt/httprouter - is extremely fast and very lightweight. It's faster than the standard library's router, and it creates 0 allocations per call, which is great in a GCed language.
Gorilla Mux - http://www.gorillatoolkit.org/pkg/mux -
Very popular, nice interface, nice community.
Example usage of httprouter:
func Hello(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
fmt.Fprintf(w, "hello, %s!\n", ps.ByName("name"))
}
func main() {
router := httprouter.New()
router.GET("/hello/:name", Hello)
log.Fatal(http.ListenAndServe(":8080", router))
}

What about trying using regex, and find a named group in your url, like playground:
package main
import (
"fmt"
"net/url"
"regexp"
)
var myExp = regexp.MustCompile(`/blob/(?P<id>\d+)/test`) // use (?P<id>[a-zA-Z]+) if the id is alphapatic
func main() {
s := "http://localhost:8080/blob/123/test"
u, err := url.Parse(s)
if err != nil {
panic(err)
}
fmt.Println(u.Path)
match := myExp.FindStringSubmatch(s) // or match := myExp.FindStringSubmatch(u.Path)
result := make(map[string]string)
for i, name := range myExp.SubexpNames() {
if i != 0 && name != "" {
result[name] = match[i]
}
}
fmt.Printf("id: %s\n", result["id"])
}
output
/blob/123/test
id: 123
Below full code to use it with url, that is receiving http://localhost:8000/hello/John/58 and returning http://localhost:8000/hello/John/58:
package main
import (
"fmt"
"net/http"
"regexp"
"strconv"
)
var helloExp = regexp.MustCompile(`/hello/(?P<name>[a-zA-Z]+)/(?P<age>\d+)`)
func hello(w http.ResponseWriter, req *http.Request) {
match := helloExp.FindStringSubmatch(req.URL.Path)
if len(match) > 0 {
result := make(map[string]string)
for i, name := range helloExp.SubexpNames() {
if i != 0 && name != "" {
result[name] = match[i]
}
}
if _, err := strconv.Atoi(result["age"]); err == nil {
fmt.Fprintf(w, "Hello, %v year old named %s!", result["age"], result["name"])
} else {
fmt.Fprintf(w, "Sorry, not accepted age!")
}
} else {
fmt.Fprintf(w, "Wrong url\n")
}
}
func main() {
http.HandleFunc("/hello/", hello)
http.ListenAndServe(":8090", nil)
}

How about writing your own url generator (extend net/url a little bit) as below.
// --- This is how does it work like --- //
url, _ := rest.NewURLGen("http", "stack.over.flow", "1234").
Pattern(foo/:foo_id/bar/:bar_id).
ParamQuery("foo_id", "abc").
ParamQuery("bar_id", "xyz").
ParamQuery("page", "1").
ParamQuery("offset", "5").
Do()
log.Printf("url: %s", url)
// url: http://stack.over.flow:1234/foo/abc/bar/xyz?page=1&offset=5
// --- Your own url generator would be like below --- //
package rest
import (
"log"
"net/url"
"strings"
"straas.io/base/errors"
"github.com/jinzhu/copier"
)
// URLGen generates request URL
type URLGen struct {
url.URL
pattern string
paramPath map[string]string
paramQuery map[string]string
}
// NewURLGen new a URLGen
func NewURLGen(scheme, host, port string) *URLGen {
h := host
if port != "" {
h += ":" + port
}
ug := URLGen{}
ug.Scheme = scheme
ug.Host = h
ug.paramPath = make(map[string]string)
ug.paramQuery = make(map[string]string)
return &ug
}
// Clone return copied self
func (u *URLGen) Clone() *URLGen {
cloned := &URLGen{}
cloned.paramPath = make(map[string]string)
cloned.paramQuery = make(map[string]string)
err := copier.Copy(cloned, u)
if err != nil {
log.Panic(err)
}
return cloned
}
// Pattern sets path pattern with placeholder (format `:<holder_name>`)
func (u *URLGen) Pattern(pattern string) *URLGen {
u.pattern = pattern
return u
}
// ParamPath builds path part of URL
func (u *URLGen) ParamPath(key, value string) *URLGen {
u.paramPath[key] = value
return u
}
// ParamQuery builds query part of URL
func (u *URLGen) ParamQuery(key, value string) *URLGen {
u.paramQuery[key] = value
return u
}
// Do returns final URL result.
// The result URL string is possible not escaped correctly.
// This is input for `gorequest`, `gorequest` will handle URL escape.
func (u *URLGen) Do() (string, error) {
err := u.buildPath()
if err != nil {
return "", err
}
u.buildQuery()
return u.String(), nil
}
func (u *URLGen) buildPath() error {
r := []string{}
p := strings.Split(u.pattern, "/")
for i := range p {
part := p[i]
if strings.Contains(part, ":") {
key := strings.TrimPrefix(p[i], ":")
if val, ok := u.paramPath[key]; ok {
r = append(r, val)
} else {
if i != len(p)-1 {
// if placeholder at the end of pattern, it could be not provided
return errors.Errorf("placeholder[%s] not provided", key)
}
}
continue
}
r = append(r, part)
}
u.Path = strings.Join(r, "/")
return nil
}
func (u *URLGen) buildQuery() {
q := u.URL.Query()
for k, v := range u.paramQuery {
q.Set(k, v)
}
u.RawQuery = q.Encode()
}

With net/http the following would trigger when calling localhost:8080/blob/123/test
http.HandleFunc("/blob/", yourHandlerFunction)
Then inside yourHandlerFunction, manually parse r.URL.Path to find 123.
Note that if you don't add a trailing / it won't work. The following would only trigger when calling localhost:8080/blob:
http.HandleFunc("/blob", yourHandlerFunction)

As of 19-Sep-22, with go version 1.19, instance of http.request URL has a method called Query, which will return a map, which is a parsed query string.
func helloHandler(res http.ResponseWriter, req *http.Request) {
// when request URL is `http://localhost:3000/?first=hello&second=world`
fmt.Println(req.URL.Query()) // outputs , map[second:[world] first:[hello]]
res.Write([]byte("Hello World Web"))
}

No way without standard library. Why you don't want to try some library? I think its not so hard to use it, just go get bla bla bla
I use Beego. Its MVC style.

how about a simple utility function ?
func withURLParams(u url.URL, param, val string) url.URL{
u.Path = strings.ReplaceAll(u.Path, param, val)
return u
}
you can use it like this:
u, err := url.Parse("http://localhost:8080/blob/:id/test")
if err != nil {
return nil, err
}
u := withURLParams(u, ":id","123")
// now u.String() is http://localhost:8080/blob/123/test

If you need a framework and you think it will be slow because it's 'bigger' than a router or net/http, then you 're wrong.
Iris is the fastest go web framework that you will ever find, so far according to all benchmarks.
Install by
go get gopkg.in/kataras/iris.v6
Django templates goes easy with iris:
import (
"gopkg.in/kataras/iris.v6"
"gopkg.in/kataras/iris.v6/adaptors/httprouter"
"gopkg.in/kataras/iris.v6/adaptors/view" // <-----
)
func main() {
app := iris.New()
app.Adapt(iris.DevLogger())
app.Adapt(httprouter.New()) // you can choose gorillamux too
app.Adapt(view.Django("./templates", ".html")) // <-----
// RESOURCE: http://127.0.0.1:8080/hi
// METHOD: "GET"
app.Get("/hi", hi)
app.Listen(":8080")
}
func hi(ctx *iris.Context){
ctx.Render("hi.html", iris.Map{"Name": "iris"})
}

Related

Why doesn't YAML.v3 marshal a struct based on the String() in golang?

I have a struct which contains a type based on an enum. I am trying to render it to a user friendly string. Here's minimum viable code:
package main
import (
"fmt"
"gopkg.in/yaml.v3"
)
type Job struct {
Engine Engine `json:"Engine" yaml:"Engine"`
}
//go:generate stringer -type=Engine --trimprefix=Engine
type Engine int
const (
engineUnknown Engine = iota // must be first
EngineDocker
engineDone // must be last
)
func main() {
j := Job{Engine: EngineDocker}
fmt.Printf("%+v\n\n", j)
out, _ := yaml.Marshal(j)
fmt.Println(string(out))
}
Here's the generated code:
// Code generated by "stringer -type=Engine --trimprefix=Engine"; DO NOT EDIT.
package main
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[engineUnknown-0]
_ = x[EngineDocker-1]
_ = x[engineDone-2]
}
const _Engine_name = "engineUnknownDockerengineDone"
var _Engine_index = [...]uint8{0, 13, 19, 29}
func (i Engine) String() string {
if i < 0 || i >= Engine(len(_Engine_index)-1) {
return "Engine(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _Engine_name[_Engine_index[i]:_Engine_index[i+1]]
}
Here's the output:
{Engine:1}
Engine: 1
Here's what I'd like the output to be:
{Engine:Docker}
Engine: Docker
I thought the String() in the generated file would accomplish this. Is there any way to do this? Thanks!
yaml marshaler doesn't use String method. Instead YAML uses encoding.TextMarshaler and encoding.TextUnmarshaler interfaces. Actually, all other codec schemes - JSON, XML, TOML, etc. - use those interfaces to read/write the values. So, if you implement those methods for your type, you will receive all other codecs for free.
Here is an example how to make a human-readable encoding for your enum: https://go.dev/play/p/pEcBmAM-oZJ
type Engine int
const (
engineUnknown Engine = iota // must be first
EngineDocker
engineDone // must be last
)
var engineNames []string
var engineNameToValue map[string]Engine
func init() {
engineNames = []string{"Unknown", "Docker"}
engineNameToValue = make(map[string]Engine)
for i, name := range engineNames {
engineNameToValue[strings.ToLower(name)] = Engine(i)
}
}
func (e Engine) String() string {
if e < 0 || int(e) >= len(engineNames) {
panic(fmt.Errorf("Invalid engine code: %d", e))
}
return engineNames[e]
}
func ParseEngine(text string) (Engine, error) {
i, ok := engineNameToValue[strings.ToLower(text)]
if !ok {
return engineUnknown, fmt.Errorf("Invalid engine name: %s", text)
}
return i, nil
}
func (e Engine) MarshalText() ([]byte, error) {
return []byte(e.String()), nil
}
func (e *Engine) UnmarshalText(text []byte) (err error) {
name := string(text)
*e, err = ParseEngine(name)
return
}
How it works:
func main() {
j := Job{Engine: EngineDocker}
fmt.Printf("%#v\n\n", j)
out, err := yaml.Marshal(j)
if err != nil {
panic(err)
}
fmt.Printf("YAML: %s\n", string(out))
var jj Job
err = yaml.Unmarshal(out, &jj)
if err != nil {
panic(err)
}
fmt.Printf("%#v\n\n", jj)
// == JSON ==
out, err = json.Marshal(j)
if err != nil {
panic(err)
}
fmt.Printf("JSON: %s\n", string(out))
var jjs Job
err = json.Unmarshal(out, &jjs)
if err != nil {
panic(err)
}
fmt.Printf("%#v\n\n", jjs)
}
the output
main.Job{Engine:1}
YAML: Engine: Docker
main.Job{Engine:1}
JSON: {"Engine":"Docker"}
main.Job{Engine:1}
See? It writes and reads strings to both YAML and JSON without any extra effort.

Finding all functions of certain type

Is there a way to list out all functions that uses/returns a specific type?
For example: I'm interested to use the following function.
func ListenAndServe(addr string, handler Handler) error
How can I find out all functions (across all Go packages) that can return a Handler?
I'd write an analysis tool using the x/tools/go/analysis framework. Here's a rough sketch that you can run on any module (it uses go/packages underneath so it fully supports modules):
import (
"bytes"
"fmt"
"go/ast"
"go/format"
"go/token"
"golang.org/x/tools/go/analysis"
"golang.org/x/tools/go/analysis/singlechecker"
)
var RtAnalysis = &analysis.Analyzer{
Name: "rtanalysis",
Doc: "finds functions by return type",
Run: run,
}
func main() {
singlechecker.Main(RtAnalysis)
}
func run(pass *analysis.Pass) (interface{}, error) {
for _, file := range pass.Files {
ast.Inspect(file, func(n ast.Node) bool {
if funcTy, ok := n.(*ast.FuncType); ok {
if funcTy.Results != nil {
for _, fl := range funcTy.Results.List {
if tv, ok := pass.TypesInfo.Types[fl.Type]; ok {
if tv.Type.String() == "net/http.Handler" {
ns := nodeString(funcTy, pass.Fset)
fmt.Printf("%s has return of type net/http.Handler\n", ns)
}
}
}
}
}
return true
})
}
return nil, nil
}
// nodeString formats a syntax tree in the style of gofmt.
func nodeString(n ast.Node, fset *token.FileSet) string {
var buf bytes.Buffer
format.Node(&buf, fset, n)
return buf.String()
}

Generic function with variable input/output types

Just playing with aws sdk for go. When listing resources of different types I tend to have alot of very similar functions like the two in the example bellow.
Is there a way to rewrite them as one generic function that will return a specific type depending on what is passed on as param?
Something like:
func generic(session, funcToCall, t, input) (interface{}, error) {}
currently I have to do this (functionality is the same just types change):
func getVolumes(s *session.Session) ([]*ec2.Volume, error) {
client := ec2.New(s)
t := []*ec2.Volume{}
input := ec2.DescribeVolumesInput{}
for {
result, err := client.DescribeVolumes(&input)
if err != nil {
return nil, err
}
t = append(t, result.Volumes...)
if result.NextToken != nil {
input.NextToken = result.NextToken
} else {
break
}
}
return t, nil
}
func getVpcs(s *session.Session) ([]*ec2.Vpc, error) {
client := ec2.New(s)
t := []*ec2.Vpc{}
input := ec2.DescribeVpcsInput{}
for {
result, err := client.DescribeVpcs(&input)
if err != nil {
return nil, err
}
t = append(t, result.Vpcs...)
if result.NextToken != nil {
input.NextToken = result.NextToken
} else {
break
}
}
return t, nil
}
Because you only deal with functions it is possible to use the reflect package to generate functions at runtime.
Using the object type (Volume, Vpc) it is possible to derive all subsequents information to provide a fully generic implementation that is really dry, at the extent at the being more complex and slower.
It is untested, you are welcome to help in testing and fixing it, but something like this should put you on the track
https://play.golang.org/p/mGjtYVG2OZS
The registry idea come from this answer https://stackoverflow.com/a/23031445/4466350
for reference the golang documentation of the reflect package is at https://golang.org/pkg/reflect/
package main
import (
"errors"
"fmt"
"reflect"
)
func main() {
fmt.Printf("%T\n", getter(Volume{}))
fmt.Printf("%T\n", getter(Vpc{}))
}
type DescribeVolumesInput struct{}
type DescribeVpcs struct{}
type Volume struct{}
type Vpc struct{}
type Session struct{}
type Client struct{}
func New(s *Session) Client { return Client{} }
var typeRegistry = make(map[string]reflect.Type)
func init() {
some := []interface{}{DescribeVolumesInput{}, DescribeVpcs{}}
for _, v := range some {
typeRegistry[fmt.Sprintf("%T", v)] = reflect.TypeOf(v)
}
}
var errV = errors.New("")
var errType = reflect.ValueOf(&errV).Elem().Type()
var zeroErr = reflect.Zero(reflect.TypeOf((*error)(nil)).Elem())
var nilErr = []reflect.Value{zeroErr}
func getter(of interface{}) interface{} {
outType := reflect.SliceOf(reflect.PtrTo(reflect.TypeOf(of)))
fnType := reflect.FuncOf([]reflect.Type{reflect.TypeOf(new(Session))}, []reflect.Type{outType, errType}, false)
fnBody := func(input []reflect.Value) []reflect.Value {
client := reflect.ValueOf(New).Call(input)[0]
t := reflect.MakeSlice(outType, 0, 0)
name := fmt.Sprintf("Describe%TsInput", of)
descInput := reflect.New(typeRegistry[name]).Elem()
mName := fmt.Sprintf("Describe%Ts", of)
meth := client.MethodByName(mName)
if !meth.IsValid() {
return []reflect.Value{
t,
reflect.ValueOf(fmt.Errorf("no such method %q", mName)),
}
}
for {
out := meth.Call([]reflect.Value{descInput.Addr()})
if len(out) > 0 {
errOut := out[len(out)-1]
if errOut.Type().Implements(errType) && errOut.IsNil() == false {
return []reflect.Value{t, errOut}
}
}
result := out[1]
fName := fmt.Sprintf("%Ts", of)
if x := result.FieldByName(fName); x.IsValid() {
t = reflect.AppendSlice(t, x)
} else {
return []reflect.Value{
t,
reflect.ValueOf(fmt.Errorf("field not found %q", fName)),
}
}
if x := result.FieldByName("NextToken"); x.IsValid() {
descInput.FieldByName("NextToken").Set(x)
} else {
break
}
}
return []reflect.Value{t, zeroErr}
}
fn := reflect.MakeFunc(fnType, fnBody)
return fn.Interface()
}
Proxying 3rd party API, is quite simple to implement with
go, here is how' it got implemented with endly e2e test runner AWS proxy
I would say that AWS API is perfect candidate for proxying, as long as reflection performance price is not an issue.
Some other 3rd party API like kubernetes
are much more challenging, but still quite easy to proxy with go, which is a combination of reflection and code generation:

HTTP request fails when executed asynchronously

I'm trying to write a tiny application in Go that can send an HTTP request to all IP addresses in hopes to find a specific content. The issue is that the application seems to crash in a very peculiar way when the call is executed asynchronously.
ip/validator.go
package ip
import (
"io/ioutil"
"net/http"
"regexp"
"time"
)
type ipValidator struct {
httpClient http.Client
path string
exp *regexp.Regexp
confirmationChannel *chan string
}
func (this *ipValidator) validateUrl(url string) bool {
response, err := this.httpClient.Get(url)
if err != nil {
return false
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return false
}
bodyBytes, _ := ioutil.ReadAll(response.Body)
result := this.exp.Match(bodyBytes)
if result && this.confirmationChannel != nil {
*this.confirmationChannel <- url
}
return result
}
func (this *ipValidator) ValidateIp(addr ip) bool {
httpResult := this.validateUrl("http://" + addr.ToString() + this.path)
httpsResult := this.validateUrl("https://" + addr.ToString() + this.path)
return httpResult || httpsResult
}
func (this *ipValidator) GetSuccessChannel() *chan string {
return this.confirmationChannel
}
func NewIpValidadtor(path string, exp *regexp.Regexp) ipValidator {
return newValidator(path, exp, nil)
}
func NewAsyncIpValidator(path string, exp *regexp.Regexp) ipValidator {
c := make(chan string)
return newValidator(path, exp, &c)
}
func newValidator(path string, exp *regexp.Regexp, c *chan string) ipValidator {
httpClient := http.Client{
Timeout: time.Second * 2,
}
return ipValidator{httpClient, path, exp, c}
}
main.go
package main
import (
"./ip"
"fmt"
"os"
"regexp"
)
func processOutput(c *chan string) {
for true {
url := <- *c
fmt.Println(url)
}
}
func main() {
args := os.Args[1:]
fmt.Printf("path: %s regex: %s", args[0], args[1])
regexp, regexpError := regexp.Compile(args[1])
if regexpError != nil {
fmt.Println("The provided regexp is not valid")
return
}
currentIp, _ := ip.NewIp("172.217.22.174")
validator := ip.NewAsyncIpValidator(args[0], regexp)
successChannel := validator.GetSuccessChannel()
go processOutput(successChannel)
for currentIp.HasMore() {
go validator.ValidateIp(currentIp)
currentIp = currentIp.Increment()
}
}
Note the line that says go validator.ValidateIp(currentIp) in main.go. Should I remove the word "go" to execute everything within the main routine, the code works as expected -> it sends requests to IP addresses starting 172.217.22.174 and should one of them return a legitimate result that matches the regexp that the ipValidator was initialized with, the URL is passed to the channel and the value is printed out by processOutput function from main.go. The issue is that simply adding go in front of validator.ValidateIp(currentIp) breaks that functionality. In fact, according to the debugger, I never seem to go past the line that says response, err := this.httpClient.Get(url) in validator.go.
The struggle is real. Should I decide to scan the whole internet, there's 256^4 IP addresses to go through. It will take years, unless I find a way to split the process into multiple routines.

Idiomatic way to validate http request in go

I need to validate that my http request has two parameters, Start and End. Currently, I set a default value that should not appear as either of the parameters and check for it along with other invalid values. However, this feels like a hack. What should be the proper way to do this?
Here is my code:
type Request struct {
Start int `json: "start"`
End int `json: "end"`
}
func HandlePost(w http.ResponseWriter, r *http.Request) {
body , _ := ioutil.ReadAll(r.Body)
reqData := Request{Start: -1, End: -1} // < whats the correct way to do this
json.Unmarshal(body, &reqData)
if reqData.Start < 0 && reqData.End < 0 {
w.WriteHeader(http.StatusBadRequest)
return
}
// rest of the logic
}
You can use https://github.com/asaskevich/govalidator for basic way of validating the request. But in case you want something more sophisticated, you need to write your own custom validator function. e.g.
type Request struct {
Start int `json: "start"`
End int `json: "end"`
}
func (a *Request) validate() url.Values {
err := url.Values{}
// Write your own validation rules
if a.Start < 0 {
err.Add("Start", "Start cannot be less than 0");
}
return err;
}
func handler(w http.ResponseWriter, r *http.Request) {
requestBody := &Request{}
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(requestBody); err != nil {
panic(err)
}
if errors := requestBody.validate(); len(errors) > 0 {
err := map[string]interface{}{"validationError": errors}
w.Header().Set("Content-type", "application/json")
w.WriteHeader(http.StatusBadRequest)
json.NewEncoder(w).Encode(err)
}
fmt.Fprint(w, "success request scenario!")
}
Here's another way to validate structures using struct tags and pointers. Note that if 0 is a valid thing to pass, then this solution will not work.
omitempty considers the 0 value to be empty. If you want this to work will considering 0 to be valid remove the pointers and modify the IsValid method
package main
import (
"encoding/json"
"fmt"
)
type Request struct {
Start *int `json: "start,omitempty"`
End *int `json: "end,omitempty"`
}
func (r Request) IsValid() (bool, error) {
if r.Start == nil {
return false, fmt.Errorf("start is missing")
}
if r.End == nil {
return false, fmt.Errorf("end is missing")
}
return true, nil
}
var (
invalidStartb = `{"end": 1}`
invalidEndb = `{"start": 1}`
valid = `{"start": 1, "end": 1}`
)
func main() {
var r Request
_ = json.Unmarshal([]byte(invalidStartb), &r)
fmt.Println(r.IsValid())
r = Request{}
_ = json.Unmarshal([]byte(invalidEndb), &r)
fmt.Println(r.IsValid())
r = Request{}
_ = json.Unmarshal([]byte(valid), &r)
fmt.Println(r.IsValid())
}
runnable version here https://goplay.space/#Z0eqLpEHO37
You can use https://github.com/buger/jsonparser getInt.
You'll get an error if the json is missing the expected key.
I recommend using benchmark and not decided by the code beauty or any other hunch

Resources