Callbacks from module - go

I'm trying to learn go so I decided to make a go module. It works but now I want the main program to get notified when changes happen. In this example I just monitor when connected, but later on for many more changes in the device controlled.
The code below works but I can't see changes that happend in the module, for example the function "onHpd1Connected" in the main program is called when connected but shows that my connection is nil (even if I can see it is set in the "emit" function). The code must be compatible with multiple devices connected (hpd1, hpd2 etc..). I've tried pointers in different ways but without any luck. What I'm I doing wrong, and is this the best way to do it? Or would channels be a better option?
package main
import (
"fmt"
"sandbox/hyperdeck"
)
type hdc struct {
client hyperdeck.Hyperdeck
}
var hpd1 = hdc{
client: hyperdeck.NewHyperdeck("10.0.20.5", 9993),
}
func (hd hdc) onHpd1Connected() {
fmt.Println("Connected 1")
fmt.Println(hd)
}
func main() {
hpd1.client.On("connected", hpd1.onHpd1Connected)
hpd1.client.Connect()
}
and the module
package hyperdeck
import (
"errors"
"fmt"
"net"
"reflect"
"strconv"
"time"
)
const (
Open ConnectionState = 1
Connecting ConnectionState = 2
Closed ConnectionState = 3
)
type ConnectionState int
type HdCallback func()
type Hyperdeck struct {
Ip string
Port int
connection net.Conn
state ConnectionState
initialized bool
listeners map[string][]HdCallback
}
func (hd *Hyperdeck) connect() error {
if hd.state != Closed {
return errors.New("already connected to server: " + hd.Ip)
}
hd.initialized = false
hd.state = Connecting
var err error
hd.connection, err = net.DialTimeout("tcp", hd.Ip+":"+strconv.Itoa(hd.Port), time.Duration(time.Second*2))
if err != nil {
hd.state = Closed
return err
} else {
hd.state = Open
hd.emit("connected")
}
return nil
}
func (hd Hyperdeck) Connect() {
for {
if !hd.Connected() {
err := hd.connect()
if err != nil {
fmt.Println(err)
}
}
time.Sleep(time.Second * 1)
}
}
func (hd *Hyperdeck) Connected() bool {
return hd.state == Open && hd.connection != nil
}
func (hd Hyperdeck) emit(event string, params ...interface{}) {
fmt.Println(hd)
if listeners, exists := hd.listeners[event]; exists {
in := make([]reflect.Value, len(params))
for k, param := range params {
in[k] = reflect.ValueOf(param)
}
for _, cb := range listeners {
reflect.ValueOf(cb).Call(in)
}
}
}
func NewHyperdeck(ip string, port int) Hyperdeck {
hd := Hyperdeck{
Ip: ip,
Port: port,
initialized: false,
connection: nil,
listeners: map[string][]HdCallback{},
state: Closed,
}
return hd
}
func (hd Hyperdeck) On(event string, callback func()) {
if _, exists := hd.listeners[event]; !exists {
hd.listeners[event] = make([]HdCallback, 0)
}
hd.listeners[event] = append(hd.listeners[event], callback)
}

Related

Why doesn't YAML.v3 marshal a struct based on the String() in golang?

I have a struct which contains a type based on an enum. I am trying to render it to a user friendly string. Here's minimum viable code:
package main
import (
"fmt"
"gopkg.in/yaml.v3"
)
type Job struct {
Engine Engine `json:"Engine" yaml:"Engine"`
}
//go:generate stringer -type=Engine --trimprefix=Engine
type Engine int
const (
engineUnknown Engine = iota // must be first
EngineDocker
engineDone // must be last
)
func main() {
j := Job{Engine: EngineDocker}
fmt.Printf("%+v\n\n", j)
out, _ := yaml.Marshal(j)
fmt.Println(string(out))
}
Here's the generated code:
// Code generated by "stringer -type=Engine --trimprefix=Engine"; DO NOT EDIT.
package main
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[engineUnknown-0]
_ = x[EngineDocker-1]
_ = x[engineDone-2]
}
const _Engine_name = "engineUnknownDockerengineDone"
var _Engine_index = [...]uint8{0, 13, 19, 29}
func (i Engine) String() string {
if i < 0 || i >= Engine(len(_Engine_index)-1) {
return "Engine(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _Engine_name[_Engine_index[i]:_Engine_index[i+1]]
}
Here's the output:
{Engine:1}
Engine: 1
Here's what I'd like the output to be:
{Engine:Docker}
Engine: Docker
I thought the String() in the generated file would accomplish this. Is there any way to do this? Thanks!
yaml marshaler doesn't use String method. Instead YAML uses encoding.TextMarshaler and encoding.TextUnmarshaler interfaces. Actually, all other codec schemes - JSON, XML, TOML, etc. - use those interfaces to read/write the values. So, if you implement those methods for your type, you will receive all other codecs for free.
Here is an example how to make a human-readable encoding for your enum: https://go.dev/play/p/pEcBmAM-oZJ
type Engine int
const (
engineUnknown Engine = iota // must be first
EngineDocker
engineDone // must be last
)
var engineNames []string
var engineNameToValue map[string]Engine
func init() {
engineNames = []string{"Unknown", "Docker"}
engineNameToValue = make(map[string]Engine)
for i, name := range engineNames {
engineNameToValue[strings.ToLower(name)] = Engine(i)
}
}
func (e Engine) String() string {
if e < 0 || int(e) >= len(engineNames) {
panic(fmt.Errorf("Invalid engine code: %d", e))
}
return engineNames[e]
}
func ParseEngine(text string) (Engine, error) {
i, ok := engineNameToValue[strings.ToLower(text)]
if !ok {
return engineUnknown, fmt.Errorf("Invalid engine name: %s", text)
}
return i, nil
}
func (e Engine) MarshalText() ([]byte, error) {
return []byte(e.String()), nil
}
func (e *Engine) UnmarshalText(text []byte) (err error) {
name := string(text)
*e, err = ParseEngine(name)
return
}
How it works:
func main() {
j := Job{Engine: EngineDocker}
fmt.Printf("%#v\n\n", j)
out, err := yaml.Marshal(j)
if err != nil {
panic(err)
}
fmt.Printf("YAML: %s\n", string(out))
var jj Job
err = yaml.Unmarshal(out, &jj)
if err != nil {
panic(err)
}
fmt.Printf("%#v\n\n", jj)
// == JSON ==
out, err = json.Marshal(j)
if err != nil {
panic(err)
}
fmt.Printf("JSON: %s\n", string(out))
var jjs Job
err = json.Unmarshal(out, &jjs)
if err != nil {
panic(err)
}
fmt.Printf("%#v\n\n", jjs)
}
the output
main.Job{Engine:1}
YAML: Engine: Docker
main.Job{Engine:1}
JSON: {"Engine":"Docker"}
main.Job{Engine:1}
See? It writes and reads strings to both YAML and JSON without any extra effort.

Golang Return a different Struct based on some logic and unmarshal to that struct

I have a case where I'm I have to support multiple versions. Each one has different data so I create 2 structs. Based on the version I will return 1 of the structs. Once I identify which struct, I then would request the data and Unmarshal into the struct. However, Since that struct satisfies an interface, I dont think the unmarshal is working correctly. I always get the zero value for the sctruct
package main
import (
"encoding/json"
"fmt"
)
// Ten300 ...
type Ten300 struct {
Map string `json:"map"`
Enabled string `json:"enabled"`
}
// Ten400 ...
type Ten400 struct {
Block1 int `json:"block_1"`
Block2 int `json:"block_2"`
}
// NET ...
type NET struct {
CMD Commands
S TenIft
}
// Commands ...
type Commands struct {
Get string
}
// TenIft ...
type TenIft interface {
Get(string) error
}
// Get ...
func (n *Ten300) Get(module string) error {
fmt.Println("Ten300", "Get()")
return nil
}
// Get ...
func (n *Ten400) Get(module string) error {
fmt.Println("Ten400", "Get()")
n.Block2 = 100
return nil
}
// TenGen easy to read fun type
type TenGen func() NET
// VersionSetup is a map of possible version
var VersionSetup = map[string]TenGen{
"3.0.0": func() NET {
return NET{
CMD: Commands{
Get: "getConfig",
},
S: &Ten300{},
}
},
"4.0.0": func() NET {
return NET{
CMD: Commands{
Get: "getSettings",
},
S: &Ten400{},
}
},
}
const input300 = `{
"map": "one",
"enabled": "two"
}`
const input400 = `{
"block_1": 1,
"block_2": 2
}`
// Setup ...
func (n *NET) Setup() error {
// This Switch is just to use hard coded data
var input string
switch n.S.(type) {
case *Ten400:
input = input400
case *Ten300:
input = input300
}
err := json.Unmarshal([]byte(input), n.S)
if err != nil {
fmt.Println("returning err:", err)
return err
}
fmt.Printf("n.s type: %T\nn.s value: %+v\n", n.S, n.S)
n.S.Get("xxx")
return nil
}
func main() {
version := "3.0.0"
if f, ok := VersionSetup[version]; ok {
net := f()
err := net.Setup()
if err != nil {
panic(err)
}
}
version = "4.0.0"
if f, ok := VersionSetup[version]; ok {
net := f()
err := net.Setup()
if err != nil {
panic(err)
}
}
}
Added go-playground ... this seems to work, but is this the best solution?

How to re-evaluate promhttp.Handler on database change?

I'm perhaps abusing promhttp.Handler() to realise the use case for my microservice to tell me the:
version
if it has database connectivity
If there is a better way to monitor my microservices, do let me know!
I'm not sure how to structure the handle in such a way that when /metrics are called, the db.Ping() is re-evaluated.
https://s.natalian.org/2019-06-02/msping.mp4
package main
import (
"log"
"net/http"
"os"
_ "github.com/go-sql-driver/mysql"
"github.com/gorilla/mux"
"github.com/jmoiron/sqlx"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
)
const version = "0.0.1"
type App struct {
Router *mux.Router
DB *sqlx.DB
}
func main() {
a := App{}
a.Initialize()
log.Fatal(http.ListenAndServe(":"+os.Getenv("PORT"), a.Router))
}
func (a *App) Initialize() {
connectionString := "root:secret#tcp(localhost:3306)/rest_api_example?multiStatements=true&sql_mode=TRADITIONAL&timeout=5s"
var err error
a.DB, err = sqlx.Open("mysql", connectionString)
if err != nil {
log.Fatal(err)
}
microservicecheck := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Name: "mscheck",
Help: "Version with DB ping check",
},
[]string{
"commit",
},
)
if a.DB.Ping() == nil {
microservicecheck.WithLabelValues(version).Set(1)
} else {
microservicecheck.WithLabelValues(version).Set(0)
}
prometheus.MustRegister(microservicecheck)
a.Router = mux.NewRouter()
a.initializeRoutes()
}
func (a *App) initializeRoutes() {
a.Router.Handle("/metrics", promhttp.Handler()).Methods("GET")
}
https://play.golang.org/p/9DdXnz77S55
You could also add a middleware hook that does a preflight routine (i.e. your ping test) before calling promhttp.Handler(). However, at collection time, I think metrics should already have been tallied; and not generated at the instance of collection. So...
Try a separate go-routine that will poll the health of the DB connection at regular intervals. This avoids any messy hooks or custom collectors:
var pingPollingFreq = 5 * time.Second // this should probably match the Prometheus scrape interval
func (a *App) Initialize() {
// ...
prometheus.MustRegister(microservicecheck)
go func() {
for {
if a.DB.Ping() == nil {
microservicecheck.WithLabelValues(version).Set(1)
} else {
microservicecheck.WithLabelValues(version).Set(0)
}
time.Sleep(pingPollingFreq)
}
}()
// ...
}

HTTP request fails when executed asynchronously

I'm trying to write a tiny application in Go that can send an HTTP request to all IP addresses in hopes to find a specific content. The issue is that the application seems to crash in a very peculiar way when the call is executed asynchronously.
ip/validator.go
package ip
import (
"io/ioutil"
"net/http"
"regexp"
"time"
)
type ipValidator struct {
httpClient http.Client
path string
exp *regexp.Regexp
confirmationChannel *chan string
}
func (this *ipValidator) validateUrl(url string) bool {
response, err := this.httpClient.Get(url)
if err != nil {
return false
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return false
}
bodyBytes, _ := ioutil.ReadAll(response.Body)
result := this.exp.Match(bodyBytes)
if result && this.confirmationChannel != nil {
*this.confirmationChannel <- url
}
return result
}
func (this *ipValidator) ValidateIp(addr ip) bool {
httpResult := this.validateUrl("http://" + addr.ToString() + this.path)
httpsResult := this.validateUrl("https://" + addr.ToString() + this.path)
return httpResult || httpsResult
}
func (this *ipValidator) GetSuccessChannel() *chan string {
return this.confirmationChannel
}
func NewIpValidadtor(path string, exp *regexp.Regexp) ipValidator {
return newValidator(path, exp, nil)
}
func NewAsyncIpValidator(path string, exp *regexp.Regexp) ipValidator {
c := make(chan string)
return newValidator(path, exp, &c)
}
func newValidator(path string, exp *regexp.Regexp, c *chan string) ipValidator {
httpClient := http.Client{
Timeout: time.Second * 2,
}
return ipValidator{httpClient, path, exp, c}
}
main.go
package main
import (
"./ip"
"fmt"
"os"
"regexp"
)
func processOutput(c *chan string) {
for true {
url := <- *c
fmt.Println(url)
}
}
func main() {
args := os.Args[1:]
fmt.Printf("path: %s regex: %s", args[0], args[1])
regexp, regexpError := regexp.Compile(args[1])
if regexpError != nil {
fmt.Println("The provided regexp is not valid")
return
}
currentIp, _ := ip.NewIp("172.217.22.174")
validator := ip.NewAsyncIpValidator(args[0], regexp)
successChannel := validator.GetSuccessChannel()
go processOutput(successChannel)
for currentIp.HasMore() {
go validator.ValidateIp(currentIp)
currentIp = currentIp.Increment()
}
}
Note the line that says go validator.ValidateIp(currentIp) in main.go. Should I remove the word "go" to execute everything within the main routine, the code works as expected -> it sends requests to IP addresses starting 172.217.22.174 and should one of them return a legitimate result that matches the regexp that the ipValidator was initialized with, the URL is passed to the channel and the value is printed out by processOutput function from main.go. The issue is that simply adding go in front of validator.ValidateIp(currentIp) breaks that functionality. In fact, according to the debugger, I never seem to go past the line that says response, err := this.httpClient.Get(url) in validator.go.
The struggle is real. Should I decide to scan the whole internet, there's 256^4 IP addresses to go through. It will take years, unless I find a way to split the process into multiple routines.

go 1.8 plugin use custom interface

I want to use custom interface based on go plugin, but I found it's not support.
Definition of filter.Filter
package filter
import (
"net/http"
"github.com/valyala/fasthttp"
)
// Context filter context
type Context interface {
SetStartAt(startAt int64)
SetEndAt(endAt int64)
GetStartAt() int64
GetEndAt() int64
GetProxyServerAddr() string
GetProxyOuterRequest() *fasthttp.Request
GetProxyResponse() *fasthttp.Response
NeedMerge() bool
GetOriginRequestCtx() *fasthttp.RequestCtx
GetMaxQPS() int
ValidateProxyOuterRequest() bool
InBlacklist(ip string) bool
InWhitelist(ip string) bool
IsCircuitOpen() bool
IsCircuitHalf() bool
GetOpenToCloseFailureRate() int
GetHalfTrafficRate() int
GetHalfToOpenSucceedRate() int
GetOpenToCloseCollectSeconds() int
ChangeCircuitStatusToClose()
ChangeCircuitStatusToOpen()
RecordMetricsForRequest()
RecordMetricsForResponse()
RecordMetricsForFailure()
RecordMetricsForReject()
GetRecentlyRequestSuccessedCount(sec int) int
GetRecentlyRequestCount(sec int) int
GetRecentlyRequestFailureCount(sec int) int
}
// Filter filter interface
type Filter interface {
Name() string
Pre(c Context) (statusCode int, err error)
Post(c Context) (statusCode int, err error)
PostErr(c Context)
}
// BaseFilter base filter support default implemention
type BaseFilter struct{}
// Pre execute before proxy
func (f BaseFilter) Pre(c Context) (statusCode int, err error) {
return http.StatusOK, nil
}
// Post execute after proxy
func (f BaseFilter) Post(c Context) (statusCode int, err error) {
return http.StatusOK, nil
}
// PostErr execute proxy has errors
func (f BaseFilter) PostErr(c Context) {
}
This pkg is in my go app project.
load plugin file
package proxy
import (
"errors"
"plugin"
"strings"
"github.com/fagongzi/gateway/pkg/conf"
"github.com/fagongzi/gateway/pkg/filter"
)
var (
// ErrKnownFilter known filter error
ErrKnownFilter = errors.New("unknow filter")
)
const (
// FilterHTTPAccess access log filter
FilterHTTPAccess = "HTTP-ACCESS"
// FilterHeader header filter
FilterHeader = "HEAD" // process header fiter
// FilterXForward xforward fiter
FilterXForward = "XFORWARD"
// FilterBlackList blacklist filter
FilterBlackList = "BLACKLIST"
// FilterWhiteList whitelist filter
FilterWhiteList = "WHITELIST"
// FilterAnalysis analysis filter
FilterAnalysis = "ANALYSIS"
// FilterRateLimiting limit filter
FilterRateLimiting = "RATE-LIMITING"
// FilterCircuitBreake circuit breake filter
FilterCircuitBreake = "CIRCUIT-BREAKE"
// FilterValidation validation request filter
FilterValidation = "VALIDATION"
)
func newFilter(filterSpec *conf.FilterSpec) (filter.Filter, error) {
if filterSpec.External {
return newExternalFilter(filterSpec)
}
input := strings.ToUpper(filterSpec.Name)
switch input {
case FilterHTTPAccess:
return newAccessFilter(), nil
case FilterHeader:
return newHeadersFilter(), nil
case FilterXForward:
return newXForwardForFilter(), nil
case FilterAnalysis:
return newAnalysisFilter(), nil
case FilterBlackList:
return newBlackListFilter(), nil
case FilterWhiteList:
return newWhiteListFilter(), nil
case FilterRateLimiting:
return newRateLimitingFilter(), nil
case FilterCircuitBreake:
return newCircuitBreakeFilter(), nil
case FilterValidation:
return newValidationFilter(), nil
default:
return nil, ErrKnownFilter
}
}
func newExternalFilter(filterSpec *conf.FilterSpec) (filter.Filter, error) {
p, err := plugin.Open(filterSpec.ExternalPluginFile)
if err != nil {
return nil, err
}
s, err := p.Lookup("NewExternalFilter")
if err != nil {
return nil, err
}
sf := s.(func() (filter.Filter, error))
return sf()
}
This is the code of load plugin in my go app project
package main
import (
"C"
"strings"
"time"
"github.com/CodisLabs/codis/pkg/utils/log"
"github.com/fagongzi/gateway/pkg/filter"
"github.com/valyala/fasthttp"
)
// AccessFilter record the http access log
// log format: $remoteip "$method $path" $code "$agent" $svr $cost
type AccessFilter struct {
}
// NewExternalFilter create a External filter
func NewExternalFilter() (filter.Filter, error) {
return &AccessFilter{}, nil
}
// Name return name of this filter
func (f *AccessFilter) Name() string {
return "HTTP-ACCESS"
}
// Pre pre process
func (f *AccessFilter) Pre(c filter.Context) (statusCode int, err error) {
return 200, nil
}
// Post execute after proxy
func (f *AccessFilter) Post(c filter.Context) (statusCode int, err error) {
cost := (c.GetStartAt() - c.GetEndAt())
log.Infof("%s %s \"%s\" %d \"%s\" %s %s",
GetRealClientIP(c.GetOriginRequestCtx()),
c.GetOriginRequestCtx().Method(),
c.GetProxyOuterRequest().RequestURI(),
c.GetProxyResponse().StatusCode(),
c.GetOriginRequestCtx().UserAgent(),
c.GetProxyServerAddr(),
time.Duration(cost))
return 200, nil
}
// PostErr post error process
func (f *AccessFilter) PostErr(c filter.Context) {
}
// GetRealClientIP get read client ip
func GetRealClientIP(ctx *fasthttp.RequestCtx) string {
xforward := ctx.Request.Header.Peek("X-Forwarded-For")
if nil == xforward {
return strings.SplitN(ctx.RemoteAddr().String(), ":", 2)[0]
}
return strings.SplitN(string(xforward), ",", 2)[0]
}
This is the definition of plugin, it's in my plugin project. The plugin project and go app project are different projects.
I found errors:
panic: interface conversion: plugin.Symbol is func() (filter.Filter, error), not func() (filter.Filter, error)
You can find code in this project https://github.com/fagongzi/gateway/tree/go18-plugin-support.
filter.Filter is in pkg/filter package.
load plugin file in proxy/factory.go
plugin go file is in another project.
Custom interfaces work just fine.
But one important thing: you can only type assert types from values looked up from plugins that are defined outside of the plugin (you can't refer types defined in plugins). This also applies to each component of "composite types", for example you can only type assert a function type whose parameter and result types are also defined outside of the plugin.
1. With a common package outside of the plugin
One solution is to define the interface in a package outside of the plugin, and both the plugin and your app can import it and refer to it.
Define it in package filter:
package filter
type Filter interface {
Name() string
Age() int
}
The plugin is in package pq and imports package filter:
package main
import (
"fmt"
"filter"
)
type plgFilter struct{}
func (plgFilter) Name() string { return "Bob" }
func (plgFilter) Age() int { return 23 }
func GetFilter() (f filter.Filter, err error) {
f = plgFilter{}
fmt.Printf("[plugin GetFilter] Returning filter: %T %v\n", f, f)
return
}
And the main app that also imports (the same) package filter, loads the plugin, looks up GetFilter(), calls it and also uses the returned Filter:
package main
import (
"fmt"
"filter"
"plugin"
)
func main() {
p, err := plugin.Open("pg/pg.so")
if err != nil {
panic(err)
}
GetFilter, err := p.Lookup("GetFilter")
if err != nil {
panic(err)
}
filter, err := GetFilter.(func() (filter.Filter, error))()
fmt.Printf("GetFilter result: %T %v %v\n", filter, filter, err)
fmt.Println("\tName:", filter.Name())
fmt.Println("\tAge:", filter.Age())
}
Output:
[plugin GetFilter] Returning filter: main.plgFilter {}
GetFilter result: main.plgFilter {} <nil>
Name: Bob
Age: 23
2. With plugin returning interface{}, and interface defined in main app
Another solution is to have the plugin function return a value of type interface{}. Your main app can define the interface it expects, and it can use type assertion on the interface{} value returned by the plugin.
No filter package this time.
The plugin is in package pq:
package main
import (
"fmt"
)
type plgFilter struct{}
func (plgFilter) Name() string { return "Bob" }
func (plgFilter) Age() int { return 23 }
func GetFilterIface() (f interface{}, err error) {
f = plgFilter{}
fmt.Printf("[plugin GetFilterIface] Returning filter: %T %v\n", f, f)
return
}
And the main app:
package main
import (
"fmt"
"plugin"
)
func main() {
p, err := plugin.Open("pg/pg.so")
if err != nil {
panic(err)
}
GetFilterIface, err := p.Lookup("GetFilterIface")
if err != nil {
panic(err)
}
filterIface, err := GetFilterIface.(func() (interface{}, error))()
fmt.Printf("GetFilterIface result: %T %v %v\n", filterIface, filterIface, err)
myfilter := filterIface.(MyFilter)
fmt.Println("\tName:", myfilter.Name())
fmt.Println("\tAge:", myfilter.Age())
}
type MyFilter interface {
Name() string
Age() int
}
Output:
[plugin GetFilterIface] Returning filter: main.plgFilter {}
GetFilterIface result: main.plgFilter {} <nil>
Name: Bob
Age: 23
Also see related question: How do Go plugin dependencies work?

Resources