I'm starting to writing more complex go code, and my object node it to convert a list from a JSON object to a map with a particular key. This operation helps me to speed up my algorithm. But I have a problem now, my container struct has several complex JSON and I'm not able to write a generic solution to achieve a generic solution. The only way that I have in mind is to use a big switch case, but I think this is not the right solution.
This is my code at the moment, where the statusChannel is a map in the code but it is a list in the JSON string
type MetricOne struct {
// Internal id to identify the metric
id int `json:"-"`
// Version of metrics format, it is used to migrate the
// JSON payload from previous version of plugin.
Version int `json:"version"`
// Name of the metrics
Name string `json:"metric_name"`
NodeId string `json:"node_id"`
Color string `json:"color"`
OSInfo *osInfo `json:"os_info"`
// timezone where the node is located
Timezone string `json:"timezone"`
// array of the up_time
UpTime []*status `json:"up_time"`
// map of informatonof channel information
ChannelsInfo map[string]*statusChannel `json:"channels_info"`
}
func (instance *MetricOne) MarshalJSON() ([]byte, error) {
jsonMap := make(map[string]interface{})
reflectType := reflect.TypeOf(*instance)
reflectValue := reflect.ValueOf(*instance)
nFiled := reflectValue.Type().NumField()
for i := 0; i < nFiled; i++ {
key := reflectType.Field(i)
valueFiled := reflectValue.Field(i)
jsonName := key.Tag.Get("json")
switch jsonName {
case "-":
// skip
continue
case "channels_info":
// TODO convert the map[string]*statusChannel in a list of statusChannel
statusChannels := make([]*statusChannel, 0)
for _, value := range valueFiled.Interface().(map[string]*statusChannel) {
statusChannels = append(statusChannels, value)
}
jsonMap[jsonName] = statusChannels
default:
jsonMap[jsonName] = valueFiled.Interface()
}
}
return json.Marshal(jsonMap)
}
func (instance *MetricOne) UnmarshalJSON(data []byte) error {
var jsonMap map[string]interface{}
err := json.Unmarshal(data, &jsonMap)
if err != nil {
log.GetInstance().Error(fmt.Sprintf("Error: %s", err))
return err
}
instance.Migrate(jsonMap)
reflectValue := reflect.ValueOf(instance)
reflectStruct := reflectValue.Elem()
// reflectType := reflectValue.Type()
for key, value := range jsonMap {
fieldName, err := utils.GetFieldName(key, "json", *instance)
if err != nil {
log.GetInstance().Info(fmt.Sprintf("Error: %s", err))
if strings.Contains(key, "dev_") {
log.GetInstance().Info("dev propriety skipped if missed")
continue
}
return err
}
field := reflectStruct.FieldByName(*fieldName)
fieldType := field.Type()
filedValue := field.Interface()
val := reflect.ValueOf(filedValue)
switch key {
case "channels_info":
statusChannelsMap := make(map[string]*statusChannel)
toArray := value.([]interface{})
for _, status := range toArray {
var statusType statusChannel
jsonVal, err := json.Marshal(status)
if err != nil {
return err
}
err = json.Unmarshal(jsonVal, &statusType)
if err != nil {
return err
}
statusChannelsMap[statusType.ChannelId] = &statusType
}
field.Set(reflect.ValueOf(statusChannelsMap))
default:
field.Set(val.Convert(fieldType))
}
}
return nil
}
And when I will decode the object I receive the following error:
➜ go-metrics-reported git:(dev) ✗ make check
go test -v ./...
? github.com/OpenLNMetrics/go-metrics-reported/cmd/go-metrics-reported [no test files]
? github.com/OpenLNMetrics/go-metrics-reported/init/persistence [no test files]
=== RUN TestJSONSerializzation
--- PASS: TestJSONSerializzation (0.00s)
=== RUN TestJSONDeserializzation
--- FAIL: TestJSONDeserializzation (0.00s)
panic: reflect.Value.Convert: value of type map[string]interface {} cannot be converted to type *plugin.osInfo [recovered]
panic: reflect.Value.Convert: value of type map[string]interface {} cannot be converted to type *plugin.osInfo
goroutine 7 [running]:
testing.tRunner.func1.1(0x61b440, 0xc0001d69a0)
/home/vincent/.gosdk/go/src/testing/testing.go:1072 +0x30d
testing.tRunner.func1(0xc000001e00)
/home/vincent/.gosdk/go/src/testing/testing.go:1075 +0x41a
panic(0x61b440, 0xc0001d69a0)
/home/vincent/.gosdk/go/src/runtime/panic.go:969 +0x1b9
reflect.Value.Convert(0x6283e0, 0xc0001bb1a0, 0x15, 0x6b93a0, 0x610dc0, 0x610dc0, 0xc00014cb40, 0x196)
/home/vincent/.gosdk/go/src/reflect/value.go:2447 +0x229
github.com/OpenLNMetrics/go-metrics-reported/internal/plugin.(*MetricOne).UnmarshalJSON(0xc00014cb00, 0xc0001d8000, 0x493, 0x500, 0x7f04d01453d8, 0xc00014cb00)
/home/vincent/Github/OpenLNMetrics/go-metrics-reported/internal/plugin/metrics_one.go:204 +0x5b3
encoding/json.(*decodeState).object(0xc00010be40, 0x657160, 0xc00014cb00, 0x16, 0xc00010be68, 0x7b)
/home/vincent/.gosdk/go/src/encoding/json/decode.go:609 +0x207c
encoding/json.(*decodeState).value(0xc00010be40, 0x657160, 0xc00014cb00, 0x16, 0xc000034698, 0x54ec19)
/home/vincent/.gosdk/go/src/encoding/json/decode.go:370 +0x6d
encoding/json.(*decodeState).unmarshal(0xc00010be40, 0x657160, 0xc00014cb00, 0xc00010be68, 0x0)
/home/vincent/.gosdk/go/src/encoding/json/decode.go:180 +0x1ea
encoding/json.Unmarshal(0xc0001d8000, 0x493, 0x500, 0x657160, 0xc00014cb00, 0x500, 0x48cba6)
/home/vincent/.gosdk/go/src/encoding/json/decode.go:107 +0x112
github.com/OpenLNMetrics/go-metrics-reported/internal/plugin.TestJSONDeserializzation(0xc000001e00)
/home/vincent/Github/OpenLNMetrics/go-metrics-reported/internal/plugin/metric_one_test.go:87 +0x95
testing.tRunner(0xc000001e00, 0x681000)
/home/vincent/.gosdk/go/src/testing/testing.go:1123 +0xef
created by testing.(*T).Run
/home/vincent/.gosdk/go/src/testing/testing.go:1168 +0x2b3
FAIL github.com/OpenLNMetrics/go-metrics-reported/internal/plugin 0.008s
? github.com/OpenLNMetrics/go-metrics-reported/pkg/db [no test files]
? github.com/OpenLNMetrics/go-metrics-reported/pkg/graphql [no test files]
? github.com/OpenLNMetrics/go-metrics-reported/pkg/log [no test files]
? github.com/OpenLNMetrics/go-metrics-reported/pkg/utils [no test files]
FAIL
make: *** [Makefile:15: check] Error 1
can someone explain how I can do this operation in a generic way?
https://play.golang.org/p/jzU_lHj1wk7
type MetricOne struct {
// ...
// Have this field be ignored.
ChannelsInfo map[string]*statusChannel `json:"-"`
}
func (m MetricOne) MarshalJSON() ([]byte, error) {
// Declare a new type using the definition of MetricOne,
// the result of this is that M will have the same structure
// as MetricOne but none of its methods (this avoids recursive
// calls to MarshalJSON).
//
// Also because M and MetricOne have the same structure you can
// easily convert between those two. e.g. M(MetricOne{}) and
// MetricOne(M{}) are valid expressions.
type M MetricOne
// Declare a new type that has a field of the "desired" type and
// also **embeds** the M type. Embedding promotes M's fields to T
// and encoding/json will marshal those fields unnested/flattened,
// i.e. at the same level as the channels_info field.
type T struct {
M
ChannelsInfo []*statusChannel `json:"channels_info"`
}
// move map elements to slice
channels := make([]*statusChannel, 0, len(m.ChannelsInfo))
for _, c := range m.ChannelsInfo {
channels = append(channels, c)
}
// Pass in an instance of the new type T to json.Marshal.
// For the embedded M field use a converted instance of the receiver.
// For the ChannelsInfo field use the channels slice.
return json.Marshal(T{
M: M(m),
ChannelsInfo: channels,
})
}
// Same as MarshalJSON but in reverse.
func (m *MetricOne) UnmarshalJSON(data []byte) error {
type M MetricOne
type T struct {
*M
ChannelsInfo []*statusChannel `json:"channels_info"`
}
t := T{M: (*M)(m)}
if err := json.Unmarshal(data, &t); err != nil {
panic(err)
}
m.ChannelsInfo = make(map[string]*statusChannel, len(t.ChannelsInfo))
for _, c := range t.ChannelsInfo {
m.ChannelsInfo[c.ChannelId] = c
}
return nil
}
Related
I have the concept of Context which is a map that can hold any structure. Basically, I want to create a generic getter that adddressably 'populates' the destination interface (similarly to how json decoding works).
Here's an example of how I want this to work:
type Context map[string]interface{}
// Random struct that will be saved in the context
type Step struct {
Name string
}
func main() {
stepA := &Step{Name: "Cool Name"}
c := Context{}
c["stepA"] = stepA
var stepB *Step
err := c.Get("stepA", stepB)
if err != nil {
panic(err)
}
fmt.Println(stepB.Name) // Cool Name
stepB.Name = "CoolName2"
fmt.Println(stepA.Name) // I want to say: CoolName2
}
func (c Context) Get(stepId string, dest interface{}) error {
context, ok := c[stepId]
if !ok {
return nil
}
destinationValue := reflect.ValueOf(dest)
contextValue := reflect.ValueOf(context)
destinationValue.Set(contextValue) // Errors here
return nil
}
I leaned towards using reflect, but maybe I don't need it? - so opened to other suggestions (except for generics as that complicates other matters) I'm getting the following error with the above:
panic: reflect: reflect.Value.Set using unaddressable value
You can test it here.
The argument passed to Get must be a pointer type whose element type is identical to the type in the context map. So if the value in the context map is of type *Step, then the argument's type must be **Step. Also the passed in argument cannot be nil, it can be a pointer to nil, but it itself cannot be nil.
So in your case you should do:
var stepB *Step
err := c.Get("stepA", &stepB) // pass pointer-to-pointer
if err != nil {
panic(err)
}
And the Get method, fixed up a bit:
func (c Context) Get(stepId string, dest interface{}) error {
context, ok := c[stepId]
if !ok {
return nil
}
dv := reflect.ValueOf(dest)
if dv.Kind() != reflect.Ptr || dv.IsNil() {
return errors.New("dest must be non-nil pointer")
}
dv = dv.Elem()
cv := reflect.ValueOf(context)
if dv.Type() != cv.Type() {
return errors.New("dest type does not match context value type")
}
dv.Set(cv)
return nil
}
https://go.dev/play/p/OECttqp1aVg
I am trying to create a CRUD and validating my request body using a Go library called [validator][1]
Validation Code sample:
func (v *Validation) Validate(i interface{}) ValidationErrors {
errs := v.validate.Struct(i).(validator.ValidationErrors) // panic originates here
if len(errs) == 0 {
return nil
}
var returnErrs []ValidationError
for _, err := range errs {
// cast the FieldError into our ValidationError and append to the slice
ve := ValidationError{err.(validator.FieldError)}
returnErrs = append(returnErrs, ve)
}
return returnErrs
}
The above validator works for invalid request body such as invalid ID.
But for a valid body, it initiates a panic
Stack Trace:
products-api 2020/07/12 15:15:11 http: panic serving 127.0.0.1:33288: interface conversion: error is nil, not validator.ValidationErrors
goroutine 21 [running]:
net/http.(*conn).serve.func1(0xc0003b6140)
/usr/local/go/src/net/http/server.go:1772 +0x139
panic(0x93d6c0, 0xc0003845d0)
/usr/local/go/src/runtime/panic.go:973 +0x3e3
github.com/AymanArif/golang-microservices/data.(*Validation).Validate(0xc0005a8108, 0x8f3480, 0xc0005aa2c0, 0x0, 0x0, 0x203000)
/home/ayman/Desktop/golang-microservices/data/validation.go:70 +0x211
interface conversion: interface conversion: error is nil, not validator.ValidationErrors
Create REST Logic:
func (p *Products) Create(rw http.ResponseWriter, r *http.Request) {
prod := r.Context().Value(KeyProduct{}).(data.Product) // Panic originate here. Check below for struct definiton
p.l.Printf("[DEBUG] Inserting product: %#v\n", prod)
data.AddProduct(prod)
}
// data.Product
type Product struct {
ID int `json:"id"` // Unique identifier for the product
Name string `json:"name" validate:"required"`
Description string `json:"description"`
SKU string `json:"sku" validate:"sku"`
}
How can I do error handling for correct requests?
[1]: https://github.com/go-playground/validator
The error you got is eloquent:
interface conversion: interface {} is *data.Product, not data.Product
The line r.Context().Value(KeyProduct{}) returns an interface type interface{}, which, the error tells you, is holding a value of concrete type *data.Product (pointer to data.Product)
Instead, you are attempting to convert it to a data.Product, without checking whether the conversion is valid.
Replace the line with:
prod := r.Context().Value(KeyProduct{}).(*data.Product)
You might want to read the Go Tour about type assertions.
After your update, the error you have now is still the same kind of issue:
interface conversion: error is nil, not validator.ValidationErrors
With the expression err.(validator.FieldError) you are attempting to convert err to a validator.FieldError when err is in fact nil.
The previous check len(errs) == 0 is only verifying that errs length is not zero, but the slice might be of non-zero length and contain nil values.
You can instead test the type assertion:
if fe, ok := err.(validator.FieldError); ok {
ve := ValidationError{fe}
returnErrs = append(returnErrs, ve)
}
If you re write the Validate function to something like this it should work. Type assertion needs to be checked for errs & err
func (v *Validation) Validate(i interface{}) ValidationErrors {
var returnErrs []ValidationError
if errs, ok := v.validate.Struct(i).(validator.ValidationErrors); ok {
if errs != nil {
for _, err := range errs {
if fe, ok := err.(validator.FieldError); ok {
ve := ValidationError{fe}
returnErrs = append(returnErrs, ve)
}
}
}
}
return returnErrs
}
I am trying to code some generic methods (CRUD approach) to share it between my services. The following example is a GetAll() method that returns all the documents present in my collection:
func GetAll(out interface{}) error {
// mongodb operations
// iterate through all documents
for cursor.Next(ctx) {
var item interface{}
// decode the document
if err := cursor.Decode(&item); err != nil {
return err
}
(*out) = append((*out), item)
// arrays.AppendToArray(out, item) // Read below :)
}
return nil // if no error
}
I also tried with some reflection, but then:
package arrays
import "reflect"
func AppendToArray(slicePtrInterface interface{}, item interface{}) {
// enter `reflect`-land
slicePtrValue := reflect.ValueOf(slicePtrInterface)
// get the type
slicePtrType := slicePtrValue.Type()
// navigate from `*[]T` to `T`
_ = slicePtrType.Elem().Elem() // crashes if input type not `*[]T`
// we'll need this to Append() to
sliceValue := reflect.Indirect(slicePtrValue)
// append requested number of zeroes
sliceValue.Set(reflect.Append(sliceValue, reflect.ValueOf(item)))
}
panic: reflect.Set: value of type primitive.D is not assignable to type *mongodb.Test [recovered]
panic: reflect.Set: value of type primitive.D is not assignable to type *mongodb.Test
What I would like is to get the same approach as cursor.Decode(&item) (you can see above)
Here's how to do it:
// GetAll decodes the cursor c to slicep where slicep is a
// pointer to a slice of pointers to values.
func GetAll(ctx context.Context, c *Cursor, slicep interface{}) error {
// Get the slice. Call Elem() because arg is pointer to the slice.
slicev := reflect.ValueOf(slicep).Elem()
// Get value type. First call to Elem() gets slice
// element type. Second call to Elem() dereferences
// the pointer type.
valuet := slicev.Type().Elem().Elem()
// Iterate through the cursor...
for c.Next(ctx) {
// Create new value.
valuep := reflect.New(valuet)
// Decode to that value.
if err := c.Decode(valuep.Interface()); err != nil {
return err
}
// Append value pointer to slice.
slicev.Set(reflect.Append(slicev, valuep))
}
return c.Err()
}
Call it like this:
var data []*T
err := GetAll(ctx, c, &data)
if err != nil {
// handle error
}
Run it on the Go Playground.
Here's a generalization of the code to work with non-pointer slice elements:
func GetAll(ctx context.Context, c *Cursor, slicep interface{}) error {
slicev := reflect.ValueOf(slicep).Elem()
valuet := slicev.Type().Elem()
isPtr := valuet.Kind() == reflect.Ptr
if isPtr {
valuet = valuet.Elem()
}
for c.Next(ctx) {
valuep := reflect.New(valuet)
if err := c.Decode(valuep.Interface()); err != nil {
return err
}
if !isPtr {
valuep = valuep.Elem()
}
slicev.Set(reflect.Append(slicev, valuep))
}
return c.Err()
}
The program will receive many msg, msg has different struct "Data", so I define the Msg struct:
type Msg struct {
MsgType int
Data interface{}
}
type Data1 struct {
//msg type 1 Data struct
}
type Data2 struct {
//msg type 2 Data struct
}
func (msgStr string) {
msg := Msg{}
if err := json.Unmarshal([]byte(msgStr), &msg); err != nil {
//log err
}
switch msg.MsgType{
case 1:
//convert msg.Data to a type 1 struct
case 2:
//convert msg.Data to a type 2 struct
}
}
But print out the msg.Data, it is a map, not interface{}, so when I convert it to Data1 by msg.Data.(Data1), got an err.
So,
1. Why interface{} auto convert to map?
2. How to convert it to Data1 struct I want?
3. What is the best practices in this scenes.
1. Because it sees a JSON object, and, as documented, a JSON object becomes a map[string]interface{} when stored into an interface{} (this is the only type that can hold whatever is in a JSON object in generality).
2. Given your current situation, you could assign each field of the map to the appropriate field of a new Data1 or Data2.
3. The ideal way to handle this is to use json.RawMessage to defer the decoding of Data until you know what it is. This can be handled like so:
type Msg struct {
MsgType int
Data interface{}
}
func (m *Msg) UnmarshalJSON(b []byte) (err error) {
var tmp struct {
MsgType int
Data json.RawMessage
}
err = json.Unmarshal(b, &tmp)
if err != nil {
return
}
m.MsgType = tmp.MsgType
switch (tmp.MsgType) {
case 1:
data := Data1{}
err = json.Unmarshal(tmp.Data, &data)
if err != nil {
return
}
m.Data = data
case 2:
data := Data2{}
err = json.Unmarshal(tmp.Data, &data)
if err != nil {
return
}
m.Data = data
default:
return errors.New("invalid DataType")
}
return
}
And then you can call json.Unmarshal or json.Decode directly on a *Msg and its Data will be decoded as you want.
I've got the following method:
func ValidateParam(conf map[string]interface{}, paramName string, out interface{}) error {
param, ok := conf[paramName]
if !ok {
return errors.New("some error")
}
// ...
}
I would like to be able to call it like so:
myVar := "some text"
err := ValidateParam(conf, "my_var_param", &myVar)
myOtherVar := &MyStruct{}
err := ValidateParam(conf, "my_struct_param", myOtherVar)
The idea is:
Get the param using the conf map
Check that this param could be converted into the same type as out
Hydrate out using the param
=> It is kind of the same process as for json.Unmarshal(data, &myVar) or when doing a query with mgo query.Collection("col").One(&myVar)
I can't find how to achieve this, any help would be more than welcome.
Cheers
One option is to use the reflect package:
The basic idea is to create reflect.Values for input and output, check if input is assignable to output and then assign.
func ValidateParam(conf map[string]interface{}, paramName string, out interface{}) error {
param, ok := conf[paramName]
if !ok {
return errors.New("some error")
}
// Output is pointer to value.
vo := reflect.ValueOf(out)
if vo.Kind() != reflect.Ptr {
return errors.New("out must be poitner")
}
vo = vo.Elem() // deref ptr
// Can input be assigned to output?
vi := reflect.ValueOf(param)
if !vi.Type().AssignableTo(vo.Type()) {
return fmt.Errorf("param %s of type %v is not assignable to %v", paramName, vi.Type(), vo.Type())
}
vo.Set(vi)
return nil
}
playground example