Related
I’m looking to iterate through an interfaces keys.
Goal:
I want to implement a kind of middleware that checks for outgoing data (being marshalled to JSON) and edits nil slices to empty slices.
It should be agnostic/generic so that I don't need to specify field names. Ideally I can pass any struct as an interface and replace nil slices with empty slices.
Controller level
type Tag struct {
Name string
}
type BaseModel struct {
ID uuid.UUID
Active bool
}
type Model struct {
BaseModel // embedded struct
Name string
Number int
Tags []Tag
}
newModel, err := GetModel()
if err != nil {
...
}
RespondAsJson(w, newModel)
Middleware / Middle man json responder
It takes an interface to be generic/agnostic and reusable
in many different controllers
//(1) Attempting to use a map
func RespondWithJson(w http.ResponseWriter, data interface{}) {
obj, ok := data.(map[string]interface{})
// (1.1) OK == false
obj, ok := data.(map[interface{}]interface{})
// (1.2) OK == false
var newMap map[string]interface{}
bytes, _ := json.Marshal(&obj)
json.unMarshal(bytes, &newMap)
// (1.3) newMap has no underlying types on fields
// Nil slice of Tags went in, and it comes out as
// value=nil and type=interface{}
}
//(2) Skipping two as I believe this works, I'd like to avoid implementing it though.
//(3)
//(3.1)
RespondWithJson(w, newModel)
func RespondWithJson(w http.ResponseWriter, data interface{}) {
e := reflect.ValueOf(&data) // data = {*interface{} | Model}
if e.Kind() == reflect.Pointer {
e = e.Elem()
// e has a flag of 22, e.Elem() has a flag of 404
}
for i := 0; i < e.NumField(); i++ {
//PANIC: reflect: call of reflect.Value.NumField on interface Value
...
}
}
//(3.2)
// Reference: https://go.dev/blog/laws-of-reflection (third law)
RespondWithJson(w, newModel)
func RespondWithJson(w http.ResponseWriter, data interface{}) {
e := reflect.ValueOf(data) // data = {interface{} | Model}
if e.Kind() == reflect.Pointer {
e = e.Elem()
}
for i := 0; i < e.NumField(); i++ {
field := e.Field(i)
if field.Kind() == reflect.Slice && field.isNil() {
ok := field.CanSet() // OK == false
// Reference third law description in the reference above
valueOfField1 := reflect.ValueOf(&field)
ok := valueOfField1 .CanSet() // OK == false
...
valueOfField2 := reflect.ValueOf(field.Interface())
ok := valueOfField2.CanSet() // OK == false
...
}
}
}
//(3.3)
// Reference: (https://stackoverflow.com/questions/64211864/setting-nil-pointers-address-with-reflections) and others like it
RespondWithJson(w, newModel)
func RespondWithJson(w http.ResponseWriter, data interface{}) {
e := reflect.ValueOf(data) // {interface{} | Model}
if e.Kind() == reflect.Pointer { e = e.Elem() }
for i := 0; i < e.NumField(); i++ {
field := e.Field(i)
if field.Kind() == reflect.Slice && field.IsNil() {
tmp := reflect.New(field.Type())
if tmp.Kind() == reflect.Pointer { tmp = tmp.Elem()}
// (3.3.1)
ok := tmp.CanSet() // OK == true
tmp.Set(reflect.MakeSlice(field.Type(),0,0))
ok := field.CanSet()
// OK == false, tmp.set doesn't affect field value && can't set
field with value of tmp
// (3.3.2)
ok := tmp.Elem().CanSet()
// PANIC - call of reflect.value.Elem on Slicevalue
...
}
}
}
//(3.4)
// I can get it to work with passing &model to the function
// Once I'm inside the function, it's seen as an interface (or a
// *interface and the above is my results
RespondWithJson(w, &newModel)
func RespondWithJson(w http.ResponseWriter, data interface{}) {
e := reflect.ValueOf(data) // Data is {interface{} | *Model}
if e.Kind() == reflect.Pointer {
e = e.Elem()
// e has a flag of 22, e.Elem() has a flag of 409
}
for i := 0; i < e.NumField(); i++ {
field := e.Field(i)
if field.Kind() == reflect.Slice && field.IsNil() {
ok := field.CanSet()
// OK == true, field is addressable
if ok {
field.Set(reflect.MakeSlice(field.Type(), 0, 0))
// Success! Tags: nil turned into Tags: []
}
}
}
}
After that and many more.. random interations, I've found a way to make it work by passing memory address of struct to function which takes interface value.
If possible, I'd like to avoid the need to do this, as the function signature won't pick it up and it just leaves a small amount of room for error for other people on my team. I can of course just document the function, but its not bullet proof :)
Does anyone have suggestions for making this work without starting with a memory address to a struct? Can I set a field of an interface? ty very much!
In general, what you're probably looking for is something involving reflection. Your current code:
func someFunction(data interface{}) {
y := reflect.ValueOf(&data)
for i := 0; i < y.NumField(); i++ {
// PANIC: y is not a value of a struct
}
}
is pretty close, but it fails because data is a pointer. You can fix this by doing:
y := reflect.ValueOf(data)
if y.Kind() == reflect.Pointer {
y = y.Elem()
}
This will ensure that you have the actual value, and not a pointer to the value, allowing you to do NumField on it. Inside the loop, you check if the field is a slice and if it's nil and then set it to the value of a new instance of a slice of your field's type.
yField := y.Field(i)
if yField.Kind() == reflect.Slice && yField.IsNil() {
yField.Set(reflect.MakeSlice(yField.Elem().Type(), 0, 0)
}
Here we use Elem again because yField points to a slice, and so to create a new slice we need the inner type.
Finally, you need to add recursion to handle inner types if any of your fields are structs:
func SomeFunction(data interface{}) ([]byte, error) {
someFunctionInner(reflect.ValueOf(data))
return json.Marshal(data)
}
func someFunctionInner(v reflect.Value) {
if v.Kind() == reflect.Pointer {
v = v.Elem()
}
for i := 0; i < v.NumField(); i++ {
vField := v.Field(i)
switch vField.Kind() {
case reflect.Slice:
if vField.IsNil() {
vField.Set(reflect.MakeSlice(vField.Type(), 0, 0))
} else {
for j := 0; j < vField.Len(); j++ {
vFieldInner := vField.Index(j)
if vFieldInner.Kind() != reflect.Struct &&
(vFieldInner.Kind() != reflect.Pointer || vFieldInner.Elem().Kind() != reflect.Struct) {
continue
}
someFunctionInner(vFieldInner.Index(j))
}
}
case reflect.Pointer, reflect.Struct:
someFunctionInner(vField)
default:
}
}
}
and then you call it like this:
func main() {
m := Model{}
b, d := SomeFunction(&m)
fmt.Printf("Data: %+v\n", m)
fmt.Printf("JSON: %s, Error: %v\n", b, d)
}
Data: {BaseModel:{ID: Active:false} Name: Number:0 Tags:[]}
JSON: {"ID":"","Active":false,"Name":"","Number":0,"Tags":[]}, Error: <nil>
Note that I haven't added any sort of error-handling. Nor have I handled anything above regular pointers. Also, this function does expect a reference to an object because it is making modifications to said object. Finally, this code doesn't touch array logic at all. Still, this is likely what you're looking for.
The program is:
package main
import (
"fmt"
"reflect"
)
type Request struct {
Method string
Resource string //path
Protocol string
}
type s struct {
ID int
Title string
Request Request
Price float64
Interface interface{}
Exists bool
Many []string
}
func main() {
s := s{}
iterateStruct(s)
}
func iterateStruct(s interface{}) {
e := reflect.ValueOf(s)
for i := 0; i < e.NumField(); i++ {
varName := e.Type().Field(i).Name
varKind := e.Field(i).Kind()
fmt.Println(e.Type().Field(i).Name)
if varKind == reflect.Struct {
//iterateStruct( <what should be here?>)
}
varType := e.Type().Field(i).Type
varValue := e.Field(i).Interface()
fmt.Printf("%v %v %v %v\n", varName, varKind, varType, varValue)
}
}
Using recursion I'd like to get the same information for Request, that is a structure part of a structure.
What would I need to pass as a parameter? I tried various ways but I have to reckon it's a lot of trial and error for me.
Try this:
if varKind == reflect.Struct {
iterateStruct(e.Field(i).Interface())
}
e.Field(i) returns the Value for the struct field. Interface{} will return the underlying value, so you can call iterateStruct using that.
Here's an example that handles fields with pointers to structs, interfaces containing struct value, etc.. As a bonus, this example indents nested structs.
func iterate(v reflect.Value, indent string) {
v = reflect.Indirect(v)
if v.Kind() != reflect.Struct {
return
}
indent += " "
for i := 0; i < v.NumField(); i++ {
varName := v.Type().Field(i).Name
varKind := v.Field(i).Kind()
varType := v.Type().Field(i).Type
varValue := v.Field(i).Interface()
fmt.Printf("%s%v %v %v %v\n", indent, varName, varKind, varType, varValue)
iterate(v.Field(i), indent)
}
}
Call it like this:
iterate(reflect.ValueOf(s), "")
https://go.dev/play/p/y1CzbKAUvD_w
Let's say I have an instance of struct Thing1 that I want to json.Marshal
type Thing1 struct {
A string `json:"a,omitempty"`
B int `json:"b,omitempty"`
C Thing2 `json:"c,omitempty"`
}
type Thing2 struct {
D bool `json:"d,omitempty"`
E int `json:"e,omitempty"`
}
...
thing1 := Thing1{
A: "test",
B: 42,
C: Thing2{D: true, E: 43},
}
How would you write a function that takes an instance of any struct and a list of fields to redact and returns a clone (or just mutates) of the incoming object, but with the redacted fields set to their zero values?
redact(thing1, []string{"B", "D"})
thing1 == Thing1{
A: "test",
B: 0,
C: Thing2{D: false, E: 43},
}
I can't use json:"-" as a field tag because the current ones in place are required for the query language I am using (Dgraph).
edit: not in the example, but objects inside arrays should also be redacted if applicable
Use reflect to manipulate the value of struct's field. Below is a proof of concept from what I have written in the comment. Since this is just a poc, you might need to adjust/modify the code to follow your needs.
This function mutate the original data. Code is self explanatory.
func redact(target interface{}, fieldsToModify []string) {
// if target is not pointer, then immediately return
// modifying struct's field requires addresable object
addrValue := reflect.ValueOf(target)
if addrValue.Kind() != reflect.Ptr {
return
}
// if target is not struct then immediatelly return
// this might need to be modified as per your needs
targetValue := addrValue.Elem()
targetType := targetValue.Type()
if targetType.Kind() != reflect.Struct {
return
}
// loop the fields
for i := 0; i < targetType.NumField(); i++ {
fType := targetType.Field(i)
fValue := targetValue.Field(i)
// if the field type is struct, then call redact() recursively
if fValue.Kind() == reflect.Struct {
redact(fValue.Addr().Interface(), fieldsToModify)
continue
}
// if the field is slice, loop then call redact() recursively
if fValue.Kind() == reflect.Array || fValue.Kind() == reflect.Slice {
for i := 0; i < fValue.Len(); i++ {
redact(fValue.Index(i).Addr().Interface(), fieldsToModify)
}
continue
}
// loop the fieldsToModify
for _, fieldToModify := range fieldsToModify {
if fieldToModify == fType.Name && fValue.CanSet() {
fValue.Set(reflect.Zero(fType.Type))
}
}
}
}
The redact() function pointer data in first parameter, since modifying fields require addresable object.
type Thing2 struct {
D bool `json:"d,omitempty"`
E int `json:"e,omitempty"`
}
type Thing1 struct {
A string `json:"a,omitempty"`
B int `json:"b,omitempty"`
C Thing2 `json:"c,omitempty"`
H []Thing2 `json:"h,omitempty"`
}
thing1 := Thing1{
A: "test",
B: 42,
C: Thing2{D: true, E: 43},
H: []Thing2{Thing2{D: true, E: 43}},
}
fmt.Printf("before: %#v \n", thing1)
// before: main.Thing1{A:"test", B:42, C:main.Thing2{D:true, E:43}, H:[]main.Thing2{main.Thing2{D:true, E:43}}}
redact(&thing1, []string{"B", "D"})
fmt.Printf("after: %#v \n", thing1)
// after: main.Thing1{A:"test", B:0, C:main.Thing2{D:false, E:43}, H:[]main.Thing2{main.Thing2{D:false, E:43}}}
Playground: https://play.golang.org/p/wy39DGdSVV7
Here's how to do it with the reflect package:
func redact(x interface{}, names []string) error {
// Starting value must be a pointer.
v := reflect.ValueOf(x)
if v.Kind() != reflect.Ptr {
return errors.New("not pointer")
}
// Create map for easy lookup.
m := make(map[string]bool)
for _, name := range names {
m[name] = true
}
redactValue(v, m)
return nil
}
func redactValue(v reflect.Value, names map[string]bool) {
switch v.Kind() {
case reflect.Ptr:
if v.IsZero() {
return
}
redactValue(v.Elem(), names)
case reflect.Interface:
if v.IsZero() {
return
}
iv := v.Elem()
switch iv.Kind() {
case reflect.Slice, reflect.Ptr:
redactValue(iv, names)
case reflect.Struct, reflect.Array:
// Copy required for modification.
copy := reflect.New(iv.Type()).Elem()
copy.Set(iv)
redactValue(copy, names)
v.Set(copy)
}
case reflect.Struct:
t := v.Type()
for i := 0; i < t.NumField(); i++ {
sf := t.Field(i)
ft := sf.Type
fv := v.Field(i)
if names[sf.Name] {
// Clobber the field.
fv.Set(reflect.Zero(ft))
continue
}
redactValue(fv, names)
}
case reflect.Slice, reflect.Array:
for i := 0; i < v.Len(); i++ {
redactValue(v.Index(i), names)
}
}
}
Run it on the playground.
This answer handles structs, slices, arrays, pointers and interfaces.
I am trying to dynamically set a field that is of type interface{}. In all of the cases below the json unmarshals into the correct structure but in the "problem" case the json unmarshals to []interface{}. For that case I am expecting []Person. Why do I get the wrong type for "problem"?
package main
import (
"encoding/json"
"fmt"
"reflect"
)
type Employees struct {
Indicator string `json:"indicator"`
Items interface{} `json:"items"`
}
type Person struct {
Name string `json:"name"`
}
func main() {
simple()
easy()
moreDifficult()
problem()
}
var j = []byte(`{"name": "bob"}`)
var jj = []byte(`[{"name": "bob"}, {"name": "jim"}, {"name": "fred"}]`)
func simple() {
p := Person{}
if err := json.Unmarshal(j, &p); err != nil {
fmt.Println(err)
}
fmt.Println("easy:", p, reflect.TypeOf(p))
}
func easy() {
p := []Person{}
if err := json.Unmarshal(jj, &p); err != nil {
fmt.Println(err)
}
fmt.Println("easy:", p, reflect.TypeOf(p))
}
func moreDifficult() {
var j = []byte(`{"indicator": "more difficult"}`)
e := Employees{}
if err := json.Unmarshal(j, &e); err != nil {
fmt.Println(err)
}
fmt.Println("moreDifficult", e.Items, reflect.TypeOf(e.Items))
}
func problem() {
var j = []byte(`{"indicator:": "problem"}`)
e := Employees{}
if err := json.Unmarshal(j, &e); err != nil {
fmt.Println(err)
}
fmt.Println("problem:", e.Items, reflect.TypeOf(e.Items)) // why not []Person???
}
func (e *Employees) UnmarshalJSON(b []byte) error {
type alias Employees
a := &alias{}
if err := json.Unmarshal(b, &a); err != nil {
return err
}
e.Indicator = a.Indicator
var k = jj
if e.Indicator == "more difficult" {
k = j
e.Items = &Person{}
} else {
e.Items = []Person{}
}
return json.Unmarshal(k, &e.Items)
}
https://play.golang.org/p/xQvjMyLTk5i
The problem is that the interface's underlying value is not a pointer:
e.Items = []Person{}
While it's true that you're passing a pointer to the interface itself:
json.Unmarshal(k, &e.Items)
That does not fix the problem of the underlying value being a non-pointer.
var a interface{} = &T{}
var b interface{} = T{}
b = &b
The types of a and b are different and will be handled differently by the unmarshaler, in b's case the unmarshaler will elect to replace the pointed-to value with a map[string]interface{}.
So to fix your immediate problem you can do something like this:
if e.Indicator == "more difficult" {
k = j
e.Items = &Person{}
} else {
e.Items = &[]Person{}
}
return json.Unmarshal(k, e.Items)
Your current problem has absolutely nothing to do with JSON unmarshaling.
You define Items as type interface{}, so obviously, when you inspect it, it will be of type interface{}. If you want it to be of type []Person, simply define it as such:
type Employees struct {
Indicator string `json:"indicator"`
Items []Person `json:"items"`
}
Once you've done that, your test case will yield the expected results. But your Items will still be empty, as your input JSON doesn't have the items field.
I'm looking to iterate over the string fields of a struct so I can do some clean-up/validation (with strings.TrimSpace, strings.Trim, etc).
Right now I have a messy switch-case that's not really scalable, and as this isn't in a hot spot of my application (a web form) it seems leveraging reflect is a good choice here.
I'm at a bit of a roadblock for how to implement this however, and the reflect docs are a little confusing to me (I've been digging through some other validation packages, but they're way too heavyweight + I'm using gorilla/schema for the unmarshalling part already):
Iterate over the struct
For each field of type string, apply whatever I need to from the strings package i.e. field = strings.TrimSpace(field)
If there exists a field.Tag.Get("max"), we'll use that value (strconv.Atoi, then unicode.RuneCountInString)
Provide an error slice that's also compatible with the error interface type
type FormError []string
type Listing struct {
Title string `max:"50"`
Location string `max:"100"`
Description string `max:"10000"`
ExpiryDate time.Time
RenderedDesc template.HTML
Contact string `max:"255"`
}
// Iterate over our struct, fix whitespace/formatting where possible
// and return errors encountered
func (l *Listing) Validate() error {
typ := l.Elem().Type()
var invalid FormError
for i = 0; i < typ.NumField(); i++ {
// Iterate over fields
// For StructFields of type string, field = strings.TrimSpace(field)
// if field.Tag.Get("max") != "" {
// check max length/convert to int/utf8.RuneCountInString
if max length exceeded, invalid = append(invalid, "errormsg")
}
if len(invalid) > 0 {
return invalid
}
return nil
}
func (f FormError) Error() string {
var fullError string
for _, v := range f {
fullError =+ v + "\n"
}
return "Errors were encountered during form processing: " + fullError
}
Thanks in advance.
What you want is primarily the methods on reflect.Value called NumFields() int and Field(int). The only thing you're really missing is the string check and SetString method.
package main
import "fmt"
import "reflect"
import "strings"
type MyStruct struct {
A,B,C string
I int
D string
J int
}
func main() {
ms := MyStruct{"Green ", " Eggs", " and ", 2, " Ham ", 15}
// Print it out now so we can see the difference
fmt.Printf("%s%s%s%d%s%d\n", ms.A, ms.B, ms.C, ms.I, ms.D, ms.J)
// We need a pointer so that we can set the value via reflection
msValuePtr := reflect.ValueOf(&ms)
msValue := msValuePtr.Elem()
for i := 0; i < msValue.NumField(); i++ {
field := msValue.Field(i)
// Ignore fields that don't have the same type as a string
if field.Type() != reflect.TypeOf("") {
continue
}
str := field.Interface().(string)
str = strings.TrimSpace(str)
field.SetString(str)
}
fmt.Printf("%s%s%s%d%s%d\n", ms.A, ms.B, ms.C, ms.I, ms.D, ms.J)
}
(Playground link)
There are two caveats here:
You need a pointer to what you're going to change. If you have a value, you'll need to return the modified result.
Attempts to modify unexported fields generally will cause reflect to panic. If you plan on modifying unexported fields, make sure to do this trick inside the package.
This code is rather flexible, you can use switch statements or type switches (on the value returned by field.Interface()) if you need differing behavior depending on the type.
Edit: As for the tag behavior, you seem to already have that figured out. Once you have field and have checked that it's a string, you can just use field.Tag.Get("max") and parse it from there.
Edit2: I made a small error on the tag. Tags are part of the reflect.Type of a struct, so to get them you can use (this is a bit long-winded) msValue.Type().Field(i).Tag.Get("max")
(Playground version of the code you posted in the comments with a working Tag get).
I got beat to the punch, but since I went to the work, here's a solution:
type FormError []*string
type Listing struct {
Title string `max:"50"`
Location string `max:"100"`
Description string `max:"10000"`
ExpiryDate time.Time
RenderedDesc template.HTML
Contact string `max:"255"`
}
// Iterate over our struct, fix whitespace/formatting where possible
// and return errors encountered
func (l *Listing) Validate() error {
listingType := reflect.TypeOf(*l)
listingValue := reflect.ValueOf(l)
listingElem := listingValue.Elem()
var invalid FormError = []*string{}
// Iterate over fields
for i := 0; i < listingElem.NumField(); i++ {
fieldValue := listingElem.Field(i)
// For StructFields of type string, field = strings.TrimSpace(field)
if fieldValue.Type().Name() == "string" {
newFieldValue := strings.TrimSpace(fieldValue.Interface().(string))
fieldValue.SetString(newFieldValue)
fieldType := listingType.Field(i)
maxLengthStr := fieldType.Tag.Get("max")
if maxLengthStr != "" {
maxLength, err := strconv.Atoi(maxLengthStr)
if err != nil {
panic("Field 'max' must be an integer")
}
// check max length/convert to int/utf8.RuneCountInString
if utf8.RuneCountInString(newFieldValue) > maxLength {
// if max length exceeded, invalid = append(invalid, "errormsg")
invalidMessage := `"`+fieldType.Name+`" is too long (max allowed: `+maxLengthStr+`)`
invalid = append(invalid, &invalidMessage)
}
}
}
}
if len(invalid) > 0 {
return invalid
}
return nil
}
func (f FormError) Error() string {
var fullError string
for _, v := range f {
fullError = *v + "\n"
}
return "Errors were encountered during form processing: " + fullError
}
I see you asked about how to do the tags. Reflection has two components: a type and a value. The tag is associated with the type, so you have to get it separately than the field: listingType := reflect.TypeOf(*l). Then you can get the indexed field and the tag from that.
I don't know if it's a good way, but I use it like this.
https://play.golang.org/p/aQ_hG2BYmMD
You can send the address of a struct to this function.
Sorry for My English is not very good.
trimStruct(&someStruct)
func trimStruct(v interface{}) {
bytes, err := json.Marshal(v)
if err != nil {
fmt.Println("[trimStruct] Marshal Error :", err)
}
var mapSI map[string]interface{}
if err := json.Unmarshal(bytes, &mapSI); err != nil {
fmt.Println("[trimStruct] Unmarshal to byte Error :", err)
}
mapSI = trimMapStringInterface(mapSI).(map[string]interface{})
bytes2, err := json.Marshal(mapSI)
if err != nil {
fmt.Println("[trimStruct] Marshal Error :", err)
}
if err := json.Unmarshal(bytes2, v); err != nil {
fmt.Println("[trimStruct] Unmarshal to b Error :", err)
}
}
func trimMapStringInterface(data interface{}) interface{} {
if values, valid := data.([]interface{}); valid {
for i := range values {
data.([]interface{})[i] = trimMapStringInterface(values[i])
}
} else if values, valid := data.(map[string]interface{}); valid {
for k, v := range values {
data.(map[string]interface{})[k] = trimMapStringInterface(v)
}
} else if value, valid := data.(string); valid {
data = strings.TrimSpace(value)
}
return data
}