Iterate fields of different structures in one array - go

I want to iterate fields of different structures in one array
i try to do it like this
package main
import "fmt"
func main() {
m := msh{"hshHeader"}
p := pid{"pidHeader"}
var i []EmptyInterface
i = append(i, m)
i = append(i, p)
for _, emptyInterface := range i {
fmt.Println(emptyInterface.header)
}
}
type msh struct {
header string
}
type pid struct {
header string
}
type EmptyInterface interface {
}
As expected, on line 15 I get the message "Unresolved reference 'header'".
Do I understand correctly that in Go I should do this using methods
package main
import "fmt"
func main() {
m := msh{"hshHeader"}
p := pid{"pidHeader"}
var i []EmptyInterface
i = append(i, m)
i = append(i, p)
for _, NotEmptyInterface := range i {
fmt.Println(NotEmptyInterface.PrintHeader())
}
}
type msh struct {
header string
}
func (m msh) PrintHeader() string {
return m.header
}
type pid struct {
header string
}
func (p pid) PrintHeader() string {
return p.header
}
type EmptyInterface interface {
PrintHeader() string
}
I don’t know what else to add so that the bot on the site does not throw an error "It looks like your post is mostly code; please add some more details.", I can suggest reading again.

Related

How to put regexp inside proto struct in golang?

I have a tree-like structure, that has string regexp and I want Go compiled *regexp.Regexp to be part of it as well, in order to run algorithms on the tree. When I marshal and pass it to a different machine I may just recompile it again from the string. What is the correct way to do that, how to force protobuf to store pointers in a structure, that it ideally wont marshal? (the only way that i see is to make uint64 field and cast its value to/from *Regexp)
pseudo-code (because required wanted features seems to be not in the language):
// struct generated by protoc
type ProtoMessage struct {
Data string
Source string
Regexp uint64 // should not be marshalled, should be forcefully omitted from payload when doing proto.Marshal, ideally it should be *regexp.Regexp
Left *ProtoMessage
Right *ProtoMessage
}
func main() {
// sender computer doSend():
mSrc := &ProtoMessage{Data:"its meee!!!", Source: "hello.+world"}
payload, _ := proto.Marshal(m)
//receiver computer: onRecv()
mDst := new(ProtoMessage)
proto.Unmarshal(payload, mDst)
r, _ := regexp.Compile(mDst.Source)
mDst.Regexp = uint64(unsafe.Pointer(r)) // not working btw
TreeMatch = func(tree* ProtoMessage, line string) string {
if *regexp.Regexp(t.Regexp).Match(line) { // not working line
return t.Data
}
if tree.Left == nil {
return ""
}
return TreeMatch(tree.Left, line)
}
assert( TreeMatch(mDst, "hello, world") == "its meee!!!") // panic if condition is false
}
With json marshal i can just pot a pointer to regexp and provide a tag json:"-" in order not to include this field into marshalled structure, and ofc its important feature of marshalling/unmarshalling system to stay efficient (eg use same structure to run algorithms on in, and avoid data copying after unmarshal). How can I do the same with protobuf?
You can't store a pointer in a protobuf, as the recipient is likely a different computer. Even if you could, you'd get a panic as soon as you tried to dereference the pointer. Easiest thing to do would be just pass the RegExp string, then compile again at the destination:
package main
import (
"fmt"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/structpb"
)
func main() {
v := structpb.NewStringValue("hello.+world")
b, err := proto.Marshal(v)
if err != nil {
panic(err)
}
fmt.Printf("%q\n", b) // "\x1a\fhello.+world"
}
Note: you can't hack around this with Gob either:
package main
import (
"bytes"
"encoding/gob"
"regexp"
)
func main() {
re := regexp.MustCompile("hello.+world")
buf := new(bytes.Buffer)
if err := gob.NewEncoder(buf).Encode(re); err != nil {
panic(err) // type regexp.Regexp has no exported fields
}
}
Found the solution, you just have to have any pointer inside your struct (no matter if its marshalling or not, you are not using its unmarshalled value on receiver side):
proto declaration:
syntax = "proto3";
package main;
option go_package = ".;main";
message Empty {
}
message ProtoMessage {
string data = 1;
string source = 2;
Empty regexp = 3; // ideally should not be marshalled at all, like `json:"-"` but for protobuf
ProtoMessage left = 4;
ProtoMessage right = 5;
}
testing code:
package main
import (
"regexp"
"testing"
"unsafe"
)
type Empty struct {
//state protoimpl.MessageState
//sizeCache protoimpl.SizeCache
//unknownFields protoimpl.UnknownFields
}
// struct generated by protoc
type ProtoMessage struct {
//state protoimpl.MessageState
//sizeCache protoimpl.SizeCache
//unknownFields protoimpl.UnknownFields
Data string `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"`
Source string `protobuf:"bytes,2,opt,name=source,proto3" json:"source,omitempty"`
Regexp *Empty `protobuf:"bytes,3,opt,name=regexp,proto3" json:"regexp,omitempty"` // ideally should not be marshalled at all, like `json:"-"` but for protobuf
Left *ProtoMessage `protobuf:"bytes,4,opt,name=left,proto3" json:"left,omitempty"`
Right *ProtoMessage `protobuf:"bytes,5,opt,name=right,proto3" json:"right,omitempty"`
}
func (p *ProtoMessage) GetCompiledRegexp() *regexp.Regexp {
return (*regexp.Regexp)(unsafe.Pointer(p.Regexp))
}
func (p *ProtoMessage) SetCompiledRegexp(r *regexp.Regexp) {
p.Regexp = (*Empty)(unsafe.Pointer(r))
}
func TreeMatch(tree *ProtoMessage, line string) string {
if tree.GetCompiledRegexp().Match([]byte(line)) { // not working line
return tree.Data
}
if tree.Left == nil {
return ""
}
return TreeMatch(tree.Left, line)
}
func TestTreeMatch(t *testing.T) {
//happening at receiver side: imagine its proto.Unmarshal(payload, receiverMsg)
receiverMsg := &ProtoMessage{
Data: "its meee!!!",
Source: "hello.+world",
}
r, _ := regexp.Compile(receiverMsg.Source)
receiverMsg.SetCompiledRegexp(r)
if TreeMatch(receiverMsg, "helloworld") != "" {
t.Fatalf("TreeMatch gives non-existing match!")
}
if TreeMatch(receiverMsg, "hello, world") != "its meee!!!" {
t.Fatalf("TreeMatch is not working!")
}
}
type ProtoMessageDirect struct {
Data string
Source string
Regexp *regexp.Regexp
Left *ProtoMessageDirect
Right *ProtoMessageDirect
}
func (p *ProtoMessageDirect) GetCompiledRegexp() *regexp.Regexp {
return p.Regexp
}
func (p *ProtoMessageDirect) SetCompiledRegexp(r *regexp.Regexp) {
p.Regexp = r
}
func TreeMatchDirect(tree *ProtoMessageDirect, line string) string {
if tree.GetCompiledRegexp().Match([]byte(line)) { // not working line
return tree.Data
}
if tree.Left == nil {
return ""
}
return TreeMatchDirect(tree.Left, line)
}
func BenchmarkRegexpCast(b *testing.B) {
receiverMsg := &ProtoMessage{
Data: "its meee!!!",
Source: "hello.+world",
}
r, _ := regexp.Compile(receiverMsg.Source)
receiverMsg.SetCompiledRegexp(r)
b.ResetTimer()
for i := 0; i < b.N; i++ {
TreeMatch(receiverMsg, "hello, world")
}
}
func BenchmarkRegexpDirect(b *testing.B) {
receiverMsg := &ProtoMessageDirect{
Data: "its meee!!!",
Source: "hello.+world",
}
r, _ := regexp.Compile(receiverMsg.Source)
receiverMsg.SetCompiledRegexp(r)
b.ResetTimer()
for i := 0; i < b.N; i++ {
TreeMatchDirect(receiverMsg, "hello, world")
}
}
TestTreeMatch is passing and Benchmarks shows that such a cast does not create any meaningful difference:
BenchmarkRegexpCast-20 2741786 376.7 ns/op 16 B/op 1 allocs/op
BenchmarkRegexpDirect-20 3075280 377.0 ns/op 16 B/op 1 allocs/op
PASS

How to manipulate a map field in a structure with reflect package?

We have a structure like:
type S struct {
M map[string]bool
}
And how can we implement a function like:
// this function modify field by name to a new map
func Modify(s *S, name string, val map[string]bool){
...
}
func Test() {
s := S{
M: map[string]bool{"Hello":true},
}
m := map[string]bool{"World":false}
Modify(&s, "M", m)
}
The reflect package support SetInt/SetString/etc, but none SetMap.
Any way to solve this problem?
Use reflect.Set()
func Modify(s *S, name string, val interface{}) {
rv := reflect.ValueOf(val)
if !rv.IsValid() {
rv = reflect.Zero(reflect.ValueOf(s).Elem().FieldByName(name).Type())
}
reflect.ValueOf(s).Elem().FieldByName(name).Set(rv)
}
Playground

Unmarshal map[string]DynamoDBAttributeValue into a struct

I'm trying to set-up an AWS-lambda using aws-sdk-go that is triggered whenever a new user is added to a certain dynamodb table.
Everything is working just fine but I can't find a way to unmarshal a map map[string]DynamoDBAttributeValue like:
{
"name": {
"S" : "John"
},
"residence_address": {
"M": {
"address": {
"S": "some place"
}
}
}
}
To a given struct, for instance, a User struct. Here is shown an example of unsmarhaling a map[string]*dynamodb.AttributeValue into a given interface, but I can't find a way to do the same thing with map[string]DynamoDBAttributeValue even though these types seem to fit the same purposes.
map[string]DynamoDBAttributeValue is returned by a events.DynamoDBEvents from package github.com/aws/aws-lambda-go/events. This is my code:
package handler
import (
"context"
"github.com/aws/aws-lambda-go/events"
"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute"
"github.com/aws/aws-sdk-go/service/dynamodb"
)
func HandleDynamoDBRequest(ctx context.Context, e events.DynamoDBEvent) {
for _, record := range e.Records {
if record.EventName == "INSERT" {
// User Struct
var dynamoUser model.DynamoDBUser
// Of course this can't be done for incompatible types
_ := dynamodbattribute.UnmarshalMap(record.Change.NewImage, &dynamoUser)
}
}
}
Of course, I can marshal record.Change.NewImage to JSON and unmarshal it back to a given struct, but then, I would have to manually initialize dynamoUser attributes starting from the latter ones.
Or I could even write a function that parses map[string]DynamoDBAttributeValue to map[string]*dynamodb.AttributeValue like:
func getAttributeValueMapFromDynamoDBStreamRecord(e events.DynamoDBStreamRecord) map[string]*dynamodb.AttributeValue {
image := e.NewImage
m := make(map[string]*dynamodb.AttributeValue)
for k, v := range image {
if v.DataType() == events.DataTypeString {
s := v.String()
m[k] = &dynamodb.AttributeValue{
S : &s,
}
}
if v.DataType() == events.DataTypeBoolean {
b := v.Boolean()
m[k] = &dynamodb.AttributeValue{
BOOL : &b,
}
}
// . . .
if v.DataType() == events.DataTypeMap {
// ?
}
}
return m
}
And then simply use dynamodbattribute.UnmarshalMap, but on events.DataTypeMap it would be quite a tricky process.
Is there a way through which I can unmarshal a DynamoDB record coming from a events.DynamoDBEvent into a struct with a similar method shown for map[string]*dynamodb.AttributeValue?
I tried the function you provided, and I met some problems with events.DataTypeList, so I managed to write the following function that does the trick:
// UnmarshalStreamImage converts events.DynamoDBAttributeValue to struct
func UnmarshalStreamImage(attribute map[string]events.DynamoDBAttributeValue, out interface{}) error {
dbAttrMap := make(map[string]*dynamodb.AttributeValue)
for k, v := range attribute {
var dbAttr dynamodb.AttributeValue
bytes, marshalErr := v.MarshalJSON(); if marshalErr != nil {
return marshalErr
}
json.Unmarshal(bytes, &dbAttr)
dbAttrMap[k] = &dbAttr
}
return dynamodbattribute.UnmarshalMap(dbAttrMap, out)
}
I was frustrated that the type of NewImage from the record wasn't map[string]*dynamodb.AttributeValue so I could use the dynamodbattribute package.
The JSON representation of events.DynamoDBAttributeValue seems to be the same as the JSON represenation of dynamodb.AttributeValue.
So I tried creating my own DynamoDBEvent type and changed the type of OldImage and NewImage, so it would be marshalled into map[string]*dynamodb.AttributeValue instead of map[string]events.DynamoDBAttributeValue
It is a little bit ugly but it works for me.
package main
import (
"github.com/aws/aws-lambda-go/events"
"github.com/aws/aws-lambda-go/lambda"
"github.com/aws/aws-sdk-go/service/dynamodb"
"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute"
"fmt"
)
func main() {
lambda.Start(lambdaHandler)
}
// changed type of event from: events.DynamoDBEvent to DynamoDBEvent (see below)
func lambdaHandler(event DynamoDBEvent) error {
for _, record := range event.Records {
change := record.Change
newImage := change.NewImage // now of type: map[string]*dynamodb.AttributeValue
var item IdOnly
err := dynamodbattribute.UnmarshalMap(newImage, &item)
if err != nil {
return err
}
fmt.Println(item.Id)
}
return nil
}
type IdOnly struct {
Id string `json:"id"`
}
type DynamoDBEvent struct {
Records []DynamoDBEventRecord `json:"Records"`
}
type DynamoDBEventRecord struct {
AWSRegion string `json:"awsRegion"`
Change DynamoDBStreamRecord `json:"dynamodb"`
EventID string `json:"eventID"`
EventName string `json:"eventName"`
EventSource string `json:"eventSource"`
EventVersion string `json:"eventVersion"`
EventSourceArn string `json:"eventSourceARN"`
UserIdentity *events.DynamoDBUserIdentity `json:"userIdentity,omitempty"`
}
type DynamoDBStreamRecord struct {
ApproximateCreationDateTime events.SecondsEpochTime `json:"ApproximateCreationDateTime,omitempty"`
// changed to map[string]*dynamodb.AttributeValue
Keys map[string]*dynamodb.AttributeValue `json:"Keys,omitempty"`
// changed to map[string]*dynamodb.AttributeValue
NewImage map[string]*dynamodb.AttributeValue `json:"NewImage,omitempty"`
// changed to map[string]*dynamodb.AttributeValue
OldImage map[string]*dynamodb.AttributeValue `json:"OldImage,omitempty"`
SequenceNumber string `json:"SequenceNumber"`
SizeBytes int64 `json:"SizeBytes"`
StreamViewType string `json:"StreamViewType"`
}
I have found the same problem and the solution is to perform a simple conversion of types. This is possible because in the end the type received by lambda events events.DynamoDBAttributeValue and the type used by the SDK V2 of AWS DynamoDB types.AttributeValue are the same. Next I show you the conversion code.
package aws_lambda
import (
"github.com/aws/aws-lambda-go/events"
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
)
func UnmarshalDynamoEventsMap(
record map[string]events.DynamoDBAttributeValue, out interface{}) error {
asTypesMap := DynamoDbEventsMapToTypesMap(record)
err := attributevalue.UnmarshalMap(asTypesMap, out)
if err != nil {
return err
}
return nil
}
func DynamoDbEventsMapToTypesMap(
record map[string]events.DynamoDBAttributeValue) map[string]types.AttributeValue {
resultMap := make(map[string]types.AttributeValue)
for key, rec := range record {
resultMap[key] = DynamoDbEventsToTypes(rec)
}
return resultMap
}
// DynamoDbEventsToTypes relates the dynamo event received by AWS Lambda with the data type that is
// used in the Amazon SDK V2 to deal with DynamoDB data.
// This function is necessary because Amazon does not provide any kind of solution to make this
// relationship between types of data.
func DynamoDbEventsToTypes(record events.DynamoDBAttributeValue) types.AttributeValue {
var val types.AttributeValue
switch record.DataType() {
case events.DataTypeBinary:
val = &types.AttributeValueMemberB{
Value: record.Binary(),
}
case events.DataTypeBinarySet:
val = &types.AttributeValueMemberBS{
Value: record.BinarySet(),
}
case events.DataTypeBoolean:
val = &types.AttributeValueMemberBOOL{
Value: record.Boolean(),
}
case events.DataTypeList:
var items []types.AttributeValue
for _, value := range record.List() {
items = append(items, DynamoDbEventsToTypes(value))
}
val = &types.AttributeValueMemberL{
Value: items,
}
case events.DataTypeMap:
items := make(map[string]types.AttributeValue)
for k, v := range record.Map() {
items[k] = DynamoDbEventsToTypes(v)
}
val = &types.AttributeValueMemberM{
Value: items,
}
case events.DataTypeNull:
val = nil
case events.DataTypeNumber:
val = &types.AttributeValueMemberN{
Value: record.Number(),
}
case events.DataTypeNumberSet:
val = &types.AttributeValueMemberNS{
Value: record.NumberSet(),
}
case events.DataTypeString:
val = &types.AttributeValueMemberS{
Value: record.String(),
}
case events.DataTypeStringSet:
val = &types.AttributeValueMemberSS{
Value: record.StringSet(),
}
}
return val
}
There is a package that allows conversion from events.DynamoDBAttributeValue to dynamodb.AttributeValue
https://pkg.go.dev/github.com/aereal/go-dynamodb-attribute-conversions/v2
From there one can unmarshal AttributeValue into struct
func Unmarshal(attribute map[string]events.DynamoDBAttributeValue, out interface{}) error {
av := ddbconversions.AttributeValueMapFrom(attribute)
return attributevalue.UnmarshalMap(av, out)
}

How to get embedded type from GO struct?

I am trying to get embedded type from Go structs. Below is an example program that demonstrates this. Is there a way to write myfunc() without enumerating every type that can come in as input?
https://play.golang.org/p/5wp14O660m
package main
import (
"fmt"
)
type ObjectMeta struct {
Name string
Namespace string
}
type A struct {
ObjectMeta
X string
}
type B struct {
ObjectMeta
X string
}
func myfunc(v interface{}) ObjectMeta {
switch u := v.(type) {
case *A:
return u.ObjectMeta
case A:
return u.ObjectMeta
case *B:
return u.ObjectMeta
case B:
return u.ObjectMeta
}
panic("No matching type")
}
func main() {
fmt.Println(myfunc(&A{}))
var v interface{} = &A{}
fmt.Println(v.(*ObjectMeta))
}
ObjectMeta, A, B structs exist in external project. I have no control over them.
It can be done using reflection, iterating through the fields of the incoming value:
func myfunc(v interface{}) ObjectMeta {
// Elem() to de-reference pointer
ifv := reflect.ValueOf(v).Elem()
ift := reflect.TypeOf(v).Elem()
for i := 0; i < ift.NumField(); i++ {
f := ift.Field(i)
if f.Name == "ObjectMeta" {
fv := ifv.Field(i)
return fv.Interface().(ObjectMeta)
}
}
panic("ObjectMeta not found")
}
Playground: https://play.golang.org/p/CzMHJWhxYr
You can define interface which will get you that embedded type:
package main
import (
"fmt"
)
type HasMeta interface {
GetMeta() ObjectMeta
}
type ObjectMeta struct {
Name string
Namespace string
}
func (o ObjectMeta) GetMeta() ObjectMeta {
return o
}
type A struct {
ObjectMeta
X string
}
type B struct {
ObjectMeta
X string
}
func myfunc(o HasMeta) ObjectMeta {
return o.GetMeta()
}
func main() {
fmt.Println(myfunc(&A{}))
fmt.Println(myfunc(A{}))
fmt.Println(myfunc(&B{}))
fmt.Println(myfunc(B{}))
}
https://play.golang.org/p/CWa4k-kvvl

Get name of struct field using reflection

What is the way of printing "Foo" here? In this example, what prints is "string".
http://play.golang.org/p/ZnK6PRwEPp
type A struct {
Foo string
}
func (a *A) PrintFoo() {
fmt.Println("Foo value is " + a.Foo)
}
func main() {
a := &A{Foo: "afoo"}
val := reflect.Indirect(reflect.ValueOf(a))
fmt.Println(val.Field(0).Type().Name())
}
You want val.Type().Field(0).Name. The Field method on reflect.Type will return a struct describing that field, which includes the name, among other information.
There is no way to retrieve the field name for a reflect.Value representing a particular field value, since that is a property of the containing struct.
I think the better way to get the fields' name in the struct is
func main() {
a := &A{Foo: "afoo"}
val := reflect.ValueOf(a).Elem()
for i:=0; i<val.NumField();i++{
fmt.Println(val.Type().Field(i).Name)
}
}
There are two tips:
use .Elem() after you reflect.ValueOf(a), because in your case, a is a pointer.
val.Field(i).Type().Name is totally different from val.Type().Field(i).Name. The latter one can get the name of the field in the struct
Hope that it is helpful..
If you want to have a look at more cases, please check my 2mins article
You need to Get the Field of the Type Definition not of the Value.
http://play.golang.org/p/7Bc7MJikbJ
package main
import "fmt"
import "reflect"
type A struct {
Foo string
}
func (a *A) PrintFoo() {
fmt.Println("Foo value is " + a.Foo)
}
func main() {
a := &A{Foo: "afoo"}
val := reflect.Indirect(reflect.ValueOf(a))
fmt.Println(val.Type().Field(0).Name)
}
With the new Names method of the structs package it's even more easier:
package main
import (
"fmt"
"github.com/fatih/structs"
)
type A struct {
Foo string
Bar int
}
func main() {
names := structs.Names(&A{})
fmt.Println(names) // ["Foo", "Bar"]
}
You can also use https://github.com/fatih/structs
// Convert the fields of a struct to a []*Field
fields := s.Fields()
for _, f := range fields {
fmt.Printf("field name: %+v\n", f.Name())
}
package main
import "fmt"
import "reflect"
type A struct {
Foo string
}
func (a *A) PrintFoo() {
fmt.Println("Foo value is " + a.Foo)
}
func main() {
a := &A{Foo: "afoo"}
//long and bored code
t := reflect.TypeOf(*a)
if t.Kind() == reflect.Struct {
for i := 0; i < t.NumField(); i++ {
fmt.Println(t.Field(i).Name)
}
} else {
fmt.Println("not a stuct")
}
//shorthanded call
fmt.Println(reflect.TypeOf(*a).Field(0).Name)//can panic if no field exists
}
You can use this function, which takes the struct as the first parameter, and then its fields. It returns the map type, which is convenient to use
If you use fields from another structure, nothing will happen
If you try to use a different type, it will cause panic
Note that the field has an ordinal number according to the list (starting from 0). All fields in the structure must start with uppercase
func GetStructFieldName(Struct interface{}, StructField ...interface{}) (fields map[int]string) {
fields = make(map[int]string)
s := reflect.ValueOf(Struct).Elem()
for r := range StructField {
f := reflect.ValueOf(StructField[r]).Elem()
for i := 0; i < s.NumField(); i++ {
valueField := s.Field(i)
if valueField.Addr().Interface() == f.Addr().Interface() {
fields[i] = s.Type().Field(i).Name
}
}
}
return fields
}
Full example and playground
package main
import (
"fmt"
"reflect"
)
type Example struct {
Apple bool
Pear int
}
func GetStructFieldName(Struct interface{}, StructField ...interface{}) (fields map[int]string) {
fields = make(map[int]string)
for r := range StructField {
s := reflect.ValueOf(Struct).Elem()
f := reflect.ValueOf(StructField[r]).Elem()
for i := 0; i < s.NumField(); i++ {
valueField := s.Field(i)
if valueField.Addr().Interface() == f.Addr().Interface() {
fields[i] = s.Type().Field(i).Name
}
}
}
return fields
}
func main() {
e := Example{}
names := GetStructFieldName(&e, &e.Apple, &e.Pear)
fmt.Println(names)
fmt.Println(names[0], names[1])
for i := range names {
fmt.Println(names[i])
}
/* Output:
map[0:Apple 1:Pear]
Apple Pear
Apple
Pear
*/
}

Resources