Trying to unmarshal YAML into complex object such as map[string]map[interface{}]string.
The problem is that I want to be able to differentiate an interface{} part between string and Source which is a struct.
type Source struct {
ID string `yaml:"id"`
Name string `yaml:"name"`
LogoID string `yaml:"logoId"`
URL string `yaml:"url"`
}
type UNFT struct {
ItemMeta map[string]map[interface{}]string `yaml:"item_meta"`
// could be
// ItemMeta map[string]map[string]string `yaml:"item_meta"`
// or
// ItemMeta map[string]map[Source]string `yaml:"item_meta"`
}
Obviously YAML does not know how to unmarshal into Source struct so I have to implement Unmarshaler interface:
type Unmarshaler interface {
UnmarshalYAML(value *Node) error
}
But I don't quite understand the big picture of unmarshaling process. In general I assume that I have to manually traverse *yaml.Node and call func UnmarshalYAML(value *Node) error on every node.
package main
import (
"fmt"
"gopkg.in/yaml.v3"
)
type Source struct {
ID string `json:"id"`
Name string `json:"name"`
LogoID string `json:"logoId"`
URL string `json:"url"`
}
var data = `
unf:
item_meta:
source:
!struct
? id: "data-watch"
name: "DataWatch"
logoid: "data-watch"
url: "https"
: "product_any('SS')"
public_usage:
"": "source_any('SDF')"
"provider": "source_any('ANO')"`
type UNFT struct {
ItemMeta map[string]map[interface{}]string `yaml:"item_meta"`
}
type MetaConverterConfigT struct {
UNFT UNFT `yaml:"unf"`
}
func main() {
cfg := MetaConverterConfigT{}
err := yaml.Unmarshal([]byte(data), &cfg)
if err != nil {
fmt.Println("%w", err)
}
fmt.Println(cfg)
}
func (s *UNFT) UnmarshalYAML(n *yaml.Node) error {
var cfg map[string]map[interface{}]string
if err := n.Decode(&cfg); err != nil {
fmt.Println("%w", err)
}
return nil
}
Go playground
type MetaKey struct {
String string
Source Source
}
func (k *MetaKey) UnmarshalYAML(n *yaml.Node) error {
if n.Tag == "!!str" {
return n.Decode(&k.String)
}
if n.Tag == "!!map" {
return n.Decode(&k.Source)
}
return fmt.Errorf("unsupported MetaKey type")
}
// ...
type UNFT struct {
ItemMeta map[string]map[MetaKey]string `yaml:"item_meta"`
}
https://go.dev/play/p/Nhtab4l-ANT
If you need the map type to remain as is, i.e. without adding the custom key type, then you can implement the unmarshaler on UNFT as well and just do a re-mapping with any:
type UNFT struct {
ItemMeta map[string]map[any]string `yaml:"item_meta"`
}
func (u *UNFT) UnmarshalYAML(n *yaml.Node) error {
var obj struct {
ItemMeta map[string]map[MetaKey]string `yaml:"item_meta"`
}
if err := n.Decode(&obj); err != nil {
return err
}
u.ItemMeta = make(map[string]map[any]string, len(obj.ItemMeta))
for k, v := range obj.ItemMeta {
m := make(map[any]string, len(v))
for k, v := range v {
if k.Source != (Source{}) {
m[k.Source] = v
} else {
m[k.String] = v
}
}
u.ItemMeta[k] = m
}
return nil
}
https://go.dev/play/p/uwboGKf3qnD
Community,
The mission
basic
Implement a func that patches all string fields on an objects
details
[done] fields shall only be patched if they match a matcher func
[done] value shall be processed via process func
patching shall be done recursive
it shall also work for []string, []*string and recursive for structs and []struct, []*struct
// update - removed old code
Solution
structs
updated the structs to use (though this does not affect the actual program, i use this for completeness
type Tag struct {
Name string `process:"yes,TagName"`
NamePtr *string `process:"no,TagNamePtr"`
}
type User struct {
ID int
Nick string
Name string `process:"yes,UserName"`
NamePtr *string `process:"yes,UserNamePtr"`
Slice []string `process:"yes,Slice"`
SlicePtr []*string `process:"yes,SlicePtr"`
SubStruct []Tag `process:"yes,SubStruct"`
SubStructPtr []*Tag `process:"yes,SubStructPtr"`
}
helper func
Further we need two helper funcs to check if a struct has a tag and to print to console
func Stringify(i interface{}) string {
s, _ := json.MarshalIndent(i, "", " ")
return string(s)
}
func HasTag(structFiled reflect.StructField, tagName string, tagValue string) bool {
tag := structFiled.Tag
if value, ok := tag.Lookup(tagName); ok {
parts := strings.Split(value, ",")
if len(parts) > 0 {
return parts[0] == tagValue
}
}
return false
}
patcher - the actual solution
type Patcher struct {
Matcher func(structFiled *reflect.StructField, v reflect.Value) bool
Process func(in string) string
}
func (p *Patcher) value(idx int, v reflect.Value, structFiled *reflect.StructField) {
if !v.IsValid() {
return
}
switch v.Kind() {
case reflect.Ptr:
p.value(idx, v.Elem(), structFiled)
case reflect.Struct:
for i := 0; i < v.NumField(); i++ {
var sf = v.Type().Field(i)
structFiled = &sf
p.value(i, v.Field(i), structFiled)
}
case reflect.Slice:
for i := 0; i < v.Len(); i++ {
p.value(i, v.Index(i), structFiled)
}
case reflect.String:
if p.Matcher(structFiled, v) {
v.SetString(p.Process(v.String()))
}
}
}
func (p *Patcher) Apply(in interface{}) {
p.value(-1, reflect.ValueOf(in).Elem(), nil)
}
how to use
func main() {
var NamePtr string = "golang"
var SubNamePtr string = "*secure"
testUser := User{
ID: 1,
Name: "lumo",
NamePtr: &NamePtr,
SubStruct: []Tag{{
Name: "go",
},
},
SubStructPtr: []*Tag{&Tag{
Name: "*go",
NamePtr: &SubNamePtr,
},
},
}
var p = Patcher{
// filter - return true if the field in struct has a tag process=true
Matcher: func(structFiled *reflect.StructField, v reflect.Value) bool {
return HasTag(*structFiled, "process", "yes")
},
// process
Process: func(in string) string {
if in != "" {
return fmt.Sprintf("!%s!", strings.ToUpper(in))
} else {
return "!empty!"
}
return in
},
}
p.Apply(&testUser)
fmt.Println("Output:")
fmt.Println(Stringify(testUser))
}
goplay
https://goplay.tools/snippet/-0MHDfKr7ax
Having a struct heirarchy like:
type DomainStore struct {
Domains []*Domain
Users []*User
}
type Domain struct {
Name string
Records []*Record
Owner *User
}
type User struct {
Name string
Email string
Domains []*Domain
}
type Record struct {
Name string
Host string
}
With a single DomainStore having a list of Domain and Users with pointer between Domain and User.
I'm looking for a way to serialize/deserialize to/from file. I have been trying to use gob, but the pointers is not (by design) serialized correct (its flattened).
Thinking about giving each object a unique id and making a func to serialize/deserialize each type, but it seems much work/boilerplate. Any suggestions for a strategy?
I would like to keep the whole DomainStore in memory, and just serialize to file on user request.
The main problem: How to serialise/deserialize and keep the pointers pointing to the same object and not different copies of the same object
Both gob and json seems to "just" copy the value of the object and afted deserializasion I end up with multiple independent copies of objects.
Using gob ang json this is what happens:
Before, A & C both points to B:
A -> B <- C
After deserialization with json/gob:
A -> B1 , C -> B2
A & C points to to different object, with the same values. But, if i change B1 it's not changed in B2.
--- Update ---
When marshalling i can obtain the memory location of the object and use it as an ID:
func (u *User) MarshalJSON() ([]byte, error) {
return json.Marshal(&JsonUser{
ID: fmt.Sprintf("%p", u),
Name: u.Name,
Email: u.Email,
})
}
And when marshalling the Domain I can replace the
func (d *Domain) MarshalJSON() ([]byte, error) {
return json.Marshal(&struct {
ID string `json:"id"`
Name string `json:"name"`
User string `json:"user"`
}{
ID: fmt.Sprintf("%p", d),
Name: d.Name,
User: fmt.Sprintf("%p", d.User),
})
}
Now I just need to be able to unmarshal this which gives me a problem in the UnmarshalJSON need to access a map of id's and their respective objects.
func (u *User) UnmarshalJSON(data []byte) error {
// need acces to a map shared by all UnmarshalJSON functions
}
It can be done using the following method:
All the objects are placed in maps in a State object.
When the objects in a State object is marshalled, all objects refered to using pointers is replaced with the memory location of the object.
When unmarshalled pointers are restored using a global list of previously read objects.
The code will run, and is just to illustrate the method, I'm new to Go, so bear with me.
package main
import (
"encoding/json"
"errors"
"fmt"
"log"
"strings"
)
type User struct {
Name string
Email string
}
type JsonUser struct {
ID string `json:"id"`
Name string `json:"name"`
Email string `json:"email"`
}
func (u *User) Print(level int) {
ident := strings.Repeat("-", level)
log.Println(ident, "Username:", u.Name, u.Email)
}
func (u *User) Id() string {
return fmt.Sprintf("%p", u)
}
func (u *User) MarshalJSON() ([]byte, error) {
return json.Marshal(&JsonUser{
ID: u.Id(),
Name: u.Name,
Email: u.Email,
})
}
func (u *User) UnmarshalJSON(data []byte) error {
aux := &JsonUser{}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
u.Name = aux.Name
u.Email = aux.Email
load_helper[aux.ID] = u
log.Println("Added user with id ", aux.ID, u.Name)
return nil
}
type Record struct {
Type string // MX / A / CNAME / TXT / REDIR / SVR
Name string // # / www
Host string // IP / address
Priority int // Used for MX
Port int // Used for SVR
}
type JsonRecord struct {
ID string
Type string
Name string
Host string
Priority int
Port int
}
func (r *Record) Print(level int) {
ident := strings.Repeat("-", level)
log.Println(ident, "", r.Type, r.Name, r.Host)
}
func (r *Record) Id() string {
return fmt.Sprintf("%p", r)
}
func (r *Record) MarshalJSON() ([]byte, error) {
return json.Marshal(&JsonRecord{
ID: r.Id(),
Name: r.Name,
Type: r.Type,
Host: r.Host,
Priority: r.Priority,
Port: r.Port,
})
}
func (r *Record) UnmarshalJSON(data []byte) error {
aux := &JsonRecord{}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
r.Name = aux.Name
r.Type = aux.Type
r.Host = aux.Host
r.Priority = aux.Priority
r.Port = aux.Port
load_helper[aux.ID] = r
log.Println("Added record with id ", aux.ID, r.Name)
return nil
}
type Domain struct {
Name string
User *User // User ID
Records []*Record // Record ID's
}
type JsonDomain struct {
ID string `json:"id"`
Name string `json:"name"`
User string `json:"user"`
Records []string `json:"records"`
}
func (d *Domain) Print(level int) {
ident := strings.Repeat("-", level)
log.Println(ident, "Domain:", d.Name)
d.User.Print(level + 1)
log.Println(ident, " Records:")
for _, r := range d.Records {
r.Print(level + 2)
}
}
func (d *Domain) Id() string {
return fmt.Sprintf("%p", d)
}
func (d *Domain) MarshalJSON() ([]byte, error) {
var record_ids []string
for _, r := range d.Records {
record_ids = append(record_ids, r.Id())
}
return json.Marshal(JsonDomain{
ID: d.Id(),
Name: d.Name,
User: d.User.Id(),
Records: record_ids,
})
}
func (d *Domain) UnmarshalJSON(data []byte) error {
log.Println("UnmarshalJSON domain")
aux := &JsonDomain{}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
d.Name = aux.Name
d.User = load_helper[aux.User].(*User) // restore pointer to domains user
for _, record_id := range aux.Records {
d.Records = append(d.Records, load_helper[record_id].(*Record))
}
return nil
}
type State struct {
Users map[string]*User
Records map[string]*Record
Domains map[string]*Domain
}
func NewState() *State {
s := &State{}
s.Users = make(map[string]*User)
s.Domains = make(map[string]*Domain)
s.Records = make(map[string]*Record)
return s
}
func (s *State) Print() {
log.Println("State:")
log.Println("Users:")
for _, u := range s.Users {
u.Print(1)
}
log.Println("Domains:")
for _, d := range s.Domains {
d.Print(1)
}
}
func (s *State) NewUser(name string, email string) *User {
u := &User{Name: name, Email: email}
id := fmt.Sprintf("%p", u)
s.Users[id] = u
return u
}
func (s *State) NewDomain(user *User, name string) *Domain {
d := &Domain{Name: name, User: user}
s.Domains[d.Id()] = d
return d
}
func (s *State) NewMxRecord(d *Domain, rtype string, name string, host string, priority int) *Record {
r := &Record{Type: rtype, Name: name, Host: host, Priority: priority}
d.Records = append(d.Records, r)
s.Records[r.Id()] = r
return r
}
func (s *State) FindDomain(name string) (*Domain, error) {
for _, v := range s.Domains {
if v.Name == name {
return v, nil
}
}
return nil, errors.New("Not found")
}
func Save(s *State) (string, error) {
b, err := json.MarshalIndent(s, "", " ")
if err == nil {
return string(b), nil
} else {
log.Println(err)
return "", err
}
}
var load_helper map[string]interface{}
func Load(s *State, blob string) {
load_helper = make(map[string]interface{})
if err := json.Unmarshal([]byte(blob), s); err != nil {
log.Println(err)
} else {
log.Println("OK")
}
}
func test_state() {
s := NewState()
u := s.NewUser("Ownername", "some#email.com")
d := s.NewDomain(u, "somedomain.com")
s.NewMxRecord(d, "MX", "#", "192.168.1.1", 10)
s.NewMxRecord(d, "A", "www", "192.168.1.1", 0)
s.Print()
x, _ := Save(s) // Saved to json string
log.Println("State saved, the json string is:")
log.Println(x)
s2 := NewState() // Create a new empty State
Load(s2, x)
s2.Print()
d, err := s2.FindDomain("somedomain.com")
if err == nil {
d.User.Name = "Changed"
} else {
log.Println("Error:", err)
}
s2.Print()
}
func main() {
test_state()
}
This is quite a lot of code and there are to much coupling between the objects and the serialization. Also the global var load_helper is bad. Ideas to improve will be appreciated.
Another approch would be to use reflection to make a more generic solution. Here is an example using this method:
package main
import (
"encoding/json"
"fmt"
"log"
"strings"
"reflect"
)
func pprint(x interface{}) {
b, err := json.MarshalIndent(x, "", " ")
if err != nil {
fmt.Println("error:", err)
}
fmt.Println(string(b))
}
var typeRegistry = make(map[string]reflect.Type)
// Register a type to make it possible for the Save/Load functions
// to serialize it.
func Register(v interface{}) {
t := reflect.TypeOf(v)
n := t.Name()
fmt.Println("Register type",n)
typeRegistry[n] = reflect.TypeOf(v)
}
// Make an instance of a type from the string name of the type.
func makeInstance(name string) reflect.Value {
v := reflect.New(typeRegistry[name]).Elem()
return v
}
// Translate a string type name tpo a real type.
func getTypeFromString(name string) reflect.Type {
return typeRegistry[name]
}
// Serializeable interface must be supported by all objects passed to the Load / Save functions.
type Serializeable interface {
Id() string
}
// GenericSave saves the object d
func GenericSave(d interface{}) (string, error) {
r := make(map[string]interface{})
v := reflect.ValueOf(d)
t := reflect.TypeOf(d)
if t.Kind()==reflect.Ptr {
t=t.Elem()
v=v.Elem()
}
r["_TYPE"]=t.Name()
r["_ID"]=fmt.Sprintf("%p", d)
for i := 0; i < t.NumField(); i++ {
f := t.Field(i)
name := f.Name
vf := v.FieldByName(name)
// fmt.Println("Field", i+1, "name is", name, "type is", f.Type.Name(), "and kind is", f.Type.Kind())
// fmt.Println("V:", vf)
if f.Tag != "" {
store:=strings.Split(f.Tag.Get("store"),",")
switch store[1] {
case "v":
switch t.Field(i).Type.Name() {
case "string":
r[store[0]]=vf.String()
case "int":
r[store[0]]=vf.Int()
}
case "p":
vals:=vf.MethodByName("Id").Call([]reflect.Value{})
r[store[0]]=vals[0].String()
case "lp":
tr:=[]string{}
for j := 0; j < vf.Len(); j++ {
vals:=vf.Index(j).MethodByName("Id").Call([]reflect.Value{})
tr=append(tr,vals[0].String())
}
r[store[0]]=tr
}
}
}
m,_:=json.Marshal(r)
return string(m),nil
}
// Save saves the list of objects.
func Save(objects []Serializeable) []byte {
lst:=[]string{}
for _,o := range(objects) {
os,_:= GenericSave(o) // o.Save()
lst=append(lst,os)
}
m,_:=json.Marshal(lst)
return m
}
func toStructPtr(obj interface{}) interface{} {
vp := reflect.New(reflect.TypeOf(obj))
vp.Elem().Set(reflect.ValueOf(obj))
return vp.Interface()
}
// Load creates a list of serializeable objects from json blob
func Load(blob []byte) []Serializeable {
objects := []Serializeable{}
loadHelper := make(map[string]interface{})
var olist []interface{}
if err := json.Unmarshal(blob, &olist); err != nil {
log.Println(err)
} else {
for _,o := range(olist) {
var omap map[string]interface{}
json.Unmarshal([]byte(o.(string)), &omap)
t:= getTypeFromString(omap["_TYPE"].(string))
obj := reflect.New(t).Elem()
for i := 0; i < t.NumField(); i++ {
// n:=t.Field(i).Name
// fmt.Println(i,n,t.Field(i).Type.Name())
if t.Field(i).Tag != "" {
store:=strings.Split(t.Field(i).Tag.Get("store"),",")
// fmt.Println(store)
switch store[1] {
case "v":
switch t.Field(i).Type.Name() {
case "string":
obj.FieldByIndex([]int{i}).SetString(omap[store[0]].(string))
case "int":
obj.FieldByIndex([]int{i}).SetInt(int64(omap[store[0]].(float64)))
}
case "p":
nObj:=loadHelper[omap[store[0]].(string)]
obj.FieldByIndex([]int{i}).Set(reflect.ValueOf(nObj.(*User)))
case "lp":
ptrItemType:=t.Field(i).Type.Elem()
slice := reflect.Zero(reflect.SliceOf( ptrItemType /* reflect.TypeOf( &Record{} ) */ ))//.Interface()
for _, pID := range(omap[store[0]].([]interface{})) {
nObj:=loadHelper[pID.(string)]
slice=reflect.Append(slice, reflect.ValueOf(nObj) )
}
obj.FieldByIndex([]int{i}).Set(slice)
}
}
}
oi:=toStructPtr(obj.Interface())
oip:=oi.(Serializeable)
objects=append(objects,oip)
loadHelper[omap["_ID"].(string)]=oip
}
}
return objects
}
/* Application data structures */
type User struct {
Name string `store:"name,v"`
Email string `store:"email,v"`
}
func (u *User) Id() string {
return fmt.Sprintf("%p", u)
}
func (u *User) Save() (string, error) {
return GenericSave(u)
}
func (u *User) Print() {
fmt.Println("User:",u.Name)
}
type Record struct {
Type string `store:"type,v"`// MX / A / CNAME / TXT / REDIR / SVR
Name string `store:"name,v"`// # / www
Host string `store:"host,v"`// IP / address
Priority int `store:"priority,v"`// Used for MX
Port int `store:"port,v"`// Used for SVR
}
func (r *Record) Id() string {
return fmt.Sprintf("%p", r)
}
func (r *Record) Save() (string, error) {
return GenericSave(r)
}
func (r *Record) Print() {
fmt.Println("Record:",r.Type,r.Name,r.Host)
}
type Domain struct {
Name string `store:"name,v"`
User *User `store:"user,p"` // User ID
Records []*Record `store:"record,lp"` // Record ID's
}
func (d *Domain) Id() string {
return fmt.Sprintf("%p", d)
}
func (d *Domain) Save() (string, error) {
return GenericSave(d)
}
func (d *Domain) Print() {
fmt.Println("Domain:",d.Name)
d.User.Print()
fmt.Println("Records:")
for _, r := range d.Records {
r.Print()
}
}
type DBM struct {
Domains []*Domain
Users []*User
Records []*Record
}
func (dbm *DBM) AddDomain(d *Domain) {
dbm.Domains=append(dbm.Domains,d)
}
func (dbm *DBM) AddUser(u *User) {
dbm.Users=append(dbm.Users,u)
}
func (dbm *DBM) AddRecord(r *Record) {
dbm.Records=append(dbm.Records,r)
}
func (dbm *DBM) GetObjects() []Serializeable {
objects:=[]Serializeable{}
for _,r := range(dbm.Records) {
objects=append(objects, r)
}
for _,u := range(dbm.Users) {
objects=append(objects, u)
}
for _,d := range(dbm.Domains) {
objects=append(objects, d)
}
return objects
}
func (dbm *DBM) SetObjects(objects []Serializeable) {
for _,o := range(objects) {
switch o.(type) {
case *Record:
fmt.Println("record")
dbm.AddRecord(o.(*Record))
case *User:
fmt.Println("record")
dbm.AddUser(o.(*User))
case *Domain:
fmt.Println("record")
dbm.AddDomain(o.(*Domain))
}
}
}
func testState() {
Register(User{})
Register(Domain{})
Register(Record{})
dbm:=DBM{}
u := &User{Name: "Martin", Email: "some#email.com"}
dbm.AddUser(u)
r1 := &Record{Name: "#", Type: "MX", Host: "mail.ishost.dk"}
r2 := &Record{Name: "#", Type: "MX", Host: "mail.infoserv.dk"}
dbm.AddRecord(r1)
dbm.AddRecord(r2)
d := &Domain{User:u, Name: "Martin", Records: []*Record{r1, r2}}
dbm.AddDomain(d)
x:=Save(dbm.GetObjects())
fmt.Println("== Saved objects")
// fmt.Println(string(x))
fmt.Println("== Loading")
dbm2:=DBM{}
dbm2.SetObjects(Load(x))
u2:=dbm2.Users[0]
u2.Print()
u2.Name="KURT"
u2.Print()
d2:=dbm2.Domains[0]
d2.Print()
d2.User.Name="ZIG"
u2.Print()
}
func main() {
testState()
}
Use encoding/json package
to marshal:
// Marshal is a function that marshals the object into an
// io.Reader.
// By default, it uses the JSON marshaller.
var Marshal = func(v interface{}) (io.Reader, error) {
b, err := json.MarshalIndent(v, "", "\t")
if err != nil {
return nil, err
}
return bytes.NewReader(b), nil
}
to unmarshal:
// Unmarshal is a function that unmarshals the data from the
// reader into the specified value.
// By default, it uses the JSON unmarshaller.
var Unmarshal = func(r io.Reader, v interface{}) error {
return json.NewDecoder(r).Decode(v)
}
Not sure there's more to this,
Another thing you can do is, store all these as json formatted strings.
I'm trying to set-up an AWS-lambda using aws-sdk-go that is triggered whenever a new user is added to a certain dynamodb table.
Everything is working just fine but I can't find a way to unmarshal a map map[string]DynamoDBAttributeValue like:
{
"name": {
"S" : "John"
},
"residence_address": {
"M": {
"address": {
"S": "some place"
}
}
}
}
To a given struct, for instance, a User struct. Here is shown an example of unsmarhaling a map[string]*dynamodb.AttributeValue into a given interface, but I can't find a way to do the same thing with map[string]DynamoDBAttributeValue even though these types seem to fit the same purposes.
map[string]DynamoDBAttributeValue is returned by a events.DynamoDBEvents from package github.com/aws/aws-lambda-go/events. This is my code:
package handler
import (
"context"
"github.com/aws/aws-lambda-go/events"
"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute"
"github.com/aws/aws-sdk-go/service/dynamodb"
)
func HandleDynamoDBRequest(ctx context.Context, e events.DynamoDBEvent) {
for _, record := range e.Records {
if record.EventName == "INSERT" {
// User Struct
var dynamoUser model.DynamoDBUser
// Of course this can't be done for incompatible types
_ := dynamodbattribute.UnmarshalMap(record.Change.NewImage, &dynamoUser)
}
}
}
Of course, I can marshal record.Change.NewImage to JSON and unmarshal it back to a given struct, but then, I would have to manually initialize dynamoUser attributes starting from the latter ones.
Or I could even write a function that parses map[string]DynamoDBAttributeValue to map[string]*dynamodb.AttributeValue like:
func getAttributeValueMapFromDynamoDBStreamRecord(e events.DynamoDBStreamRecord) map[string]*dynamodb.AttributeValue {
image := e.NewImage
m := make(map[string]*dynamodb.AttributeValue)
for k, v := range image {
if v.DataType() == events.DataTypeString {
s := v.String()
m[k] = &dynamodb.AttributeValue{
S : &s,
}
}
if v.DataType() == events.DataTypeBoolean {
b := v.Boolean()
m[k] = &dynamodb.AttributeValue{
BOOL : &b,
}
}
// . . .
if v.DataType() == events.DataTypeMap {
// ?
}
}
return m
}
And then simply use dynamodbattribute.UnmarshalMap, but on events.DataTypeMap it would be quite a tricky process.
Is there a way through which I can unmarshal a DynamoDB record coming from a events.DynamoDBEvent into a struct with a similar method shown for map[string]*dynamodb.AttributeValue?
I tried the function you provided, and I met some problems with events.DataTypeList, so I managed to write the following function that does the trick:
// UnmarshalStreamImage converts events.DynamoDBAttributeValue to struct
func UnmarshalStreamImage(attribute map[string]events.DynamoDBAttributeValue, out interface{}) error {
dbAttrMap := make(map[string]*dynamodb.AttributeValue)
for k, v := range attribute {
var dbAttr dynamodb.AttributeValue
bytes, marshalErr := v.MarshalJSON(); if marshalErr != nil {
return marshalErr
}
json.Unmarshal(bytes, &dbAttr)
dbAttrMap[k] = &dbAttr
}
return dynamodbattribute.UnmarshalMap(dbAttrMap, out)
}
I was frustrated that the type of NewImage from the record wasn't map[string]*dynamodb.AttributeValue so I could use the dynamodbattribute package.
The JSON representation of events.DynamoDBAttributeValue seems to be the same as the JSON represenation of dynamodb.AttributeValue.
So I tried creating my own DynamoDBEvent type and changed the type of OldImage and NewImage, so it would be marshalled into map[string]*dynamodb.AttributeValue instead of map[string]events.DynamoDBAttributeValue
It is a little bit ugly but it works for me.
package main
import (
"github.com/aws/aws-lambda-go/events"
"github.com/aws/aws-lambda-go/lambda"
"github.com/aws/aws-sdk-go/service/dynamodb"
"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute"
"fmt"
)
func main() {
lambda.Start(lambdaHandler)
}
// changed type of event from: events.DynamoDBEvent to DynamoDBEvent (see below)
func lambdaHandler(event DynamoDBEvent) error {
for _, record := range event.Records {
change := record.Change
newImage := change.NewImage // now of type: map[string]*dynamodb.AttributeValue
var item IdOnly
err := dynamodbattribute.UnmarshalMap(newImage, &item)
if err != nil {
return err
}
fmt.Println(item.Id)
}
return nil
}
type IdOnly struct {
Id string `json:"id"`
}
type DynamoDBEvent struct {
Records []DynamoDBEventRecord `json:"Records"`
}
type DynamoDBEventRecord struct {
AWSRegion string `json:"awsRegion"`
Change DynamoDBStreamRecord `json:"dynamodb"`
EventID string `json:"eventID"`
EventName string `json:"eventName"`
EventSource string `json:"eventSource"`
EventVersion string `json:"eventVersion"`
EventSourceArn string `json:"eventSourceARN"`
UserIdentity *events.DynamoDBUserIdentity `json:"userIdentity,omitempty"`
}
type DynamoDBStreamRecord struct {
ApproximateCreationDateTime events.SecondsEpochTime `json:"ApproximateCreationDateTime,omitempty"`
// changed to map[string]*dynamodb.AttributeValue
Keys map[string]*dynamodb.AttributeValue `json:"Keys,omitempty"`
// changed to map[string]*dynamodb.AttributeValue
NewImage map[string]*dynamodb.AttributeValue `json:"NewImage,omitempty"`
// changed to map[string]*dynamodb.AttributeValue
OldImage map[string]*dynamodb.AttributeValue `json:"OldImage,omitempty"`
SequenceNumber string `json:"SequenceNumber"`
SizeBytes int64 `json:"SizeBytes"`
StreamViewType string `json:"StreamViewType"`
}
I have found the same problem and the solution is to perform a simple conversion of types. This is possible because in the end the type received by lambda events events.DynamoDBAttributeValue and the type used by the SDK V2 of AWS DynamoDB types.AttributeValue are the same. Next I show you the conversion code.
package aws_lambda
import (
"github.com/aws/aws-lambda-go/events"
"github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue"
"github.com/aws/aws-sdk-go-v2/service/dynamodb/types"
)
func UnmarshalDynamoEventsMap(
record map[string]events.DynamoDBAttributeValue, out interface{}) error {
asTypesMap := DynamoDbEventsMapToTypesMap(record)
err := attributevalue.UnmarshalMap(asTypesMap, out)
if err != nil {
return err
}
return nil
}
func DynamoDbEventsMapToTypesMap(
record map[string]events.DynamoDBAttributeValue) map[string]types.AttributeValue {
resultMap := make(map[string]types.AttributeValue)
for key, rec := range record {
resultMap[key] = DynamoDbEventsToTypes(rec)
}
return resultMap
}
// DynamoDbEventsToTypes relates the dynamo event received by AWS Lambda with the data type that is
// used in the Amazon SDK V2 to deal with DynamoDB data.
// This function is necessary because Amazon does not provide any kind of solution to make this
// relationship between types of data.
func DynamoDbEventsToTypes(record events.DynamoDBAttributeValue) types.AttributeValue {
var val types.AttributeValue
switch record.DataType() {
case events.DataTypeBinary:
val = &types.AttributeValueMemberB{
Value: record.Binary(),
}
case events.DataTypeBinarySet:
val = &types.AttributeValueMemberBS{
Value: record.BinarySet(),
}
case events.DataTypeBoolean:
val = &types.AttributeValueMemberBOOL{
Value: record.Boolean(),
}
case events.DataTypeList:
var items []types.AttributeValue
for _, value := range record.List() {
items = append(items, DynamoDbEventsToTypes(value))
}
val = &types.AttributeValueMemberL{
Value: items,
}
case events.DataTypeMap:
items := make(map[string]types.AttributeValue)
for k, v := range record.Map() {
items[k] = DynamoDbEventsToTypes(v)
}
val = &types.AttributeValueMemberM{
Value: items,
}
case events.DataTypeNull:
val = nil
case events.DataTypeNumber:
val = &types.AttributeValueMemberN{
Value: record.Number(),
}
case events.DataTypeNumberSet:
val = &types.AttributeValueMemberNS{
Value: record.NumberSet(),
}
case events.DataTypeString:
val = &types.AttributeValueMemberS{
Value: record.String(),
}
case events.DataTypeStringSet:
val = &types.AttributeValueMemberSS{
Value: record.StringSet(),
}
}
return val
}
There is a package that allows conversion from events.DynamoDBAttributeValue to dynamodb.AttributeValue
https://pkg.go.dev/github.com/aereal/go-dynamodb-attribute-conversions/v2
From there one can unmarshal AttributeValue into struct
func Unmarshal(attribute map[string]events.DynamoDBAttributeValue, out interface{}) error {
av := ddbconversions.AttributeValueMapFrom(attribute)
return attributevalue.UnmarshalMap(av, out)
}
If one is working with a database, a Null* type is useful for most scenarios as one typically does not want a "zero" value going through, you want the NOT NULL constraints etc. to kick up and remind you that you haven't passed in all the data necessary.
So you create a structure like the following:
type Role struct {
Id sql.NullInt64
Code sql.NullString
}
Thats great, but now you cannot get direct access to the properties and have to use Role.Id.Value to both get and set, this is going to get pretty old in a large app when you have to have the extra step every single time you want access to the properties.
It would be nice if you could assign directly eg. Role.Code = "Fsfs", and be able to do something like Role.Code.IsNull when you are interested in null checking. Is such a thing possible?
Is using intermediate pointer value(s) an option?
package main
import "fmt"
type tmp struct {
Value int
}
func getInt() *int {
i := 123
return &i
}
func main() {
// Re
var v *int
v = nil
fmt.Printf("%T / %v\n", v, v)
if v == nil {
println("nil...")
}
v = getInt()
fmt.Printf("%T / %v\n", v, *v)
if v != nil {
println("not nil...")
}
s := tmp{*v}
fmt.Printf("%T / %v\n", s, s)
}
http://play.golang.org/p/lBrwTKh6-v
You can access Role.Code like that:
var r *Role
r.Code = *code
You can check for null like this:
fmt.Println(r.Code, r.Code.Valid)
If you change the value of r.Code manually without using an sql.Scanner a Setter could be helpful:
func (r *Role) SetCode(code string) {
r.Code.String = code
r.Code.Valid = true
}
func main() {
var r *Role
r.SetCode("mi")
if r.Code.Valid {
fmt.Println(r.Code)
}
I tried this out here: https://play.golang.org/p/faxQUm-2lr
Keep app and database code separate.
// Role belongs to app code, no compromises.
type Role struct {
Id int64
Code string
}
Model the database.
// Database has tables with columns.
type Table struct {
Name string
Columns []string
}
var RoleTable = Table{
Name: "roles",
Columns: []string{
"id",
"code",
},
}
Write code once to convert between model and database row.
// Database package needs to make it work.
// Write a model to database row.
type Writer struct {
Role
}
func (w *Writer) Write() []interface{} {
return []interface{}{
w.Role.Id,
sql.NullString{
Valid: len(w.Role.Code) > 0,
String: w.Role.String,
},
}
}
// Read a database row into model.
type Reader struct {
Id int64
Code sql.NullString
}
func (r *Reader) Scan(row *sql.Row) error {
return row.Scan(
&r.Id,
&r.Code,
)
}
func (r *Reader) Read() Role {
return Role{
Id: r.Id,
Code: r.Code.String,
}
}
Your schema is decoupled from app model. You can flatten structures like user contact details when saving or loading.
// Nested struct in app code.
type User struct {
TwitterProfile struct {
Id string
ScreenName string
}
}
// Database row is normalized flat.
var UserTable = Table{
Name: "users",
Columns: []string{
"twitter_id",
"twitter_screen_name",
},
}
It's flexible. You can even scan join rows without intermediate structs.
type RowMux struct {
vs []interface{}
}
func (mux *RowMux) Scan(vs ...interface{}) error {
mux.vs = append(mux.vs, vs...)
return nil
}
func (mux *RowMux) Mux(row *sql.Row) error {
return row.Scan(mux.vs...)
}
// Scan join rows!
row := db.QueryRow(`
SELECT users.*, roles.*
FROM users
JOIN roles ON users.id = roles.user_id
WHERE users.twitter_id = "123"
`)
mux := &RowMux{}
userReader := &UserReader{}
userReader.Scan(mux)
roleReader := &RoleReader{}
roleReader.Scan(mux)
if err := mux.Mux(row); err != nil {
panic(err)
}
user := userReader.Read()
role := roleReader.Read()