Unmarshal dynamic interface using UnmarshalBSON based on known key - go

I have an object of type foo containing an interface ActivationInterface ; this object is saved in MongoDB and I have trouble fetching it back as the underlying type of the inner object is not known.
I implemented UnmarshalBSON as follow without success, as it seems even after setting the concrete type of the interface, the unmarshaller still does now the underlying type, as I still get the error:
error decoding key act: no decoder found for main.ActivationInterface
Do you have any idea how I can achieve this ?
I found somethign close woking here, so I don't get why mine isn't: Unmarshal dynamic JSON based on a type key
I cannot see what I'm doing wrong and different...!
EDIT: I updated the code to compare with json. UnmarshalJSON is working great with exactly the same code while UnmarshalBSON still fails.
package main
import (
"fmt"
"log"
"go.mongodb.org/mongo-driver/bson"
)
type foo struct {
Type string `bson:"type"`
Act ActivationInterface
}
type ActivationInterface interface{}
type Activation1 struct {
Name string `bson:"name"`
}
type Activation2 struct {
Address string `bson:"adress"`
}
func (q *foo) UnmarshalBSON(data []byte) error {
// Unmarshall only the type
fooTemp := new(struct {
Type string `bson:"type"`
})
if err := bson.Unmarshal(data, fooTemp); err != nil {
return err
}
fmt.Println(fooTemp.Type)
// Set the type to the prop
switch fooTemp.Type {
case "act1":
// q.Act = &Activation1{}
q.Act = new(Activation1)
case "act2":
// q.Act = &Activation2{}
q.Act = new(Activation2)
default:
fmt.Println("DEFAULT")
}
// Call Unmarshal again
type Alias foo // avoids infinite recursion using a type alias
return bson.Unmarshal(data, (*Alias)(q))
}
func main() {
foo1 := foo{
Type: "act1",
Act: Activation1{
Name: "name: act1",
},
}
foo2 := foo{
Type: "act2",
Act: Activation2{
Address: "adress: act2",
},
}
// Marshal
m1, err := bson.Marshal(foo1)
if err != nil {
log.Fatal(err)
}
m2, err := bson.Marshal(foo2)
if err != nil {
log.Fatal(err)
}
//fmt.Println(m1, m2)
// Unmarshal
var u1, u2 foo
err = bson.Unmarshal(m1, &u1)
if err != nil {
fmt.Println("1 -> ", err) // error decoding key act: no decoder found for main.ActivationInterface
}
err = bson.Unmarshal(m2, &u2)
if err != nil {
fmt.Println("2 -> ", err) // error decoding key act: no decoder found for main.ActivationInterface
}
fmt.Println(foo1.Type, ":", u1.Act.(*Activation1).Name)
fmt.Println(foo2.Type, ":", u2.Act.(*Activation2).Address)
}
Go playground: https://go.dev/play/p/bHMy6-ZLsYQ
Almost the same code, but using JSON and working: https://go.dev/play/p/V5HLrQ_-ls3
Thanks !

When using an interface in your struct, unmarshall cannot figure out which "implementation" to choose... You have to do it manually, in your case based on the "type" field. Usually unmarshall methods put a map for interface{}, a key value store.
Anyway back on your problem, you have to store the interface data in bson.Raw (slice of bytes) and do the unmarshalling manually by choosing the right struct.
package main
import (
"fmt"
"log"
"go.mongodb.org/mongo-driver/bson"
)
type foo struct {
Type string `bson:"type"`
Act ActivationInterface
}
type ActivationInterface interface{}
type Activation1 struct {
Name string `bson:"name"`
}
type Activation2 struct {
Address string `bson:"adress"`
}
func (q *foo) UnmarshalBSON(data []byte) error {
// Unmarshall only the type
fooTemp := new(struct {
Type string `bson:"type"`
Act bson.Raw
})
if err := bson.Unmarshal(data, fooTemp); err != nil {
return err
}
fmt.Println(fooTemp.Type)
// Set the type to the prop
switch fooTemp.Type {
case "act1":
// q.Act = &Activation1{}
a := Activation1{}
err := bson.Unmarshal(fooTemp.Act, &a)
if err != nil {
return err
}
q.Act = a
case "act2":
// q.Act = &Activation2{}
a := Activation2{}
err := bson.Unmarshal(fooTemp.Act, &a)
if err != nil {
return err
}
q.Act = a
default:
fmt.Println("DEFAULT")
return fmt.Errorf("unknown type: %v", fooTemp.Type)
}
return nil
}
func main() {
foo1 := foo{
Type: "act1",
Act: Activation1{
Name: "name: act1",
},
}
foo2 := foo{
Type: "act2",
Act: Activation2{
Address: "adress: act2",
},
}
// Marshal
m1, err := bson.Marshal(foo1)
if err != nil {
log.Fatal(err)
}
m2, err := bson.Marshal(foo2)
if err != nil {
log.Fatal(err)
}
//fmt.Println(m1, m2)
// Unmarshal
var u1, u2 foo
err = bson.Unmarshal(m1, &u1)
if err != nil {
fmt.Println("1 -> ", err) // error decoding key act: no decoder found for main.ActivationInterface
}
err = bson.Unmarshal(m2, &u2)
if err != nil {
fmt.Println("2 -> ", err) // error decoding key act: no decoder found for main.ActivationInterface
}
fmt.Println(foo1.Type, ":", u1.Act.(Activation1).Name)
fmt.Println(foo2.Type, ":", u2.Act.(Activation2).Address)
}
https://go.dev/play/p/CG2SlEknNrO

Related

Golang Unmarshal an JSON response, then marshal with Struct field names

So I am hitting an API that returns a JSON response and I am unmarshalling it into a struct like so:
package main
type ProcessedRecords struct {
SLMIndividualID string `json:"individual_id"`
HouseholdPosition int `json:"Household Position"`
IndividualFirstName string `json:"individual_first_name"`
}
func main() {
req, _ := http.NewRequest(method, url, payload)
res, err := client.Do(req)
if err != nil {
fmt.Println(err)
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
fmt.Println(err)
}
fmt.Println(body)
var responseObject Response
json.Unmarshal(body, &responseObject)
fmt.Println(responseObject)
which works great. However I need to marshal this struct again but I want to use the Struct Fields as keys instead of the json: ... fields. I am using the following code:
recordsInput := []*firehose.Record{}
for i := 0; i < len(records); i++ {
if len(recordsInput) == 500 {
* code to submit records, this part works fine *
}
b, err := json.Marshal(records[i])
if err != nil {
log.Printf("Error: %v", err)
}
record := &firehose.Record{Data: b}
recordsInput = append(recordsInput, record)
}
This does submit records successfully but it's in the format:
{"individual_id":"33c05b49-149b-480f-b1c2-3a3b30e0cb6f","Household Position":1...}
and I'd like it in the format:
{"SLMIndividualId":"33c05b49-149b-480f-b1c2-3a3b30e0cb6f","HouseholdPosition":1...}
How can I achieve this?
Those tags say how the struct should be marshalled, so if they are present, that is how the output will be. You'll need to convert it to a matching struct that does not have the json: tags:
type ProcessedRecords struct {
SLMIndividualID string `json:"individual_id"`
HouseholdPosition int `json:"Household Position"`
IndividualFirstName string `json:"individual_first_name"`
}
type ProcessedRecordsOut struct {
SLMIndividualID string
HouseholdPosition int
IndividualFirstName string
}
func process() {
var in ProcessedRecords
json.Unmarshal(data, &in)
// Convert to same type w/o tags
out := ProcessedRecordsOut(in)
payload, _ := json.Marshal(out)
// ...
}
See a working example here: https://play.golang.org/p/p0Fc8DJotYE
You can omit fields one-way by defining a custom type and implementing the correct interface, e.g.
package main
import (
"encoding/json"
"fmt"
)
type Animal struct {
Name ReadOnlyString
Order string
}
type ReadOnlyString string
func (ReadOnlyString) UnmarshalJSON([]byte) error { return nil }
func main() {
x := Animal{"Bob", "First"}
js, err := json.Marshal(&x)
if err != nil {
fmt.Println("error:", err)
}
fmt.Printf("%s\n", js)
var jsonBlob = []byte(`{"Name": "Platypus", "Order": "Monotremata"}`)
if err := json.Unmarshal(jsonBlob, &x); err != nil {
fmt.Println("error:", err)
}
fmt.Printf("%#v\n\n", x)
}
https://go.dev/play/p/-mwBL0kIqM
Found this answer here: https://github.com/golang/go/issues/19423#issuecomment-284607677

How to extend go-yaml to support custom tags

I have spent some time reading the code and docs of go-yaml, but I have not found any way to do this, except forking the project..
I want to extend the YAML unmarshaller so that it can accept a custom YAML tag (!include <file> in this case), which in turn would allow me to add support for including files. This is easily implemented with other YAML libraries, like in this answer.
Is there any way to accomplish this, using the public interface of the library (or another yaml library)?
Yes, this is possible (since v3). You can load the whole YAML file into a yaml.Node and then walk over the structure. The trick is that yaml.Node is an intermediate representation which you can only access if you define an unmarshaler.
For example:
package main
import (
"errors"
"fmt"
"io/ioutil"
"gopkg.in/yaml.v3"
)
// used for loading included files
type Fragment struct {
content *yaml.Node
}
func (f *Fragment) UnmarshalYAML(value *yaml.Node) error {
var err error
// process includes in fragments
f.content, err = resolveIncludes(value)
return err
}
type IncludeProcessor struct {
target interface{}
}
func (i *IncludeProcessor) UnmarshalYAML(value *yaml.Node) error {
resolved, err := resolveIncludes(value)
if err != nil {
return err
}
return resolved.Decode(i.target)
}
func resolveIncludes(node *yaml.Node) (*yaml.Node, error) {
if node.Tag == "!include" {
if node.Kind != yaml.ScalarNode {
return nil, errors.New("!include on a non-scalar node")
}
file, err := ioutil.ReadFile(node.Value)
if err != nil {
return nil, err
}
var f Fragment
err = yaml.Unmarshal(file, &f)
return f.content, err
}
if node.Kind == yaml.SequenceNode || node.Kind == yaml.MappingNode {
var err error
for i := range node.Content {
node.Content[i], err = resolveIncludes(node.Content[i])
if err != nil {
return nil, err
}
}
}
return node, nil
}
type MyStructure struct {
// this structure holds the values you want to load after processing
// includes, e.g.
Num int
}
func main() {
var s MyStructure
yaml.Unmarshal([]byte("!include foo.yaml"), &IncludeProcessor{&s})
fmt.Printf("Num: %v", s.Num)
}
Code prints Num: 42 when a file foo.yaml exists with the content num: 42.
Modified #flyx's original code a little to make it modular for adding custom resolvers.
package main
import (
"errors"
"fmt"
"io/ioutil"
"os"
"gopkg.in/yaml.v3"
)
var tagResolvers = make(map[string]func(*yaml.Node) (*yaml.Node, error))
type Fragment struct {
content *yaml.Node
}
func (f *Fragment) UnmarshalYAML(value *yaml.Node) error {
var err error
// process includes in fragments
f.content, err = resolveTags(value)
return err
}
type CustomTagProcessor struct {
target interface{}
}
func (i *CustomTagProcessor) UnmarshalYAML(value *yaml.Node) error {
resolved, err := resolveTags(value)
if err != nil {
return err
}
return resolved.Decode(i.target)
}
func resolveTags(node *yaml.Node) (*yaml.Node, error) {
for tag, fn := range tagResolvers {
if node.Tag == tag {
return fn(node)
}
}
if node.Kind == yaml.SequenceNode || node.Kind == yaml.MappingNode {
var err error
for i := range node.Content {
node.Content[i], err = resolveTags(node.Content[i])
if err != nil {
return nil, err
}
}
}
return node, nil
}
func resolveIncludes(node *yaml.Node) (*yaml.Node, error) {
if node.Kind != yaml.ScalarNode {
return nil, errors.New("!include on a non-scalar node")
}
file, err := ioutil.ReadFile(node.Value)
if err != nil {
return nil, err
}
var f Fragment
err = yaml.Unmarshal(file, &f)
return f.content, err
}
func resolveGetValueFromEnv(node *yaml.Node) (*yaml.Node, error) {
if node.Kind != yaml.ScalarNode {
return nil, errors.New("!getValueFromEnv on a non-scalar node")
}
value := os.Getenv(node.Value)
if value == "" {
return nil, fmt.Errorf("environment variable %v not set", node.Value)
}
var f Fragment
err := yaml.Unmarshal([]byte(value), &f)
return f.content, err
}
func AddResolvers(tag string, fn func(*yaml.Node) (*yaml.Node, error)) {
tagResolvers[tag] = fn
}
func main() {
// Register custom tag resolvers
AddResolvers("!include", resolveIncludes)
AddResolvers("!getValueFromEnv", resolveGetValueFromEnv)
type MyStructure struct {
// this structure holds the values you want to load after processing
// includes, e.g.
Num int
}
var s MyStructure
os.Setenv("FOO", `{"num": 42}`)
err := yaml.Unmarshal([]byte("!getValueFromEnv FOO"), &CustomTagProcessor{&s})
if err != nil {
panic("Error encountered during unmarshalling")
}
fmt.Printf("\nNum: %v", s.Num)
err = yaml.Unmarshal([]byte("!include foo.yaml"), &CustomTagProcessor{&s})
if err != nil {
panic("Error encountered during unmarshalling")
}
fmt.Printf("\nNum: %v", s.Num)
}

Gob decode cannot decode interface after register type

I have these types defined:
func init() {
gob.RegisterName("MyMessageHeader", MyMessageHeader{})
gob.RegisterName("OtherMsg", OtherMsg{})
}
//
// Messages
//
type MyMessageHeader struct {
MessageId InstanceIdType
OtherId uint64
}
type MyMessage interface {
Id() *MyMessageHeader
}
type otherStruct struct {
X uint8
Y uint8
}
type OtherMsg struct {
MyMessageHeader
OtherField *otherStruct
}
func (m *OtherMsg) Id() *MyMessageHeader {
return &m.MyMessageHeader
}
func MarshalMessage(m MyMessage) ([]byte, error) {
var buff bytes.Buffer
encoder := gob.NewEncoder(&buff)
if err := encoder.Encode(&m); err != nil {
return nil, errors.Newf("Could not marshal message: %v", err)
}
return buff.Bytes(), nil
}
func UnmarshalMessage(buf []byte) (MyMessage, error) {
var msg MyMessage
decoder := gob.NewDecoder(bytes.NewReader(buf))
if err := decoder.Decode(&msg); err != nil {
return nil, errors.Newf("Could not decode my message: %v", err)
}
return msg, nil
}
When I try to use UnmarshalMessage I got the error gob: OtherMsg is not assignable to type MyMessage.
I have no difference with the usage in the go examples for encoding and decoding an interface. What am I doing wrong?
The error message says:
gob: OtherMsg is not assignable to type MyMessage
Which is true. MyMessage is an interface which requires implemener types to have a method:
Id() *MyMessageHeader
A value of type OtherMsg does not have such method in its method set, only a pointer to it, that is a value of type *OtherMsg (because OtherMsg.Id() has pointer receiver).
To make it work, register a value of type *OtherMsg:
gob.RegisterName("OtherMsg", &OtherMsg{})
Or simply:
gob.Register(&OtherMsg{})
See a working example on the Go Playground:
func init() {
gob.Register(&OtherMsg{})
}
func main() {
var m MyMessage = &OtherMsg{
MyMessageHeader: MyMessageHeader{
MessageId: 1,
OtherId: 2,
},
OtherField: "othervalue",
}
data, err := MarshalMessage(m)
if err != nil {
panic(err)
}
m2, err := UnmarshalMessage(data)
if err != nil {
panic(err)
}
fmt.Printf("%+v\n", m2)
}
Output (try it on the Go Playground):
&{MyMessageHeader:{MessageId:1 OtherId:2} OtherField:othervalue}

cannot use function (type func()) as type in argument

package main
import (
"log"
"strings"
"asl.com/asl"
)
/*
Trivial service to demonstrate chaining service together
Message starts in originator, travels through a couple formatters, and then gets back to originator
*/
type MessageTest struct {
Body string `json:"body"`
}
var s *asl.Service
func main() {
var (
err error
cid string
)
//var m MessageDelivery
var g asl.MessageHandler
g = UpperCaseHandler
// UpperCaser := asl.NewService("UpperCaser", "", false)
UpperCaser := asl.NewService("UpperCaser")
if err = UpperCaser.ConsumeFunc("asl-service-uc", []string{"asl-service-uc"},func() interface{} {
return ""
},g); err != nil {
log.Fatalf("Error starting consumer: %v", err)
}
// Repeater := asl.NewService("Repeater", "", false)
Repeater := asl.NewService("Repeater")
if err = Repeater.ConsumeFunc("asl-service-repeat", []string{"asl-service-repeat"}, func() interface{} {
return ""
}, RepeatHandler); err != nil {
//if err = Repeater.ConsumeFunc("asl-service-repeat", []string{"asl-service-repeat"}, mg asl.MessageGenerator, mh asl.MessageHandler); err != nil {
log.Fatalf("Error starting consumer: %v", err)
}
// originator := asl.NewService("Originator", "", false)
originator := asl.NewService("Originator")
deliveryChan := make(chan asl.MessageDelivery)
m := asl.MessagePublishing{
Message: MessageTest{"this is a test"},
RoutingKeys: []string{"asl-service-uc", "asl-service-repeat"},
}
if cid, err = originator.RPCPublish(m, deliveryChan); err != nil {
log.Fatalf("Failed to publish: %v", err)
}
message := <-deliveryChan
log.Printf("Originator Got: %+v", message.Message)
originator.RemoveQueue(cid)
UpperCaser.Wait()
}
func UpperCaseHandler(md asl.MessageDelivery) {
s.Reply(MessageTest{strings.ToUpper(md.Message.(string))}, md.Delivery)
}
func RepeatHandler(md asl.MessageDelivery) {
s.Reply(MessageTest{strings.Repeat(md.Message.(string), 5)}, md.Delivery)
}
package asl
Error
./chains.go:26:10: cannot use UpperCaseHandler (typefunc(asl.MessageDelivery)) as type asl.MessageHandler in
assignment
./chains.go:37:86: cannot use RepeatHandler (type func(asl.MessageDelivery)) as type asl.MessageHandler in
argument to Repeater.ConsumeFunc
type MessageDelivery struct {
Delivery amqp.Delivery
Message interface{}
Error error
Context *Context
}
type MessageGenerator func() interface{}
type MessageHandler func(MessageDelivery) (interface{}, error)
I tried running the code,where am i going wrong,how do i right pass function as argument to another function. The function returns
interface and error. Though function is taking MessageDelivery Struct as argument,function signature is same.where am i going wrong
You are passing the function as an argument correctly but they do not match the expected signature. Change your functions to:
func UpperCaseHandler(md busboy.MessageDelivery) (interface{}, error} {
s.Reply(MessageTest{strings.ToUpper(md.Message.(string))}, md.Delivery)
return nil, nil
}
func RepeatHandler(md busboy.MessageDelivery) (interface{}, error} {
s.Reply(MessageTest{strings.Repeat(md.Message.(string), 5)}, md.Delivery)
return nil, nil
}

What's the purpose of gob.Register method?

I have read the documentation of ( gob) and I have some problems :
Now I know how to encode structure and decode like that:
func main() {
s1 := &S{
Field1: "Hello Gob",
Field2: 999,
}
log.Println("Original value:", s1)
buf := new(bytes.Buffer)
err := gob.NewEncoder(buf).Encode(s1)
if err != nil {
log.Println("Encode:", err)
return
}
s2 := &S{}
err = gob.NewDecoder(buf).Decode(s2)
if err != nil {
log.Println("Decode:", err)
return
}
log.Println("Decoded value:", s2)
}
But I don't know the purpose of this method gob.Register() can someone explain to me when to use it and why?
If you're dealing with concrete types (structs) only, you don't really need it. Once you're dealing with interfaces you must register your concrete type first.
For example, let's assume we have these struct and interface (the struct implements the interface):
type Getter interface {
Get() string
}
type Foo struct {
Bar string
}
func (f Foo)Get() string {
return f.Bar
}
To send a Foo over gob as a Getter and decode it back, we must first call
gob.Register(Foo{})
So the flow would be:
// init and register
buf := bytes.NewBuffer(nil)
gob.Register(Foo{})
// create a getter of Foo
g := Getter(Foo{"wazzup"})
// encode
enc := gob.NewEncoder(buf)
enc.Encode(&g)
// decode
dec := gob.NewDecoder(buf)
var gg Getter
if err := dec.Decode(&gg); err != nil {
panic(err)
}
Now try removing the Register and this won't work because gob wouldn't know how to map things back to their appropriate type.
As http://golang.org/pkg/encoding/gob/#Register said:
Only types that will be transferred as implementations of interface
values need to be registered.
So it doesn't needed by your demo.
If you want to encode / decode a map[string]interface{}, since the field of the map is enclosed as interface type, then we need to register the specific type before.
package main
import (
"bytes"
"encoding/gob"
"fmt"
"log"
)
type SomeStruct struct {
Text string
}
func main() {
var bytes bytes.Buffer
// Remove one of these, then the decoding will produce error
gob.Register(SomeStruct{})
gob.Register([]interface{}{})
gob.Register([]SomeStruct{})
gob.Register(map[string]SomeStruct{})
writer := gob.NewEncoder(&bytes)
err := writer.Encode(map[string]interface{}{
"SomeStruct": SomeStruct{"Halo"},
"SomeSlice": []interface{}{},
"SomeSliceStruct": []SomeStruct{
{
Text: "SomeText",
},
},
"SomeMapStruct": map[string]SomeStruct{
"S": {"Test"},
},
})
if err != nil {
log.Fatalf("Error on encode process: %v\n", err)
return
}
reader := gob.NewDecoder(&bytes)
var aMap map[string]interface{}
err = reader.Decode(&aMap)
if err != nil {
log.Fatalf("Error on decode process: %v\n", err)
return
}
fmt.Printf("Decode is successful: %+v\n", aMap)
}

Resources