How to gob encode struct with unexported pointer? - go

I have a struct that I can successfully gob encode and decode as follows:
type Node struct {
Value int
Next *Node
}
myNode := Node{
Value: 1,
Next: &Node{
Value: 2,
},
}
var buf bytes.Buffer
enc := gob.NewEncoder(&buf)
dec := gob.NewDecoder(&buf)
if err := enc.Encode(&myNode); err != nil {
panic(err)
}
var decodedNode Node
if err := dec.Decode(&decodedNode); err != nil {
panic(err)
}
I'd like to now hide the fields of Node:
type Node struct {
value int
next *Node
}
Because the fields are no longer exported I have to now write custom GobEncode and GobDecode functions:
func (d *Node) GobEncode() ([]byte, error) {
var buf bytes.Buffer
encoder := gob.NewEncoder(&buf)
if err := encoder.Encode(d.value); err != nil {
return nil, err
}
if err := encoder.Encode(d.next); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func (d *Node) GobDecode(b []byte) error {
buf := bytes.NewBuffer(b)
decoder := gob.NewDecoder(buf)
if err := decoder.Decode(&d.value); err != nil {
return err
}
if err := decoder.Decode(&d.next); err != nil {
return err
}
return nil
}
This doesn't work because of a nil value in Node.next:
panic: gob: cannot encode nil pointer of type *main.Node [recovered]
panic: gob: cannot encode nil pointer of type *main.Node [recovered]
panic: gob: cannot encode nil pointer of type *main.Node
I'm re-using gob.NewEncoder inside my custom function because I'm trying to re-use as much of the encode package as possible.
Any suggestions for getting this working?

Related

How to set slice interface values with reflection

I would like to build a function that takes a generic pointer array and fill that list based on mongo results.
I don't know how to set the value I got from mongo into my pointer array. In the below attempt, program panics with following error : reflect.Set: value of type []interface {} is not assignable to type []Person
When I print total / documents found, it corresponds to what I am expecting. So I think question is about reflection.
func getListWithCount(ctx context.Context, receiver interface{}) (int, error) {
//my mongo query here
var mongoResp struct {
Total int `bson:"total"`
Documents interface{} `bson:"documents"`
}
if err := cursor.Decode(&mongoResp); err != nil {
return 0, err
}
receiverValue := reflect.ValueOf(receiver)
docs := []interface{}(mongoResp.Documents.(primitive.A))
receiverValue.Elem().Set(reflect.ValueOf(docs))
return mongoResp.Total, nil
}
type Person struct {
Name string `bson:"name"`
}
func main() {
var persons []Person
count, err := getListWithCount(context.Background(), &persons)
if err != nil {
log.Fatal(err)
}
fmt.Println(count)
fmt.Println(persons)
}
You should be able to decode first into bson.RawValue and then Unmarshal it into the receiver.
func getListWithCount(ctx context.Context, receiver interface{}) (int, error) {
//my mongo query here
var mongoResp struct {
Total int `bson:"total"`
Documents bson.RawValue `bson:"documents"`
}
if err := cursor.Decode(&mongoResp); err != nil {
return 0, err
}
if err := mongoResp.Documents.Unmarshal(receiver); err != nil {
return 0, err
}
return mongoResp.Total, nil
}
You can also implement it as a custom bson.Unmarshaler.
type MongoResp struct {
Total int `bson:"total"`
Documents interface{} `bson:"documents"`
}
func (r *MongoResp) UnmarshalBSON(data []byte) error {
var temp struct {
Total int `bson:"total"`
Documents bson.RawValue `bson:"documents"`
}
if err := bson.Unmarshal(data, &temp); err != nil {
return err
}
r.Total = temp.Total
return temp.Documents.Unmarshal(r.Documents)
}
With that you would use it in the function like so:
func getListWithCount(ctx context.Context, receiver interface{}) (int, error) {
//my mongo query here
mongoResp := MongoResp{Documents: receiver}
if err := cursor.Decode(&mongoResp); err != nil {
return 0, err
}
return mongoResp.Total, nil
}
Dynamically create a struct type that matches the queried document. See commentary below for details.
func getListWithCount(receiver interface{}) (int, error) {
dst := reflect.ValueOf(receiver).Elem()
// Your mongo query here
// Create a struct type that matches the document.
doct := reflect.StructOf([]reflect.StructField{
reflect.StructField{Name: "Total", Type: reflect.TypeOf(0), Tag: `bson:"total"`},
reflect.StructField{Name: "Documents", Type: dst.Type(), Tag: `bson:"documents"`},
})
// Decode to a value of the type.
docp := reflect.New(doct)
if err := cursor.Decode(docp.Interface()); err != nil {
return 0, err
}
docv := docp.Elem()
// Copy the Documents field to *receiver.
dst.Set(docv.Field(1))
// Return the total
return docv.Field(0).Interface().(int), nil
}
there is no need to use reflect here, you can decode it directly to your Person slices
func getPersons(ctx context.Context, coll *mongo.Collection, results interface{}) error {
cur, err := coll.Find(ctx, bson.D{})
if err != nil {
return err
}
err = cur.All(ctx, results)
if err != nil {
return err
}
return nil
}
and the len is the count of the results.
err = getPersons(ctx, coll, &persons)
require.NoError(t, err)
t.Logf("Got %d persons: %v", len(persons), persons)
see https://gist.github.com/xingyongtao/459f92490bdcbf7d5afe9f5d1ae6c04a

Custom unmarshall to a type than input out different type

I'm learning Go and I have a question:
I have an Info type that is defined like that:
type Info struct {
ID ID `json:"id,omitempty"`
DisplayName string `json:"display_name,omitempty"`
}
I made a custom UnmarshallJSON function to unmarshall this struct because as an input I have either:
An []interface{} with at position [0] an int and [1] a string
A boolean always equals to false meaning that the field is null
I want that when the input is false, the Info is nil.
Here's the UnmarshallJSON function
func (i *Info) UnmarshalJSON(data []byte) error {
var v []interface{}
if err := json.Unmarshal(data, &v); err != nil {
var v bool
if err = json.Unmarshal(data, &v); err != nil {
return err
}
return nil
}
i.ID = ID(v[0].(float64))
i.DisplayName = v[1].(string)
return nil
}
It's ugly, and I would like to know if there's a better option.
Thank you very much.
Fist you should be more defensive about unexpected types and length to avoid a panic. Then you can unmarshal into a []json.RawMessage to defer unmarshaling of the elements until you are ready. Finally you should guard against your invalid true.
Here is my best effort, please others feel free to edit (here is a playground):
func (i *Info) UnmarshalJSON(data []byte) error {
var raw interface{}
if err := json.Unmarshal(data, &raw); err != nil {
return err
}
switch r := raw.(type) {
case []interface{}:
case bool:
if r {
return errors.New("unexpected true, must be array or false")
}
return nil
default:
return fmt.Errorf("unexpected type %T, must be array or false", r)
}
var v []json.RawMessage
if err := json.Unmarshal(data, &v); err != nil {
return err
}
if len(v) != 2 {
return fmt.Errorf("unexpected length %d, must be 2", len(v))
}
if err := json.Unmarshal(v[0], &i.ID); err != nil {
return err
}
if err := json.Unmarshal(v[1], &i.DisplayName); err != nil {
return err
}
return nil
}

golang type mismatch when using inherited struct

type MongoDBConfig struct {
*mgo.DialInfo
}
func ConfigureMongoDB() (*MongoDBConfig, error) {
//Get gopath
GOPATH := os.Getenv("GOPATH")
file, err := os.Open(GOPATH+RESOURCE_PATH)
if err != nil {
return nil, err
}
decoder := json.NewDecoder(file)
mongoConfig := MongoDBConfig{}
er := decoder.Decode(&mongoConfig)
if er != nil {
return nil, er
}
return &mongoConfig, nil
}
func InitMongoDB() (*Session, error){
mongoConfig, err := ConfigureMongoDB()
if err != nil {
return nil, err
}
session, mongoerr := mgo.DialWithInfo(mongoConfig)
}
Getting error in the last line while passing mongoConfig. I created the struct type of MongoDBConfig using DialInfo type.
Cannot use mongoConfig (type * MongoDBConfig) as type *DialInfo
Access embeded field explicitly:
session, mongoerr := mgo.DialWithInfo(mongoConfig.DialInfo)

pass interface pointer and assignment value

I want to write a file cache in Go. I am using gob encoding, and saving to a file, but my get function has some problem:
package main
import (
"encoding/gob"
"fmt"
"os"
)
var (
file = "tmp.txt"
)
type Data struct {
Expire int64
D interface{}
}
type User struct {
Id int
Name string
}
func main() {
user := User{
Id: 1,
Name: "lei",
}
err := set(file, user, 10)
if err != nil {
fmt.Println(err)
return
}
user = User{}
err = get(file, &user)
if err != nil {
fmt.Println(err)
return
}
//user not change.
fmt.Println(user)
}
func set(file string, v interface{}, expire int64) error {
f, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
return err
}
defer f.Close()
//wrapper data
//save v in data.D
data := Data{
Expire: expire,
D: v,
}
gob.Register(v)
enc := gob.NewEncoder(f)
err = enc.Encode(data)
if err != nil {
return err
}
return nil
}
func get(file string, v interface{}) error {
f, err := os.OpenFile(file, os.O_RDONLY, 0600)
if err != nil {
return err
}
defer f.Close()
var data Data
dec := gob.NewDecoder(f)
err = dec.Decode(&data)
if err != nil {
return err
}
//get v
v = data.D
fmt.Println(v)
return nil
}
The get function passes interface type and I want to change the value, but not change.
http://play.golang.org/p/wV7rBH028o
In order to insert an unknown value into v of type interface{}, you need to use reflection. This is somewhat involved, but if you want to support this in full, you can see how its done by walking through the decoding process in some of the encoding packages (json, gob).
To get you started, here's a basic version of your get function using reflection. This skips a number of checks, and will only decode something that was encoded as a pointer.
func get(file string, v interface{}) error {
f, err := os.OpenFile(file, os.O_RDONLY, 0600)
if err != nil {
return err
}
defer f.Close()
rv := reflect.ValueOf(v)
if rv.Kind() != reflect.Ptr || rv.IsNil() {
panic("need a non nil pointer")
}
var data Data
dec := gob.NewDecoder(f)
err = dec.Decode(&data)
if err != nil {
return err
}
dv := reflect.ValueOf(data.D)
if dv.Kind() != reflect.Ptr {
panic("didn't decode a pointer")
}
rv.Elem().Set(dv.Elem())
return nil
}
I would actually suggest an easier way to handle this in your own code, which is to have the Get function return an interface{}. Since you will know what the possible types are at that point, you can use a type switch to assert the correct value.
An alternative approach is to return directly the value from the file:
func get(file string) (interface{}, error) {
f, err := os.OpenFile(file, os.O_RDONLY, 0600)
if err != nil {
return nil, err
}
defer f.Close()
var data Data
dec := gob.NewDecoder(f)
err = dec.Decode(&data)
if err != nil {
return nil,err
}
fmt.Println(data.D)
return data.D,nil
}
full working example: http://play.golang.org/p/178U_LVC5y

Generic function to store Go Encoded Data

I need to write a generic function which can store objects as gobjects.
func hash_store(data map[string]string) {
//initialize a *bytes.Buffer
m := new(bytes.Buffer)
//the *bytes.Buffer satisfies the io.Writer interface and can
//be used in gob.NewEncoder()
enc := gob.NewEncoder(m)
//gob.Encoder has method Encode that accepts data items as parameter
enc.Encode(data)
//the bytes.Buffer type has method Bytes() that returns type []byte,
//and can be used as a parameter in ioutil.WriteFile()
err := ioutil.WriteFile("dep_data", m.Bytes(), 0600)
if err != nil {
panic(err)
}
fmt.Printf("just saved all depinfo with %v\n", data)
n,err := ioutil.ReadFile("dep_data")
if err != nil {
fmt.Printf("cannot read file")
panic(err)
}
//create a bytes.Buffer type with n, type []byte
p := bytes.NewBuffer(n)
//bytes.Buffer satisfies the interface for io.Writer and can be used
//in gob.NewDecoder()
dec := gob.NewDecoder(p)
//make a map reference type that we'll populate with the decoded gob
//e := make(map[int]string)
e := make(map[string]string)
//we must decode into a pointer, so we'll take the address of e
err = dec.Decode(&e)
if err != nil {
fmt.Printf("cannot decode")
panic(err)
}
fmt.Println("after reading dep_data printing ",e)
}
In this function I know the data type to be stored in map[string]string . But I need to write a generic function where I don't know data type and still store it as a gobject in a file.
Change your concrete type (map[string]string) to the empty interface type (interface{}).
See this related question why this works.
Encoding:
func store(data interface{}) {
m := new(bytes.Buffer)
enc := gob.NewEncoder(m)
err := enc.Encode(data)
if err != nil { panic(err) }
err = ioutil.WriteFile("dep_data", m.Bytes(), 0600)
if err != nil { panic(err) }
}
Decoding:
func load(e interface{}) {
n,err := ioutil.ReadFile("dep_data")
if err != nil { panic(err) }
p := bytes.NewBuffer(n)
dec := gob.NewDecoder(p)
err = dec.Decode(e)
if err != nil { panic(err) }
}
The value you put in load must be a pointer of the type you stored in the file using gob.
Example for map[string]string:
org := map[string]string{"foo": "bar"}
store(org)
var loadedMap map[string]string
load(&loadedMap)
fmt.Println(loadedMap["foo"]) // bar
When you encode the data, give the Encoder a *interface{}, then you can decoded with a *interface{}
var to_enc interface{} = ...;
god.NewEncoder(...).Encode(&to_enc);
...
var to_dec interface{}
god.NewDecoder(...).Decode(&to_dec);

Resources