Serialization of struct with pointers - go

Having a struct heirarchy like:
type DomainStore struct {
Domains []*Domain
Users []*User
}
type Domain struct {
Name string
Records []*Record
Owner *User
}
type User struct {
Name string
Email string
Domains []*Domain
}
type Record struct {
Name string
Host string
}
With a single DomainStore having a list of Domain and Users with pointer between Domain and User.
I'm looking for a way to serialize/deserialize to/from file. I have been trying to use gob, but the pointers is not (by design) serialized correct (its flattened).
Thinking about giving each object a unique id and making a func to serialize/deserialize each type, but it seems much work/boilerplate. Any suggestions for a strategy?
I would like to keep the whole DomainStore in memory, and just serialize to file on user request.
The main problem: How to serialise/deserialize and keep the pointers pointing to the same object and not different copies of the same object
Both gob and json seems to "just" copy the value of the object and afted deserializasion I end up with multiple independent copies of objects.
Using gob ang json this is what happens:
Before, A & C both points to B:
A -> B <- C
After deserialization with json/gob:
A -> B1 , C -> B2
A & C points to to different object, with the same values. But, if i change B1 it's not changed in B2.
--- Update ---
When marshalling i can obtain the memory location of the object and use it as an ID:
func (u *User) MarshalJSON() ([]byte, error) {
return json.Marshal(&JsonUser{
ID: fmt.Sprintf("%p", u),
Name: u.Name,
Email: u.Email,
})
}
And when marshalling the Domain I can replace the
func (d *Domain) MarshalJSON() ([]byte, error) {
return json.Marshal(&struct {
ID string `json:"id"`
Name string `json:"name"`
User string `json:"user"`
}{
ID: fmt.Sprintf("%p", d),
Name: d.Name,
User: fmt.Sprintf("%p", d.User),
})
}
Now I just need to be able to unmarshal this which gives me a problem in the UnmarshalJSON need to access a map of id's and their respective objects.
func (u *User) UnmarshalJSON(data []byte) error {
// need acces to a map shared by all UnmarshalJSON functions
}

It can be done using the following method:
All the objects are placed in maps in a State object.
When the objects in a State object is marshalled, all objects refered to using pointers is replaced with the memory location of the object.
When unmarshalled pointers are restored using a global list of previously read objects.
The code will run, and is just to illustrate the method, I'm new to Go, so bear with me.
package main
import (
"encoding/json"
"errors"
"fmt"
"log"
"strings"
)
type User struct {
Name string
Email string
}
type JsonUser struct {
ID string `json:"id"`
Name string `json:"name"`
Email string `json:"email"`
}
func (u *User) Print(level int) {
ident := strings.Repeat("-", level)
log.Println(ident, "Username:", u.Name, u.Email)
}
func (u *User) Id() string {
return fmt.Sprintf("%p", u)
}
func (u *User) MarshalJSON() ([]byte, error) {
return json.Marshal(&JsonUser{
ID: u.Id(),
Name: u.Name,
Email: u.Email,
})
}
func (u *User) UnmarshalJSON(data []byte) error {
aux := &JsonUser{}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
u.Name = aux.Name
u.Email = aux.Email
load_helper[aux.ID] = u
log.Println("Added user with id ", aux.ID, u.Name)
return nil
}
type Record struct {
Type string // MX / A / CNAME / TXT / REDIR / SVR
Name string // # / www
Host string // IP / address
Priority int // Used for MX
Port int // Used for SVR
}
type JsonRecord struct {
ID string
Type string
Name string
Host string
Priority int
Port int
}
func (r *Record) Print(level int) {
ident := strings.Repeat("-", level)
log.Println(ident, "", r.Type, r.Name, r.Host)
}
func (r *Record) Id() string {
return fmt.Sprintf("%p", r)
}
func (r *Record) MarshalJSON() ([]byte, error) {
return json.Marshal(&JsonRecord{
ID: r.Id(),
Name: r.Name,
Type: r.Type,
Host: r.Host,
Priority: r.Priority,
Port: r.Port,
})
}
func (r *Record) UnmarshalJSON(data []byte) error {
aux := &JsonRecord{}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
r.Name = aux.Name
r.Type = aux.Type
r.Host = aux.Host
r.Priority = aux.Priority
r.Port = aux.Port
load_helper[aux.ID] = r
log.Println("Added record with id ", aux.ID, r.Name)
return nil
}
type Domain struct {
Name string
User *User // User ID
Records []*Record // Record ID's
}
type JsonDomain struct {
ID string `json:"id"`
Name string `json:"name"`
User string `json:"user"`
Records []string `json:"records"`
}
func (d *Domain) Print(level int) {
ident := strings.Repeat("-", level)
log.Println(ident, "Domain:", d.Name)
d.User.Print(level + 1)
log.Println(ident, " Records:")
for _, r := range d.Records {
r.Print(level + 2)
}
}
func (d *Domain) Id() string {
return fmt.Sprintf("%p", d)
}
func (d *Domain) MarshalJSON() ([]byte, error) {
var record_ids []string
for _, r := range d.Records {
record_ids = append(record_ids, r.Id())
}
return json.Marshal(JsonDomain{
ID: d.Id(),
Name: d.Name,
User: d.User.Id(),
Records: record_ids,
})
}
func (d *Domain) UnmarshalJSON(data []byte) error {
log.Println("UnmarshalJSON domain")
aux := &JsonDomain{}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
d.Name = aux.Name
d.User = load_helper[aux.User].(*User) // restore pointer to domains user
for _, record_id := range aux.Records {
d.Records = append(d.Records, load_helper[record_id].(*Record))
}
return nil
}
type State struct {
Users map[string]*User
Records map[string]*Record
Domains map[string]*Domain
}
func NewState() *State {
s := &State{}
s.Users = make(map[string]*User)
s.Domains = make(map[string]*Domain)
s.Records = make(map[string]*Record)
return s
}
func (s *State) Print() {
log.Println("State:")
log.Println("Users:")
for _, u := range s.Users {
u.Print(1)
}
log.Println("Domains:")
for _, d := range s.Domains {
d.Print(1)
}
}
func (s *State) NewUser(name string, email string) *User {
u := &User{Name: name, Email: email}
id := fmt.Sprintf("%p", u)
s.Users[id] = u
return u
}
func (s *State) NewDomain(user *User, name string) *Domain {
d := &Domain{Name: name, User: user}
s.Domains[d.Id()] = d
return d
}
func (s *State) NewMxRecord(d *Domain, rtype string, name string, host string, priority int) *Record {
r := &Record{Type: rtype, Name: name, Host: host, Priority: priority}
d.Records = append(d.Records, r)
s.Records[r.Id()] = r
return r
}
func (s *State) FindDomain(name string) (*Domain, error) {
for _, v := range s.Domains {
if v.Name == name {
return v, nil
}
}
return nil, errors.New("Not found")
}
func Save(s *State) (string, error) {
b, err := json.MarshalIndent(s, "", " ")
if err == nil {
return string(b), nil
} else {
log.Println(err)
return "", err
}
}
var load_helper map[string]interface{}
func Load(s *State, blob string) {
load_helper = make(map[string]interface{})
if err := json.Unmarshal([]byte(blob), s); err != nil {
log.Println(err)
} else {
log.Println("OK")
}
}
func test_state() {
s := NewState()
u := s.NewUser("Ownername", "some#email.com")
d := s.NewDomain(u, "somedomain.com")
s.NewMxRecord(d, "MX", "#", "192.168.1.1", 10)
s.NewMxRecord(d, "A", "www", "192.168.1.1", 0)
s.Print()
x, _ := Save(s) // Saved to json string
log.Println("State saved, the json string is:")
log.Println(x)
s2 := NewState() // Create a new empty State
Load(s2, x)
s2.Print()
d, err := s2.FindDomain("somedomain.com")
if err == nil {
d.User.Name = "Changed"
} else {
log.Println("Error:", err)
}
s2.Print()
}
func main() {
test_state()
}
This is quite a lot of code and there are to much coupling between the objects and the serialization. Also the global var load_helper is bad. Ideas to improve will be appreciated.
Another approch would be to use reflection to make a more generic solution. Here is an example using this method:
package main
import (
"encoding/json"
"fmt"
"log"
"strings"
"reflect"
)
func pprint(x interface{}) {
b, err := json.MarshalIndent(x, "", " ")
if err != nil {
fmt.Println("error:", err)
}
fmt.Println(string(b))
}
var typeRegistry = make(map[string]reflect.Type)
// Register a type to make it possible for the Save/Load functions
// to serialize it.
func Register(v interface{}) {
t := reflect.TypeOf(v)
n := t.Name()
fmt.Println("Register type",n)
typeRegistry[n] = reflect.TypeOf(v)
}
// Make an instance of a type from the string name of the type.
func makeInstance(name string) reflect.Value {
v := reflect.New(typeRegistry[name]).Elem()
return v
}
// Translate a string type name tpo a real type.
func getTypeFromString(name string) reflect.Type {
return typeRegistry[name]
}
// Serializeable interface must be supported by all objects passed to the Load / Save functions.
type Serializeable interface {
Id() string
}
// GenericSave saves the object d
func GenericSave(d interface{}) (string, error) {
r := make(map[string]interface{})
v := reflect.ValueOf(d)
t := reflect.TypeOf(d)
if t.Kind()==reflect.Ptr {
t=t.Elem()
v=v.Elem()
}
r["_TYPE"]=t.Name()
r["_ID"]=fmt.Sprintf("%p", d)
for i := 0; i < t.NumField(); i++ {
f := t.Field(i)
name := f.Name
vf := v.FieldByName(name)
// fmt.Println("Field", i+1, "name is", name, "type is", f.Type.Name(), "and kind is", f.Type.Kind())
// fmt.Println("V:", vf)
if f.Tag != "" {
store:=strings.Split(f.Tag.Get("store"),",")
switch store[1] {
case "v":
switch t.Field(i).Type.Name() {
case "string":
r[store[0]]=vf.String()
case "int":
r[store[0]]=vf.Int()
}
case "p":
vals:=vf.MethodByName("Id").Call([]reflect.Value{})
r[store[0]]=vals[0].String()
case "lp":
tr:=[]string{}
for j := 0; j < vf.Len(); j++ {
vals:=vf.Index(j).MethodByName("Id").Call([]reflect.Value{})
tr=append(tr,vals[0].String())
}
r[store[0]]=tr
}
}
}
m,_:=json.Marshal(r)
return string(m),nil
}
// Save saves the list of objects.
func Save(objects []Serializeable) []byte {
lst:=[]string{}
for _,o := range(objects) {
os,_:= GenericSave(o) // o.Save()
lst=append(lst,os)
}
m,_:=json.Marshal(lst)
return m
}
func toStructPtr(obj interface{}) interface{} {
vp := reflect.New(reflect.TypeOf(obj))
vp.Elem().Set(reflect.ValueOf(obj))
return vp.Interface()
}
// Load creates a list of serializeable objects from json blob
func Load(blob []byte) []Serializeable {
objects := []Serializeable{}
loadHelper := make(map[string]interface{})
var olist []interface{}
if err := json.Unmarshal(blob, &olist); err != nil {
log.Println(err)
} else {
for _,o := range(olist) {
var omap map[string]interface{}
json.Unmarshal([]byte(o.(string)), &omap)
t:= getTypeFromString(omap["_TYPE"].(string))
obj := reflect.New(t).Elem()
for i := 0; i < t.NumField(); i++ {
// n:=t.Field(i).Name
// fmt.Println(i,n,t.Field(i).Type.Name())
if t.Field(i).Tag != "" {
store:=strings.Split(t.Field(i).Tag.Get("store"),",")
// fmt.Println(store)
switch store[1] {
case "v":
switch t.Field(i).Type.Name() {
case "string":
obj.FieldByIndex([]int{i}).SetString(omap[store[0]].(string))
case "int":
obj.FieldByIndex([]int{i}).SetInt(int64(omap[store[0]].(float64)))
}
case "p":
nObj:=loadHelper[omap[store[0]].(string)]
obj.FieldByIndex([]int{i}).Set(reflect.ValueOf(nObj.(*User)))
case "lp":
ptrItemType:=t.Field(i).Type.Elem()
slice := reflect.Zero(reflect.SliceOf( ptrItemType /* reflect.TypeOf( &Record{} ) */ ))//.Interface()
for _, pID := range(omap[store[0]].([]interface{})) {
nObj:=loadHelper[pID.(string)]
slice=reflect.Append(slice, reflect.ValueOf(nObj) )
}
obj.FieldByIndex([]int{i}).Set(slice)
}
}
}
oi:=toStructPtr(obj.Interface())
oip:=oi.(Serializeable)
objects=append(objects,oip)
loadHelper[omap["_ID"].(string)]=oip
}
}
return objects
}
/* Application data structures */
type User struct {
Name string `store:"name,v"`
Email string `store:"email,v"`
}
func (u *User) Id() string {
return fmt.Sprintf("%p", u)
}
func (u *User) Save() (string, error) {
return GenericSave(u)
}
func (u *User) Print() {
fmt.Println("User:",u.Name)
}
type Record struct {
Type string `store:"type,v"`// MX / A / CNAME / TXT / REDIR / SVR
Name string `store:"name,v"`// # / www
Host string `store:"host,v"`// IP / address
Priority int `store:"priority,v"`// Used for MX
Port int `store:"port,v"`// Used for SVR
}
func (r *Record) Id() string {
return fmt.Sprintf("%p", r)
}
func (r *Record) Save() (string, error) {
return GenericSave(r)
}
func (r *Record) Print() {
fmt.Println("Record:",r.Type,r.Name,r.Host)
}
type Domain struct {
Name string `store:"name,v"`
User *User `store:"user,p"` // User ID
Records []*Record `store:"record,lp"` // Record ID's
}
func (d *Domain) Id() string {
return fmt.Sprintf("%p", d)
}
func (d *Domain) Save() (string, error) {
return GenericSave(d)
}
func (d *Domain) Print() {
fmt.Println("Domain:",d.Name)
d.User.Print()
fmt.Println("Records:")
for _, r := range d.Records {
r.Print()
}
}
type DBM struct {
Domains []*Domain
Users []*User
Records []*Record
}
func (dbm *DBM) AddDomain(d *Domain) {
dbm.Domains=append(dbm.Domains,d)
}
func (dbm *DBM) AddUser(u *User) {
dbm.Users=append(dbm.Users,u)
}
func (dbm *DBM) AddRecord(r *Record) {
dbm.Records=append(dbm.Records,r)
}
func (dbm *DBM) GetObjects() []Serializeable {
objects:=[]Serializeable{}
for _,r := range(dbm.Records) {
objects=append(objects, r)
}
for _,u := range(dbm.Users) {
objects=append(objects, u)
}
for _,d := range(dbm.Domains) {
objects=append(objects, d)
}
return objects
}
func (dbm *DBM) SetObjects(objects []Serializeable) {
for _,o := range(objects) {
switch o.(type) {
case *Record:
fmt.Println("record")
dbm.AddRecord(o.(*Record))
case *User:
fmt.Println("record")
dbm.AddUser(o.(*User))
case *Domain:
fmt.Println("record")
dbm.AddDomain(o.(*Domain))
}
}
}
func testState() {
Register(User{})
Register(Domain{})
Register(Record{})
dbm:=DBM{}
u := &User{Name: "Martin", Email: "some#email.com"}
dbm.AddUser(u)
r1 := &Record{Name: "#", Type: "MX", Host: "mail.ishost.dk"}
r2 := &Record{Name: "#", Type: "MX", Host: "mail.infoserv.dk"}
dbm.AddRecord(r1)
dbm.AddRecord(r2)
d := &Domain{User:u, Name: "Martin", Records: []*Record{r1, r2}}
dbm.AddDomain(d)
x:=Save(dbm.GetObjects())
fmt.Println("== Saved objects")
// fmt.Println(string(x))
fmt.Println("== Loading")
dbm2:=DBM{}
dbm2.SetObjects(Load(x))
u2:=dbm2.Users[0]
u2.Print()
u2.Name="KURT"
u2.Print()
d2:=dbm2.Domains[0]
d2.Print()
d2.User.Name="ZIG"
u2.Print()
}
func main() {
testState()
}

Use encoding/json package
to marshal:
// Marshal is a function that marshals the object into an
// io.Reader.
// By default, it uses the JSON marshaller.
var Marshal = func(v interface{}) (io.Reader, error) {
b, err := json.MarshalIndent(v, "", "\t")
if err != nil {
return nil, err
}
return bytes.NewReader(b), nil
}
to unmarshal:
// Unmarshal is a function that unmarshals the data from the
// reader into the specified value.
// By default, it uses the JSON unmarshaller.
var Unmarshal = func(r io.Reader, v interface{}) error {
return json.NewDecoder(r).Decode(v)
}
Not sure there's more to this,
Another thing you can do is, store all these as json formatted strings.

Related

Unmarshal YAML into complex object which may be either struct or string

Trying to unmarshal YAML into complex object such as map[string]map[interface{}]string.
The problem is that I want to be able to differentiate an interface{} part between string and Source which is a struct.
type Source struct {
ID string `yaml:"id"`
Name string `yaml:"name"`
LogoID string `yaml:"logoId"`
URL string `yaml:"url"`
}
type UNFT struct {
ItemMeta map[string]map[interface{}]string `yaml:"item_meta"`
// could be
// ItemMeta map[string]map[string]string `yaml:"item_meta"`
// or
// ItemMeta map[string]map[Source]string `yaml:"item_meta"`
}
Obviously YAML does not know how to unmarshal into Source struct so I have to implement Unmarshaler interface:
type Unmarshaler interface {
UnmarshalYAML(value *Node) error
}
But I don't quite understand the big picture of unmarshaling process. In general I assume that I have to manually traverse *yaml.Node and call func UnmarshalYAML(value *Node) error on every node.
package main
import (
"fmt"
"gopkg.in/yaml.v3"
)
type Source struct {
ID string `json:"id"`
Name string `json:"name"`
LogoID string `json:"logoId"`
URL string `json:"url"`
}
var data = `
unf:
item_meta:
source:
!struct
? id: "data-watch"
name: "DataWatch"
logoid: "data-watch"
url: "https"
: "product_any('SS')"
public_usage:
"": "source_any('SDF')"
"provider": "source_any('ANO')"`
type UNFT struct {
ItemMeta map[string]map[interface{}]string `yaml:"item_meta"`
}
type MetaConverterConfigT struct {
UNFT UNFT `yaml:"unf"`
}
func main() {
cfg := MetaConverterConfigT{}
err := yaml.Unmarshal([]byte(data), &cfg)
if err != nil {
fmt.Println("%w", err)
}
fmt.Println(cfg)
}
func (s *UNFT) UnmarshalYAML(n *yaml.Node) error {
var cfg map[string]map[interface{}]string
if err := n.Decode(&cfg); err != nil {
fmt.Println("%w", err)
}
return nil
}
Go playground
type MetaKey struct {
String string
Source Source
}
func (k *MetaKey) UnmarshalYAML(n *yaml.Node) error {
if n.Tag == "!!str" {
return n.Decode(&k.String)
}
if n.Tag == "!!map" {
return n.Decode(&k.Source)
}
return fmt.Errorf("unsupported MetaKey type")
}
// ...
type UNFT struct {
ItemMeta map[string]map[MetaKey]string `yaml:"item_meta"`
}
https://go.dev/play/p/Nhtab4l-ANT
If you need the map type to remain as is, i.e. without adding the custom key type, then you can implement the unmarshaler on UNFT as well and just do a re-mapping with any:
type UNFT struct {
ItemMeta map[string]map[any]string `yaml:"item_meta"`
}
func (u *UNFT) UnmarshalYAML(n *yaml.Node) error {
var obj struct {
ItemMeta map[string]map[MetaKey]string `yaml:"item_meta"`
}
if err := n.Decode(&obj); err != nil {
return err
}
u.ItemMeta = make(map[string]map[any]string, len(obj.ItemMeta))
for k, v := range obj.ItemMeta {
m := make(map[any]string, len(v))
for k, v := range v {
if k.Source != (Source{}) {
m[k.Source] = v
} else {
m[k.String] = v
}
}
u.ItemMeta[k] = m
}
return nil
}
https://go.dev/play/p/uwboGKf3qnD

golang patch string values on an object, recursive with filtering

Community,
The mission
basic
Implement a func that patches all string fields on an objects
details
[done] fields shall only be patched if they match a matcher func
[done] value shall be processed via process func
patching shall be done recursive
it shall also work for []string, []*string and recursive for structs and []struct, []*struct
// update - removed old code
Solution
structs
updated the structs to use (though this does not affect the actual program, i use this for completeness
type Tag struct {
Name string `process:"yes,TagName"`
NamePtr *string `process:"no,TagNamePtr"`
}
type User struct {
ID int
Nick string
Name string `process:"yes,UserName"`
NamePtr *string `process:"yes,UserNamePtr"`
Slice []string `process:"yes,Slice"`
SlicePtr []*string `process:"yes,SlicePtr"`
SubStruct []Tag `process:"yes,SubStruct"`
SubStructPtr []*Tag `process:"yes,SubStructPtr"`
}
helper func
Further we need two helper funcs to check if a struct has a tag and to print to console
func Stringify(i interface{}) string {
s, _ := json.MarshalIndent(i, "", " ")
return string(s)
}
func HasTag(structFiled reflect.StructField, tagName string, tagValue string) bool {
tag := structFiled.Tag
if value, ok := tag.Lookup(tagName); ok {
parts := strings.Split(value, ",")
if len(parts) > 0 {
return parts[0] == tagValue
}
}
return false
}
patcher - the actual solution
type Patcher struct {
Matcher func(structFiled *reflect.StructField, v reflect.Value) bool
Process func(in string) string
}
func (p *Patcher) value(idx int, v reflect.Value, structFiled *reflect.StructField) {
if !v.IsValid() {
return
}
switch v.Kind() {
case reflect.Ptr:
p.value(idx, v.Elem(), structFiled)
case reflect.Struct:
for i := 0; i < v.NumField(); i++ {
var sf = v.Type().Field(i)
structFiled = &sf
p.value(i, v.Field(i), structFiled)
}
case reflect.Slice:
for i := 0; i < v.Len(); i++ {
p.value(i, v.Index(i), structFiled)
}
case reflect.String:
if p.Matcher(structFiled, v) {
v.SetString(p.Process(v.String()))
}
}
}
func (p *Patcher) Apply(in interface{}) {
p.value(-1, reflect.ValueOf(in).Elem(), nil)
}
how to use
func main() {
var NamePtr string = "golang"
var SubNamePtr string = "*secure"
testUser := User{
ID: 1,
Name: "lumo",
NamePtr: &NamePtr,
SubStruct: []Tag{{
Name: "go",
},
},
SubStructPtr: []*Tag{&Tag{
Name: "*go",
NamePtr: &SubNamePtr,
},
},
}
var p = Patcher{
// filter - return true if the field in struct has a tag process=true
Matcher: func(structFiled *reflect.StructField, v reflect.Value) bool {
return HasTag(*structFiled, "process", "yes")
},
// process
Process: func(in string) string {
if in != "" {
return fmt.Sprintf("!%s!", strings.ToUpper(in))
} else {
return "!empty!"
}
return in
},
}
p.Apply(&testUser)
fmt.Println("Output:")
fmt.Println(Stringify(testUser))
}
goplay
https://goplay.tools/snippet/-0MHDfKr7ax

Golang flag: Ignore missing flag and parse multiple duplicate flags

I am new to Golang and I have been unable to find a solution to this problem using flag.
How can I use flag so my program can handle calls like these, where the -term flag may be present a variable number of times, including 0 times:
./myprogram -f flag1
./myprogram -f flag1 -term t1 -term t2 -term t3
You need to declare your own type which implements the Value interface. Here is an example.
// Created so that multiple inputs can be accecpted
type arrayFlags []string
func (i *arrayFlags) String() string {
// change this, this is just can example to satisfy the interface
return "my string representation"
}
func (i *arrayFlags) Set(value string) error {
*i = append(*i, strings.TrimSpace(value))
return nil
}
then in the main function where you are parsing the flags
var myFlags arrayFlags
flag.Var(&myFlags, "term", "my terms")
flag.Parse()
Now all the terms are contained in the slice myFlags
This question is an interesting one and can play in many variations.
Array
Map
Struct
The core content is the same as #reticentroot answered,
Complete the definition of this interface: Flag.Value
The following are examples to share and provide relevant links as much as possible
Example
expected usage:
type Books []string
func (*Books) String() string { return "" }
func (*Books) Set(string) error { return nil }
type Dict map[string]string
func (*Dict) String() string { return "" }
func (*Dict) Set(string) error { return nil }
type Person struct {
Name string
Age int
}
func (*Person) String() string { return "" }
func (*Person) Set(string) error { return nil }
func pseudocode() {
flagSetTest := flag.NewFlagSet("test", flag.ContinueOnError)
books := Books{}
flagSetTest.Var(&books, "book", "-book C++ -book Go -book javascript")
// expected output: books: []string{C++,Go,javascript}
dict := Dict{}
flagSetTest.Var(&dict, "dict", "-dict A:65|B:66")
// expected output: dict: map[string]string{"A":"65", "B":"66"}
// map
person := Person{}
flagSetTest.Var(&person, "person", "-person Name:foo|Age:18")
// output: {Name:foo Age:18}
flagSetTest.Parse(os.Args[1:])
fmt.Println(person, books, dict)
}
Full code
package main
import (
"bufio"
"errors"
"flag"
"fmt"
"os"
"reflect"
"strconv"
"strings"
)
type BooksValue []string
// https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L298
func (arr *BooksValue) String() string {
/*
value.String(): https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L870
DefValue string:
- https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L348
- https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L914-L920
- https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L529-L536
- https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L464
*/
return ""
}
// https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L299
func (arr *BooksValue) Set(value string) error {
/*
value: https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L947
bool: Set(value): https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L966-L975
else: Set(value): https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L986-L988
*/
*arr = append(*arr, strings.TrimSpace(value))
return nil
}
type DictValue map[string]string
func (m *DictValue) String() string {
return ""
}
func (m *DictValue) Set(value string) error {
arr := strings.Split(value, "|") // "key1:val1|key2:val2|..."
for _, curPairStr := range arr {
itemArr := strings.Split(curPairStr, ":")
key := itemArr[0]
val := itemArr[1]
(*m)[key] = val
}
return nil
}
type PersonValue struct {
Name string
Age int
Msg string
IsActive bool
}
func (s *PersonValue) String() string {
return ""
}
func (s *PersonValue) Set(value string) error {
arr := strings.Split(value, "|") // "Field1:Value1|F2:V2|...|FN:VN"
for _, curPairStr := range arr {
itemArr := strings.Split(curPairStr, ":")
key := itemArr[0]
val := itemArr[1]
// [Access struct property by name](https://stackoverflow.com/a/66470232/9935654)
pointToStruct := reflect.ValueOf(s)
curStruct := pointToStruct.Elem()
curField := curStruct.FieldByName(key)
if !curField.IsValid() {
return errors.New("not found")
}
// CanSet one of conditions: Name starts with a capital
if !curField.CanSet() {
return errors.New("can't set")
}
t := reflect.TypeOf(*s)
structFieldXXX, isFound := t.FieldByName(key)
if !isFound {
return errors.New("not found")
}
switch structFieldXXX.Type.Name() {
case "int":
// https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L146-L153
intValue, err := strconv.ParseInt(val, 0, strconv.IntSize)
if err != nil {
return errors.New("parse error: [int]")
}
curField.SetInt(intValue)
case "bool":
// https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L117-L121
boolValue, err := strconv.ParseBool(val)
if err != nil {
return errors.New("parse error: [bool]")
}
curField.SetBool(boolValue)
case "string":
curField.SetString(val)
default:
return errors.New("not support type=" + structFieldXXX.Type.Name())
}
}
return nil
}
func main() {
flagSetTest := flag.NewFlagSet("test", flag.ContinueOnError)
// array
books := BooksValue{}
flagSetTest.Var(&books, "book", "-book Go -book javascript ...")
// map
myMap := DictValue{}
flagSetTest.Var(&myMap, "map", "-dict A:65|B:66")
// struct
person := PersonValue{Msg: "Hello world"}
flagSetTest.Var(&person, "person", "-person Name:string|Age:int|Msg:string|IsActive:bool")
testArgs := []string{"test",
"-book", "Go", "-book", "javascript", // testArray
"-map", "A:65|B:66|Name:Carson", // testMap
"-person", "Name:Carson|Age:30|IsActive:true", // testStruct
}
testFunc := func(args []string, reset bool) {
if reset {
books = BooksValue{}
myMap = DictValue{}
person = PersonValue{}
}
if err := flagSetTest.Parse(args); err != nil {
fmt.Printf(err.Error())
}
fmt.Printf("%+v\n", books)
fmt.Printf("%+v\n", myMap)
fmt.Printf("%+v\n", person)
}
testFunc(testArgs[1:], false)
// ↓ play by yourself
scanner := bufio.NewScanner(os.Stdin)
for {
fmt.Println("Enter CMD: ") // example: test -book item1 -book item2 -map key1:value1|key2:v2 -person Age:18|Name:Neil|IsActive:true
scanner.Scan() // Scans a line from Stdin(Console)
text := scanner.Text() // Holds the string that scanned
args := strings.Split(text, " ")
switch args[0] {
case "quit":
return
case "test":
testFunc(args[1:], true)
}
}
}
go playground

Create custom type that will seem like the another when checking types Golang

I am a experienced python programmer but I am still new to Golang so my apologies if this is an obvious or silly question. But I am trying to create my own type that I want to act exactly like the base type with the exception of several methods being overridden. The reason for this is because several libraries I am using are checking the type against time.Time and I want it to match.
type PythonTime struct {
time.Time
}
var pythonTimeFormatStr = "2006-01-02 15:04:05-0700"
func (self *PythonTime) UnmarshalJSON(b []byte) (err error) {
// removes prepending/trailing " in the string
if b[0] == '"' && b[len(b)-1] == '"' {
b = b[1 : len(b)-1]
}
self.Time, err = time.Parse(pythonTimeFormatStr, string(b))
return
}
func (self *PythonTime) MarshalJSON() ([]byte, error) {
return []byte(self.Time.Format(pythonTimeFormatStr)), nil
}
type OtherType struct {
Uuid string `json:"uuid`
Second PythonTime `json:"second"`
Location string `json:"location"`
Action string `json:"action"`
Duration int `json:"duration"`
Value string `json:"value"`
}
So the the above works fine for marshalling and unmarshalling JSON. However, for my library that I am using (gocql and cqlr) they are checking if the type is a time.Time type so they can make some other modifications before putting it in C*. How do I get my PythonTime type to equate to either show as time.Time or override the default marshalling/unmarshalling for a time.Time object just for the use of my OtherType objects?
My temporary solution has been to modify their code and add a special case for the PythonTime type that does the same thing as the time.Time type. However, this is causing me circular imports and is not the best solution. Here is their code with my modifications.
func marshalTimestamp(info TypeInfo, value interface{}) ([]byte, error) {
switch v := value.(type) {
case Marshaler:
return v.MarshalCQL(info)
case int64:
return encBigInt(v), nil
case time.Time:
if v.IsZero() {
return []byte{}, nil
}
x := int64(v.UTC().Unix()*1e3) + int64(v.UTC().Nanosecond()/1e6)
return encBigInt(x), nil
case models.PythonTime:
x := int64(v.UTC().Unix()*1e3) + int64(v.UTC().Nanosecond()/1e6)
return encBigInt(x), nil
}
if value == nil {
return nil, nil
}
rv := reflect.ValueOf(value)
switch rv.Type().Kind() {
case reflect.Int64:
return encBigInt(rv.Int()), nil
}
return nil, marshalErrorf("can not marshal %T into %s", value, info)
}
Don't do this. You're checking for a time.Time object when you should be checking that it satisfies an interface.
type TimeLike interface {
Day() int
Format(string) string
... // whatever makes a "time" object to your code!
// looks like in this case it's
UTC() time.Time
IsZero() bool
}
then any code that expects a time.Time that can be substituted with a PythonTime, expect a TimeLike instead.
function Foo(value interface{}) int {
switch v := value.(type) {
case TimeLike:
return v.Day() // works for either time.Time or models.PythonTime
}
return 0
}
Just like you have done with the json.Marshaler and json.Unamrshaler, you can also implement the gocql.Marshaler gocql.Unamrshaler interfaces.
func (t *PythonTime) MarshalCQL(info gocql.TypeInfo) ([]byte, error) {
b := make([]byte, 8)
x := t.UnixNano() / int64(time.Millisecond)
binary.BigEndian.PutUint64(b, uint64(x))
return b, nil
}
func (t *PythonTime) UnmarshalCQL(info gocql.TypeInfo, data []byte) error {
x := int64(binary.BigEndian.Uint64(data)) * int64(time.Millisecond)
t.Time = time.Unix(0, x)
return nil
}
(note, untested in the context of CQL, but this does round-trip with itself)
Unfortunately, that will not work in Go. Your best option would be to create some import and export methods, so that you can cast your PythonTime to a time.Time and vice versa. That will give you flexibility you desire along with compatibility with other libraries.
package main
import (
"fmt"
"reflect"
"time"
)
func main() {
p, e := NewFromTime(time.Now())
if e != nil {
panic(e)
}
v, e := p.MarshalJSON()
if e != nil {
panic(e)
}
fmt.Println(string(v), reflect.TypeOf(p))
t, e := p.GetTime()
if e != nil {
panic(e)
}
fmt.Println(t.String(), reflect.TypeOf(t))
}
type PythonTime struct {
time.Time
}
var pythonTimeFormatStr = "2006-01-02 15:04:05-0700"
func NewFromTime(t time.Time) (*PythonTime, error) {
b, e := t.GobEncode()
if e != nil {
return nil, e
}
p := new(PythonTime)
e = p.GobDecode(b)
if e != nil {
return nil, e
}
return p, nil
}
func (self *PythonTime) GetTime() (time.Time, error) {
return time.Parse(pythonTimeFormatStr, self.Format(pythonTimeFormatStr))
}
func (self *PythonTime) UnmarshalJSON(b []byte) (err error) {
// removes prepending/trailing " in the string
if b[0] == '"' && b[len(b)-1] == '"' {
b = b[1 : len(b)-1]
}
self.Time, err = time.Parse(pythonTimeFormatStr, string(b))
return
}
func (self *PythonTime) MarshalJSON() ([]byte, error) {
return []byte(self.Time.Format(pythonTimeFormatStr)), nil
}
That should give output like this:
2016-02-04 14:32:17-0700 *main.PythonTime
2016-02-04 14:32:17 -0700 MST time.Time

How to marshal struct when some members are protected/inner/hidden

How do ensure the fields in this LookupCode struct are included when marshalling?
package main
import (
"encoding/json"
"fmt"
)
type LookupCode struct {
code string `json:"code"`
name string `json:"name"`
}
func (l *LookupCode) GetCode() string {
return l.code
}
func main() {
c := &LookupCode{
code: "A",
name: "Apple",
}
b, _ := json.MarshalIndent(c, "", "\t")
fmt.Println(string(b))
}
http://play.golang.org/p/my52DAn0-Z
You can by implementing the json.Marshaller interface:
Full Example: http://play.golang.org/p/8mIcPwX92P
// Implement json.Unmarshaller
func (l *LookupCode) UnmarshalJSON(b []byte) error {
var tmp struct {
Code string `json:"code"`
Name string `json:"name"`
}
err := json.Unmarshal(b, &tmp)
if err != nil {
return err
}
l.code = tmp.Code
l.name = tmp.Name
return nil
}
func (l *LookupCode) MarshalJSON() ([]byte, error) {
return json.Marshal(struct {
Code string `json:"code"`
Name string `json:"name"`
}{
Code: l.code,
Name: l.name,
})
}
encode/json cannot marshal unexported fields. Change your code to:
type LookupCode struct {
Code string `json:"code"`
Name string `json:"name"`
}
and do the same wherever you use code or name.
Playground: http://play.golang.org/p/rak0nVCNGI
Edit
The limitation is due to the reflection used when marshalling the struct. If you need to keep your values unexported, you must implement the json.Marshaller interface and do the encoding manually.
if the struct has only string-type fields,you can try this hack way.
package main
import (
"fmt"
"reflect"
"github.com/bitly/go-simplejson"
)
type A struct {
name string `json:"name"`
code string `json:"code"`
}
func marshal(a A) ([]byte, error) {
j := simplejson.New()
va := reflect.ValueOf(&a)
vt := va.Elem()
types := reflect.TypeOf(a)
for i := 0; i < vt.NumField(); i++ {
j.Set(types.Field(i).Tag.Get("json"), fmt.Sprintf("%v", reflect.Indirect(va).Field(i)))
}
return j.MarshalJSON()
}
func main() {
a := A{name: "jessonchan", code: "abc"}
b, _ := marshal(a)
fmt.Println(string(b))
}

Resources