MongoDB is not marshaling a value before storing it - go

I'm using a custom JSON marshaller/unmarshaller for a mapping between integers and strings in Go. The problem is that values are being stored in the database as integers instead of strings. In the example below, I would expect this to be stored in the MongoDB database:
{ "_id" : "id123", "desc" : "Red Delicious", "value" : "apple" }
Instead I get:
{ "_id" : "id123", "desc" : "Red Delicious", "value" : 1 }
As the test shows, marshalling and unmarshalling are working fine. What's going on?
Here's an example as a Go test (save to unmarshal_test.go and "go test").
package testunmarshal
import (
"fmt"
"testing"
"encoding/json"
mgo "gopkg.in/mgo.v2"
"gopkg.in/mgo.v2/bson"
)
type Const int
const (
Apple Const = 1
Banana = 2
Cherry = 4
)
type Record struct {
Id string `bson:"_id" json:"id"`
Desc string `bson:"desc" json:"desc"`
Value Const `bson:"value" json:"value`
}
func (intValue Const) Code() string {
switch intValue {
case Apple: return "apple"
case Banana: return "banana"
case Cherry: return "cherry"
}
return "invalid"
}
func (intValue *Const) UnmarshalJSON(data []byte) (err error) {
switch string(data) {
case `"apple"`:
*intValue = Apple
case `"banana"`:
*intValue = Banana
case `"cherry"`:
*intValue = Cherry
default:
return fmt.Errorf("Invalid fruit %s", data)
}
return nil
}
func (intValue *Const) MarshalJSON() ([]byte, error) {
return []byte(fmt.Sprintf(`"%s"`, intValue.Code())), nil
}
func TestMarshalJSON(t *testing.T) {
var orig = Record {
Id: "id456",
Desc: "Cavendish",
Value: Banana,
}
var copy Record
bytes, err := json.Marshal(&orig)
if err != nil {
t.Errorf("Marshal failed: %s", err.Error())
return
}
err = json.Unmarshal(bytes, &copy)
if err != nil {
t.Errorf("Unmarshal failed: %s", err.Error())
return
}
if orig.Value != copy.Value {
t.Errorf("Expected %d=%s, got %d=%s", orig.Value, orig.Value.Code(), copy.Value, copy.Value.Code())
}
}
func TestMarshalBSON(t *testing.T) {
var orig = Record {
Id: "id456",
Desc: "Cavendish",
Value: Banana,
}
var copy Record
bytes, err := bson.Marshal(&orig)
if err != nil {
t.Errorf("Marshal failed: %s", err.Error())
return
}
err = bson.Unmarshal(bytes, &copy)
if err != nil {
t.Errorf("Unmarshal failed: %s", err.Error())
return
}
if orig.Value != copy.Value {
t.Errorf("Expected %d=%s, got %d=%s", orig.Value, orig.Value.Code(), copy.Value, copy.Value.Code())
}
}
func TestMongo(t *testing.T) {
var rec1 = Record {
Id: "id123",
Desc: "Red Delicious",
Value: Apple,
}
var rec2 Record
sess, err := mgo.Dial("localhost")
if err != nil {
t.Errorf(err.Error())
return
}
db := sess.DB("test")
if db == nil {
t.Fatal("Failed to connect to database")
return
}
col := db.C("fruit")
if col == nil {
t.Fatal("Failed to open collection")
return
}
// defer col.DropCollection()
err = col.Insert(&rec1)
if err != nil {
t.Fatal("Failed to insert: %s", err.Error())
return
}
err = col.Find(bson.M{}).One(&rec2)
if err != nil {
t.Fatal("Failed to retrieve stored object: %s", err.Error())
return
}
if rec1.Value != rec2.Value {
t.Errorf("Expected %d=%s, got %d=%s", rec1.Value, rec1.Value.Code(), rec1.Value, rec2.Value.Code())
}
}
Edit: Added more tests to demonstrate that marshalling and unmarshalling are working.

The bson encoder does not use the JSON marshaling interfaces. Implement the Getter interface:
func (intValue Const) GetBSON() (interface{}, error) {
return intValue.Code(), nil
}
You will also want to implement the Setter interface.
func (intValue *Const) SetBSON(raw bson.Raw) error {
var data int
if err := raw.Unmarshal(&data); err != nil {
return err
}
switch data {
case `"apple"`:
*intValue = Apple
case `"banana"`:
*intValue = Banana
case `"cherry"`:
*intValue = Cherry
default:
return fmt.Errorf("Invalid fruit %s", data)
}
return nil
}

Related

Elegant way to eliminate multiple condition checks in function

There are multiple condition checks in multiple functions
type VA struct {
A string
}
func (va *VA) CheckA(s string) error {
if s != va.A {
return errors.New("invalid str ")
}
return nil
}
type VB struct {
B int
}
func (vb *VB) CheckB(i int) error {
if i == vb.B {
return errors.New("invalid int")
}
return nil
}
func FuncA(s string, i int) error {
a := &VA{A: "testa"}
errA := a.CheckA(s)
if errA != nil {
return errA
}
b := &VB{B: 3}
errB := b.CheckB(i)
if errB != nil {
return errB
}
// more logic ...
return nil
}
func FuncB(sb string, v int32) error {
a := &VA{A: "testb"}
errA := a.CheckA(sb)
if errA != nil {
return errA
}
// more logic ...
return nil
}
func FuncC(sc string, vv int) error {
b := &VB{B: 3}
errB := b.CheckB(vv)
if errB != nil {
return errB
}
// more logic ...
return nil
}
We do CheckA and CheckB in function FuncA and do CheckA in function FuncB. However, only do CheckB in function FuncC. There is one pitfall that when the return value of CheckA is changed, both FuncA and FuncB would be changed.
We want to refactor the above codes. Is there any elegant way to do that in Golang?
What we have tried, combine CheckA and CheckB in one function ValidateFunc like below
type VALIDATE int
const (
_ VALIDATE = 1 << iota
VALIDATEA
VALIDATEB
)
func ValidateFunc(vs, s string, vi, i int, validate VALIDATE) error {
if validate&VALIDATEA == VALIDATEA {
a := &VA{A: vs}
errA := a.CheckA(s)
if errA != nil {
return errA
}
}
if validate&VALIDATEB == VALIDATEB {
b := &VB{B: vi}
errB := b.CheckB(i)
if errB != nil {
return errB
}
}
return nil
}
func FuncA(s string, i int) error {
err := ValidateFunc("testa", s, 3, i, VALIDATEA|VALIDATEB)
if err != nil {
return err
}
// more logic ...
return nil
}
Refer to Option pattern, it seems the codes will be simpler than before.
type Validator struct {
A VA
B VB
}
type Validate func(v *Validator) error
func WithVA(s string) Validate {
return func(v *Validator) error {
if err := v.A.CheckA(s); err != nil {
return err
}
return nil
}
}
func WithVB(i int) Validate {
return func(v *Validator) error {
if err := v.B.CheckB(i); err != nil {
return err
}
return nil
}
}
func DoValidate(vs string, vi int, vals ...func(v *Validator) error) error {
v := &Validator{A: VA{A: vs}, B: VB{B: vi}}
for _, val := range vals {
if err := val(v); err != nil {
return err
}
}
return nil
}
func FuncA(s string, i int) error {
err := DoValidate("testa", 3, WithVA(s), WithVB(i))
if err != nil {
return err
}
// more logic ...
return nil
}
Perhaps combine VA & VB into a new struct
Then validate that one?

How to receive a json to insert

I am receiving some values per post and I have a json type field but it arrives empty and if I enter a normal text it works and I do not see the error in the field
the model was updated so that it receives the fields and allows inserting in mysql
POSTman
{
"Code":"1234",//it works
"Desc":"desc",//it works
"Config":{"link":"https://stackoverflow.com/" }, //not works
"Dev":[ {"item":1},{"item":2}]//not works
}
type User struct {
gorm.Model
Code string `gorm:"type:varchar(100);unique_index"`
Desc string `gorm:"type:varchar(255);"`
Config JSON `json:"currencies" gorm:"type:varchar(255);"`
Dev JSON `json:"currencies" gorm:"type:varchar(255);"`
}
func CreateUser(c *gin.Context) {
var usuario models.User
var bodyBytes []byte
if c.Request.Body != nil {
bodyBytes, _ = ioutil.ReadAll(c.Request.Body)
}
data := bytes.NewBuffer(bodyBytes)
fmt.Println(data.Config)
c.BindJSON(&usuario)
db.DB.Create(&usuario)
c.JSON(200, usuario)
}
Model update. receive post form with json fields and insert in mysql
package models
import (
"bytes"
"database/sql/driver"
"errors"
)
type JSON []byte
func (j JSON) Value() (driver.Value, error) {
if j.IsNull() {
return nil, nil
}
return string(j), nil
}
func (j *JSON) Scan(value interface{}) error {
if value == nil {
*j = nil
return nil
}
s, ok := value.([]byte)
if !ok {
errors.New("error")
}
*j = append((*j)[0:0], s...)
return nil
}
func (m JSON) MarshalJSON() ([]byte, error) {
if m == nil {
return []byte("null"), nil
}
return m, nil
}
func (m *JSON) UnmarshalJSON(data []byte) error {
if m == nil {
return errors.New("error")
}
*m = append((*m)[0:0], data...)
return nil
}
func (j JSON) IsNull() bool {
return len(j) == 0 || string(j) == "null"
}
func (j JSON) Equals(j1 JSON) bool {
return bytes.Equal([]byte(j), []byte(j1))
}
Thank you very much to everyone who helped me, I consider that the functionality of receiving a json and saving it in mysql is very common and this can be useful to many people
You can change the JSON like below or You can change the Struct like below (I prefer struct approach)
{
"Code": "1234",
"Desc": "desc",
"Config": {
"Link": "https://stackoverflow.com/"
},
"Dev": [
{
"Item": 1
},
{
"Item": 2
}
]
}
Struct:
type User struct {
gorm.Model
Code string `json:"Code" gorm:"type:varchar(100);unique_index"`
Desc string `json:"Desc" gorm:"type:varchar(255);"`
Config []struct {
Link string `json:"link" gorm:"type:varchar(255);"`
Title string `json:"title" gorm:"type:varchar(255);"`
}
Dev []struct {
Item string `json:"item" gorm:"type:varchar(255);"`
}
}
You have made two kind of mistakes
Your json decoding cannot work because your struct does not match your json. Config is defined as a array of something but in your json you have an object not array, and in Dev the property Item is a int not a string
Your model may not be well defined as you have not defined you joined table. Well I never seen a working example with this kind of definition. I suggest you to declare your nested struct as independent struct.
Here a full working example :
package main
import (
"database/sql"
"encoding/json"
"fmt"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
)
const data = `{
"Code":"1234",
"Desc":"desc",
"Config":{"link":"https://stackoverflow.com/" },
"Dev":[ {"item":1},{"item":2}]
}`
type Config struct {
Id int `gorm:"primaryKey"`
Link string `json:"link"`
Title string
UserId int
}
type Item struct {
Id int `gorm:"primaryKey"`
Item int `json:"item"`
UserId int
}
type User struct {
Id int `gorm:"primaryKey"`
Code string
Desc string
Config Config `gorm:"foreignkey:UserId"`
Dev []Item `gorm:"foreignkey:UserId"`
}
func initDb(url string) (*gorm.DB, *sql.DB, error) {
connexion := sqlite.Open(url)
db, err := gorm.Open(connexion, &gorm.Config{})
if err != nil {
return nil, nil, err
}
sql, err := db.DB()
if err != nil {
return nil, nil, err
}
err = db.AutoMigrate(&User{})
if err != nil {
return nil, nil, err
}
err = db.AutoMigrate(&Item{})
if err != nil {
return nil, nil, err
}
err = db.AutoMigrate(&Config{})
if err != nil {
return nil, nil, err
}
return db, sql, nil
}
func run() error {
db, sql, err := initDb("file::memory:?cache=shared")
if err != nil {
return err
}
defer sql.Close()
var user User
err = json.Unmarshal([]byte(data), &user)
fmt.Printf("%#v\n", user)
err = db.Create(&user).Error
if err != nil {
return err
}
var loaded User
db.Preload("Config").Preload("Dev").First(&loaded)
fmt.Printf("%#v\n", loaded)
return nil
}
func main() {
if err := run(); err != nil {
fmt.Println("failed", err)
}
}
try adding this JSON Field in your model
import (
"errors"
"database/sql/driver"
"encoding/json"
)
// JSON Interface for JSON Field of yourTableName Table
type JSON interface{}
// Value Marshal
func (a JSON) Value() (driver.Value, error) {
return json.Marshal(a)
}
// Scan Unmarshal
func (a *JSON) Scan(value interface{}) error {
b, ok := value.([]byte)
if !ok {
return errors.New("type assertion to byte failed")
}
return json.Unmarshal(b,&a)
}
All these answers didn't work for me, but this will work for everyone
Model
// This is the max Thing you need
import "gorm.io/datatypes"
import "encoding/json"
type CMSGenericModel struct {
gorm.Model
//... Other Posts
ExtraData datatypes.JSON `json:"data"`
}
In Handler Function
type CmsReqBody struct {
// ============= RAW ========
Data json.RawMessage `json:"data"`
// other props...
}
cmsBodyRecord := new(models.CMSGenericModel)
cmsBodyPayload := new(CmsReqBody)
if err := c.BodyParser(cmsBodyPayload); err != nil {
return c.Status(503).SendString(err.Error())
}
cmsBodyRecord.ExtraData = datatypes.JSON(cmsBodyPayload.Data)
My Sample Data
{
"title": "Blog Post 1",
"subtitle": "first",
"description": "Updated",
"type": "blog",
"isActive": true,
"uuid": "new",
"data": {
"complex1": ["kkkk", "yyyy"],
"complex2": [
{
"name": "sourav"
},
{
"name": "yahooo"
},
{
"yahoo": "name",
"kjk": ["abbsb", {"data": "abcd"}]
}
]
}
}

Unmarshalling YAML to ordered maps

I am trying to unmarshal the following YAML with Go YAML v3.
model:
name: mymodel
default-children:
- payment
pipeline:
accumulator_v1:
by-type:
type: static
value: false
result-type:
type: static
value: 3
item_v1:
amount:
type: schema-path
value: amount
start-date:
type: schema-path
value: start-date
Under pipeline is an arbitrary number of ordered items. The struct to which this should be unmarshalled looks like this:
type PipelineItemOption struct {
Type string
Value interface{}
}
type PipelineItem struct {
Options map[string]PipelineItemOption
}
type Model struct {
Name string
DefaultChildren []string `yaml:"default-children"`
Pipeline orderedmap[string]PipelineItem // "pseudo code"
}
How does this work with Golang YAML v3? In v2 there was MapSlice, but that is gone in v3.
You claim that marshaling to an intermediate yaml.Node is highly non-generic, but I don't really see why. It looks like this:
package main
import (
"fmt"
"gopkg.in/yaml.v3"
)
type PipelineItemOption struct {
Type string
Value interface{}
}
type PipelineItem struct {
Name string
Options map[string]PipelineItemOption
}
type Pipeline []PipelineItem
type Model struct {
Name string
DefaultChildren []string `yaml:"default-children"`
Pipeline Pipeline
}
func (p *Pipeline) UnmarshalYAML(value *yaml.Node) error {
if value.Kind != yaml.MappingNode {
return fmt.Errorf("pipeline must contain YAML mapping, has %v", value.Kind)
}
*p = make([]PipelineItem, len(value.Content)/2)
for i := 0; i < len(value.Content); i += 2 {
var res = &(*p)[i/2]
if err := value.Content[i].Decode(&res.Name); err != nil {
return err
}
if err := value.Content[i+1].Decode(&res.Options); err != nil {
return err
}
}
return nil
}
var input []byte = []byte(`
model:
name: mymodel
default-children:
- payment
pipeline:
accumulator_v1:
by-type:
type: static
value: false
result-type:
type: static
value: 3
item_v1:
amount:
type: schema-path
value: amount
start-date:
type: schema-path
value: start-date`)
func main() {
var f struct {
Model Model
}
var err error
if err = yaml.Unmarshal(input, &f); err != nil {
panic(err)
}
fmt.Printf("%v", f)
}
For me it was a bit of a learning curve to figure out what v3 expects instead of MapSlice. Similar to answer from #flyx, the yaml.Node tree needs to be walked, particularly its []Content.
Here is a utility to provide an ordered map[string]interface{} that is a little more reusable and tidy. (Though it is not as constrained as the question specified.)
Per structure above, redefine Pipeline generically:
type Model struct {
Name string
DefaultChildren []string `yaml:"default-children"`
Pipeline *yaml.Node
}
Use a utility fn to traverse yaml.Node content:
// fragment
var model Model
if err := yaml.Unmarshal(&model) ; err != nil {
return err
}
om, err := getOrderedMap(model.Pipeline)
if err != nil {
return err
}
for _,k := range om.Order {
v := om.Map[k]
fmt.Printf("%s=%v\n", k, v)
}
The utility fn:
type OrderedMap struct {
Map map[string]interface{}
Order []string
}
func getOrderedMap(node *yaml.Node) (om *OrderedMap, err error) {
content := node.Content
end := len(content)
count := end / 2
om = &OrderedMap{
Map: make(map[string]interface{}, count),
Order: make([]string, 0, count),
}
for pos := 0 ; pos < end ; pos += 2 {
keyNode := content[pos]
valueNode := content[pos + 1]
if keyNode.Tag != "!!str" {
err = fmt.Errorf("expected a string key but got %s on line %d", keyNode.Tag, keyNode.Line)
return
}
var k string
if err = keyNode.Decode(&k) ; err != nil {
return
}
var v interface{}
if err = valueNode.Decode(&v) ; err != nil {
return
}
om.Map[k] = v
om.Order = append(om.Order, k)
}
return
}
Building from #jws's solution and adding recursion:
func Encode(obj any) (string, error) {
var buffer bytes.Buffer
yamlEncoder := yaml.NewEncoder(&buffer)
yamlEncoder.SetIndent(2)
encodeErr := yamlEncoder.Encode(obj)
if encodeErr != nil {
return "", encodeErr
}
return buffer.String(), nil
}
type OrderedMap struct {
Map map[string]interface{}
Order []string
}
func (om *OrderedMap) MarshalYAML() (interface{}, error) {
node, err := EncodeDocumentNode(om)
if err != nil {
return nil, err
}
return node.Content[0], nil
}
// DecodeDocumentNode decodes a root yaml node into an OrderedMap
func DecodeDocumentNode(node *yaml.Node) (*OrderedMap, error) {
if node.Kind != yaml.DocumentNode {
return nil, fmt.Errorf("node %v is not a document node", node)
}
om, err := decodeMap(node.Content[0])
if err != nil {
return nil, err
}
return om, err
}
func decode(node *yaml.Node) (any, error) {
switch node.Tag {
case "!!null":
return decodeNull(node)
case "!!str":
return decodeStr(node)
case "!!map":
return decodeMap(node)
case "!!seq":
return decodeSeq(node)
default:
return nil, fmt.Errorf("unknown node tag %s", node.Tag)
}
}
func decodeNull(_ *yaml.Node) (any, error) {
return nil, nil
}
func decodeStr(node *yaml.Node) (string, error) {
var s string
if err := node.Decode(&s); err != nil {
return "", fmt.Errorf("decode error for %v: %v", node, err)
}
return s, nil
}
func decodeMap(node *yaml.Node) (*OrderedMap, error) {
keyValuePairs := lo.Map(lo.Chunk(node.Content, 2), func(c []*yaml.Node, _ int) mo.Result[lo.Entry[string, any]] {
if len(c) != 2 {
return mo.Err[lo.Entry[string, any]](fmt.Errorf("invalid yaml; expected key/value pair"))
}
keyNode := c[0]
valueNode := c[1]
if keyNode.Tag != "!!str" {
return mo.Err[lo.Entry[string, any]](fmt.Errorf("expected a string key but got %s on line %d", keyNode.Tag, keyNode.Line))
}
key, err := decodeStr(keyNode)
if err != nil {
return mo.Err[lo.Entry[string, any]](fmt.Errorf("key decode error: %v", err))
}
value, err := decode(valueNode)
if err != nil {
return mo.Err[lo.Entry[string, any]](fmt.Errorf("value decode error: %v", err))
}
return mo.Ok(lo.Entry[string, any]{
Key: key,
Value: value,
})
})
validErrGroups := lo.GroupBy(keyValuePairs, func(kvp mo.Result[lo.Entry[string, any]]) bool {
return kvp.IsOk()
})
errs := validErrGroups[false]
if len(errs) != 0 {
return nil, fmt.Errorf("%v", lo.Map(errs, func(e mo.Result[lo.Entry[string, any]], _ int) error {
return e.Error()
}))
}
kvps := lo.Map(validErrGroups[true], func(kvp mo.Result[lo.Entry[string, any]], _ int) lo.Entry[string, any] {
return kvp.MustGet()
})
return &OrderedMap{
Map: lo.FromEntries(kvps),
Order: lo.Map(kvps, func(kvp lo.Entry[string, any], _ int) string {
return kvp.Key
}),
}, nil
}
func decodeSeq(node *yaml.Node) ([]*OrderedMap, error) {
seq := lo.Map(node.Content, func(n *yaml.Node, _ int) mo.Result[*OrderedMap] {
return mo.Try(func() (*OrderedMap, error) {
return decodeMap(n)
})
})
validErrGroups := lo.GroupBy(seq, func(kvp mo.Result[*OrderedMap]) bool {
return kvp.IsOk()
})
errs := validErrGroups[false]
if len(errs) != 0 {
return nil, fmt.Errorf("%v", lo.Map(errs, func(e mo.Result[*OrderedMap], _ int) error {
return e.Error()
}))
}
oms := validErrGroups[true]
return lo.Map(oms, func(om mo.Result[*OrderedMap], _ int) *OrderedMap {
return om.MustGet()
}), nil
}
// EncodeDocumentNode encodes an OrderedMap into a root yaml node
func EncodeDocumentNode(om *OrderedMap) (*yaml.Node, error) {
node, err := encodeMap(om)
if err != nil {
return nil, err
}
return &yaml.Node{
Kind: yaml.DocumentNode,
Content: []*yaml.Node{node},
Line: 1,
Column: 1,
}, nil
}
func encode(x any) (*yaml.Node, error) {
if x == nil {
return encodeNull()
}
switch reflect.ValueOf(x).Kind() {
case reflect.String:
return encodeStr(x.(string))
case reflect.Ptr:
return encodeMap(x.(*OrderedMap))
case reflect.Slice:
return encodeSeq(x.([]*OrderedMap))
default:
return nil, fmt.Errorf("unable to encode %v with kind %v", x, reflect.ValueOf(x).Kind())
}
}
func encodeNull() (*yaml.Node, error) {
return &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!null",
}, nil
}
func encodeStr(s string) (*yaml.Node, error) {
return &yaml.Node{
Kind: yaml.ScalarNode,
Tag: "!!str",
Value: s,
}, nil
}
func encodeMap(om *OrderedMap) (*yaml.Node, error) {
content := lo.FlatMap(om.Order, func(key string, _ int) []mo.Result[*yaml.Node] {
return []mo.Result[*yaml.Node]{
mo.Try(func() (*yaml.Node, error) {
return encodeStr(key)
}),
mo.Try(func() (*yaml.Node, error) {
return encode(om.Map[key])
}),
}
})
validErrGroups := lo.GroupBy(content, func(kvp mo.Result[*yaml.Node]) bool {
return kvp.IsOk()
})
errs := validErrGroups[false]
if len(errs) != 0 {
return nil, fmt.Errorf("%v", lo.Map(errs, func(e mo.Result[*yaml.Node], _ int) error {
return e.Error()
}))
}
nodes := validErrGroups[true]
return &yaml.Node{
Kind: yaml.MappingNode,
Tag: "!!map",
Content: lo.Map(nodes, func(c mo.Result[*yaml.Node], _ int) *yaml.Node {
return c.MustGet()
}),
}, nil
}
func encodeSeq(oms []*OrderedMap) (*yaml.Node, error) {
content := lo.Map(oms, func(om *OrderedMap, _ int) mo.Result[*yaml.Node] {
return mo.Try(func() (*yaml.Node, error) {
return encodeMap(om)
})
})
validErrGroups := lo.GroupBy(content, func(kvp mo.Result[*yaml.Node]) bool {
return kvp.IsOk()
})
errs := validErrGroups[false]
if len(errs) != 0 {
return nil, fmt.Errorf("%v", lo.Map(errs, func(e mo.Result[*yaml.Node], _ int) error {
return e.Error()
}))
}
nodes := validErrGroups[true]
return &yaml.Node{
Kind: yaml.SequenceNode,
Tag: "!!seq",
Content: lo.Map(nodes, func(c mo.Result[*yaml.Node], _ int) *yaml.Node {
return c.MustGet()
}),
}, nil
}
End-to-end test:
func TestDecodeEncodeE2E(t *testing.T) {
y := heredoc.Doc(`
root:
outer-key-4:
- inner-key-7:
key-8: value-8
key-9: value-9
- inner-key-10:
key-11: value-11
key-12: value-12
outer-key-3:
- inner-key-5: inner-value-5
- inner-key-6: inner-value-6
outer-key-1:
inner-key-1: inner-value-1
inner-key-2: inner-value-2
outer-key-2:
inner-key-3: inner-value-3
inner-key-4: inner-value-4
key-1: value-1
key-2: value-2
`)
var documentNode yaml.Node
err := yaml.Unmarshal([]byte(y), &documentNode)
require.NoError(t, err)
decodeActual, decodeErr := DecodeDocumentNode(&documentNode)
require.NoError(t, decodeErr)
stringifiedOrderedMap, stringifiedOrderedMapErr := Encode(decodeActual)
assert.NoError(t, stringifiedOrderedMapErr)
assert.Equal(t, y, stringifiedOrderedMap)
encodeActual, encodeErr := EncodeDocumentNode(decodeActual)
require.NoError(t, encodeErr)
// for troubleshooting purposes; commented out because lines and columns don't match
// assert.Equal(t, &documentNode, encodeActual)
stringifiedNode, stringifiedNodeErr := Encode(encodeActual)
assert.NoError(t, stringifiedNodeErr)
assert.Equal(t, y, stringifiedNode)
}

How to extend go-yaml to support custom tags

I have spent some time reading the code and docs of go-yaml, but I have not found any way to do this, except forking the project..
I want to extend the YAML unmarshaller so that it can accept a custom YAML tag (!include <file> in this case), which in turn would allow me to add support for including files. This is easily implemented with other YAML libraries, like in this answer.
Is there any way to accomplish this, using the public interface of the library (or another yaml library)?
Yes, this is possible (since v3). You can load the whole YAML file into a yaml.Node and then walk over the structure. The trick is that yaml.Node is an intermediate representation which you can only access if you define an unmarshaler.
For example:
package main
import (
"errors"
"fmt"
"io/ioutil"
"gopkg.in/yaml.v3"
)
// used for loading included files
type Fragment struct {
content *yaml.Node
}
func (f *Fragment) UnmarshalYAML(value *yaml.Node) error {
var err error
// process includes in fragments
f.content, err = resolveIncludes(value)
return err
}
type IncludeProcessor struct {
target interface{}
}
func (i *IncludeProcessor) UnmarshalYAML(value *yaml.Node) error {
resolved, err := resolveIncludes(value)
if err != nil {
return err
}
return resolved.Decode(i.target)
}
func resolveIncludes(node *yaml.Node) (*yaml.Node, error) {
if node.Tag == "!include" {
if node.Kind != yaml.ScalarNode {
return nil, errors.New("!include on a non-scalar node")
}
file, err := ioutil.ReadFile(node.Value)
if err != nil {
return nil, err
}
var f Fragment
err = yaml.Unmarshal(file, &f)
return f.content, err
}
if node.Kind == yaml.SequenceNode || node.Kind == yaml.MappingNode {
var err error
for i := range node.Content {
node.Content[i], err = resolveIncludes(node.Content[i])
if err != nil {
return nil, err
}
}
}
return node, nil
}
type MyStructure struct {
// this structure holds the values you want to load after processing
// includes, e.g.
Num int
}
func main() {
var s MyStructure
yaml.Unmarshal([]byte("!include foo.yaml"), &IncludeProcessor{&s})
fmt.Printf("Num: %v", s.Num)
}
Code prints Num: 42 when a file foo.yaml exists with the content num: 42.
Modified #flyx's original code a little to make it modular for adding custom resolvers.
package main
import (
"errors"
"fmt"
"io/ioutil"
"os"
"gopkg.in/yaml.v3"
)
var tagResolvers = make(map[string]func(*yaml.Node) (*yaml.Node, error))
type Fragment struct {
content *yaml.Node
}
func (f *Fragment) UnmarshalYAML(value *yaml.Node) error {
var err error
// process includes in fragments
f.content, err = resolveTags(value)
return err
}
type CustomTagProcessor struct {
target interface{}
}
func (i *CustomTagProcessor) UnmarshalYAML(value *yaml.Node) error {
resolved, err := resolveTags(value)
if err != nil {
return err
}
return resolved.Decode(i.target)
}
func resolveTags(node *yaml.Node) (*yaml.Node, error) {
for tag, fn := range tagResolvers {
if node.Tag == tag {
return fn(node)
}
}
if node.Kind == yaml.SequenceNode || node.Kind == yaml.MappingNode {
var err error
for i := range node.Content {
node.Content[i], err = resolveTags(node.Content[i])
if err != nil {
return nil, err
}
}
}
return node, nil
}
func resolveIncludes(node *yaml.Node) (*yaml.Node, error) {
if node.Kind != yaml.ScalarNode {
return nil, errors.New("!include on a non-scalar node")
}
file, err := ioutil.ReadFile(node.Value)
if err != nil {
return nil, err
}
var f Fragment
err = yaml.Unmarshal(file, &f)
return f.content, err
}
func resolveGetValueFromEnv(node *yaml.Node) (*yaml.Node, error) {
if node.Kind != yaml.ScalarNode {
return nil, errors.New("!getValueFromEnv on a non-scalar node")
}
value := os.Getenv(node.Value)
if value == "" {
return nil, fmt.Errorf("environment variable %v not set", node.Value)
}
var f Fragment
err := yaml.Unmarshal([]byte(value), &f)
return f.content, err
}
func AddResolvers(tag string, fn func(*yaml.Node) (*yaml.Node, error)) {
tagResolvers[tag] = fn
}
func main() {
// Register custom tag resolvers
AddResolvers("!include", resolveIncludes)
AddResolvers("!getValueFromEnv", resolveGetValueFromEnv)
type MyStructure struct {
// this structure holds the values you want to load after processing
// includes, e.g.
Num int
}
var s MyStructure
os.Setenv("FOO", `{"num": 42}`)
err := yaml.Unmarshal([]byte("!getValueFromEnv FOO"), &CustomTagProcessor{&s})
if err != nil {
panic("Error encountered during unmarshalling")
}
fmt.Printf("\nNum: %v", s.Num)
err = yaml.Unmarshal([]byte("!include foo.yaml"), &CustomTagProcessor{&s})
if err != nil {
panic("Error encountered during unmarshalling")
}
fmt.Printf("\nNum: %v", s.Num)
}

Golang Gorilla Websocket stops receiving information at 120 seconds

I'm currently trying to connect to the CEX.IO bitcoin exchange's websocket, but have been having issues not only with CEX.IO but with others too. All of my connections drop around the 120-second mark which makes me think there is some TTL problem going on. The Process() goroutine in the main package ends up just hanging and waiting for data from the readLoop which just stops receiving data. I've included some read-only API keys in the code so you can test if you'd like.
package main
import (
"fmt"
"bitbucket.org/tradedefender/cryptocurrency/exchange-connector/cexio"
"github.com/shopspring/decimal"
"encoding/json"
"time"
)
type OrderBook struct {
Asks []Ask
Bids []Bid
}
type Ask struct {
Rate decimal.Decimal
Amount decimal.Decimal
}
type Bid struct {
Rate decimal.Decimal
Amount decimal.Decimal
}
func main() {
cexioConn := new(cexio.Connection)
err := cexioConn.Connect()
if err != nil {
fmt.Errorf("error: %s", err.Error())
}
err = cexioConn.Authenticate("TLwYkktLf7Im6nqSKt6UO1IrU", "9ImOJcR7Qj3LMIyPCzky0D7WE")
if err != nil {
fmt.Errorf("error: %s", err.Error())
}
readChannel := make(chan cexio.IntraAppMessage, 25)
go cexioConn.ReadLoop(readChannel)
processor := Processor{
WatchPairs: [][2]string{
[2]string{
"BTC", "USD",
},
},
conn: cexioConn,
}
go processor.Process(readChannel)
// LOL
for {
continue
}
}
type Processor struct {
WatchPairs [][2]string
conn *cexio.Connection
}
func (p *Processor) Process(ch <-chan cexio.IntraAppMessage) {
p.conn.SubscribeToOrderBook(p.WatchPairs[0])
pingTimer := time.Now().Unix()
for {
fmt.Printf("(%v)\n", time.Now().Unix())
if (time.Now().Unix() - pingTimer) >= 10 {
fmt.Println("sending ping")
p.conn.SendPing()
pingTimer = time.Now().Unix()
}
readMsg := <- ch
output, _ := json.Marshal(readMsg.SocketMessage)
fmt.Println(string(output))
if readMsg.SocketMessage.Event == "ping" {
fmt.Println("sending pong")
p.conn.SendPong()
pingTimer = time.Now().Unix()
}
}
}
Below is the connector to the cexio websocket. Here is a link to their API: https://cex.io/websocket-api
package cexio
import (
"github.com/gorilla/websocket"
//"github.com/shopspring/decimal"
"github.com/satori/go.uuid"
"encoding/hex"
"encoding/json"
"crypto/hmac"
"crypto/sha256"
"bytes"
"strconv"
"time"
"fmt"
)
const Url = "wss://ws.cex.io/ws/"
type Connection struct {
conn *websocket.Conn
}
type IntraAppMessage struct {
SocketMessage GenericMessage
ProgramMessage ProgramMessage
}
type GenericMessage struct {
Event string `json:"e"`
Data interface{} `json:"data"`
Auth AuthData `json:"auth,omitempty"`
Ok string `json:"ok,omitempty"`
Oid string `json:"oid,omitempty"`
Time int64 `json:"time,omitempty"`
}
type ProgramMessage struct {
Error string
}
type AuthData struct {
Key string `json:"key"`
Signature string `json:"signature"`
Timestamp int64 `json:"timestamp"`
}
type OrderBookSubscribeData struct {
Pair [2]string `json:"pair"`
Subscribe bool `json:"subscribe"`
Depth int `json:"depth"`
}
func (c *Connection) SendPong() error {
pongMsg := GenericMessage{
Event: "pong",
}
err := c.conn.WriteJSON(pongMsg)
if err != nil {
return nil
}
deadline := time.Now().Add(15*time.Second)
err = c.conn.WriteControl(websocket.PongMessage, nil, deadline)
if err != nil {
return err
}
return nil
}
func (c *Connection) SendPing() error {
pingMsg := GenericMessage{
Event: "get-balance",
Oid: uuid.NewV4().String(),
}
err := c.conn.WriteJSON(pingMsg)
if err != nil {
return err
}
deadline := time.Now().Add(15*time.Second)
err = c.conn.WriteControl(websocket.PingMessage, nil, deadline)
if err != nil {
return err
}
return nil
}
func (c *Connection) Connect() error {
dialer := *websocket.DefaultDialer
wsConn, _, err := dialer.Dial(Url, nil)
if err != nil {
return err
}
c.conn = wsConn
//c.conn.SetPingHandler(c.HandlePing)
for {
_, msgBytes, err := c.conn.ReadMessage()
if err != nil {
c.Disconnect()
return err
}
fmt.Println(string(msgBytes))
var m GenericMessage
err = json.Unmarshal(msgBytes, &m)
if err != nil {
c.Disconnect()
return err
}
if m.Event != "connected" {
c.Disconnect()
return err
} else {
break
}
}
return nil
}
func (c *Connection) Disconnect() error {
return c.conn.Close()
}
func (c *Connection) ReadLoop(ch chan<- IntraAppMessage) {
for {
fmt.Println("starting new read")
_, msgBytes, err := c.conn.ReadMessage()
if err != nil {
ch <- IntraAppMessage{
ProgramMessage: ProgramMessage{
Error: err.Error(),
},
}
continue
}
var m GenericMessage
err = json.Unmarshal(msgBytes, &m)
if err != nil {
ch <- IntraAppMessage{
ProgramMessage: ProgramMessage{
Error: err.Error(),
},
}
continue
}
ch <- IntraAppMessage{
SocketMessage: m,
}
}
}
func CreateSignature(timestamp int64, key, secret string) string {
secretBytes := []byte(secret)
h := hmac.New(sha256.New, secretBytes)
var buffer bytes.Buffer
buffer.WriteString(strconv.FormatInt(timestamp, 10))
buffer.WriteString(key)
h.Write(buffer.Bytes())
return hex.EncodeToString(h.Sum(nil))
}
func (c *Connection) Authenticate(key, secret string) error {
timestamp := time.Now().Unix()
signature := CreateSignature(timestamp, key, secret)
var authMsg GenericMessage
authMsg.Event = "auth"
authMsg.Auth = AuthData{
Key: key,
Signature: signature,
Timestamp: timestamp,
}
err := c.conn.WriteJSON(authMsg)
if err != nil {
return err
}
for {
_, msgBytes, err := c.conn.ReadMessage()
if err != nil {
c.Disconnect()
return err
}
fmt.Println(string(msgBytes))
var m GenericMessage
err = json.Unmarshal(msgBytes, &m)
if err != nil {
c.Disconnect()
return err
}
if m.Event != "auth" && m.Ok != "ok" {
c.Disconnect()
return err
} else {
break
}
}
return nil
}
func (c *Connection) SubscribeToOrderBook(pair [2]string) error {
sendMsg := GenericMessage{
Event: "order-book-subscribe",
Data: OrderBookSubscribeData{
Pair: pair,
Subscribe: true,
Depth: 0,
},
Oid: uuid.NewV4().String(),
}
err := c.conn.WriteJSON(sendMsg)
if err != nil {
return err
}
return nil
}
func (c *Connection) GetBalance() error {
sendMsg := GenericMessage{
Event: "get-balance",
Oid: uuid.NewV4().String(),
}
err := c.conn.WriteJSON(sendMsg)
if err != nil {
return err
}
return nil
}
Solution was to remove the
for {
continue
}
at the end of the main function

Resources