I am trying to convert an old version jsonline into a new one (with a different structure).
Now the old file has the following structure
{"user": "myname", "uuid": "1242315425", "data": {"Niveau1": ["AUTRE", "RC"], "Niveau2": ["RA06"], "Niveau3": ["RA06_01"]}}
however, Niveau2 and Niveau3 are not always present and the length of the lists is not always the same.
The new file has a more complicated structure
{"user": "myname", "uuid": "1242315425","annotation":{"classifications":{"Niveau1":{"labels":[{"value":"AUTRE"}, {"value":"RC"}]}, "Niveau2": {"labels": [{"value":"RA06"}], "Niveau3": {"labels": [{"value":"RA06_01"}]}}}}
What I have done so far is (after parsing the files in appropriate structures) the following function
func convert(oldAnnots []AnnotV1) (newAnnots []AnnotV2) {
for _, element := range oldAnnots {
var newAnnot AnnotV2
newAnnot.User = element.User
newAnnot.Uuid = element.Uuid
if element.Data.Niveau1 != nil {
for i, annot1 := range element.Data.Niveau1 {
newAnnot.Annotation.Classif.Niveau1.Labels[i].Value = annot1
}
}
if element.Data.Niveau2 != nil {
for j, annot2 := range element.Data.Niveau2 {
newAnnot.Annotation.Classif.Niveau2.Labels[j].Value = annot2
}
}
if element.Data.Niveau3 != nil {
for k, annot3 := range element.Data.Niveau3 {
newAnnot.Annotation.Classif.Niveau3.Labels[k].Value = annot3
}
}
newAnnots = append(newAnnots, newAnnot)
}
return
}
However, I got the error saying the index [0] is out of range for my slice.
panic: runtime error: index out of range [0] with length 0
Definitions of the two structures are the following
type AnnotV1 struct {
Uuid string `json:"uuid"`
Data struct {
Niveau1 []string `json:"Niveau1"`
Niveau2 []string `json:"Niveau2"`
Niveau3 []string `json:"Niveau3"`
} `json:"data"`
User string `json:"user"`
}
and
type AnnotV2 struct {
Uuid string `json:"uuid"`
Annotation struct {
Classif struct {
Niveau1 struct {
Labels []struct {
Value string `json:value`
} `json:"labels"`
}
Niveau2 struct {
Labels []struct {
Value string `json:value`
} `json:"labels"`
}
Niveau3 struct {
Labels []struct {
Value string `json:value`
} `json:"labels"`
}
} `json:"classifications"`
} `json:"annotation"`
User string `json:"user"`
}
type Label struct {
Value string `json:"value"`
}
type AnnotV2 struct {
Uuid string `json:"uuid"`
Annotation struct {
Classif struct {
Niveau1 struct {
Labels []Label `json:"labels"`
}
Niveau2 struct {
Labels []Label `json:"labels"`
}
Niveau3 struct {
Labels []Label `json:"labels"`
}
} `json:"classifications"`
} `json:"annotation"`
User string `json:"user"`
}
pre-allocate the slice
if element.Data.Niveau2 != nil {
newAnnot.Annotation.Classif.Niveau2.Labels = make([]Label, len(element.Data.Niveau2))
for j, annot2 := range element.Data.Niveau2 {
newAnnot.Annotation.Classif.Niveau2.Labels[j].Value = annot2
}
}
or use append
if element.Data.Niveau2 != nil {
for _, annot2 := range element.Data.Niveau2 {
newAnnot.Annotation.Classif.Niveau2.Labels = append(newAnnot.Annotation.Classif.Niveau2.Labels, Label{annot2})
}
}
Related
Trying to unmarshal YAML into complex object such as map[string]map[interface{}]string.
The problem is that I want to be able to differentiate an interface{} part between string and Source which is a struct.
type Source struct {
ID string `yaml:"id"`
Name string `yaml:"name"`
LogoID string `yaml:"logoId"`
URL string `yaml:"url"`
}
type UNFT struct {
ItemMeta map[string]map[interface{}]string `yaml:"item_meta"`
// could be
// ItemMeta map[string]map[string]string `yaml:"item_meta"`
// or
// ItemMeta map[string]map[Source]string `yaml:"item_meta"`
}
Obviously YAML does not know how to unmarshal into Source struct so I have to implement Unmarshaler interface:
type Unmarshaler interface {
UnmarshalYAML(value *Node) error
}
But I don't quite understand the big picture of unmarshaling process. In general I assume that I have to manually traverse *yaml.Node and call func UnmarshalYAML(value *Node) error on every node.
package main
import (
"fmt"
"gopkg.in/yaml.v3"
)
type Source struct {
ID string `json:"id"`
Name string `json:"name"`
LogoID string `json:"logoId"`
URL string `json:"url"`
}
var data = `
unf:
item_meta:
source:
!struct
? id: "data-watch"
name: "DataWatch"
logoid: "data-watch"
url: "https"
: "product_any('SS')"
public_usage:
"": "source_any('SDF')"
"provider": "source_any('ANO')"`
type UNFT struct {
ItemMeta map[string]map[interface{}]string `yaml:"item_meta"`
}
type MetaConverterConfigT struct {
UNFT UNFT `yaml:"unf"`
}
func main() {
cfg := MetaConverterConfigT{}
err := yaml.Unmarshal([]byte(data), &cfg)
if err != nil {
fmt.Println("%w", err)
}
fmt.Println(cfg)
}
func (s *UNFT) UnmarshalYAML(n *yaml.Node) error {
var cfg map[string]map[interface{}]string
if err := n.Decode(&cfg); err != nil {
fmt.Println("%w", err)
}
return nil
}
Go playground
type MetaKey struct {
String string
Source Source
}
func (k *MetaKey) UnmarshalYAML(n *yaml.Node) error {
if n.Tag == "!!str" {
return n.Decode(&k.String)
}
if n.Tag == "!!map" {
return n.Decode(&k.Source)
}
return fmt.Errorf("unsupported MetaKey type")
}
// ...
type UNFT struct {
ItemMeta map[string]map[MetaKey]string `yaml:"item_meta"`
}
https://go.dev/play/p/Nhtab4l-ANT
If you need the map type to remain as is, i.e. without adding the custom key type, then you can implement the unmarshaler on UNFT as well and just do a re-mapping with any:
type UNFT struct {
ItemMeta map[string]map[any]string `yaml:"item_meta"`
}
func (u *UNFT) UnmarshalYAML(n *yaml.Node) error {
var obj struct {
ItemMeta map[string]map[MetaKey]string `yaml:"item_meta"`
}
if err := n.Decode(&obj); err != nil {
return err
}
u.ItemMeta = make(map[string]map[any]string, len(obj.ItemMeta))
for k, v := range obj.ItemMeta {
m := make(map[any]string, len(v))
for k, v := range v {
if k.Source != (Source{}) {
m[k.Source] = v
} else {
m[k.String] = v
}
}
u.ItemMeta[k] = m
}
return nil
}
https://go.dev/play/p/uwboGKf3qnD
I have a json like following, where value can be int or string
{
"data": [
{
"Name": "a_name",
"value": 1
},
{
"Name": "b_name",
"value": "val"
},
{
"Name": "c_name",
"value": 2
}
]
}
Now I want to convert that json into following struct, like only extract a_name and b_name value.
type Data struct {
AName int `json: "a_name"`
BName string `json: "b_name"`
}
I can do it by following way
import (
"encoding/json"
"fmt"
)
type TmpData struct {
Data []struct {
Name string `json:"Name"`
Value interface{} `json:"value"`
} `json:"data"`
}
type ExpectedData struct {
AName int `json: "a_name"`
BName string `json: "b_name"`
}
func main() {
data := `{
"data": [
{
"Name": "a_name",
"value": 1
},
{
"Name": "b_name",
"value": "val"
},
{
"Name": "c_name",
"value": 2
}
]
}`
tmpData := &TmpData{}
json.Unmarshal([]byte(data), tmpData)
ans := &ExpectedData{}
for _, d := range tmpData.Data {
if d.Name == "a_name" {
ans.AName = int(d.Value.(float64))
} else if d.Name == "b_name" {
ans.BName = d.Value.(string)
}
}
fmt.Println(ans)
}
Is there any better solution for this?
Not possible with the standard JSON un-marshalling unless you write a custom un-marshaller for your Data type.
The key here is to define the type for value to be an interface{}, so that multiple types could be stored in your b_name record.
func (d *Data) UnmarshalJSON(data []byte) error {
var result Details
if err := json.Unmarshal(data, &result); err != nil {
return err
}
for _, value := range result.Data {
switch value.Name {
// The json package will assume float64 when Unmarshalling with an interface{}
case "a_name":
v, ok := (value.Value).(float64)
if !ok {
return fmt.Errorf("a_name got data of type %T but wanted float64", value.Value)
}
d.AName = int(v)
case "b_name":
v, ok := (value.Value).(string)
if !ok {
return fmt.Errorf("b_name got data of type %T but wanted string", value.Value)
}
d.BName = v
}
}
return nil
}
Playground - https://go.dev/play/p/GrXKAE87d1F
Community,
The mission
basic
Implement a func that patches all string fields on an objects
details
[done] fields shall only be patched if they match a matcher func
[done] value shall be processed via process func
patching shall be done recursive
it shall also work for []string, []*string and recursive for structs and []struct, []*struct
// update - removed old code
Solution
structs
updated the structs to use (though this does not affect the actual program, i use this for completeness
type Tag struct {
Name string `process:"yes,TagName"`
NamePtr *string `process:"no,TagNamePtr"`
}
type User struct {
ID int
Nick string
Name string `process:"yes,UserName"`
NamePtr *string `process:"yes,UserNamePtr"`
Slice []string `process:"yes,Slice"`
SlicePtr []*string `process:"yes,SlicePtr"`
SubStruct []Tag `process:"yes,SubStruct"`
SubStructPtr []*Tag `process:"yes,SubStructPtr"`
}
helper func
Further we need two helper funcs to check if a struct has a tag and to print to console
func Stringify(i interface{}) string {
s, _ := json.MarshalIndent(i, "", " ")
return string(s)
}
func HasTag(structFiled reflect.StructField, tagName string, tagValue string) bool {
tag := structFiled.Tag
if value, ok := tag.Lookup(tagName); ok {
parts := strings.Split(value, ",")
if len(parts) > 0 {
return parts[0] == tagValue
}
}
return false
}
patcher - the actual solution
type Patcher struct {
Matcher func(structFiled *reflect.StructField, v reflect.Value) bool
Process func(in string) string
}
func (p *Patcher) value(idx int, v reflect.Value, structFiled *reflect.StructField) {
if !v.IsValid() {
return
}
switch v.Kind() {
case reflect.Ptr:
p.value(idx, v.Elem(), structFiled)
case reflect.Struct:
for i := 0; i < v.NumField(); i++ {
var sf = v.Type().Field(i)
structFiled = &sf
p.value(i, v.Field(i), structFiled)
}
case reflect.Slice:
for i := 0; i < v.Len(); i++ {
p.value(i, v.Index(i), structFiled)
}
case reflect.String:
if p.Matcher(structFiled, v) {
v.SetString(p.Process(v.String()))
}
}
}
func (p *Patcher) Apply(in interface{}) {
p.value(-1, reflect.ValueOf(in).Elem(), nil)
}
how to use
func main() {
var NamePtr string = "golang"
var SubNamePtr string = "*secure"
testUser := User{
ID: 1,
Name: "lumo",
NamePtr: &NamePtr,
SubStruct: []Tag{{
Name: "go",
},
},
SubStructPtr: []*Tag{&Tag{
Name: "*go",
NamePtr: &SubNamePtr,
},
},
}
var p = Patcher{
// filter - return true if the field in struct has a tag process=true
Matcher: func(structFiled *reflect.StructField, v reflect.Value) bool {
return HasTag(*structFiled, "process", "yes")
},
// process
Process: func(in string) string {
if in != "" {
return fmt.Sprintf("!%s!", strings.ToUpper(in))
} else {
return "!empty!"
}
return in
},
}
p.Apply(&testUser)
fmt.Println("Output:")
fmt.Println(Stringify(testUser))
}
goplay
https://goplay.tools/snippet/-0MHDfKr7ax
I'm struggling with updating a slice inside map rulesByCountry without any success.
The value of enabled changes only inside the function UpdateName but the map itself still sees this value unchanged. I assume it's something to do with pointers. I guess I did not grasp the concept of it. Can someone direct me what I'm doing wrong here? I tried a lot of things and run out of options. I would appreciate any kind of help.
import (
"fmt"
)
// Consts
const RuleSeparateStreetNameFromHome string = "Separate street number from home"
// Types
type Address struct {
AddressLines []string
Country string
}
type RuleChain []RuleDefinition
type RuleDefinition struct {
Name string
Enabled bool
}
//Map
var rulesByCountry map[string][]RuleDefinition = map[string][]RuleDefinition{
"DE": {
{
Name: RuleSeparateStreetNameFromHome,
// TODO some logic,
Enabled: false,
},
},
}
func main() {
var addr *Address
addr = &Address{
AddressLines: []string{
"Street3",
},
Country: "DE",
}
rules := GetRulesForCountry(addr.GetCountry())
rules.UpdateName(RuleSeparateStreetNameFromHome)
fmt.Println(rules)
}
func GetRulesForCountry(country string) RuleChain {
if rules, ok := rulesByCountry[country]; ok {
return rules
}
return nil
}
func (a *Address) GetFirstAddressLine() string {
return a.GetAddressLine(1)
}
func (a *Address) GetAddressLine(lineNumber int) string {
if lineNumber <= 0 {
return ""
}
lines := a.GetAddressLines()
if len(lines) >= lineNumber {
return lines[lineNumber-1]
}
return ""
}
func (m *Address) GetAddressLines() []string {
if m != nil {
return m.AddressLines
}
return nil
}
func (r *RuleChain) UpdateName(name string) {
for _, rule := range *r {
if rule.Name == name {
rule.Enabled = true
fmt.Print(rule)
}
}
}
func (m *Address) GetCountry() string {
if m != nil {
return m.Country
}
return ""
}
Based on the inputs of mkopriva
package main
import (
"fmt"
)
// Consts
const RuleSeparateStreetNameFromHome string = "Separate street number from home"
// Types
type Address struct {
AddressLines []string
Country string
}
type RuleChain []*RuleDefinition
type RuleDefinition struct {
Name string
Enabled bool
}
//Map
var rulesByCountry map[string][]*RuleDefinition = map[string][]*RuleDefinition{
"DE": {
{
Name: RuleSeparateStreetNameFromHome,
// TODO some logic,
Enabled: false,
},
},
}
func main() {
var addr *Address
addr = &Address{
AddressLines: []string{
"Street3",
},
Country: "DE",
}
rules := GetRulesForCountry(addr.GetCountry())
rules.UpdateName(RuleSeparateStreetNameFromHome)
fmt.Println(rules[0])
}
func GetRulesForCountry(country string) RuleChain {
if rules, ok := rulesByCountry[country]; ok {
return rules
}
return nil
}
func (a *Address) GetFirstAddressLine() string {
return a.GetAddressLine(1)
}
func (a *Address) GetAddressLine(lineNumber int) string {
if lineNumber <= 0 {
return ""
}
lines := a.GetAddressLines()
if len(lines) >= lineNumber {
return lines[lineNumber-1]
}
return ""
}
func (m *Address) GetAddressLines() []string {
if m != nil {
return m.AddressLines
}
return nil
}
func (r *RuleChain) UpdateName(name string) {
for _, rule := range *r {
if rule.Name == name {
rule.Enabled = true
fmt.Print(rule)
}
}
}
func (m *Address) GetCountry() string {
if m != nil {
return m.Country
}
return ""
}
Output:
&{Separate street number from home true}&{Separate street number from home true}
type Player struct {
id bson.ObjectId
test map[int]int
}
func (this *Player) Id() bson.ObjectId {
return this.id
}
func (this *Player) DbObj() bson.D {
testBson := bson.D{}
for k,v := range this.test {
testBson = append(testBson, bson.M{"id":k, "v":v}) // compile error
}
return bson.D{
{"_id",this.id},
{"test", testBson},
}
}
The bson document should be:
{'_id':ObjectId(xx),'test':[{'id':1,'v':1},..., {'id':2,'v':2}]}
But I don't know how to convert the map object to [{'id':1,'v':1},..., {'id':2,'v':2}], bson.M is not the type DocElem which D contain
Use a Go slice to represent the BSON array:
func (this *Player) DbObj() bson.D {
var testBson []bson.M
for k,v := range this.test {
testBson = append(testBson, bson.M{"id":k, "v":v})
}
return bson.D{
{"_id",this.id},
{"test", testBson},
}
}