For example, we have 3 CSV files and common for all is Email column. In first file are Name and Email, in another are Email (plus different info) and no Name field. So, if I need to fill in 2 and 3 files field Name based on the correspondence of the Name and Đ•mail from the first file than... I wrote code like this:
package main
import (
"fmt"
"io/ioutil"
"log"
"path/filepath"
"strings"
"github.com/jszwec/csvutil"
)
type User struct {
Name string `csv:"name"`
Email string `csv:"email"`
}
type Good struct {
User
Dt string `csv:"details"`
}
type Strange struct {
User
St string `csv:"status"`
Dt string `csv:"details"`
}
var lst map[string]string
func readCSV(fn string, dat interface{}) error {
raw, err := ioutil.ReadFile(fn)
if err != nil {
return fmt.Errorf("Cannot read CSV: %w", err)
}
if err := csvutil.Unmarshal(raw, dat); err != nil {
return fmt.Errorf("Cannot unmarshal CSV: %w", err)
}
return nil
}
func fixNames(fl string, in interface{}) error {
if err := readCSV(fl, in); err != nil {
return fmt.Errorf("CSV: %w", err)
}
switch in.(type) {
case *[]Good:
var vals []Good
for _, v := range *in.(*[]Good) {
v.Name = lst[strings.TrimSpace(strings.ToLower(v.Email))]
vals = append(vals, v)
}
in = vals
case *[]Strange:
var vals []Strange
for _, v := range *in.(*[]Strange) {
v.Name = lst[strings.TrimSpace(strings.ToLower(v.Email))]
vals = append(vals, v)
}
in = vals
}
b, err := csvutil.Marshal(in)
if err != nil {
return fmt.Errorf("Cannot marshal CSV: %w", err)
}
ext := filepath.Ext(fl)
bas := filepath.Base(fl)
err = ioutil.WriteFile(bas[:len(bas)-len(ext)]+"-XIAOSE"+ext, b, 0644)
if err != nil {
return fmt.Errorf("Cannot save CSV: %w", err)
}
return nil
}
func main() {
var users []User
if err := readCSV("./Guitar_Contacts.csv", &users); err != nil {
log.Fatalf("CSV: %s", err)
}
lst = make(map[string]string)
for _, v := range users {
lst[strings.TrimSpace(strings.ToLower(v.Email))] = v.Name
}
var usersGood []Good
if err := fixNames("./Guitar-Good.csv", &usersGood); err != nil {
log.Fatalf("fix: %s", err)
}
var usersStrange []Strange
if err := fixNames("./Guitar-Uknown.csv", &usersStrange); err != nil {
log.Fatalf("fix: %s", err)
}
fmt.Println("OK")
}
in this code I don't like part in func fixNames where is switch:
switch in.(type) {
case *[]Good:
var vals []Good
for _, v := range *in.(*[]Good) {
v.Name = lst[strings.TrimSpace(strings.ToLower(v.Email))]
vals = append(vals, v)
}
in = vals
case *[]Strange:
var vals []Strange
for _, v := range *in.(*[]Strange) {
v.Name = lst[strings.TrimSpace(strings.ToLower(v.Email))]
vals = append(vals, v)
}
in = vals
}
because I just repeat code in part where *in.(SOME_TYPE). I want one loop and one action for different types, structs where are Name and Email fields...
Also was idea to do it with reflection smth. like this:
v := reflect.ValueOf(in)
v = v.Elem()
for i := 0; i < v.Len(); i++ {
fmt.Println(v.Index(i))
}
but I do not know what to do next, how to add in that v value for Name
You don't need reflection for this particular case. You can clean the code up by realizing that you are only working on the User part of the structs, and that you can simplify the type switch:
fix:=func(in *User) {
in.Name = lst[strings.TrimSpace(strings.ToLower(in.Email))]
}
switch k:=in.(type) {
case *[]Good:
for i := range *k {
fix( &(*k)[i].User )
}
case *[]Strange:
for i := range *k {
fix( &(*k)[i].User )
}
}
You have to repeat the for loop, but above code does the correction in place.
You can clean up a bit more by not passing a reference to the slice.
With reflect package, you can do that like this.
func fixNames(fl string, in interface{}) error {
//other code
v := reflect.ValueOf(in)
if v.Kind() == reflect.Ptr {
arr := v.Elem()
fmt.Println(arr.Len())
if arr.Kind() == reflect.Slice || arr.Kind() == reflect.Array {
for i := 0; i < arr.Len(); i++ {
elem := arr.Index(i)
f := elem.FieldByName("Name")
f.SetString("NameOfUser")
}
}
}
// other code
}
Also playground example: https://play.golang.org/p/KrGvLVprslH
Related
I was trying to test a bit with Redis for the first time and I bumped into some confusion with HGET/HSET/HGETALL. My main problem was that I needed to store time, and I wanted to use a hash as I'll continuously update the time.
At first I read about how a MarshalBinary function such as this would save me:
func (f Foo) MarshalBinary() ([]byte, error) {
return json.Marshal(f)
}
What that did was that it saved the struct as a json string, but only as a string and not as an actual Redis hash. What I ended up doing in the end was a fairly large boilerplate code that makes my struct I want to save into a map, and that one is properly stored as a hash in Redis.
type Foo struct {
Number int `json:"number"`
ATime time.Time `json:"atime"`
String string `json:"astring"`
}
func (f Foo) toRedis() map[string]interface{} {
res := make(map[string]interface{})
rt := reflect.TypeOf(f)
rv := reflect.ValueOf(f)
if rt.Kind() == reflect.Ptr {
rt = rt.Elem()
rv = rv.Elem()
}
for i := 0; i < rt.NumField(); i++ {
f := rt.Field(i)
v := rv.Field(i)
switch t := v.Interface().(type) {
case time.Time:
res[f.Tag.Get("json")] = t.Format(time.RFC3339)
default:
res[f.Tag.Get("json")] = t
}
}
return res
}
Then to parse back into my Foo struct when calling HGetAll(..).Result(), I'm getting the result as a map[string]string and create a new Foo with these functions:
func setRequestParam(arg *Foo, i int, value interface{}) {
v := reflect.ValueOf(arg).Elem()
f := v.Field(i)
if f.IsValid() {
if f.CanSet() {
if f.Kind() == reflect.String {
f.SetString(value.(string))
return
} else if f.Kind() == reflect.Int {
f.Set(reflect.ValueOf(value))
return
} else if f.Kind() == reflect.Struct {
f.Set(reflect.ValueOf(value))
}
}
}
}
func fromRedis(data map[string]string) (f Foo) {
rt := reflect.TypeOf(f)
rv := reflect.ValueOf(f)
for i := 0; i < rt.NumField(); i++ {
field := rt.Field(i)
v := rv.Field(i)
switch v.Interface().(type) {
case time.Time:
if val, ok := data[field.Tag.Get("json")]; ok {
if ti, err := time.Parse(time.RFC3339, val); err == nil {
setRequestParam(&f, i, ti)
}
}
case int:
if val, ok := data[field.Tag.Get("json")]; ok {
in, _ := strconv.ParseInt(val, 10, 32)
setRequestParam(&f, i, int(in))
}
default:
if val, ok := data[field.Tag.Get("json")]; ok {
setRequestParam(&f, i, val)
}
}
}
return
}
The whole code in its ungloryness is here
I'm thinking that there must be a saner way to solve this problem? Or am I forced to do something like this? The struct I need to store only contains ints, strings and time.Times.
*edit
The comment field is a bit short so doing an edit instead:
I did originally solve it like 'The Fool' suggested in comments and as an answer. The reason I changed to the above part, while more complex a solution, I think it's more robust for changes. If I go with a hard coded map solution, I'd "have to" have:
Constants with hash keys for the fields, since they'll be used at least in two places (from and to Redis), it'll be a place for silly mistakes not picked up by the compiler. Can of course skip that but knowing my own spelling it's likely to happen
If someone just wants to add a new field and doesn't know the code well, it will compile just fine but the new field won't be added in Redis. An easy mistake to do, especially for junior developers being a bit naive, or seniors with too much confidence.
I can put these helper functions in a library, and things will just magically work for all our code when a time or complex type is needed.
My intended question/hope though was: Do I really have to jump through hoops like this to store time in Redis hashes with go? Fair, time.Time isn't a primitive and Redis isn't a (no)sql database, but I would consider timestamps in cache a very common use case (in my case a heartbeat to keep track of timed out sessions together with metadata enough to permanently store it, thus the need to update them). But maybe I'm misusing Redis, and I should rather have two entries, one for the data and one for the timestamp, which would then leave me with two simple get/set functions taking in time.Time and returning time.Time.
You can use redigo/redis#Args.AddFlat to convert struct to redis hash we can map the value using redis tag.
package main
import (
"fmt"
"time"
"github.com/gomodule/redigo/redis"
)
type Foo struct {
Number int64 `json:"number" redis:"number"`
ATime time.Time `json:"atime" redis:"atime"`
AString string `json:"astring" redis:"astring"`
}
func main() {
c, err := redis.Dial("tcp", ":6379")
if err != nil {
fmt.Println(err)
return
}
defer c.Close()
t1 := time.Now().UTC()
var foo Foo
foo.Number = 10000000000
foo.ATime = t1
foo.AString = "Hello"
tmp := redis.Args{}.Add("id1").AddFlat(&foo)
if _, err := c.Do("HMSET", tmp...); err != nil {
fmt.Println(err)
return
}
v, err := redis.StringMap(c.Do("HGETALL", "id1"))
if err != nil {
fmt.Println(err)
return
}
fmt.Printf("%#v\n", v)
}
Then to update ATime you can use redis HSET
if _, err := c.Do("HMSET", "id1", "atime", t1.Add(-time.Hour * (60 * 60 * 24))); err != nil {
fmt.Println(err)
return
}
And to retrieve it back to struct we have to do some reflect magic
func structFromMap(src map[string]string, dst interface{}) error {
dt := reflect.TypeOf(dst).Elem()
dv := reflect.ValueOf(dst).Elem()
for i := 0; i < dt.NumField(); i++ {
sf := dt.Field(i)
sv := dv.Field(i)
if v, ok := src[strings.ToLower(sf.Name)]; ok {
switch sv.Interface().(type) {
case time.Time:
format := "2006-01-02 15:04:05 -0700 MST"
ti, err := time.Parse(format, v)
if err != nil {
return err
}
sv.Set(reflect.ValueOf(ti))
case int, int64:
x, err := strconv.ParseInt(v, 10, sv.Type().Bits())
if err != nil {
return err
}
sv.SetInt(x)
default:
sv.SetString(v)
}
}
}
return nil
}
Final Code
package main
import (
"fmt"
"time"
"reflect"
"strings"
"strconv"
"github.com/gomodule/redigo/redis"
)
type Foo struct {
Number int64 `json:"number" redis:"number"`
ATime time.Time `json:"atime" redis:"atime"`
AString string `json:"astring" redis:"astring"`
}
func main() {
c, err := redis.Dial("tcp", ":6379")
if err != nil {
fmt.Println(err)
return
}
defer c.Close()
t1 := time.Now().UTC()
var foo Foo
foo.Number = 10000000000
foo.ATime = t1
foo.AString = "Hello"
tmp := redis.Args{}.Add("id1").AddFlat(&foo)
if _, err := c.Do("HMSET", tmp...); err != nil {
fmt.Println(err)
return
}
v, err := redis.StringMap(c.Do("HGETALL", "id1"))
if err != nil {
fmt.Println(err)
return
}
fmt.Printf("%#v\n", v)
if _, err := c.Do("HMSET", "id1", "atime", t1.Add(-time.Hour * (60 * 60 * 24))); err != nil {
fmt.Println(err)
return
}
var foo2 Foo
structFromMap(v, &foo2)
fmt.Printf("%#v\n", foo2)
}
func structFromMap(src map[string]string, dst interface{}) error {
dt := reflect.TypeOf(dst).Elem()
dv := reflect.ValueOf(dst).Elem()
for i := 0; i < dt.NumField(); i++ {
sf := dt.Field(i)
sv := dv.Field(i)
if v, ok := src[strings.ToLower(sf.Name)]; ok {
switch sv.Interface().(type) {
case time.Time:
format := "2006-01-02 15:04:05 -0700 MST"
ti, err := time.Parse(format, v)
if err != nil {
return err
}
sv.Set(reflect.ValueOf(ti))
case int, int64:
x, err := strconv.ParseInt(v, 10, sv.Type().Bits())
if err != nil {
return err
}
sv.SetInt(x)
default:
sv.SetString(v)
}
}
}
return nil
}
Note: The struct field name is matched with the redis tag
How can i programmatically access the msg value "Design" in the go language structure shown below?
after subIssues[28].Fields.Unknowns["customfield_11801"] i dont find a language construct to access the data structure.
To convert into a struct and work with that, have a look at this repo: https://github.com/mitchellh/mapstructure.
And if you want to do it your self, something like this: (Adjust implementation to match your project)
func SetField(obj interface{}, name string, value interface{}) error {
structValue := reflect.ValueOf(obj).Elem()
structFieldValue := structValue.FieldByName(name)
if !structFieldValue.IsValid() {
return fmt.Errorf("No such field: %s in obj", name)
}
if !structFieldValue.CanSet() {
return fmt.Errorf("Cannot set %s field value", name)
}
structFieldType := structFieldValue.Type()
val := reflect.ValueOf(value)
if structFieldType != val.Type() {
return errors.New("Provided value type didn't match obj field type")
}
structFieldValue.Set(val)
return nil
}
type MyStruct struct {
Name string
Age int64
}
func (s *MyStruct) FillStruct(m map[string]interface{}) error {
for k, v := range m {
err := SetField(s, k, v)
if err != nil {
return err
}
}
return nil
}
func main() {
myData := make(map[string]interface{})
myData["Name"] = "Tony"
myData["Age"] = int64(23)
result := &MyStruct{}
err := result.FillStruct(myData)
if err != nil {
fmt.Println(err)
}
fmt.Println(result)
}
I have an error in the function DeleteCharactersID, it removes the entire multidimensional array but I need to delete a single element or value using the id that I gave it.
This my domain.go
type CharactersBB []struct {
CharID int `json:"char_id"`
Name string `json:"name"`
Birthday string `json:"birthday"`
Occupation []string `json:"occupation"`
Img string `json:"img"`
Status string `json:"status"`
Nickname string `json:"nickname"`
Appearance []int `json:"appearance"`
Portrayed string `json:"portrayed"`
Category string `json:"category"`
BetterCallSaulAppearance []interface{} `json:"better_call_saul_appearance"`
}
This my services.go
var characters []domain.CharactersBB
func DeleteCharactersID(i string) ([]domain.CharactersBB, error) {
id, err := validateID(i)
if err != nil {
return []domain.CharactersBB{}, ErrorInvalidID
}
c, err := searchCharacter(id)
if err != nil {
return []domain.CharactersBB{}, err
}
for i := 0; i < len(characters); i++ {
for j := 0; j < len(characters[i]); j++ {
if characters[i][j].CharID == c[j].CharID {
characters[j] = append(characters[i][:j], characters[i][j+1:]...)
break
}
}
}
fmt.Printf("%T\n", c)
return characters, nil
}
func validateID(id string) (int, error) {
num, err := strconv.Atoi(id)
if err != nil {
return -1, err
}
return num, nil
}
func searchCharacter(id int) (domain.CharactersBB, error) {
var c domain.CharactersBB
for a := range characters {
if c[a].CharID == id {
return c, nil
}
}
return domain.CharactersBB{}, ErrorCharactersNotFound
}
func init() {
var c domain.CharactersBB
url := baseurl + "characters"
resp, _ := http.Get(url)
Data, _ := ioutil.ReadAll(resp.Body)
json.Unmarshal([]byte(Data), &c)
characters = append(characters, c)
//s, _ := json.MarshalIndent(characters, "", "\t")
//fmt.Println(string(s))
fmt.Printf("%T\n", characters)
}
func GetCharacters() ([]domain.CharactersBB, error) {
return characters, nil
}
In this line:
characters[j] = append(characters[i][:j], characters[i][j+1:]...)
Your inner array counter is j, your outer array counter is i, so it seems like this should be:
characters[i] = append(characters[i][:j], characters[i][j+1:]...)
// ^^^ fix here
Assuming we have two yaml files
master.yaml
someProperty: "someVaue"
anotherProperty: "anotherValue"
override.yaml
someProperty: "overriddenVaue"
Is it possible to unmarshall, merge, and then write those changes to a file without having to define a struct for every property in the yaml file?
The master file has over 500 properties in it that are not at all important to the service at this point of execution, so ideally I'd be able to just unmarshal into a map, do a merge and write out in yaml again but I'm relatively new to go so wanted some opinions.
I've got some code to read the yaml into an interface but i'm unsure on the best approach to then merge the two.
var masterYaml interface{}
yamlBytes, _ := ioutil.ReadFile("master.yaml")
yaml.Unmarshal(yamlBytes, &masterYaml)
var overrideYaml interface{}
yamlBytes, _ = ioutil.ReadFile("override.yaml")
yaml.Unmarshal(yamlBytes, &overrideYaml)
I've looked into libraries like mergo but i'm not sure if that's the right approach.
I'm hoping that after the master I would be able to write out to file with properties
someProperty: "overriddenVaue"
anotherProperty: "anotherValue"
Assuming that you just want to merge at the top level, you can unmarshal into maps of type map[string]interface{}, as follows:
package main
import (
"io/ioutil"
"gopkg.in/yaml.v2"
)
func main() {
var master map[string]interface{}
bs, err := ioutil.ReadFile("master.yaml")
if err != nil {
panic(err)
}
if err := yaml.Unmarshal(bs, &master); err != nil {
panic(err)
}
var override map[string]interface{}
bs, err = ioutil.ReadFile("override.yaml")
if err != nil {
panic(err)
}
if err := yaml.Unmarshal(bs, &override); err != nil {
panic(err)
}
for k, v := range override {
master[k] = v
}
bs, err = yaml.Marshal(master)
if err != nil {
panic(err)
}
if err := ioutil.WriteFile("merged.yaml", bs, 0644); err != nil {
panic(err)
}
}
For a broader solution (with n input files), you can use this function. I have used #robox answer to do my solution:
func ReadValues(filenames ...string) (string, error) {
if len(filenames) <= 0 {
return "", errors.New("You must provide at least one filename for reading Values")
}
var resultValues map[string]interface{}
for _, filename := range filenames {
var override map[string]interface{}
bs, err := ioutil.ReadFile(filename)
if err != nil {
log.Info(err)
continue
}
if err := yaml.Unmarshal(bs, &override); err != nil {
log.Info(err)
continue
}
//check if is nil. This will only happen for the first filename
if resultValues == nil {
resultValues = override
} else {
for k, v := range override {
resultValues[k] = v
}
}
}
bs, err := yaml.Marshal(resultValues)
if err != nil {
log.Info(err)
return "", err
}
return string(bs), nil
}
So for this example you should call it with this order:
result, _ := ReadValues("master.yaml", "overwrite.yaml")
In the case you have an extra file newFile.yaml, you could also use this function:
result, _ := ReadValues("master.yaml", "overwrite.yaml", "newFile.yaml")
DEEP MERGE TWO YAML FILES
package main
import (
"fmt"
"io/ioutil"
"sigs.k8s.io/yaml"
)
func main() {
// declare two map to hold the yaml content
base := map[string]interface{}{}
currentMap := map[string]interface{}{}
// read one yaml file
data, _ := ioutil.ReadFile("conf.yaml")
if err := yaml.Unmarshal(data, &base); err != nil {
}
// read another yaml file
data1, _ := ioutil.ReadFile("conf1.yaml")
if err := yaml.Unmarshal(data1, ¤tMap); err != nil {
}
// merge both yaml data recursively
base = mergeMaps(base, currentMap)
// print merged map
fmt.Println(base)
}
func mergeMaps(a, b map[string]interface{}) map[string]interface{} {
out := make(map[string]interface{}, len(a))
for k, v := range a {
out[k] = v
}
for k, v := range b {
if v, ok := v.(map[string]interface{}); ok {
if bv, ok := out[k]; ok {
if bv, ok := bv.(map[string]interface{}); ok {
out[k] = mergeMaps(bv, v)
continue
}
}
}
out[k] = v
}
return out
}
I am get leveldb's all key-val to a map[string][]byte, but it is not running as my expection.
code is as below
package main
import (
"fmt"
"strconv"
"github.com/syndtr/goleveldb/leveldb"
)
func main() {
db, err := leveldb.OpenFile("db", nil)
if err != nil {
panic(err)
}
defer db.Close()
for i := 0; i < 10; i++ {
err := db.Put([]byte("key"+strconv.Itoa(i)), []byte("value"+strconv.Itoa(i)), nil)
if err != nil {
panic(err)
}
}
snap, err := db.GetSnapshot()
if err != nil {
panic(err)
}
if snap == nil {
panic("snap shot is nil")
}
data := make(map[string][]byte)
iter := snap.NewIterator(nil, nil)
for iter.Next() {
Key := iter.Key()
Value := iter.Value()
data[string(Key)] = Value
}
iter.Release()
if iter.Error() != nil {
panic(iter.Error())
}
for k, v := range data {
fmt.Println(string(k) + ":" + string(v))
}
}
but the result is below
key3:value9
key6:value9
key7:value9
key8:value9
key1:value9
key2:value9
key4:value9
key5:value9
key9:value9
key0:value9
rather not key0:value0
Problem is with casting around types (byte[] to string, etc.).
You are trying to print string values. To avoid unnecessary casting apply the following modifications:
Change data initialization into data := make(map[string]string)
Assign values into data with `data[string(Key)] = string(Value) (by the way, don't use capitalization for variables you aren't intend to export)
Print data's values with fmt.Println(k + ":" + v))
This should produce the following result:
key0:value0
key1:value1
key7:value7
key2:value2
key3:value3
key4:value4
key5:value5
key6:value6
key8:value8
key9:value9