I'm having a csv file, and want to read:
Header names
Fields types
So, I wrote the below:
package main
import (
"encoding/csv"
"fmt"
"os"
"log"
"reflect"
"strconv"
)
func main() {
filePath := "./file.csv"
headerNames := make(map[int]string)
headerTypes := make(map[int]string)
// Load a csv file.
f, _ := os.Open(filePath)
// Create a new reader.
r := csv.NewReader(f)
// Read first row only
header, err := r.Read()
checkError("Some other error occurred", err)
// Add mapping: Column/property name --> record index
for i, v := range header {
headerNames[i] = v
}
// Read second row
record, err := r.Read()
checkError("Some other error occurred", err)
// Check record fields types
for i, v := range record {
var value interface{}
if value, err = strconv.Atoi(v); err != nil {
if value, err = strconv.ParseFloat(v, 64); err != nil {
if value, err = strconv.ParseBool(v); err != nil {
if value, err = strconv.ParseBool(v); err != nil { // <== How to do this with unknown layout
// Value is a string
headerTypes[i] = "string"
value = v
fmt.Println(reflect.TypeOf(value), reflect.ValueOf(value))
} else {
// Value is a timestamp
headerTypes[i] = "time"
fmt.Println(reflect.TypeOf(value), reflect.ValueOf(value))
}
} else {
// Value is a bool
headerTypes[i] = "bool"
fmt.Println(reflect.TypeOf(value), reflect.ValueOf(value))
}
} else {
// Value is a float
headerTypes[i] = "float"
fmt.Println(reflect.TypeOf(value), reflect.ValueOf(value))
}
} else {
// Value is an int
headerTypes[i] = "int"
fmt.Println(reflect.TypeOf(value), reflect.ValueOf(value))
}
}
for i, _ := range header {
fmt.Printf("Header: %v \tis\t %v\n", headerNames[i], headerTypes[i])
}
}
func checkError(message string, err error) {
// Error Logging
if err != nil {
log.Fatal(message, err)
}
}
And with csv file as:
name,age,developer
"Hasan","46.4","true"
I got an output as:
Header: name is string
Header: age is float
Header: developer is bool
The output is correct.
The thing that I could not do is the one is checking if the field is string as I do not know what layout the field could be.
I aware I can pasre string to time as per the format stated at https://go.dev/src/time/format.go, and can build a custom parser, something like:
test, err := fmtdate.Parse("MM/DD/YYYY", "10/15/1983")
if err != nil {
panic(err)
}
But this will work only (as per my knowledge) if I know the layout?
So, again my question is, how can I parse time, or what shall I do to be able to parse it, if I do not know the layout?
Thanks to the comment by Burak, I found the solution by using this package: github.com/araddon/dateparse
// Normal parse. Equivalent Timezone rules as time.Parse()
t, err := dateparse.ParseAny("3/1/2014")
// Parse Strict, error on ambigous mm/dd vs dd/mm dates
t, err := dateparse.ParseStrict("3/1/2014")
> returns error
// Return a string that represents the layout to parse the given date-time.
layout, err := dateparse.ParseFormat("May 8, 2009 5:57:51 PM")
> "Jan 2, 2006 3:04:05 PM"
Related
I'm getting a message from server like [0,"on",[6,1,5,"market",45.7]] and save it to []interface{} variable. I want to initialize struct with values of this array.
I'm totally new in Go and try to do it like:
import "golang.org/x/net/websocket"
...
var msg []interface{}
// Server send response: `[0,"on",[6,1,5,"market",45.7]]`
if err := websocket.Message.Receive(ws, &msg); err != nil {
logger.Println(err)
} else {
type Order struct {
ID int32,
GID int32,
CID int32,
Type string,
Amount float64
}
// here msg is [0,"on",[6,1,5,"market",45.7]]
switch msg[1] {
case "on":
if rawOrder, ok := msg[2].([]interface{}); ok {
order := Order{int32(rawOrder[0]), int32(rawOrder[1]), int32(rawOrder[2]), string(rawOrder[3]), float64(rawOrder[4])}
}
}
But I'm getting an error "Cannot convert an expression of the type 'interface{}' to the type 'int32'" and the next step is use switch for every rawOrder[i] type, but it's toooo long.
How can I do it easilly?
If you know that the codec used on the websocket will always be json, you can formally define Order and give it an UnmarshalJSON function to do the decoding for you.
import "golang.org/x/net/websocket"
type Order struct {
ID, GID, CID int32
Type string
Amount float64
}
func (o *Order) UnmarshalJSON(data []byte) error {
var first []json.RawMessage
err := json.Unmarshal(data, &first)
if err != nil {
return fmt.Errorf("invalid order, must be array: %w", err)
}
if len(first) != 3 {
return fmt.Errorf("invalid order, length must be 3, got %d", len(first))
}
var second string
err = json.Unmarshal(first[1], &second)
if err != nil {
return fmt.Errorf("invalid order, second element must be string: %w", err)
}
switch second {
case "on":
var third []json.RawMessage
err = json.Unmarshal(first[2], &third)
if err != nil {
return fmt.Errorf("invalid order, third element must be array: %w", err)
}
if len(third) != 5 {
return fmt.Errorf("invalid order, element 3 length must be 5, got %d", len(third))
}
for i, f := range []interface{}{&o.ID, &o.GID, &o.CID, &o.Type, &o.Amount} {
err = json.Unmarshal(third[i], f)
if err != nil {
return fmt.Errorf("invalid order, wrong type for element 3[%d]: %w", i, err)
}
}
return nil
}
return fmt.Errorf("invalid order, unknown type %q", second)
}
...
var msg *Order
// Server send response: `[0,"on",[6,1,5,"market",45.7]]`
if err := websocket.JSON.Receive(ws, &msg); err != nil {
logger.Println(err)
}
// msg is now an &Order{ID:6, GID:1, CID:5, Type:"market", Amount:45.7}
}
The reason the UnmarshalJSON function is huge is because your API is bad. If you control the Server; then you should avoid using mixed types in the same array, and you should avoid using arrays for relational data.
So I am hitting an API that returns a JSON response and I am unmarshalling it into a struct like so:
package main
type ProcessedRecords struct {
SLMIndividualID string `json:"individual_id"`
HouseholdPosition int `json:"Household Position"`
IndividualFirstName string `json:"individual_first_name"`
}
func main() {
req, _ := http.NewRequest(method, url, payload)
res, err := client.Do(req)
if err != nil {
fmt.Println(err)
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
fmt.Println(err)
}
fmt.Println(body)
var responseObject Response
json.Unmarshal(body, &responseObject)
fmt.Println(responseObject)
which works great. However I need to marshal this struct again but I want to use the Struct Fields as keys instead of the json: ... fields. I am using the following code:
recordsInput := []*firehose.Record{}
for i := 0; i < len(records); i++ {
if len(recordsInput) == 500 {
* code to submit records, this part works fine *
}
b, err := json.Marshal(records[i])
if err != nil {
log.Printf("Error: %v", err)
}
record := &firehose.Record{Data: b}
recordsInput = append(recordsInput, record)
}
This does submit records successfully but it's in the format:
{"individual_id":"33c05b49-149b-480f-b1c2-3a3b30e0cb6f","Household Position":1...}
and I'd like it in the format:
{"SLMIndividualId":"33c05b49-149b-480f-b1c2-3a3b30e0cb6f","HouseholdPosition":1...}
How can I achieve this?
Those tags say how the struct should be marshalled, so if they are present, that is how the output will be. You'll need to convert it to a matching struct that does not have the json: tags:
type ProcessedRecords struct {
SLMIndividualID string `json:"individual_id"`
HouseholdPosition int `json:"Household Position"`
IndividualFirstName string `json:"individual_first_name"`
}
type ProcessedRecordsOut struct {
SLMIndividualID string
HouseholdPosition int
IndividualFirstName string
}
func process() {
var in ProcessedRecords
json.Unmarshal(data, &in)
// Convert to same type w/o tags
out := ProcessedRecordsOut(in)
payload, _ := json.Marshal(out)
// ...
}
See a working example here: https://play.golang.org/p/p0Fc8DJotYE
You can omit fields one-way by defining a custom type and implementing the correct interface, e.g.
package main
import (
"encoding/json"
"fmt"
)
type Animal struct {
Name ReadOnlyString
Order string
}
type ReadOnlyString string
func (ReadOnlyString) UnmarshalJSON([]byte) error { return nil }
func main() {
x := Animal{"Bob", "First"}
js, err := json.Marshal(&x)
if err != nil {
fmt.Println("error:", err)
}
fmt.Printf("%s\n", js)
var jsonBlob = []byte(`{"Name": "Platypus", "Order": "Monotremata"}`)
if err := json.Unmarshal(jsonBlob, &x); err != nil {
fmt.Println("error:", err)
}
fmt.Printf("%#v\n\n", x)
}
https://go.dev/play/p/-mwBL0kIqM
Found this answer here: https://github.com/golang/go/issues/19423#issuecomment-284607677
I am writing a trie DS, json gzipped into a file trieSample.json.gz, and reading it back into the struct. Strangely, the unmarshal succeeds but the struct is not populated.
I have tried both json.Unmarshal and json.Decoder to no avail. Need help in finding what I am missing here. There is no error thrown while reading, just that the struct doesn't have any keys.
If I try normal json marshal -> Write to file and Read from file -> Unmarshal, it works properly.
var charSet = "0123456789bcdefghjkmnopqrstuvwxyz"
const logTagSlice = "trie.log"
type trieSlice struct {
Children []*tNode `json:"c"`
Charset map[int32]int8 `json:"l"` // Charset is the legend of what charset is used to create the keys and how to position them in trie array
logger loggingapi.Logger `json:"-"`
capacity int `json:"-"` // capacity is the slice capacity to have enough to hold all the characters in the charset
}
type tNode struct {
Children []*tNode `json:"c"` // children represents the next valid runes AFTER the current one
IsLeaf bool `json:"e,omitempty"` // isLeaf represents if this node represents the end of a valid word
Value int16 `json:"v,omitempty"` // value holds the corresponding value for key value pair, where key is the whole tree of nodes starting from parent representing a valid word
}
// NewTrieSlice returns a Trie, charset represents how the children need to be positioned in the array
func NewTrieSlice(charset string, logger loggingapi.Logger) *trieSlice {
m := map[int32]int8{}
for index, r := range charset {
m[r] = int8(index)
}
return &trieSlice{
Charset: m,
Children: make([]*tNode, len(charset)),
logger: logger,
capacity: len(charset),
}
}
func newNode(capacity int) *tNode {
return &tNode{
Children: make([]*tNode, capacity),
}
}
// getPosition gets the array index position that the rune should be put in
func (t *trieSlice) getPosition(r int32) (index int8, found bool) {
if index, ok := t.Charset[r]; ok {
return index, true
}
return -1, false
}
// Add ...
func (t *trieSlice) Add(key string, val int16) {
if len(key) == 0 {
t.logger.Info(logTagSlice, "trying to add empty key, return with no action")
return
}
runes := []rune(key)
prefix := runes[0]
var child *tNode
var pos int
index, ok := t.getPosition(prefix)
if !ok {
t.logger.Info(logTagSlice, "key is not present in the charset %s, cannot add to trieSlice", prefix)
return
}
// trie node with same prefix doesnt exist
if child = t.Children[index]; child == nil {
child = newNode(len(t.Charset))
t.Children[index] = child
}
pos = 1
for pos <= len(runes) {
// base condition
if pos == len(key) {
child.IsLeaf = true
child.Value = val
return
}
prefix := runes[pos]
index, ok := t.getPosition(prefix)
if !ok {
t.logger.Info(logTagSlice, "key is not present in the charset %s, cannot add to trieSlice", prefix)
return
}
// repeat with child node if prefix is already present
if newChild := child.Children[index]; newChild == nil {
child.Children[index] = newNode(len(t.Charset))
child = child.Children[index]
} else {
child = newChild
}
pos++
}
}
// Test using gzip writer, reader
func TestSample(t *testing.T) {
// Create trie and add a few keys
trie := NewTrieSlice(charSet, loggingapi.NewStdOut())
trie.Add("test", 10)
trie.Add("test1", 20)
trie.Add("test2", 30)
trie.Add("test3", 40)
trie.Add("test4", 50)
// Write gzipped json to file
var network bytes.Buffer
b, err := json.Marshal(trie)
if err != nil {
fmt.Println("error in marshal ... ", err.Error())
t.Fail()
}
w := gzip.NewWriter(&network)
w.Write(b)
ioutil.WriteFile("../resources/trieSample.json.gz", []byte(network.String()), 0644)
w.Close()
// Read gzipped json from file into struct
trieUnmarshal := NewTrieSlice(charSet, loggingapi.NewStdOut())
trieDecoder := NewTrieSlice(charSet, loggingapi.NewStdOut())
// attempt via json Unmarshal
file, err := os.Open("../resources/trieSample.json.gz")
if err != nil {
fmt.Println(err.Error())
t.Fail()
}
r, err := gzip.NewReader(file)
if err != nil {
fmt.Println(err.Error())
t.Fail()
}
sc := bufio.NewScanner(r)
json.Unmarshal(sc.Bytes(), trieUnmarshal)
// attempt via json Decoder
b, err = ioutil.ReadFile("../resources/trieSample.json.gz")
if err != nil {
fmt.Println(err.Error())
t.Fail()
}
bReader := bytes.NewReader(b)
json.NewDecoder(bReader).Decode(trieDecoder)
// spew.Dump shows that object is not populated
spew.Dump(trieUnmarshal)
spew.Dump(trieDecoder)
}
spew.Dump shows that the trieSlice Children array has all nil elements
Close the compressor before using the data. Decompress the data before using it. Don't chop it up with inappropriate use of bufio.Scanner.
var network bytes.Buffer
b, err := json.Marshal(trie)
if err != nil {
fmt.Println("error in marshal ... ", err.Error())
t.Fail()
}
w := gzip.NewWriter(&network)
w.Write(b)
w.Close()
err = ioutil.WriteFile("trieSample.json.gz", network.Bytes(), 0644)
if err != nil {
log.Fatal(err)
}
trieDecoder := NewTrieSlice(charSet)
// attempt via json Unmarshal
file, err := os.Open("trieSample.json.gz")
if err != nil {
log.Fatal(err)
}
r, err := gzip.NewReader(file)
if err != nil {
log.Fatal(err)
}
err = json.NewDecoder(r).Decode(trieDecoder)
if err != nil {
log.Fatal(err)
}
spew.Dump(trieDecoder)
https://play.golang.org/p/pYup3v8-f4c
Assuming we have two yaml files
master.yaml
someProperty: "someVaue"
anotherProperty: "anotherValue"
override.yaml
someProperty: "overriddenVaue"
Is it possible to unmarshall, merge, and then write those changes to a file without having to define a struct for every property in the yaml file?
The master file has over 500 properties in it that are not at all important to the service at this point of execution, so ideally I'd be able to just unmarshal into a map, do a merge and write out in yaml again but I'm relatively new to go so wanted some opinions.
I've got some code to read the yaml into an interface but i'm unsure on the best approach to then merge the two.
var masterYaml interface{}
yamlBytes, _ := ioutil.ReadFile("master.yaml")
yaml.Unmarshal(yamlBytes, &masterYaml)
var overrideYaml interface{}
yamlBytes, _ = ioutil.ReadFile("override.yaml")
yaml.Unmarshal(yamlBytes, &overrideYaml)
I've looked into libraries like mergo but i'm not sure if that's the right approach.
I'm hoping that after the master I would be able to write out to file with properties
someProperty: "overriddenVaue"
anotherProperty: "anotherValue"
Assuming that you just want to merge at the top level, you can unmarshal into maps of type map[string]interface{}, as follows:
package main
import (
"io/ioutil"
"gopkg.in/yaml.v2"
)
func main() {
var master map[string]interface{}
bs, err := ioutil.ReadFile("master.yaml")
if err != nil {
panic(err)
}
if err := yaml.Unmarshal(bs, &master); err != nil {
panic(err)
}
var override map[string]interface{}
bs, err = ioutil.ReadFile("override.yaml")
if err != nil {
panic(err)
}
if err := yaml.Unmarshal(bs, &override); err != nil {
panic(err)
}
for k, v := range override {
master[k] = v
}
bs, err = yaml.Marshal(master)
if err != nil {
panic(err)
}
if err := ioutil.WriteFile("merged.yaml", bs, 0644); err != nil {
panic(err)
}
}
For a broader solution (with n input files), you can use this function. I have used #robox answer to do my solution:
func ReadValues(filenames ...string) (string, error) {
if len(filenames) <= 0 {
return "", errors.New("You must provide at least one filename for reading Values")
}
var resultValues map[string]interface{}
for _, filename := range filenames {
var override map[string]interface{}
bs, err := ioutil.ReadFile(filename)
if err != nil {
log.Info(err)
continue
}
if err := yaml.Unmarshal(bs, &override); err != nil {
log.Info(err)
continue
}
//check if is nil. This will only happen for the first filename
if resultValues == nil {
resultValues = override
} else {
for k, v := range override {
resultValues[k] = v
}
}
}
bs, err := yaml.Marshal(resultValues)
if err != nil {
log.Info(err)
return "", err
}
return string(bs), nil
}
So for this example you should call it with this order:
result, _ := ReadValues("master.yaml", "overwrite.yaml")
In the case you have an extra file newFile.yaml, you could also use this function:
result, _ := ReadValues("master.yaml", "overwrite.yaml", "newFile.yaml")
DEEP MERGE TWO YAML FILES
package main
import (
"fmt"
"io/ioutil"
"sigs.k8s.io/yaml"
)
func main() {
// declare two map to hold the yaml content
base := map[string]interface{}{}
currentMap := map[string]interface{}{}
// read one yaml file
data, _ := ioutil.ReadFile("conf.yaml")
if err := yaml.Unmarshal(data, &base); err != nil {
}
// read another yaml file
data1, _ := ioutil.ReadFile("conf1.yaml")
if err := yaml.Unmarshal(data1, ¤tMap); err != nil {
}
// merge both yaml data recursively
base = mergeMaps(base, currentMap)
// print merged map
fmt.Println(base)
}
func mergeMaps(a, b map[string]interface{}) map[string]interface{} {
out := make(map[string]interface{}, len(a))
for k, v := range a {
out[k] = v
}
for k, v := range b {
if v, ok := v.(map[string]interface{}); ok {
if bv, ok := out[k]; ok {
if bv, ok := bv.(map[string]interface{}); ok {
out[k] = mergeMaps(bv, v)
continue
}
}
}
out[k] = v
}
return out
}
I am get leveldb's all key-val to a map[string][]byte, but it is not running as my expection.
code is as below
package main
import (
"fmt"
"strconv"
"github.com/syndtr/goleveldb/leveldb"
)
func main() {
db, err := leveldb.OpenFile("db", nil)
if err != nil {
panic(err)
}
defer db.Close()
for i := 0; i < 10; i++ {
err := db.Put([]byte("key"+strconv.Itoa(i)), []byte("value"+strconv.Itoa(i)), nil)
if err != nil {
panic(err)
}
}
snap, err := db.GetSnapshot()
if err != nil {
panic(err)
}
if snap == nil {
panic("snap shot is nil")
}
data := make(map[string][]byte)
iter := snap.NewIterator(nil, nil)
for iter.Next() {
Key := iter.Key()
Value := iter.Value()
data[string(Key)] = Value
}
iter.Release()
if iter.Error() != nil {
panic(iter.Error())
}
for k, v := range data {
fmt.Println(string(k) + ":" + string(v))
}
}
but the result is below
key3:value9
key6:value9
key7:value9
key8:value9
key1:value9
key2:value9
key4:value9
key5:value9
key9:value9
key0:value9
rather not key0:value0
Problem is with casting around types (byte[] to string, etc.).
You are trying to print string values. To avoid unnecessary casting apply the following modifications:
Change data initialization into data := make(map[string]string)
Assign values into data with `data[string(Key)] = string(Value) (by the way, don't use capitalization for variables you aren't intend to export)
Print data's values with fmt.Println(k + ":" + v))
This should produce the following result:
key0:value0
key1:value1
key7:value7
key2:value2
key3:value3
key4:value4
key5:value5
key6:value6
key8:value8
key9:value9