Custom xml decoder issue - go

I have multiple test cases which pass, however this one fails. What am I missing here that is causing the decoder read the content of my target keys incorrectly?
const respGenericFault1 = `<?xml version='1.0' encoding='UTF-8'?>
<SOAP-ENV:Envelope
xmlns:SOAP-ENV="http://schemas.xmlsoap.org/soap/envelope/"
xmlns:xsi="http://www.w3.org/1999/XMLSchema-instance"
xmlns:xsd="http://www.w3.org/1999/XMLSchema">
<SOAP-ENV:Body>
<SOAP-ENV:Fault>
<faultcode xsi:type="xsd:string">SOAP-ENV:Client</faultcode>
<faultstring xsi:type="xsd:string">Failed to validate</faultstring>
</SOAP-ENV:Fault>
</SOAP-ENV:Body>
</SOAP-ENV:Envelope>`
type Fault struct {
FaultCode, FaultString string
}
func (f Fault) Error() string {
return "Fault Code: '" + f.FaultCode + "' FaultString: '" + f.FaultString + "'"
}
func ParseFault(b []byte) error {
reader := bytes.NewReader(b)
d := xml.NewDecoder(reader)
var start xml.StartElement
fault := Fault{}
found := false
// iterate through the tokens
for {
tok, _ := d.Token()
if tok == nil {
break
}
// switch on token type
switch t := tok.(type) {
case xml.StartElement:
start = t.Copy()
fmt.Println(start.Name.Local)
case xml.CharData:
key := strings.ToLower(start.Name.Local)
// fault was found, capture the values and mark as found
if key == "faultcode" {
found = true
fault.FaultCode = string(t)
fmt.Printf("%#v\n", string(t))
} else if key == "faultstring" {
found = true
fault.FaultString = string(t)
}
}
}
if found {
return fault
}
return nil
}
func main() {
err := ParseFault([]byte(respGenericFault1))
fmt.Printf("%#v\n", err)
}
Here is the playground url: http://play.golang.org/p/7PFPNsmast

Your code successfully captures the faultstring and faultcode, but then unintentionally overwrites it with xml.CharData containing the whitespace between tags.
Here is a fixed version: http://play.golang.org/p/s1aFFYtwcX . Comment out line 52 to see the failure mode.
Alternatively, you can use encoding/xml Unmarshal to parse the XML directly into a struct. See http://play.golang.org/p/lOsZRUJ63B

Related

Why doesn't YAML.v3 marshal a struct based on the String() in golang?

I have a struct which contains a type based on an enum. I am trying to render it to a user friendly string. Here's minimum viable code:
package main
import (
"fmt"
"gopkg.in/yaml.v3"
)
type Job struct {
Engine Engine `json:"Engine" yaml:"Engine"`
}
//go:generate stringer -type=Engine --trimprefix=Engine
type Engine int
const (
engineUnknown Engine = iota // must be first
EngineDocker
engineDone // must be last
)
func main() {
j := Job{Engine: EngineDocker}
fmt.Printf("%+v\n\n", j)
out, _ := yaml.Marshal(j)
fmt.Println(string(out))
}
Here's the generated code:
// Code generated by "stringer -type=Engine --trimprefix=Engine"; DO NOT EDIT.
package main
import "strconv"
func _() {
// An "invalid array index" compiler error signifies that the constant values have changed.
// Re-run the stringer command to generate them again.
var x [1]struct{}
_ = x[engineUnknown-0]
_ = x[EngineDocker-1]
_ = x[engineDone-2]
}
const _Engine_name = "engineUnknownDockerengineDone"
var _Engine_index = [...]uint8{0, 13, 19, 29}
func (i Engine) String() string {
if i < 0 || i >= Engine(len(_Engine_index)-1) {
return "Engine(" + strconv.FormatInt(int64(i), 10) + ")"
}
return _Engine_name[_Engine_index[i]:_Engine_index[i+1]]
}
Here's the output:
{Engine:1}
Engine: 1
Here's what I'd like the output to be:
{Engine:Docker}
Engine: Docker
I thought the String() in the generated file would accomplish this. Is there any way to do this? Thanks!
yaml marshaler doesn't use String method. Instead YAML uses encoding.TextMarshaler and encoding.TextUnmarshaler interfaces. Actually, all other codec schemes - JSON, XML, TOML, etc. - use those interfaces to read/write the values. So, if you implement those methods for your type, you will receive all other codecs for free.
Here is an example how to make a human-readable encoding for your enum: https://go.dev/play/p/pEcBmAM-oZJ
type Engine int
const (
engineUnknown Engine = iota // must be first
EngineDocker
engineDone // must be last
)
var engineNames []string
var engineNameToValue map[string]Engine
func init() {
engineNames = []string{"Unknown", "Docker"}
engineNameToValue = make(map[string]Engine)
for i, name := range engineNames {
engineNameToValue[strings.ToLower(name)] = Engine(i)
}
}
func (e Engine) String() string {
if e < 0 || int(e) >= len(engineNames) {
panic(fmt.Errorf("Invalid engine code: %d", e))
}
return engineNames[e]
}
func ParseEngine(text string) (Engine, error) {
i, ok := engineNameToValue[strings.ToLower(text)]
if !ok {
return engineUnknown, fmt.Errorf("Invalid engine name: %s", text)
}
return i, nil
}
func (e Engine) MarshalText() ([]byte, error) {
return []byte(e.String()), nil
}
func (e *Engine) UnmarshalText(text []byte) (err error) {
name := string(text)
*e, err = ParseEngine(name)
return
}
How it works:
func main() {
j := Job{Engine: EngineDocker}
fmt.Printf("%#v\n\n", j)
out, err := yaml.Marshal(j)
if err != nil {
panic(err)
}
fmt.Printf("YAML: %s\n", string(out))
var jj Job
err = yaml.Unmarshal(out, &jj)
if err != nil {
panic(err)
}
fmt.Printf("%#v\n\n", jj)
// == JSON ==
out, err = json.Marshal(j)
if err != nil {
panic(err)
}
fmt.Printf("JSON: %s\n", string(out))
var jjs Job
err = json.Unmarshal(out, &jjs)
if err != nil {
panic(err)
}
fmt.Printf("%#v\n\n", jjs)
}
the output
main.Job{Engine:1}
YAML: Engine: Docker
main.Job{Engine:1}
JSON: {"Engine":"Docker"}
main.Job{Engine:1}
See? It writes and reads strings to both YAML and JSON without any extra effort.

Changing value of field in a go struct

I'm making an http request in golang to an external api. It gives a general response of {"error":[]string, "result":changing interface{}}. depending on the function that is making the request, the Result field changes. Since I know the structure of the Result field for each function I run, I want to be able to change the value of Result before unmarshalling to json. I've tried to do this with the following code:
func GetAssets(output *Resp, resultType interface{}) error {
return publicRequest("/Assets", output, resultType)
}
func publicRequest(endPoint string, output *Resp, resultType interface{}) error {
url := Rest_url + Pub_rest_url + endPoint //"https://api.kraken.com/0/public/Assets in this case
resp, err := http.Get(url)
if err != nil {
return err
}
defer resp.Body.Close()
output.Result = resultType
return json.NewDecoder(resp.Body).Decode(&output)
}
Here is how it's being ran in main
type Resp struct {
Error []string `json:"error"`
Result interface{} `json:"result"`
}
type AssetInfo struct {
Aclass string `json:"aclass"`
Altname string `json:"altname"`
Decimals int `json:"decimals"`
Display int `json:"display_decimals"`
}
func main() {
var result map[string]AssetInfo
jsonData := Resp{}
rest_api_client.GetAssets(&jsonData, result)
fmt.Println(jsonData)
}
The issue is that it doesn't unmarshal correctly. A map is created for each asset, but the data contained inside of each asset is also being stored inside of a map. I'm not sure if I explained this well, but here is the current response after unmarshalling to understand what I mean.
Here is the data type of Resp.Result: map[string]interface {}
{[] map[1INCH:map[aclass:currency altname:1INCH decimals:10 display_decimals:5] AAVE:map[aclass:currency altname:AAVE decimals:10 display_decimals:5] ACA:map[aclass:currency altname:ACA decimals:10 display_decimals:5] ADA:map[aclass:currency altname:ADA decimals:8 display_decimals:6]...}
The response type I'm looking for is map[string]AssetInfo. Hopefully it could be unmarshalled like this:
{[] map[1INCH:{currency 1INCH 10 5} AAVE:{currency AAVE 10 5} ACA:{currency ACA 10 5} ADA:{currency ADA 8 6} ADA.S:{currency ADA.S 8 6}...}
Any help? I'd rather keep the Resp struct as generic as possible and just change the value of the Result field (if this is even possible to do correctly) since I plan to have multiple functions that call different endpoints of the api, and they'll all have the same underlying response type of the Resp struct with different Result types
You can view a working example in the following repo:
https://github.com/alessiosavi/GoArbitrage/blob/e107af466852b1ed30c2413eb4401595f7412b4f/markets/kraken/kraken.go
Basically, I've defined the following structure:
type Tickers struct {
Error []interface{} `json:"error"`
Result map[string]Ticker `json:"result"`
}
type Ticker struct {
Aclass string `json:"aclass"`
Altname string `json:"altname"`
Decimals int `json:"decimals"`
DisplayDecimals int `json:"display_decimals"`
}
Than I execute the request in the following way:
const KRAKEN_TICKERS_URL string = `https://api.kraken.com/0/public/Assets`
type Kraken struct {
PairsNames []string `json:"pairs_name"`
Pairs map[string]datastructure.KrakenPair `json:"pairs"`
OrderBook map[string]datastructure.KrakenOrderBook `json:"orderbook"`
MakerFee float64 `json:"maker_fee"`
TakerFees float64 `json:"taker_fee"`
// FeePercent is delegated to save if the fee is in percent or in coin
FeePercent bool `json:"fee_percent"`
Tickers []string
}
// Init is delegated to initialize the maps for the kraken
func (k *Kraken) Init() {
k.Pairs = make(map[string]datastructure.KrakenPair)
k.OrderBook = make(map[string]datastructure.KrakenOrderBook)
k.SetFees()
}
// SetFees is delegated to initialize the fee type/amount for the given market
func (k *Kraken) SetFees() {
k.MakerFee = 0.16
k.TakerFees = 0.26
k.FeePercent = true
}
func (k *Kraken) GetTickers() error {
res := datastructure.Tickers{}
var err error
var request req.Request
var data []byte
var tickers []string
resp := request.SendRequest(KRAKEN_TICKERS_URL, "GET", nil, nil, false, 10*time.Second)
if resp.Error != nil {
zap.S().Debugw("Error during http request. Err: " + resp.Error.Error())
return resp.Error
}
if resp.StatusCode != 200 {
zap.S().Warnw("Received a non 200 status code: " + strconv.Itoa(resp.StatusCode))
return errors.New("NON_200_STATUS_CODE")
}
data = resp.Body
if err = json.Unmarshal(data, &res); err != nil {
zap.S().Warn("ERROR! :" + err.Error())
return err
}
zap.S().Infof("Data: %v", res.Result)
tickers = make([]string, len(res.Result))
i := 0
for key := range res.Result {
tickers[i] = res.Result[key].Altname
i++
}
k.Tickers = tickers
return nil
}

Idiomatic way to validate http request in go

I need to validate that my http request has two parameters, Start and End. Currently, I set a default value that should not appear as either of the parameters and check for it along with other invalid values. However, this feels like a hack. What should be the proper way to do this?
Here is my code:
type Request struct {
Start int `json: "start"`
End int `json: "end"`
}
func HandlePost(w http.ResponseWriter, r *http.Request) {
body , _ := ioutil.ReadAll(r.Body)
reqData := Request{Start: -1, End: -1} // < whats the correct way to do this
json.Unmarshal(body, &reqData)
if reqData.Start < 0 && reqData.End < 0 {
w.WriteHeader(http.StatusBadRequest)
return
}
// rest of the logic
}
You can use https://github.com/asaskevich/govalidator for basic way of validating the request. But in case you want something more sophisticated, you need to write your own custom validator function. e.g.
type Request struct {
Start int `json: "start"`
End int `json: "end"`
}
func (a *Request) validate() url.Values {
err := url.Values{}
// Write your own validation rules
if a.Start < 0 {
err.Add("Start", "Start cannot be less than 0");
}
return err;
}
func handler(w http.ResponseWriter, r *http.Request) {
requestBody := &Request{}
defer r.Body.Close()
if err := json.NewDecoder(r.Body).Decode(requestBody); err != nil {
panic(err)
}
if errors := requestBody.validate(); len(errors) > 0 {
err := map[string]interface{}{"validationError": errors}
w.Header().Set("Content-type", "application/json")
w.WriteHeader(http.StatusBadRequest)
json.NewEncoder(w).Encode(err)
}
fmt.Fprint(w, "success request scenario!")
}
Here's another way to validate structures using struct tags and pointers. Note that if 0 is a valid thing to pass, then this solution will not work.
omitempty considers the 0 value to be empty. If you want this to work will considering 0 to be valid remove the pointers and modify the IsValid method
package main
import (
"encoding/json"
"fmt"
)
type Request struct {
Start *int `json: "start,omitempty"`
End *int `json: "end,omitempty"`
}
func (r Request) IsValid() (bool, error) {
if r.Start == nil {
return false, fmt.Errorf("start is missing")
}
if r.End == nil {
return false, fmt.Errorf("end is missing")
}
return true, nil
}
var (
invalidStartb = `{"end": 1}`
invalidEndb = `{"start": 1}`
valid = `{"start": 1, "end": 1}`
)
func main() {
var r Request
_ = json.Unmarshal([]byte(invalidStartb), &r)
fmt.Println(r.IsValid())
r = Request{}
_ = json.Unmarshal([]byte(invalidEndb), &r)
fmt.Println(r.IsValid())
r = Request{}
_ = json.Unmarshal([]byte(valid), &r)
fmt.Println(r.IsValid())
}
runnable version here https://goplay.space/#Z0eqLpEHO37
You can use https://github.com/buger/jsonparser getInt.
You'll get an error if the json is missing the expected key.
I recommend using benchmark and not decided by the code beauty or any other hunch

Parsing plist xml

How to parse xml in such silly format:
<key>KEY1</key><string>VALUE OF KEY1</string>
<key>KEY2</key><string>VALUE OF KEY2</string>
<key>KEY3</key><integer>42</integer>
<key>KEY3</key><array>
<integer>1</integer>
<integer>2</integer>
</array>
Parsing would be very simple if all values would have same type - for example strings. But in my case each value could be string, data, integer, boolean, array or dict.
This xml looks nearly like json, but unfortunately format is fixed, and I cannot change it. And I would prefer solution without any external packages.
Use a lower-level parsing interface provided by encoding/xml which allows you to iterate over individual tokens in the XML stream (such as "start element", "end element" etc).
See the Token() method of the encoding/xml's Decoder type.
Since the data is not well structured, and you can't modify the format, you can't use xml.Unmarshal, so you can process the XML elements by creating a new Decoder, then iterate over the tokens and use DecodeElement to process them one by one. In my sample code below, it puts everything in a map. The code is also on github here...
package main
import (
"encoding/xml"
"strings"
"fmt"
)
type PlistArray struct {
Integer []int `xml:"integer"`
}
const in = "<key>KEY1</key><string>VALUE OF KEY1</string><key>KEY2</key><string>VALUE OF KEY2</string><key>KEY3</key><integer>42</integer><key>KEY3</key><array><integer>1</integer><integer>2</integer></array>"
func main() {
result := map[string]interface{}{}
dec := xml.NewDecoder(strings.NewReader(in))
dec.Strict = false
var workingKey string
for {
token, _ := dec.Token()
if token == nil {
break
}
switch start := token.(type) {
case xml.StartElement:
fmt.Printf("startElement = %+v\n", start)
switch start.Name.Local {
case "key":
var k string
err := dec.DecodeElement(&k, &start)
if err != nil {
fmt.Println(err.Error())
}
workingKey = k
case "string":
var s string
err := dec.DecodeElement(&s, &start)
if err != nil {
fmt.Println(err.Error())
}
result[workingKey] = s
workingKey = ""
case "integer":
var i int
err := dec.DecodeElement(&i, &start)
if err != nil {
fmt.Println(err.Error())
}
result[workingKey] = i
workingKey = ""
case "array":
var ai PlistArray
err := dec.DecodeElement(&ai, &start)
if err != nil {
fmt.Println(err.Error())
}
result[workingKey] = ai
workingKey = ""
default:
fmt.Errorf("Unrecognized token")
}
}
}
fmt.Printf("%+v", result)
}

Iterate Over String Fields in Struct

I'm looking to iterate over the string fields of a struct so I can do some clean-up/validation (with strings.TrimSpace, strings.Trim, etc).
Right now I have a messy switch-case that's not really scalable, and as this isn't in a hot spot of my application (a web form) it seems leveraging reflect is a good choice here.
I'm at a bit of a roadblock for how to implement this however, and the reflect docs are a little confusing to me (I've been digging through some other validation packages, but they're way too heavyweight + I'm using gorilla/schema for the unmarshalling part already):
Iterate over the struct
For each field of type string, apply whatever I need to from the strings package i.e. field = strings.TrimSpace(field)
If there exists a field.Tag.Get("max"), we'll use that value (strconv.Atoi, then unicode.RuneCountInString)
Provide an error slice that's also compatible with the error interface type
type FormError []string
type Listing struct {
Title string `max:"50"`
Location string `max:"100"`
Description string `max:"10000"`
ExpiryDate time.Time
RenderedDesc template.HTML
Contact string `max:"255"`
}
// Iterate over our struct, fix whitespace/formatting where possible
// and return errors encountered
func (l *Listing) Validate() error {
typ := l.Elem().Type()
var invalid FormError
for i = 0; i < typ.NumField(); i++ {
// Iterate over fields
// For StructFields of type string, field = strings.TrimSpace(field)
// if field.Tag.Get("max") != "" {
// check max length/convert to int/utf8.RuneCountInString
if max length exceeded, invalid = append(invalid, "errormsg")
}
if len(invalid) > 0 {
return invalid
}
return nil
}
func (f FormError) Error() string {
var fullError string
for _, v := range f {
fullError =+ v + "\n"
}
return "Errors were encountered during form processing: " + fullError
}
Thanks in advance.
What you want is primarily the methods on reflect.Value called NumFields() int and Field(int). The only thing you're really missing is the string check and SetString method.
package main
import "fmt"
import "reflect"
import "strings"
type MyStruct struct {
A,B,C string
I int
D string
J int
}
func main() {
ms := MyStruct{"Green ", " Eggs", " and ", 2, " Ham ", 15}
// Print it out now so we can see the difference
fmt.Printf("%s%s%s%d%s%d\n", ms.A, ms.B, ms.C, ms.I, ms.D, ms.J)
// We need a pointer so that we can set the value via reflection
msValuePtr := reflect.ValueOf(&ms)
msValue := msValuePtr.Elem()
for i := 0; i < msValue.NumField(); i++ {
field := msValue.Field(i)
// Ignore fields that don't have the same type as a string
if field.Type() != reflect.TypeOf("") {
continue
}
str := field.Interface().(string)
str = strings.TrimSpace(str)
field.SetString(str)
}
fmt.Printf("%s%s%s%d%s%d\n", ms.A, ms.B, ms.C, ms.I, ms.D, ms.J)
}
(Playground link)
There are two caveats here:
You need a pointer to what you're going to change. If you have a value, you'll need to return the modified result.
Attempts to modify unexported fields generally will cause reflect to panic. If you plan on modifying unexported fields, make sure to do this trick inside the package.
This code is rather flexible, you can use switch statements or type switches (on the value returned by field.Interface()) if you need differing behavior depending on the type.
Edit: As for the tag behavior, you seem to already have that figured out. Once you have field and have checked that it's a string, you can just use field.Tag.Get("max") and parse it from there.
Edit2: I made a small error on the tag. Tags are part of the reflect.Type of a struct, so to get them you can use (this is a bit long-winded) msValue.Type().Field(i).Tag.Get("max")
(Playground version of the code you posted in the comments with a working Tag get).
I got beat to the punch, but since I went to the work, here's a solution:
type FormError []*string
type Listing struct {
Title string `max:"50"`
Location string `max:"100"`
Description string `max:"10000"`
ExpiryDate time.Time
RenderedDesc template.HTML
Contact string `max:"255"`
}
// Iterate over our struct, fix whitespace/formatting where possible
// and return errors encountered
func (l *Listing) Validate() error {
listingType := reflect.TypeOf(*l)
listingValue := reflect.ValueOf(l)
listingElem := listingValue.Elem()
var invalid FormError = []*string{}
// Iterate over fields
for i := 0; i < listingElem.NumField(); i++ {
fieldValue := listingElem.Field(i)
// For StructFields of type string, field = strings.TrimSpace(field)
if fieldValue.Type().Name() == "string" {
newFieldValue := strings.TrimSpace(fieldValue.Interface().(string))
fieldValue.SetString(newFieldValue)
fieldType := listingType.Field(i)
maxLengthStr := fieldType.Tag.Get("max")
if maxLengthStr != "" {
maxLength, err := strconv.Atoi(maxLengthStr)
if err != nil {
panic("Field 'max' must be an integer")
}
// check max length/convert to int/utf8.RuneCountInString
if utf8.RuneCountInString(newFieldValue) > maxLength {
// if max length exceeded, invalid = append(invalid, "errormsg")
invalidMessage := `"`+fieldType.Name+`" is too long (max allowed: `+maxLengthStr+`)`
invalid = append(invalid, &invalidMessage)
}
}
}
}
if len(invalid) > 0 {
return invalid
}
return nil
}
func (f FormError) Error() string {
var fullError string
for _, v := range f {
fullError = *v + "\n"
}
return "Errors were encountered during form processing: " + fullError
}
I see you asked about how to do the tags. Reflection has two components: a type and a value. The tag is associated with the type, so you have to get it separately than the field: listingType := reflect.TypeOf(*l). Then you can get the indexed field and the tag from that.
I don't know if it's a good way, but I use it like this.
https://play.golang.org/p/aQ_hG2BYmMD
You can send the address of a struct to this function.
Sorry for My English is not very good.
trimStruct(&someStruct)
func trimStruct(v interface{}) {
bytes, err := json.Marshal(v)
if err != nil {
fmt.Println("[trimStruct] Marshal Error :", err)
}
var mapSI map[string]interface{}
if err := json.Unmarshal(bytes, &mapSI); err != nil {
fmt.Println("[trimStruct] Unmarshal to byte Error :", err)
}
mapSI = trimMapStringInterface(mapSI).(map[string]interface{})
bytes2, err := json.Marshal(mapSI)
if err != nil {
fmt.Println("[trimStruct] Marshal Error :", err)
}
if err := json.Unmarshal(bytes2, v); err != nil {
fmt.Println("[trimStruct] Unmarshal to b Error :", err)
}
}
func trimMapStringInterface(data interface{}) interface{} {
if values, valid := data.([]interface{}); valid {
for i := range values {
data.([]interface{})[i] = trimMapStringInterface(values[i])
}
} else if values, valid := data.(map[string]interface{}); valid {
for k, v := range values {
data.(map[string]interface{})[k] = trimMapStringInterface(v)
}
} else if value, valid := data.(string); valid {
data = strings.TrimSpace(value)
}
return data
}

Resources