I have some code for handling a YAML config file that's getting a little out-of-control w/ type assertions and I feel like there must be a better way to do this.
Here's the relevant snippet from my config file:
plugins:
taxii20:
default: default
api_roots:
default:
auth:
- ldap
- mutualtls
collections:
all:
selector: g.V().Save("<type>").Save("<created>").All()
selector_query_lang: gizmo
And here's my parsing code:
func parseTaxiiConfig() {
plg.ConfigMutex.Lock()
taxiiConfig := plg.ConfigData.Plugins["taxii20"].(map[interface{}]interface{})
ConfigData = &Config{}
if taxiiConfig["default"] != nil {
ConfigData.DefaultRoot = taxiiConfig["default"].(string)
}
if taxiiConfig["api_roots"] != nil {
ConfigData.APIRoots = make([]model.APIRoot, 0)
iroots := taxiiConfig["api_roots"].(map[interface{}]interface{})
for iname, iroot := range iroots {
root := model.APIRoot{Name: iname.(string)}
authMethods := iroot.(map[interface{}]interface{})["auth"].([]interface{})
root.AuthMethods = make([]string, 0)
for _, method := range authMethods {
root.AuthMethods = append(root.AuthMethods, method.(string))
}
collections := iroot.(map[interface{}]interface{})["collections"].(map[interface{}]interface{})
root.Collections = make([]model.Collection, 0)
for icolName, icollection := range collections {
collection := model.Collection{Name: icolName.(string)}
collection.Selector = icollection.(map[interface{}]interface{})["selector"].(string)
collection.SelectorQueryLang = icollection.(map[interface{}]interface{})["selector_query_lang"].(string)
root.Collections = append(root.Collections, collection)
}
ConfigData.APIRoots = append(ConfigData.APIRoots, root)
}
}
plg.ConfigMutex.Unlock()
// debug
fmt.Println(ConfigData)
}
The code works as intended, but there's just so many type assertions here and I can't shake the feeling that I'm missing a better way.
One possible critical item of note, as the config implies, this is configuration for a Caddy-style plugin system, so the main config parser cannot know ahead of time what the shape of the plugin config will look like. It has to delegate processing of the plugin's portion of the config file to the plugin itself.
Here's what I came up with instead. Much more readable.
// Config represents TAXII 2.0 plugin structure
type Config struct {
DefaultRoot string
APIRoots []model.APIRoot
}
// Intermediate config for mapstructure
type configRaw struct {
DefaultRoot string `mapstructure:"default"`
APIRoots map[string]apiRootRaw `mapstructure:"api_roots"`
}
type apiRootRaw struct {
AuthMethods []string `mapstructure:"auth"`
Collections map[string]collectionRaw `mapstructure:"collections"`
}
type collectionRaw struct {
Selector string `mapstructure:"selector"`
SelectorQueryLang string `mapstructure:"selector_query_lang"`
}
func parseTaxiiConfig() error {
plg.ConfigMutex.Lock()
defer plg.ConfigMutex.Unlock()
taxiiConfig := plg.ConfigData.Plugins["taxii20"].(map[interface{}]interface{})
fmt.Println(taxiiConfig)
ConfigData = &Config{}
raw := &configRaw{}
err := mapstructure.Decode(taxiiConfig, raw)
if err != nil {
return err
}
ConfigData.DefaultRoot = raw.DefaultRoot
ConfigData.APIRoots = make([]model.APIRoot, 0)
for name, root := range raw.APIRoots {
apiRoot := model.APIRoot{Name: name}
apiRoot.AuthMethods = root.AuthMethods
apiRoot.Collections = make([]model.Collection, 0)
for colName, col := range root.Collections {
collection := model.Collection{Name: colName}
collection.Selector = col.Selector
collection.SelectorQueryLang = col.SelectorQueryLang
apiRoot.Collections = append(apiRoot.Collections, collection)
}
ConfigData.APIRoots = append(ConfigData.APIRoots, apiRoot)
}
return nil
}
Related
I'm working on a resolver function for a GraphQL query for a BE I'm writing in Go. In the resolver, I have user data that I want to update, using an input value containing several possible update properties.
In JavaScript, this can be done quickly through destructuring (pseudo):
const mergedObj = {...oldProps, ...newProps}
For now, my resolver function looks like this (using gqlgen for GraphQL Go resolvers):
func (r *mutationResolver) ModifyUser(ctx context.Context, input *model.ModifyUserInput) (*model.User, error) {
id := input.ID
us, ok := r.Resolver.UserStore[id]
if !ok {
return nil, fmt.Errorf("not found")
}
if input.FirstName != nil {
us.FirstName = *input.FirstName
}
if input.LastName != nil {
us.LastName = *input.LastName
}
if input.ProfileImage != nil {
us.ProfileImage = input.ProfileImage
}
if input.Password != nil {
us.Password = *input.Password
}
if input.Email != nil {
us.Email = *input.Email
}
if input.InTomorrow != nil {
us.InTomorrow = input.InTomorrow
}
if input.DefaultDaysIn != nil {
us.DefaultDaysIn = input.DefaultDaysIn
}
r.Resolver.UserStore[id] = us
return &us, nil
}
This feels quite boilerplatey. Would it make sense in this situation to iterate through struct keys? Or is there another pattern I'm missing?
Use a function to reduce the boilerplate:
func mergef[T any](a, b *T) {
if b != nil {
*a = *b
}
}
...
mergef(&us.FirstName, input.FirstName)
mergef(&us.LastName, input.LastName)
...
Use the reflect package to reduce more boilerplate:
// merge sets fields in struct pointed to by d to
// dereferenced fields in struct pointed to by s.
//
// Argument s must point to a struct with pointer type
// fields.
// Argument d must point to a struct with fields that
// correspond to the fields in s: there must be a field
// in d with the same name as a field in s; the type of
// the field in s must be a pointer to the type of the field
// in d.
func merge(d, s any) {
sv := reflect.ValueOf(s).Elem()
dv := reflect.ValueOf(d).Elem()
for i := 0; i < sv.NumField(); i++ {
sf := sv.Field(i)
if sf.IsNil() {
continue
}
df := dv.FieldByName(sv.Type().Field(i).Name)
df.Set(sf.Elem())
}
}
Employ the function like this:
merge(us, input)
I have a GO script that generates pager duty on call reporting, and it has its own config.yaml file as such:
# PagerDuty auth token
pdAuthToken: 12345
# Explicitly set report time range (RFC822)
reportTimeRange:
start: 01 Jan 20 00:00 UTC
end: 01 Feb 20 00:00 UTC
# Rotation general information
rotationInfo:
dailyRotationStartsAt: 8
checkRotationChangeEvery: 30 # minutes
I need to pass environment variables in the config.yaml file. I tried to use ${THE_VARIABLE} as such:
reportTimeRange:
start: ${THE_VARIABLE}
Can anyone help on how I can passmy Linux environment variables in the config.yaml file without the need of editing the script.
After unmarshaling the yaml file you can use reflect on the result to update any string fields whose value matches variable format of your choice.
var reVar = regexp.MustCompile(`^\${(\w+)}$`)
func fromenv(v interface{}) {
_fromenv(reflect.ValueOf(v).Elem())// assumes pointer to struct
}
// recursive
func _fromenv(rv reflect.Value) {
for i := 0; i < rv.NumField(); i++ {
fv := rv.Field(i)
if fv.Kind() == reflect.Ptr {
fv = fv.Elem()
}
if fv.Kind() == reflect.Struct {
_fromenv(fv)
continue
}
if fv.Kind() == reflect.String {
match := reVar.FindStringSubmatch(fv.String())
if len(match) > 1 {
fv.SetString(os.Getenv(match[1]))
}
}
}
}
https://play.golang.org/p/1zuK7Mhtvsa
Alternatively, you could declare types that implement the yaml.Unmarshaler interface and use those types for fields in the config struct that expect the corresponding yaml properties to contain environment variables.
type Config struct {
ReportTimeRange struct {
Start StringFromEnv `yaml:"start"`
} `yaml:"reportTimeRange"`
}
var reVar = regexp.MustCompile(`^\${(\w+)}$`)
type StringFromEnv string
func (e *StringFromEnv) UnmarshalYAML(value *yaml.Node) error {
var s string
if err := value.Decode(&s); err != nil {
return err
}
if match := reVar.FindStringSubmatch(s); len(match) > 0 {
*e = StringFromEnv(os.Getenv(match[1]))
} else {
*e = StringFromEnv(s)
}
return nil
}
https://play.golang.org/p/Zy0rXJ7RRdC
What I understand is that you want to replace the placeholder {{ .THE_VARIABLE }} with a environment var at program start in memory, without modifying the yaml file.
The idea is to load the yaml file into a var, use template.Execute to replace the data. Finally Unmarshal the string.
I just kept it simple.
YAML FILE:
Start: {{ .THE_VARIABLE }}
Replacing data:
fileData, _ := ioutil.ReadFile("test.yml")
var finalData bytes.Buffer
t := template.New("config")
t, err := t.Parse(string(fileData))
if err != nil {
panic(err)
}
data := struct {
THE_VARIABLE int
}{
THE_VARIABLE: 30, // replace with os.Getenv("FOO")
}
t.Execute(&finalData, data)
str := finalData.String()
fmt.Println(str)
// unmarshal YAML here - from finalData.Bytes()
Output:
Start: 30
Is there a way to search for a value in Hashicorp Vault? I am trying to write Golang code to search and list all locations a value appears in vault. It would be similar to golang's walk function on directories. Does anyone have a good approach for this? I was thinking of using concurrency to search vault for a value. Thank you
Below is a sample of the code I came up with. I am looking on how to make this faster by using concurrency. Is there a way to traverse a directory concurrently?
func walkDir(client *api.Client, path string) {
var value *api.Secret
var err error
if path != "" {
value, err = client.Logical().List(path)
} else {
path = vault_path
value, err = client.Logical().List(path)
}
if err != nil {
fmt.Println(err)
}
var datamap map[string]interface{}
datamap = value.Data
data := datamap["keys"].([]interface{})
for _, item := range data {
itemString := item.(string)
if strings.HasSuffix(itemString, "/") {
walkDir(client, path+itemString)
} else {
//its a secret
data := read(client, path+itemString)
if *searchKey!="" && searchForKey(data,*searchKey){
fmt.Println(path + itemString)
}
if *searchValue!="" && searchForValue(data,*searchValue){
fmt.Println(path + itemString)
}
}
}
}
func read(client *api.Client, path string) map[string]interface{} {
value, err := client.Logical().Read(path)
if err != nil {
fmt.Println(err)
}
values := value.Data
return values
}
func searchForValue(mapp map[string]interface{}, searchValue string) bool {
for _, value := range mapp {
if searchValue == value {
return true
}
}
return false
}
func searchForKey(mapp map[string]interface{}, searchKey string) bool {
for key := range mapp {
if searchKey == key {
return true
}
}
return false
}
You can LIST "directories" in Vault (I'm assuming you're just looking at the kv engine). So treat it somewhat like a regular file-system: start at the root, list the entries, check the contents of each of them for that value, then iterate through each entry, listing its contents, and so forth.
https://www.vaultproject.io/api-docs/secret/kv/kv-v1#list-secrets
I have similar setup as bellow, how can I access my extension values from XYZ enum using "github.com/golang/protobuf/proto"?
extend google.protobuf.EnumValueOptions {
Details meta = 50001;
}
message Details {
string description = 1;
}
enum MyEnum {
MY_ENUM_UNSPECIFIED = 0;
XYZ = 1 [deprecated=true, (meta) = {description: "lorem ipsum"}];
}
I'm aware of proto.GetExtension(proto.Message, proto.ExtensionDesc), however I wasn't able to figure out how it can be used for the enum...
Bit late but I've just run into the same; you can do it like this:
fd, _ := descriptor.ForMessage(&pb.Details{})
for _, e := range fd.EnumType {
if e.GetName() == "MyEnum" {
for _, v := range e.Value {
ext, err := proto.GetExtension(v.Options, pb.E_Meta)
if err == nil {
details := ext.(*pb.Details)
// do stuff with details
}
}
}
}
There might be a more direct way of getting the enum descriptor, although I haven't managed after a bit of wrangling.
Some of the methods used in current best answer is now deprecated and it is a bit lengthy.
Here is how I got it:
// pd is the module of your complied protobuf files
fd := pd.File_name_of_your_proto_file_proto
enumDesc := fd.Enums().ByName("MyEnum")
if enumDesc == nil {
panic()
}
enumValDesc := enumDesc.Values().ByName("XYZ")
if enumValDesc == nil {
panic()
}
ext := proto.GetExtension(enumValDesc.Options(), pd.E_Meta)
if enumValDesc == nil {
panic()
}
meta := ext.(*Details)
Let me know if there is a better way.
After many hours, I have found a method to access the description for the enum. Here is my implementation, I hope it helps.
In a file called enum.go in the same package as the generated .pb file, I added this method to the enum type that retrieves the description.
func (t MyEnum) GetValue() (*Details, error) {
tt, err := proto.GetExtension(proto.MessageV1(t.Descriptor().Values().ByNumber(t.Number()).Options()), E_Details)
if err != nil {
return nil, err
}
return tt.(*Details), nil
}
I am sure there is an easier way, but until somebody finds one, this should work.
Trying to Unmarshal a hcl config file to a struct, using viper, this error is returned: 1 error(s) decoding:\n\n* 'NATS' expected a map, got 'slice'. What is missing?
The code:
func lab() {
var c conf
// config file
viper.SetConfigName("draft")
viper.AddConfigPath(".")
viper.SetConfigType("hcl")
if err := viper.ReadInConfig(); err != nil {
log.Error(err)
return
}
log.Info(viper.Get("NATS")) // gives [map[port:10041 username:cl1 password:__Psw__4433__ http_port:10044]]
if err := viper.Unmarshal(&c); err != nil {
log.Error(err)
return
}
log.Infow("got conf", "conf", c)
}
type conf struct {
NATS struct {
HTTPPort int
Port int
Username string
Password string
}
}
And the config file (draft.hcl inside current directory):
NATS {
HTTPPort = 10044
Port = 10041
Username = "cl1"
Password = "__Psw__4433__"
}
Edit
Have checked this struct with hcl package and it gets marshaled/unmarshalled correctly. Also this works correctly with yaml and viper.
There is a difference between these two where log.Info(viper.Get("NATS")) is called. While the hcl version returns a slice of maps, the yaml version returns a map: map[password:__psw__4433__ httpport:10044 port:10041 username:cl1].
Your conf struct is not matching the HCL. When converted to json the HCL looks like below
{
"NATS": [
{
"HTTPPort": 10044,
"Password": "__Psw__4433__",
"Port": 10041,
"Username": "cl1"
}
]
}
So the Conf Struct should look like this
type Conf struct {
NATS []struct{
HTTPPort int
Port int
Username string
Password string
}
}
Modified code
package main
import (
"log"
"github.com/spf13/viper"
"fmt"
)
type Conf struct {
NATS []struct{
HTTPPort int
Port int
Username string
Password string
}
}
func main() {
var c Conf
// config file
viper.SetConfigName("draft")
viper.AddConfigPath(".")
viper.SetConfigType("hcl")
if err := viper.ReadInConfig(); err != nil {
log.Fatal(err)
}
fmt.Println(viper.Get("NATS")) // gives [map[port:10041 username:cl1 password:__Psw__4433__ http_port:10044]]
if err := viper.Unmarshal(&c); err != nil {
log.Fatal(err)
}
fmt.Println(c.NATS[0].Username)
}
I know this question is more than two years old now, but I came across the same issue recently.
I'm using viper to be able to load different configuration files into a Go struct, allowing configuration in JSON, YAML, TOML, HCL, just pick your favourite :)
HCL file format does wrap a map into a slice because it allows redefining a section like:
section = {
key1 = "value"
}
section = {
key2 = "value"
}
which is something that is not supported by the other formats.
And here's how I fixed it:
My solution implies each new block will override any previous definition of the same key, and keep all the others. You can do some merging magic but I didn't need to.
You need to make a hook to convert a slice of maps into a map:
// sliceOfMapsToMapHookFunc merges a slice of maps to a map
func sliceOfMapsToMapHookFunc() mapstructure.DecodeHookFunc {
return func(from reflect.Type, to reflect.Type, data interface{}) (interface{}, error) {
if from.Kind() == reflect.Slice && from.Elem().Kind() == reflect.Map && (to.Kind() == reflect.Struct || to.Kind() == reflect.Map) {
source, ok := data.([]map[string]interface{})
if !ok {
return data, nil
}
if len(source) == 0 {
return data, nil
}
if len(source) == 1 {
return source[0], nil
}
// flatten the slice into one map
convert := make(map[string]interface{})
for _, mapItem := range source {
for key, value := range mapItem {
convert[key] = value
}
}
return convert, nil
}
return data, nil
}
}
then you need to create a DecodeHook:
configOption := viper.DecodeHook(mapstructure.ComposeDecodeHookFunc(
sliceOfMapsToMapHookFunc(),
mapstructure.StringToTimeDurationHookFunc(),
mapstructure.StringToSliceHookFunc(","),
))
the two other hooks are the default ones so you might want to keep them
then you pass the option to the Unmarshal method
viper.Unmarshal(&c, configOption)
With this method you don't need a slice around your structs or your maps. Also that makes it compatible with the other configuration file formats