Error unknown type Go - go

i'm really new to Go, su just bear with me here. I'm trying to write a code for loading mysql data to redis cluster using following :redis-go-cluster,
load2redis
This is the code. Its a little bit long, just bear with me here.
package main
import (
"bytes"
"database/sql"
"flag"
// "github.com/garyburd/redigo/redis"
_ "github.com/go-sql-driver/mysql"
//"gopkg.in/redis.v4"
"github.com/chasex/redis-go-cluster"
"log"
"runtime"
// "strings"
"sync"
"time"
)
var client *redis.Cluster
type Task interface {
Execute()
}
type Pool struct {
mu sync.Mutex
size int
tasks chan Task
kill chan struct{}
wg sync.WaitGroup
}
func NewPool(size int) *Pool {
pool := &Pool{
tasks: make(chan Task, 128),
kill: make(chan struct{}),
}
pool.Resize(size)
return pool
}
func (p *Pool) worker() {
defer p.wg.Done()
for {
select {
case task, ok := <-p.tasks:
if !ok {
return
}
task.Execute()
case <-p.kill:
return
}
}
}
func (p *Pool) Resize(n int) {
p.mu.Lock()
defer p.mu.Unlock()
for p.size < n {
p.size++
p.wg.Add(1)
go p.worker()
}
for p.size > n {
p.size--
p.kill <- struct{}{}
}
}
func (p *Pool) Close() {
close(p.tasks)
}
func (p *Pool) Wait() {
p.wg.Wait()
}
func (p *Pool) Exec(task Task) {
p.tasks <- task
}
type RedisTask struct {
Index int
Command string
Key string
Value string
MapData map[string]string
}
func (e RedisTask) Execute() {
log.Println("executing:", e.Key, ",", e.Index)
if e.Command == "SET" {
_,err := redis.String(client.Do("SET", e.Key, e.Value))
checkErr(err, "set error:")
} else if e.Command == "SADD" {
_,err := redis.Strings(client.Do("SADD", e.Key, e.Value))
checkErr(err, "sadd error:")
} else if e.Command == "HMSET" {
_,err := redis.StringMap(client.Do("HMSET", e.Key, e.MapData))
checkErr(err, "hmset error:")
}
// TODO: clean data
}
func main() {
runtime.GOMAXPROCS(runtime.NumCPU())
startTime := time.Now().UnixNano() / int64(time.Millisecond)
host := flag.String("s", "localhost:3306", "mysql server host and port ,eg localhost:3306")
username := flag.String("u", "test", "username to login mysql")
password := flag.String("p", "test", "password for mysql")
database := flag.String("d", "test", "database you want to execute query")
query := flag.String("q", "select 1;", "your query sql")
ds := flag.String("ds", "key", "redis structure")
PK := flag.String("pk", "Rkey", "the redis Key in the fields of mysql query result")
//redisHost := flag.String("rs", "localhost:6379", "redis host and port ,eg localhost:6379")
//redisPassword := flag.String("rp", "test", "redis password")
poolSize := flag.Int("size", 10000, "redis pool size")
flag.Parse()
var buf bytes.Buffer = bytes.Buffer{}
buf.WriteString(*username)
buf.WriteString(":")
buf.WriteString(*password)
buf.WriteString("#tcp(")
buf.WriteString(*host)
buf.WriteString(")/")
buf.WriteString(*database)
db, err := sql.Open("mysql", buf.String())
checkErr(err, "connect to mysql error !")
defer db.Close()
poolWorker := NewPool(*poolSize)
// Execute the query
rows, err := db.Query(*query)
checkErr(err, "execute sql error!")
// pool = newPool(*redisHost, *redisPassword, *poolSize)
//client = redis.NewClient(&redis.Options{
// Addr: *redisHost,
// Password: *redisPassword, // no password set
// DB: 0, // use default DB
//})
client,_ = redis.NewCluster(&redis.Options{
StartNodes: []string{"10.x.x.x:6000", "10.x.x.x:6001", "10.x.x.x:6002"},
ConnTimeout: 50 * time.Millisecond,
ReadTimeout: 50 * time.Millisecond,
WriteTimeout: 50 * time.Millisecond,
KeepAlive: 16,
AliveTime: 60 * time.Second,
})
//checkErr(err, "client error:")
//pong, err := client.Ping().Result()
//checkErr(err, "redis client error:")
//log.Println(pong)
columns, err := rows.Columns()
checkErr(err, "get columns error!")
length := len(columns)
values := make([]sql.RawBytes, length)
scanArgs := make([]interface{}, len(values))
for i := range values {
scanArgs[i] = &values[i]
}
count := 0
for rows.Next() {
count += 1
err = rows.Scan(scanArgs...)
checkErr(err, "scan error")
var value string
var key string
var task RedisTask
if *ds == "key" {
key = getStringData(values[0])
value = getStringData(values[1])
if value != "" {
task = RedisTask{
Index: count,
Command: "SET",
Key: key,
Value: value,
}
}
} else if *ds == "set" {
key = getStringData(values[0])
value = getStringData(values[1])
if value != "" {
task = RedisTask{
Index: count,
Command: "SADD",
Key: key,
Value: value,
}
}
} else if *ds == "hash" {
key = getStringData(values[0])
// args := redis.Args{}.Add(key)
m := make(map[string]string)
for i, col := range values {
if col != nil && columns[i] != *PK {
value = getStringData(col)
m[columns[i]] = value
}
}
task = RedisTask{
Index: count,
Command: "HMSET",
Key: key,
MapData: m,
}
}
poolWorker.Exec(task)
}
if err = rows.Err(); err != nil {
panic(err.Error()) // proper error handling instead of panic in your app
}
poolWorker.Close()
poolWorker.Wait()
EndTime := time.Now().UnixNano() / int64(time.Millisecond)
log.Println("======================================== executing time:", EndTime-startTime, " ms, total:", count)
}
func getStringData(data sql.RawBytes) string {
if data == nil {
return ""
}
value := string(data)
return clearBad(value)
}
func clearBad(str string) string {
// str = strings.Trim(str, "`")
// str = strings.Trim(str, "ï½€")
// str = strings.Trim(str, "-")
// str = strings.Trim(str, ".")
// str = strings.Trim(str, " ")
// str = strings.Trim(str, ";")
// str = strings.Trim(str, ",")
// str = strings.Trim(str, ":")
// str = strings.Trim(str, ";")
// str = strings.Trim(str, "'")
// str = strings.Trim(str, "!")
return str
}
func checkErr(err error, msg string) {
if err != nil {
log.Fatalln(msg, err)
}
}
When i'm executing it, i'm getting following exception:
./rak -u user -p user -s 10.X.X.X:8080 -d test -q "SELECT CONCAT( 'student:', c.sid ) Rkey, c.sname SNAME, c.saddress SADDRESS, c.sage SAGE FROM STUDENT c WHERE c.sid > 0;" -ds hash -size 1200
2017/07/21 10:29:09 rak.go:93: executing: student:2 , 2
2017/07/21 10:29:09 rak.go:93: executing: student:1 , 1
2017/07/21 10:29:09 rak.go:93: executing: student:3 , 3
2017/07/21 10:29:09 rak.go:268: hmset error: Do: unknown type map[string]string
$
Can somebody explain to me what i'm doing wrong here? I'd be very grateful.

As pointed out, Do does not work with maps. This is one way you could fix it.
} else if e.Command == "HMSET" {
// Build up a string slice to hold the key value pairs
args := make([]string, 0, len(e.MapData) * 2)
for k, v := range e.MapData {
args = append(args, k, v)
}
_,err := redis.StringMap(client.Do("HMSET", e.Key, args...))
checkErr(err, "hmset error:")
}
The Do method maps to the Redis command set and arguments are expected in the same way. For example.
127.0.0.1:6379> HMSET myKey foo bar baz boff
OK
127.0.0.1:6379> HGETALL myKey
1) "foo"
2) "bar"
3) "baz"
4) "boff"
127.0.0.1:6379>
The same map-set operation using the redis client in your code would be
client.Do("HMSET", "myKey", "foo", "bar", "baz", "boff")
When the arguments for keys and values of the map are dynamic, the most straight forward way is
client.Do("HMSET", "myKey", []string{"foo", "bar", "baz", "boff"}...)
which is exactly what the first code block above does.

Accepted values for Do function arguements are int64, float64, string or []byte .Since you passed a map it failed with error unknown type %T
Here is a reference to the respective code that does it
redis-go-cluster/node.go

Related

Why my counter doesn't work in "uber-go/tally"?

Can you please help me to find out why my event counters don't get reported. I even put the debugger on the reporter and it is not triggering. Time recorder triggers fine though.
Here is the implementation.
I have a Kafka consumer which I have defined the followings: (consumer.go)
reporter := metrics.NewPrintStatsReporter()
// Report every 100*time.Millisecond
rootScope, closer := tally.NewRootScope(tally.ScopeOptions{
Reporter: reporter,
}, time.Second)
defer func(closer io.Closer) {
err := closer.Close()
if err != nil {
...
}
}(closer)
subScope := rootScope.SubScope("Events")
then I do:
supportedEventCount := subScope.Counter(metrics.SupportedEventTypes)
unSupportedEventCount := subScope.Counter(metrics.UnsupportedEventTypes)
totalEventTypes := subScope.Counter(metrics.TotalEventTypes)
kafkaReadLatency := rootScope.Timer(metrics.KafkaReadLatency)
in the same file
go func() {
defer close(messages)
for { ...
st := time.Now()
m, err := c.r.ReadMessage(ctx)
kafkaConsumerReadLatency := time.Since(st)
if kafkaConsumerReadLatency < 5*time.Minute {
kafkaReadLatency.Record(kafkaConsumerReadLatency)
}
// Reporting any kind of event
totalEventTypes.Inc(1)
if helpers.IsSupportedEvent(logger, &kafkaEvent) {
supportedEventCount.Inc(1)
messages <- kafkaEvent
} else {
unSupportedEventCount.Inc(1)
}
}()
...
kafkaReadLatency.Record(kafkaConsumerReadLatency) works fine I get timer kafka_read_latency 34.479058541s
However non of the counters are working... which is very strange.
Here is my tally interface reporter: (very standard, 100% matches the supplied example)
package metrics
import (
"fmt"
tally "github.com/uber-go/tally/v4"
"time"
)
const (
SupportedEventTypes = "supported_event_types"
UnsupportedEventTypes = "unsupported_event_types"
TotalEventTypes = "total_event_types"
KafkaReadLatency = "kafka_read_latency"
)
type printStatsReporter struct{}
func NewPrintStatsReporter() tally.StatsReporter {
return &printStatsReporter{}
}
func (r *printStatsReporter) ReportCounter(name string, _ map[string]string, value int64) {
fmt.Printf("count %s %d\n", name, value)
}
func (r *printStatsReporter) ReportGauge(name string, _ map[string]string, value float64) {
fmt.Printf("gauge %s %f\n", name, value)
}
func (r *printStatsReporter) ReportTimer(name string, _ map[string]string, interval time.Duration) {
fmt.Printf("timer %s %s\n", name, interval.String())
}
...
func (r *printStatsReporter) Capabilities() tally.Capabilities {
return r
}
func (r *printStatsReporter) Reporting() bool {
return true
}
func (r *printStatsReporter) Tagging() bool {
return true
}
func (r *printStatsReporter) Flush() {
fmt.Printf("flush\n")
}
Here is the example:
https://github.com/uber-go/tally/blob/master/example/main.go

gob decoder returns only first element in the array

So I was trying to create a mock DB, and in the current implementation, I am trying to make an insert and select which insert rows and select returns them. I decided to use a bytes.Buffer to help maintain a memory block I could insert a slice of rows in, and deserialize that memory block when I call select but it seems select just returns the first row instead of all the rows passed to the array.
main.go
func main() {
inputBuffer := compiler.NewInputBuffer()
scanner := bufio.NewScanner(os.Stdin)
for {
PrintPrompt()
scanner.Scan()
command := scanner.Text()
inputBuffer.Buffer = command
if strings.HasPrefix(inputBuffer.Buffer, ".") {
switch compiler.DoMetaCommand(inputBuffer) {
case compiler.MetaCommandSuccess:
continue
case compiler.MetaCommandUnrecognizedCommand:
fmt.Printf("Unrecognized command %q \n", inputBuffer.Buffer)
continue
}
}
var statement compiler.Statement
switch compiler.PrepareStatement(inputBuffer, &statement) {
case compiler.PrepareSuccess:
case compiler.PrepareUnrecognizedStatement:
fmt.Printf("Unrecognized command at start of %q \n", inputBuffer.Buffer)
continue
case compiler.PrepareSyntaxError:
fmt.Println("Syntax error. Could not parse statement.")
continue
}
compiler.ExecuteStatement(statement)
fmt.Println("Executed")
}
}
func PrintPrompt() {
fmt.Printf("db > ")
}
Above is the code responsible for collecting user input etc.
package compiler
import (
"bytes"
"log"
"os"
"strconv"
"strings"
)
type Row struct {
ID int32
Username string
Email string
}
type Statement struct {
RowToInsert Row
Type StatementType
}
var (
RowsTable = make([]Row, 0)
RowsTableBuffer bytes.Buffer
)
func DoMetaCommand(buffer InputBuffer) MetaCommandResult {
if buffer.Buffer == ".exit" {
os.Exit(0)
} else {
return MetaCommandUnrecognizedCommand
}
return MetaCommandSuccess
}
func PrepareStatement(buffer InputBuffer, statement *Statement) PrepareResult {
if len(buffer.Buffer) > 6 {
bufferArguments := strings.Fields(buffer.Buffer)
if bufferArguments[0] == "insert" {
statement.Type = StatementInsert
if len(bufferArguments) < 4 {
return PrepareSyntaxError
} else {
i, err := strconv.Atoi(bufferArguments[1])
if err != nil {
log.Printf("%q is not a valid id\n", bufferArguments[1])
return PrepareSyntaxError
} else {
statement.RowToInsert.ID = int32(i)
}
statement.RowToInsert.Username = bufferArguments[2]
statement.RowToInsert.Email = bufferArguments[3]
}
RowsTable = append(RowsTable, statement.RowToInsert)
return PrepareSuccess
}
}
if buffer.Buffer == "select" {
statement.Type = StatementSelect
return PrepareSuccess
}
return PrepareUnrecognizedStatement
}
func ExecuteStatement(statement Statement) {
switch statement.Type {
case (StatementInsert):
SerializeRow(RowsTable)
case (StatementSelect):
DeserializeRow()
}
}
The code above is for parsing and appending the entries into statements and depending on the keywords, it's either an insert or select [Took the code for defining enums out and left core logic]
func SerializeRow(r []Row) {
encoder := gob.NewEncoder(&RowsTableBuffer)
err := encoder.Encode(r)
if err != nil {
log.Println("encode error:", err)
}
}
func DeserializeRow() {
var rowsBuffer = RowsTableBuffer
rowsTable := make([]Row, 0)
decoder := gob.NewDecoder(&rowsBuffer)
err := decoder.Decode(&rowsTable)
if err != nil {
log.Println("decode error:", err)
}
fmt.Println(rowsTable)
}
So the code above uses a global buffer in which the slice being appended to in PrepareStatement()will be encoded after an insert is done. A select ought to return the slice of all rows but just returns the first element for some reason.
Example (in terminal):
If I make 2 inserts:
db > insert 1 john c#mail.com
db > insert 2 collins k#mail.com
Then I make a select:
select
=> it returns [{1 john c#mail.com}] only.
Is there anything I am missing here? Thanks for your support.
So the answer was pretty simple. We were creating a new encoder in the SerializeRow function instead of creating it once. We pulled it out of the function and created a global.
var (
encoder = gob.NewEncoder(&RowsTableBuffer)
decoder = gob.NewDecoder(&RowsTableBuffer)
)
func SerializeRow(r Row) {
err := encoder.Encode(r)
if err != nil {
log.Println("encode error:", err)
}
}
func DeserializeRow() {
var rows Row
err := decoder.Decode(&rows)
for err == nil {
if err != nil {
log.Fatal("decode error:", err)
}
fmt.Printf("%d %s %s\n", rows.ID, rows.Username, rows.Email)
err = decoder.Decode(&rows)
}
}

nil point deference in golang using govmomi library

I'm trying to use the govmomi library (https://github.com/vmware/govmomi) to make some automated changes to a VM but I can't seem to get around the nil pointer exceptions on line 134 and 136 for the last few hours. I got it to work at one point but then I'm not sure what I'm doing wrong now as I've tried so many combos but nothing seems to work now...
package main
import (
"context"
"flag"
"github.com/vmware/govmomi"
"github.com/vmware/govmomi/vim25"
"github.com/vmware/govmomi/vim25/soap"
"net/url"
"os"
"strings"
"fmt"
"github.com/vmware/govmomi/govc/flags"
"github.com/vmware/govmomi/vim25/types"
"log"
)
type change struct {
*flags.VirtualMachineFlag
}
const (
envURL = "GOVC_URL"
envUserName = "GOVC_USERNAME"
envPassword = "GOVC_PASSWORD"
envInsecure = "GOVC_INSECURE"
)
// getEnvString returns string from environment variable.
func getEnvString(v string, def string) string {
r := os.Getenv(v)
if r == "" {
return def
}
return r
}
// getEnvBool returns boolean from environment variable.
func getEnvBool(v string, def bool) bool {
r := os.Getenv(v)
if r == "" {
return def
}
switch strings.ToLower(r[0:1]) {
case "t", "y", "1":
return true
}
return false
}
var urlDescription = fmt.Sprintf("ESX or vCenter URL [%s]", envURL)
var urlFlag = flag.String("url", getEnvString(envURL, "https://username:password#host"+vim25.Path), urlDescription)
var insecureDescription = fmt.Sprintf("Don't verify the server's certificate chain [%s]", envInsecure)
var insecureFlag = flag.Bool("insecure", getEnvBool(envInsecure, false), insecureDescription)
func processOverride(u *url.URL) {
envUsername := os.Getenv(envUserName)
envPassword := os.Getenv(envPassword)
// Override username if provided
if envUsername != "" {
var password string
var ok bool
if u.User != nil {
password, ok = u.User.Password()
}
if ok {
u.User = url.UserPassword(envUsername, password)
} else {
u.User = url.User(envUsername)
}
}
// Override password if provided
if envPassword != "" {
var username string
if u.User != nil {
username = u.User.Username()
}
u.User = url.UserPassword(username, envPassword)
}
}
func NewClient(ctx context.Context) (*govmomi.Client, error) {
flag.Parse()
// Parse URL from string
u, err := soap.ParseURL(*urlFlag)
if err != nil {
return nil, err
}
// Override username and/or password as required
processOverride(u)
// Connect and log in to ESX or vCenter
return govmomi.NewClient(ctx, u, *insecureFlag)
}
func main() {
ctx := context.Background()
// Connect and login to ESX or vCenter
c, err := NewClient(ctx)
if err != nil {
log.Fatal(err)
}
defer c.Logout(ctx)
var spec *types.VirtualMachineConfigSpec
var cmd *change
var flip bool
flip = false
spec.VPMCEnabled = &flip
vm, err := cmd.VirtualMachine()
task, err := vm.Reconfigure(ctx, *spec)
if err != nil {
println(err)
}
fmt.Println(task.Wait(ctx))
}
the 2 lines that are throwing errors are:
spec.VPMCEnabled = &flip
and
vm, err := cmd.VirtualMachine()
both seem to throw the same panic: runtime error: invalid memory address or nil pointer dereference. If I comment out the first one throwing the error, the second one throws it then.
I think both are unrelated but I can't quite figure how how to do the dereferencing correctly.
----UPDATE EDIT 1--------------
I made some changes to this with the below response but still can't get around the error on at task, err := vm.Reconfigure(ctx, *spec)...
package main
import (
"context"
"flag"
"github.com/vmware/govmomi"
"github.com/vmware/govmomi/object"
"github.com/vmware/govmomi/vim25"
"github.com/vmware/govmomi/vim25/soap"
"net/url"
"os"
"strings"
"fmt"
"github.com/vmware/govmomi/vim25/types"
"log"
)
const (
envURL = "GOVC_URL"
envUserName = "GOVC_USERNAME"
envPassword = "GOVC_PASSWORD"
envInsecure = "GOVC_INSECURE"
)
// getEnvString returns string from environment variable.
func getEnvString(v string, def string) string {
r := os.Getenv(v)
if r == "" {
return def
}
return r
}
// getEnvBool returns boolean from environment variable.
func getEnvBool(v string, def bool) bool {
r := os.Getenv(v)
if r == "" {
return def
}
switch strings.ToLower(r[0:1]) {
case "t", "y", "1":
return true
}
return false
}
var urlDescription = fmt.Sprintf("ESX or vCenter URL [%s]", envURL)
var urlFlag = flag.String("url", getEnvString(envURL, "https://username:password#host"+vim25.Path), urlDescription)
var insecureDescription = fmt.Sprintf("Don't verify the server's certificate chain [%s]", envInsecure)
var insecureFlag = flag.Bool("insecure", getEnvBool(envInsecure, false), insecureDescription)
func processOverride(u *url.URL) {
envUsername := os.Getenv(envUserName)
envPassword := os.Getenv(envPassword)
// Override username if provided
if envUsername != "" {
var password string
var ok bool
if u.User != nil {
password, ok = u.User.Password()
}
if ok {
u.User = url.UserPassword(envUsername, password)
} else {
u.User = url.User(envUsername)
}
}
// Override password if provided
if envPassword != "" {
var username string
if u.User != nil {
username = u.User.Username()
}
u.User = url.UserPassword(username, envPassword)
}
}
func NewClient(ctx context.Context) (*govmomi.Client, error) {
flag.Parse()
// Parse URL from string
u, err := soap.ParseURL(*urlFlag)
if err != nil {
return nil, err
}
// Override username and/or password as required
processOverride(u)
// Connect and log in to ESX or vCenter
return govmomi.NewClient(ctx, u, *insecureFlag)
}
func main() {
ctx := context.Background()
// Connect and login to ESX or vCenter
c, err := NewClient(ctx)
if err != nil {
log.Fatal(err)
}
defer c.Logout(ctx)
var spec *types.VirtualMachineConfigSpec
spec = new(types.VirtualMachineConfigSpec)
var flip bool
flip = false
spec.VPMCEnabled = &flip
var vm *object.VirtualMachine
vm = new(object.VirtualMachine)
task, err := vm.Reconfigure(ctx, *spec)
if err != nil {
println(err)
}
fmt.Println(task.Wait(ctx))
}
the full error I'm getting is this:
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x280 pc=0x14e18f6]
goroutine 1 [running]:
github.com/vmware/govmomi/vim25.(*Client).RoundTrip(0x0, 0x1886f00, 0xc000016088, 0x1884140, 0xc000298720, 0x1884140, 0xc000298740, 0x300, 0x16e8ac0)
/Users/ronakpatel/go/src/github.com/vmware/govmomi/vim25/client.go:89 +0x26
github.com/vmware/govmomi/vim25/methods.ReconfigVM_Task(0x1886f00, 0xc000016088, 0x1884060, 0x0, 0xc0002d0000, 0xc000288000, 0xc000080400, 0x0)
/Users/ronakpatel/go/src/github.com/vmware/govmomi/vim25/methods/methods.go:10879 +0xb8
github.com/vmware/govmomi/object.VirtualMachine.Reconfigure(0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1886f00, 0xc000016088, 0x0, ...)
/Users/ronakpatel/go/src/github.com/vmware/govmomi/object/virtual_machine.go:207 +0x19b
main.main()
/Users/ronakpatel/go/src/awesomeProject1/main.go:143 +0x1ec
exit status 2
----UPDATE EDIT 2----------
I changed somethings around and used the answer provided below but now I'm getting the error again but at this part: task, err := vm.Reconfigure(ctx, spec1)
Error:
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x0 pc=0x153d05a]
goroutine 1 [running]:
main.main()
/Users/ronakpatel/go/src/awesomeProject1/main.go:134 +0x19a
exit status 2
Code:
spec1 := types.VirtualMachineConfigSpec{
VPMCEnabled: &[]bool{false}[0],
}
var vm *object.VirtualMachine
var vmf *flags.VirtualMachineFlag
vmf, ctx = flags.NewVirtualMachineFlag(ctx)
vm, _ = vmf.VirtualMachine()
task, err := vm.Reconfigure(ctx, spec1)
if err != nil {
println(err)
}
fmt.Println(task.Wait(ctx))
var spec *types.VirtualMachineConfigSpec // error: spec is nil pointer
spec = new(types.VirtualMachineConfigSpec) // fix: spec is not nil
var cmd *change
var flip bool
flip = false
// used here
spec.VPMCEnabled = &flip
vm, err := cmd.VirtualMachine()

Golang flag: Ignore missing flag and parse multiple duplicate flags

I am new to Golang and I have been unable to find a solution to this problem using flag.
How can I use flag so my program can handle calls like these, where the -term flag may be present a variable number of times, including 0 times:
./myprogram -f flag1
./myprogram -f flag1 -term t1 -term t2 -term t3
You need to declare your own type which implements the Value interface. Here is an example.
// Created so that multiple inputs can be accecpted
type arrayFlags []string
func (i *arrayFlags) String() string {
// change this, this is just can example to satisfy the interface
return "my string representation"
}
func (i *arrayFlags) Set(value string) error {
*i = append(*i, strings.TrimSpace(value))
return nil
}
then in the main function where you are parsing the flags
var myFlags arrayFlags
flag.Var(&myFlags, "term", "my terms")
flag.Parse()
Now all the terms are contained in the slice myFlags
This question is an interesting one and can play in many variations.
Array
Map
Struct
The core content is the same as #reticentroot answered,
Complete the definition of this interface: Flag.Value
The following are examples to share and provide relevant links as much as possible
Example
expected usage:
type Books []string
func (*Books) String() string { return "" }
func (*Books) Set(string) error { return nil }
type Dict map[string]string
func (*Dict) String() string { return "" }
func (*Dict) Set(string) error { return nil }
type Person struct {
Name string
Age int
}
func (*Person) String() string { return "" }
func (*Person) Set(string) error { return nil }
func pseudocode() {
flagSetTest := flag.NewFlagSet("test", flag.ContinueOnError)
books := Books{}
flagSetTest.Var(&books, "book", "-book C++ -book Go -book javascript")
// expected output: books: []string{C++,Go,javascript}
dict := Dict{}
flagSetTest.Var(&dict, "dict", "-dict A:65|B:66")
// expected output: dict: map[string]string{"A":"65", "B":"66"}
// map
person := Person{}
flagSetTest.Var(&person, "person", "-person Name:foo|Age:18")
// output: {Name:foo Age:18}
flagSetTest.Parse(os.Args[1:])
fmt.Println(person, books, dict)
}
Full code
package main
import (
"bufio"
"errors"
"flag"
"fmt"
"os"
"reflect"
"strconv"
"strings"
)
type BooksValue []string
// https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L298
func (arr *BooksValue) String() string {
/*
value.String(): https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L870
DefValue string:
- https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L348
- https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L914-L920
- https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L529-L536
- https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L464
*/
return ""
}
// https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L299
func (arr *BooksValue) Set(value string) error {
/*
value: https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L947
bool: Set(value): https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L966-L975
else: Set(value): https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L986-L988
*/
*arr = append(*arr, strings.TrimSpace(value))
return nil
}
type DictValue map[string]string
func (m *DictValue) String() string {
return ""
}
func (m *DictValue) Set(value string) error {
arr := strings.Split(value, "|") // "key1:val1|key2:val2|..."
for _, curPairStr := range arr {
itemArr := strings.Split(curPairStr, ":")
key := itemArr[0]
val := itemArr[1]
(*m)[key] = val
}
return nil
}
type PersonValue struct {
Name string
Age int
Msg string
IsActive bool
}
func (s *PersonValue) String() string {
return ""
}
func (s *PersonValue) Set(value string) error {
arr := strings.Split(value, "|") // "Field1:Value1|F2:V2|...|FN:VN"
for _, curPairStr := range arr {
itemArr := strings.Split(curPairStr, ":")
key := itemArr[0]
val := itemArr[1]
// [Access struct property by name](https://stackoverflow.com/a/66470232/9935654)
pointToStruct := reflect.ValueOf(s)
curStruct := pointToStruct.Elem()
curField := curStruct.FieldByName(key)
if !curField.IsValid() {
return errors.New("not found")
}
// CanSet one of conditions: Name starts with a capital
if !curField.CanSet() {
return errors.New("can't set")
}
t := reflect.TypeOf(*s)
structFieldXXX, isFound := t.FieldByName(key)
if !isFound {
return errors.New("not found")
}
switch structFieldXXX.Type.Name() {
case "int":
// https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L146-L153
intValue, err := strconv.ParseInt(val, 0, strconv.IntSize)
if err != nil {
return errors.New("parse error: [int]")
}
curField.SetInt(intValue)
case "bool":
// https://github.com/golang/go/blob/2580d0e/src/flag/flag.go#L117-L121
boolValue, err := strconv.ParseBool(val)
if err != nil {
return errors.New("parse error: [bool]")
}
curField.SetBool(boolValue)
case "string":
curField.SetString(val)
default:
return errors.New("not support type=" + structFieldXXX.Type.Name())
}
}
return nil
}
func main() {
flagSetTest := flag.NewFlagSet("test", flag.ContinueOnError)
// array
books := BooksValue{}
flagSetTest.Var(&books, "book", "-book Go -book javascript ...")
// map
myMap := DictValue{}
flagSetTest.Var(&myMap, "map", "-dict A:65|B:66")
// struct
person := PersonValue{Msg: "Hello world"}
flagSetTest.Var(&person, "person", "-person Name:string|Age:int|Msg:string|IsActive:bool")
testArgs := []string{"test",
"-book", "Go", "-book", "javascript", // testArray
"-map", "A:65|B:66|Name:Carson", // testMap
"-person", "Name:Carson|Age:30|IsActive:true", // testStruct
}
testFunc := func(args []string, reset bool) {
if reset {
books = BooksValue{}
myMap = DictValue{}
person = PersonValue{}
}
if err := flagSetTest.Parse(args); err != nil {
fmt.Printf(err.Error())
}
fmt.Printf("%+v\n", books)
fmt.Printf("%+v\n", myMap)
fmt.Printf("%+v\n", person)
}
testFunc(testArgs[1:], false)
// ↓ play by yourself
scanner := bufio.NewScanner(os.Stdin)
for {
fmt.Println("Enter CMD: ") // example: test -book item1 -book item2 -map key1:value1|key2:v2 -person Age:18|Name:Neil|IsActive:true
scanner.Scan() // Scans a line from Stdin(Console)
text := scanner.Text() // Holds the string that scanned
args := strings.Split(text, " ")
switch args[0] {
case "quit":
return
case "test":
testFunc(args[1:], true)
}
}
}
go playground

out of memory in golang when parsing Freebase RDF

I'm parsing through the triples of the Freebase RDF compressed and streaming with the XML package in Golang. However, I'm getting an out of memory error.
Do I have to garbage-collect? How can I do that? How can I clear the memory after I'm doing writing that triple to the XML file?
Here's my code: http://play.golang.org/p/dWvbtcs7wy
package main
import(
"bufio"
"flag"
"fmt"
"io"
"net/url"
"os"
"regexp"
"strings"
)
var inputFile = flag.String("infile", "freebase-rdf", "Input file path")
var filter, _ = regexp.Compile("^file:.*|^talk:.*|^special:.*|^wikipedia:.*|^wiktionary:.*|^user:.*|^user_talk:.*")
type Redirect struct {
Title string `xml:"title,attr"`
}
type Page struct {
Title string `xml:"title"`
Abstract string `xml:""`
}
func CanonicaliseTitle(title string) string{
can := strings.ToLower(title)
can = strings.Replace(can, " ", "_", -1)
can = url.QueryEscape(can)
return can
}
func convertFreebaseId(uri string) string{
if strings.HasPrefix(uri, "<") && strings.HasSuffix(uri, ">") {
var id = uri[1 : len(uri)-1]
id = strings.Replace(id, "http://rdf.freebase.com/ns", "", -1)
id = strings.Replace(id, ".", "/", -1)
return id
}
return uri
}
func parseTriple(line string) (string, string, string){
var parts = strings.Split(line, "\t")
subject := convertFreebaseId(parts[0])
predicate := convertFreebaseId(parts[1])
object := convertFreebaseId(parts[2])
return subject, predicate, object
}
var (
validRegexp = regexp.MustCompile("^[A-Za-z0-9][A-Za-z0-9_-]*$")
englishRegexp = regexp.MustCompile("#en")
)
func validTitle(content []string) bool{
for _, v := range content{
if !englishRegexp.MatchString(v) && len(v) > 1 && strings.Index(v, "[]") != -1{
}
}
return true
}
func validText(content []string) bool{
for _, v := range content{
if !validRegexp.MatchString(v) && len(v) > 1 && strings.Index(v, "[]") != -1{
return false
}
}
return true
}
func processTopic(id string, properties map[string][]string, file io.Writer){
if validTitle(properties["/type/object/name"]) && validText(properties["/common/document/text"]){
fmt.Fprintf(file, "<card>\n")
fmt.Fprintf(file, "<title>\"%s\"</title>\n", properties["/type/object/name"])
fmt.Fprintf(file, "<image>\"%s/%s\"</image>\n", "https://usercontent.googleapis.com/freebase/v1/image", id)
fmt.Fprintf(file, "<text>\"%s\"</text>\n", properties["/common/document/text"])
fmt.Fprintf(file, "<facts>\n")
for k, v := range properties{
for _, value := range v{
fmt.Fprintf(file, "<fact property=\"%s\">%s</fact>\n", k, value)
}
}
fmt.Fprintf(file, "</facts>\n")
fmt.Fprintf(file, "</card>\n")
}
}
func main(){
var current_mid = ""
current_topic := make(map[string][]string)
f, err := os.Open(*inputFile)
if err != nil {
fmt.Println(err)
return
}
r := bufio.NewReader(f)
xmlFile, _ := os.Create("freebase.xml")
line, err := r.ReadString('\n')
for err == nil{
subject, predicate, object := parseTriple(line)
if subject == current_mid{
current_topic[predicate] = append(current_topic[predicate], object)
}else if len(current_mid) > 0{
processTopic(current_mid, current_topic, xmlFile)
current_topic = make(map[string][]string)
}
current_mid = subject
line, err = r.ReadString('\n')
}
processTopic(current_mid, current_topic, xmlFile)
if err != io.EOF {
fmt.Println(err)
return
}
}
I'm not sure that this is your problem, although reading your code it seems you're not leaking anything - but you can tune GC behavior a bit with SetGCPercent() http://golang.org/pkg/runtime/debug/#SetGCPercent
According to TFM, a collection is triggered when the ratio of freshly allocated data to live data remaining after the previous collection reaches this percentage.. The default rate is 100%, meaning for programs that make lots of small allocations and hold lots of RAM, the overhead can be huge. I had an HTTP cache take up over 200% the cache size once. Try tuning the percentage to somewhere around 10% and see if it helps.

Resources