Please consider the code https://play.golang.org/p/aO07_PoQLuh
I've a list of structs, which I read from to get an idea of the members I've generated enroute.
For each struct, I've a method to raise a counter, however I am missing the sauce here. As you can see from the o/p, I've incremented SBytesSent, but when I read the list of struct and inspect it, it is at 0.
What's the best way to handle this? Thanks!
package main
import (
"fmt"
"sync"
)
type destination struct {
Name string
SBytesSent int64
ABytesSent int64
LastSeenAlive int64
Mutex *sync.Mutex
}
type destinations []destination
var (
destination_list destinations
myHosts = []string{"host1", "host2", "host3"}
)
func main() {
fmt.Println("Hello, playground")
for i, _ := range myHosts {
newDest := myHosts[i]
newd := destination{Name: newDest}
newd.Mutex = &sync.Mutex{}
destination_list = append(destination_list, newd)
}
i := 0
for {
increment()
status()
i += 1
if i == 3 {
break
}
}
}
func (self *destination) incrementSBytes(a int) {
self.Mutex.Lock()
defer self.Mutex.Unlock()
self.SBytesSent += int64(a)
fmt.Printf("new val %d\n", self.SBytesSent)
}
func (self *destination) Status() {
fmt.Printf("my val %d\n", self.SBytesSent)
}
func increment() {
for i, _ := range destination_list {
dest := destination_list[i]
dest.incrementSBytes(33)
}
}
func status() {
for i, _ := range destination_list {
dest := destination_list[i]
dest.Status()
}
}
Edit 1
Please see https://play.golang.org/p/5uqqc3OKYDs - I've incremented host3 to 6 - yet towards the end they all show 99. How can I make host3 retain the previous increment and show 99 + 6 = 105?
It doesn't work, because you're copying, and modifying the copy:
dest := destination_list[i]
dest.incrementSBytes(33)
Above, you first copy the array element to dest, and then modify dest. Array element never changes. Instead try this:
destination_list[i].incrementsSBytes(33)
All the magic lies in the range, it creates a copy of that element and hence the modification wasn't visible.
Read more about it: Change values while iterating
package main
import (
"fmt"
"sync"
)
type destination struct {
Name string
SBytesSent int64
ABytesSent int64
LastSeenAlive int64
Mutex *sync.Mutex
}
type destinations []destination
var (
destination_list destinations
myHosts = []string{"host1", "host2", "host3"}
)
func main() {
fmt.Println("Hello, playground")
for i := range myHosts {
newDest := myHosts[i]
newd := destination{Name: newDest}
newd.Mutex = &sync.Mutex{}
destination_list = append(destination_list, newd)
}
i := 0
for {
increment()
status()
i++
if i == 3 {
break
}
}
}
func (self *destination) incrementSBytes(a int) {
self.Mutex.Lock()
defer self.Mutex.Unlock()
self.SBytesSent += int64(a)
fmt.Printf("new val %d\n", self.SBytesSent)
}
func (self *destination) Status() {
fmt.Printf("my val %d\n", self.SBytesSent)
}
func increment() {
for i:=0; i<len(destination_list); i++ {
destination_list[i].incrementSBytes(33)
}
}
func status() {
for i:=0; i<len(destination_list); i++ {
destination_list[i].Status()
}
}
Output:
Hello, playground
new val 33
new val 33
new val 33
my val 33
my val 33
my val 33
new val 66
new val 66
new val 66
my val 66
my val 66
my val 66
new val 99
new val 99
new val 99
my val 99
my val 99
my val 99
Related
I have a tree-like structure, that has string regexp and I want Go compiled *regexp.Regexp to be part of it as well, in order to run algorithms on the tree. When I marshal and pass it to a different machine I may just recompile it again from the string. What is the correct way to do that, how to force protobuf to store pointers in a structure, that it ideally wont marshal? (the only way that i see is to make uint64 field and cast its value to/from *Regexp)
pseudo-code (because required wanted features seems to be not in the language):
// struct generated by protoc
type ProtoMessage struct {
Data string
Source string
Regexp uint64 // should not be marshalled, should be forcefully omitted from payload when doing proto.Marshal, ideally it should be *regexp.Regexp
Left *ProtoMessage
Right *ProtoMessage
}
func main() {
// sender computer doSend():
mSrc := &ProtoMessage{Data:"its meee!!!", Source: "hello.+world"}
payload, _ := proto.Marshal(m)
//receiver computer: onRecv()
mDst := new(ProtoMessage)
proto.Unmarshal(payload, mDst)
r, _ := regexp.Compile(mDst.Source)
mDst.Regexp = uint64(unsafe.Pointer(r)) // not working btw
TreeMatch = func(tree* ProtoMessage, line string) string {
if *regexp.Regexp(t.Regexp).Match(line) { // not working line
return t.Data
}
if tree.Left == nil {
return ""
}
return TreeMatch(tree.Left, line)
}
assert( TreeMatch(mDst, "hello, world") == "its meee!!!") // panic if condition is false
}
With json marshal i can just pot a pointer to regexp and provide a tag json:"-" in order not to include this field into marshalled structure, and ofc its important feature of marshalling/unmarshalling system to stay efficient (eg use same structure to run algorithms on in, and avoid data copying after unmarshal). How can I do the same with protobuf?
You can't store a pointer in a protobuf, as the recipient is likely a different computer. Even if you could, you'd get a panic as soon as you tried to dereference the pointer. Easiest thing to do would be just pass the RegExp string, then compile again at the destination:
package main
import (
"fmt"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/structpb"
)
func main() {
v := structpb.NewStringValue("hello.+world")
b, err := proto.Marshal(v)
if err != nil {
panic(err)
}
fmt.Printf("%q\n", b) // "\x1a\fhello.+world"
}
Note: you can't hack around this with Gob either:
package main
import (
"bytes"
"encoding/gob"
"regexp"
)
func main() {
re := regexp.MustCompile("hello.+world")
buf := new(bytes.Buffer)
if err := gob.NewEncoder(buf).Encode(re); err != nil {
panic(err) // type regexp.Regexp has no exported fields
}
}
Found the solution, you just have to have any pointer inside your struct (no matter if its marshalling or not, you are not using its unmarshalled value on receiver side):
proto declaration:
syntax = "proto3";
package main;
option go_package = ".;main";
message Empty {
}
message ProtoMessage {
string data = 1;
string source = 2;
Empty regexp = 3; // ideally should not be marshalled at all, like `json:"-"` but for protobuf
ProtoMessage left = 4;
ProtoMessage right = 5;
}
testing code:
package main
import (
"regexp"
"testing"
"unsafe"
)
type Empty struct {
//state protoimpl.MessageState
//sizeCache protoimpl.SizeCache
//unknownFields protoimpl.UnknownFields
}
// struct generated by protoc
type ProtoMessage struct {
//state protoimpl.MessageState
//sizeCache protoimpl.SizeCache
//unknownFields protoimpl.UnknownFields
Data string `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"`
Source string `protobuf:"bytes,2,opt,name=source,proto3" json:"source,omitempty"`
Regexp *Empty `protobuf:"bytes,3,opt,name=regexp,proto3" json:"regexp,omitempty"` // ideally should not be marshalled at all, like `json:"-"` but for protobuf
Left *ProtoMessage `protobuf:"bytes,4,opt,name=left,proto3" json:"left,omitempty"`
Right *ProtoMessage `protobuf:"bytes,5,opt,name=right,proto3" json:"right,omitempty"`
}
func (p *ProtoMessage) GetCompiledRegexp() *regexp.Regexp {
return (*regexp.Regexp)(unsafe.Pointer(p.Regexp))
}
func (p *ProtoMessage) SetCompiledRegexp(r *regexp.Regexp) {
p.Regexp = (*Empty)(unsafe.Pointer(r))
}
func TreeMatch(tree *ProtoMessage, line string) string {
if tree.GetCompiledRegexp().Match([]byte(line)) { // not working line
return tree.Data
}
if tree.Left == nil {
return ""
}
return TreeMatch(tree.Left, line)
}
func TestTreeMatch(t *testing.T) {
//happening at receiver side: imagine its proto.Unmarshal(payload, receiverMsg)
receiverMsg := &ProtoMessage{
Data: "its meee!!!",
Source: "hello.+world",
}
r, _ := regexp.Compile(receiverMsg.Source)
receiverMsg.SetCompiledRegexp(r)
if TreeMatch(receiverMsg, "helloworld") != "" {
t.Fatalf("TreeMatch gives non-existing match!")
}
if TreeMatch(receiverMsg, "hello, world") != "its meee!!!" {
t.Fatalf("TreeMatch is not working!")
}
}
type ProtoMessageDirect struct {
Data string
Source string
Regexp *regexp.Regexp
Left *ProtoMessageDirect
Right *ProtoMessageDirect
}
func (p *ProtoMessageDirect) GetCompiledRegexp() *regexp.Regexp {
return p.Regexp
}
func (p *ProtoMessageDirect) SetCompiledRegexp(r *regexp.Regexp) {
p.Regexp = r
}
func TreeMatchDirect(tree *ProtoMessageDirect, line string) string {
if tree.GetCompiledRegexp().Match([]byte(line)) { // not working line
return tree.Data
}
if tree.Left == nil {
return ""
}
return TreeMatchDirect(tree.Left, line)
}
func BenchmarkRegexpCast(b *testing.B) {
receiverMsg := &ProtoMessage{
Data: "its meee!!!",
Source: "hello.+world",
}
r, _ := regexp.Compile(receiverMsg.Source)
receiverMsg.SetCompiledRegexp(r)
b.ResetTimer()
for i := 0; i < b.N; i++ {
TreeMatch(receiverMsg, "hello, world")
}
}
func BenchmarkRegexpDirect(b *testing.B) {
receiverMsg := &ProtoMessageDirect{
Data: "its meee!!!",
Source: "hello.+world",
}
r, _ := regexp.Compile(receiverMsg.Source)
receiverMsg.SetCompiledRegexp(r)
b.ResetTimer()
for i := 0; i < b.N; i++ {
TreeMatchDirect(receiverMsg, "hello, world")
}
}
TestTreeMatch is passing and Benchmarks shows that such a cast does not create any meaningful difference:
BenchmarkRegexpCast-20 2741786 376.7 ns/op 16 B/op 1 allocs/op
BenchmarkRegexpDirect-20 3075280 377.0 ns/op 16 B/op 1 allocs/op
PASS
package main
import (
"fmt"
"reflect"
)
type Aservice struct {
}
type Adata struct {
msg string
}
type Bdata struct {
more string
}
var amap map[string]interface{} = make(map[string]interface{}, 1024)
func (aser *Aservice) Bar(data *Adata) error {
return nil
}
func (aser *Aservice) Foo(data *Bdata) error {
return nil
}
func main() {
var ser *Aservice
typeOfService := reflect.TypeOf(ser)
valueOfService := reflect.ValueOf(ser)
for i := 0; i < valueOfService.NumMethod(); i++ {
nref := valueOfService.Method(i).Type().In(0)
fmt.Println("++", nref.Elem().Name())
amap[typeOfService.Method(i).Name] = nref
}
}
Currently "Adata" and "Bdata" can be printed correctly
But I don’t know how to store the empty structure pointers of "Adata" and "Bdata" in amap
No idea for the next step
I want to use Method(i).Name() in amap to store the parameters that need to be passed in for the Method
Based on the suggestions in comments :
package main
import (
"fmt"
"reflect"
)
type Aservice struct {
}
type Adata struct {
msg string
}
type Bdata struct {
more string
}
var amap = map[string]interface{}{}
func (aser *Aservice) Bar(data *Adata) error {
return nil
}
func (aser *Aservice) Foo(data *Bdata) error {
return nil
}
func main() {
var ser *Aservice
typeOfService := reflect.TypeOf(ser)
valueOfService := reflect.ValueOf(ser)
for i := 0; i < valueOfService.NumMethod(); i++ {
nref := valueOfService.Method(i).Type().In(0)
amap[typeOfService.Method(i).Name] = reflect.New(nref.Elem()).Interface()
}
for k, v := range amap {
fmt.Printf("%s %#v\n", k, v)
}
}
Output:
Bar &main.Adata{msg:""}
Foo &main.Bdata{more:""}
I'm working on one of our system applications, specifically in the configuration file handling bits. We currently have 3 different places where a configuration file can be stored, and that can possibly be extended later. What I'm trying to do is simplify the way we need to add a new managed field.
The solution I have so far looks something like this:
package main
import (
"reflect"
"strconv"
"strings"
)
type Datastore interface{}
type MyInt struct {
intVal int
}
func NewMyInt(key string, dv int, db *Datastore) *MyInt {
// Do something here to construct MyInt
return &MyInt{intVal: dv}
}
type Config struct {
myInts map[string]*MyInt
// Tag is of form "<key in DB>:<default value>"
Value1 MyInt "value1_key:12345"
Value2 MyInt "value2_key:54321"
}
func NewConfig(db *Datastore) *Config {
c := &Config{
myInts: make(map[string]*MyInt),
}
cType := reflect.TypeOf(c)
for i := 0; i < cType.NumField(); i++ {
f := cType.Field(i)
if f.Name == "myInts" {
continue
}
tag := string(f.Tag)
fields := strings.Split(tag, ":")
switch f.Type.Name() {
case "myInt":
intVal, _ := strconv.Atoi(fields[1])
val := NewMyInt(fields[0], intVal, db)
c.myInts[fields[0]] = val
// How do I set the i'th field to this newly constructed value?
}
}
return c
}
So far I'm just missing this piece to do the assignment.
For this question, you can try
func NewConfig(db *Datastore) *Config {
c := &Config{
myInts: make(map[string]*MyInt),
}
cType := reflect.TypeOf(c).Elem() // have to use Elem() to get actual value
cValue := reflect.ValueOf(c).Elem()
for i := 0; i < cType.NumField(); i++ {
f := cType.Field(i)
if f.Name == "myInts" {
continue
}
tag := string(f.Tag)
fields := strings.Split(tag, ":")
switch f.Type.Name() {
case "MyInt":
intVal, _ := strconv.Atoi(fields[1])
val := NewMyInt(fields[0], intVal, db)
c.myInts[fields[0]] = val
// How do I set the i'th field to this newly constructed value?
cValue.Field(i).Set(reflect.ValueOf(val).Elem())
}
}
fmt.Println(c.Value1.intVal, c.Value2.intVal)
return c
}
In my Go program I start multiple worker groups for every department.
I want to wait for workers from each department to complete before exiting the program
I cannot use a single WaitGroups because in the actual scenario I may have to end any particular department and need to wait only on that.
This is simplified version of code, but it panics with a message
panic: runtime error: invalid memory address or nil pointer dereference
package main
import (
"fmt"
"sync"
"time"
)
var wgMap map[string]*sync.WaitGroup
func deptWorker(dName string, id int) {
defer wgMap[dName].Done()
fmt.Printf("Department %s : Worker %d starting\n", dName, id)
time.Sleep(time.Second)
fmt.Printf("Department %s : Worker %d done\n", dName, id)
}
func department(dName string) {
var wg sync.WaitGroup
for i := 1; i <= 3; i++ {
wg.Add(1)
go deptWorker(dName, i)
}
wgMap[dName] = &wg
}
func main() {
go department("medical")
go department("electronics")
wgMap["medical"].Wait()
wgMap["electronics"].Wait()
}
Two fix nil panic you simply need to use
var wgMap = map[string]*sync.WaitGroup{}
It will initialize the map. However, in my view, it's better here to create a new abstraction, let's name it 'WaitMap'.
It can be implemented in this way:
package main
import (
"fmt"
"sync"
"time"
)
type WaitMapObject struct {
wg map[string]int
mu sync.Mutex
cond sync.Cond
}
func WaitMap() *WaitMapObject {
m := &WaitMapObject{}
m.wg = make(map[string]int)
m.cond.L = &m.mu
return m
}
func (m *WaitMapObject) Wait(name string) {
m.mu.Lock()
for m.wg[name] != 0 {
m.cond.Wait()
}
m.mu.Unlock()
}
func (m *WaitMapObject) Done(name string) {
m.mu.Lock()
no := m.wg[name] - 1
if no < 0 {
panic("")
}
m.wg[name] = no
m.mu.Unlock()
m.cond.Broadcast()
}
func (m *WaitMapObject) Add(name string, no int) {
m.mu.Lock()
m.wg[name] = m.wg[name] + no
m.mu.Unlock()
}
func deptWorker(dName string, id int, wm *WaitMapObject) {
defer wm.Done(dName)
fmt.Printf("Department %s : Worker %d starting\n", dName, id)
time.Sleep(time.Second)
fmt.Printf("Department %s : Worker %d done\n", dName, id)
}
func department(dName string, wm *WaitMapObject) {
for i := 1; i <= 3; i++ {
wm.Add(dName,1)
go deptWorker(dName, i, wm)
}
wm.Done(dName)
}
func main() {
wm := WaitMap()
wm.Add("mediacal",1)
go department("medical", wm)
wm.Add("electronics",1)
go department("electronics", wm)
wm.Wait("medical")
wm.Wait("electronics")
}
I have two different struct as mentioned below A abd B and two process functions. Is there any way by means of which i can write a common function to generate the map[string]struct for the both the struct. Moreover, is there any way using reflection given the struct name i can create the object of the same?
type A struct {
name string
// more fields
}
type B struct {
name string
// more fields
}
func ProcessA(input []A) map[string]A {
output := make(map[string]A)
for _, v := range input {
output[v.name] = v
}
return output
}
func ProcessB(input []B) map[string]B {
output := make(map[string]B)
for _, v := range input {
output[v.name] = v
}
return output
}
Idiomatic way in Go would be to use interface.
type Named interface {
Name() string
}
type letter struct {
name string
}
func (l letter) Name() string {
return l.name
}
type A struct {
letter
// more fields
}
type B struct {
letter
// more fields
}
func ProcessNameds(input []Named) map[string]Named {
output := make(map[string]Named, len(input))
for _, v := range input {
output[v.Name()] = v
}
return output
}
Well, see if something like this would help:
package main
import (
"fmt"
"strconv"
)
type A struct {
name string
// more fields
}
type B struct {
name string
// more fields
}
func Process(x interface{}) interface{} {
ma := make(map[string]int)
mb := make(map[string]string)
if x == nil {
return nil
} else if a, ok := x.([]A); ok {
fmt.Printf("Type A argument passed %s\n", x)
ma[a[0].name] = 1
ma[a[1].name] = 2
return ma //you can return whatever type you want here
} else if b, ok := x.([]B); ok {
fmt.Printf("Type B argument passed %s\n", x)
mb[b[0].name] = "a"
mb[b[1].name] = "b"
return mb //you can return whatever type you want here
} else {
panic(fmt.Sprintf("Unexpected type %T: %v", x, x))
}
return nil
}
func main() {
a := make([]A, 5)
for i := 0; i < len(a); i++ {
a[i].name = strconv.Itoa(i) + "A"
}
b := make([]B, 7)
for i := 0; i < len(b); i++ {
b[i].name = strconv.Itoa(i) + "B"
}
fmt.Println(Process(a))
fmt.Println(Process(b))
//Uncomment line below to see the panic
//fmt.Println(Process(8))
}
https://play.golang.org/p/irdCsbpvUv_t