I have a tree-like structure, that has string regexp and I want Go compiled *regexp.Regexp to be part of it as well, in order to run algorithms on the tree. When I marshal and pass it to a different machine I may just recompile it again from the string. What is the correct way to do that, how to force protobuf to store pointers in a structure, that it ideally wont marshal? (the only way that i see is to make uint64 field and cast its value to/from *Regexp)
pseudo-code (because required wanted features seems to be not in the language):
// struct generated by protoc
type ProtoMessage struct {
Data string
Source string
Regexp uint64 // should not be marshalled, should be forcefully omitted from payload when doing proto.Marshal, ideally it should be *regexp.Regexp
Left *ProtoMessage
Right *ProtoMessage
}
func main() {
// sender computer doSend():
mSrc := &ProtoMessage{Data:"its meee!!!", Source: "hello.+world"}
payload, _ := proto.Marshal(m)
//receiver computer: onRecv()
mDst := new(ProtoMessage)
proto.Unmarshal(payload, mDst)
r, _ := regexp.Compile(mDst.Source)
mDst.Regexp = uint64(unsafe.Pointer(r)) // not working btw
TreeMatch = func(tree* ProtoMessage, line string) string {
if *regexp.Regexp(t.Regexp).Match(line) { // not working line
return t.Data
}
if tree.Left == nil {
return ""
}
return TreeMatch(tree.Left, line)
}
assert( TreeMatch(mDst, "hello, world") == "its meee!!!") // panic if condition is false
}
With json marshal i can just pot a pointer to regexp and provide a tag json:"-" in order not to include this field into marshalled structure, and ofc its important feature of marshalling/unmarshalling system to stay efficient (eg use same structure to run algorithms on in, and avoid data copying after unmarshal). How can I do the same with protobuf?
You can't store a pointer in a protobuf, as the recipient is likely a different computer. Even if you could, you'd get a panic as soon as you tried to dereference the pointer. Easiest thing to do would be just pass the RegExp string, then compile again at the destination:
package main
import (
"fmt"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/structpb"
)
func main() {
v := structpb.NewStringValue("hello.+world")
b, err := proto.Marshal(v)
if err != nil {
panic(err)
}
fmt.Printf("%q\n", b) // "\x1a\fhello.+world"
}
Note: you can't hack around this with Gob either:
package main
import (
"bytes"
"encoding/gob"
"regexp"
)
func main() {
re := regexp.MustCompile("hello.+world")
buf := new(bytes.Buffer)
if err := gob.NewEncoder(buf).Encode(re); err != nil {
panic(err) // type regexp.Regexp has no exported fields
}
}
Found the solution, you just have to have any pointer inside your struct (no matter if its marshalling or not, you are not using its unmarshalled value on receiver side):
proto declaration:
syntax = "proto3";
package main;
option go_package = ".;main";
message Empty {
}
message ProtoMessage {
string data = 1;
string source = 2;
Empty regexp = 3; // ideally should not be marshalled at all, like `json:"-"` but for protobuf
ProtoMessage left = 4;
ProtoMessage right = 5;
}
testing code:
package main
import (
"regexp"
"testing"
"unsafe"
)
type Empty struct {
//state protoimpl.MessageState
//sizeCache protoimpl.SizeCache
//unknownFields protoimpl.UnknownFields
}
// struct generated by protoc
type ProtoMessage struct {
//state protoimpl.MessageState
//sizeCache protoimpl.SizeCache
//unknownFields protoimpl.UnknownFields
Data string `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"`
Source string `protobuf:"bytes,2,opt,name=source,proto3" json:"source,omitempty"`
Regexp *Empty `protobuf:"bytes,3,opt,name=regexp,proto3" json:"regexp,omitempty"` // ideally should not be marshalled at all, like `json:"-"` but for protobuf
Left *ProtoMessage `protobuf:"bytes,4,opt,name=left,proto3" json:"left,omitempty"`
Right *ProtoMessage `protobuf:"bytes,5,opt,name=right,proto3" json:"right,omitempty"`
}
func (p *ProtoMessage) GetCompiledRegexp() *regexp.Regexp {
return (*regexp.Regexp)(unsafe.Pointer(p.Regexp))
}
func (p *ProtoMessage) SetCompiledRegexp(r *regexp.Regexp) {
p.Regexp = (*Empty)(unsafe.Pointer(r))
}
func TreeMatch(tree *ProtoMessage, line string) string {
if tree.GetCompiledRegexp().Match([]byte(line)) { // not working line
return tree.Data
}
if tree.Left == nil {
return ""
}
return TreeMatch(tree.Left, line)
}
func TestTreeMatch(t *testing.T) {
//happening at receiver side: imagine its proto.Unmarshal(payload, receiverMsg)
receiverMsg := &ProtoMessage{
Data: "its meee!!!",
Source: "hello.+world",
}
r, _ := regexp.Compile(receiverMsg.Source)
receiverMsg.SetCompiledRegexp(r)
if TreeMatch(receiverMsg, "helloworld") != "" {
t.Fatalf("TreeMatch gives non-existing match!")
}
if TreeMatch(receiverMsg, "hello, world") != "its meee!!!" {
t.Fatalf("TreeMatch is not working!")
}
}
type ProtoMessageDirect struct {
Data string
Source string
Regexp *regexp.Regexp
Left *ProtoMessageDirect
Right *ProtoMessageDirect
}
func (p *ProtoMessageDirect) GetCompiledRegexp() *regexp.Regexp {
return p.Regexp
}
func (p *ProtoMessageDirect) SetCompiledRegexp(r *regexp.Regexp) {
p.Regexp = r
}
func TreeMatchDirect(tree *ProtoMessageDirect, line string) string {
if tree.GetCompiledRegexp().Match([]byte(line)) { // not working line
return tree.Data
}
if tree.Left == nil {
return ""
}
return TreeMatchDirect(tree.Left, line)
}
func BenchmarkRegexpCast(b *testing.B) {
receiverMsg := &ProtoMessage{
Data: "its meee!!!",
Source: "hello.+world",
}
r, _ := regexp.Compile(receiverMsg.Source)
receiverMsg.SetCompiledRegexp(r)
b.ResetTimer()
for i := 0; i < b.N; i++ {
TreeMatch(receiverMsg, "hello, world")
}
}
func BenchmarkRegexpDirect(b *testing.B) {
receiverMsg := &ProtoMessageDirect{
Data: "its meee!!!",
Source: "hello.+world",
}
r, _ := regexp.Compile(receiverMsg.Source)
receiverMsg.SetCompiledRegexp(r)
b.ResetTimer()
for i := 0; i < b.N; i++ {
TreeMatchDirect(receiverMsg, "hello, world")
}
}
TestTreeMatch is passing and Benchmarks shows that such a cast does not create any meaningful difference:
BenchmarkRegexpCast-20 2741786 376.7 ns/op 16 B/op 1 allocs/op
BenchmarkRegexpDirect-20 3075280 377.0 ns/op 16 B/op 1 allocs/op
PASS
Related
Let's say we have a library provide a function Double to double the integer, we use pointer i to get the result value not by return:
package api
type Action interface {
Double(i *int) error
}
type NUM struct{}
func (n NUM) Double(i *int) error {
*i *= 2
return nil
}
in our main function we use this library to do our task. like this:
package app
import (
"fmt"
"github.com/hotsnow/api"
)
func main() {
j := job{a: &api.NUM{}}
d := j.task(3)
fmt.Println(3, d)
}
type job struct {
a api.Action
}
// double me
func (j job) task(i int) int {
j.a.Double(&i)
return i
}
Now we need to test the task() function, how can we get the pointer return bye mock the Double function?
Here is the test:
package app
import (
"github.com/golang/mock/gomock"
"github.com/hotsnow/mocks"
"testing"
)
func TestReq(t *testing.T) {
ctrl := gomock.NewController(t)
defer ctrl.Finish()
m := mocks.NewMockAction(ctrl)
m.EXPECT().Double(gomock.Any()).Return(nil)
j := job{a: m}
got := j.task(3)
if got != 6 {
t.Errorf("got = %#v; want 6", got)
}
}
The code here: https://github.com/hotsnow/mock.git (stackoverflow branch)
you can use gomock setarg function for this
yourPackage.EXPECT().insert(&pointer).SetArg(0, newPointer)
You can achieve this with the provided Eq() matcher, which internally calls reflect.DeepEqual() on the expected and actual values; as per the documentation for this method:
Pointer values are deeply equal if they are equal using Go's == operator or if they point to deeply equal values.
Say we have a function that depends upon an interface method that takes a pointer parameter:
package resource
type ServiceRequest struct {
Name string
Owner *string // this is a pointer so it can be omitted with `nil`
}
type Model struct {
// resource model...
}
type ResourceService interface {
Fetch(req *ServiceRequest) (Model, error)
}
type getResourceHandler struct {
resourceService ResourceService
}
type GetResourceEvent struct {
Resource string
Owner *string
}
func NewResourceHandler(resourceService ResourceService) *getResourceHandler {
return &getResourceHandler{resourceService}
}
func (h *getResourceHandler) Handle(event GetResourceEvent) (Model, error) {
return h.resourceService.Fetch(&ServiceRequest{event.Resource, event.Owner})
}
We can use the Eq() matcher when setting up the expectation against our generated mock of the ResourceService interface:
package test
import (
"testing"
"github.com/golang/mock/gomock"
"github.com/stretchr/testify/assert"
"github.com/org/repo/internal/mock"
"github.com/org/repo/internal/resource"
)
func optionalString(str string) *string {
return &str
}
func Test_GetResourceHandler_ReturnsResultFromService(t *testing.T) {
resourceName := "my-resource"
owner := optionalString("Joe Bloggs")
resourceReq := &resource.ServiceRequest{resourceName, owner}
event := resource.GetResourceEvent{resourceName, owner}
model := resource.Model{ /* fields here... */ }
ctrl := gomock.NewController(t)
mockResourceService := mock.NewMockResourceService(ctrl)
handler := resource.NewResourceHandler(mockResourceService)
mockResourceService.EXPECT().Fetch(gomock.Eq(resourceReq)).Return(model, nil)
res, err := handler.Handle(event)
assert.Nil(t, err)
assert.Equal(t, model, res)
}
If you change the contents of the service request in either the test or the unit under test, you'll see that the test no longer passes. Otherwise, it will pass in spite of the test and the unit under test having their own respective pointers to separate ServiceRequest{} values.
It seems you don't have to use gomock to test the task method.
Since you have an interface, why not just create a mock implementation of the interface, for example:
type dummy struct{
callCount int
}
func (d *dummy) Double(i *int) error {
d.callCount++
return nil
}
d := dummy{}
j := job{a: &d}
got := j.task(3)
if d.callCount != 1 {
// XXX
}
I am writing a program which has several structs and functions to handle these structs differently. I am having a generic function which calls the required function based on the inputs. Is there a generic way to use the returned value from getStruct()?
package main
var X func(s []string) A
var Y func(s []string) B
type A struct {
Name string
Place string
}
type B struct {
Name string
Place string
Value string
}
func newA(s []string) A {
a := A{
Name: s[0],
Place: s[1],
}
return a
}
func newB(s []string) B {
a := B{
Name: s[0],
Place: s[1],
Value: s[2],
}
return a
}
func getStruct(t string) interface{} {
switch {
case t == "A":
return X
case t == "B":
return Y
default:
return //someStruct
}
}
func main() {
buildNewStruct := getStruct("A") //Lets assume "A" here is got as an argument
var strSlice = []string{"Bob", "US"}
buildNewStruct(strSlice) //How to do this operation?
//I am hoping to use buildNewStruct(strSlice) to dynamically call
//either of newA(strSlice) or newB(strSlice) function
}
I have tried looking at this and this the later is not exactly the same as my question.
Since I am new to go, I am not sure if something like this is possible.
you can use the reflect package to set the struct properties to the equivalent index positioned value from an []interface{} slice.
package main
import (
"fmt"
"log"
"reflect"
)
func main() {
var a A
err := decode(&a, []interface{}{"Name", "Place"})
log.Println(err)
log.Println(a)
}
func decode(dst interface{}, values []interface{}) error {
rvptr := reflect.ValueOf(dst)
if rvptr.Kind() != reflect.Ptr {
return fmt.Errorf("value must be ptr")
}
rv := rvptr.Elem()
if rv.NumField() < len(values) {
return fmt.Errorf("too many values")
}
if rv.NumField() > len(values) {
return fmt.Errorf("too few values")
}
rvalues := reflect.ValueOf(values)
for i := range values {
f := rv.FieldByIndex([]int{i})
f.Set(rvalues.Index(i).Elem())
}
return nil
}
type A struct {
Name string
Place string
}
type B struct {
Name string
Place string
Value string
}
prints
$ go run main.go
2019/11/21 17:00:17 <nil>
2019/11/21 17:00:17 {Name Place}
The problem is the return type for the function.
func newA(in []string) interface{} {...}
func newB(in []string) interface{} {...}
func getStruct(name string) func([]string) interface{} {
switch name {
case "A": return newA
case "B": return newB
}
return nil
}
func main() {
buildNewStruct := getStruct("A")
var strSlice = []string{"Bob", "US"}
str:=buildNewStruct(strSlice)
if a, ok:=str.(A); ok {
...
}
}
With this approach, even though you saved some code by calling a unified buildNewStruct(), you have to use type assertions to figure out what is returned from that function, so this may not make a lot of sense. It depends on your exact use case though.
I would like to know if there is a native way to transform strings like:
a.string
a-string
a_string
a string
In a string that follows the convention for public field members of structs, in Go.
The idea is to write a function that accepts a string and try to get the value of the field, even if the passed string is not using the PascalCase convention, example:
type MyStruct struct {
Debug bool
AString bool
SomethingMoreComplex
}
var myStruct MyStruct
func GetField(s string) reflect.Value {
v := reflect.ValueOf(myStruct)
return v.FieldByName(s)
}
function main() {
GetField("debug")
GetField("a.string")
GetField("a-string")
GetField("a_string")
GetField("-a.string")
GetField("something-more-complex")
}
I was using the strcase package, but it only works for ASCII.
By the magic of regular expressions
https://goplay.space/#xHfxG249CsH
package main
import (
"fmt"
"regexp"
"strings"
)
func ConvertFieldName(s string) string {
r := regexp.MustCompile("(\\b|-|_|\\.)[a-z]")
return r.ReplaceAllStringFunc(s, func(t string) string {
if len(t) == 1 {
return strings.ToUpper(t)
} else {
return strings.ToUpper(string(t[1]))
}
})
}
func main() {
fmt.Println(ConvertFieldName("debug"))
fmt.Println(ConvertFieldName("a.string"))
fmt.Println(ConvertFieldName("a-string"))
fmt.Println(ConvertFieldName("a_string"))
fmt.Println(ConvertFieldName("-a.string"))
fmt.Println(ConvertFieldName("something-more-complex"))
}
Outputs
Debug
AString
AString
AString
AString
SomethingMoreComplex
package main
import (
"fmt"
"reflect"
)
type MyStruct struct {
Debug bool
AString bool
}
var myStruct MyStruct
func GetField(s string) (reflect.Value, error) {
t := reflect.TypeOf(myStruct)
v := reflect.ValueOf(myStruct)
for fieldIndex := 0; fieldIndex < v.NumField(); fieldIndex++ {
if t.Field(fieldIndex).Name == s {
return v.Field(fieldIndex), nil
}
}
return reflect.Value{}, fmt.Errorf("%s not exist", s)
}
func main() {
var v reflect.Value
var err error
v, err = GetField("Debug")
fmt.Println(v, err)
v, err = GetField("debug")
fmt.Println(v, err)
}
the other way, you can try define your own field's tag, like json tag
I'm not quite sure how to address this question, please feel free to edit.
With the first code block below, I am able to check if a all fields of a struct are nil.
In reality however, the values injected in the struct, are received as args.Review (see second code block below).
In the second code block, how can I check if all fields from args.Review are nil?
Try it on Golang Playground
package main
import (
"fmt"
"reflect"
)
type review struct {
Stars *int32 `json:"stars" bson:"stars,omitempty" `
Commentary *string `json:"commentary" bson:"commentary,omitempty"`
}
func main() {
newReview := &review{
Stars: nil,
// Stars: func(i int32) *int32 { return &i }(5),
Commentary: nil,
// Commentary: func(i string) *string { return &i }("Great"),
}
if reflect.DeepEqual(review{}, *newReview) {
fmt.Println("Nothing")
} else {
fmt.Println("Hello")
}
}
Try the second code on Golang Playground
This code below gets two errors:
prog.go:32:14: type args is not an expression
prog.go:44:27: args.Review is not a type
package main
import (
"fmt"
"reflect"
)
type review struct {
Stars *int32 `json:"stars" bson:"stars,omitempty" `
Commentary *string `json:"commentary" bson:"commentary,omitempty"`
}
type reviewInput struct {
Stars *int32
Commentary *string
}
type args struct {
PostSlug string
Review *reviewInput
}
func main() {
f := &args {
PostSlug: "second-post",
Review: &reviewInput{
Stars: func(i int32) *int32 { return &i }(5),
Commentary: func(i string) *string { return &i }("Great"),
},
}
createReview(args)
}
func createReview(args *struct {
PostSlug string
Review *reviewInput
}) {
g := &review{
Stars: args.Review.Stars,
Commentary: args.Review.Commentary,
}
if reflect.DeepEqual(args.Review{}, nil) {
fmt.Println("Nothing")
} else {
fmt.Println("Something")
}
}
If you're dealing with a small number of fields you should use simple if statements to determine whether they are nil or not.
if args.Stars == nil && args.Commentary == nil {
// ...
}
If you're dealing with more fields than you would like to manually spell out in if statements you could use a simple helper function that takes a variadic number of interface{} arguments. Just keep in mind that there is this: Check for nil and nil interface in Go
func AllNil(vv ...interface{}) bool {
for _, v := range vv {
if v == nil {
continue
}
if rv := reflect.ValueOf(v); !rv.IsNil() {
return false
}
}
return true
}
if AllNil(args.Stars, args.Commentary, args.Foo, args.Bar, args.Baz) {
// ...
}
Or you can use the reflect package to do your bidding.
func NilFields(x interface{}) bool {
rv := reflect.ValueOf(args)
rv = rv.Elem()
for i := 0; i < rv.NumField(); i++ {
if f := rv.Field(i); f.IsValid() && !f.IsNil() {
return false
}
}
return true
}
if NilFields(args) {
// ...
}
What is the way of printing "Foo" here? In this example, what prints is "string".
http://play.golang.org/p/ZnK6PRwEPp
type A struct {
Foo string
}
func (a *A) PrintFoo() {
fmt.Println("Foo value is " + a.Foo)
}
func main() {
a := &A{Foo: "afoo"}
val := reflect.Indirect(reflect.ValueOf(a))
fmt.Println(val.Field(0).Type().Name())
}
You want val.Type().Field(0).Name. The Field method on reflect.Type will return a struct describing that field, which includes the name, among other information.
There is no way to retrieve the field name for a reflect.Value representing a particular field value, since that is a property of the containing struct.
I think the better way to get the fields' name in the struct is
func main() {
a := &A{Foo: "afoo"}
val := reflect.ValueOf(a).Elem()
for i:=0; i<val.NumField();i++{
fmt.Println(val.Type().Field(i).Name)
}
}
There are two tips:
use .Elem() after you reflect.ValueOf(a), because in your case, a is a pointer.
val.Field(i).Type().Name is totally different from val.Type().Field(i).Name. The latter one can get the name of the field in the struct
Hope that it is helpful..
If you want to have a look at more cases, please check my 2mins article
You need to Get the Field of the Type Definition not of the Value.
http://play.golang.org/p/7Bc7MJikbJ
package main
import "fmt"
import "reflect"
type A struct {
Foo string
}
func (a *A) PrintFoo() {
fmt.Println("Foo value is " + a.Foo)
}
func main() {
a := &A{Foo: "afoo"}
val := reflect.Indirect(reflect.ValueOf(a))
fmt.Println(val.Type().Field(0).Name)
}
With the new Names method of the structs package it's even more easier:
package main
import (
"fmt"
"github.com/fatih/structs"
)
type A struct {
Foo string
Bar int
}
func main() {
names := structs.Names(&A{})
fmt.Println(names) // ["Foo", "Bar"]
}
You can also use https://github.com/fatih/structs
// Convert the fields of a struct to a []*Field
fields := s.Fields()
for _, f := range fields {
fmt.Printf("field name: %+v\n", f.Name())
}
package main
import "fmt"
import "reflect"
type A struct {
Foo string
}
func (a *A) PrintFoo() {
fmt.Println("Foo value is " + a.Foo)
}
func main() {
a := &A{Foo: "afoo"}
//long and bored code
t := reflect.TypeOf(*a)
if t.Kind() == reflect.Struct {
for i := 0; i < t.NumField(); i++ {
fmt.Println(t.Field(i).Name)
}
} else {
fmt.Println("not a stuct")
}
//shorthanded call
fmt.Println(reflect.TypeOf(*a).Field(0).Name)//can panic if no field exists
}
You can use this function, which takes the struct as the first parameter, and then its fields. It returns the map type, which is convenient to use
If you use fields from another structure, nothing will happen
If you try to use a different type, it will cause panic
Note that the field has an ordinal number according to the list (starting from 0). All fields in the structure must start with uppercase
func GetStructFieldName(Struct interface{}, StructField ...interface{}) (fields map[int]string) {
fields = make(map[int]string)
s := reflect.ValueOf(Struct).Elem()
for r := range StructField {
f := reflect.ValueOf(StructField[r]).Elem()
for i := 0; i < s.NumField(); i++ {
valueField := s.Field(i)
if valueField.Addr().Interface() == f.Addr().Interface() {
fields[i] = s.Type().Field(i).Name
}
}
}
return fields
}
Full example and playground
package main
import (
"fmt"
"reflect"
)
type Example struct {
Apple bool
Pear int
}
func GetStructFieldName(Struct interface{}, StructField ...interface{}) (fields map[int]string) {
fields = make(map[int]string)
for r := range StructField {
s := reflect.ValueOf(Struct).Elem()
f := reflect.ValueOf(StructField[r]).Elem()
for i := 0; i < s.NumField(); i++ {
valueField := s.Field(i)
if valueField.Addr().Interface() == f.Addr().Interface() {
fields[i] = s.Type().Field(i).Name
}
}
}
return fields
}
func main() {
e := Example{}
names := GetStructFieldName(&e, &e.Apple, &e.Pear)
fmt.Println(names)
fmt.Println(names[0], names[1])
for i := range names {
fmt.Println(names[i])
}
/* Output:
map[0:Apple 1:Pear]
Apple Pear
Apple
Pear
*/
}