Multi WATCH in Golang using go-redis/redis - go

I am trying to do this 4 redis calls in one transaction using https://github.com/go-redis/redis :
func getHost(key) host, score, error{
numberOfHosts, err := rdb.ZCard(key).Result()
if err != nil {
return nil, 0, err
}
host, err := rdb.ZRangeByScoreWithScores(key, redis.ZRangeBy{
Min: "0",
Max: strconv.Itoa(maxScore),
Offset: numberOfHosts - 1,
Count: 1,
}).Result()
if err != nil {
return nil, 0, err
}
score := host[0].Score
member, err := json.Marshal(host[0].Member)
if err != nil {
return nil, 0, err
}
if int(score) < maxCapacity {
rdb.ZIncrBy(key, score+1, string(member))
}
host, err := rdb.GetHost(string(member))
if err != nil {
return nil, 0, err
}
return host, int(score), nil
}
I found this https://godoc.org/github.com/go-redis/redis#Client.Watch in the doc and I am trying to do something with it but it is a bit difficult at the moment.
Is anybody already done this and can provide advice ?
EDIT :
I tried to do this :
count := func(key string) error {
txf := func(tx *redis.Tx) error {
numberOfHosts, err := tx.ZCard(key).Result()
if err != nil {
return err
}
numberOfHosts--
_, err = tx.Pipelined(func(pipe redis.Pipeliner) error {
getHostKey := func(key string, numberOfHosts int) error {
txf := func(tx *redis.Tx) error {
host, err := tx.ZRangeByScoreWithScores(key, redis.ZRangeBy{
Min: "0",
Max: strconv.Itoa(maxCapacity),
Offset: numberOfHosts,
Count: 1,
}).Result()
if err != nil {
return err
}
if len(host) == 0 {
return fmt.Errorf("Get Host did not return a host for the key: %v", key)
}
score := host[0].Score
member, err := json.Marshal(host[0].Member)
if err != nil {
return err
}
score++
_, err = tx.Pipelined(func(pipe redis.Pipeliner) error {
if int(score) < maxCapacity {
incScore := func(key string, score int, member string) (*Host, int, error) {
txf := func(tx *redis.Tx) error {
tx.ZIncrBy(key, score, string(member))
host, err := rc.GetHost(string(member))
if err != nil {
return nil, 0, err
}
return host, int(score), nil
}
}
err := rc.Client.Watch(txf, key)
if err != redis.TxFailedErr {
return err
}
}
}
})
}
err := rc.Client.Watch(txf, key)
if err != redis.TxFailedErr {
return err
}
}
if err := getHostKey(key, numberOfHosts); err != nil {
return err
}
})
}
err := rc.Client.Watch(txf, key)
if err != redis.TxFailedErr {
return err
}
}
if err := count(key); err != nil {
return nil, 0, err
But the compiler is not happy with all the variable coming from the previous watch.
I will look into LUA script

Related

I get an error in golang test and would like to know how to improve it

sta9_test.go:124: Cannot convert id to numeric, got = id
sta9_test.go:124: Cannot convert id to numericCannot convert id to numeric, got = id
sta9_test.go:145: It is a value we do not expect.
map[string]any{
- "UpdatedAt": string("2022-10-27T15:19:10Z"),
- "createdAenter code heret": string("2022-10-27T15:19:10Z"),
"description": string(""),}
func TestStation9(t *testing.T) {
dbPath := "./temp_test.db"
if err := os.Setenv("DB_PATH", dbPath); err != nil {
t.Error("dbPathのセットに失敗しました。", err)
return
}
t.Cleanup(func() {
if err := os.Remove(dbPath); err != nil {
t.Errorf("テスト用のDBファイルの削除に失敗しました: %v", err)
return
}
})
todoDB, err := db.NewDB(dbPath)
if err != nil {
t.Error("DBの作成に失敗しました。", err)
return
}
defer func(todoDB *sql.DB) {
err := todoDB.Close()
if err != nil {
t.Error("DBのクローズに失敗しました.", err)
}
}(todoDB)
r := router.NewRouter(todoDB)
srv := httptest.NewServer(r)
defer srv.Close()
testcases := map[string]struct {
Subject string
Description string
WantHTTPStatusCode int
}{
"Subject is empty": {
WantHTTPStatusCode: http.StatusBadRequest,
},
"Description is empty": {
Subject: "todo subject",
WantHTTPStatusCode: http.StatusOK,
},
"Subject and Description is not empty": {
Subject: "todo subject",
Description: "todo description",
WantHTTPStatusCode: http.StatusOK,
},
}
for name, tc := range testcases {
name := name
tc := tc
t.Run(name, func(t *testing.T) {
resp, err := http.Post(srv.URL+"/todos", "application/json",
bytes.NewBufferString(fmt.Sprintf(`{"subject":"%s","description":"%s"}`, tc.Subject, tc.Description)))
if err != nil {
t.Error("リクエストの送信に失敗しました。", err)
return
}
defer func() {
if err := resp.Body.Close(); err != nil {
t.Error("レスポンスのクローズに失敗しました。", err)
return
}
}()
if resp.StatusCode != tc.WantHTTPStatusCode {
t.Errorf("期待していない HTTP status code です, got = %d, want = %d", resp.StatusCode, tc.WantHTTPStatusCode)
return
}
if tc.WantHTTPStatusCode != http.StatusOK {
return
}
var m map[string]interface{}
if err := json.NewDecoder(resp.Body).Decode(&m); err != nil {
t.Error("レスポンスのデコードに失敗しました。", err)
return
}
v, ok := m["todo"]
if !ok {
t.Error("レスポンスの中にtodoがありません。")
return
}
got, ok := v.(map[string]interface{})
if !ok {
t.Error("レスポンスの中のtodoがmapではありません。")
return
}
want := map[string]interface{}{
"subject": tc.Subject,
"description": tc.Description,
}
now := time.Now().UTC()
fmt.Println(got)
fmt.Println(want)
diff := cmp.Diff(got, want, cmpopts.IgnoreMapEntries(func(k string, v interface{}) bool {
switch k {
case "id":
if vv, _ := v.(float64); vv == 0 {
t.Errorf("id を数値に変換できません, got = %s", k)
}
return true
case "created_at", "updated_at":
vv, ok := v.(string)
if !ok {
t.Errorf("日付が文字列に変換できません, got = %+v", k)
return true
}
if tt, err := time.Parse(time.RFC3339, vv); err != nil {
t.Errorf("日付が期待しているフォーマットではありません, got = %s", k)
} else if now.Before(tt) {
t.Errorf("日付が未来の日付になっています, got = %s", tt)
}
return true
}
return false
}))
if diff != "" {
t.Error("期待していない値です\n", diff)
}
})
}
}
type TODOHandler struct {
svc *service.TODOService
}
// NewTODOHandler returns TODOHandler based http.Handler.
func NewTODOHandler(svc *service.TODOService) *TODOHandler {
return &TODOHandler{
svc: svc,
}
}
func (h *TODOHandler)ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPost{
var todo model.CreateTODORequest
if err := json.NewDecoder(r.Body).Decode(&todo); err != nil {
log.Fatal(err)
}
if todo.Subject == ""{
w.WriteHeader(400)
}else{
createdTodo, _ := h.Create(r.Context(),&todo)
e := json.NewEncoder(w)
if err:= e.Encode(createdTodo); err != nil{
log.Fatal("ListenAndServe:", err)
}
}
}
}
// Create handles the endpoint that creates the TODO.
func (h *TODOHandler) Create(ctx context.Context, req *model.CreateTODORequest) (*model.CreateTODOResponse, error) {
createtodo, err := h.svc.CreateTODO(ctx, req.Subject, req.Description)
if err != nil{
log.Fatal(err)
}
return &model.CreateTODOResponse{TODO: *createtodo}, nil
}
type TODOService struct {
db *sql.DB
}
// NewTODOService returns new TODOService.
func NewTODOService(db *sql.DB) *TODOService {
return &TODOService{
db: db,
}
}
// CreateTODO creates a TODO on DB.
func (s *TODOService) CreateTODO(ctx context.Context, subject, description string) (*model.TODO, error) {
const (
insert = `INSERT INTO todos(subject, description) VALUES(?, ?)`
confirm = `SELECT subject, description, created_at, updated_at FROM todos WHERE id = ?`
)
var todo model.TODO
if subject == ""{
msg := "subjectがありません"
return &todo,fmt.Errorf("err %s", msg)
}
stmt,err := s.db.PrepareContext(ctx,insert)
if err != nil{
log.Fatal("server/todo.go s.db.ExecContext(ctx,insert,subject,description)",err)
}
defer stmt.Close()
res, err := stmt.ExecContext(ctx, subject,description)
if err != nil {
return nil, err
}
insert_id,err := res.LastInsertId()
if err != nil{
log.Fatal("server/todo.go stmt.RowsAffected()",err)
}
err = s.db.QueryRowContext(ctx,confirm,insert_id).Scan(&todo.Subject,&todo.Description,&todo.CreatedAt,&todo.UpdatedAt)
if err != nil{
log.Fatal("server/todo.go s.db.QueryRowContext(ctx,confirm,insert_id).Scan(&todo.Subject,&todo.Description,&todo.CreatedAt,&todo.UpdatedAt)",err)
}
return &todo, err
}
I am building a TODO application with TDD and have a question. I get the error at the top. What kind of error is this and how can I improve it?

How to return the errors in golang?

In this code, I am taking the data from the environment variable and using it to send the email to the given address. It only works if I exclude the errors from the code. I returned the errors and used them in middleware because I don't want to use log.Fatal(err) every time. But it gives me two errors.
1. cannot use "" (untyped string constant) as [4]string value in return statement
2. code can not execute unreachable return statement
func LoadEnvVariable(key string) (string, error) {
viper.SetConfigFile(".env")
err := viper.ReadInConfig()
return "", err
value, ok := viper.Get(key).(string)
if !ok {
return "", err
}
return value, nil
}
func Email(value [4]string) ([4]string, error) {
mail := gomail.NewMessage()
myEmail, err := LoadEnvVariable("EMAIL")
return "", err
appPassword, err := LoadEnvVariable("APP_PASSWORD")
return "", err
mail.SetHeader("From", myEmail)
mail.SetHeader("To", myEmail)
mail.SetHeader("Reply-To", value[1])
mail.SetHeader("subject", value[2])
mail.SetBody("text/plain", value[3])
a := gomail.NewDialer("smtp.gmail.com", 587, myEmail, appPassword)
err = a.DialAndSend(mail)
return "", err
return value, nil
}
use this to check error
if err != nil {
return "", err
}
func LoadEnvVariable(key string) (string, error) {
viper.SetConfigFile(".env")
err := viper.ReadInConfig()
if err != nil {
return "", err
}
value, ok := viper.Get(key).(string)
if !ok {
return "", err
}
return value, nil
}
func Email(value [4]string) ([4]string, error) {
mail := gomail.NewMessage()
myEmail, err := LoadEnvVariable("EMAIL")
if err != nil {
return "", err
}
appPassword, err := LoadEnvVariable("APP_PASSWORD")
if err != nil {
return "", err
}
mail.SetHeader("From", myEmail)
mail.SetHeader("To", myEmail)
mail.SetHeader("Reply-To", value[1])
mail.SetHeader("subject", value[2])
mail.SetBody("text/plain", value[3])
a := gomail.NewDialer("smtp.gmail.com", 587, myEmail, appPassword)
err = a.DialAndSend(mail)
if err != nil {
return "", err
}
return value, nil
}
Firstly, to error handle, check if err is nil before returning!
So not:
myEmail, err := LoadEnvVariable("EMAIL")
return "", err
but instead:
myEmail, err := LoadEnvVariable("EMAIL")
if err != nil {
return "", err
}
Secondly, a single string type "" is compatible to a fixed array size of [4]string. It may make your life easier to use named return variables, so you don't need to pass a explicit "zero" value in the event of an error. So define your function signature like so:
func Email(in [4]string) (out [4]string, err error) {
myEmail, err := LoadEnvVariable("EMAIL")
if err != nil {
return // implicitly returns 'out' and 'err'
}
// ...
out = in
return // implicitly returns 'out' and nil 'err'
}

I have got panic: runtime error: invalid memory address or nil pointer dereference in Golang

panic: runtime error: invalid memory address or nil pointer dereference
when i add
statusCOD, err := c.route.FindStatusCODByOrigin(ctx, req.Origin)
if err != nil {
if err != sql.ErrNoRows {
ddlogger.Log(ctx, span, "Find status cod destination and cod origin", err)
errChan <- err
return
}
}
if statusCOD != nil {
IsCODOrigin = statusCOD.IsCODOrigin
IsCODDestination = statusCOD.IsCODDestination
}
in this func
for i, v := range detailShipments {
var dtPackage repo.PackageBaseModel
go func(idx int, vShipment repo.ShipmentDetailBaseModel, dataShipmentNew repo.ShipmentCODCreateModel) {
defer wg1.Done()
randomID := commonString.RandomWithCustomCharList(c.config.ShipmentCODIDRandom, c.config.ShipmentIDCharlist)
shipmentID := fmt.Sprintf("%s%s", prefix, randomID)
dataShipmentNew.ShipmentBaseModel.ShipmentID = strings.ToUpper(shipmentID)
dataShipmentNew.ShipmentDetailBaseModel = vShipment
var commodityName string
sCategory, err := c.shipmentCategoryRepo.FindOneShipmentCategoryByID(ctx, vShipment.ShipmentCategoryID.Int64)
if err != err && err != sql.ErrNoRows {
ddlogger.Log(ctx, span, "shipmentService-CreateShipmentCOD "+shipmentID, " Failed shipmentCategoryRepo.FindOneShipmentCategoryByID", err)
} else {
if sCategory != nil {
commodityName = sCategory.CommodityName.String
}
}
req := &repo.TariffRequestModelV2{
Origin: form.DetailSender.Origin,
Destination: vShipment.Destination,
Weight: vShipment.GrossWeight / 1000,
Commodity: commodityName,
GoodsValue: int64(vShipment.GoodValues.Float64),
IsInsurance: vShipment.IsInsurance.Bool,
Product: vShipment.ProductType,
Width: vShipment.DimensionWidth.Float64,
Height: vShipment.DimensionHeight.Float64,
Length: vShipment.DimensionLength.Float64,
IsCOD: true,
CODValue: vShipment.CODValue.Float64,
}
tariff, err := c.coreSystemRepo.CheckTariffV3(ctx, req)
if err != nil {
ddlogger.Log(ctx, span, "[shipmentService-CreateShipmentCOD [coreSystemRepo.CheckTariffV3]]", err)
errChan <- err
return
}
statusCOD, err := c.route.FindStatusCODByOrigin(ctx, req.Origin)
if err != nil {
if err != sql.ErrNoRows {
ddlogger.Log(ctx, span, "Find status cod destination and cod origin", err)
errChan <- err
return
}
}
if statusCOD != nil {
IsCODOrigin = statusCOD.IsCODOrigin
IsCODDestination = statusCOD.IsCODDestination
}
................
}
wg1.Wait()
close(shipmentChan)
close(errChan)
close(amountChan)
It looks like you just created the pointer, and it doesn't point to any memory spaces, you can check all the variables in your new code to find it. e.g. ctx

Purpose of a select statement with a single case in Go?

I'm reading the source code of MicroMDM, specifically the implementation of pollCommands (https://github.com/micromdm/micromdm/blob/5c70048c14592fc16c2107f1af1bf8e0d3a52e0b/platform/queue/queue.go#L223-L284):
func (db *Store) pollCommands(pubsub pubsub.PublishSubscriber) error {
commandEvents, err := pubsub.Subscribe(context.TODO(), "command-queue", command.CommandTopic)
if err != nil {
return errors.Wrapf(err,
"subscribing push to %s topic", command.CommandTopic)
}
go func() {
for {
select {
case event := <-commandEvents:
var ev command.Event
if err := command.UnmarshalEvent(event.Message, &ev); err != nil {
level.Info(db.logger).Log("msg", "unmarshal command event in queue", "err", err)
continue
}
cmd := new(DeviceCommand)
cmd.DeviceUDID = ev.DeviceUDID
byUDID, err := db.DeviceCommand(ev.DeviceUDID)
if err == nil && byUDID != nil {
cmd = byUDID
}
newPayload, err := plist.Marshal(ev.Payload)
if err != nil {
level.Info(db.logger).Log("msg", "marshal event payload", "err", err)
continue
}
newCmd := Command{
UUID: ev.Payload.CommandUUID,
Payload: newPayload,
}
cmd.Commands = append(cmd.Commands, newCmd)
if err := db.Save(cmd); err != nil {
level.Info(db.logger).Log("msg", "save command in db", "err", err)
continue
}
level.Info(db.logger).Log(
"msg", "queued event for device",
"device_udid", ev.DeviceUDID,
"command_uuid", ev.Payload.CommandUUID,
"request_type", ev.Payload.Command.RequestType,
)
cq := new(QueueCommandQueued)
cq.DeviceUDID = ev.DeviceUDID
cq.CommandUUID = ev.Payload.CommandUUID
msgBytes, err := MarshalQueuedCommand(cq)
if err != nil {
level.Info(db.logger).Log("msg", "marshal queued command", "err", err)
continue
}
if err := pubsub.Publish(context.TODO(), CommandQueuedTopic, msgBytes); err != nil {
level.Info(db.logger).Log("msg", "publish command to queued topic", "err", err)
}
}
}
}()
return nil
}
What puzzles me is why there is a select statement in the for loop with a single case. Would this code not be equivalent to
for {
event := <-commandEvents
var ev command.Event
...
}
It seems to me that the code could be simplified by doing away with the select statement?

Create a service as autorun

OS: windows/7/8/8.1/10 32bit
I have one question. How to create a service that would work like autorun?
Most applications install themselves in autorun through the registry or through C:\Users\Anon\AppData\Roaming\Microsoft\Windows\Start Menu\Programs\Startup. But there are those that are installing through the services, or rather as a service.
I have a code:
package main
import (
"fmt"
"strings"
"time"
"syscall"
"golang.org/x/sys/windows/svc"
"golang.org/x/sys/windows/svc/mgr"
"golang.org/x/sys/windows/svc/debug"
"log"
"os"
"path/filepath"
"golang.org/x/sys/windows/svc/eventlog"
)
var elog debug.Log
type myservice struct{}
func (m *myservice) Execute(args []string, r <-chan svc.ChangeRequest, changes chan<- svc.Status) (ssec bool, errno uint32) {
const cmdsAccepted = svc.AcceptStop | svc.AcceptShutdown | svc.AcceptPauseAndContinue
changes <- svc.Status{State: svc.StartPending}
fasttick := time.Tick(500 * time.Millisecond)
slowtick := time.Tick(2 * time.Second)
tick := fasttick
changes <- svc.Status{State: svc.Running, Accepts: cmdsAccepted}
elog.Info(1, strings.Join(args, "-"))
loop:
for {
select {
case <-tick:
beep()
elog.Info(1, "beep")
case c := <-r:
switch c.Cmd {
case svc.Interrogate:
changes <- c.CurrentStatus
// Testing deadlock from https://code.google.com/p/winsvc/issues/detail?id=4
time.Sleep(100 * time.Millisecond)
changes <- c.CurrentStatus
case svc.Stop, svc.Shutdown:
break loop
case svc.Pause:
changes <- svc.Status{State: svc.Paused, Accepts: cmdsAccepted}
tick = slowtick
case svc.Continue:
changes <- svc.Status{State: svc.Running, Accepts: cmdsAccepted}
tick = fasttick
default:
elog.Error(1, fmt.Sprintf("unexpected control request #%d", c))
}
}
}
changes <- svc.Status{State: svc.StopPending}
return
}
func runService(name string, isDebug bool) {
var err error
if isDebug {
elog = debug.New(name)
} else {
elog, err = eventlog.Open(name)
if err != nil {
return
}
}
defer elog.Close()
elog.Info(1, fmt.Sprintf("starting %s service", name))
run := svc.Run
if isDebug {
run = debug.Run
}
err = run(name, &myservice{})
if err != nil {
elog.Error(1, fmt.Sprintf("%s service failed: %v", name, err))
return
}
elog.Info(1, fmt.Sprintf("%s service stopped", name))
}
func startService(name string) error {
m, err := mgr.Connect()
if err != nil {
return err
}
defer m.Disconnect()
s, err := m.OpenService(name)
if err != nil {
return fmt.Errorf("could not access service: %v", err)
}
defer s.Close()
err = s.Start("is", "auto-started")
if err != nil {
return fmt.Errorf("could not start service: %v", err)
}
return nil
}
func controlService(name string, c svc.Cmd, to svc.State) error {
m, err := mgr.Connect()
if err != nil {
return err
}
defer m.Disconnect()
s, err := m.OpenService(name)
if err != nil {
return fmt.Errorf("could not access service: %v", err)
}
defer s.Close()
status, err := s.Control(c)
if err != nil {
return fmt.Errorf("could not send control=%d: %v", c, err)
}
timeout := time.Now().Add(10 * time.Second)
for status.State != to {
if timeout.Before(time.Now()) {
return fmt.Errorf("timeout waiting for service to go to state=%d", to)
}
time.Sleep(300 * time.Millisecond)
status, err = s.Query()
if err != nil {
return fmt.Errorf("could not retrieve service status: %v", err)
}
}
return nil
}
func main() {
const svcName = "Best Service"
isIntSess, err := svc.IsAnInteractiveSession()
if err != nil {
log.Fatalf("failed to determine if we are running in an interactive session: %v", err)
}
if !isIntSess {
runService(svcName, false)
return
}
/*err = controlService(svcName, svc.Stop, svc.Stopped)
err = removeService(svcName)*/
err = installService(svcName, "Best Service")
runService(svcName, true)
if err != nil {
log.Fatalf("failed to %s: %v", svcName, err)
}
return
}
func exePath() (string, error) {
prog := os.Args[0]
p, err := filepath.Abs(prog)
if err != nil {
return "", err
}
fi, err := os.Stat(p)
if err == nil {
if !fi.Mode().IsDir() {
return p, nil
}
err = fmt.Errorf("%s is directory", p)
}
if filepath.Ext(p) == "" {
p += ".exe"
fi, err := os.Stat(p)
if err == nil {
if !fi.Mode().IsDir() {
return p, nil
}
err = fmt.Errorf("%s is directory", p)
}
}
return "", err
}
func installService(name, desc string) error {
exepath, err := exePath()
if err != nil {
return err
}
m, err := mgr.Connect()
if err != nil {
return err
}
defer m.Disconnect()
s, err := m.OpenService(name)
if err == nil {
s.Close()
return fmt.Errorf("service %s already exists", name)
}
s, err = m.CreateService(name, exepath, mgr.Config{DisplayName: desc, Description: "BB service"}, "is", "auto-started")
if err != nil {
return err
}
defer s.Close()
err = eventlog.InstallAsEventCreate(name, eventlog.Error|eventlog.Warning|eventlog.Info)
if err != nil {
s.Delete()
return fmt.Errorf("SetupEventLogSource() failed: %s", err)
}
return nil
}
func removeService(name string) error {
m, err := mgr.Connect()
if err != nil {
return err
}
defer m.Disconnect()
s, err := m.OpenService(name)
if err != nil {
return fmt.Errorf("service %s is not installed", name)
}
defer s.Close()
err = s.Delete()
if err != nil {
return err
}
err = eventlog.Remove(name)
if err != nil {
return fmt.Errorf("RemoveEventLogSource() failed: %s", err)
}
return nil
}
var (
beepFunc = syscall.MustLoadDLL("user32.dll").MustFindProc("MessageBeep")
)
func beep() {
beepFunc.Call(0xffffffff)
}
Application is installed and every time I exit the application the service stops. I need that even after restarting the PC the service worked and the application started. How can I do it?
maybe it's not actual but during the creation of the service you should extend and pass Config
mgr.Config{DisplayName: desc, StartType: mgr.StartAutomatic}
like here:
s, err = m.CreateService(name, exepath, mgr.Config{DisplayName: desc, StartType: mgr.StartAutomatic}, "is", "auto-started")
if err != nil {
return err
}
Here you can find all necessary constants and functions:
https://github.com/golang/sys/blob/master/windows/svc/mgr/config.go
On Windows 10 go to Task Scheduler > Task Scheduler Library > Create Basic Task > Trigger: when the computer starts > Action: Start a program.
When running the task, use the following user account: SYSTEM.
There is a package in the standard library that does this:
https://godoc.org/golang.org/x/sys/windows/svc
example: https://github.com/golang/sys/tree/master/windows/svc/example

Resources