I get an error in golang test and would like to know how to improve it - go

sta9_test.go:124: Cannot convert id to numeric, got = id
sta9_test.go:124: Cannot convert id to numericCannot convert id to numeric, got = id
sta9_test.go:145: It is a value we do not expect.
map[string]any{
- "UpdatedAt": string("2022-10-27T15:19:10Z"),
- "createdAenter code heret": string("2022-10-27T15:19:10Z"),
"description": string(""),}
func TestStation9(t *testing.T) {
dbPath := "./temp_test.db"
if err := os.Setenv("DB_PATH", dbPath); err != nil {
t.Error("dbPathのセットに失敗しました。", err)
return
}
t.Cleanup(func() {
if err := os.Remove(dbPath); err != nil {
t.Errorf("テスト用のDBファイルの削除に失敗しました: %v", err)
return
}
})
todoDB, err := db.NewDB(dbPath)
if err != nil {
t.Error("DBの作成に失敗しました。", err)
return
}
defer func(todoDB *sql.DB) {
err := todoDB.Close()
if err != nil {
t.Error("DBのクローズに失敗しました.", err)
}
}(todoDB)
r := router.NewRouter(todoDB)
srv := httptest.NewServer(r)
defer srv.Close()
testcases := map[string]struct {
Subject string
Description string
WantHTTPStatusCode int
}{
"Subject is empty": {
WantHTTPStatusCode: http.StatusBadRequest,
},
"Description is empty": {
Subject: "todo subject",
WantHTTPStatusCode: http.StatusOK,
},
"Subject and Description is not empty": {
Subject: "todo subject",
Description: "todo description",
WantHTTPStatusCode: http.StatusOK,
},
}
for name, tc := range testcases {
name := name
tc := tc
t.Run(name, func(t *testing.T) {
resp, err := http.Post(srv.URL+"/todos", "application/json",
bytes.NewBufferString(fmt.Sprintf(`{"subject":"%s","description":"%s"}`, tc.Subject, tc.Description)))
if err != nil {
t.Error("リクエストの送信に失敗しました。", err)
return
}
defer func() {
if err := resp.Body.Close(); err != nil {
t.Error("レスポンスのクローズに失敗しました。", err)
return
}
}()
if resp.StatusCode != tc.WantHTTPStatusCode {
t.Errorf("期待していない HTTP status code です, got = %d, want = %d", resp.StatusCode, tc.WantHTTPStatusCode)
return
}
if tc.WantHTTPStatusCode != http.StatusOK {
return
}
var m map[string]interface{}
if err := json.NewDecoder(resp.Body).Decode(&m); err != nil {
t.Error("レスポンスのデコードに失敗しました。", err)
return
}
v, ok := m["todo"]
if !ok {
t.Error("レスポンスの中にtodoがありません。")
return
}
got, ok := v.(map[string]interface{})
if !ok {
t.Error("レスポンスの中のtodoがmapではありません。")
return
}
want := map[string]interface{}{
"subject": tc.Subject,
"description": tc.Description,
}
now := time.Now().UTC()
fmt.Println(got)
fmt.Println(want)
diff := cmp.Diff(got, want, cmpopts.IgnoreMapEntries(func(k string, v interface{}) bool {
switch k {
case "id":
if vv, _ := v.(float64); vv == 0 {
t.Errorf("id を数値に変換できません, got = %s", k)
}
return true
case "created_at", "updated_at":
vv, ok := v.(string)
if !ok {
t.Errorf("日付が文字列に変換できません, got = %+v", k)
return true
}
if tt, err := time.Parse(time.RFC3339, vv); err != nil {
t.Errorf("日付が期待しているフォーマットではありません, got = %s", k)
} else if now.Before(tt) {
t.Errorf("日付が未来の日付になっています, got = %s", tt)
}
return true
}
return false
}))
if diff != "" {
t.Error("期待していない値です\n", diff)
}
})
}
}
type TODOHandler struct {
svc *service.TODOService
}
// NewTODOHandler returns TODOHandler based http.Handler.
func NewTODOHandler(svc *service.TODOService) *TODOHandler {
return &TODOHandler{
svc: svc,
}
}
func (h *TODOHandler)ServeHTTP(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPost{
var todo model.CreateTODORequest
if err := json.NewDecoder(r.Body).Decode(&todo); err != nil {
log.Fatal(err)
}
if todo.Subject == ""{
w.WriteHeader(400)
}else{
createdTodo, _ := h.Create(r.Context(),&todo)
e := json.NewEncoder(w)
if err:= e.Encode(createdTodo); err != nil{
log.Fatal("ListenAndServe:", err)
}
}
}
}
// Create handles the endpoint that creates the TODO.
func (h *TODOHandler) Create(ctx context.Context, req *model.CreateTODORequest) (*model.CreateTODOResponse, error) {
createtodo, err := h.svc.CreateTODO(ctx, req.Subject, req.Description)
if err != nil{
log.Fatal(err)
}
return &model.CreateTODOResponse{TODO: *createtodo}, nil
}
type TODOService struct {
db *sql.DB
}
// NewTODOService returns new TODOService.
func NewTODOService(db *sql.DB) *TODOService {
return &TODOService{
db: db,
}
}
// CreateTODO creates a TODO on DB.
func (s *TODOService) CreateTODO(ctx context.Context, subject, description string) (*model.TODO, error) {
const (
insert = `INSERT INTO todos(subject, description) VALUES(?, ?)`
confirm = `SELECT subject, description, created_at, updated_at FROM todos WHERE id = ?`
)
var todo model.TODO
if subject == ""{
msg := "subjectがありません"
return &todo,fmt.Errorf("err %s", msg)
}
stmt,err := s.db.PrepareContext(ctx,insert)
if err != nil{
log.Fatal("server/todo.go s.db.ExecContext(ctx,insert,subject,description)",err)
}
defer stmt.Close()
res, err := stmt.ExecContext(ctx, subject,description)
if err != nil {
return nil, err
}
insert_id,err := res.LastInsertId()
if err != nil{
log.Fatal("server/todo.go stmt.RowsAffected()",err)
}
err = s.db.QueryRowContext(ctx,confirm,insert_id).Scan(&todo.Subject,&todo.Description,&todo.CreatedAt,&todo.UpdatedAt)
if err != nil{
log.Fatal("server/todo.go s.db.QueryRowContext(ctx,confirm,insert_id).Scan(&todo.Subject,&todo.Description,&todo.CreatedAt,&todo.UpdatedAt)",err)
}
return &todo, err
}
I am building a TODO application with TDD and have a question. I get the error at the top. What kind of error is this and how can I improve it?

Related

Getting More than 1 push notification using go lang cron

I am creating GO rest APIs. We are using AWS server.
I want to send push notification to mobile. Then I used
https://pkg.go.dev/github.com/robfig/cron (https://github.com/robfig/cron )
for creating cron job.
We are using 2 version of API, V1(old one) and V1.1(new one)
we have more than 1 environment dev,QA,preproduction,production
in our go lang code I created a cron job for sending push notification to mobile. and the function called inside main().
But we are getting 2 notification each interval.
I didn't understand why 2 one is getting at a time
I am attaching my code.
const title = "This Week’s Activity"
func NotifyWeeklyActivity(db *sql.DB, logger *zap.Logger) {
logger.Info("NotifyWeeklyActivity- start")
c := cron.New()
c.AddFunc("*/5 * * * *", func() {
lastweekTime := CurrentUTC().AddDate(0, 0, -7)
type PostCount struct {
HiveID uint64 `json:"hive_id"`
Post uint64 `json:"post"`
NotificationTopicArn null.String `json:"notification_topic_arn"`
}
var posts []PostCount
err := queries.Raw(`
select count(post_id) as post , post.hive_id as hive_id , hive.notification_topic_arn
from post
join hive on post.hive_id=hive.hive_id and hive.deleted_at is null
where post.deleted_at is null
and hive.deleted_at is null
and post.created_at between ? and ?
group by hive_id
having count(post_id)>3 ;
`, lastweekTime, CurrentUTC()).Bind(context.TODO(), db, &posts)
if err != nil {
logger.Error("error while fetching data ", zap.Error(err))
// return err
}
cfg, _ := config.GetImpart()
if cfg.Env != config.Local {
notification := NewImpartNotificationService(db, string(cfg.Env), cfg.Region, cfg.IOSNotificationARN, logger)
logger.Info("Notification- fetching complted")
for _, hive := range posts {
pushNotification := Alert{
Title: aws.String(title),
Body: aws.String(
fmt.Sprintf("Check out %d new posts in your Hive this week", hive.Post),
),
}
additionalData := NotificationData{
EventDatetime: CurrentUTC(),
HiveID: hive.HiveID,
}
Logger.Info("Notification",
zap.Any("pushNotification", pushNotification),
zap.Any("additionalData", additionalData),
zap.Any("hive", hive),
)
err = notification.NotifyTopic(context.Background(), additionalData, pushNotification, hive.NotificationTopicArn.String)
if err != nil {
logger.Error("error sending notification to topic", zap.Error(err))
}
}
}
})
c.Start()
}
func NewImpartNotificationService(db *sql.DB, stage, region, platformApplicationARN string, logger *zap.Logger) NotificationService {
//SNS not available in us-east-2
if strings.EqualFold(region, "us-east-2") {
region = "us-east-1"
}
sess, err := session.NewSession(&aws.Config{
Region: aws.String(region),
HTTPClient: NewHttpClient(10 * time.Second),
})
if err != nil {
logger.Fatal("unable to create aws session", zap.Error(err))
}
snsAppleNotificationService := &snsAppleNotificationService{
stage: stage,
Logger: logger,
SNS: sns.New(sess),
platformApplicationARN: platformApplicationARN,
db: db,
}
logger.Debug("created new NotificationService",
zap.String("stage", stage),
zap.String("arn", platformApplicationARN))
return snsAppleNotificationService
}
Why I am getting 2 notification at a time ?
How can I Solve this
func (ns *snsAppleNotificationService) NotifyTopic(ctx context.Context, data NotificationData, alert Alert, topicARN string) error {
var b []byte
var err error
if strings.TrimSpace(topicARN) == "" {
return nil
}
ns.Logger.Debug("sending push notification",
zap.Any("data", data),
zap.Any("msg", alert),
zap.String("platformEndpoint", topicARN),
zap.String("arn", ns.platformApplicationARN))
if b, err = json.Marshal(apnsMessageWrapper{
APNSData: APNSMessage{
Alert: alert,
Sound: aws.String("default"),
Data: data,
Badge: aws.Int(0),
},
}); err != nil {
return err
}
msg := awsSNSMessage{Default: *alert.Body}
msg.APNS = string(b)
msg.APNSSandbox = string(b)
if b, err = json.Marshal(msg); err != nil {
return err
}
input := &sns.PublishInput{
Message: aws.String(string(b)),
MessageStructure: aws.String("json"),
TopicArn: aws.String(topicARN),
}
// print()
_, err = ns.Publish(input)
if err != nil {
ns.Logger.Error("push-notification : After publish input",
zap.Any("topicARN", topicARN),
zap.Error(err),
)
}
return err
}
main fuction
func main() {
logger, err := zap.NewProduction()
if err != nil {
log.Fatal(err)
}
cfg, err := config.GetImpart()
if err != nil {
logger.Fatal("error parsing config", zap.Error(err))
}
if cfg == nil {
logger.Fatal("nil config")
return
}
if cfg.Debug {
gin.SetMode(gin.DebugMode)
//boil.DebugMode = true
boil.WithDebugWriter(context.TODO(), &config.ZapBoilWriter{Logger: logger})
logger, _ = zap.NewDevelopment()
if cfg.Env == config.Local || cfg.Env == config.Development {
logger.Debug("config startup", zap.Any("config", *cfg))
}
} else {
gin.SetMode(gin.ReleaseMode)
}
//init the sentry logger ,either debug
logger, err = impart.InitSentryLogger(cfg, logger, cfg.Debug)
if err != nil {
logger.Error("error on sentry init", zap.Any("error", err))
}
migrationDB, err := cfg.GetMigrationDBConnection()
if err != nil {
logger.Fatal("unable to connect to DB", zap.Error(err))
}
//Trap sigterm during migraitons
migrationsDoneChan := make(chan bool)
shutdownMigrationsChan := make(chan bool)
sigc := make(chan os.Signal, 1)
signal.Notify(sigc,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGQUIT)
go func() {
select {
case <-sigc:
logger.Info("received a shutdown request during migrations, sending shutdown signal")
shutdownMigrationsChan <- true
case <-migrationsDoneChan:
logger.Info("migrations complete, no longer waiting for sig int")
return
}
}()
err = migrater.RunMigrationsUp(migrationDB, cfg.MigrationsPath, logger, shutdownMigrationsChan)
if err != nil {
logger.Fatal("error running migrations", zap.Error(err))
}
migrationsDoneChan <- true
if err := migrationDB.Close(); err != nil {
logger.Fatal("error closing migrations DB connection", zap.Error(err))
}
boil.SetLocation(time.UTC)
db, err := cfg.GetDBConnection()
if err != nil {
logger.Fatal("unable to connect to DB", zap.Error(err))
}
defer db.Close()
defer logger.Sync()
// if err := migrater.BootStrapAdminUsers(db, cfg.Env, logger); err != nil {
// logger.Fatal("unable to bootstrap user", zap.Error(err))
// }
// if err := migrater.BootStrapTopicHive(db, cfg.Env, logger); err != nil {
// logger.Fatal("unable to bootstrap user", zap.Error(err))
// }
// initiate global profanity detector
impart.InitProfanityDetector(db, logger)
impart.NotifyWeeklyActivity(db, logger)
services := setupServices(cfg, db, logger)
r := gin.New()
r.Use(CORS)
r.Use(secure.Secure(secure.Options{
//AllowedHosts: []string{"*"},
// AllowedHosts: []string{"localhost:3000", "ssl.example.com"},
//SSLRedirect: true,
// SSLHost: "*",
SSLProxyHeaders: map[string]string{"X-Forwarded-Proto": "https"},
STSIncludeSubdomains: true,
FrameDeny: true,
ContentTypeNosniff: true,
BrowserXssFilter: true,
ContentSecurityPolicy: "default-src 'self'",
}))
r.RedirectTrailingSlash = true
r.Use(ginzap.RecoveryWithZap(logger, true)) // panics don't stop server
r.Use(ginzap.Ginzap(logger, time.RFC3339, true)) // logs all requests
r.NoRoute(noRouteFunc)
r.GET("/ping", func(ctx *gin.Context) {
_, err := dbmodels.Pings(dbmodels.PingWhere.Ok.EQ(true)).One(ctx, db)
if err != nil {
ctx.AbortWithStatus(http.StatusInternalServerError)
}
ctx.String(http.StatusOK, "pong")
})
var v1Route string
var v2Route string
if cfg.Env == config.Production || cfg.Env == config.Local {
v1Route = "v1"
v2Route = "v1.1"
} else {
v1Route = fmt.Sprintf("%s/v1", cfg.Env)
v2Route = fmt.Sprintf("%s/v1.1", cfg.Env)
}
err = mailchimp.SetKey(impart.MailChimpApiKey)
if err != nil {
logger.Info("Error connecting Mailchimp", zap.Error(err),
zap.Any("MailchimpApikey", cfg.MailchimpApikey))
}
v1 := r.Group(v1Route)
setRouter(v1, services, logger, db)
v2 := r.Group(v2Route)
setRouter(v2, services, logger, db)
server := cfg.GetHttpServer()
server.Handler = r
logger.Info("Impart backend started.", zap.Int("port", cfg.Port), zap.String("env", string(cfg.Env)))
if err := graceful.Graceful(server.ListenAndServe, server.Shutdown); err != nil {
logger.Fatal("error serving", zap.Error(err))
}
logger.Info("done serving")
}
publish
// See also, https://docs.aws.amazon.com/goto/WebAPI/sns-2010-03-31/Publish
func (c *SNS) Publish(input *PublishInput) (*PublishOutput, error) {
req, out := c.PublishRequest(input)
return out, req.Send()
}

I am trying to test an REST Api client in Golang using mux and httptest

Here I am trying to write a test for a REST client, by writing the apiOutput into the http.ResponseWriter but I always receive {nil nil} as apiResponse.
Can someone help me in pointing out the error ?
func Test_Do(t *testing.T) {
mux := http.NewServeMux()
server := httptest.NewServer(mux)
t.Cleanup(server.Close)
config := NewClientConfig()
config.BaseURL = server.URL
client, err := NewClient(config)
if err != nil {
t.Fatal(err)
}
apiResponse := struct {
Id string `json:"id"`
Error error `json:"error"`
Data interface{} `json:"data"`
}{}
apiOutput := []byte(`{
"id":"1",
"error":nil,
"data":[{"IP": "1.2.2.3"}]}`)
mux.HandleFunc("/api/v1/hosts", func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.Write(apiOutput)
})
t.Run("Get Host Details", func(t *testing.T) {
req, err := client.MakeGetRequest(http.MethodGet, "/api/v1/hosts", nil)
if err != nil {
t.Fatal(err)
}
resp, err1 := client.Do(req, &apiResponse)
if err1 != nil {
t.Fatalf("failed with statusCode: %d, error: %s", resp.StatusCode, err1)
}
if apiResponse.Id != "1" || apiResponse.Error != nil {
t.Errorf("Client.Do() problem unmarshaling problem: got %#v", apiResponse)
}
fmt.Printf("%v", apiResponse)
})
}
func (c *Client) Do(req *http.Request, v interface{}) (*http.Response, error) {
resp, err := c.HTTPClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if errorResponse := c.CheckResponse(resp); errorResponse != nil {
return resp, errorResponse
}
if v != nil {
if w, ok := v.(io.Writer); ok {
io.Copy(w, resp.Body)
} else {
decErr := json.NewDecoder(resp.Body).Decode(v)
if decErr == io.EOF {
decErr = nil // ignore EOF errors caused by empty response body
}
if decErr != nil {
err = decErr
}
}
}
return resp, err
}
Output:
Test_Do/Get_Host_Details: clients_test.go:269: Client.Do() problem unmarshaling problem: got struct { Id string "json:\"id\""; Error error "json:\"error\""; Data interface {} "json:\"data\"" }{Id:"", Error:error(nil), Data:interface {}(nil)}
{ <nil> <nil>}

database/sql rows.scan hangs after 350K rows

I have a task to pull data from an Oracle Database and I am trying to pull huge data > 6MM records with 100 columns for processing.
Need to convert the data to a Map. I was successfully able to process them for 350K records in less than 35 seconds. After that the server hangs and does not proceed further.
Is there a way I can batch these based on the row size or batch them to free up my space.
func FetchUsingGenericResult(ctx context.Context, dsConnection *string, sqlStatement string) (*entity.GenericResultCollector, error) {
columnTypes := make(map[string]string)
var resultCollection entity.GenericResultCollector
db, err := sql.Open("godror", *dsConnection)
if err != nil {
return &resultCollection, errors.Wrap(err, "error connecting to Oracle")
}
log := logger.FromContext(ctx).Sugar()
log.Infof("start querying from Oracle at :%v", time.Now())
rows, err := db.Query(sqlStatement, godror.FetchRowCount(defaultFetchCount))
if err != nil {
return &resultCollection, errors.Wrap(err, "error querying")
}
objects, err := rows2Strings(ctx, rows)
log.Infof("total Rows converted are :%v by %v", len(*objects), time.Now())
resultCollection = entity.GenericResultCollector{
Columns: columnTypes,
Rows: objects,
}
return &resultCollection, nil
}
func rows2Strings(ctx context.Context, rows *sql.Rows) (*[]map[string]string, error) {
result := make(map[string]string)
resultsSlice := []map[string]string{}
fields, err := rows.Columns()
if err != nil {
return nil, err
}
log := logger.FromContext(ctx).Sugar()
waitGroup, ctx := errgroup.WithContext(ctx)
counter := 0
for rows.Next() {
counter++
if counter%defaultFetchCount == 0 {
log.Infof("finished converting %v rows by %v", counter, time.Now())
}
waitGroup.Go(func() error {
result, err = row2mapStr(rows, fields)
if err != nil {
return err
}
resultsSlice = append(resultsSlice, result)
return nil
})
if err := waitGroup.Wait(); err != nil {
return nil, err
}
}
return &resultsSlice, nil
}
func row2mapStr(rows *sql.Rows, fields []string) (resultsMap map[string]string, err error) {
result := make(map[string]string)
scanResultContainers := make([]interface{}, len(fields))
for i := 0; i < len(fields); i++ {
var scanResultContainer interface{}
scanResultContainers[i] = &scanResultContainer
}
if err := rows.Scan(scanResultContainers...); err != nil {
return nil, err
}
for ii, key := range fields {
rawValue := reflect.Indirect(reflect.ValueOf(scanResultContainers[ii]))
// if row is null then as empty string
if rawValue.Interface() == nil {
result[key] = ""
continue
}
if data, err := value2String(&rawValue); err == nil {
result[key] = data
} else {
return nil, err
}
}
return result, nil
}
func value2String(rawValue *reflect.Value) (str string, err error) {
aa := reflect.TypeOf((*rawValue).Interface())
vv := reflect.ValueOf((*rawValue).Interface())
switch aa.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
str = strconv.FormatInt(vv.Int(), 10)
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
str = strconv.FormatUint(vv.Uint(), 10)
case reflect.Float32, reflect.Float64:
str = strconv.FormatFloat(vv.Float(), 'f', -1, 64)
case reflect.String:
str = vv.String()
case reflect.Array, reflect.Slice:
switch aa.Elem().Kind() {
case reflect.Uint8:
data := rawValue.Interface().([]byte)
str = string(data)
if str == "\x00" {
str = "0"
}
default:
err = fmt.Errorf("Unsupported struct type %v", vv.Type().Name())
}
// time type
case reflect.Struct:
if aa.ConvertibleTo(timeType) {
str = vv.Convert(timeType).Interface().(time.Time).Format(time.RFC3339Nano)
} else {
err = fmt.Errorf("Unsupported struct type %v", vv.Type().Name())
}
case reflect.Bool:
str = strconv.FormatBool(vv.Bool())
case reflect.Complex128, reflect.Complex64:
str = fmt.Sprintf("%v", vv.Complex())
default:
err = fmt.Errorf("Unsupported struct type %v", vv.Type().Name())
}
return
}
Has anyone encountered a similar problem?
Modified the code as below:
func FetchUsingGenericResult(ctx context.Context, dsConnection *string, sqlStatement string) (*entity.GenericResultCollector, error) {
columnTypes := make(map[string]string)
var resultCollection entity.GenericResultCollector
db, err := sql.Open("godror", *dsConnection)
if err != nil {
return &resultCollection, errors.Wrap(err, "error connecting to Oracle")
}
log := logger.FromContext(ctx).Sugar()
log.Infof("start querying from Oracle at :%v", time.Now())
rows, err := db.Query(sqlStatement, godror.FetchRowCount(defaultFetchCount))
if err != nil {
return &resultCollection, errors.Wrap(err, "error querying")
}
objects, err := rows2Strings(ctx, rows)
log.Infof("total Rows converted are :%v by %v", len(*objects), time.Now())
resultCollection = entity.GenericResultCollector{
Columns: columnTypes,
Rows: objects,
}
return &resultCollection, nil
}
func rows2Strings(ctx context.Context, rows *sql.Rows) (*[]map[string]string, error) {
result := make(map[string]string)
resultsSlice := []map[string]string{}
fields, err := rows.Columns()
if err != nil {
return nil, err
}
log := logger.FromContext(ctx).Sugar()
counter := 0
for rows.Next() {
counter++
if counter%defaultFetchCount == 0 {
log.Infof("finished converting %v rows by %v", counter, time.Now())
}
result, err = row2mapStr(rows, fields)
if err != nil {
return nil, err
}
resultsSlice = append(resultsSlice, result)
}
return &resultsSlice, nil
}
func row2mapStr(rows *sql.Rows, fields []string) (resultsMap map[string]string, err error) {
result := make(map[string]string)
scanResultContainers := make([]interface{}, len(fields))
for i := 0; i < len(fields); i++ {
var scanResultContainer interface{}
scanResultContainers[i] = &scanResultContainer
}
if err := rows.Scan(scanResultContainers...); err != nil {
return nil, err
}
for ii, key := range fields {
rawValue := reflect.Indirect(reflect.ValueOf(scanResultContainers[ii]))
// if row is null then as empty string
if rawValue.Interface() == nil {
result[key] = ""
continue
}
if data, err := value2String(&rawValue); err == nil {
result[key] = data
} else {
return nil, err
}
}
return result, nil
}
func value2String(rawValue *reflect.Value) (str string, err error) {
aa := reflect.TypeOf((*rawValue).Interface())
vv := reflect.ValueOf((*rawValue).Interface())
switch aa.Kind() {
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
str = strconv.FormatInt(vv.Int(), 10)
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
str = strconv.FormatUint(vv.Uint(), 10)
case reflect.Float32, reflect.Float64:
str = strconv.FormatFloat(vv.Float(), 'f', -1, 64)
case reflect.String:
str = vv.String()
case reflect.Array, reflect.Slice:
switch aa.Elem().Kind() {
case reflect.Uint8:
data := rawValue.Interface().([]byte)
str = string(data)
if str == "\x00" {
str = "0"
}
default:
err = fmt.Errorf("Unsupported struct type %v", vv.Type().Name())
}
// time type
case reflect.Struct:
if aa.ConvertibleTo(timeType) {
str = vv.Convert(timeType).Interface().(time.Time).Format(time.RFC3339Nano)
} else {
err = fmt.Errorf("Unsupported struct type %v", vv.Type().Name())
}
case reflect.Bool:
str = strconv.FormatBool(vv.Bool())
case reflect.Complex128, reflect.Complex64:
str = fmt.Sprintf("%v", vv.Complex())
default:
err = fmt.Errorf("Unsupported struct type %v", vv.Type().Name())
}
return
}
As the server memory was limited the array could was stuck without proceeding further.
I have started process the data as it gets scanned and this solved my problem.

Create a service as autorun

OS: windows/7/8/8.1/10 32bit
I have one question. How to create a service that would work like autorun?
Most applications install themselves in autorun through the registry or through C:\Users\Anon\AppData\Roaming\Microsoft\Windows\Start Menu\Programs\Startup. But there are those that are installing through the services, or rather as a service.
I have a code:
package main
import (
"fmt"
"strings"
"time"
"syscall"
"golang.org/x/sys/windows/svc"
"golang.org/x/sys/windows/svc/mgr"
"golang.org/x/sys/windows/svc/debug"
"log"
"os"
"path/filepath"
"golang.org/x/sys/windows/svc/eventlog"
)
var elog debug.Log
type myservice struct{}
func (m *myservice) Execute(args []string, r <-chan svc.ChangeRequest, changes chan<- svc.Status) (ssec bool, errno uint32) {
const cmdsAccepted = svc.AcceptStop | svc.AcceptShutdown | svc.AcceptPauseAndContinue
changes <- svc.Status{State: svc.StartPending}
fasttick := time.Tick(500 * time.Millisecond)
slowtick := time.Tick(2 * time.Second)
tick := fasttick
changes <- svc.Status{State: svc.Running, Accepts: cmdsAccepted}
elog.Info(1, strings.Join(args, "-"))
loop:
for {
select {
case <-tick:
beep()
elog.Info(1, "beep")
case c := <-r:
switch c.Cmd {
case svc.Interrogate:
changes <- c.CurrentStatus
// Testing deadlock from https://code.google.com/p/winsvc/issues/detail?id=4
time.Sleep(100 * time.Millisecond)
changes <- c.CurrentStatus
case svc.Stop, svc.Shutdown:
break loop
case svc.Pause:
changes <- svc.Status{State: svc.Paused, Accepts: cmdsAccepted}
tick = slowtick
case svc.Continue:
changes <- svc.Status{State: svc.Running, Accepts: cmdsAccepted}
tick = fasttick
default:
elog.Error(1, fmt.Sprintf("unexpected control request #%d", c))
}
}
}
changes <- svc.Status{State: svc.StopPending}
return
}
func runService(name string, isDebug bool) {
var err error
if isDebug {
elog = debug.New(name)
} else {
elog, err = eventlog.Open(name)
if err != nil {
return
}
}
defer elog.Close()
elog.Info(1, fmt.Sprintf("starting %s service", name))
run := svc.Run
if isDebug {
run = debug.Run
}
err = run(name, &myservice{})
if err != nil {
elog.Error(1, fmt.Sprintf("%s service failed: %v", name, err))
return
}
elog.Info(1, fmt.Sprintf("%s service stopped", name))
}
func startService(name string) error {
m, err := mgr.Connect()
if err != nil {
return err
}
defer m.Disconnect()
s, err := m.OpenService(name)
if err != nil {
return fmt.Errorf("could not access service: %v", err)
}
defer s.Close()
err = s.Start("is", "auto-started")
if err != nil {
return fmt.Errorf("could not start service: %v", err)
}
return nil
}
func controlService(name string, c svc.Cmd, to svc.State) error {
m, err := mgr.Connect()
if err != nil {
return err
}
defer m.Disconnect()
s, err := m.OpenService(name)
if err != nil {
return fmt.Errorf("could not access service: %v", err)
}
defer s.Close()
status, err := s.Control(c)
if err != nil {
return fmt.Errorf("could not send control=%d: %v", c, err)
}
timeout := time.Now().Add(10 * time.Second)
for status.State != to {
if timeout.Before(time.Now()) {
return fmt.Errorf("timeout waiting for service to go to state=%d", to)
}
time.Sleep(300 * time.Millisecond)
status, err = s.Query()
if err != nil {
return fmt.Errorf("could not retrieve service status: %v", err)
}
}
return nil
}
func main() {
const svcName = "Best Service"
isIntSess, err := svc.IsAnInteractiveSession()
if err != nil {
log.Fatalf("failed to determine if we are running in an interactive session: %v", err)
}
if !isIntSess {
runService(svcName, false)
return
}
/*err = controlService(svcName, svc.Stop, svc.Stopped)
err = removeService(svcName)*/
err = installService(svcName, "Best Service")
runService(svcName, true)
if err != nil {
log.Fatalf("failed to %s: %v", svcName, err)
}
return
}
func exePath() (string, error) {
prog := os.Args[0]
p, err := filepath.Abs(prog)
if err != nil {
return "", err
}
fi, err := os.Stat(p)
if err == nil {
if !fi.Mode().IsDir() {
return p, nil
}
err = fmt.Errorf("%s is directory", p)
}
if filepath.Ext(p) == "" {
p += ".exe"
fi, err := os.Stat(p)
if err == nil {
if !fi.Mode().IsDir() {
return p, nil
}
err = fmt.Errorf("%s is directory", p)
}
}
return "", err
}
func installService(name, desc string) error {
exepath, err := exePath()
if err != nil {
return err
}
m, err := mgr.Connect()
if err != nil {
return err
}
defer m.Disconnect()
s, err := m.OpenService(name)
if err == nil {
s.Close()
return fmt.Errorf("service %s already exists", name)
}
s, err = m.CreateService(name, exepath, mgr.Config{DisplayName: desc, Description: "BB service"}, "is", "auto-started")
if err != nil {
return err
}
defer s.Close()
err = eventlog.InstallAsEventCreate(name, eventlog.Error|eventlog.Warning|eventlog.Info)
if err != nil {
s.Delete()
return fmt.Errorf("SetupEventLogSource() failed: %s", err)
}
return nil
}
func removeService(name string) error {
m, err := mgr.Connect()
if err != nil {
return err
}
defer m.Disconnect()
s, err := m.OpenService(name)
if err != nil {
return fmt.Errorf("service %s is not installed", name)
}
defer s.Close()
err = s.Delete()
if err != nil {
return err
}
err = eventlog.Remove(name)
if err != nil {
return fmt.Errorf("RemoveEventLogSource() failed: %s", err)
}
return nil
}
var (
beepFunc = syscall.MustLoadDLL("user32.dll").MustFindProc("MessageBeep")
)
func beep() {
beepFunc.Call(0xffffffff)
}
Application is installed and every time I exit the application the service stops. I need that even after restarting the PC the service worked and the application started. How can I do it?
maybe it's not actual but during the creation of the service you should extend and pass Config
mgr.Config{DisplayName: desc, StartType: mgr.StartAutomatic}
like here:
s, err = m.CreateService(name, exepath, mgr.Config{DisplayName: desc, StartType: mgr.StartAutomatic}, "is", "auto-started")
if err != nil {
return err
}
Here you can find all necessary constants and functions:
https://github.com/golang/sys/blob/master/windows/svc/mgr/config.go
On Windows 10 go to Task Scheduler > Task Scheduler Library > Create Basic Task > Trigger: when the computer starts > Action: Start a program.
When running the task, use the following user account: SYSTEM.
There is a package in the standard library that does this:
https://godoc.org/golang.org/x/sys/windows/svc
example: https://github.com/golang/sys/tree/master/windows/svc/example

Golang Gorilla Websocket stops receiving information at 120 seconds

I'm currently trying to connect to the CEX.IO bitcoin exchange's websocket, but have been having issues not only with CEX.IO but with others too. All of my connections drop around the 120-second mark which makes me think there is some TTL problem going on. The Process() goroutine in the main package ends up just hanging and waiting for data from the readLoop which just stops receiving data. I've included some read-only API keys in the code so you can test if you'd like.
package main
import (
"fmt"
"bitbucket.org/tradedefender/cryptocurrency/exchange-connector/cexio"
"github.com/shopspring/decimal"
"encoding/json"
"time"
)
type OrderBook struct {
Asks []Ask
Bids []Bid
}
type Ask struct {
Rate decimal.Decimal
Amount decimal.Decimal
}
type Bid struct {
Rate decimal.Decimal
Amount decimal.Decimal
}
func main() {
cexioConn := new(cexio.Connection)
err := cexioConn.Connect()
if err != nil {
fmt.Errorf("error: %s", err.Error())
}
err = cexioConn.Authenticate("TLwYkktLf7Im6nqSKt6UO1IrU", "9ImOJcR7Qj3LMIyPCzky0D7WE")
if err != nil {
fmt.Errorf("error: %s", err.Error())
}
readChannel := make(chan cexio.IntraAppMessage, 25)
go cexioConn.ReadLoop(readChannel)
processor := Processor{
WatchPairs: [][2]string{
[2]string{
"BTC", "USD",
},
},
conn: cexioConn,
}
go processor.Process(readChannel)
// LOL
for {
continue
}
}
type Processor struct {
WatchPairs [][2]string
conn *cexio.Connection
}
func (p *Processor) Process(ch <-chan cexio.IntraAppMessage) {
p.conn.SubscribeToOrderBook(p.WatchPairs[0])
pingTimer := time.Now().Unix()
for {
fmt.Printf("(%v)\n", time.Now().Unix())
if (time.Now().Unix() - pingTimer) >= 10 {
fmt.Println("sending ping")
p.conn.SendPing()
pingTimer = time.Now().Unix()
}
readMsg := <- ch
output, _ := json.Marshal(readMsg.SocketMessage)
fmt.Println(string(output))
if readMsg.SocketMessage.Event == "ping" {
fmt.Println("sending pong")
p.conn.SendPong()
pingTimer = time.Now().Unix()
}
}
}
Below is the connector to the cexio websocket. Here is a link to their API: https://cex.io/websocket-api
package cexio
import (
"github.com/gorilla/websocket"
//"github.com/shopspring/decimal"
"github.com/satori/go.uuid"
"encoding/hex"
"encoding/json"
"crypto/hmac"
"crypto/sha256"
"bytes"
"strconv"
"time"
"fmt"
)
const Url = "wss://ws.cex.io/ws/"
type Connection struct {
conn *websocket.Conn
}
type IntraAppMessage struct {
SocketMessage GenericMessage
ProgramMessage ProgramMessage
}
type GenericMessage struct {
Event string `json:"e"`
Data interface{} `json:"data"`
Auth AuthData `json:"auth,omitempty"`
Ok string `json:"ok,omitempty"`
Oid string `json:"oid,omitempty"`
Time int64 `json:"time,omitempty"`
}
type ProgramMessage struct {
Error string
}
type AuthData struct {
Key string `json:"key"`
Signature string `json:"signature"`
Timestamp int64 `json:"timestamp"`
}
type OrderBookSubscribeData struct {
Pair [2]string `json:"pair"`
Subscribe bool `json:"subscribe"`
Depth int `json:"depth"`
}
func (c *Connection) SendPong() error {
pongMsg := GenericMessage{
Event: "pong",
}
err := c.conn.WriteJSON(pongMsg)
if err != nil {
return nil
}
deadline := time.Now().Add(15*time.Second)
err = c.conn.WriteControl(websocket.PongMessage, nil, deadline)
if err != nil {
return err
}
return nil
}
func (c *Connection) SendPing() error {
pingMsg := GenericMessage{
Event: "get-balance",
Oid: uuid.NewV4().String(),
}
err := c.conn.WriteJSON(pingMsg)
if err != nil {
return err
}
deadline := time.Now().Add(15*time.Second)
err = c.conn.WriteControl(websocket.PingMessage, nil, deadline)
if err != nil {
return err
}
return nil
}
func (c *Connection) Connect() error {
dialer := *websocket.DefaultDialer
wsConn, _, err := dialer.Dial(Url, nil)
if err != nil {
return err
}
c.conn = wsConn
//c.conn.SetPingHandler(c.HandlePing)
for {
_, msgBytes, err := c.conn.ReadMessage()
if err != nil {
c.Disconnect()
return err
}
fmt.Println(string(msgBytes))
var m GenericMessage
err = json.Unmarshal(msgBytes, &m)
if err != nil {
c.Disconnect()
return err
}
if m.Event != "connected" {
c.Disconnect()
return err
} else {
break
}
}
return nil
}
func (c *Connection) Disconnect() error {
return c.conn.Close()
}
func (c *Connection) ReadLoop(ch chan<- IntraAppMessage) {
for {
fmt.Println("starting new read")
_, msgBytes, err := c.conn.ReadMessage()
if err != nil {
ch <- IntraAppMessage{
ProgramMessage: ProgramMessage{
Error: err.Error(),
},
}
continue
}
var m GenericMessage
err = json.Unmarshal(msgBytes, &m)
if err != nil {
ch <- IntraAppMessage{
ProgramMessage: ProgramMessage{
Error: err.Error(),
},
}
continue
}
ch <- IntraAppMessage{
SocketMessage: m,
}
}
}
func CreateSignature(timestamp int64, key, secret string) string {
secretBytes := []byte(secret)
h := hmac.New(sha256.New, secretBytes)
var buffer bytes.Buffer
buffer.WriteString(strconv.FormatInt(timestamp, 10))
buffer.WriteString(key)
h.Write(buffer.Bytes())
return hex.EncodeToString(h.Sum(nil))
}
func (c *Connection) Authenticate(key, secret string) error {
timestamp := time.Now().Unix()
signature := CreateSignature(timestamp, key, secret)
var authMsg GenericMessage
authMsg.Event = "auth"
authMsg.Auth = AuthData{
Key: key,
Signature: signature,
Timestamp: timestamp,
}
err := c.conn.WriteJSON(authMsg)
if err != nil {
return err
}
for {
_, msgBytes, err := c.conn.ReadMessage()
if err != nil {
c.Disconnect()
return err
}
fmt.Println(string(msgBytes))
var m GenericMessage
err = json.Unmarshal(msgBytes, &m)
if err != nil {
c.Disconnect()
return err
}
if m.Event != "auth" && m.Ok != "ok" {
c.Disconnect()
return err
} else {
break
}
}
return nil
}
func (c *Connection) SubscribeToOrderBook(pair [2]string) error {
sendMsg := GenericMessage{
Event: "order-book-subscribe",
Data: OrderBookSubscribeData{
Pair: pair,
Subscribe: true,
Depth: 0,
},
Oid: uuid.NewV4().String(),
}
err := c.conn.WriteJSON(sendMsg)
if err != nil {
return err
}
return nil
}
func (c *Connection) GetBalance() error {
sendMsg := GenericMessage{
Event: "get-balance",
Oid: uuid.NewV4().String(),
}
err := c.conn.WriteJSON(sendMsg)
if err != nil {
return err
}
return nil
}
Solution was to remove the
for {
continue
}
at the end of the main function

Resources