gobwas-ws clean conn close - go

I use high-level structures to create a ws server.
func wsHandler(w http.ResponseWriter, r *http.Request) {
re := regexp.MustCompile(`^\/ws\/([0-9a-zA-Z]+)\/*`)
match := re.FindStringSubmatch(r.URL.Path)
if len(match) != 2 {
http.NotFound(w, r)
return
}
conn, _, _, err := ws.UpgradeHTTP(r, w)
if err != nil {
log.Printf("gobwas/ws: %s", err)
}
go func() {
defer conn.Close()
channels.Lock()
channels.Channels[match[1]] = append(channels.Channels[match[1]], &conn)
channels.Unlock()
for {
msg, op, err := wsutil.ReadClientData(conn)
if err != nil {
if err != io.EOF {
log.Printf("gobwas/ws/wsutil: %s", err)
}
break
}
if len(msg) > 0 {
go sendMessage(&conn, op, match[1], msg)
}
}
deleteConn(&conn, match[1])
}()
}
Then, when I implement a graceful disconnection, then with the usual conn.Close (), the connection simply breaks outside the protocol. I use the following code to close:
func onShutdown() {
channels.RLock()
for _, channel := range channels.Channels {
for _, conn := range channel {
if err := ws.WriteFrame(*conn, ws.NewCloseFrame(ws.NewCloseFrameBody(ws.StatusNormalClosure, "Server shutdown"))); err != nil {
fmt.Println(err)
}
(*conn).Close()
}
}
channels.RUnlock()
}
And even if I wait for a response from the client and then close the connection, then anyway, the client fixes the disconnection. (The client is the browser and JS WebSocket).
Sample JS code:
var socket = new WebSocket(`${location.protocol === "https:" ? "wss:" : "ws:"}//${window.location.host}/ws/valid`);
socket.onclose = function (event) {
if (event.wasClean) {
console.log('Clean');
} else {
console.log('disconnect');
}
console.log('Code: ' + event.code + ' reason: ' + event.reason);
};

Related

Getting More than 1 push notification using go lang cron

I am creating GO rest APIs. We are using AWS server.
I want to send push notification to mobile. Then I used
https://pkg.go.dev/github.com/robfig/cron (https://github.com/robfig/cron )
for creating cron job.
We are using 2 version of API, V1(old one) and V1.1(new one)
we have more than 1 environment dev,QA,preproduction,production
in our go lang code I created a cron job for sending push notification to mobile. and the function called inside main().
But we are getting 2 notification each interval.
I didn't understand why 2 one is getting at a time
I am attaching my code.
const title = "This Week’s Activity"
func NotifyWeeklyActivity(db *sql.DB, logger *zap.Logger) {
logger.Info("NotifyWeeklyActivity- start")
c := cron.New()
c.AddFunc("*/5 * * * *", func() {
lastweekTime := CurrentUTC().AddDate(0, 0, -7)
type PostCount struct {
HiveID uint64 `json:"hive_id"`
Post uint64 `json:"post"`
NotificationTopicArn null.String `json:"notification_topic_arn"`
}
var posts []PostCount
err := queries.Raw(`
select count(post_id) as post , post.hive_id as hive_id , hive.notification_topic_arn
from post
join hive on post.hive_id=hive.hive_id and hive.deleted_at is null
where post.deleted_at is null
and hive.deleted_at is null
and post.created_at between ? and ?
group by hive_id
having count(post_id)>3 ;
`, lastweekTime, CurrentUTC()).Bind(context.TODO(), db, &posts)
if err != nil {
logger.Error("error while fetching data ", zap.Error(err))
// return err
}
cfg, _ := config.GetImpart()
if cfg.Env != config.Local {
notification := NewImpartNotificationService(db, string(cfg.Env), cfg.Region, cfg.IOSNotificationARN, logger)
logger.Info("Notification- fetching complted")
for _, hive := range posts {
pushNotification := Alert{
Title: aws.String(title),
Body: aws.String(
fmt.Sprintf("Check out %d new posts in your Hive this week", hive.Post),
),
}
additionalData := NotificationData{
EventDatetime: CurrentUTC(),
HiveID: hive.HiveID,
}
Logger.Info("Notification",
zap.Any("pushNotification", pushNotification),
zap.Any("additionalData", additionalData),
zap.Any("hive", hive),
)
err = notification.NotifyTopic(context.Background(), additionalData, pushNotification, hive.NotificationTopicArn.String)
if err != nil {
logger.Error("error sending notification to topic", zap.Error(err))
}
}
}
})
c.Start()
}
func NewImpartNotificationService(db *sql.DB, stage, region, platformApplicationARN string, logger *zap.Logger) NotificationService {
//SNS not available in us-east-2
if strings.EqualFold(region, "us-east-2") {
region = "us-east-1"
}
sess, err := session.NewSession(&aws.Config{
Region: aws.String(region),
HTTPClient: NewHttpClient(10 * time.Second),
})
if err != nil {
logger.Fatal("unable to create aws session", zap.Error(err))
}
snsAppleNotificationService := &snsAppleNotificationService{
stage: stage,
Logger: logger,
SNS: sns.New(sess),
platformApplicationARN: platformApplicationARN,
db: db,
}
logger.Debug("created new NotificationService",
zap.String("stage", stage),
zap.String("arn", platformApplicationARN))
return snsAppleNotificationService
}
Why I am getting 2 notification at a time ?
How can I Solve this
func (ns *snsAppleNotificationService) NotifyTopic(ctx context.Context, data NotificationData, alert Alert, topicARN string) error {
var b []byte
var err error
if strings.TrimSpace(topicARN) == "" {
return nil
}
ns.Logger.Debug("sending push notification",
zap.Any("data", data),
zap.Any("msg", alert),
zap.String("platformEndpoint", topicARN),
zap.String("arn", ns.platformApplicationARN))
if b, err = json.Marshal(apnsMessageWrapper{
APNSData: APNSMessage{
Alert: alert,
Sound: aws.String("default"),
Data: data,
Badge: aws.Int(0),
},
}); err != nil {
return err
}
msg := awsSNSMessage{Default: *alert.Body}
msg.APNS = string(b)
msg.APNSSandbox = string(b)
if b, err = json.Marshal(msg); err != nil {
return err
}
input := &sns.PublishInput{
Message: aws.String(string(b)),
MessageStructure: aws.String("json"),
TopicArn: aws.String(topicARN),
}
// print()
_, err = ns.Publish(input)
if err != nil {
ns.Logger.Error("push-notification : After publish input",
zap.Any("topicARN", topicARN),
zap.Error(err),
)
}
return err
}
main fuction
func main() {
logger, err := zap.NewProduction()
if err != nil {
log.Fatal(err)
}
cfg, err := config.GetImpart()
if err != nil {
logger.Fatal("error parsing config", zap.Error(err))
}
if cfg == nil {
logger.Fatal("nil config")
return
}
if cfg.Debug {
gin.SetMode(gin.DebugMode)
//boil.DebugMode = true
boil.WithDebugWriter(context.TODO(), &config.ZapBoilWriter{Logger: logger})
logger, _ = zap.NewDevelopment()
if cfg.Env == config.Local || cfg.Env == config.Development {
logger.Debug("config startup", zap.Any("config", *cfg))
}
} else {
gin.SetMode(gin.ReleaseMode)
}
//init the sentry logger ,either debug
logger, err = impart.InitSentryLogger(cfg, logger, cfg.Debug)
if err != nil {
logger.Error("error on sentry init", zap.Any("error", err))
}
migrationDB, err := cfg.GetMigrationDBConnection()
if err != nil {
logger.Fatal("unable to connect to DB", zap.Error(err))
}
//Trap sigterm during migraitons
migrationsDoneChan := make(chan bool)
shutdownMigrationsChan := make(chan bool)
sigc := make(chan os.Signal, 1)
signal.Notify(sigc,
syscall.SIGINT,
syscall.SIGTERM,
syscall.SIGQUIT)
go func() {
select {
case <-sigc:
logger.Info("received a shutdown request during migrations, sending shutdown signal")
shutdownMigrationsChan <- true
case <-migrationsDoneChan:
logger.Info("migrations complete, no longer waiting for sig int")
return
}
}()
err = migrater.RunMigrationsUp(migrationDB, cfg.MigrationsPath, logger, shutdownMigrationsChan)
if err != nil {
logger.Fatal("error running migrations", zap.Error(err))
}
migrationsDoneChan <- true
if err := migrationDB.Close(); err != nil {
logger.Fatal("error closing migrations DB connection", zap.Error(err))
}
boil.SetLocation(time.UTC)
db, err := cfg.GetDBConnection()
if err != nil {
logger.Fatal("unable to connect to DB", zap.Error(err))
}
defer db.Close()
defer logger.Sync()
// if err := migrater.BootStrapAdminUsers(db, cfg.Env, logger); err != nil {
// logger.Fatal("unable to bootstrap user", zap.Error(err))
// }
// if err := migrater.BootStrapTopicHive(db, cfg.Env, logger); err != nil {
// logger.Fatal("unable to bootstrap user", zap.Error(err))
// }
// initiate global profanity detector
impart.InitProfanityDetector(db, logger)
impart.NotifyWeeklyActivity(db, logger)
services := setupServices(cfg, db, logger)
r := gin.New()
r.Use(CORS)
r.Use(secure.Secure(secure.Options{
//AllowedHosts: []string{"*"},
// AllowedHosts: []string{"localhost:3000", "ssl.example.com"},
//SSLRedirect: true,
// SSLHost: "*",
SSLProxyHeaders: map[string]string{"X-Forwarded-Proto": "https"},
STSIncludeSubdomains: true,
FrameDeny: true,
ContentTypeNosniff: true,
BrowserXssFilter: true,
ContentSecurityPolicy: "default-src 'self'",
}))
r.RedirectTrailingSlash = true
r.Use(ginzap.RecoveryWithZap(logger, true)) // panics don't stop server
r.Use(ginzap.Ginzap(logger, time.RFC3339, true)) // logs all requests
r.NoRoute(noRouteFunc)
r.GET("/ping", func(ctx *gin.Context) {
_, err := dbmodels.Pings(dbmodels.PingWhere.Ok.EQ(true)).One(ctx, db)
if err != nil {
ctx.AbortWithStatus(http.StatusInternalServerError)
}
ctx.String(http.StatusOK, "pong")
})
var v1Route string
var v2Route string
if cfg.Env == config.Production || cfg.Env == config.Local {
v1Route = "v1"
v2Route = "v1.1"
} else {
v1Route = fmt.Sprintf("%s/v1", cfg.Env)
v2Route = fmt.Sprintf("%s/v1.1", cfg.Env)
}
err = mailchimp.SetKey(impart.MailChimpApiKey)
if err != nil {
logger.Info("Error connecting Mailchimp", zap.Error(err),
zap.Any("MailchimpApikey", cfg.MailchimpApikey))
}
v1 := r.Group(v1Route)
setRouter(v1, services, logger, db)
v2 := r.Group(v2Route)
setRouter(v2, services, logger, db)
server := cfg.GetHttpServer()
server.Handler = r
logger.Info("Impart backend started.", zap.Int("port", cfg.Port), zap.String("env", string(cfg.Env)))
if err := graceful.Graceful(server.ListenAndServe, server.Shutdown); err != nil {
logger.Fatal("error serving", zap.Error(err))
}
logger.Info("done serving")
}
publish
// See also, https://docs.aws.amazon.com/goto/WebAPI/sns-2010-03-31/Publish
func (c *SNS) Publish(input *PublishInput) (*PublishOutput, error) {
req, out := c.PublishRequest(input)
return out, req.Send()
}

How I can close the connection by timeout if the client doesn't respond in 10 seconds?

I have code (I use https://github.com/fiorix/go-smpp):
// -----------------------------------------------
// handleConnection new clients.
// -----------------------------------------------
func (_srv *ServerSmpp) handleConnection(_cfg *ConfigSmpp, c *conn) {
defer c.Close()
if err := _srv.auth(_cfg, c); err != nil {
if err != io.EOF {
log.Printf("smpp_server: server auth failed: %s\n", err)
}
return
}
notify := make(chan error)
go func() {
for {
pb, err := c.Read()
if err != nil {
notify <- err
return
}
err = _srv.Handler(_srv.RemoteProvider, c, pb)
if err != nil {
fmt.Printf("%s\n", err)
notify <- err
return
}
}
}()
for {
select {
case err:= <-notify:
if io.EOF == err {
fmt.Printf("Smpp server (read): %s\n", err)
return
}
case <-time.After(time.Second * 10):
fmt.Printf("Client disconnected by timeout.\n")
return
}
}
}
Code for invoked handleConnection:
func (_srv *ServerSmpp) Serve(_cfg *ConfigSmpp) {
for {
client, err := _srv.NetListener.Accept()
if err != nil {
break
}
c := newConn(client)
go _srv.handleConnection(_cfg, c)
}
}
When this code work, the server disconnects all clients by timeout 10 sec, but how I can disconnect the client when it's doesn't work 10 sec?
Your client object seems to be a net.Conn,
choose a way to call client.SetReadDeadline() with the appropriate time.Time value before blocking on client.Read() :
c.client.SetDeadline( time.Now().Add(10 * time.Second )
pb, err := c.Read() { ...

Golang: Recursive function for reconnecting a TCP client... Bad idea?

I have this working TCP client code. When it fails to Write or Read on the TCP connection, it creates a new connection with the recursive function tcpReconnect().
Is this safe or will it fill up the RAM? It is possible that it may be trying to reconnect over several days (weekend or holidays). This is code is part of a Driver that monitors the state of an industrial Machine.
Maybe there is a better solution for this problem. I was not able to find one.
PS: I don't like polling
package main
import (
"fmt"
"net"
"time"
)
var pollTime = 1000 //ms
var host = "127.0.0.1"
var port = "11000"
func main() {
finished := make(chan bool)
go Driver()
<-finished
}
func tcpReconnect() net.Conn {
newConn, err := net.Dial("tcp", host+":"+port)
if err != nil {
fmt.Println("Failed to reconnect:", err.Error())
time.Sleep(time.Millisecond * time.Duration(2000))
newConn = tcpReconnect()
}
return newConn
}
func Driver() {
var conn net.Conn
conn, err := net.Dial("tcp", host+":"+port)
if err != nil {
fmt.Println("Failed to initialize Connection, trying to reconnect:", err.Error())
conn = tcpReconnect()
}
for {
_, err = conn.Write([]byte("11|5546|STATUS" + "\r\n"))
if err != nil {
println("Write to server failed:", err.Error())
println("Trying reset the connection...")
conn = tcpReconnect()
}
var replyBuffer = make([]byte, 256)
_, err = conn.Read(replyBuffer)
if err != nil {
println("Read from server failed:", err.Error())
println("Trying reset the connection...")
conn = tcpReConnect()
}
var reply string
for i, val := range replyBuffer {
if val == 13 { //13 is CR and marks the end of the message
reply = string(replyBuffer[:i])
break
}
}
fmt.Printf("reply from server=%s\n", reply)
time.Sleep(time.Millisecond * time.Duration(pollTime))
}
}
This is what I came up with. Credits go to #tkausl and #ThunderCat
func Driver() {
for {
conn, err := net.Dial("tcp", host+":"+port)
if err != nil {
fmt.Println("Failed to connect:", err.Error())
fmt.Println("Trying reset the connection...")
time.Sleep(time.Millisecond * time.Duration(2000))
} else {
for {
_, err = conn.Write([]byte("11|5546|STATUS" + "\r\n"))
if err != nil {
fmt.Println("Write to server failed:", err.Error())
fmt.Println("Trying reset the connection...")
break
}
var replyBuffer = make([]byte, 256)
_, err = conn.Read(replyBuffer)
if err != nil {
fmt.Println("Read from server failed:", err.Error())
fmt.Println("Trying reset the connection...")
break
}
var reply string
for i, val := range replyBuffer {
if val == 13 { //13 is CR and marks the end of the message
reply = string(replyBuffer[:i])
break
}
}
fmt.Printf("reply from server=_%s_\n", reply)
time.Sleep(time.Millisecond * time.Duration(pollTime))
}
}
}
}

How do I add an object to a channel from a goroutine that is receiving data from that channel?

Basically, I am trying to write a concurrent sitemap crawler using goroutines. One sitemap can contain links to multiple sitemaps which can contain links to other sitemaps etc.
Right now, this is my design:
worker:
- receives url from channel
- processesUrl(url)
processUrl:
for each link in lookup(url):
- if link is sitemap:
channel <- url
else:
print(url)
main:
- create 10 workers
- chanel <- root url
the problem is that the worker won't accept a new url from the channel until processUrl() is finished and processUrl won't finish until a worker accepts a new url from the channel if it is adding a url to the channel. What concurrent design can I use to add the url to a task queue without a channel and without busy-waiting or without waiting for channel <- url?
Here is the actual code if it helps:
func (c *SitemapCrawler) worker() {
for {
select {
case url := <-urlChan:
fmt.Println(url)
c.crawlSitemap(url)
}
}
}
func crawlUrl(url string) {
defer crawlWg.Done()
crawler := NewCrawler(url)
for i := 0; i < MaxCrawlRate*20; i++ {
go crawler.worker()
}
crawler.getSitemaps()
pretty.Println(crawler.sitemaps)
crawler.crawlSitemaps()
}
func (c SitemapCrawler) crawlSitemap(url string) {
c.limiter.Take()
resp, err := MakeRequest(url)
if err != nil || resp.StatusCode != 200 {
crawlWg.Done()
return
}
var resp_txt []byte
if strings.Contains(resp.Header.Get("Content-Type"), "html") {
crawlWg.Done()
return
} else if strings.Contains(url, ".gz") || resp.Header.Get("Content-Encoding") == "gzip" {
reader, err := gzip.NewReader(resp.Body)
if err != nil {
crawlWg.Done()
panic(err)
} else {
resp_txt, err = ioutil.ReadAll(reader)
if err != nil {
crawlWg.Done()
panic(err)
}
}
reader.Close()
} else {
resp_txt, err = ioutil.ReadAll(resp.Body)
if err != nil {
//panic(err)
crawlWg.Done()
return
}
}
io.Copy(ioutil.Discard, resp.Body)
resp.Body.Close()
d, err := libxml2.ParseString(string(resp_txt))
if err != nil {
crawlWg.Done()
return
}
results, err := d.Find("//*[contains(local-name(), 'loc')]")
if err != nil {
crawlWg.Done()
return
}
locs := results.NodeList()
printLock.Lock()
for i := 0; i < len(locs); i++ {
newUrl := locs[i].TextContent()
if strings.Contains(newUrl, ".xml") {
crawlWg.Add(1)
//go c.crawlSitemap(newUrl)
urlChan <- newUrl
} else {
fmt.Println(newUrl)
}
}
printLock.Unlock()
crawlWg.Done()
}
Write operations to channels are blocking when the channel is not buffered.
To create a buffered channel:
urlChan := make(chan string, len(allUrls))
When this channel is full however, write operations will block again.
Alternatively you could use a switch. When the write 'fails' it will immediately fall through to default
select {
case urlChan <- url:
fmt.Println("received message")
default:
fmt.Println("no activity")
}
To have a timeout on writing to the channel do the following
select {
case urlChan <- url:
fmt.Println("received message")
case <-time.After(5 * time.Second):
fmt.Println("timed out")
}
Or finally put the write event in a separate go channel
func write() {
urlChan <- url
}
go write()

Golang http: multiple response.WriteHeader calls

These days I was working on send message via websoket,using Beego framework.
but meet the wrong message http: multiple response.WriteHeader calls
Where is the problem?
Any tips would be great!
func (this *WsController) Get() {
fmt.Println("connected")
handler(this.Ctx.ResponseWriter, this.Ctx.Request, this);
conn, err := upgrader.Upgrade(this.Ctx.ResponseWriter, this.Ctx.Request, nil)
if _, ok := err.(websocket.HandshakeError); ok {
http.Error(this.Ctx.ResponseWriter, "Not a websocket handshake", 400)
return
} else if err != nil {
return
}
fmt.Println("connected")
connection := consumer.New(beego.AppConfig.String("LoggregatorAddress"), &tls.Config{InsecureSkipVerify: true}, nil)
fmt.Println("===== Tailing messages")
msgChan, err := connection.Tail(this.Ctx.Input.Param(":appGuid"), this.Ctx.Input.Param(":token"))
if err != nil {
fmt.Printf("===== Error tailing: %v\n", err)
} else {
for msg := range msgChan {
// if closeRealTimeLogFlag{
// consumer.Close()
// break
// }
if err = conn.WriteMessage(websocket.TextMessage, msg.Message); err != nil {
fmt.Println(err)
}
fmt.Printf("%v \n", msg)
}
}
}
because you write more than statusCode

Resources