How to stop stacking on high load websocket app - go

Wrote an app for collecting transactions from equities of NYSE and sending it through websocket. I am using gorilla websocket and what I have:
import (
"log"
"net/http"
"github.com/gorilla/websocket"
)
func (tm *TicksModel) websocketConnection() {
go tm.writeMessages()
http.HandleFunc("/", tm.echo)
log.Println("http server started on :", 8126)
err := http.ListenAndServe("0.0.0.0:8126", nil)
if err != nil {
log.Fatal("8126 ERROR ListenAndServe: ", err)
}
}
func (tm *TicksModel) echo(w http.ResponseWriter, r *http.Request, Collectingchan chan string) {
var (
message []byte
ticker string
sendMessage string
err error
v *websocket.Conn
i int
whichS int
tickerObj *tickerModel
ok bool
)
tm.upgrader.CheckOrigin = func(r *http.Request) bool { return true }
c, err := tm.upgrader.Upgrade(w, r, nil)
if err != nil {
log.Print("upgrade:", err)
return
}
defer c.Close()
for {
message, err = nil, nil
_, message, err = c.ReadMessage()
log.Println(">>", string(message))
if err != nil {
log.Println("ERROR", err)
tm.removeConnection(c)
return
}
if string(message) == "Client" {
log.Println("NEW CLIENT", c.RemoteAddr())
}
if message[0] == 'a' {
ticker = string(message)[1:]
tm.mx.Lock()
if tickerObj, ok = tm.Tickers[ticker]; ok {
if len(tickerObj.WSclients) == 0 {
Collectingchan <- message
}
tickerObj.WSclients = append(tickerObj.WSclients, c)
log.Println("TICK LISTENERS ADD", ticker, tickerObj)
} else {
tickerObj = &tickerModel{}
Collectingchan <- message
tickerObj.WSclients = append(tickerObj.WSclients, c)
tm.Tickers[ticker] = tickerObj
log.Println("TICK LISTENERS ADD", ticker, tickerObj)
}
tm.mx.Unlock()
} else if message[0] == 'r' {
ticker = string(message)[1:]
tm.mx.Lock()
if tickerObj, ok = tm.Tickers[ticker]; ok {
for i, v = range tickerObj.WSclients {
if v == c {
tickerObj.WSclients = append(tickerObj.WSclients[:i], tickerObj.WSclients[i+1:]...)
if len(tickerObj.WSclients) == 0 {
sendMessage = "r" + ticker + "\n"
Collectingchan <- message
}
log.Println("TICK LISTENERS REMOVE", ticker, tickerObj)
}
}
}
tm.mx.Unlock()
}
}
}
func (tm *TicksModel) writeMessages() {
var (
toWSMessage wsSendModel
tickerObj *tickerModel
wsocket *websocket.Conn
err error
ok bool
)
for {
toWSMessage = <-tm.WSocketChan
if tickerObj, ok = tm.Tickers[toWSMessage.Ticker]; ok {
for _, wsocket = range tickerObj.WSclients {
err = wsocket.WriteJSON(toWSMessage)
if err != nil {
log.Println("write error ticks", wsocket.RemoteAddr(), ":", err)
tm.removeConnection(wsocket)
}
}
}
}
}
func (tm *TicksModel) removeConnection(c *websocket.Conn) {
var (
wsocket *websocket.Conn
i int
sendMessage string
)
tm.mx.Lock()
for ticker, tickerObj := range tm.Tickers {
for i, wsocket = range tickerObj.WSclients {
if wsocket == c {
tickerObj.WSclients = append(tickerObj.WSclients[:i], tickerObj.WSclients[i+1:]...)
if len(tickerObj.WSclients) == 0 {
tm.socketChanObject.getChan(tickerObj.SocketNumber, true) <- "r" + ticker
}
log.Println("REMOVE CONNECTION FROM TICK", ticker, tickerObj)
}
}
}
tm.mx.Unlock()
}
The map looks like this:
type TicksModel struct {
Tickers map[string]*tickerModel
upgrader websocket.Upgrader
socketChanObject socketModel
WSocketChan chan wsSendModel
mx sync.Mutex
}
type tickerModel struct {
WSclients []*websocket.Conn
SocketNumber int
}
After some time, new connections connects than disconnects after a minute. Application doesn't send new data, it stacks. What do you think?

Related

How to use "ListenForWebhook" in go-telegram-bot-api with gin?

if i use the following construction, everything is OK. But this construction uses tgbotapi.Update - it doesn't really work for me. I want to use tgbotapi.UpdatesChannel via function tgbotapi.ListenForWebhook().
import (
"github.com/gin-gonic/gin"
tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api/v5"
)
type TelegramBot struct {
API *tgbotapi.BotAPI
Upd tgbotapi.Update
UpdCh tgbotapi.UpdatesChannel
Log *logger.Logger
}
func (tlg *TelegramBot) InitTg(cfg *config.Config) {
bot, err := tgapi.NewBot(cfg.Telegram.Token)
if err != nil {
tlg.Log.Fatal(err)
}
tlg.API = bot
router := gin.Default()
if cfg.TelegramMode != "webhook" {
botUpdate := tgbotapi.NewUpdate(0)
botUpdate.Timeout = 60
tlg.UpdCh = tlg.API.GetUpdatesChan(botUpdate)
go tlg.Start()
} else {
router.POST(cfg.Telegram.Path+"*any", tlg.WebhookHandler)
}
go router.Run(cfg.Listen.BindIP + ":" + cfg.Listen.Port)
//TODO fix me :)
for {
}
}
func (tlg *TelegramBot) WebhookHandler(c *gin.Context) {
defer c.Request.Body.Close()
bytes, err := ioutil.ReadAll(c.Request.Body)
if err != nil {
tlg.Log.Println(err)
return
}
err = json.Unmarshal(bytes, &tlg.Upd)
if err != nil {
log.Println(err)
return
}
go tlg.Start()
c.JSON(http.StatusOK, gin.H{"data": "not you :)"})
}
func (tlg *TelegramBot) Start() {
//TODO: for webhook
if tlg.Upd.Message != nil {
chatID := tlg.Upd.Message.Chat.ID
tlg.analyzeUpdate(chatID)
} else if tlg.Upd.CallbackQuery != nil {
chatID := tlg.Upd.CallbackQuery.Message.Chat.ID
tlg.analyzeUpdate(chatID)
}
//TODO: for polling
for update := range tlg.UpdCh {
if update.Message != nil {
tlg.Upd = update
chatID := tlg.Upd.Message.Chat.ID
tlg.analyzeUpdate(chatID)
} else if update.CallbackQuery != nil {
tlg.Upd = update
chatID := update.CallbackQuery.Message.Chat.ID
tlg.analyzeUpdate(chatID)
}
}
}
How can I do this, if possible (to get the webhook data into the tgbotapi.UpdatesChannel as in polling) :
router.POST(cfg.Telegram.Path+"*any", func(c *gin.Context) {
tlg.UpdCh = bot.ListenForWebhook(cfg.Listen.BindIP + ":" + cfg.Listen.Port + cfg.Telegram.Path + bot.Token)
go tlg.Start()
c.JSON(http.StatusOK, gin.H{"data": "not you :)"})
})

How to properly control a listener in golang

I want to control a listener depending on the state of something, let's say my program will listen if the content of a file is 1 and don't listen if the content is 0.
The problem is that there's not way, once a listener is initialized, to tell him to refuse connection. There is only the 'accept' state ; I have to close the listener.
So I found a way to do it using the following code, but I feel that's not a good way to do because i'm using a global and usually it's not a good idea.
Is there a better way to do it ?
var healthStatus bool
func listen(l net.Listener, port int) {
var err error
l, err = net.Listen("tcp", ":"+strconv.Itoa(port))
if err != nil {
panic(err)
}
defer l.Close()
for {
if healthStatus == false {
_ = l.Close()
return
}
logrus.Debug("Going to listen !")
conn, err := l.Accept()
if err != nil {
panic(err)
}
go func(c net.Conn) {
_ = c.Close()
}(conn)
}
}
func main() {
healthStatus = true
var listener net.Listener
var isListening = false
for {
logrus.Debug("Performing checks...")
healthStatus = healthcheck()
if healthStatus {
if !isListening {
isListening = true
//Restart listener
go listen(listener, config.port)
}
}
if !healthStatus {
if isListening {
isListening = false
}
}
time.Sleep(time.Second * 10)
}
}
EDIT :
With channel
package main
import (
"net"
"strconv"
"time"
)
var listening = make(chan bool)
func listen(l net.Listener, port int) {
var err error
l, err = net.Listen("tcp", ":"+strconv.Itoa(port))
if err != nil {
panic(err)
}
defer l.Close()
for {
localstatus := <- listening
if localstatus == false {
_ = l.Close()
return
}
conn, _ := l.Accept()
go func(c net.Conn) {
// Shut down the connection.
_ = c.Close()
listening <- true
}(conn)
}
}
func main() {
healthStatus := true
var listener net.Listener
var isListening = false
for {
healthStatus = healthcheck()
if healthStatus {
if !isListening {
isListening = true
//Restart listener
go listen(listener, config.port)
}
listening <- true
}
if !healthStatus {
if isListening {
isListening = false
listening <- false
}
}
time.Sleep(time.Second * 10)
}
}
Close the listener when the health goes bad. Use a channel to signal the accept loop that it's a clean shutdown.
var listener net.Listener
var done chan struct{}
for {
if healthcheck() {
if listener == nil {
var err error
listener, err = net.Listen("tcp", ":"+strconv.Itoa(conig.port))
if err != nil {
panic(err)
}
done = make(chan struct{})
go accept(listener, done)
}
} else {
if listener != nil {
close(done)
listener.Close()
done = nil
listener = nil
}
}
time.Sleep(time.Second * 10)
}
The accept function is:
func accept(l net.Listener, done chan struct{}) {
for {
conn, err := l.Accept()
select {
case <-done:
return
default:
}
if err != nil {
panic(err)
}
go func(c net.Conn) {
_ = c.Close()
}(conn)
}
}

When my number of goroutine is large, the code will get an error

My code works fine with a small number of goroutine, but a large number of errors can occur with memory and pointers.I guess it is my lock used improperly.Can you help me see the tools I wrote? I have been debugging for a long time but I have no way to start.
I thought about adding a read lock to the clock function.But still will report an error.I also tried to use sync.Map. But it did not solve the problem
package main
import (
"fmt"
"io/ioutil"
"net/http"
"encoding/json"
"strings"
"sync"
"time"
"strconv"
)
type UrlArray struct {
Url string `json:"url"`
Method string `json:"method"`
Params string `json:"params"`
}
type MsgRequest struct {
Command string `json:"command"`
Concurrent int `json:"concurrent"`
IncrementalRatio float64 `json:"incrementalRatio"`
InitialRatio float64 `json:"initialRatio"`
Intervals int `json:"intervals"`
UrlArray []UrlArray `json:"urls"`
}
type GroupData struct {
TotalCount int `json:"totalCount"`
FailCount int `json:"failCount"`
SuccessRate float64 `json:"successRate"`
CostTime float64 `json:"costTime"`
AvergeTime float64 `json:"avergeTime"`
}
type MsgResponse struct {
TimeObject map[string]GroupData `json:"timeObject"`
TotalCount int `json:"totalCount"`
FailCount int `json:"failCount"`
SuccessRate float64 `json:"successRate"`
CostTime float64 `json:"costTime"`
AvergeTime float64 `json:"avergeTime"`
}
type LevelData struct {
result map[string]GroupData
sync.RWMutex
}
type ResultStatic struct {
result map[string]MsgResponse
}
var p = fmt.Println
var levelOutput LevelData
var output ResultStatic
var closeAllChan chan int
var isHandle = false
//write to map
func (r *LevelData) recordStatic(url string, status bool, useTimeSec float64) {
r.Lock()
defer r.Unlock()
val, ok := r.result[url]
if ok {
val.TotalCount += 1
val.CostTime += useTimeSec
if status == false {
val.FailCount += 1
}
val.SuccessRate = 1.00 - (float64(val.FailCount) * 1.00) / (float64(val.TotalCount) * 1.00)
val.AvergeTime = val.CostTime / float64(val.TotalCount)
levelOutput.result[url] = val
} else {
failcnt := 0
if status == false {
failcnt = 1
}
successRate := float64(1 - failcnt)
val := GroupData{TotalCount: 1, FailCount: failcnt, SuccessRate: successRate, CostTime: useTimeSec, AvergeTime: useTimeSec}
levelOutput.result[url] = val
}
}
//get request
func get(url string, params string) {
start := time.Now()
data := url + "?" + params
response, err:= http.Get(data)
end := time.Now()
costTime := float64((end.Sub(start)).Seconds())
if err != nil {
levelOutput.recordStatic(url, false, costTime)
} else {
levelOutput.recordStatic(url, true, costTime)
}
defer response.Body.Close()
}
// post request
func post(url string, params string) {
start := time.Now()
response, err := http.Post(url, "application/x-www-form-urlencoded", strings.NewReader(params))
end := time.Now()
costTime := float64((end.Sub(start)).Seconds())
if err != nil {
levelOutput.recordStatic(url, false, costTime)
} else {
levelOutput.recordStatic(url, true, costTime)
}
response.Body.Close()
}
//clock ticker
func (r *LevelData) clock(urlArrays []UrlArray, concurrent int, initialRatio float64, incrementalRatio float64, intervals int) {
start := int(float64(concurrent) * initialRatio)
increment := int(float64(concurrent) * incrementalRatio)
ticker := time.NewTicker(time.Duration(intervals) * time.Second)
endtotal := increment + concurrent
objectKey := 0
for range ticker.C {
select {
case <-closeAllChan:
p("死循环退出")
return
default:
if objectKey != 0 {
for _, v := range(urlArrays) {
val1, ok1 := output.result[v.Url]
p(val1, ok1)
if !ok1 {
val1 = MsgResponse{}
val1.TimeObject = make(map[string]GroupData)
}
val1.TimeObject[strconv.Itoa(objectKey)] = levelOutput.result[v.Url]
val1.TotalCount += (val1.TimeObject[strconv.Itoa(objectKey)]).TotalCount
val1.FailCount += (val1.TimeObject[strconv.Itoa(objectKey)]).FailCount
val1.CostTime += (val1.TimeObject[strconv.Itoa(objectKey)]).CostTime
val1.SuccessRate = 1.00 - (float64(val1.FailCount) * 1.00) / (float64(val1.TotalCount) * 1.00)
val1.AvergeTime = val1.CostTime / float64(val1.TotalCount)
output.result[v.Url] = val1
}
}
levelOutput = LevelData{result: map[string]GroupData{}}
if start >= endtotal {
p("start == concurrent")
close(closeAllChan)
time.Sleep(time.Second)
return
}
if objectKey == 0 {
for i := 0; i < start; i++ {
go work(urlArrays)
}
} else {
for i := 0; i < increment; i++ {
go work(urlArrays)
}
}
objectKey += 1
start += increment
}
}
}
//work
func work(urlArrays []UrlArray) {
i := 0
for {
select {
case <-closeAllChan:
return
default:
index := i % len(urlArrays)
i++
url := urlArrays[index].Url
method := urlArrays[index].Method
params := urlArrays[index].Params
if method == "Get" {
get(url, params)
} else {
post(url, params)
}
}
}
}
func setupResponse(w *http.ResponseWriter, req *http.Request) {
(*w).Header().Set("Access-Control-Allow-Origin", "*")
(*w).Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS")
(*w).Header().Set("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization")
}
func myfunc(w http.ResponseWriter, r *http.Request) {
setupResponse(&w, r)
if (r).Method == "OPTIONS" {
return
}
b, err := ioutil.ReadAll(r.Body)
defer r.Body.Close()
if err != nil {
http.Error(w, err.Error(), 501)
return
}
//json decode
var msgs MsgRequest
err = json.Unmarshal(b, &msgs)
if err != nil {
p(err.Error())
http.Error(w, err.Error(), 502)
return
}
var command = msgs.Command
if command == "stop" {
p("stop start")
if isHandle {
close(closeAllChan)
isHandle = false
} else {
p("no handle")
http.Error(w, "未有数据在执行", 504)
return
}
} else if command == "get" {
p("get start")
if isHandle {
p(output.result)
js, err := json.Marshal(output.result)
if err != nil {
http.Error(w, err.Error(), 503)
return
}
w.Header().Set("Content-Type", "application/json")
w.Write(js)
} else {
p("no handle")
http.Error(w, "未有数据在执行", 504)
return
}
} else {
isHandle = true
closeAllChan = make(chan int)
output = ResultStatic{result: map[string]MsgResponse{}}
urlArrays := msgs.UrlArray
concurrent := msgs.Concurrent
initialRatio := msgs.InitialRatio
incrementalRatio := msgs.IncrementalRatio
intervals := msgs.Intervals
levelOutput.clock(urlArrays, concurrent, initialRatio, incrementalRatio, intervals)
isHandle = false
js, err := json.Marshal(output.result)
if err != nil {
http.Error(w, err.Error(), 503)
return
}
p(output.result)
w.Header().Set("Content-Type", "application/json")
w.Write(js)
}
return
}
func main() {
http.HandleFunc("/handle", myfunc)
err := http.ListenAndServe(":8080", nil)
if err != nil {
panic(err)
}
}
// json example
// {
// "command": "handle/get/stop",
// "concurrent": 100,
// "initialRatio": 0.60,
// "incrementalRatio": 0.20,
// "intervals": 10,
// "urls": [
// {
// "url": "http://google.com",
// "method": "GET",
// "params": ""
// }
// ]
// }
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x40 pc=0x6782c4]
goroutine 64655 [running]:
main.post(0xc00001e0c0, 0x3c, 0xc000188510, 0x2f)
/root/toolv2.go:114 +0x1b4
main.work(0xc000391380, 0x1, 0x4)
/root/toolv2.go:193 +0xcf
created by main.(*LevelData).clock
/root/toolv2.go:161 +0x657
exit status 2
"invalid memory address or nil pointer dereference" does not indicate a data race. You are dereferencing a pointer that is nil:
func post(url string, params string) {
start := time.Now()
response, err := http.Post(url, "application/x-www-form-urlencoded", strings.NewReader(params))
end := time.Now()
costTime := float64((end.Sub(start)).Seconds())
if err != nil {
levelOutput.recordStatic(url, false, costTime)
} else {
levelOutput.recordStatic(url, true, costTime)
}
response.Body.Close()
}
You must not dereference response after an error occurs, because then response is nil.

Golang Gorilla Websocket stops receiving information at 120 seconds

I'm currently trying to connect to the CEX.IO bitcoin exchange's websocket, but have been having issues not only with CEX.IO but with others too. All of my connections drop around the 120-second mark which makes me think there is some TTL problem going on. The Process() goroutine in the main package ends up just hanging and waiting for data from the readLoop which just stops receiving data. I've included some read-only API keys in the code so you can test if you'd like.
package main
import (
"fmt"
"bitbucket.org/tradedefender/cryptocurrency/exchange-connector/cexio"
"github.com/shopspring/decimal"
"encoding/json"
"time"
)
type OrderBook struct {
Asks []Ask
Bids []Bid
}
type Ask struct {
Rate decimal.Decimal
Amount decimal.Decimal
}
type Bid struct {
Rate decimal.Decimal
Amount decimal.Decimal
}
func main() {
cexioConn := new(cexio.Connection)
err := cexioConn.Connect()
if err != nil {
fmt.Errorf("error: %s", err.Error())
}
err = cexioConn.Authenticate("TLwYkktLf7Im6nqSKt6UO1IrU", "9ImOJcR7Qj3LMIyPCzky0D7WE")
if err != nil {
fmt.Errorf("error: %s", err.Error())
}
readChannel := make(chan cexio.IntraAppMessage, 25)
go cexioConn.ReadLoop(readChannel)
processor := Processor{
WatchPairs: [][2]string{
[2]string{
"BTC", "USD",
},
},
conn: cexioConn,
}
go processor.Process(readChannel)
// LOL
for {
continue
}
}
type Processor struct {
WatchPairs [][2]string
conn *cexio.Connection
}
func (p *Processor) Process(ch <-chan cexio.IntraAppMessage) {
p.conn.SubscribeToOrderBook(p.WatchPairs[0])
pingTimer := time.Now().Unix()
for {
fmt.Printf("(%v)\n", time.Now().Unix())
if (time.Now().Unix() - pingTimer) >= 10 {
fmt.Println("sending ping")
p.conn.SendPing()
pingTimer = time.Now().Unix()
}
readMsg := <- ch
output, _ := json.Marshal(readMsg.SocketMessage)
fmt.Println(string(output))
if readMsg.SocketMessage.Event == "ping" {
fmt.Println("sending pong")
p.conn.SendPong()
pingTimer = time.Now().Unix()
}
}
}
Below is the connector to the cexio websocket. Here is a link to their API: https://cex.io/websocket-api
package cexio
import (
"github.com/gorilla/websocket"
//"github.com/shopspring/decimal"
"github.com/satori/go.uuid"
"encoding/hex"
"encoding/json"
"crypto/hmac"
"crypto/sha256"
"bytes"
"strconv"
"time"
"fmt"
)
const Url = "wss://ws.cex.io/ws/"
type Connection struct {
conn *websocket.Conn
}
type IntraAppMessage struct {
SocketMessage GenericMessage
ProgramMessage ProgramMessage
}
type GenericMessage struct {
Event string `json:"e"`
Data interface{} `json:"data"`
Auth AuthData `json:"auth,omitempty"`
Ok string `json:"ok,omitempty"`
Oid string `json:"oid,omitempty"`
Time int64 `json:"time,omitempty"`
}
type ProgramMessage struct {
Error string
}
type AuthData struct {
Key string `json:"key"`
Signature string `json:"signature"`
Timestamp int64 `json:"timestamp"`
}
type OrderBookSubscribeData struct {
Pair [2]string `json:"pair"`
Subscribe bool `json:"subscribe"`
Depth int `json:"depth"`
}
func (c *Connection) SendPong() error {
pongMsg := GenericMessage{
Event: "pong",
}
err := c.conn.WriteJSON(pongMsg)
if err != nil {
return nil
}
deadline := time.Now().Add(15*time.Second)
err = c.conn.WriteControl(websocket.PongMessage, nil, deadline)
if err != nil {
return err
}
return nil
}
func (c *Connection) SendPing() error {
pingMsg := GenericMessage{
Event: "get-balance",
Oid: uuid.NewV4().String(),
}
err := c.conn.WriteJSON(pingMsg)
if err != nil {
return err
}
deadline := time.Now().Add(15*time.Second)
err = c.conn.WriteControl(websocket.PingMessage, nil, deadline)
if err != nil {
return err
}
return nil
}
func (c *Connection) Connect() error {
dialer := *websocket.DefaultDialer
wsConn, _, err := dialer.Dial(Url, nil)
if err != nil {
return err
}
c.conn = wsConn
//c.conn.SetPingHandler(c.HandlePing)
for {
_, msgBytes, err := c.conn.ReadMessage()
if err != nil {
c.Disconnect()
return err
}
fmt.Println(string(msgBytes))
var m GenericMessage
err = json.Unmarshal(msgBytes, &m)
if err != nil {
c.Disconnect()
return err
}
if m.Event != "connected" {
c.Disconnect()
return err
} else {
break
}
}
return nil
}
func (c *Connection) Disconnect() error {
return c.conn.Close()
}
func (c *Connection) ReadLoop(ch chan<- IntraAppMessage) {
for {
fmt.Println("starting new read")
_, msgBytes, err := c.conn.ReadMessage()
if err != nil {
ch <- IntraAppMessage{
ProgramMessage: ProgramMessage{
Error: err.Error(),
},
}
continue
}
var m GenericMessage
err = json.Unmarshal(msgBytes, &m)
if err != nil {
ch <- IntraAppMessage{
ProgramMessage: ProgramMessage{
Error: err.Error(),
},
}
continue
}
ch <- IntraAppMessage{
SocketMessage: m,
}
}
}
func CreateSignature(timestamp int64, key, secret string) string {
secretBytes := []byte(secret)
h := hmac.New(sha256.New, secretBytes)
var buffer bytes.Buffer
buffer.WriteString(strconv.FormatInt(timestamp, 10))
buffer.WriteString(key)
h.Write(buffer.Bytes())
return hex.EncodeToString(h.Sum(nil))
}
func (c *Connection) Authenticate(key, secret string) error {
timestamp := time.Now().Unix()
signature := CreateSignature(timestamp, key, secret)
var authMsg GenericMessage
authMsg.Event = "auth"
authMsg.Auth = AuthData{
Key: key,
Signature: signature,
Timestamp: timestamp,
}
err := c.conn.WriteJSON(authMsg)
if err != nil {
return err
}
for {
_, msgBytes, err := c.conn.ReadMessage()
if err != nil {
c.Disconnect()
return err
}
fmt.Println(string(msgBytes))
var m GenericMessage
err = json.Unmarshal(msgBytes, &m)
if err != nil {
c.Disconnect()
return err
}
if m.Event != "auth" && m.Ok != "ok" {
c.Disconnect()
return err
} else {
break
}
}
return nil
}
func (c *Connection) SubscribeToOrderBook(pair [2]string) error {
sendMsg := GenericMessage{
Event: "order-book-subscribe",
Data: OrderBookSubscribeData{
Pair: pair,
Subscribe: true,
Depth: 0,
},
Oid: uuid.NewV4().String(),
}
err := c.conn.WriteJSON(sendMsg)
if err != nil {
return err
}
return nil
}
func (c *Connection) GetBalance() error {
sendMsg := GenericMessage{
Event: "get-balance",
Oid: uuid.NewV4().String(),
}
err := c.conn.WriteJSON(sendMsg)
if err != nil {
return err
}
return nil
}
Solution was to remove the
for {
continue
}
at the end of the main function

Change the sample by using goroutine?

I found a good web invalid links checker. But how to change it for a complete sample by using goroutine? The web page is: How To Crawl A Website In Golang. The codes dynamically add the url that will be searched to the pending slice. but I have some difficulties to use goroutine to do it.
package main
import (
"crypto/tls"
"errors"
"fmt"
"golang.org/x/net/html"
"io"
"net/http"
"net/url"
"strings"
"time"
)
var alreadyCrawledList []string
var pending []string
var brokenLinks []string
const localHostWithPort = "localhost:8080"
func IsLinkInPendingQueue(link string) bool {
for _, x := range pending {
if x == link {
return true
}
}
return false
}
func IsLinkAlreadyCrawled(link string) bool {
for _, x := range alreadyCrawledList {
if x == link {
return true
}
}
return false
}
func AddLinkInAlreadyCrawledList(link string) {
alreadyCrawledList = append(alreadyCrawledList, link)
}
func AddLinkInPendingQueue(link string) {
pending = append(pending, link)
}
func AddLinkInBrokenLinksQueue(link string) {
brokenLinks = append(brokenLinks, link)
}
func main() {
start := time.Now()
AddLinkInPendingQueue("http://" + localHostWithPort)
for count := 0; len(pending) > 0; count++ {
x := pending[0]
pending = pending[1:] //it dynamicly change the search url
if err := crawlPage(x); err != nil { //how to use it by using goroutine?
t.Errorf(err.Error())
}
}
duration := time.Since(start)
fmt.Println("________________")
count = 0
for _, l := range brokenLinks {
count++
fmt.Println(count, "Broken. | ", l)
}
fmt.Println("Time taken:", duration)
}
func crawlPage(uri string) error {
if IsLinkAlreadyCrawled(uri) {
fmt.Println("Already visited: Ignoring uri | ", uri)
return nil
}
transport := &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
}
client := http.Client{Transport: transport}
resp, err := client.Get(uri)
if err != nil {
fmt.Println("Got error: ", err.Error())
return err
}
if resp.StatusCode != http.StatusOK {
AddLinkInBrokenLinksQueue(uri)
return errors.New(fmt.Sprintf("Got %v instead of 200", resp.StatusCode))
}
defer resp.Body.Close()
links := ParseLinks(resp.Body)
links = ConvertLinksToLocalHost(links)
for _, link := range links {
if !InOurDomain(link) {
continue
}
absolute := FixURL(link, uri)
if !IsLinkAlreadyCrawled(absolute) && !IsLinkInPendingQueue(absolute) && absolute != uri { // Don't enqueue a page twice!
AddLinkInPendingQueue(absolute)
}
}
AddLinkInAlreadyCrawledList(uri)
return nil
}
func InOurDomain(link string) bool {
uri, err := url.Parse(link)
if err != nil {
return false
}
if uri.Scheme == "http" || uri.Scheme == "https" {
if uri.Host == localHostWithPort {
return true
}
return false
}
return true
}
func ConvertLinksToLocalHost(links []string) []string {
var convertedLinks []string
for _, link := range links {
convertedLinks = append(convertedLinks, strings.Replace(link, "leantricks.com", localHostWithPort, 1))
}
return convertedLinks
}
func FixURL(href, base string) string {
uri, err := url.Parse(href)
if err != nil {
return ""
}
baseURL, err := url.Parse(base)
if err != nil {
return ""
}
uri = baseURL.ResolveReference(uri)
return uri.String()
}
func ParseLinks(httpBody io.Reader) []string {
var links []string
page := html.NewTokenizer(httpBody)
for {
tokenType := page.Next()
if tokenType == html.ErrorToken {
return links
}
token := page.Token()
switch tokenType {
case html.StartTagToken:
fallthrough
case html.SelfClosingTagToken:
switch token.DataAtom.String() {
case "a":
fallthrough
case "link":
fallthrough
case "script":
for _, attr := range token.Attr {
if attr.Key == "href" {
links = append(links, attr.Val)
}
}
}
}
}
}
You could invoke the crawlPage() concurrently and handle alreadyCrawledList, pending and brokenLinks variables with mutexes (not so performant though). On the other hand, the code needs to be modified a lot to get it more performant.
I did a quick check with 4 links and seems to half the duration. I did a sample code with a simple http server and its here
Thanks,
- Anoop

Resources