High CPU usage when reading /dev/gpiomem - go

Is there something in my Golang code that would be causing high cpu usage?
The code should read a button on the GPIO pin on a Raspberry Pi 4 (4GB) and when pressed send an REST post message to another program.
I tried using more sleep and no sleep, ive tried changing the code to use a falling edge detection method on the pin. Nothing has changed the high CPU usage.
import (
"fmt"
"github.com/stianeikeland/go-rpio"
"os"
"io/ioutil"
"net/http"
"strings"
"time"
)
func main() {
fmt.Println("Starting Stream Control 0.1")
pinOpenError := rpio.Open()
if pinOpenError != nil {
fmt.Println("Pin Open Error: ", pinOpenError.Error())
os.Exit(1)
}
pin := rpio.Pin(18)
pin.Input()
pin.PullUp()
var PID string
start := time.Now()
StreamRunning := false;
fmt.Println("Reading and Controlling")
for {
res := pin.Read()
if res == 0 {
if StreamRunning == false && time.Since(start).Seconds() > 10{
var StartStreamError error
PID, StartStreamError = StartStream()
if StartStreamError != nil {
fmt.Println("Start Stream Error: ", StartStreamError.Error())
}
fmt.Println("PID = " + PID)
start = time.Now()
StreamRunning = true;
} else {
if time.Since(start).Seconds() > 10 {
StopStream(PID)
StreamRunning = false;
start = time.Now()
}
}
}
time.Sleep(100)
}
}
func postHTTP(url string, requestString string) ([]byte, error){
var blankBody []byte
payload := strings.NewReader(requestString)
client := &http.Client{}
req, clientError := http.NewRequest("POST", url, payload)
if clientError != nil {
return blankBody, clientError
}
req.Header.Add("Content-Type", "application/json")
res, requestError := client.Do(req)
if requestError != nil {
return blankBody, requestError
}
defer res.Body.Close()
returnBody, returnError := ioutil.ReadAll(res.Body)
return returnBody, returnError
}
func StartStream()(string, error) {
fmt.Println("Start Stream")
response, StreamOnError := postHTTP("http://localhost:3000/startStream","{\"devicePath\":\"/dev/video0\",\"streamName\":\"LumiPi-003\"}")
if StreamOnError != nil {
fmt.Println("Stream On Error: ", StreamOnError.Error())
return "0", StreamOnError
}
splitstrings := strings.Split(string(response),",")
for i := range splitstrings {
if strings.Contains(splitstrings[i], "\"pid\":") {
linesplit := strings.Split(splitstrings[i],":")
return linesplit[1], nil
}
}
return "0", nil
}
func StopStream(PID string)(error) {
fmt.Println("Stop Stream")
command := "{\"pid\":" + PID + "}"
_, StreamOnError := postHTTP("http://localhost:3000/stopStream",command)
if StreamOnError != nil {
fmt.Println("Stream Off Error: ", StreamOnError.Error())
return StreamOnError
}
return nil
}
Picture below shows the affect time.sleep has on the program.

The time.Sleep() function was causing the PI to crash when rpio.Open() was still open.
Opening, reading the pins and closing allowed time.Sleep() to be used with a much larger delay which in turn dropped the cpu usage because its not constantly looping.
import (
"fmt"
"github.com/stianeikeland/go-rpio"
"os"
"io/ioutil"
"net/http"
"strings"
"time"
)
func main() {
fmt.Println("Starting Stream Control 0.1")
var PID string
start := time.Now()
StreamRunning := false;
fmt.Println("Reading and Controlling")
for {
pinOpenError := rpio.Open()
if pinOpenError != nil {
fmt.Println("Pin Open Error: ", pinOpenError.Error())
os.Exit(1)
}
pin := rpio.Pin(18)
pin.Input()
pin.PullUp()
res := pin.Read()
rpio.Close()
if res == 0 {
if StreamRunning == false && time.Since(start).Seconds() > 10{
var StartStreamError error
PID, StartStreamError = StartStream()
if StartStreamError != nil {
fmt.Println("Start Stream Error: ", StartStreamError.Error())
}
fmt.Println("PID = " + PID)
start = time.Now()
StreamRunning = true;
} else {
if time.Since(start).Seconds() > 10 {
StopStream(PID)
StreamRunning = false;
start = time.Now()
}
}
}
time.Sleep(100*time.Millisecond)
}
}
func postHTTP(url string, requestString string) ([]byte, error){
var blankBody []byte
payload := strings.NewReader(requestString)
client := &http.Client{}
req, clientError := http.NewRequest("POST", url, payload)
if clientError != nil {
return blankBody, clientError
}
req.Header.Add("Content-Type", "application/json")
res, requestError := client.Do(req)
if requestError != nil {
return blankBody, requestError
}
defer res.Body.Close()
returnBody, returnError := ioutil.ReadAll(res.Body)
return returnBody, returnError
}
func StartStream()(string, error) {
fmt.Println("Start Stream")
response, StreamOnError := postHTTP("http://localhost:3000/startStream","{\"devicePath\":\"/dev/video0\",\"streamName\":\"LumiPi-003\"}")
if StreamOnError != nil {
fmt.Println("Stream On Error: ", StreamOnError.Error())
return "0", StreamOnError
}
splitstrings := strings.Split(string(response),",")
for i := range splitstrings {
if strings.Contains(splitstrings[i], "\"pid\":") {
linesplit := strings.Split(splitstrings[i],":")
return linesplit[1], nil
}
}
return "0", nil
}
func StopStream(PID string)(error) {
fmt.Println("Stop Stream")
command := "{\"pid\":" + PID + "}"
_, StreamOnError := postHTTP("http://localhost:3000/stopStream",command)
if StreamOnError != nil {
fmt.Println("Stream Off Error: ", StreamOnError.Error())
return StreamOnError
}
return nil
}

Related

How to use res.body.Close in "for" loop as I need to make multiple calls to the same api?

I want to use res.body.Close in for loop but before being able to close the tcp connection it is being recalled.What is the optimal way to use it?
This is my code snippet for reference
//function call
serviceList := RecursiveResourceMapApiCall(resourceItem)
.
.
.
.
.
func RecursiveResourceMapApiCall(resourceItem types.ResourceItem) (serviceList []types.Services) {
fmt.Println("RecursiveResourceMapApiCall")
//var err error
if len(resourceItem.ServiceRefs) == 0 {
fmt.Println("service at this point : ", resourceItem.ID)
serviceDetail := types.Services{
Name: resourceItem.ID,
Version: resourceItem.Version,
}
serviceList = append(serviceList, serviceDetail)
//should i close the response body here? I have to change the return type of function then
return serviceList
}
var resourceMapResponse types.ResourceMapResponse
for i := 0; i < len(resourceItem.ServiceRefs); i++ {
//FetchDetailsFromResourceMapService is the api call
response, err := allocatorStore.FetchDetailsFromResourceMapService(resourceItem.ServiceRefs[i])
if err == nil && response.StatusCode == 200 {
data, _ := ioutil.ReadAll(response.Body)
err := json.Unmarshal(data, &resourceMapResponse)
if err != nil {
fmt.Println("error in unmarshalling resource map response", err)
}
fmt.Println("Length of data in response : ", len(resourceMapResponse.Data.Resources), "Resource Data received : ", resourceMapResponse.Data.Resources)
if resourceMapResponse.StatusCode == 0 {
//Making recursive call here as there is data to process still
serviceList = RecursiveResourceMapApiCall(resourceMapResponse.Data.Resources[0])
} else if err != nil {
fmt.Println("Error on resource request", err.Error())
} else {
fmt.Println("Error on resource request", response.StatusCode, response.Body)
}
response.Body.Close()
}
}
return serviceList
}
I know that response.body.Close stops resource leak by closing tcp layer connection. But I am not able to close it as it is being called recursively.
I thought of putting it in base condition. But it doesn't feel right. whats the best practise here?
Also can there be any repercussions if I use res.body.Close() in loop?
You can close right after the read:
package main
import (
"net/http"
"os"
"time"
)
func main() {
for i := 0; i < 9; i++ {
res, err := http.Get("https://cdn.sstatic.net/Js/webpack-chunks/8374.en.js")
if err != nil {
panic(err)
}
os.Stdout.ReadFrom(res.Body)
if err := res.Body.Close(); err != nil {
panic(err)
}
time.Sleep(time.Second)
}
}
or you can move it to its own function:
package main
import (
"net/http"
"os"
"time"
)
func get() error {
res, err := http.Get("https://cdn.sstatic.net/Js/webpack-chunks/8374.en.js")
if err != nil {
return err
}
defer res.Body.Close()
if _, err := os.Stdout.ReadFrom(res.Body); err != nil {
return err
}
return nil
}
func main() {
for i := 0; i < 9; i++ {
err := get()
if err != nil {
panic(err)
}
time.Sleep(time.Second)
}
}

Can't run the go routines while building chat app with golang

I'm trying to build a chat app with golang. However, I've encountered a problem that if I wrap the code of the for loop into a Sender() function and using goroutines, the client will shut down immediately. But if I put it in the main function, it can run correctly.
Here is the client code.
package main
import (
"fmt"
"net"
"os"
"os/signal"
"syscall"
)
const (
SERVER_HOST = "localhost"
SERVER_PORT = "9988"
SERVER_TYPE = "tcp"
)
var connection net.Conn
var err error
func main() {
SetupCloseHandler()
//establish connection
connection, err = net.Dial(SERVER_TYPE, SERVER_HOST+":"+SERVER_PORT)
if err != nil {
panic(err)
}
//receive some data
go Receiver()
// go Sender()
for {
var input string
fmt.Print("Enter text: ")
fmt.Scan(&input)
_, err = connection.Write([]byte(input))
if err != nil {
panic(err)
}
}
}
// func Sender() {
// for {
// var input string
// fmt.Print("Enter text: ")
// fmt.Scan(&input)
// _, err = connection.Write([]byte(input))
// if err != nil {
// panic(err)
// }
// }
// }
func Receiver() {
for {
buffer := make([]byte, 1024)
mLen, err := connection.Read(buffer)
if err != nil {
panic(err)
}
fmt.Println("Received: ", string(buffer[:mLen]))
}
}
func SetupCloseHandler() {
c := make(chan os.Signal, 2)
signal.Notify(c, os.Interrupt, syscall.SIGTERM)
go func() {
<-c
fmt.Println("\r- Ctrl+C pressed in Terminal")
os.Exit(0)
}()
}
Here is the server code.
package main
import (
"fmt"
"net"
"os"
"github.com/google/uuid"
)
const (
SERVER_HOST = "localhost"
SERVER_PORT = "9988"
SERVER_TYPE = "tcp"
)
var clientMap = make(map[string]net.Conn)
func main() {
fmt.Println("Server Running...")
server, err := net.Listen(SERVER_TYPE, SERVER_HOST+":"+SERVER_PORT)
if err != nil {
fmt.Println("Error listening:", err.Error())
os.Exit(1)
}
defer server.Close()
fmt.Println("Listening on " + SERVER_HOST + ":" + SERVER_PORT)
fmt.Println("Waiting for client...")
for {
connection, err := server.Accept()
id := uuid.New().String()
clientMap[id] = connection
if err != nil {
fmt.Println("Error accepting: ", err.Error())
os.Exit(1)
}
fmt.Printf("client %s connected\n", id)
go processClient(connection, id)
}
}
func processClient(connection net.Conn, id string) {
defer connection.Close()
for {
buffer := make([]byte, 1024)
mLen, err := connection.Read(buffer)
if err != nil {
fmt.Println("Error reading:", err.Error())
delete(clientMap, id)
connection.Close()
return
}
fmt.Println("Received: ", string(buffer[:mLen]))
for k, c := range clientMap {
if k != id {
c.Write(buffer[:mLen])
}
}
}
}

When my number of goroutine is large, the code will get an error

My code works fine with a small number of goroutine, but a large number of errors can occur with memory and pointers.I guess it is my lock used improperly.Can you help me see the tools I wrote? I have been debugging for a long time but I have no way to start.
I thought about adding a read lock to the clock function.But still will report an error.I also tried to use sync.Map. But it did not solve the problem
package main
import (
"fmt"
"io/ioutil"
"net/http"
"encoding/json"
"strings"
"sync"
"time"
"strconv"
)
type UrlArray struct {
Url string `json:"url"`
Method string `json:"method"`
Params string `json:"params"`
}
type MsgRequest struct {
Command string `json:"command"`
Concurrent int `json:"concurrent"`
IncrementalRatio float64 `json:"incrementalRatio"`
InitialRatio float64 `json:"initialRatio"`
Intervals int `json:"intervals"`
UrlArray []UrlArray `json:"urls"`
}
type GroupData struct {
TotalCount int `json:"totalCount"`
FailCount int `json:"failCount"`
SuccessRate float64 `json:"successRate"`
CostTime float64 `json:"costTime"`
AvergeTime float64 `json:"avergeTime"`
}
type MsgResponse struct {
TimeObject map[string]GroupData `json:"timeObject"`
TotalCount int `json:"totalCount"`
FailCount int `json:"failCount"`
SuccessRate float64 `json:"successRate"`
CostTime float64 `json:"costTime"`
AvergeTime float64 `json:"avergeTime"`
}
type LevelData struct {
result map[string]GroupData
sync.RWMutex
}
type ResultStatic struct {
result map[string]MsgResponse
}
var p = fmt.Println
var levelOutput LevelData
var output ResultStatic
var closeAllChan chan int
var isHandle = false
//write to map
func (r *LevelData) recordStatic(url string, status bool, useTimeSec float64) {
r.Lock()
defer r.Unlock()
val, ok := r.result[url]
if ok {
val.TotalCount += 1
val.CostTime += useTimeSec
if status == false {
val.FailCount += 1
}
val.SuccessRate = 1.00 - (float64(val.FailCount) * 1.00) / (float64(val.TotalCount) * 1.00)
val.AvergeTime = val.CostTime / float64(val.TotalCount)
levelOutput.result[url] = val
} else {
failcnt := 0
if status == false {
failcnt = 1
}
successRate := float64(1 - failcnt)
val := GroupData{TotalCount: 1, FailCount: failcnt, SuccessRate: successRate, CostTime: useTimeSec, AvergeTime: useTimeSec}
levelOutput.result[url] = val
}
}
//get request
func get(url string, params string) {
start := time.Now()
data := url + "?" + params
response, err:= http.Get(data)
end := time.Now()
costTime := float64((end.Sub(start)).Seconds())
if err != nil {
levelOutput.recordStatic(url, false, costTime)
} else {
levelOutput.recordStatic(url, true, costTime)
}
defer response.Body.Close()
}
// post request
func post(url string, params string) {
start := time.Now()
response, err := http.Post(url, "application/x-www-form-urlencoded", strings.NewReader(params))
end := time.Now()
costTime := float64((end.Sub(start)).Seconds())
if err != nil {
levelOutput.recordStatic(url, false, costTime)
} else {
levelOutput.recordStatic(url, true, costTime)
}
response.Body.Close()
}
//clock ticker
func (r *LevelData) clock(urlArrays []UrlArray, concurrent int, initialRatio float64, incrementalRatio float64, intervals int) {
start := int(float64(concurrent) * initialRatio)
increment := int(float64(concurrent) * incrementalRatio)
ticker := time.NewTicker(time.Duration(intervals) * time.Second)
endtotal := increment + concurrent
objectKey := 0
for range ticker.C {
select {
case <-closeAllChan:
p("死循环退出")
return
default:
if objectKey != 0 {
for _, v := range(urlArrays) {
val1, ok1 := output.result[v.Url]
p(val1, ok1)
if !ok1 {
val1 = MsgResponse{}
val1.TimeObject = make(map[string]GroupData)
}
val1.TimeObject[strconv.Itoa(objectKey)] = levelOutput.result[v.Url]
val1.TotalCount += (val1.TimeObject[strconv.Itoa(objectKey)]).TotalCount
val1.FailCount += (val1.TimeObject[strconv.Itoa(objectKey)]).FailCount
val1.CostTime += (val1.TimeObject[strconv.Itoa(objectKey)]).CostTime
val1.SuccessRate = 1.00 - (float64(val1.FailCount) * 1.00) / (float64(val1.TotalCount) * 1.00)
val1.AvergeTime = val1.CostTime / float64(val1.TotalCount)
output.result[v.Url] = val1
}
}
levelOutput = LevelData{result: map[string]GroupData{}}
if start >= endtotal {
p("start == concurrent")
close(closeAllChan)
time.Sleep(time.Second)
return
}
if objectKey == 0 {
for i := 0; i < start; i++ {
go work(urlArrays)
}
} else {
for i := 0; i < increment; i++ {
go work(urlArrays)
}
}
objectKey += 1
start += increment
}
}
}
//work
func work(urlArrays []UrlArray) {
i := 0
for {
select {
case <-closeAllChan:
return
default:
index := i % len(urlArrays)
i++
url := urlArrays[index].Url
method := urlArrays[index].Method
params := urlArrays[index].Params
if method == "Get" {
get(url, params)
} else {
post(url, params)
}
}
}
}
func setupResponse(w *http.ResponseWriter, req *http.Request) {
(*w).Header().Set("Access-Control-Allow-Origin", "*")
(*w).Header().Set("Access-Control-Allow-Methods", "POST, GET, OPTIONS")
(*w).Header().Set("Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization")
}
func myfunc(w http.ResponseWriter, r *http.Request) {
setupResponse(&w, r)
if (r).Method == "OPTIONS" {
return
}
b, err := ioutil.ReadAll(r.Body)
defer r.Body.Close()
if err != nil {
http.Error(w, err.Error(), 501)
return
}
//json decode
var msgs MsgRequest
err = json.Unmarshal(b, &msgs)
if err != nil {
p(err.Error())
http.Error(w, err.Error(), 502)
return
}
var command = msgs.Command
if command == "stop" {
p("stop start")
if isHandle {
close(closeAllChan)
isHandle = false
} else {
p("no handle")
http.Error(w, "未有数据在执行", 504)
return
}
} else if command == "get" {
p("get start")
if isHandle {
p(output.result)
js, err := json.Marshal(output.result)
if err != nil {
http.Error(w, err.Error(), 503)
return
}
w.Header().Set("Content-Type", "application/json")
w.Write(js)
} else {
p("no handle")
http.Error(w, "未有数据在执行", 504)
return
}
} else {
isHandle = true
closeAllChan = make(chan int)
output = ResultStatic{result: map[string]MsgResponse{}}
urlArrays := msgs.UrlArray
concurrent := msgs.Concurrent
initialRatio := msgs.InitialRatio
incrementalRatio := msgs.IncrementalRatio
intervals := msgs.Intervals
levelOutput.clock(urlArrays, concurrent, initialRatio, incrementalRatio, intervals)
isHandle = false
js, err := json.Marshal(output.result)
if err != nil {
http.Error(w, err.Error(), 503)
return
}
p(output.result)
w.Header().Set("Content-Type", "application/json")
w.Write(js)
}
return
}
func main() {
http.HandleFunc("/handle", myfunc)
err := http.ListenAndServe(":8080", nil)
if err != nil {
panic(err)
}
}
// json example
// {
// "command": "handle/get/stop",
// "concurrent": 100,
// "initialRatio": 0.60,
// "incrementalRatio": 0.20,
// "intervals": 10,
// "urls": [
// {
// "url": "http://google.com",
// "method": "GET",
// "params": ""
// }
// ]
// }
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x40 pc=0x6782c4]
goroutine 64655 [running]:
main.post(0xc00001e0c0, 0x3c, 0xc000188510, 0x2f)
/root/toolv2.go:114 +0x1b4
main.work(0xc000391380, 0x1, 0x4)
/root/toolv2.go:193 +0xcf
created by main.(*LevelData).clock
/root/toolv2.go:161 +0x657
exit status 2
"invalid memory address or nil pointer dereference" does not indicate a data race. You are dereferencing a pointer that is nil:
func post(url string, params string) {
start := time.Now()
response, err := http.Post(url, "application/x-www-form-urlencoded", strings.NewReader(params))
end := time.Now()
costTime := float64((end.Sub(start)).Seconds())
if err != nil {
levelOutput.recordStatic(url, false, costTime)
} else {
levelOutput.recordStatic(url, true, costTime)
}
response.Body.Close()
}
You must not dereference response after an error occurs, because then response is nil.

Golang Gorilla Websocket stops receiving information at 120 seconds

I'm currently trying to connect to the CEX.IO bitcoin exchange's websocket, but have been having issues not only with CEX.IO but with others too. All of my connections drop around the 120-second mark which makes me think there is some TTL problem going on. The Process() goroutine in the main package ends up just hanging and waiting for data from the readLoop which just stops receiving data. I've included some read-only API keys in the code so you can test if you'd like.
package main
import (
"fmt"
"bitbucket.org/tradedefender/cryptocurrency/exchange-connector/cexio"
"github.com/shopspring/decimal"
"encoding/json"
"time"
)
type OrderBook struct {
Asks []Ask
Bids []Bid
}
type Ask struct {
Rate decimal.Decimal
Amount decimal.Decimal
}
type Bid struct {
Rate decimal.Decimal
Amount decimal.Decimal
}
func main() {
cexioConn := new(cexio.Connection)
err := cexioConn.Connect()
if err != nil {
fmt.Errorf("error: %s", err.Error())
}
err = cexioConn.Authenticate("TLwYkktLf7Im6nqSKt6UO1IrU", "9ImOJcR7Qj3LMIyPCzky0D7WE")
if err != nil {
fmt.Errorf("error: %s", err.Error())
}
readChannel := make(chan cexio.IntraAppMessage, 25)
go cexioConn.ReadLoop(readChannel)
processor := Processor{
WatchPairs: [][2]string{
[2]string{
"BTC", "USD",
},
},
conn: cexioConn,
}
go processor.Process(readChannel)
// LOL
for {
continue
}
}
type Processor struct {
WatchPairs [][2]string
conn *cexio.Connection
}
func (p *Processor) Process(ch <-chan cexio.IntraAppMessage) {
p.conn.SubscribeToOrderBook(p.WatchPairs[0])
pingTimer := time.Now().Unix()
for {
fmt.Printf("(%v)\n", time.Now().Unix())
if (time.Now().Unix() - pingTimer) >= 10 {
fmt.Println("sending ping")
p.conn.SendPing()
pingTimer = time.Now().Unix()
}
readMsg := <- ch
output, _ := json.Marshal(readMsg.SocketMessage)
fmt.Println(string(output))
if readMsg.SocketMessage.Event == "ping" {
fmt.Println("sending pong")
p.conn.SendPong()
pingTimer = time.Now().Unix()
}
}
}
Below is the connector to the cexio websocket. Here is a link to their API: https://cex.io/websocket-api
package cexio
import (
"github.com/gorilla/websocket"
//"github.com/shopspring/decimal"
"github.com/satori/go.uuid"
"encoding/hex"
"encoding/json"
"crypto/hmac"
"crypto/sha256"
"bytes"
"strconv"
"time"
"fmt"
)
const Url = "wss://ws.cex.io/ws/"
type Connection struct {
conn *websocket.Conn
}
type IntraAppMessage struct {
SocketMessage GenericMessage
ProgramMessage ProgramMessage
}
type GenericMessage struct {
Event string `json:"e"`
Data interface{} `json:"data"`
Auth AuthData `json:"auth,omitempty"`
Ok string `json:"ok,omitempty"`
Oid string `json:"oid,omitempty"`
Time int64 `json:"time,omitempty"`
}
type ProgramMessage struct {
Error string
}
type AuthData struct {
Key string `json:"key"`
Signature string `json:"signature"`
Timestamp int64 `json:"timestamp"`
}
type OrderBookSubscribeData struct {
Pair [2]string `json:"pair"`
Subscribe bool `json:"subscribe"`
Depth int `json:"depth"`
}
func (c *Connection) SendPong() error {
pongMsg := GenericMessage{
Event: "pong",
}
err := c.conn.WriteJSON(pongMsg)
if err != nil {
return nil
}
deadline := time.Now().Add(15*time.Second)
err = c.conn.WriteControl(websocket.PongMessage, nil, deadline)
if err != nil {
return err
}
return nil
}
func (c *Connection) SendPing() error {
pingMsg := GenericMessage{
Event: "get-balance",
Oid: uuid.NewV4().String(),
}
err := c.conn.WriteJSON(pingMsg)
if err != nil {
return err
}
deadline := time.Now().Add(15*time.Second)
err = c.conn.WriteControl(websocket.PingMessage, nil, deadline)
if err != nil {
return err
}
return nil
}
func (c *Connection) Connect() error {
dialer := *websocket.DefaultDialer
wsConn, _, err := dialer.Dial(Url, nil)
if err != nil {
return err
}
c.conn = wsConn
//c.conn.SetPingHandler(c.HandlePing)
for {
_, msgBytes, err := c.conn.ReadMessage()
if err != nil {
c.Disconnect()
return err
}
fmt.Println(string(msgBytes))
var m GenericMessage
err = json.Unmarshal(msgBytes, &m)
if err != nil {
c.Disconnect()
return err
}
if m.Event != "connected" {
c.Disconnect()
return err
} else {
break
}
}
return nil
}
func (c *Connection) Disconnect() error {
return c.conn.Close()
}
func (c *Connection) ReadLoop(ch chan<- IntraAppMessage) {
for {
fmt.Println("starting new read")
_, msgBytes, err := c.conn.ReadMessage()
if err != nil {
ch <- IntraAppMessage{
ProgramMessage: ProgramMessage{
Error: err.Error(),
},
}
continue
}
var m GenericMessage
err = json.Unmarshal(msgBytes, &m)
if err != nil {
ch <- IntraAppMessage{
ProgramMessage: ProgramMessage{
Error: err.Error(),
},
}
continue
}
ch <- IntraAppMessage{
SocketMessage: m,
}
}
}
func CreateSignature(timestamp int64, key, secret string) string {
secretBytes := []byte(secret)
h := hmac.New(sha256.New, secretBytes)
var buffer bytes.Buffer
buffer.WriteString(strconv.FormatInt(timestamp, 10))
buffer.WriteString(key)
h.Write(buffer.Bytes())
return hex.EncodeToString(h.Sum(nil))
}
func (c *Connection) Authenticate(key, secret string) error {
timestamp := time.Now().Unix()
signature := CreateSignature(timestamp, key, secret)
var authMsg GenericMessage
authMsg.Event = "auth"
authMsg.Auth = AuthData{
Key: key,
Signature: signature,
Timestamp: timestamp,
}
err := c.conn.WriteJSON(authMsg)
if err != nil {
return err
}
for {
_, msgBytes, err := c.conn.ReadMessage()
if err != nil {
c.Disconnect()
return err
}
fmt.Println(string(msgBytes))
var m GenericMessage
err = json.Unmarshal(msgBytes, &m)
if err != nil {
c.Disconnect()
return err
}
if m.Event != "auth" && m.Ok != "ok" {
c.Disconnect()
return err
} else {
break
}
}
return nil
}
func (c *Connection) SubscribeToOrderBook(pair [2]string) error {
sendMsg := GenericMessage{
Event: "order-book-subscribe",
Data: OrderBookSubscribeData{
Pair: pair,
Subscribe: true,
Depth: 0,
},
Oid: uuid.NewV4().String(),
}
err := c.conn.WriteJSON(sendMsg)
if err != nil {
return err
}
return nil
}
func (c *Connection) GetBalance() error {
sendMsg := GenericMessage{
Event: "get-balance",
Oid: uuid.NewV4().String(),
}
err := c.conn.WriteJSON(sendMsg)
if err != nil {
return err
}
return nil
}
Solution was to remove the
for {
continue
}
at the end of the main function

Getting EOF from server as client in Go

I have some a Go client for a custom protocol. The protocol is lz4-compressed JSON-RPC with a four byte header giving the length of the compressed JSON.
func ReceiveMessage(conn net.Conn) ([]byte, error) {
start := time.Now()
bodyLen := 0
body := make([]byte, 0, 4096)
buf := make([]byte, 0, 256)
for bodyLen == 0 || len(body) < bodyLen {
if len(body) > 4 {
header := body[:4]
body = body[:4]
bodyLen = int(unpack(header))
}
n, err := conn.Read(buf[:])
if err != nil {
if err != io.EOF {
return body, err
}
}
body = append(body, buf[0:n]...)
now := time.Now()
if now.Sub(start) > time.Duration(readTimeout) * time.Millisecond {
return body, fmt.Errorf("Timed-out while reading from socket.")
}
time.Sleep(time.Duration(1) * time.Millisecond)
}
return lz4.Decode(nil, body)
}
The client:
func main() {
address := os.Args[1]
msg := []byte(os.Args[2])
fmt.Printf("Sending %s to %s\n", msg, address)
conn, err := net.Dial(address)
if err != nil {
fmt.Printf("%v\n", err)
return
}
// Another library call
_, err = SendMessage(conn, []byte(msg))
if err != nil {
fmt.Printf("%v\n", err)
return
}
response, err := ReceiveMessage(conn)
conn.Close()
if err != nil {
fmt.Printf("%v\n", err)
return
}
fmt.Printf("Response: %s\n", response)
}
When I call it, I get no response and it just times out. (If I do not explicitly ignore the EOF, it returns there with io.EOF error.) I have another library for this written in Python that also works against the same endpoint with the same payload. Do you see anything immediately?
[JimB just beat me to an answer but here goes anyway.]
The root issue is that you did body = body[:4]
when you wanted body = body[4:].
The former keeps only the first four header bytes
while the latter tosses
the four header bytes just decoded.
Here is a self contained version with some debug logs
that works.
It has some of the other changes I mentioned.
(I guessed at various things that you didn't include, like the lz4 package used, the timeout, unpack, etc.)
package main
import (
"encoding/binary"
"errors"
"fmt"
"io"
"log"
"net"
"time"
"github.com/bkaradzic/go-lz4"
)
const readTimeout = 30 * time.Second // XXX guess
func ReceiveMessage(conn net.Conn) ([]byte, error) {
bodyLen := 0
body := make([]byte, 0, 4096)
var buf [256]byte
conn.SetDeadline(time.Now().Add(readTimeout))
defer conn.SetDeadline(time.Time{}) // disable deadline
for bodyLen == 0 || len(body) < bodyLen {
if bodyLen == 0 && len(body) >= 4 {
bodyLen = int(unpack(body[:4]))
body = body[4:]
if bodyLen <= 0 {
return nil, errors.New("invalid body length")
}
log.Println("read bodyLen:", bodyLen)
continue
}
n, err := conn.Read(buf[:])
body = append(body, buf[:n]...)
log.Printf("appended %d bytes, len(body) now %d", n, len(body))
// Note, this is checked *after* handing any n bytes.
// An io.Reader is allowed to return data with an error.
if err != nil {
if err != io.EOF {
return nil, err
}
break
}
}
if len(body) != bodyLen {
return nil, fmt.Errorf("got %d bytes, expected %d",
len(body), bodyLen)
}
return lz4.Decode(nil, body)
}
const address = ":5678"
var msg = []byte(`{"foo":"bar"}`)
func main() {
//address := os.Args[1]
//msg := []byte(os.Args[2])
fmt.Printf("Sending %s to %s\n", msg, address)
conn, err := net.Dial("tcp", address)
if err != nil {
fmt.Printf("%v\n", err)
return
}
// Another library call
_, err = SendMessage(conn, msg)
if err != nil {
fmt.Printf("%v\n", err)
return
}
response, err := ReceiveMessage(conn)
conn.Close()
if err != nil {
fmt.Printf("%v\n", err)
return
}
fmt.Printf("Response: %s\n", response)
}
// a guess at what your `unpack` does
func unpack(b []byte) uint32 {
return binary.LittleEndian.Uint32(b)
}
func SendMessage(net.Conn, []byte) (int, error) {
// stub
return 0, nil
}
func init() {
// start a simple test server in the same process as a go-routine.
ln, err := net.Listen("tcp", address)
if err != nil {
log.Fatal(err)
}
go func() {
defer ln.Close()
for {
conn, err := ln.Accept()
if err != nil {
log.Fatalln("accept:", err)
}
go Serve(conn)
}
}()
}
func Serve(c net.Conn) {
defer c.Close()
// skip readding the initial request/message and just respond
const response = `{"somefield": "someval"}`
// normally (de)compression in Go is done streaming via
// an io.Reader or io.Writer but we need the final length.
data, err := lz4.Encode(nil, []byte(response))
if err != nil {
log.Println("lz4 encode:", err)
return
}
log.Println("sending len:", len(data))
if err = binary.Write(c, binary.LittleEndian, uint32(len(data))); err != nil {
log.Println("writing len:", err)
return
}
log.Println("sending data")
if _, err = c.Write(data); err != nil {
log.Println("writing compressed response:", err)
return
}
log.Println("Serve done, closing connection")
}
Playground (but not runnable there).
You have a number of issues with the server code. Without a full reproducing case, it's hard to tell if these will fix everything.
for bodyLen == 0 || len(body) < bodyLen {
if len(body) > 4 {
header := body[:4]
body = body[:4]
bodyLen = int(unpack(header))
}
every iteration, if len(body) > 4, you slice body back to the first 4 bytes. Body might never get to be >= bodyLen.
n, err := conn.Read(buf[:])
You don't need to re-slice buf here, use conn.Read(buf)
if err != nil {
if err != io.EOF {
return body, err
}
}
io.EOF is the end of the stream, and you need to handle it. Note that n might still be > 0 when you get an EOF. Check after processing the body for io.EOF or you could loop indefinitely.
body = append(body, buf[0:n]...)
now := time.Now()
if now.Sub(start) > time.Duration(readTimeout) * time.Millisecond {
return body, fmt.Errorf("Timed-out while reading from socket.")
you would be better off using conn.SetReadDeadline before each read, so a stalled Read could be interrupted.

Resources