I am working on a personal project that will run on a Raspberry Pi with some sensors attached to it.
The function that read from the sensors and the function that handle the socket connection are executed in different goroutines, so, in order to send data on the socket when they are read from the sensors, I create a chan []byte in the main function and pass it to the goroutines.
My problem came out here: if I do multiple writes in a row, only the first data arrives to the client, but the others don't. But if I put a little time.Sleep in the sender function, all the data arrives correctly to the client.
Anyway, that's a simplified version of this little program :
package main
import (
"net"
"os"
"sync"
"time"
)
const socketName string = "./test_socket"
// create to the socket and launch the accept client routine
func launchServerUDS(ch chan []byte) {
if err := os.RemoveAll(socketName); err != nil {
return
}
l, err := net.Listen("unix", socketName)
if err != nil {
return
}
go acceptConnectionRoutine(l, ch)
}
// accept incoming connection on the socket and
// 1) launch the routine to handle commands from the client
// 2) launch the routine to send data when the server reads from the sensors
func acceptConnectionRoutine(l net.Listener, ch chan []byte) {
defer l.Close()
for {
conn, err := l.Accept()
if err != nil {
return
}
go commandsHandlerRoutine(conn, ch)
go autoSendRoutine(conn, ch)
}
}
// routine that sends data to the client
func autoSendRoutine(c net.Conn, ch chan []byte) {
for {
data := <-ch
if string(data) == "exit" {
return
}
c.Write(data)
}
}
// handle client connection and calls functions to execute commands
func commandsHandlerRoutine(c net.Conn, ch chan []byte) {
for {
buf := make([]byte, 1024)
n, err := c.Read(buf)
if err != nil {
ch <- []byte("exit")
break
}
// now, for sake of simplicity , only echo commands back to the client
_, err = c.Write(buf[:n])
if err != nil {
ch <- []byte("exit")
break
}
}
}
// write on the channel to the autosend routine so the data are written on the socket
func sendDataToClient(data []byte, ch chan []byte) {
select {
case ch <- data:
// if i put a little sleep here, no problems
// i i remove the sleep, only data1 is sent to the client
// time.Sleep(1 * time.Millisecond)
default:
}
}
func dummyReadDataRoutine(ch chan []byte) {
for {
// read data from the sensors every 5 seconds
time.Sleep(5 * time.Second)
// read first data and send it
sendDataToClient([]byte("dummy data1\n"), ch)
// read second data and send it
sendDataToClient([]byte("dummy data2\n"), ch)
// read third data and send it
sendDataToClient([]byte("dummy data3\n"), ch)
}
}
func main() {
ch := make(chan []byte)
wg := sync.WaitGroup{}
wg.Add(2)
go dummyReadDataRoutine(ch)
go launchServerUDS(ch)
wg.Wait()
}
I don't think it's correct to use a sleep to synchronize writes. How do I fix this while keeping the functions running on a different different goroutines.
The primary problem was in the function:
func sendDataToClient(data []byte, ch chan []byte) {
select {
case ch <- data:
// if I put a little sleep here, no problems
// if I remove the sleep, only data1 is sent to the client
// time.Sleep(1 * time.Millisecond)
default:
}
If the channel ch isn't ready at the moment the function is called, the default case will be taken and the data will never be sent. In this case you should eliminate the function and send to the channel directly.
Buffering the channel is orthogonal to the problem at hand, and should be done for the similar reasons as you would buffered IO, i.e. provide a "buffer" for writes that can't immediately progress. If the code were not able progress without a buffer, adding one only delays possible deadlocks.
You also don't need the exit sentinel value here, as you could range over the channel and close it when you're done. This however still ignores write errors, but again that requires some re-design.
for data := range ch {
c.Write(data)
}
You should also be careful passing slices over channels, as it's all too easy to lose track of which logical process has ownership and is going to modify the backing array. I can't say from the information given if passing the read+write data over channels improves the architecture, but this is not a pattern you will find in most go networking code.
JimB gave a good explanation, so I think his answer is the better one.
I have included my partial solution in this answer.
I was thinking that my code was clear and simplified, but as Jim said I can do it simpler and clearer. I leave my old code posted so people can understand better how you can post simpler code and not do a mess like I did.
As chmike said, my issue wasn't related to the socket like I was thinking, but was only related to the channel. Write on a unbuffered channel was one of the problems. After change the unbuffered channel to a buffered one, the issue was resolved. Anyway, this code is not "good code" and can be improved following the principles that JimB has written in his answer.
So here is the new code:
package main
import (
"net"
"os"
"sync"
"time"
)
const socketName string = "./test_socket"
// create the socket and accept clients connections
func launchServerUDS(ch chan []byte, wg *sync.WaitGroup) {
defer wg.Done()
if err := os.RemoveAll(socketName); err != nil {
return
}
l, err := net.Listen("unix", socketName)
if err != nil {
return
}
defer l.Close()
for {
conn, err := l.Accept()
if err != nil {
return
}
// this goroutine are launched when a client is connected
// routine that listen and echo commands
go commandsHandlerRoutine(conn, ch)
// routine to send data read from the sensors to the client
go autoSendRoutine(conn, ch)
}
}
// routine that sends data to the client
func autoSendRoutine(c net.Conn, ch chan []byte) {
for {
data := <-ch
if string(data) == "exit" {
return
}
c.Write(data)
}
}
// handle commands received from the client
func commandsHandlerRoutine(c net.Conn, ch chan []byte) {
for {
buf := make([]byte, 1024)
n, err := c.Read(buf)
if err != nil {
// if i can't read send an exit command to autoSendRoutine and exit
ch <- []byte("exit")
break
}
// now, for sake of simplicity , only echo commands back to the client
_, err = c.Write(buf[:n])
if err != nil {
// if i can't write back send an exit command to autoSendRoutine and exit
ch <- []byte("exit")
break
}
}
}
// this goroutine reads from the sensors and write to the channel , so data are sent
// to the client if a client is connected
func dummyReadDataRoutine(ch chan []byte, wg *sync.WaitGroup) {
x := 0
for x < 100 {
// read data from the sensors every 5 seconds
time.Sleep(1 * time.Second)
// read first data and send it
ch <- []byte("data1\n")
// read second data and send it
ch <- []byte("data2\n")
// read third data and send it
ch <- []byte("data3\n")
x++
}
wg.Done()
}
func main() {
// create a BUFFERED CHANNEL
ch := make(chan []byte, 1)
wg := sync.WaitGroup{}
wg.Add(2)
// launch the goruotines that handle the socket connections
// and read data from the sensors
go dummyReadDataRoutine(ch, &wg)
go launchServerUDS(ch, &wg)
wg.Wait()
}
Related
I found some example with "Catch values from Goroutines"-> link
There is show how to fetch value but if I want to return value from several goroutines, it wont work.So, does anybody know, how to do it?
package main
import (
"fmt"
"io/ioutil"
"log"
"net/http"
"sync"
)
// WaitGroup is used to wait for the program to finish goroutines.
var wg sync.WaitGroup
func responseSize(url string, nums chan int) {
// Schedule the call to WaitGroup's Done to tell goroutine is completed.
defer wg.Done()
response, err := http.Get(url)
if err != nil {
log.Fatal(err)
}
defer response.Body.Close()
body, err := ioutil.ReadAll(response.Body)
if err != nil {
log.Fatal(err)
}
// Send value to the unbuffered channel
nums <- len(body)
}
func main() {
nums := make(chan int) // Declare a unbuffered channel
wg.Add(1)
go responseSize("https://www.golangprograms.com", nums)
go responseSize("https://gobyexample.com/worker-pools", nums)
go responseSize("https://stackoverflow.com/questions/ask", nums)
fmt.Println(<-nums) // Read the value from unbuffered channel
wg.Wait()
close(nums) // Closes the channel
// >> loading forever
Also, this example, worker pools
Is it possible to get value from result: fmt.Println(<-results) <- will be error.
Yes, just read from the channel multiple times:
answerOne := <-nums
answerTwo := <-nums
answerThree := <-nums
Channels function like thread-safe queues, allowing you to queue up values and read them out one by one
P.S. You should either add 3 to the wait group or not have one at all. The <-nums will block until a value is available on nums so it is not necessary
I'm using two concurrent goroutines to copy stdin/stdout from my terminal to a net.Conn target. For some reason, I can't manage to completely stop the two go routines without getting a panic error (for trying to close a closed connection). This is my code:
func interact(c net.Conn, sessionMap map[int]net.Conn) {
quit := make(chan bool) //the channel to quit
copy := func(r io.ReadCloser, w io.WriteCloser) {
defer func() {
r.Close()
w.Close()
close(quit) //this is how i'm trying to close it
}()
_, err := io.Copy(w, r)
if err != nil {
//
}
}
go func() {
for {
select {
case <-quit:
return
default:
copy(c, os.Stdout)
}
}
}()
go func() {
for {
select {
case <-quit:
return
default:
copy(os.Stdin, c)
}
}
}()
}
This errors as panic: close of closed channel
I want to terminate the two go routines, and then normally proceed to another function. What am I doing wrong?
You can't call close on a channel more than once, there's no reason to call copy in a for loop, since it can only operate one time, and you're copying in the wrong direction, writing to stdin and reading from stdout.
Simply asking how to quit 2 goroutines is simple, but that's not the only thing you need to do here. Since io.Copy is blocking, you don't need the extra synchronization to determine when the call is complete. This lets you simplify the code significantly, which will make it a lot easier to reason about.
func interact(c net.Conn) {
go func() {
// You want to close this outside the goroutine if you
// expect to send data back over a half-closed connection
defer c.Close()
// Optionally close stdout here if you need to signal the
// end of the stream in a pipeline.
defer os.Stdout.Close()
_, err := io.Copy(os.Stdout, c)
if err != nil {
//
}
}()
_, err := io.Copy(c, os.Stdin)
if err != nil {
//
}
}
Also note that you may not be able to break out of the io.Copy from stdin, so you can't expect the interact function to return. Manually doing the io.Copy in the function body and checking for a half-closed connection on every loop may be a good idea, then you can break out sooner and ensure that you fully close the net.Conn.
Also could be like this
func scanReader(quit chan int, r io.Reader) chan string {
line := make(chan string)
go func(quit chan int) {
defer close(line)
scan := bufio.NewScanner(r)
for scan.Scan() {
select {
case <- quit:
return
default:
s := scan.Text()
line <- s
}
}
}(quit)
return line
}
stdIn := scanReader(quit, os.Stdin)
conIn := scanReader(quit, c)
for {
select {
case <-quit:
return
case l <- stdIn:
_, e := fmt.Fprintf(c, l)
if e != nil {
quit <- 1
return
}
case l <- conIn:
fmt.Println(l)
}
}
I need to send echo aaa three times to redis server,
but its get stuck in the middle of process, I also check
if read and write operation get error message, but it doesn't.
so, why it get stuck in the middle of process?
package main
import (
"fmt"
"os"
"io"
"net"
"sync"
)
var (
wg = new(sync.WaitGroup)
)
func readFromServer(isWrite chan bool, r io.Reader) {
for {
select {
case <-isWrite:
_ , err := io.Copy(os.Stdout, r)
if err != nil {
panic(err)
}
}
}
}
func writeToServer(conn net.Conn , isWrite chan bool ){
defer wg.Done()
for i :=0; i<3; i++{
_ , err := conn.Write([]byte("*2\r\n$4\r\necho\r\n$3\r\naaa\r\n"))
if err != nil {
panic(err)
}
isWrite<- true
}
}
func main(){
wg.Add(1)
conn ,err := net.Dial("tcp","127.0.0.1:6379")
isWrite := make(chan bool)
if err != nil {
panic(err)
}
go readFromServer(isWrite, conn)
go writeToServer(conn , isWrite)
wg.Wait()
fmt.Println("finished...")
}
Output:
$3
aaa
$3
aaa
Stuck here...
The readFromServer function receives one value from the isWrite channel and then blocks in the call to io.Copy. The io.Copy function does not return until EOF or some error reading or writing data. All of the program output is from the single call to io.Copy.
The second send to isWrite in sendToServer blocks.
The isWrite channel is an unbuffered channel. A send on an unbuffered channel does not proceed until there's a receiver. There is no receiver on the channel because readFromServer is blocked in the call to io.Copy.
Possible fixes are:
The fix is to modify readFromServer to parse the RESP protocol and read exactly one message per iteration in the loop.
Replace for loop in readFromServer with a single call to io.Copy.
The isWrite channel is not needed.
The program does not ensure that readFromServer reads all of the responses from writeToServer before the program exits.
I'm looking for a solution to multiplex some channel output in go.
I have a source of data which is a read from an io.Reader that I send to a single channel. On the other side I have a websocket request handler that reads from the channel. Now it happens that two clients create a websocket connection, both reading from the same channel but each of them only getting a part of the messages.
Code example (simplified):
func (b *Bootloader) ReadLog() (<-chan []byte, error) {
if b.logCh != nil {
logrus.Warn("ReadLog called while channel already exists!")
return b.logCh, nil // This is where we get problems
}
b.logCh = make(chan []byte, 0)
go func() {
buf := make([]byte, 1024)
for {
n, err := b.p.Read(buf)
if err == nil {
msg := make([]byte, n)
copy(msg, buf[:n])
b.logCh <- msg
} else {
break
}
}
close(b.logCh)
b.logCh = nil
}()
return b.logCh, nil
}
Now when ReadLog() is called twice, the second call just returns the channel created in the first call, which leads to the problem explained above.
The question is: How to do proper multiplexing?
Is it better/easier/more ideomatic to care about the multiplexing on the sending or receiving site?
Should I hide the channel from the receiver and work with callbacks?
I'm a little stuck at the moment. Any hints are welcome.
Mutiplexing is pretty straightforward: make a slice of channels you want to multiplex to, start up a goroutine that reads from the original channel and copies each message to each channel in the slice:
// Really this should be in Bootloader but this is just an example
var consumers []chan []byte
func (b *Bootloader) multiplex() {
// We'll use a sync.once to make sure we don't start a bunch of these.
sync.Once(func(){
go func() {
// Every time a message comes over the channel...
for v := range b.logCh {
// Loop over the consumers...
for _,cons := range consumers {
// Send each one the message
cons <- v
}
}
}()
})
}
I'm here to find out the most idiomatic way to do the follow task.
Task:
Write data from a channel to a file.
Problem:
I have a channel ch := make(chan int, 100)
I need to read from the channel and write the values I read from the channel to a file. My question is basically how do I do so given that
If channel ch is full, write the values immediately
If channel ch is not full, write every 5s.
So essentially, data needs to be written to the file at least every 5s (assuming that data will be filled into the channel at least every 5s)
Whats the best way to use select, for and range to do my above task?
Thanks!
There is no such "event" as "buffer of channel is full", so you can't detect that [*]. This means you can't idiomatically solve your problem with language primitives using only 1 channel.
[*] Not entirely true: you could detect if the buffer of a channel is full by using select with default case when sending on the channel, but that requires logic from the senders, and repetitive attempts to send.
I would use another channel from which I would receive as values are sent on it, and "redirect", store the values in another channel which has a buffer of 100 as you mentioned. At each redirection you may check if the internal channel's buffer is full, and if so, do an immediate write. If not, continue to monitor the "incoming" channel and a timer channel with a select statement, and if the timer fires, do a "regular" write.
You may use len(chInternal) to check how many elements are in the chInternal channel, and cap(chInternal) to check its capacity. Note that this is "safe" as we are the only goroutine handling the chInternal channel. If there would be multiple goroutines, value returned by len(chInternal) could be outdated by the time we use it to something (e.g. comparing it).
In this solution chInternal (as its name says) is for internal use only. Others should only send values on ch. Note that ch may or may not be a buffered channel, solution works in both cases. However, you may improve efficiency if you also give some buffer to ch (so chances that senders get blocked will be lower).
var (
chInternal = make(chan int, 100)
ch = make(chan int) // You may (should) make this a buffered channel too
)
func main() {
delay := time.Second * 5
timer := time.NewTimer(delay)
for {
select {
case v := <-ch:
chInternal <- v
if len(chInternal) == cap(chInternal) {
doWrite() // Buffer is full, we need to write immediately
timer.Reset(delay)
}
case <-timer.C:
doWrite() // "Regular" write: 5 seconds have passed since last write
timer.Reset(delay)
}
}
}
If an immediate write happens (due to a "buffer full" situation), this solution will time the next "regular" write 5 seconds after this. If you don't want this and you want the 5-second regular writes be independent from the immediate writes, simply do not reset the timer following the immediate write.
An implementation of doWrite() may be as follows:
var f *os.File // Make sure to open file for writing
func doWrite() {
for {
select {
case v := <-chInternal:
fmt.Fprintf(f, "%d ", v) // Write v to the file
default: // Stop when no more values in chInternal
return
}
}
}
We can't use for ... range as that only returns when the channel is closed, but our chInternal channel is not closed. So we use a select with a default case so when no more values are in the buffer of chInternal, we return.
Improvements
Using a slice instead of 2nd channel
Since the chInternal channel is only used by us, and only on a single goroutine, we may also choose to use a single []int slice instead of a channel (reading/writing a slice is much faster than a channel).
Showing only the different / changed parts, it could look something like this:
var (
buf = make([]int, 0, 100)
)
func main() {
// ...
for {
select {
case v := <-ch:
buf = append(buf, v)
if len(buf) == cap(buf) {
// ...
}
}
func doWrite() {
for _, v := range buf {
fmt.Fprintf(f, "%d ", v) // Write v to the file
}
buf = buf[:0] // "Clear" the buffer
}
With multiple goroutines
If we stick to leave chInternal a channel, the doWrite() function may be called on another goroutine to not block the other one, e.g. go doWrite(). Since data to write is read from a channel (chInternal), this requires no further synchronization.
if you just use 5 seconds write, to increase the file write performance,
you may fill the channel any time you need,
then writer goroutine writes that data to the buffered file,
see this very simple and idiomatic sample without using timer
with just using for...range:
package main
import (
"bufio"
"fmt"
"os"
"sync"
)
var wg sync.WaitGroup
func WriteToFile(filename string, ch chan int) {
f, e := os.Create(filename)
if e != nil {
panic(e)
}
w := bufio.NewWriterSize(f, 4*1024*1024)
defer wg.Done()
defer f.Close()
defer w.Flush()
for v := range ch {
fmt.Fprintf(w, "%d ", v)
}
}
func main() {
ch := make(chan int, 100)
wg.Add(1)
go WriteToFile("file.txt", ch)
for i := 0; i < 500000; i++ {
ch <- i // do the job
}
close(ch) // Finish the job and close output file
wg.Wait()
}
and notice the defers order.
and in case of 5 seconds write, you may add one interval timer just to flush the buffer of this file to the disk, like this:
package main
import (
"bufio"
"fmt"
"os"
"sync"
"time"
)
var wg sync.WaitGroup
func WriteToFile(filename string, ch chan int) {
f, e := os.Create(filename)
if e != nil {
panic(e)
}
w := bufio.NewWriterSize(f, 4*1024*1024)
ticker := time.NewTicker(5 * time.Second)
quit := make(chan struct{})
go func() {
for {
select {
case <-ticker.C:
if w.Buffered() > 0 {
fmt.Println(w.Buffered())
w.Flush()
}
case <-quit:
ticker.Stop()
return
}
}
}()
defer wg.Done()
defer f.Close()
defer w.Flush()
defer close(quit)
for v := range ch {
fmt.Fprintf(w, "%d ", v)
}
}
func main() {
ch := make(chan int, 100)
wg.Add(1)
go WriteToFile("file.txt", ch)
for i := 0; i < 25; i++ {
ch <- i // do the job
time.Sleep(500 * time.Millisecond)
}
close(ch) // Finish the job and close output file
wg.Wait()
}
here I used time.NewTicker(5 * time.Second) for interval timer with quit channel, you may use time.AfterFunc() or time.Tick() or time.Sleep().
with some optimizations ( removing quit channel):
package main
import (
"bufio"
"fmt"
"os"
"sync"
"time"
)
var wg sync.WaitGroup
func WriteToFile(filename string, ch chan int) {
f, e := os.Create(filename)
if e != nil {
panic(e)
}
w := bufio.NewWriterSize(f, 4*1024*1024)
ticker := time.NewTicker(5 * time.Second)
defer wg.Done()
defer f.Close()
defer w.Flush()
for {
select {
case v, ok := <-ch:
if ok {
fmt.Fprintf(w, "%d ", v)
} else {
fmt.Println("done.")
ticker.Stop()
return
}
case <-ticker.C:
if w.Buffered() > 0 {
fmt.Println(w.Buffered())
w.Flush()
}
}
}
}
func main() {
ch := make(chan int, 100)
wg.Add(1)
go WriteToFile("file.txt", ch)
for i := 0; i < 25; i++ {
ch <- i // do the job
time.Sleep(500 * time.Millisecond)
}
close(ch) // Finish the job and close output file
wg.Wait()
}
I hope this helps.