How to use TLS Conn.Read to read a large size of data - go

I'm trying with tls.Conn i.e, conn.Read from golang crypto/tls to read a large size of data. However when all the data is read out, then the program will never stop. Why? My simplified code is as follow:
// server
func main() {
log.SetFlags(log.Lshortfile)
cer, err := tls.LoadX509KeyPair("server.pem", "server.key")
if err != nil {
log.Println(err)
return
}
config := &tls.Config{Certificates: []tls.Certificate{cer}}
ln, err := tls.Listen("tcp", ":2000", config)
if err != nil {
log.Println(err)
return
}
defer ln.Close()
for {
conn, err := ln.Accept()
if err != nil {
log.Println(err)
continue
}
go handleConnection(conn)
}
}
func handleConnection(conn net.Conn) {
defer conn.Close()
buf := make([]byte, 0)
tmp := make([]byte, 1000)
totalSize := 0
for {
n, err := conn.Read(tmp) //after read all the data, server stucked here
totalSize += n
if err != nil {
if err != io.EOF {
log.Printf("prover: conn: read: %s", err)
}
break
}
buf = append(buf, tmp[:n]...)
}
n, err := conn.Write([]byte("finished"))
if err != nil {
log.Println(n, err)
return
}
}
The logic of the client:
// client
func main() {
log.SetFlags(log.Lshortfile)
conf := &tls.Config{
InsecureSkipVerify: true,
}
conn, err := tls.Dial("tcp", "127.0.0.1:2000", conf)
if err != nil {
log.Println(err)
return
}
defer conn.Close()
writeData := make([]byte, 4096)
n, err := conn.Write(writeData)
if err != nil {
log.Println(n, err)
return
}
fmt.Println("finish writing")
buf := make([]byte, 4096)
n, err = conn.Read(buf)
if err != nil {
log.Println(n, err)
}
fmt.Print("finish reading")
}
How can I read a large size of data in the server and after the data is completely read out, then the server will send a response to the client. I tried with net.Conn (TCPConn), this logic works. WHy?

Related

Can we send request form server to client and get response through quic/http3?

I'm using quic-go to implement my thought, I need the server to forwardly send request to client to get response, just like we do that client sends request to web server commonly. But with quic-go, after connection is setup, can server initialize streams to send request to client and get responses? I did a trying but haven't made it work. The code below is from the echo.go of example dir, the two parts between comment lines are added by me.
package main
import (
"context"
"crypto/rand"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"fmt"
"io"
"log"
"math/big"
"github.com/lucas-clemente/quic-go"
)
const addr = "localhost:4242"
const message = "foobar"
// We start a server echoing data on the first stream the client opens,
// then connect with a client, send the message, and wait for its receipt.
func main() {
go func() { log.Fatal(echoServer()) }()
err := clientMain()
if err != nil {
panic(err)
}
}
// Start a server that echos all data on the first stream opened by the client
func echoServer() error {
listener, err := quic.ListenAddr(addr, generateTLSConfig(), nil)
if err != nil {
return err
}
conn, err := listener.Accept(context.Background())
if err != nil {
return err
}
stream, err := conn.AcceptStream(context.Background())
if err != nil {
panic(err)
}
// Echo through the loggingWriter
_, err = io.Copy(loggingWriter{stream}, stream)
if err != nil {
panic(err)
}
//------------------------------
stream1, err := conn.OpenStream()
if err != nil {
panic(err)
}
message := "aaaaa"
fmt.Printf("2-Server: Sending '%s'\n", message)
_, err = stream1.Write([]byte(message))
//------------------------------
return err
}
func clientMain() error {
tlsConf := &tls.Config{
InsecureSkipVerify: true,
NextProtos: []string{"quic-echo-example"},
}
conn, err := quic.DialAddr(addr, tlsConf, nil)
if err != nil {
return err
}
stream, err := conn.OpenStreamSync(context.Background())
if err != nil {
return err
}
fmt.Printf("Client: Sending '%s'\n", message)
_, err = stream.Write([]byte(message))
if err != nil {
return err
}
buf := make([]byte, len(message))
_, err = io.ReadFull(stream, buf)
if err != nil {
return err
}
fmt.Printf("Client: Got '%s'\n", buf)
err = stream.Close()
if err != nil {
return err
}
//-------------------------------
for {
stream1, err := conn.AcceptStream(context.Background())
if err != nil {
panic(err)
}
buf1 := make([]byte, len(message))
_, err = io.ReadFull(stream1, buf1)
if err != nil {
panic(err)
}
fmt.Printf("2-Client: Got '%s'\n", buf1)
err = stream1.Close()
if err != nil {
panic(err)
}
}
//-------------------------------
return nil
}
// A wrapper for io.Writer that also logs the message.
type loggingWriter struct{ io.Writer }
func (w loggingWriter) Write(b []byte) (int, error) {
fmt.Printf("Server: Got '%s'\n", string(b))
return w.Writer.Write(b)
}
// Setup a bare-bones TLS config for the server
func generateTLSConfig() *tls.Config {
key, err := rsa.GenerateKey(rand.Reader, 1024)
if err != nil {
panic(err)
}
template := x509.Certificate{SerialNumber: big.NewInt(1)}
certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
if err != nil {
panic(err)
}
keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
if err != nil {
panic(err)
}
return &tls.Config{
Certificates: []tls.Certificate{tlsCert},
NextProtos: []string{"quic-echo-example"},
}
}
You are close.
Notice how the server writes, then returns, it reaches out to log.Fatal before the client had a chance to read and handle the data.
Take care to the message variable, one is 6 length long, the other one is only 5 length long.
Properly close your stream, the server was not ending it ending before handing over;
package main
import (
"context"
"crypto/rand"
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"fmt"
"io"
"log"
"math/big"
"github.com/lucas-clemente/quic-go"
)
const addr = "localhost:4242"
const message = "foobar"
// We start a server echoing data on the first stream the client opens,
// then connect with a client, send the message, and wait for its receipt.
func main() {
go func() {
err := echoServer()
if err != nil {
log.Println(err)
}
}()
err := clientMain()
if err != nil {
panic(err)
}
}
// Start a server that echos all data on the first stream opened by the client
func echoServer() error {
listener, err := quic.ListenAddr(addr, generateTLSConfig(), nil)
if err != nil {
return err
}
conn, err := listener.Accept(context.Background())
if err != nil {
return err
}
stream, err := conn.AcceptStream(context.Background())
if err != nil {
panic(err)
}
// Echo through the loggingWriter
_, err = io.Copy(loggingWriter{stream}, stream)
if err != nil {
log.Println(err)
return err
}
//------------------------------
fmt.Printf("Server: open stream\n")
stream1, err := conn.OpenStream()
if err != nil {
return err
}
fmt.Printf("2-Server: Sending '%s'\n", message)
var n int
n, err = stream1.Write([]byte(message))
fmt.Printf("Server: write %v %v\n", n, err)
stream1.Close()
//------------------------------
return err
}
func clientMain() error {
tlsConf := &tls.Config{
InsecureSkipVerify: true,
NextProtos: []string{"quic-echo-example"},
}
conn, err := quic.DialAddr(addr, tlsConf, nil)
if err != nil {
return err
}
stream, err := conn.OpenStreamSync(context.Background())
if err != nil {
return err
}
fmt.Printf("Client: Sending '%s'\n", message)
_, err = stream.Write([]byte(message))
if err != nil {
return err
}
buf := make([]byte, len(message))
_, err = io.ReadFull(stream, buf)
if err != nil {
return err
}
fmt.Printf("Client: Got '%s'\n", buf)
err = stream.Close()
if err != nil {
return err
}
//-------------------------------
for {
fmt.Printf("Client: accept stream\n")
stream1, err := conn.AcceptStream(context.Background())
if err != nil {
return err
}
fmt.Printf("Client: got stream %v\n", err)
buf1 := make([]byte, len(message))
_, err = io.ReadFull(stream1, buf1)
if err != nil {
return err
}
fmt.Printf("2-Client: Got '%s'\n", buf1)
err = stream1.Close()
if err != nil {
return err
}
break
}
//-------------------------------
return nil
}
// A wrapper for io.Writer that also logs the message.
type loggingWriter struct{ io.Writer }
func (w loggingWriter) Write(b []byte) (int, error) {
fmt.Printf("Server: Got '%s'\n", string(b))
return w.Writer.Write(b)
}
// Setup a bare-bones TLS config for the server
func generateTLSConfig() *tls.Config {
key, err := rsa.GenerateKey(rand.Reader, 1024)
if err != nil {
panic(err)
}
template := x509.Certificate{SerialNumber: big.NewInt(1)}
certDER, err := x509.CreateCertificate(rand.Reader, &template, &template, &key.PublicKey, key)
if err != nil {
panic(err)
}
keyPEM := pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)})
certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: certDER})
tlsCert, err := tls.X509KeyPair(certPEM, keyPEM)
if err != nil {
panic(err)
}
return &tls.Config{
Certificates: []tls.Certificate{tlsCert},
NextProtos: []string{"quic-echo-example"},
}
}

How do I make it such that a TCP connection will timeout if the connection doesn't receive a response every second?

I'm trying to create a TCP server that will timeout if the client does not respond within the span of every second.
I tried:
func main() {
listener, err := net.Listen("tcp", "localhost:8000")
if err != nil {
log.Fatal(err)
}
for {
conn, err := listener.Accept()
conn.SetDeadline(time.Now().Add(timeout))
if err != nil {
log.Print(err)
}
go handleConn(conn)
}
}
where the timeout is a single second but the disconnects immediately, not even waiting for a reply.
What you want can be achieved by setting socket options on your listener. Tweak the values as per your needs
Note that this is its own KeepAlive and does not depend on incoming/outgoing data by application
func enableTCPKeepAlive(listener *net.TCPListener) error {
rawConn, err := listener.SyscallConn()
if err != nil {
return err
}
cfg := config.TLSServerConfig()
rawConn.Control(
func(fdPtr uintptr) {
// got socket file descriptor. Setting parameters.
fd := int(fdPtr)
//Idle time before sending probe.
err = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPIDLE, cfg.TCPKAIdleTime)
if err != nil {
return err
}
//Number of probes.
err := syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPCNT, cfg.TCPKANumProbes)
if err != nil {
return err
}
//Wait time after an unsuccessful probe.
err = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, cfg.TCPKAInterval)
if err != nil {
return err
}
// go syscall doesn't have the constant 0x12 (18) for TCP_USER_TIMEOUT.
// 0x12 value referenced from linux kernel source code header:
// include/uapi/linux/tcp.h
err = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, 0x12, cfg.TCPKAUserTimeout)
if err != nil {
return err
}
})
return nil
}
There are more options available than the ones I have mentioned above.
Call this function on your listener before the for loop.
func main() {
listener, err := net.Listen("tcp", "localhost:8000")
if err != nil {
log.Fatal(err)
}
err = enableTCPKeepAlive(listener)
if err != nil {
log.Fatal(err)
}
for {
conn, err := listener.Accept()
conn.SetDeadline(time.Now().Add(timeout))
if err != nil {
log.Print(err)
}
go handleConn(conn)
}
}
The problem is almost always in code that is not posted here. The function obviously works like a charme:
package main
import (
"crypto/rand"
"log"
"net"
"time"
)
func main() {
listener, err := net.Listen("tcp", "localhost:8000")
if err != nil {
log.Fatal(err)
}
go func() {
for {
conn, err := listener.Accept()
if err != nil {
log.Print(err)
return
}
go func(c net.Conn) {
defer c.Close()
conn.SetDeadline(time.Now().Add(2 * time.Second))
if err != nil {
log.Print(err)
return
}
buf := make([]byte, 1<<19) // 512 KB
for {
_, err := conn.Read(buf)
if err != nil {
log.Print(err)
break
}
}
}(conn)
}
}()
payload := make([]byte, 1<<20)
_, err = rand.Read(payload) // generate a random payload
if err != nil {
log.Print(err)
}
conn, err := net.Dial("tcp", listener.Addr().String())
if err != nil {
log.Fatal(err)
}
log.Println("Connected to server.")
time.Sleep(5 * time.Second)
_, err = conn.Write(payload)
if err != nil {
log.Print(err)
}
listener.Close()
}

Post multi-part form data for audio file

I'm wrote a function that posts multipart form data to an endpoint but doesn't seem to be working properly. Here's the code
func Upload(filePath string) error {
client := &http.Client{
Timeout: time.Second * 10,
}
// New multipart writer.
body := &bytes.Buffer{}
file, err := os.Open(filePath)
if err != nil {
return err
}
writer := multipart.NewWriter(body)
fw, err := writer.CreateFormFile("file", filepath.Base(filePath))
if err != nil {
return err
}
_, err = io.Copy(fw, bufio.NewReader(file))
if err != nil {
return err
}
// Close multipart writer.
writer.Close()
req, err := http.NewRequest("POST", "http://localhost:5050/upload", bytes.NewReader(body.Bytes()))
if err != nil {
return err
}
req.Header.Set("Content-Type", writer.FormDataContentType())
rsp, _ := client.Do(req)
if rsp == nil {
return fmt.Errorf("failed to upload")
}
if rsp.StatusCode != http.StatusOK {
fmt.Printf("Request failed with response code: %d", rsp.StatusCode)
}
fmt.Println("rsp: ", rsp.StatusCode)
return nil
}
The handler on the API receiving the POST is this:
func (m *MetadataService) uploadHandler(res http.ResponseWriter, req *http.Request) {
file, handler, err := req.FormFile("file")
if err != nil {
panic(err) //dont do this
}
defer file.Close()
fmt.Println("req: ", req)
// Create a buffer to store the header of the file in
fileHeader := make([]byte, 512)
// Copy the headers into the FileHeader buffer
if _, err := file.Read(fileHeader); err != nil {
panic(err) //dont do this
}
// set position back to start.
if _, err := file.Seek(0, 0); err != nil {
panic(err) //dont do this
}
// copy example
f, err := os.OpenFile(handler.Filename, os.O_WRONLY|os.O_CREATE, 0666)
if err != nil {
panic(err) //please dont
}
defer f.Close()
buf := bytes.NewBuffer(nil)
if _, err := io.Copy(buf, file); err != nil {
panic(err)
}
fmt.Println("filename:", handler.Filename)
fmt.Println("file.(Sizer).Size():", file.(Sizer).Size())
fmt.Println("contentType:", http.DetectContentType(fileHeader))
io.WriteString(res, id)
}
The code panics when reading the fileHeader thought. The content type when sent is "application/octet-stream" when I'd expect it to be "audio/mpeg" since i'm uploading a song. Not sure what's wrong with the aforementioned Upload function. Need help! Thank you!

How to close client socket in udp

I have a simple UDP server all that I want is if the IP is equal to 1.1.1.1 for example the client socket will close its like a blacklist system
My code
func main() {
arguments := os.Args
if len(arguments) == 1 {
fmt.Println("Please provide a port number!")
return
}
PORT := ":" + arguments[1]
s, err := net.ResolveUDPAddr("udp4", PORT)
if err != nil {
fmt.Println(err)
return
}
connection, err := net.ListenUDP("udp4", s)
if err != nil {
fmt.Println(err)
return
}
defer connection.Close()
buffer := make([]byte, 1024)
for {
n, addr, err := connection.ReadFromUDP(buffer)
fmt.Print("-> ", string(buffer[0:n-1]))
data := []byte(strconv.Itoa(random(1, 1001)))
fmt.Printf("data: %s\n", string(data))
_, err = connection.WriteToUDP(data, addr)
if err != nil {
fmt.Println(err)
return
}
}
}

Write multiple time on TCP conn

I have a TCP server server :
l, err := net.Listen("tcp", "localhost:"+strconv.Itoa(tcpPort))
The server listens to incoming client requests as is :
for {
c, err := l.Accept()
if err != nil {
log.Fatal(err)
}
b := make([]byte, 1024)
c.Read(b)
fmt.Println(string(b)) // "Hello"
}
I have a client :
conn, err := net.Dial("tcp", address)
Now I I write once with conn.Write([]byte("Hello"))
The server catches Hello. But if I have these :
_, err := conn.Write([]byte("Hello"))
if err != nil {
log.Fatal(err)
}
_, err = conn.Write([]byte("World"))
if err != nil {
log.Fatal(err)
}
Then I will get Hello, but not World.
How can I write multiple time on the same connection ?
Full function below
func main() {
l, err := net.Listen("tcp", "localhost:1234")
if err != nil {
log.Fatal(err)
}
defer l.Close()
go func() {
for {
c, err := l.Accept()
if err != nil {
log.Fatal(err)
}
b := make([]byte, 1024)
c.Read(b)
fmt.Println(string(b))
}
}()
conn, err := net.Dial("tcp", "localhost:1234")
if err != nil {
log.Fatal(err)
}
_, err = conn.Write([]byte("hello"))
if err != nil {
log.Fatal(err)
}
_, err = conn.Write([]byte("world"))
if err != nil {
log.Fatal(err)
}
}
Your problem is in the server code, the one that receives data. It only reads once from the tcp stream. If you want it to read "world", replace the single read operation by a loop:
go func() {
for {
c, err := l.Accept()
if err != nil {
log.Fatal(err)
}
for {
b := make([]byte, 1024)
c.Read(b)
fmt.Println(string(b))
}
}
}()
Your problem is not with conn.Write but with reading from connection. Now you read just once from every new opened connection accepted by l.Accept(). Solution is to read repeatedly.
Your code is also limited to handle just one connection. And do not forget to check error from c.Read(b) to know when stop listen on this connection.
go func() {
for {
c, err := l.Accept()
if err != nil {
log.Fatal(err)
}
go func(conn net.Conn) {
for {
b := make([]byte, 1024)
_, err := conn.Read(b)
if err != nil {
if err != io.EOF {
fmt.Println("read error:", err)
}
break
}
fmt.Println(string(b))
}
fmt.Println("Stopping handle connection")
}(c)
}
}()

Resources