tcp reset connection by peer - go

I have a problem launching multiple go routines over 90000 to do http requests via POST to the same server, both the server and the client are locally, some of the requests are executed successfully and some of them giving me this response read: connection reset by peer
Please notice I am posting a small amount of data in the provided example
but actually I am sending a huge amount of data using protobuf.
This is the server https://play.golang.org/p/r1-rYNuAos
package main
import (
"net/http"
"log"
"encoding/json"
)
var port string
type problem struct{
}
func main() {
p := &problem{}
p.server(":9090")
}
func (self *problem)server(port string) {
s := &http.Server{
Addr: port,
Handler: self,
}
log.Println("Server started")
// Should be last line as it is a blocking.
log.Fatal(s.ListenAndServe())
}
func (self *problem) ServeHTTP(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusCreated)
if err := json.NewEncoder(w).Encode(map[string]interface{}{"created": true}); err != nil {
log.Fatal(err.Error())
}
}
And this is the client https://play.golang.org/p/Xx5qQoqrYm
package main
import (
"net/http"
"bytes"
"io/ioutil"
"log"
"fmt"
)
type problem struct{
}
func main() {
p := &problem{}
p.client(":9090")
}
func (self *problem)client(port string) {
var k int
for k=0;k<90000;k++ {
go func(){
nativeRequest, err := http.NewRequest(
"POST",
"http://127.0.0.1" + port + "/",
bytes.NewBuffer([]byte(`testing`)),
)
nativeRequest.Close = true
if err != nil {
fmt.Println(err.Error())
}
client := &http.Transport{
}
nativeResponse, err := client.RoundTrip(nativeRequest)
if err != nil {
log.Println(err.Error())
}
if nativeResponse != nil {
defer nativeResponse.Body.Close()
if err != nil {
fmt.Println(err.Error())
}
body, err := ioutil.ReadAll(nativeResponse.Body)
if err != nil {
fmt.Println(err.Error())
}
fmt.Println(string(body))
}
}()
}
}

Related

I only have *.pem and *.key files, how to make sure the http client is not attacked

I only got the two files *.pem and *.key through the certificate authority.
I am worried about man-in-the-middle attacks, which will replace the certificate and deceive the client, so I wrote the following code, the client directly reads the *.pem file and compares the certificate obtained by http to ensure that the connection is correct when they are the same.
I would like to know if my solution is correct and if there is a better solution to my problem?
server code
package main
import (
"fmt"
"net/http"
"time"
)
func main() {
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
_, _ = fmt.Fprintf(w, "hello %s", time.Now().String())
})
err := http.ListenAndServeTLS(":443", `xx.pem`, `xx.key`, nil)
if err != nil {
panic(err)
}
}
client code
package main
import (
"bytes"
"crypto/tls"
"crypto/x509"
"encoding/pem"
"errors"
"fmt"
"io"
"net/http"
"os"
)
func main() {
err := getPem()
if err != nil {
panic(err)
}
err = httpGet()
if err != nil {
panic(err)
}
}
var (
pemRawCerts [][]byte // xx.pem读取的原始数据
errCheckPem = errors.New("check xx.pem error")
)
func getPem() error {
certPEMBlock, err := os.ReadFile(`xxx.pem`)
if err != nil {
return err
}
var certDERBlock *pem.Block
for {
certDERBlock, certPEMBlock = pem.Decode(certPEMBlock)
if certDERBlock == nil {
break
}
if certDERBlock.Type == "CERTIFICATE" {
tmp := make([]byte, len(certDERBlock.Bytes))
copy(tmp, certDERBlock.Bytes)
pemRawCerts = append(pemRawCerts, tmp)
}
}
return nil
}
func httpGet() error {
c := http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
VerifyPeerCertificate: func(data [][]byte, verifiedChains [][]*x509.Certificate) error {
if len(data) == len(pemRawCerts) {
for i := len(data) - 1; i >= 0; i-- {
if !bytes.Equal(data[i], pemRawCerts[i]) {
return errCheckPem
}
}
// Only when every item of the certificate is correct can the current connection be determined to be ok
return nil
}
return errCheckPem
},
},
},
}
resp, err := c.Get("https://janbar.com")
if err != nil {
return err
}
defer resp.Body.Close()
n, err := io.Copy(io.Discard, resp.Body)
fmt.Println(n)
return err
}

Golang ServeHTTP make proxy error EOF on POST request

I’m working on proxy server with gin and ServeHTTP. 
Actually GET and OPTIONS request works well. But when I trying multiples POST request I get EOF error one in two request.
I’ve test to make repeat request without proxy service and its work well, so there is something not working in my code.
Edit :
I have test POST proxy with https://ptsv2.com/ and all request response return 200 status code.
// main.go
package main
import (
"fmt"
"../pkg/api"
)
func main() {
fmt.Println("Starting server")
api.InitServer()
}
// routes.go
package api
import (
"github.com/gin-gonic/gin"
)
const serviceUrl = "http://localhost:8732"
func InitServer() {
router := gin.Default()
defineRoutes(router)
router.Run()
}
func defineRoutes(router *gin.Engine) {
router.GET("/ping", Ping)
router.POST("/*any", Proxy(serviceUrl))
}
// controller.go
package api
import (
"bytes"
"fmt"
"io/ioutil"
"net/http"
"net/http/httputil"
"net/url"
"strconv"
"github.com/gin-gonic/gin"
)
type transport struct {
http.RoundTripper
}
func (t *transport) RoundTrip(req *http.Request) (resp *http.Response, err error) {
resp, err = t.RoundTripper.RoundTrip(req)
if err != nil {
// EOF ERROR HERE
return nil, err
}
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
err = resp.Body.Close()
if err != nil {
return nil, err
}
b = bytes.Replace(b, []byte("server"), []byte("schmerver"), -1)
body := ioutil.NopCloser(bytes.NewReader(b))
response := resp
response.Body = body
response.ContentLength = int64(len(b))
response.Header.Set("Content-Length", strconv.Itoa(len(b)))
response.Header.Set("Access-Control-Allow-Origin", "*")
return response, nil
}
func Proxy(targetUrl string) gin.HandlerFunc {
fn := func(c *gin.Context) {
remote, err := url.Parse(targetUrl)
if err != nil {
panic(err)
}
proxy := httputil.NewSingleHostReverseProxy(remote)
proxy.Director = func(req *http.Request) {
req.Header = c.Request.Header
req.Host = remote.Host
req.URL.Scheme = remote.Scheme
req.URL.Host = remote.Host
req.Close = true // True / False same result
}
proxy.Transport = &transport{http.DefaultTransport}
proxy.ServeHTTP(c.Writer, c.Request)
}
return fn
}
conclusion
your code have no bug. it works. maybe your network setting is wrong.
explain
I download your code and test it with a local backend server. It works.
appendix
backend server code
package main
import (
"fmt"
"github.com/gin-gonic/gin"
)
func main() {
fmt.Println("Starting server")
router := gin.Default()
router.POST("/*any", func(c *gin.Context) {
c.JSON(200, gin.H{
"mes": "hello",
})
})
router.Run(":8888")
}

Web server and listening nats at the same time

My code reads input from terminal and send those value to nats while it needs to have an http endpoint.
Separately it works but when I combine all of them it does not read from nats. If you could point me to a right direction I would appreciate.
package main
import (
"bufio"
"fmt"
nats "github.com/nats-io/nats.go"
"html/template"
"log"
"net/http"
"os"
)
func main() {
wd, err := os.Getwd()
if err != nil {
log.Fatal(err)
}
tmpl := template.Must(template.ParseFiles(wd + "/template/main.html"))
http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
data := TodoPageData{
PageTitle: "Demo",
}
tmpl.Execute(w, data)
})
http.ListenAndServe(":8081", nil)
type message struct {
content string
}
var messages []message
nc, err := nats.Connect(
nats.DefaultURL,
)
if err != nil {
log.Fatal(err)
}
defer nc.Close()
// Subscribe
if _, err := nc.Subscribe("updates", func(m *nats.Msg) {
fmt.Printf("Received a message: %s\n", string(m.Data))
}); err != nil {
log.Fatal(err)
}
// io r/w
scanner := bufio.NewScanner(os.Stdin)
for scanner.Scan() {
if err := nc.Publish("updates", []byte(scanner.Text())); err != nil {
log.Fatal(err)
}
messages = append(messages, message{scanner.Text()})
for _, message := range messages {
fmt.Println(message.content)
}
}
if scanner.Err() != nil {
// handle error.
}
}
http.ListenAndServe is a blocking call. Start it on a new goroutine:
go http.ListenAndServe(":8081", nil)

Poll API, pass result to chan, pass from chan to Websocket. Panic

I'm writing a small package which does a GET request to an external API every 2 seconds. It takes the value from this request and passes it into a channel. I have made this channel available to a http.handler (chi router) which upgrades to a websocket where the front-end will grab the value in realtime. the panic error is a lot of lines but i guess the most important is this:
2018/11/14 16:47:55 http: response.WriteHeader on hijacked connection
2018/11/14 16:47:55 http: response.Write on hijacked connection
Aside from that I'm sure there is a better way of doing this. Any experienced Gophers out there have any pointers to help a noob such as myself improve this?
package currencyticker
import (
"bitbucket.org/special/api/config"
"encoding/json"
"fmt"
"github.com/go-chi/chi"
"github.com/go-chi/render"
"github.com/gorilla/websocket"
"github.com/leekchan/accounting"
"io/ioutil"
"log"
"math/big"
"net/http"
"time"
)
var (
ac = accounting.Accounting{Precision: 2}
from = "USD"
to = "EUR,SWK"
url = "https://min-api.currencyapi.com/data/price?fsym=" + from + "&tsyms=" + to
messages = make(chan float64)
)
var wsupgrader = websocket.Upgrader{
ReadBufferSize: 1024,
WriteBufferSize: 1024,
CheckOrigin: func(r *http.Request) bool {
return true // Disable CORS for testing
},
}
// Config - init
type Config struct {
*config.Config
}
type result map[string]float64
// New - init the configs
func New(configuration *config.Config) *Config {
return &Config{configuration}
}
// Routes - api urls
func (config *Config) Routes() *chi.Mux {
router := chi.NewRouter()
router.Use(
render.SetContentType(render.ContentTypeHTML), // Set content-Type headers as application/json
)
router.Get("/", config.GetPrice) // subscribe to new tweets
return router
}
func (config *Config) GetPrice(w http.ResponseWriter, r *http.Request) {
conn, err := wsupgrader.Upgrade(w, r, nil)
if err != nil {
fmt.Println(fmt.Printf("Failed to set websocket upgrade: %+v ", err))
return
}
for {
time.Sleep(1 * time.Second)
price := <-messages
w, err := conn.NextWriter(websocket.TextMessage)
if err != nil {
fmt.Println("ws error", err)
}
currVal := ac.FormatMoneyBigFloat(big.NewFloat(price))
if _, err := w.Write([]byte(currVal)); err != nil {
fmt.Printf("w.Write() returned %v", err)
}
w.Close()
}
}
// start getting the price of ether as soon as they ap starts
func init() {
go startPollingPriceAPI()
}
// Go Routine to start polling
func startPollingPriceAPI() {
for {
time.Sleep(2 * time.Second)
go getPriceFromAPI()
}
}
func getPriceFromAPI() {
w := http.Client{
// Timeout: time.Second * 3,
}
req, _ := http.NewRequest(http.MethodGet, url, nil)
res, err := w.Do(req)
if err != nil {
log.Println("err getting price [req]: ", err)
}
body, err := ioutil.ReadAll(res.Body)
if err != nil {
log.Println("err getting price [io-read]: ", err)
}
r := result{}
if jsonErr := json.Unmarshal(body, &r); jsonErr != nil {
log.Println("err getting price [json]: ", jsonErr)
}
fmt.Println("1 Dollar = €", r["EUR"])
messages <- r["EUR"]
}

TCP client not working correcty golang

I have rest-ful interface written in golang. I need to do the authentication on each endpoint. once authentication is done I have to forward it back to a tcp server.
I created a tcp client and any value that is coming from the channel is to be send to tcp server. the channel is populated from the http request body.
The issue is that once I issue curl command the client is stuck with no response; so obviously I am doing something wrong not sure what is wrong. does anyone have any insights on what my problem might be?
package main
import (
"bufio"
"encoding/json"
"flag"
"fmt"
"io/ioutil"
"log"
"net"
"net/http"
"net/http/httputil"
"net/url"
"os"
"strconv"
auth "github.com/abbot/go-http-auth"
)
type Configuration struct {
Server string
Port int64
UserName string
Pwd string
Realm string
ProxyPort int64
Edeserver string
}
var (
Config *Configuration
logp = flag.Bool("log", false, "enable logging")
)
func ReadConfiguration() {
file, _ := os.Open("Config.json")
decoder := json.NewDecoder(file)
Config = &Configuration{}
err := decoder.Decode(&Config)
if err != nil {
fmt.Println("error:", err)
}
}
func Secret(user, realm string) string {
if user == Config.UserName {
// password is "hello"
return Config.Pwd
}
return ""
}
func reverseProxyTows(w http.ResponseWriter, authenticatedRequest *auth.AuthenticatedRequest) {
req := &authenticatedRequest.Request
if *logp {
log.Println(" Authenticated Username ", authenticatedRequest.Username)
log.Println(" Authenticated URL ", req.URL.RequestURI())
}
destinationURL := fmt.Sprintf("http://%s:%d", Config.Server, Config.Port)
u, err := url.Parse(destinationURL)
if err != nil {
log.Fatal(err)
}
if *logp {
log.Println("reverse_proxy", u)
}
reverseProxy := httputil.NewSingleHostReverseProxy(u)
reverseProxy.ServeHTTP(w, req)
}
func openConnectionTotcp(edechannel chan string) {
conn, _ := net.Dial("tcp", Config.Edeserver)
text := <-edechannel
fmt.Fprintf(conn, text+"\n")
message, _ := bufio.NewReader(conn).ReadString('\n')
fmt.Print("Message from server: " + message)
}
func main() {
ReadConfiguration()
flag.Parse()
c := make(chan string)
go openConnectionTotcp(c)
fmt.Printf("Started proxy to destination server %v:%d and is listening on %d ", Config.Server, Config.Port, Config.ProxyPort)
authenticator := auth.NewBasicAuthenticator(Config.Realm, Secret)
http.HandleFunc("/", authenticator.Wrap(reverseProxyTows))
http.HandleFunc("/tyrion/1", authenticator.Wrap(func(w http.ResponseWriter, authenticatedRequest *auth.AuthenticatedRequest) {
req := &authenticatedRequest.Request
bodyBytes, err2 := ioutil.ReadAll(req.Body)
if err2 != nil {
log.Fatal(err2)
}
bodyString := string(bodyBytes)
c <- bodyString
fmt.Fprintf(w, "success")
}))
http.ListenAndServe(":"+strconv.FormatInt(Config.ProxyPort, 10), nil)
}
Your code execution blocks at c <- bodyString because nothing appears to be reading from that unbuffered channel. That line will pause execution until another routine reads from the channel.

Resources