I am trying to do the following.
|Upload file in HTML post file form|
|
⌄
|Server A forwards the multipart request|
|
⌄
|Server B receives and stores the file from the forwarded multipart request|
|
⌄
|Server A receives response from Server B when Server B is done|
Processing the multipart request on Server A is straightforward, but when I try to process the forwarded request on Server B it fails with multipart: NextPart: EOF.
I am trying to create separate frontend/backend services. Frontend only handles UI related processing, while backend will actually do some processing on the file, hence the multipart request forwarding needed.
The forwarding code on Server A is as follows.
The solution has been taken from here.
https://stackoverflow.com/a/34725635/6569715
func forwardRequest(address string, path string, r *http.Request) (interface{}, error) {
body, err := ioutil.ReadAll(r.Body)
if err != nil {
return nil, err
}
r.Body = ioutil.NopCloser(bytes.NewReader(body))
proxyReq, err := http.NewRequest(r.Method, fmt.Sprintf("%s%s", address, path), bytes.NewReader(body))
if err != nil {
return nil, err
}
for header, values := range r.Header {
for _, value := range values {
proxyReq.Header.Add(header, value)
}
}
client := &http.Client{}
resp, err := client.Do(proxyReq)
if err != nil {
return nil, err
}
defer resp.Body.Close()
return resp, nil
}
And the code on Server B to process the forwarded request:
func testMultiPart(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(10 << 20); err != nil {
err = errors.Wrap(errors.WithStack(err), "Backend: Failed to parse form")
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprint(w, fmt.Sprintf("{\"error\":\"%s\"}", err.Error())
return
}
}
Any help is appreciated.
I managed to make it work. I believe it was just my own mistake not filling in the URI properly. In any case I will post my snippets from my solution for future reference.
The client html file form part:
<form action="/test-main/file-test" enctype="multipart/form-data" method="post">
<label for="file-upload">Upload your file :</label>
<input type="file" id="file-upload" name="file-upload" accept="image/*">
</form>
Server A code:
import (
"net/http"
"errors"
"fmt"
"log"
"io/ioutil"
"bytes"
"github.com/gorilla/mux"
)
func fileUpload(w http.ResponseWriter, r *http.Request) {
body, err := ioutil.ReadAll(r.Body)
if err != nil {
return log.Fatal(err)
}
r.Body = ioutil.NopCloser(bytes.NewReader(body))
// If Server A and B are separate docker images, you may need to use their docker subnet IP, like below.
proxyReq, err := http.NewRequest(r.Method, fmt.Sprintf("http://172.18.0.2:8082%s", r.RequestURI), bytes.NewReader(body))
if err != nil {
return log.Fatal(err)
}
for header, values := range r.Header {
for _, value := range values {
proxyReq.Header.Add(header, value)
}
}
client := &http.Client{}
resp, err := client.Do(proxyReq)
if err != nil {
return log.Fatal(err)
}
defer resp.Body.Close()
respBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
return log.Fatal(err)
}
// Process Server B response
// ...
}
func createRouter() *mux.Router {
r := mux.NewRouter()
testPath := r.PathPrefix("/test-main").Subrouter()
testPath.HandleFunc("/file-test", fileUpload)
return r
}
func main() {
// Create Server and Route Handlers
srv := &http.Server{
Handler: createRouter(),
Addr: ":8081",
ReadTimeout: 30 * time.Second,
WriteTimeout: 30 * time.Second,
}
// Start Server
go func() {
log.Println("Starting Server")
if err := srv.ListenAndServe(); err != nil {
log.Fatal(err)
}
}()
}
And Server B code:
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"github.com/gorilla/mux"
)
func uploadFile(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(10 << 20); err != nil {
log.Fatal(err)
}
file, handler, err := r.FormFile("file-upload")
if err == http.ErrMissingFile {
return nil
}
if err != nil {
log.Fatal(err)
}
fmt.Printf("Uploaded File: %+v\n", handler.Filename)
fmt.Printf("File Size: %+v\n", handler.Size)
fmt.Printf("MIME Header: %+v\n", handler.Header)
defer file.Close()
// Create file
dst, err := os.Create(fmt.Sprintf("/some-destination-folder/%s", handler.Filename))
if err != nil {
log.Fatal(err)
}
// Copy the uploaded file to the created file on the file system.
if _, err := io.Copy(dst, file); err != nil {
if err2 := dst.Close(); err2 != nil {
log.Fatal(err)
}
log.Fatal(err)
}
dst.Close()
return nil
}
func (c *Controller) createRouter() *mux.Router {
r := mux.NewRouter()
testPath := r.PathPrefix("/test-main").Subrouter()
testPath.HandleFunc("/file-test", uploadFile)
return r
}
func main() {
// Create Server and Route Handlers
srv := &http.Server{
Handler: createRouter(),
Addr: ":8082",
ReadTimeout: 30 * time.Second,
WriteTimeout: 30 * time.Second,
}
// Start Server
go func() {
log.Println("Starting Server")
if err := srv.ListenAndServe(); err != nil {
log.Fatal(err)
}
}()
}
Good luck for future readers.
Related
I am trying to implement distributed tracing with basic GO client-server app. Using default Jaeger docker-compose all-in-one.
What was done to fix and doesn't help:
Changed collector to agent and agent to collector.
Checked logs, nothing about "client" there
Tried to inject headers (propagation)
Tried without injecting to headers (propagation)
CLIENT CODE:
import(
...
tracesdk "go.opentelemetry.io/otel/sdk/trace"
...
)
func main() {
.....
exporter, err := jaeger.New(
jaeger.WithAgentEndpoint(
jaeger.WithAgentHost("localhost"),
jaeger.WithAgentPort("6831"),
),
)
if err != nil {
return err
}
tp := tracesdk.NewTracerProvider(
tracesdk.WithBatcher(exporter),
tracesdk.WithResource(resource.NewWithAttributes(
semconv.SchemaURL,
semconv.ServiceNameKey.String("client"),
semconv.ServiceVersionKey.String("1.0.0"),
semconv.DeploymentEnvironmentKey.String("local"),
)),
)
defer func() {
if err := tp.Shutdown(context.Background()); err != nil {
log.Fatal(err)
}
}()
otel.SetTracerProvider(tp)
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
propagation.TraceContext{},
propagation.Baggage{},
propagators.Jaeger{},
))
.......
}
....
func fetcherSuccess() error {
tr := otel.Tracer("clientHTTP")
ctx, span := tr.Start(context.Background(), "client.fetcherSuccess")
defer span.End()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/success", nil)
if err != nil {
AddSpanError(span, err)
FailSpan(span, "request error")
return err
}
// Try to inject headers to the context using this otel.GetTextMapPropagator().Inject(ctx, h)
headers := InjectHTTPHeaders(ctx)
for k, v := range headers {
req.Header.Add(k, v)
}
res, _ := http.DefaultClient.Do(req)
return nil
}
SERVER CODE:
import(
...
tracesdk "go.opentelemetry.io/otel/sdk/trace"
...
)
func main() {
.....
exporter, err := jaeger.New(
jaeger.WithAgentEndpoint(
jaeger.WithAgentHost("localhost"),
jaeger.WithAgentPort("6831"),
),
)
if err != nil {
return err
}
tp := tracesdk.NewTracerProvider(
tracesdk.WithBatcher(exporter),
tracesdk.WithResource(resource.NewWithAttributes(
semconv.SchemaURL,
semconv.ServiceNameKey.String("server"),
semconv.ServiceVersionKey.String("1.0.0"),
semconv.DeploymentEnvironmentKey.String("local"),
)),
)
defer func() {
if err := tp.Shutdown(context.Background()); err != nil {
log.Fatal(err)
}
}()
otel.SetTracerProvider(tp)
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
propagation.TraceContext{},
propagation.Baggage{},
propagators.Jaeger{},
))
if err := handleRequests(); err != nil {
panic("unable to create handler")
}
.......
}
func handleRequests() error {
router := http.NewServeMux()
router.HandleFunc("/success", handleSuccess)
router.HandleFunc("/error", handleError)
fmt.Println("Server is listening on port: 8080")
if err := http.ListenAndServe(":8080", router); err != nil {
return err
}
return nil
}
....
func handleSuccess(w http.ResponseWriter, r *http.Request) {
tr := otel.Tracer("serverHTTP")
//Extract headers using otel.GetTextMapPropagator().Extract()
ctx := ExtractHTTPHeaders(r.Context(), r.Header)
ctx, span := tr.Start(ctx, "server.handleSuccess")
defer span.End()
//Add some tags here to help debug.
AddSpanTags(span, map[string]string{"name": "Ivan"})
//Add some event.
AddSpanEvents(span, "testEvent", map[string]string{"eventInfo": "some info"})
initCall(ctx, false)
w.WriteHeader(http.StatusOK)
w.Write([]byte("done"))
}
In Jaeger UI I see only "server" spans but not "client". So what I understand that for some reasons trace from "client" unable to reach the agent/collector(I tried both). Is there any problems with my code? Jaeger init seems equal why one app doesn't send anything is not clear for me.
How I can implement this kind of scenario:
1.I have LoginHandler which receives some user data - email and signedXml:
func LoginHandler(c *gin.Context) {
var (
err error
data LoginPost
)
if err = c.BindJSON(&data); err != nil {
c.JSON(http.StatusBadRequest, gin.H{"status": "error"})
return
}
...
c.JSON(http.StatusOK, gin.H{"status": "ok"})
}
2.I need to send signedXml to another server via websocket
3.Save result (success or error)
4.Close connection
Every HTTP request will open connection, send 1 message, get 1 result and finally close socket. I was trying with channel, but no success. Is this possible to implement my case?
UPDATE
package main
import (
"log"
"net/url"
"github.com/gorilla/mux"
"github.com/gorilla/websocket"
"net/http"
)
func indexHandler(w http.ResponseWriter, r *http.Request) {
message := r.FormValue("message")
w.Write([]byte(message))
}
func postHandler(w http.ResponseWriter, r *http.Request) {
var (
message = r.FormValue("message")
u = url.URL{Scheme: "ws", Host: "echo.websocket.org", Path: "/"}
err error
out []byte
conn *websocket.Conn
)
log.Printf("message: %s\n", message)
log.Printf("connecting to %s\n", u.String())
conn, _, err = websocket.DefaultDialer.Dial(u.String(), nil)
if err != nil {
log.Fatal("dial:", err)
}
log.Println("write")
if err = conn.WriteMessage(websocket.TextMessage, []byte(message)); err != nil {
log.Fatal("write:", err)
}
log.Println("read")
if _, out, err = conn.ReadMessage(); err != nil {
log.Fatal("read:", err)
}
w.Write(out)
log.Println("close")
conn.Close()
}
func main() {
r := mux.NewRouter()
r.HandleFunc("/", indexHandler).Methods("GET")
r.HandleFunc("/post", postHandler).Methods("POST")
http.Handle("/", r)
http.ListenAndServe(":8080", nil)
}
Call Dial, WriteMessage, ReadMessage and Close in sequence.
c, _, err := websocket.DefaultDialer.Dial(url, nil)
if err != nil {
// handle error
}
err := c.WriteMessage(websocket.TextMessage, signedXML)
if err != nil {
// handle error
}
_, p, err := c.ReadMessage()
if err != nil {
// handle error
}
c.Close()
// p is a []byte with the first received message.
I want to get server's session value when client creates a new request.
But, server returns nil always.
I don't know what is the problem in this case.
Client
package main
import (
"io/ioutil"
"log"
"net/http"
"github.com/gorilla/mux"
"github.com/gorilla/securecookie"
"github.com/gorilla/sessions"
)
var (
store *sessions.CookieStore = sessions.NewCookieStore(securecookie.GenerateRandomKey(64))
session *sessions.Session
SessionName = "client"
)
func main() {
store.Options = &sessions.Options{
Path: "/",
MaxAge: 60 * 15,
Secure: false,
HttpOnly: true,
}
router := mux.NewRouter()
router.HandleFunc("/", cIndex)
router.HandleFunc("/test", cTest)
http.ListenAndServe(":7000", router)
}
func cIndex(w http.ResponseWriter, r *http.Request) {
session, err := store.Get(r, SessionName)
if err != nil {
log.Println("cIndex Session Error : ", err.Error())
return
}
session.Values["foo"] = "bar"
session.Save(r, w)
w.Header().Set("Location", "http://localhost:8080?foo=bar")
w.Header().Set("Content-Type", "application/x-www-form-urlencoded")
w.WriteHeader(http.StatusFound)
}
func cTest(w http.ResponseWriter, r *http.Request) {
req, err := http.NewRequest("GET", "http://localhost:8080/test", nil)
if err != nil {
log.Println("cTest Request Error : ", err.Error())
return
}
req.SetBasicAuth("sample_id", "sample_secret")
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
c := &http.Client{
Transport: &http.Transport{},
}
resp, err := c.Do(req)
if err != nil {
log.Println("cTest Response Error : ", err.Error())
return
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Println("cTest Body Error : ", err.Error())
return
}
log.Println(string(body))
}
Server
package main
import (
"encoding/json"
"log"
"net/http"
"github.com/gorilla/mux"
"github.com/gorilla/securecookie"
"github.com/gorilla/sessions"
)
var (
store *sessions.CookieStore = sessions.NewCookieStore(securecookie.GenerateRandomKey(64))
session *sessions.Session
SessionName = "server"
)
func main() {
store.Options = &sessions.Options{
Path: "/",
MaxAge: 60 * 15,
Secure: false,
HttpOnly: true,
}
router := mux.NewRouter()
router.HandleFunc("/", sIndex)
router.HandleFunc("/test", sTest)
http.ListenAndServe(":8080", router)
}
func sIndex(w http.ResponseWriter, r *http.Request) {
r.ParseForm()
v := r.FormValue("foo")
session, err := store.Get(r, SessionName)
if err != nil {
log.Println("sIndex Session Error : ", err.Error())
return
}
session.Values["foo"] = v
session.Save(r, w)
w.Header().Set("Location", "http://localhost:7000/test")
w.Header().Set("Content-Type", "application/x-www-form-urlencoded")
w.WriteHeader(http.StatusFound)
}
func sTest(w http.ResponseWriter, r *http.Request) {
session, err := store.Get(r, SessionName)
if err != nil {
log.Println("sTest Session Error : ", err.Error())
return
}
v, ok := session.Values["foo"].(string)
if !ok {
log.Printf("foo = %v\n", v)
}
data := struct {
Foo string
}{
v, // I expected v is bar,but v was nil.
}
bytes, err := json.Marshal(data)
if err != nil {
log.Println("sTest JSON Error : ", err.Error())
return
}
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(http.StatusOK)
w.Write(bytes)
}
Details
Client sends request to server with query foo=bar.
Server receives the request and parsing.
Server saves query parameter in session.(I used gorilla/session)
Client makes *http.Client and send new Request to server.
Server receives request, and loads value in session, and set the value at response.
Client receives value bar
In 5, I expected that server can load the value bar from session, but session value was nil.
I wrote some codes and it moves.
Is it correct way?
Client
func cTest(w http.ResponseWrite, r *http.Request) {
serverURL := "http://localhost:8080/test"
r.Method = http.MethodGet
r.Host = serverURL // server
v, err := url.Parse(serverURL)
if err != nil {
log.Println("url parsing error occurred : ", err.Error())
return
}
c := &http.Client{Transport: &http.Transport{}}
resp, err := c.Transport.RoundTrip(r)
if err != nil {
log.Println("client roundtrip error occurred : ", err.Error())
return
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Println("response body reading error occurred : ", err.Error())
return
}
w.Write(body)
}
I have more than one ip to go to the internet. I am making request choosing interface. In this case how should I set headers?
tcpAddr := &net.TCPAddr{
IP: addrs[3].(*net.IPNet).IP, // Choosing ip address number 3
}
d := net.Dialer{LocalAddr: tcpAddr}
conn, err2 := d.Dial("tcp", "www.whatismyip.com:80")
if err2 != nil {
log.Fatal(err2)
}
defer conn.Close()
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{LocalAddr: tcpAddr}).Dial,
TLSHandshakeTimeout: 10 * time.Second,
}
client := &http.Client{
Transport: transport,
}
response, err := client.Get("https://www.whatismyip.com/")
Usually headers are set in this way:
req.Header.Set("name", "value")
But cannot figure out how to set them to my code.
I guess they must be set somewhere in http.Transport or http.Client. But how exactly?
My full code:
package main
import (
"bytes"
"fmt"
"github.com/PuerkitoBio/goquery"
"io/ioutil"
"log"
"net"
"net/http"
"os"
"time"
)
func main() {
ief, err := net.InterfaceByName("eth0")
if err != nil {
log.Fatal(err)
}
addrs, err := ief.Addrs()
if err != nil {
log.Fatal(err)
}
tcpAddr := &net.TCPAddr{
IP: addrs[3].(*net.IPNet).IP, // Choosing ip address number 3
}
d := net.Dialer{LocalAddr: tcpAddr}
conn, err2 := d.Dial("tcp", "www.whatismyip.com:80")
if err2 != nil {
log.Fatal(err2)
}
defer conn.Close()
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{LocalAddr: tcpAddr}).Dial,
TLSHandshakeTimeout: 10 * time.Second,
}
client := &http.Client{
Transport: transport,
}
response, err := client.Get("https://www.whatismyip.com/")
if err != nil {
fmt.Printf("%s", err)
os.Exit(1)
} else {
defer response.Body.Close()
contents, err := ioutil.ReadAll(response.Body)
if err != nil {
fmt.Printf("%s", err)
os.Exit(1)
}
var contentsStr = string(contents)
fmt.Printf("%s\n", contentsStr)
var doc = DocByHtmlString(contentsStr)
doc.Find("div").Each(func(i int, s *goquery.Selection) {
attr, exists := s.Attr("class")
if exists {
if attr == "ip" {
fmt.Println(s.Text())
}
}
})
}
}
func DocByHtmlString(html string) *goquery.Document {
doc, err := goquery.NewDocumentFromReader(bytes.NewBufferString(html))
if err != nil {
panic(err)
}
return doc
}
Create a request:
req, err := http.NewRequest("GET", "https://www.whatismyip.com/", nil)
if err != nil {
// handle error
}
Set the headers:
req.Header.Set("name", "value")
Run the request using client as configured in the question:
resp, err := client.Do(req)
if err != nil {
// handle error
}
Handle the response as shown in the question.
I have the following Go program: https://play.golang.org/p/-TUtJ7DIhi
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strconv"
)
func main() {
body, err := get("https://hacker-news.firebaseio.com/v0/topstories.json")
if err != nil {
panic(err)
}
var ids [500]int
if err = json.Unmarshal(body, &ids); err != nil {
panic(err)
}
var contents []byte
for _, value := range ids[0:10] {
body, err := get("https://hacker-news.firebaseio.com/v0/item/" + strconv.Itoa(value) + ".json")
if err != nil {
fmt.Println(err)
} else {
contents = append(contents, body...)
}
}
fmt.Println(contents)
}
func get(url string) ([]byte, error) {
res, err := http.Get(url)
if err != nil {
return nil, err
}
body, err := ioutil.ReadAll(res.Body)
res.Body.Close()
return body, err
}
When run it throws EOF json errors on the iterative get requests, but when I hit the URLs individually they do not appear to be malformed.
What am I missing?
It looks like there's something wrong with their server, and it's closing connections without sending a Connection: close header. The client therefore tries to reuse the connection per the HTTP/1.1 specification.
You can work around this by creating your own request, and setting Close = true, or using a custom Transport with DisableKeepAlives = true
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Close = true
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}