set headers for request using http.Client and http.Transport - go

I have more than one ip to go to the internet. I am making request choosing interface. In this case how should I set headers?
tcpAddr := &net.TCPAddr{
IP: addrs[3].(*net.IPNet).IP, // Choosing ip address number 3
}
d := net.Dialer{LocalAddr: tcpAddr}
conn, err2 := d.Dial("tcp", "www.whatismyip.com:80")
if err2 != nil {
log.Fatal(err2)
}
defer conn.Close()
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{LocalAddr: tcpAddr}).Dial,
TLSHandshakeTimeout: 10 * time.Second,
}
client := &http.Client{
Transport: transport,
}
response, err := client.Get("https://www.whatismyip.com/")
Usually headers are set in this way:
req.Header.Set("name", "value")
But cannot figure out how to set them to my code.
I guess they must be set somewhere in http.Transport or http.Client. But how exactly?
My full code:
package main
import (
"bytes"
"fmt"
"github.com/PuerkitoBio/goquery"
"io/ioutil"
"log"
"net"
"net/http"
"os"
"time"
)
func main() {
ief, err := net.InterfaceByName("eth0")
if err != nil {
log.Fatal(err)
}
addrs, err := ief.Addrs()
if err != nil {
log.Fatal(err)
}
tcpAddr := &net.TCPAddr{
IP: addrs[3].(*net.IPNet).IP, // Choosing ip address number 3
}
d := net.Dialer{LocalAddr: tcpAddr}
conn, err2 := d.Dial("tcp", "www.whatismyip.com:80")
if err2 != nil {
log.Fatal(err2)
}
defer conn.Close()
transport := &http.Transport{
Proxy: http.ProxyFromEnvironment,
Dial: (&net.Dialer{LocalAddr: tcpAddr}).Dial,
TLSHandshakeTimeout: 10 * time.Second,
}
client := &http.Client{
Transport: transport,
}
response, err := client.Get("https://www.whatismyip.com/")
if err != nil {
fmt.Printf("%s", err)
os.Exit(1)
} else {
defer response.Body.Close()
contents, err := ioutil.ReadAll(response.Body)
if err != nil {
fmt.Printf("%s", err)
os.Exit(1)
}
var contentsStr = string(contents)
fmt.Printf("%s\n", contentsStr)
var doc = DocByHtmlString(contentsStr)
doc.Find("div").Each(func(i int, s *goquery.Selection) {
attr, exists := s.Attr("class")
if exists {
if attr == "ip" {
fmt.Println(s.Text())
}
}
})
}
}
func DocByHtmlString(html string) *goquery.Document {
doc, err := goquery.NewDocumentFromReader(bytes.NewBufferString(html))
if err != nil {
panic(err)
}
return doc
}

Create a request:
req, err := http.NewRequest("GET", "https://www.whatismyip.com/", nil)
if err != nil {
// handle error
}
Set the headers:
req.Header.Set("name", "value")
Run the request using client as configured in the question:
resp, err := client.Do(req)
if err != nil {
// handle error
}
Handle the response as shown in the question.

Related

Golang socks5 proxy client

I'm trying to make a proxy:
I need to listen to port 1080 (socks 5), and complete a request to a destination using an external ip:port socks 5, I managed to open this connection with the external ip, but I don't know how I could complete the request to the target destination using that external ip.
could someone help me with this?
package main
import (
"bufio"
"errors"
"fmt"
"io"
"log"
"net"
)
const (
ConnectCommand = uint8(1)
BindCommand = uint8(2)
AssociateCommand = uint8(3)
ipv4Address = uint8(1)
fqdnAddress = uint8(3)
ipv6Address = uint8(4)
)
type AddrSpec struct {
FQDN string
IP net.IP
Port int
}
func main() {
l, err := net.Listen("tcp", "127.0.0.1:1080")
if err != nil {
fmt.Print(err)
}
defer l.Close()
for {
conn, err := l.Accept()
if err != nil {
fmt.Print(err)
}
go handle(conn)
}
}
func handle(conn net.Conn) {
defer func() {
_ = conn.Close()
}()
bufConn := bufio.NewReader(conn)
version := []byte{0}
if _, err := bufConn.Read(version); err != nil {
log.Fatalf("cannot read version: %s", err.Error())
}
if version[0] != uint8(5) {
log.Fatalf("unsupported SOCKS version: %v", version)
}
socks5ExternalConn, err := net.Dial("tcp", externalSOCKS5Proxy())
if err != nil {
log.Printf("Connection error: %s", err.Error())
}
dest, err := readAddrSpec(bufConn)
if err != nil {
}
// how i can send request to server with external conn?
}
func externalSOCKS5Proxy() string {
return "externalip:externalport"
}
func readAddrSpec(r io.Reader) (*AddrSpec, error) {
d := &AddrSpec{}
// Get the address type
addrType := []byte{0}
if _, err := r.Read(addrType); err != nil {
return nil, err
}
// Handle on a per type basis
switch addrType[0] {
case ipv4Address:
addr := make([]byte, 4)
if _, err := io.ReadAtLeast(r, addr, len(addr)); err != nil {
return nil, err
}
d.IP = net.IP(addr)
case ipv6Address:
addr := make([]byte, 16)
if _, err := io.ReadAtLeast(r, addr, len(addr)); err != nil {
return nil, err
}
d.IP = net.IP(addr)
case fqdnAddress:
if _, err := r.Read(addrType); err != nil {
return nil, err
}
addrLen := int(addrType[0])
fqdn := make([]byte, addrLen)
if _, err := io.ReadAtLeast(r, fqdn, addrLen); err != nil {
return nil, err
}
d.FQDN = string(fqdn)
default:
return nil, errors.New("unrecognizedAddrType")
}
// Read the port
port := []byte{0, 0}
if _, err := io.ReadAtLeast(r, port, 2); err != nil {
return nil, err
}
d.Port = (int(port[0]) << 8) | int(port[1])
return d, nil
}

Cannot receive response packets from multicast server in golang

I setup a udp server listening multicast trafic, and create a client to sent test packet. When the server receives the package, it will send a response. Everything is ok, except the client cannot receive the packet from the server. Why?
package main
import (
"log"
"net"
"os"
"time"
"golang.org/x/net/ipv4"
)
func SetupMulticast(ifn, addr string) {
ifi, err := net.InterfaceByName(ifn)
if err != nil {
log.Fatal(err)
}
//server
c, err := net.ListenPacket("udp4", addr)
if err != nil {
log.Fatal(err)
}
log.Printf("listen ad:%s\n", addr)
p := ipv4.NewPacketConn(c)
gAddr, err2 := net.ResolveUDPAddr("udp4", addr)
if err2 != nil {
log.Fatal(err2)
}
if err := p.JoinGroup(ifi, gAddr); err != nil {
log.Fatal(err)
}
if err := p.SetControlMessage(ipv4.FlagDst|ipv4.FlagSrc, true); err != nil {
log.Fatal(err)
}
go func() {
b := make([]byte, 1500)
for {
n, cm, src, err := p.ReadFrom(b)
if err != nil {
break
}
log.Println("receive:", string(b[:n]), cm.Dst.IsMulticast(), cm.Dst)
if n2, err := p.WriteTo([]byte("world!"), nil, src); err != nil {
log.Printf("fail to write back:%v\n", err)
} else {
log.Printf("write back addr: %s length:%d\n", src, n2)
}
}
}()
//client
if conn, err2 := net.DialUDP("udp4", nil /*src*/, gAddr); err2 != nil {
log.Fatal(err)
} else {
go func() {
for {
log.Println("write hello...")
conn.Write([]byte("hello"))
time.Sleep(time.Second * 2)
}
}()
go func() {
b2 := make([]byte, 1500)
for {
n, err := conn.Read(b2)
if err != nil {
log.Panic(err)
}
log.Printf("sender received response:%s\n", string(b2[:n]))
}
}()
}
}
func main() {
If := os.Args[1] //ens33
Addr := os.Args[2] //224.0.0.248:1025
SetupMulticast(If, Addr)
for {
}
}
And the output:
2022/08/17 22:53:53 listen ad:224.0.0.248:1025
2022/08/17 22:53:53 write hello...
2022/08/17 22:53:53 receive: hello true 224.0.0.248
2022/08/17 22:53:53 write back addr: 192.168.19.131:43925 length:6
2022/08/17 22:53:55 write hello...
2022/08/17 22:53:55 receive: hello true 224.0.0.248
2022/08/17 22:53:55 write back addr: 192.168.19.131:43925 length:6
From the logs, there are no any "sender received response" record. I don't know why?

Distributed tracing doesn't work Jaeger+OpenTelemetry

I am trying to implement distributed tracing with basic GO client-server app. Using default Jaeger docker-compose all-in-one.
What was done to fix and doesn't help:
Changed collector to agent and agent to collector.
Checked logs, nothing about "client" there
Tried to inject headers (propagation)
Tried without injecting to headers (propagation)
CLIENT CODE:
import(
...
tracesdk "go.opentelemetry.io/otel/sdk/trace"
...
)
func main() {
.....
exporter, err := jaeger.New(
jaeger.WithAgentEndpoint(
jaeger.WithAgentHost("localhost"),
jaeger.WithAgentPort("6831"),
),
)
if err != nil {
return err
}
tp := tracesdk.NewTracerProvider(
tracesdk.WithBatcher(exporter),
tracesdk.WithResource(resource.NewWithAttributes(
semconv.SchemaURL,
semconv.ServiceNameKey.String("client"),
semconv.ServiceVersionKey.String("1.0.0"),
semconv.DeploymentEnvironmentKey.String("local"),
)),
)
defer func() {
if err := tp.Shutdown(context.Background()); err != nil {
log.Fatal(err)
}
}()
otel.SetTracerProvider(tp)
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
propagation.TraceContext{},
propagation.Baggage{},
propagators.Jaeger{},
))
.......
}
....
func fetcherSuccess() error {
tr := otel.Tracer("clientHTTP")
ctx, span := tr.Start(context.Background(), "client.fetcherSuccess")
defer span.End()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/success", nil)
if err != nil {
AddSpanError(span, err)
FailSpan(span, "request error")
return err
}
// Try to inject headers to the context using this otel.GetTextMapPropagator().Inject(ctx, h)
headers := InjectHTTPHeaders(ctx)
for k, v := range headers {
req.Header.Add(k, v)
}
res, _ := http.DefaultClient.Do(req)
return nil
}
SERVER CODE:
import(
...
tracesdk "go.opentelemetry.io/otel/sdk/trace"
...
)
func main() {
.....
exporter, err := jaeger.New(
jaeger.WithAgentEndpoint(
jaeger.WithAgentHost("localhost"),
jaeger.WithAgentPort("6831"),
),
)
if err != nil {
return err
}
tp := tracesdk.NewTracerProvider(
tracesdk.WithBatcher(exporter),
tracesdk.WithResource(resource.NewWithAttributes(
semconv.SchemaURL,
semconv.ServiceNameKey.String("server"),
semconv.ServiceVersionKey.String("1.0.0"),
semconv.DeploymentEnvironmentKey.String("local"),
)),
)
defer func() {
if err := tp.Shutdown(context.Background()); err != nil {
log.Fatal(err)
}
}()
otel.SetTracerProvider(tp)
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
propagation.TraceContext{},
propagation.Baggage{},
propagators.Jaeger{},
))
if err := handleRequests(); err != nil {
panic("unable to create handler")
}
.......
}
func handleRequests() error {
router := http.NewServeMux()
router.HandleFunc("/success", handleSuccess)
router.HandleFunc("/error", handleError)
fmt.Println("Server is listening on port: 8080")
if err := http.ListenAndServe(":8080", router); err != nil {
return err
}
return nil
}
....
func handleSuccess(w http.ResponseWriter, r *http.Request) {
tr := otel.Tracer("serverHTTP")
//Extract headers using otel.GetTextMapPropagator().Extract()
ctx := ExtractHTTPHeaders(r.Context(), r.Header)
ctx, span := tr.Start(ctx, "server.handleSuccess")
defer span.End()
//Add some tags here to help debug.
AddSpanTags(span, map[string]string{"name": "Ivan"})
//Add some event.
AddSpanEvents(span, "testEvent", map[string]string{"eventInfo": "some info"})
initCall(ctx, false)
w.WriteHeader(http.StatusOK)
w.Write([]byte("done"))
}
In Jaeger UI I see only "server" spans but not "client". So what I understand that for some reasons trace from "client" unable to reach the agent/collector(I tried both). Is there any problems with my code? Jaeger init seems equal why one app doesn't send anything is not clear for me.

GCP - get project NAT GW's

We have account on GCP which contain valid cloud Nat, now we want to get those values via
GCP sdk, I've tried the following and get empty response (maybe I use the wrong API and it not ListExternalVpnGatewaysRequest)
package main
import (
"context"
"fmt"
compute "cloud.google.com/go/compute/apiv1"
"google.golang.org/api/iterator"
computepb "google.golang.org/genproto/googleapis/cloud/compute/v1"
)
func main() {
ctx := context.Background()
c, err := compute.NewExternalVpnGatewaysRESTClient(ctx)
if err != nil {
fmt.Println(err)
}
defer c.Close()
proj := "dev-proj"
req := &computepb.ListExternalVpnGatewaysRequest{
//Filter: new(string),
//MaxResults: new(uint32),
//OrderBy: new(string),
//PageToken: new(string),
Project: proj,
//ReturnPartialSuccess: new(bool),
}
it := c.List(ctx, req)
for {
resp, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
fmt.Println(err)
}
// TODO: Use resp.
_ = resp
fmt.Println(resp)
}
}
I need to get the following values using GCP GO SDK
update
I tried the following as-is and I got error
package main
import (
"context"
"fmt"
"google.golang.org/api/compute/v1"
"log"
)
func main() {
project := "my-proj"
region := "my-region"
ctx := context.Background()
computeService, err := compute.New(ctx)
if err != nil {
log.Fatal(err)
}
req := computeService.Routers.List(project, region)
if err := req.Pages(ctx, func(page *compute.RouterList) error {
for _, router := range page.Items {
// process each `router` resource:
fmt.Printf("%#v\n", router)
// NAT Gateways are found in router.nats
}
return nil
}); err != nil {
log.Fatal(err)
}
}
Error is: ./main.go:16:36: cannot use ctx (type context.Context) as type *http.Client in argument to compute.New
A VPN Gateway is not the same as a NAT Gateway.
Use this code to list routers. Within the list of routers, is the NAT Gateways
import "google.golang.org/api/compute/v1"
// Replace with valid values for your project
project := "my-project"
region := "my-region"
ctx := context.Background()
c, err := google.DefaultClient(ctx, compute.CloudPlatformScope)
if err != nil {
log.Fatal(err)
}
computeService, err := compute.New(c)
if err != nil {
log.Fatal(err)
}
req := computeService.Routers.List(project, region)
if err := req.Pages(ctx, func(page *compute.RouterList) error {
for _, router := range page.Items {
// process each `router` resource:
fmt.Printf("%#v\n", router)
// NAT Gateways are found in router.nats
}
return nil
}); err != nil {
log.Fatal(err)
}
SDK Documentation

How to redirect multipart POST request to a second server in Golang?

I am trying to do the following.
|Upload file in HTML post file form|
|
⌄
|Server A forwards the multipart request|
|
⌄
|Server B receives and stores the file from the forwarded multipart request|
|
⌄
|Server A receives response from Server B when Server B is done|
Processing the multipart request on Server A is straightforward, but when I try to process the forwarded request on Server B it fails with multipart: NextPart: EOF.
I am trying to create separate frontend/backend services. Frontend only handles UI related processing, while backend will actually do some processing on the file, hence the multipart request forwarding needed.
The forwarding code on Server A is as follows.
The solution has been taken from here.
https://stackoverflow.com/a/34725635/6569715
func forwardRequest(address string, path string, r *http.Request) (interface{}, error) {
body, err := ioutil.ReadAll(r.Body)
if err != nil {
return nil, err
}
r.Body = ioutil.NopCloser(bytes.NewReader(body))
proxyReq, err := http.NewRequest(r.Method, fmt.Sprintf("%s%s", address, path), bytes.NewReader(body))
if err != nil {
return nil, err
}
for header, values := range r.Header {
for _, value := range values {
proxyReq.Header.Add(header, value)
}
}
client := &http.Client{}
resp, err := client.Do(proxyReq)
if err != nil {
return nil, err
}
defer resp.Body.Close()
return resp, nil
}
And the code on Server B to process the forwarded request:
func testMultiPart(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(10 << 20); err != nil {
err = errors.Wrap(errors.WithStack(err), "Backend: Failed to parse form")
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprint(w, fmt.Sprintf("{\"error\":\"%s\"}", err.Error())
return
}
}
Any help is appreciated.
I managed to make it work. I believe it was just my own mistake not filling in the URI properly. In any case I will post my snippets from my solution for future reference.
The client html file form part:
<form action="/test-main/file-test" enctype="multipart/form-data" method="post">
<label for="file-upload">Upload your file :</label>
<input type="file" id="file-upload" name="file-upload" accept="image/*">
</form>
Server A code:
import (
"net/http"
"errors"
"fmt"
"log"
"io/ioutil"
"bytes"
"github.com/gorilla/mux"
)
func fileUpload(w http.ResponseWriter, r *http.Request) {
body, err := ioutil.ReadAll(r.Body)
if err != nil {
return log.Fatal(err)
}
r.Body = ioutil.NopCloser(bytes.NewReader(body))
// If Server A and B are separate docker images, you may need to use their docker subnet IP, like below.
proxyReq, err := http.NewRequest(r.Method, fmt.Sprintf("http://172.18.0.2:8082%s", r.RequestURI), bytes.NewReader(body))
if err != nil {
return log.Fatal(err)
}
for header, values := range r.Header {
for _, value := range values {
proxyReq.Header.Add(header, value)
}
}
client := &http.Client{}
resp, err := client.Do(proxyReq)
if err != nil {
return log.Fatal(err)
}
defer resp.Body.Close()
respBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
return log.Fatal(err)
}
// Process Server B response
// ...
}
func createRouter() *mux.Router {
r := mux.NewRouter()
testPath := r.PathPrefix("/test-main").Subrouter()
testPath.HandleFunc("/file-test", fileUpload)
return r
}
func main() {
// Create Server and Route Handlers
srv := &http.Server{
Handler: createRouter(),
Addr: ":8081",
ReadTimeout: 30 * time.Second,
WriteTimeout: 30 * time.Second,
}
// Start Server
go func() {
log.Println("Starting Server")
if err := srv.ListenAndServe(); err != nil {
log.Fatal(err)
}
}()
}
And Server B code:
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"github.com/gorilla/mux"
)
func uploadFile(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(10 << 20); err != nil {
log.Fatal(err)
}
file, handler, err := r.FormFile("file-upload")
if err == http.ErrMissingFile {
return nil
}
if err != nil {
log.Fatal(err)
}
fmt.Printf("Uploaded File: %+v\n", handler.Filename)
fmt.Printf("File Size: %+v\n", handler.Size)
fmt.Printf("MIME Header: %+v\n", handler.Header)
defer file.Close()
// Create file
dst, err := os.Create(fmt.Sprintf("/some-destination-folder/%s", handler.Filename))
if err != nil {
log.Fatal(err)
}
// Copy the uploaded file to the created file on the file system.
if _, err := io.Copy(dst, file); err != nil {
if err2 := dst.Close(); err2 != nil {
log.Fatal(err)
}
log.Fatal(err)
}
dst.Close()
return nil
}
func (c *Controller) createRouter() *mux.Router {
r := mux.NewRouter()
testPath := r.PathPrefix("/test-main").Subrouter()
testPath.HandleFunc("/file-test", uploadFile)
return r
}
func main() {
// Create Server and Route Handlers
srv := &http.Server{
Handler: createRouter(),
Addr: ":8082",
ReadTimeout: 30 * time.Second,
WriteTimeout: 30 * time.Second,
}
// Start Server
go func() {
log.Println("Starting Server")
if err := srv.ListenAndServe(); err != nil {
log.Fatal(err)
}
}()
}
Good luck for future readers.

Resources