Distributed tracing doesn't work Jaeger+OpenTelemetry - go

I am trying to implement distributed tracing with basic GO client-server app. Using default Jaeger docker-compose all-in-one.
What was done to fix and doesn't help:
Changed collector to agent and agent to collector.
Checked logs, nothing about "client" there
Tried to inject headers (propagation)
Tried without injecting to headers (propagation)
CLIENT CODE:
import(
...
tracesdk "go.opentelemetry.io/otel/sdk/trace"
...
)
func main() {
.....
exporter, err := jaeger.New(
jaeger.WithAgentEndpoint(
jaeger.WithAgentHost("localhost"),
jaeger.WithAgentPort("6831"),
),
)
if err != nil {
return err
}
tp := tracesdk.NewTracerProvider(
tracesdk.WithBatcher(exporter),
tracesdk.WithResource(resource.NewWithAttributes(
semconv.SchemaURL,
semconv.ServiceNameKey.String("client"),
semconv.ServiceVersionKey.String("1.0.0"),
semconv.DeploymentEnvironmentKey.String("local"),
)),
)
defer func() {
if err := tp.Shutdown(context.Background()); err != nil {
log.Fatal(err)
}
}()
otel.SetTracerProvider(tp)
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
propagation.TraceContext{},
propagation.Baggage{},
propagators.Jaeger{},
))
.......
}
....
func fetcherSuccess() error {
tr := otel.Tracer("clientHTTP")
ctx, span := tr.Start(context.Background(), "client.fetcherSuccess")
defer span.End()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:8080/success", nil)
if err != nil {
AddSpanError(span, err)
FailSpan(span, "request error")
return err
}
// Try to inject headers to the context using this otel.GetTextMapPropagator().Inject(ctx, h)
headers := InjectHTTPHeaders(ctx)
for k, v := range headers {
req.Header.Add(k, v)
}
res, _ := http.DefaultClient.Do(req)
return nil
}
SERVER CODE:
import(
...
tracesdk "go.opentelemetry.io/otel/sdk/trace"
...
)
func main() {
.....
exporter, err := jaeger.New(
jaeger.WithAgentEndpoint(
jaeger.WithAgentHost("localhost"),
jaeger.WithAgentPort("6831"),
),
)
if err != nil {
return err
}
tp := tracesdk.NewTracerProvider(
tracesdk.WithBatcher(exporter),
tracesdk.WithResource(resource.NewWithAttributes(
semconv.SchemaURL,
semconv.ServiceNameKey.String("server"),
semconv.ServiceVersionKey.String("1.0.0"),
semconv.DeploymentEnvironmentKey.String("local"),
)),
)
defer func() {
if err := tp.Shutdown(context.Background()); err != nil {
log.Fatal(err)
}
}()
otel.SetTracerProvider(tp)
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
propagation.TraceContext{},
propagation.Baggage{},
propagators.Jaeger{},
))
if err := handleRequests(); err != nil {
panic("unable to create handler")
}
.......
}
func handleRequests() error {
router := http.NewServeMux()
router.HandleFunc("/success", handleSuccess)
router.HandleFunc("/error", handleError)
fmt.Println("Server is listening on port: 8080")
if err := http.ListenAndServe(":8080", router); err != nil {
return err
}
return nil
}
....
func handleSuccess(w http.ResponseWriter, r *http.Request) {
tr := otel.Tracer("serverHTTP")
//Extract headers using otel.GetTextMapPropagator().Extract()
ctx := ExtractHTTPHeaders(r.Context(), r.Header)
ctx, span := tr.Start(ctx, "server.handleSuccess")
defer span.End()
//Add some tags here to help debug.
AddSpanTags(span, map[string]string{"name": "Ivan"})
//Add some event.
AddSpanEvents(span, "testEvent", map[string]string{"eventInfo": "some info"})
initCall(ctx, false)
w.WriteHeader(http.StatusOK)
w.Write([]byte("done"))
}
In Jaeger UI I see only "server" spans but not "client". So what I understand that for some reasons trace from "client" unable to reach the agent/collector(I tried both). Is there any problems with my code? Jaeger init seems equal why one app doesn't send anything is not clear for me.

Related

Golang Chromedp: pdf file download without saving in server

How to chromedp pdf download without saving in server?
Below code is working for generating pdf file and saving in server side. But I want to download pdf file without saving in server side.
func PDFInvoice(c *gin.Context) {
session := sessions.Default(c)
id := c.Params.ByName("id")
token := session.Get("login_session").(string)
// create context
ctx, cancel := chromedp.NewContext(context.Background())
defer cancel()
// capture pdf
var buf []byte
url := "http://localhost:8080/invoice/" + id + "/" + token
if err := chromedp.Run(ctx, printToPDF(url, &buf)); err != nil {
log.Fatal(err)
}
buff := new(bytes.Buffer)
if _, err := buff.WriteTo(c.Writer); err != nil {
panic(err)
}
if err := os.WriteFile("sample.pdf", buf, 0o644); err != nil {
log.Fatal(err)
}
//ioutil.WriteFile("sample.pdf", buf, 0644)
c.JSON(200, id+" "+token)
}
// print a specific pdf page.
func printToPDF(urlstr string, res *[]byte) chromedp.Tasks {
return chromedp.Tasks{
chromedp.Navigate(urlstr),
chromedp.ActionFunc(func(ctx context.Context) error {
buf, _, err := page.PrintToPDF().WithPrintBackground(false).Do(ctx)
if err != nil {
return err
}
*res = buf
return nil
}),
}
}
You can write the bytes to http.ResponseWriter directly. See the demo below:
package main
import (
"context"
"log"
"net/http"
"sync"
"github.com/chromedp/cdproto/page"
"github.com/chromedp/chromedp"
)
func main() {
http.Handle("/pdf", http.HandlerFunc(servePDF))
log.Fatal(http.ListenAndServe(":8080", http.DefaultServeMux))
}
func servePDF(w http.ResponseWriter, r *http.Request) {
buf, err := createPDF()
if err != nil {
log.Fatalln(err)
w.WriteHeader(http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/pdf")
w.Write(buf)
}
func createPDF() ([]byte, error) {
ctx, cancel := newTabContext()
defer cancel()
html := `<html>
<body>
<div>text</div>
<img src="https://pkg.go.dev/static/shared/gopher/package-search-700x300.jpeg"/>
<img src="https://go.dev/images/gophers/motorcycle.svg"/>
<img src="https://go.dev/images/go_google_case_study_carousel.png" />
</body>
</html>`
var buf []byte
if err := chromedp.Run(ctx,
chromedp.Navigate("about:blank"),
// set the page content and wait until the page is loaded (including its resources).
chromedp.ActionFunc(func(ctx context.Context) error {
lctx, cancel := context.WithCancel(ctx)
defer cancel()
var wg sync.WaitGroup
wg.Add(1)
chromedp.ListenTarget(lctx, func(ev interface{}) {
if _, ok := ev.(*page.EventLoadEventFired); ok {
// It's a good habit to remove the event listener if we don't need it anymore.
cancel()
wg.Done()
}
})
frameTree, err := page.GetFrameTree().Do(ctx)
if err != nil {
return err
}
if err := page.SetDocumentContent(frameTree.Frame.ID, html).Do(ctx); err != nil {
return err
}
wg.Wait()
return nil
}),
chromedp.ActionFunc(func(ctx context.Context) error {
var err error
buf, _, err = page.PrintToPDF().WithPrintBackground(false).Do(ctx)
if err != nil {
return err
}
return nil
}),
); err != nil {
return nil, err
}
return buf, nil
}
var (
browserCtx context.Context
once sync.Once
)
// newTabContext creates a tab context with the global browser context as its parent context.
//
// When tasks is run with the returned context, a new tab will be created in the browser.
func newTabContext() (context.Context, context.CancelFunc) {
once.Do(func() { initBrowser() })
if browserCtx == nil || browserCtx.Err() != nil {
log.Fatalf("browser is not available: %v", browserCtx.Err())
}
return chromedp.NewContext(browserCtx)
}
// initBrowser starts a browser in which to create new tab for running tasks.
func initBrowser() {
browserCtx, _ = chromedp.NewContext(context.Background())
// to start the browser
if err := chromedp.Run(browserCtx); err != nil {
log.Fatal(err)
}
}
Usage:
go run main.go
curl http://localhost:8080/pdf > sample.pdf
References:
https://github.com/chromedp/chromedp/issues/941
https://github.com/chromedp/chromedp/issues/836

GCP - get project NAT GW's

We have account on GCP which contain valid cloud Nat, now we want to get those values via
GCP sdk, I've tried the following and get empty response (maybe I use the wrong API and it not ListExternalVpnGatewaysRequest)
package main
import (
"context"
"fmt"
compute "cloud.google.com/go/compute/apiv1"
"google.golang.org/api/iterator"
computepb "google.golang.org/genproto/googleapis/cloud/compute/v1"
)
func main() {
ctx := context.Background()
c, err := compute.NewExternalVpnGatewaysRESTClient(ctx)
if err != nil {
fmt.Println(err)
}
defer c.Close()
proj := "dev-proj"
req := &computepb.ListExternalVpnGatewaysRequest{
//Filter: new(string),
//MaxResults: new(uint32),
//OrderBy: new(string),
//PageToken: new(string),
Project: proj,
//ReturnPartialSuccess: new(bool),
}
it := c.List(ctx, req)
for {
resp, err := it.Next()
if err == iterator.Done {
break
}
if err != nil {
fmt.Println(err)
}
// TODO: Use resp.
_ = resp
fmt.Println(resp)
}
}
I need to get the following values using GCP GO SDK
update
I tried the following as-is and I got error
package main
import (
"context"
"fmt"
"google.golang.org/api/compute/v1"
"log"
)
func main() {
project := "my-proj"
region := "my-region"
ctx := context.Background()
computeService, err := compute.New(ctx)
if err != nil {
log.Fatal(err)
}
req := computeService.Routers.List(project, region)
if err := req.Pages(ctx, func(page *compute.RouterList) error {
for _, router := range page.Items {
// process each `router` resource:
fmt.Printf("%#v\n", router)
// NAT Gateways are found in router.nats
}
return nil
}); err != nil {
log.Fatal(err)
}
}
Error is: ./main.go:16:36: cannot use ctx (type context.Context) as type *http.Client in argument to compute.New
A VPN Gateway is not the same as a NAT Gateway.
Use this code to list routers. Within the list of routers, is the NAT Gateways
import "google.golang.org/api/compute/v1"
// Replace with valid values for your project
project := "my-project"
region := "my-region"
ctx := context.Background()
c, err := google.DefaultClient(ctx, compute.CloudPlatformScope)
if err != nil {
log.Fatal(err)
}
computeService, err := compute.New(c)
if err != nil {
log.Fatal(err)
}
req := computeService.Routers.List(project, region)
if err := req.Pages(ctx, func(page *compute.RouterList) error {
for _, router := range page.Items {
// process each `router` resource:
fmt.Printf("%#v\n", router)
// NAT Gateways are found in router.nats
}
return nil
}); err != nil {
log.Fatal(err)
}
SDK Documentation

How to redirect multipart POST request to a second server in Golang?

I am trying to do the following.
|Upload file in HTML post file form|
|
⌄
|Server A forwards the multipart request|
|
⌄
|Server B receives and stores the file from the forwarded multipart request|
|
⌄
|Server A receives response from Server B when Server B is done|
Processing the multipart request on Server A is straightforward, but when I try to process the forwarded request on Server B it fails with multipart: NextPart: EOF.
I am trying to create separate frontend/backend services. Frontend only handles UI related processing, while backend will actually do some processing on the file, hence the multipart request forwarding needed.
The forwarding code on Server A is as follows.
The solution has been taken from here.
https://stackoverflow.com/a/34725635/6569715
func forwardRequest(address string, path string, r *http.Request) (interface{}, error) {
body, err := ioutil.ReadAll(r.Body)
if err != nil {
return nil, err
}
r.Body = ioutil.NopCloser(bytes.NewReader(body))
proxyReq, err := http.NewRequest(r.Method, fmt.Sprintf("%s%s", address, path), bytes.NewReader(body))
if err != nil {
return nil, err
}
for header, values := range r.Header {
for _, value := range values {
proxyReq.Header.Add(header, value)
}
}
client := &http.Client{}
resp, err := client.Do(proxyReq)
if err != nil {
return nil, err
}
defer resp.Body.Close()
return resp, nil
}
And the code on Server B to process the forwarded request:
func testMultiPart(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(10 << 20); err != nil {
err = errors.Wrap(errors.WithStack(err), "Backend: Failed to parse form")
w.WriteHeader(http.StatusInternalServerError)
fmt.Fprint(w, fmt.Sprintf("{\"error\":\"%s\"}", err.Error())
return
}
}
Any help is appreciated.
I managed to make it work. I believe it was just my own mistake not filling in the URI properly. In any case I will post my snippets from my solution for future reference.
The client html file form part:
<form action="/test-main/file-test" enctype="multipart/form-data" method="post">
<label for="file-upload">Upload your file :</label>
<input type="file" id="file-upload" name="file-upload" accept="image/*">
</form>
Server A code:
import (
"net/http"
"errors"
"fmt"
"log"
"io/ioutil"
"bytes"
"github.com/gorilla/mux"
)
func fileUpload(w http.ResponseWriter, r *http.Request) {
body, err := ioutil.ReadAll(r.Body)
if err != nil {
return log.Fatal(err)
}
r.Body = ioutil.NopCloser(bytes.NewReader(body))
// If Server A and B are separate docker images, you may need to use their docker subnet IP, like below.
proxyReq, err := http.NewRequest(r.Method, fmt.Sprintf("http://172.18.0.2:8082%s", r.RequestURI), bytes.NewReader(body))
if err != nil {
return log.Fatal(err)
}
for header, values := range r.Header {
for _, value := range values {
proxyReq.Header.Add(header, value)
}
}
client := &http.Client{}
resp, err := client.Do(proxyReq)
if err != nil {
return log.Fatal(err)
}
defer resp.Body.Close()
respBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
return log.Fatal(err)
}
// Process Server B response
// ...
}
func createRouter() *mux.Router {
r := mux.NewRouter()
testPath := r.PathPrefix("/test-main").Subrouter()
testPath.HandleFunc("/file-test", fileUpload)
return r
}
func main() {
// Create Server and Route Handlers
srv := &http.Server{
Handler: createRouter(),
Addr: ":8081",
ReadTimeout: 30 * time.Second,
WriteTimeout: 30 * time.Second,
}
// Start Server
go func() {
log.Println("Starting Server")
if err := srv.ListenAndServe(); err != nil {
log.Fatal(err)
}
}()
}
And Server B code:
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"github.com/gorilla/mux"
)
func uploadFile(w http.ResponseWriter, r *http.Request) {
if err := r.ParseMultipartForm(10 << 20); err != nil {
log.Fatal(err)
}
file, handler, err := r.FormFile("file-upload")
if err == http.ErrMissingFile {
return nil
}
if err != nil {
log.Fatal(err)
}
fmt.Printf("Uploaded File: %+v\n", handler.Filename)
fmt.Printf("File Size: %+v\n", handler.Size)
fmt.Printf("MIME Header: %+v\n", handler.Header)
defer file.Close()
// Create file
dst, err := os.Create(fmt.Sprintf("/some-destination-folder/%s", handler.Filename))
if err != nil {
log.Fatal(err)
}
// Copy the uploaded file to the created file on the file system.
if _, err := io.Copy(dst, file); err != nil {
if err2 := dst.Close(); err2 != nil {
log.Fatal(err)
}
log.Fatal(err)
}
dst.Close()
return nil
}
func (c *Controller) createRouter() *mux.Router {
r := mux.NewRouter()
testPath := r.PathPrefix("/test-main").Subrouter()
testPath.HandleFunc("/file-test", uploadFile)
return r
}
func main() {
// Create Server and Route Handlers
srv := &http.Server{
Handler: createRouter(),
Addr: ":8082",
ReadTimeout: 30 * time.Second,
WriteTimeout: 30 * time.Second,
}
// Start Server
go func() {
log.Println("Starting Server")
if err := srv.ListenAndServe(); err != nil {
log.Fatal(err)
}
}()
}
Good luck for future readers.

Want to add a FormFile in unit test Golang

I want to test a httpRequest with a json body and a test file.
I don't know how to add the created test file to the request beside body json.
body := strings.NewReader(URLTest.RequestBody)
request, err := http.NewRequest(URLTest.MethodType, "localhost:"+string(listeningPort)+URLTest.URL, body)
if err != nil {
t.Fatalf("HTTP NOT WORKING")
}
fileBuffer := new(bytes.Buffer)
mpWriter := multipart.NewWriter(fileBuffer)
fileWriter, err := mpWriter.CreateFormFile("file", "testfile.pdf")
if err != nil {
t.Fatalf(err.Error())
}
file, err := os.Open("testfile.pdf")
if err != nil {
t.Fatalf(err.Error())
}
defer file.Close()
_, err = io.Copy(fileWriter, file)
if err != nil {
t.Fatalf(err.Error())
}
rec := httptest.NewRecorder()
UploadFiles(rec, request, nil)
response := rec.Result()
if response.StatusCode != URLTest.ExpectedStatusCode {
t.Errorf(URLTest.URL + " status mismatch")
}
responseBody, err := ioutil.ReadAll(response.Body)
defer response.Body.Close()
if err != nil {
t.Errorf(URLTest.URL + " cant read response")
} else {
if strings.TrimSpace(string(responseBody)) != URLTest.ExpectedResponseBody {
t.Errorf(URLTest.URL + " response mismatch - have: " + string(responseBody) + " want: " + URLTest.ExpectedResponseBody)
}
}
}
Can I add file as a value like request.FormFile.Add(...) or something?
Regarding your question about how to send a file in an HTTP request with Go, here's some sample code.
And you will need the mime/multipart package to build the form.
package main
import (
"bytes"
"fmt"
"io"
"mime/multipart"
"net/http"
"net/http/httptest"
"net/http/httputil"
"os"
"strings"
)
func main() {
var client *http.Client
var remoteURL string
{
//setup a mocked http client.
ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
b, err := httputil.DumpRequest(r, true)
if err != nil {
panic(err)
}
fmt.Printf("%s", b)
}))
defer ts.Close()
client = ts.Client()
remoteURL = ts.URL
}
//prepare the reader instances to encode
values := map[string]io.Reader{
"file": mustOpen("main.go"), // lets assume its this file
"other": strings.NewReader("hello world!"),
}
err := Upload(client, remoteURL, values)
if err != nil {
panic(err)
}
}
func Upload(client *http.Client, url string, values map[string]io.Reader) (err error) {
// Prepare a form that you will submit to that URL.
var b bytes.Buffer
w := multipart.NewWriter(&b)
for key, r := range values {
var fw io.Writer
if x, ok := r.(io.Closer); ok {
defer x.Close()
}
// Add an image file
if x, ok := r.(*os.File); ok {
if fw, err = w.CreateFormFile(key, x.Name()); err != nil {
return
}
} else {
// Add other fields
if fw, err = w.CreateFormField(key); err != nil {
return
}
}
if _, err = io.Copy(fw, r); err != nil {
return err
}
}
// Don't forget to close the multipart writer.
// If you don't close it, your request will be missing the terminating boundary.
w.Close()
// Now that you have a form, you can submit it to your handler.
req, err := http.NewRequest("POST", url, &b)
if err != nil {
return
}
// Don't forget to set the content type, this will contain the boundary.
req.Header.Set("Content-Type", w.FormDataContentType())
// Submit the request
res, err := client.Do(req)
if err != nil {
return
}
// Check the response
if res.StatusCode != http.StatusOK {
err = fmt.Errorf("bad status: %s", res.Status)
}
return
}
Hope you can use this in your unit test

Requesting multiple URLs in Go

I have the following Go program: https://play.golang.org/p/-TUtJ7DIhi
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strconv"
)
func main() {
body, err := get("https://hacker-news.firebaseio.com/v0/topstories.json")
if err != nil {
panic(err)
}
var ids [500]int
if err = json.Unmarshal(body, &ids); err != nil {
panic(err)
}
var contents []byte
for _, value := range ids[0:10] {
body, err := get("https://hacker-news.firebaseio.com/v0/item/" + strconv.Itoa(value) + ".json")
if err != nil {
fmt.Println(err)
} else {
contents = append(contents, body...)
}
}
fmt.Println(contents)
}
func get(url string) ([]byte, error) {
res, err := http.Get(url)
if err != nil {
return nil, err
}
body, err := ioutil.ReadAll(res.Body)
res.Body.Close()
return body, err
}
When run it throws EOF json errors on the iterative get requests, but when I hit the URLs individually they do not appear to be malformed.
What am I missing?
It looks like there's something wrong with their server, and it's closing connections without sending a Connection: close header. The client therefore tries to reuse the connection per the HTTP/1.1 specification.
You can work around this by creating your own request, and setting Close = true, or using a custom Transport with DisableKeepAlives = true
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
req.Close = true
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}

Resources