How to solve the problem of access conflict to shared resources? - go

There is a test service with 2 requests. Those requests use a shared resource in the form of the ActualOrders variable. Suppose that hundreds of parallel queries are running, there is a chance that a data conflict will occur in the ActualOrders variable. Especially when I'm looping through an array. To prevent this, will it be enough to use a Mutex, as I did in the example below?
main.go:
package main
import (
"encoding/json"
"errors"
"fmt"
"net/http"
"os"
"time"
)
type Order struct {
Room string `json:"room"`
UserEmail string `json:"email"`
From time.Time `json:"from"`
To time.Time `json:"to"`
}
var ActualOrders = []Order{}
var mutex sync.Mutex
func getOrders(responseWriter http.ResponseWriter, request *http.Request) {
userEmail := request.URL.Query().Get("email")
results := []Order{}
mutex.Lock()
for _, item := range ActualOrders {
if item.UserEmail == userEmail {
results = append(results, item)
}
}
mutex.Unlock()
bytes, err := json.Marshal(results)
if err != nil {
http.Error(responseWriter, err.Error(), http.StatusInternalServerError)
return
}
responseWriter.Header().Set("Content-type", "application/json")
responseWriter.WriteHeader(http.StatusOK)
responseWriter.Write(bytes)
}
func createOrder(responseWriter http.ResponseWriter, request *http.Request) {
var newOrder Order
requestBody := request.Body
defer request.Body.Close()
err := json.NewDecoder(requestBody).Decode(&newOrder)
if err != nil {
http.Error(responseWriter, err.Error(), http.StatusBadRequest)
return
}
mutex.Lock()
for _, order := range ActualOrders {
if !(newOrder.To.Before(order.From) || newOrder.From.After(order.To)) {
http.Error(responseWriter, http.StatusText(http.StatusConflict), http.StatusConflict)
return
}
}
ActualOrders = append(ActualOrders, newOrder)
mutex.Unlock()
responseWriter.WriteHeader(http.StatusCreated)
}
func main() {
mux := http.NewServeMux()
mux.HandleFunc("/orders", getOrders)
mux.HandleFunc("/order", createOrder)
err := http.ListenAndServe(":8080", mux)
if errors.Is(err, http.ErrServerClosed) {
fmt.Printf("server closed\n")
} else if err != nil {
fmt.Printf("error starting server: %s\n", err)
os.Exit(1)
}
}

Using a mutex as you did will protect from data races. Your implementation can be improved though.
You can use a RWMutex, use a read-lock for the getOrders function, and a lock for the createOrder function. This will allow exclusive access to the ActualOrders variable when you are writing to it, but shared reads will be allowed:
var mutex sync.RWMutex
func getOrders(responseWriter http.ResponseWriter, request *http.Request) {
...
mutex.RLock()
...
mutex.RUnlock()
}
func createOrder(responseWriter http.ResponseWriter, request *http.Request) {
...
mutex.Lock()
for _, order := range ActualOrders {
...
}
ActualOrders = append(ActualOrders, newOrder)
mutex.Unlock()
}

Related

variable is empty but later has a value

I'm trying to develop a Terraform provider but I have a problem of the first request body. Here is the code:
type Body struct {
id string
}
func resourceServerCreate(d *schema.ResourceData, m interface{}) error {
key := d.Get("key").(string)
token := d.Get("token").(string)
workspace_name := d.Get("workspace_name").(string)
board_name := d.Get("board_name").(string)
resp, err := http.Post("https://api.trello.com/1/organizations?key="+key+"&token="+token+"&displayName="+workspace_name,"application/json",nil)
if err != nil {
log.Fatalln(err)
}
defer resp.Body.Close()
//lettura body.
body := new(Body)
json.NewDecoder(resp.Body).Decode(body)
log.Println("[ORCA MADONNA] il log funzia "+body.id)
d.Set("board_id",body.id)
resp1, err1 := http.Post("https://api.trello.com/1/boards?key="+key+"&token="+token+"&idOrganization="+body.id+"&=&name="+board_name,"application/json",nil)
if err1 != nil {
log.Fatalln(resp1)
}
defer resp1.Body.Close()
d.SetId(board_name)
return resourceServerRead(d, m)
}
In the log is empty, but the second call have it and work fine. How is it possible?
Go doesn't force you to check error responses, therefore it's easy to make silly mistakes. Had you checked the return value from Decode(), you would have immediately discovered a problem.
err := json.NewDecoder(resp.Body).Decode(body)
if err != nil {
log.Fatal("Decode error: ", err)
}
Decode error: json: Unmarshal(non-pointer main.Body)
So your most immediate fix is to use & to pass a pointer to Decode():
json.NewDecoder(resp.Body).Decode(&body)
Also of note, some programming editors will highlight this mistake for you:
Here's a working demonstration, including a corrected Body structure as described at json.Marshal(struct) returns “{}”:
package main
import (
"bytes"
"encoding/json"
"fmt"
"log"
"net/http"
"time"
)
type JSON = map[string]interface{}
type JSONArray = []interface{}
func ErrFatal(err error, msg string) {
if err != nil {
log.Fatal(msg+": ", err)
}
}
func handleTestRequest(w http.ResponseWriter, req *http.Request) {
w.Write(([]byte)("{\"id\":\"yourid\"}"))
}
func launchTestServer() {
http.HandleFunc("/", handleTestRequest)
go http.ListenAndServe(":8080", nil)
time.Sleep(1 * time.Second) // allow server to get started
}
// Medium: "Don’t use Go’s default HTTP client (in production)"
var restClient = &http.Client{
Timeout: time.Second * 10,
}
func DoREST(method, url string, headers, payload JSON) *http.Response {
requestPayload, err := json.Marshal(payload)
ErrFatal(err, "json.Marshal(payload")
request, err := http.NewRequest(method, url, bytes.NewBuffer(requestPayload))
ErrFatal(err, "NewRequest "+method+" "+url)
for k, v := range headers {
request.Header.Add(k, v.(string))
}
response, err := restClient.Do(request)
ErrFatal(err, "DoRest client.Do")
return response
}
type Body struct {
Id string `json:"id"`
}
func clientDemo() {
response := DoREST("POST", "http://localhost:8080", JSON{}, JSON{})
defer response.Body.Close()
var body Body
err := json.NewDecoder(response.Body).Decode(&body)
ErrFatal(err, "Decode")
fmt.Printf("Body: %#v\n", body)
}
func main() {
launchTestServer()
for i := 0; i < 5; i++ {
clientDemo()
}
}

cannot encode json.decoded request body

I have a server implementation. Now I am writing unit test to check it's functionalities.
I cannot prepare request, that would unmarshall on the server side well. Code below results with InvalidUnmarshallError. I don't know, how to debug it further.
Client side code:
body := PatchCatRequest{Adopted: true}
bodyBuf := &bytes.Buffer{}
err := json.NewEncoder(bodyBuf).Encode(body)
assert.NoError(t, err)
req, err := http.NewRequest("PATCH", URL+"/"+catId, bodyBuf)
recorder := httptest.NewRecorder()
handler.PatchCat(recorder, req.WithContext(ctx))
Server side code:
type PatchCatRequest struct {
Adopted bool `json:"adopted"`
}
func (h *Handler) PatchCat (rw http.ResponseWriter, req *http.Request) {
var patchRequest *PatchCatRequest
if err := json.NewDecoder(req.Body).Decode(patchRequest); err != nil {
rw.WriteHeader(http.StatusBadRequest)
logger.WithField("error", err.Error()).Error(ErrDocodeRequest.Error())
return
}
...
}
You are unmarshaling into a nil pointer, as the error message says:
package main
import (
"encoding/json"
"fmt"
)
type PatchCatRequest struct {
Adopted bool
}
func main() {
var patchRequest *PatchCatRequest // nil pointer
err := json.Unmarshal([]byte(`{"Adopted":true}`), patchRequest)
fmt.Println(err) // json: Unmarshal(nil *main.PatchCatRequest)
}
https://play.golang.org/p/vt7t5BgT3lA
Initialize the pointer before unmarshaling:
func main() {
patchRequest := new(PatchCatRequest) // non-nil pointer
err := json.Unmarshal([]byte(`{"Adopted":true}`), patchRequest)
fmt.Println(err) // <nil>
}
https://play.golang.org/p/BqliguktWmr

Go equivalent of Python's requests.Session for making many requests with the same basic authentication?

Consider this example for making an HTTP request in Go with basic authentication:
package main
import (
"encoding/base64"
"fmt"
"io/ioutil"
"net/http"
"net/http/httptest"
"strings"
)
var userName = "myUserName"
var password = "myPassword"
func main() {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if !checkAuth(w, r) {
http.Error(w, "You're not authorized!", http.StatusUnauthorized)
return
}
w.Write([]byte("You're authorized!"))
}))
defer ts.Close()
req, err := http.NewRequest("GET", ts.URL, nil)
check(err)
req.SetBasicAuth(userName, password+"foo")
resp, err := http.DefaultClient.Do(req)
check(err)
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
check(err)
fmt.Println(string(body))
}
// checkAuth checks authentication (cf. https://stackoverflow.com/questions/21936332/idiomatic-way-of-requiring-http-basic-auth-in-go/21937924#21937924)
func checkAuth(w http.ResponseWriter, r *http.Request) bool {
s := strings.SplitN(r.Header.Get("Authorization"), " ", 2)
if len(s) != 2 {
return false
}
b, err := base64.StdEncoding.DecodeString(s[1])
if err != nil {
return false
}
pair := strings.SplitN(string(b), ":", 2)
if len(pair) != 2 {
return false
}
return pair[0] == userName && pair[1] == password
}
func check(err error) {
if err != nil {
panic(err)
}
}
Note that SetBasicAuth is a method of an *http.Request, so if I want to make many requests, I would have to call this method on each request.
In Python, you can define a requests.Session like in this example (from https://requests.readthedocs.io/en/master/user/advanced/#session-objects):
s = requests.Session()
s.auth = ('user', 'pass')
s.headers.update({'x-test': 'true'})
# both 'x-test' and 'x-test2' are sent
s.get('https://httpbin.org/headers', headers={'x-test2': 'true'})
Is there an idiomatic way of defining the equivalent of a requests.Session in Go (preferably using the standard library)? All I can think of is defining a custom client struct with its own Do() method:
type MyClient struct {
UserName, Password string
}
func (client *MyClient) Do(req *http.Request) (*http.Response, error) {
req.SetBasicAuth(client.UserName, client.Password)
return http.DefaultClient.Do(req)
}
and invoking it in the above script like
client := MyClient{UserName: userName, Password: password}
resp, err := client.Do(req)
Would this be an idiomatic way to avoid multiple calls to SetBasicAuth()?

How can you longpoll multiple urls in Go?

Here's what I have thus far:
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
)
func subscribe(urls Urls) []byte {
req, err := http.NewRequest("GET", urls.Url, nil)
if err != nil {
log.Fatal(err)
}
req.Header.Set("authentication", "Bearer " + urls.Token)
http_client := &http.Client{}
res, err := http_client.Do(req)
if err != nil {
log.Fatal(err)
}
defer res.Body.Close()
resourceResp, err := ioutil.ReadAll(res.Body)
if err != nil {
log.Fatal(err)
}
fmt.Println(string(resourceResp))
var data map[string]interface{}
error := json.Unmarshal([]byte(resourceResp), &data)
if error != nil {
log.Fatal(error)
}
return subscribe(urls)
}
type Urls struct {
Url string
Token string
}
func main() {
var urls [2]Urls
urls[0] = Urls{
Url: "https://example.com/users/8",
Token: "abcdefg",
}
urls[1] = Urls{
Url: "https://example.com/users/9",
Token: "hijklmnop",
}
subscribe(urls[0])
subscribe(urls[1])
}
The end goal is to "subscribe" to the multiple urls and pull any updated data (eventually adding it to a queue, but one step at a time). After that, reestablish the connection. Right now, only the first subscribe gets run. Thanks!
I think what you're asking is for the subscribe functions to be run in parallel. One way is to wrap them in goroutines and wait for all the goroutines to finish:
func main() {
...
...
var wg sync.WaitGroup
wg.Add(len(urls))
for _, url := range(urls) {
go func() {
defer wg.Done()
subscribe(url)
}()
}
wg.Wait()
}

Poll API, pass result to chan, pass from chan to Websocket. Panic

I'm writing a small package which does a GET request to an external API every 2 seconds. It takes the value from this request and passes it into a channel. I have made this channel available to a http.handler (chi router) which upgrades to a websocket where the front-end will grab the value in realtime. the panic error is a lot of lines but i guess the most important is this:
2018/11/14 16:47:55 http: response.WriteHeader on hijacked connection
2018/11/14 16:47:55 http: response.Write on hijacked connection
Aside from that I'm sure there is a better way of doing this. Any experienced Gophers out there have any pointers to help a noob such as myself improve this?
package currencyticker
import (
"bitbucket.org/special/api/config"
"encoding/json"
"fmt"
"github.com/go-chi/chi"
"github.com/go-chi/render"
"github.com/gorilla/websocket"
"github.com/leekchan/accounting"
"io/ioutil"
"log"
"math/big"
"net/http"
"time"
)
var (
ac = accounting.Accounting{Precision: 2}
from = "USD"
to = "EUR,SWK"
url = "https://min-api.currencyapi.com/data/price?fsym=" + from + "&tsyms=" + to
messages = make(chan float64)
)
var wsupgrader = websocket.Upgrader{
ReadBufferSize: 1024,
WriteBufferSize: 1024,
CheckOrigin: func(r *http.Request) bool {
return true // Disable CORS for testing
},
}
// Config - init
type Config struct {
*config.Config
}
type result map[string]float64
// New - init the configs
func New(configuration *config.Config) *Config {
return &Config{configuration}
}
// Routes - api urls
func (config *Config) Routes() *chi.Mux {
router := chi.NewRouter()
router.Use(
render.SetContentType(render.ContentTypeHTML), // Set content-Type headers as application/json
)
router.Get("/", config.GetPrice) // subscribe to new tweets
return router
}
func (config *Config) GetPrice(w http.ResponseWriter, r *http.Request) {
conn, err := wsupgrader.Upgrade(w, r, nil)
if err != nil {
fmt.Println(fmt.Printf("Failed to set websocket upgrade: %+v ", err))
return
}
for {
time.Sleep(1 * time.Second)
price := <-messages
w, err := conn.NextWriter(websocket.TextMessage)
if err != nil {
fmt.Println("ws error", err)
}
currVal := ac.FormatMoneyBigFloat(big.NewFloat(price))
if _, err := w.Write([]byte(currVal)); err != nil {
fmt.Printf("w.Write() returned %v", err)
}
w.Close()
}
}
// start getting the price of ether as soon as they ap starts
func init() {
go startPollingPriceAPI()
}
// Go Routine to start polling
func startPollingPriceAPI() {
for {
time.Sleep(2 * time.Second)
go getPriceFromAPI()
}
}
func getPriceFromAPI() {
w := http.Client{
// Timeout: time.Second * 3,
}
req, _ := http.NewRequest(http.MethodGet, url, nil)
res, err := w.Do(req)
if err != nil {
log.Println("err getting price [req]: ", err)
}
body, err := ioutil.ReadAll(res.Body)
if err != nil {
log.Println("err getting price [io-read]: ", err)
}
r := result{}
if jsonErr := json.Unmarshal(body, &r); jsonErr != nil {
log.Println("err getting price [json]: ", jsonErr)
}
fmt.Println("1 Dollar = €", r["EUR"])
messages <- r["EUR"]
}

Resources