I'm writing a REST API, through which users can register/login and a number of other things that would require them to send JSON in the request body. The JSON sent in the request body will look something like this for registering:
{
"name": "luke",
"email": "l#g.com",
"date_of_birth": "2012-04-23T18:25:43.511Z",
"location": "Galway",
"profile_image_id": 1,
"password": "password",
"is_subscribee_account": false,
"broker_id": 1
}
This data will then be decoded into a struct and the individual items inserted into the database. I'm not familiar with how I should sanitize the data to remove any Javascript, HTML, SQL etc that may interfere with the correct running of the application and possibly result in a compromised system.
The below code is what I have currently to read in and validate the data but it doesn't sanitize the data for html, javascript, sql etc.
Any input would be appreciated on how I should go about sanitizing it. The REST API is written in Go.
//The below code decodes the JSON into the dst variable.
func (app *application) readJSON(w http.ResponseWriter, r *http.Request, dst interface{}) error {
maxBytes := 1_048_576
r.Body = http.MaxBytesReader(w, r.Body, int64(maxBytes))
dec := json.NewDecoder(r.Body)
dec.DisallowUnknownFields()
err := dec.Decode(dst)
if err != nil {
var syntaxError *json.SyntaxError
var unmarshalTypeError *json.UnmarshalTypeError
var invalidUnmarshalError *json.InvalidUnmarshalError
switch {
case errors.As(err, &syntaxError):
return fmt.Errorf("body contains badly-formed JSON (at character %d)", syntaxError.Offset)
case errors.Is(err, io.ErrUnexpectedEOF):
return errors.New("body contains badly-formed JSON")
case errors.As(err, &unmarshalTypeError):
if unmarshalTypeError.Field != "" {
return fmt.Errorf("body contains incorrect JSON type for field %q", unmarshalTypeError.Field)
}
return fmt.Errorf("body contains incorrect JSON type (at character %d)", unmarshalTypeError.Offset)
case errors.Is(err, io.EOF):
return errors.New("body must not be empty")
case strings.HasPrefix(err.Error(), "json: unknown field "):
fieldName := strings.TrimPrefix(err.Error(), "json: unknown field ")
return fmt.Errorf("body contains unknown key %s", fieldName)
case err.Error() == "http: request body too large":
return fmt.Errorf("body must not be larger than %d bytes", maxBytes)
case errors.As(err, &invalidUnmarshalError):
panic(err)
default:
return err
}
}
err = dec.Decode(&struct{}{})
if err != io.EOF {
return errors.New("body must only contain a single JSON value")
}
return nil
}
//The below code validates the email, password and user. If the first parameter in V.Check() is false the following two parameter are returned in the response to the user.
func ValidateEmail(v *validator.Validator, email string) {
v.Check(email != "", "email", "must be provided")
v.Check(validator.Matches(email, validator.EmailRX), "email", "must be a valid email address")
}
func ValidatePasswordPlaintext(v *validator.Validator, password string) {
v.Check(password != "", "password", "must be provided")
v.Check(len(password) >= 8, "password", "must be at least 8 bytes long")
v.Check(len(password) <= 72, "password", "must not be more than 72 bytes long")
}
func ValidateUser(v *validator.Validator, user *User) {
v.Check(user.Name != "", "name", "must be provided")
v.Check(len(user.Name) <= 500, "name", "must not be more than 500 bytes long")
ValidateEmail(v, user.Email)
if user.Password.plaintext != nil {
ValidatePasswordPlaintext(v, *user.Password.plaintext)
}
if user.Password.hash == nil {
panic("missing password hash for user")
}
}
How do I serialize the struct data with this in mind? So it wont cause issues if later inserted in a webpage. Is there a package function that I can pass the string data to that will return safe data?
Related
I'm currently using gRPC for my API communication. Now I need the value of my request can accept any data type, be it a struct object or just a primitive data type like single integer, string, or boolean. I tried using google.protobuf.Any as below but it doesn't work when I want to marshal and unmarshal it.
message MyRequest {
google.protobuf.Any item = 1; // could be 9, "a string", true, false, {"key": value}, etc.
}
proto-compiled result:
type MyRequest struct {
Item *anypb.Any `protobuf:"bytes,1,opt,name=item,proto3" json:"item,omitempty"`
}
I will marshal the incoming request through gRPC and save it as a JSON string in the database:
bytes, err = json.Marshal(m.request.Item)
if err != nil {
return err
}
itemStr := string(bytes)
But when I want to unmarshal the string back into *anypb.Any by:
func getItem(itemStr string) (structpb.Value, error) {
var item structpb.Value
err := json.Unmarshal([]byte(itemStr), &item)
if err != nil {
return item, err
}
// also I've no idea on how to convert `structpb.Value` to `anypb.Any`
return item, nil
}
It returns an error which seems to be because the struct *anypb.Any doesn't contain any of the fields in my encoded item string. Is there a correct way to solve this problem?
Consider using anypb
In the documentation it shows an example of how to unmarshal it
m := new(foopb.MyMessage)
if err := any.UnmarshalTo(m); err != nil {
... // handle error
}
... // make use of m
I am trying to ensure the body of a post request for example contains exact structure of the body and if not ahould throw an error
for example i have the following function
func UpdatePassword(c *fiber.Ctx) error {
type UpdatePasswordData struct {
Password string `json:"password" form:"password"`
NewPassword string `json:"new_password" form:"new_password"`
NewPasswordConfirm string `json:"new_password_confirm" form:"new_password_confirm"`
}
data := UpdatePasswordData{}
if err := c.BodyParser(&data); err != nil {
return err
}
var user models.User
if data.NewPassword != data.NewPasswordConfirm {
c.Status(400)
return c.JSON(fiber.Map{
"message": "passwords do not match",
})
}
email, _ := middlewares.GetUserEmail(c)
newPassword := models.HashPassword(data.NewPassword)
database.DB.Model(&user).Select("Password").Where("email = ?", email).Updates(map[string]interface{}{"Password": newPassword})
return c.JSON(user)
}
the POST request should be looking for body with this structure
{
"password": "oldpassword",
"new_password": "newpassword",
"new_password_confirm": "newpassword",
}
but currently this endpoint accepts body that does not have this exact structure. So how do i enforce the structure in the body of request, so that if structure does not match, i throw an error?
do not like gin, fiber has not builtin validate package
use go-playground/validator
go get github.com/go-playground/validator
example
type UpdatePasswordData struct {
Password string `json:"password" validate:"required,min=8,max=32"`
NewPassword string `json:"new_password" validate:"required,min=8,max=32"`
NewPasswordConfirm string `json:"new_password_confirm" validate:"eqfield=NewPassword"`
}
func UpdatePassword(c *fiber.Ctx) error {
var body UpdatePasswordData
if err := c.BodyParser(&body); err != nil {
return err
}
validate := validator.New()
if err := validate.Struct(body); err != nil {
return err
}
// do others
// get current user, check password == hash(body.password)
// save new passworld
}
or you can see fiber office docs https://docs.gofiber.io/guide/validation#validator-package
We can use struct tag
`validate:"required"`
to ensure that all the mandatory fields are there in the request payload.
Moreover we can validate the fields with the provided tags of the validator package and for additional validations we can implement custom validators and register them like this:
validate := validator.New()
validate.RegisterValidation("password-validator", PasswordValidator)
I have something like a data pipeline.
API response (10k) rows as JSON.
=> Sanitize some of the data into a new structure
=> Create a CSV File
I can currently do that by getting the full response and doing that step by step.
I was wondering if there's a simpler way to stream the response reading into CSV right away and also writing in the file as it goes over the request-response.
Current code:
I will have a JSON like { "name": "Full Name", ...( 20 columns)} and that data repeats about 10-20k times with different values.
For request
var res *http.Response
if res, err = client.Do(request); err != nil {
return errors.Wrap(err, "failed to perform request")
}
For Unmarshal
var record []RecordStruct
if err = json.NewDecoder(res.Body).Decode(&record); err != nil {
return err
}
For CSV
var row []byte
if row, err = csvutil.Marshal(record); err != nil {
return err
}
To stream an array of JSON objects you have to decode nested objects instead of root object. To do this you need read data using tokens (check out Token method). According to the documentation:
Token returns the next JSON token in the input stream. At the end of the input stream, Token returns nil, io.EOF.
Token guarantees that the delimiters [ ] { } it returns are properly nested and matched: if Token encounters an unexpected delimiter in the input, it will return an error.
The input stream consists of basic JSON values—bool, string, number, and null—along with delimiters [ ] { } of type Delim to mark the start and end of arrays and objects. Commas and colons are elided.
That mean you can decode document part by part. Find an official example how to do it here
I will post a code snippet that shows how you can combine json stream technic with writing result to the CSV:
package main
import (
"encoding/csv"
"encoding/json"
"log"
"os"
"strings"
)
type RecordStruct struct {
Name string `json:"name"`
Info string `json:"info"`
// ... any field you want
}
func (rs *RecordStruct) CSVRecord() []string {
// Here we form data for CSV writer
return []string{rs.Name, rs.Info}
}
const jsonData =
`[
{ "name": "Full Name", "info": "..."},
{ "name": "Full Name", "info": "..."},
{ "name": "Full Name", "info": "..."},
{ "name": "Full Name", "info": "..."},
{ "name": "Full Name", "info": "..."}
]`
func main() {
// Create file for storing our result
file, err := os.Create("result.csv")
if err != nil {
log.Fatalln(err)
}
defer file.Close()
// Create CSV writer using standard "encoding/csv" package
var w = csv.NewWriter(file)
// Put your reader here. In this case I use strings.Reader
// If you are getting data through http it will be resp.Body
var jsonReader = strings.NewReader(jsonData)
// Create JSON decoder using "encoding/json" package
decoder := json.NewDecoder(jsonReader)
// Token returns the next JSON token in the input stream.
// At the end of the input stream, Token returns nil, io.EOF.
// In this case our first token is '[', i.e. array start
_, err = decoder.Token()
if err != nil {
log.Fatalln(err)
}
// More reports whether there is another element in the
// current array or object being parsed.
for decoder.More() {
var record RecordStruct
// Decode only the one item from our array
if err := decoder.Decode(&record); err != nil {
log.Fatalln(err)
}
// Convert and put out record to the csv file
if err := writeToCSV(w, record.CSVRecord()); err != nil {
log.Fatalln(err)
}
}
// Our last token is ']', i.e. array end
_, err = decoder.Token()
if err != nil {
log.Fatalln(err)
}
}
func writeToCSV(w *csv.Writer, record []string) error {
if err := w.Write(record); err != nil {
return err
}
w.Flush()
return nil
}
You can also use 3d party packages like github.com/bcicen/jstream
I am trying to test my UserRegister functionality, it takes http request.
If user enters already existing email, UserRegister returns an error log (using logrus).
logs "github.com/sirupsen/logrus"
func UserRegister(res http.ResponseWriter, req *http.Request) {
requestID := req.FormValue("uid")
email := req.FormValue("email")
logs.WithFields(logs.Fields{
"Service": "User Service",
"package": "register",
"function": "UserRegister",
"uuid": requestID,
"email": email,
}).Info("Received data to insert to users table")
// check user entered new email address
hasAccount := checkemail.Checkmail(email, requestID) // returns true/false
if hasAccount != true { // User doesn't have an account
db := dbConn()
// Inserting token to login_token table
insertUser, err := db.Prepare("INSERT INTO users (email) VALUES(?)")
if err != nil {
logs.WithFields(logs.Fields{
"Service": "User Service",
"package": "register",
"function": "UserRegister",
"uuid": requestID,
"Error": err,
}).Error("Couldnt prepare insert statement for users table")
}
insertUser.Exec(email)
defer db.Close()
return
} // user account created
logs.WithFields(logs.Fields{
"Service": "User Service",
"package": "register",
"function": "UserRegister",
"uuid": requestID,
"email": email,
}).Error("User has an account for this email")
}
In my test module, I used following.
func TestUserRegister(t *testing.T) {
rec := httptest.NewRecorder()
req, _ := http.NewRequest("POST", "http://localhost:7071/register?email=sachit45345h#gmail.com&uid=sjfkjsdkf9w89w83490w", nil)
UserRegister(rec, req)
expected := "User has an account for this email"
res := rec.Result()
content, err := ioutil.ReadAll(res.Body)
if err != nil {
t.Error("Couldnt read body", err)
}
val, err := strconv.Atoi(string(bytes.TrimSpace(content)))
if err != nil {
log.Println("Error parsing response", err)
}
if string(val) != expected {
t.Errorf("Expected %s, got %s", expected, string(content))
}
}
Result : Error parsing response strconv.Atoi: parsing "": invalid syntax
Why response can not be converted?
Checked threads:
Why is this Golang code to convert a string to an integer failing.
Edit : after #chmike answer.
This is a part of microservice. All the responses are written to API-Gateway. Using a function.
But here I just want to perform unit test and check whether my UserRegister works as expected.
The function UserRegister never writes to res or sets the status. As a consequence you get an empty string from res in TestUserRegister. content is an empty string and the conversion of an empty string to an integer with Atoi fails since there is no integer to convert.
I can only explain what happens. I can’t tell you what to do to fix the problem because you don’t explain what you want to do or get in the question.
The http response you are reading is not the response to your request. You create a response and expect it to have something in it. it will not. So you end up trying to create an integer from an empty string.
Look at some examples to see where the real response would come from. https://golang.org/pkg/net/http
This is my struct, when I get a socket message I readJson and the structs gets filled with the data and all is fine. It goes through some functions, but once it goes through the Send function it serializes it in a weird way that eventually I get back a bunch of numbers and when I convert it to string, data is missing.
type Reply struct {
Topic string `redis:"topic" json:"topic"`
Ref string `redis:"ref" json:"ref"`
Payload struct {
Status string `redis:"status" json:"status"`
Response map[string]interface{} `redis:"response" json:"response"`
} `json:"payload"`
}
I just want to broadcast messages in this format.
This is where I get the modified and problematic data
func (rr *redisReceiver) run() error {
l := log.WithField("channel", Channel)
conn := rr.pool.Get()
defer conn.Close()
psc := redis.PubSubConn{Conn: conn}
psc.Subscribe(Channel)
go rr.connHandler()
for {
switch v := psc.Receive().(type) {
case redis.Message:
rr.broadcast(v.Data)
case redis.Subscription:
l.WithFields(logrus.Fields{
"kind": v.Kind,
"count": v.Count,
}).Println("Redis Subscription Received")
log.Println("Redis Subscription Received")
case error:
return errors.New("Error while subscribed to Redis channel")
default:
l.WithField("v", v).Info("Unknown Redis receive during subscription")
log.Println("Unknown Redis receive during subscription")
}
}
}
Does Redigo not support that type of data structure?
This is the format I get and the format I'm supposed to get.
//Get
"{{spr_reply sketchpad map[] 1} {ok map[success:Joined successfully]}}"
//Supposed to get
{event: "spr_reply", topic: "sketchpad", ref: "45", payload: {status:
"ok", response: {}}}
On line 55 is where I get back the "corrupted" data - https://play.golang.org/p/TOzJuvewlP
Redigo supports the following conversions to Redis bulk strings:
Go Type Conversion
[]byte Sent as is
string Sent as is
int, int64 strconv.FormatInt(v)
float64 strconv.FormatFloat(v, 'g', -1, 64)
bool true -> "1", false -> "0"
nil ""
all other types fmt.Print(v)
The Reply type is encoding using fmt.Print(v).
It looks like you want to encode the value as JSON. If so, do the encoding in the application. You can remove the redis field tags.
writeToRedis(conn redis.Conn, data Reply) error {
p, err := json.Marshl(data)
if err != nil {
return errors.Wrap(err, "Unable to encode message to json")
}
if err := conn.Send("PUBLISH", Channel, p); err != nil {
return errors.Wrap(err, "Unable to publish message to Redis")
}
if err := conn.Flush(); err != nil {
return errors.Wrap(err, "Unable to flush published message to Redis")
}
return nil
}