How to fix error with Query dynamodb request? - go

In DynamoDB I Have a table that contains:
- email (primary key)
- password (attribute)
- rname (attribute)
I'm using V1 of the AWS Go SDK, to implement to perform a query using just the primary key to my database:
My struct to unMarshal to is:
type Item struct {
Email string `json:"email"`
Password string `json:"password"`
Rname string `json:"rname"`
}
and the code:
result, err := client.Query(&dynamodb.QueryInput{
TableName: aws.String("accountsTable"),
KeyConditions: map[string]*dynamodb.Condition{
"email": {
ComparisonOperator: aws.String("EQ"),
AttributeValueList: []*dynamodb.AttributeValue{
{
S: aws.String(email),
},
},
},
},
})
if err != nil {
return false, err
}
item := []Item{}
err = dynamodbattribute.UnmarshalListOfMaps(result.Items, &item)
if err != nil {
return false, err
}
However, I get the issue that the key is invalid. I check the key in the database and it matches the one i print out to the console too.
Not sure how to get round this issue as example's i've seen seem to work for their's and look the same.
Any help in fixing this issue would be appreciated thanks :)

You need to set the values of password and rname to omitempty so that it's not set to empty values as they are not keys they should not be included on a Query as it throws an invalid key exception:
type Item struct {
Email string `json:"email" dynamodbav:"email,omitempty"`
Password string `json:"password" dynamodbav:"password,omitempty"`
Rname string `json:"rname" dynamodbav:"rname,omitempty"`
}
Update
I believe the issue is due to the fact you try to marshall the entire response in a single command, however, iterating works for me. (I do not use Go).
package main
import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/dynamodb"
"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute"
"fmt"
)
func main() {
// Create Session
sess := session.Must(session.NewSessionWithOptions(session.Options{
SharedConfigState: session.SharedConfigEnable,
}))
// Create DynamoDB Client with Logging
client := dynamodb.New(sess, aws.NewConfig())
type Item struct {
Email string `dynamodbav: "email"`
Password string `dynamodbav: "password,omitempty"`
Rname string `dynamodbav: "rname,omitempty"`
}
result, err := client.Query(&dynamodb.QueryInput{
TableName: aws.String("accountsTable"),
KeyConditions: map[string]*dynamodb.Condition{
"email": {
ComparisonOperator: aws.String("EQ"),
AttributeValueList: []*dynamodb.AttributeValue{
{
S: aws.String("lhnng#amazon.com"),
},
},
},
},
})
if err != nil {
fmt.Println("Query API call failed:")
fmt.Println((err.Error()))
}
for _, i := range result.Items {
item := Item{}
err = dynamodbattribute.UnmarshalMap(i, &item)
if err != nil {
fmt.Println("Got error unmarshalling: %s", err)
}
fmt.Println("Email: ", item.Email)
fmt.Println()
}
}
Moreover, as you use a single key of email, it means there is at most 1 item with the same email address, meaning you should use GetItem rather than Query:
package main
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/dynamodb"
"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute"
)
func main() {
// Item to Get
type Item struct {
Email string `dynamodbav: "email"`
Password string `dynamodbav: "password,omitempty"`
Rname string `dynamodbav: "rname,omitempty"`
}
// Create Session
sess := session.Must(session.NewSessionWithOptions(session.Options{
SharedConfigState: session.SharedConfigEnable,
}))
// Create DynamoDB Client
client := dynamodb.New(sess, aws.NewConfig())
// Get Item
result, err := client.GetItem(&dynamodb.GetItemInput{
TableName: aws.String("accountsTable"),
Key: map[string]*dynamodb.AttributeValue{
"email": {
S: aws.String("lhnng#amazon.com"),
},
},
})
// Catch Error
if err != nil {
fmt.Println("GetItem API call failed:")
fmt.Println((err.Error()))
}
item := Item{}
// Unmarhsall
err = dynamodbattribute.UnmarshalMap(result.Item, &item)
if err != nil {
panic(fmt.Sprintf("Failed to unmarshal Record, %v", err))
}
// If Item Returns Empty
if item.Email == "" {
fmt.Println("Could not find Item")
return
}
// Print Result
fmt.Println("Found item:")
fmt.Println("Email: ", item.Email)
}

Related

dynamodbattribute.MarshalMap returns empty map

I'm attempting to write a really simple Go function to insert an entry into a DynamoDB table.
I'm following the tutorial provided on the AWS Documentation site, but for some reason, the function dynamodbattribute.MarshalMap is returning an empty map.
The code compiles and runs, but breaks when attempting to insert the record as it can't find the required keys in the map. A little println action shows that the map is empty, even though the struct it's created from isn't.
Any help is appreciated!
package main
import (
"context"
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-lambda-go/lambda"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/dynamodb"
"github.com/aws/aws-sdk-go/service/dynamodb/dynamodbattribute"
)
// Item Struct for Fruit Item
type Item struct {
fruitID int
fruitName string
}
/*HandleRequest handles the lambda request.*/
func HandleRequest(ctx context.Context) (string, error) {
// Initialize a session that the SDK will use to load
// credentials from the shared credentials file ~/.aws/credentials
// and region from the shared configuration file ~/.aws/config.
sess := session.Must(session.NewSessionWithOptions(session.Options{
SharedConfigState: session.SharedConfigEnable,
}))
// Create DynamoDB client
svc := dynamodb.New(sess)
item := Item{
fruitID: 2,
fruitName: "Orange",
}
fmt.Println(item)
av, err := dynamodbattribute.MarshalMap(item)
if err != nil {
fmt.Println("Got error marshalling new fruit item: ")
fmt.Println(err.Error())
return "", err
}
fmt.Println("av: ", av)
fmt.Println("ln: ", len(av))
fmt.Println("KEY VALUE PAIRS::")
for key, value := range av {
fmt.Println("Key:", key, "Value:", value)
}
tableName := "fruits"
input := &dynamodb.PutItemInput{
Item: av,
TableName: aws.String(tableName),
}
_, err = svc.PutItem(input)
if err != nil {
fmt.Println("Got error inserting new fruit item: ")
fmt.Println(err.Error())
return "", err
}
return "SUCCESS!", nil
}
func main() {
lambda.Start(HandleRequest)
}
Log output from Lambda
START RequestId: 82f120d9-6bba-40d9-9204-136d491dbb88 Version: $LATEST
{2 Orange}
av: map[]
ln: 0
KEY VALUE PAIRS::
Got error inserting new fruit item:
ValidationException: One or more parameter values were invalid: Missing the key fruitId in the item
status code: 400, request id: QUJG8R8EHPO65O1JFVI98PRQ1VVV4KQNSO5AEMVJF66Q9ASUAAJG
ValidationException: One or more parameter values were invalid: Missing the key fruitId in the item
status code: 400, request id: QUJG8R8EHPO65O1JFVI98PRQ1VVV4KQNSO5AEMVJF66Q9ASUAAJG: requestError
null
END RequestId: 82f120d9-6bba-40d9-9204-136d491dbb88
The fields in struct starts with Capital letter is only exported and can be used in outside of this package.
Can you try with this struct and see if that works ?
type Item struct {
FruitID int
FruitName string
}

Scan a dynamodb table and using contains on a list with go sdk

I have a dynamodb field that looks like this:
[ { "S" : "test#gmail.com" }, { "S" : "test2#gmail.com" } ]
I am trying to run a scan to return any record that e.g. contain test#gmail.com . I am not sure I should use contains to do this, it's currently not returning any records, any pointers as to what I should use?
My go is setup like this:
type Site struct {
ID string `json:"id"`
Site string `json:"site"`
Emails []string `json:"emails,omitempty"`
}
func (ds *datastore) GetEmail(email string, out interface{}) error {
filt := expression.Name("emails").Contains(strings.ToLower(email))
fmt.Println("Get Email", filt)
//filt := expression.Contains(expression.Name("emails"), expression.Value(email))
proj := expression.NamesList(
expression.Name("emails"),
expression.Name("site"),
)
expr, err := expression.NewBuilder().
WithFilter(filt).
WithProjection(proj).
Build()
if err != nil {
fmt.Println(err)
}
scanInput := &dynamodb.ScanInput{
ExpressionAttributeNames: expr.Names(),
ExpressionAttributeValues: expr.Values(),
FilterExpression: expr.Filter(),
ProjectionExpression: expr.Projection(),
TableName: aws.String(ds.TableName),
}
result, err := ds.DDB.Scan(scanInput)
if err != nil {
fmt.Println("what is the err", err)
return err
}
if len(result.Items) == 0 {
fmt.Println("No Email found")
return errors.New(http.StatusText(http.StatusNotFound))
}
err = ds.Marshaler.UnmarshalMap(result.Items[0], out)
return err
}
If you're doing a contains on a partial email it won't match when the filter is applied to a set. It will have to be an exact email match.
{
"Email": "test#gmail.com"
}
// This will match a contains on "test#g"
{
"Emails": ["test#gmail.com", "another#gmail.com"]
}
// this will not match a contains on "test#g" but will match a contains of "test#gmail.com"
See contains: https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Condition.html
Also note you're doing a scan. Scans perform poorly in Dynamodb as soon as your data is of any significant size. Think of storing your data in a different format so you query it via partition keys or use an AWS RDS as an alternative.

What is the proper way to save a slice of structs into Cloud Datastore (Firestore in Datastore Mode)?

I want to save a slice of structs in Google Cloud Datastore (Firestore in Datastore mode).
Take this Phonebook and Contact for example.
type Contact struct {
Key *datastore.Key `json:"id" datastore:"__key__"`
Email string `json:"email" datastore:",noindex"`
Name string `json:"name" datastore:",noindex"`
}
type Phonebook struct {
Contacts []Contact
Title string
}
Saving and loading this struct is no problem as the Datastore library takes care of it.
Due to the presence of some complex properties in my actual code, I need to implement PropertyLoadSaver methods.
Saving the Title property is straightforward. But I have problems storing the slice of Contact structs.
I tried using the SaveStruct method:
func (pb *Phonebook) Save() ([]datastore.Property, error) {
ps := []datastore.Property{
{
Name: "Title",
Value: pb.Title,
NoIndex: true,
},
}
ctt, err := datastore.SaveStruct(pb.Contacts)
if err != nil {
return nil, err
}
ps = append(ps, datastore.Property{
Name: "Contacts",
Value: ctt,
NoIndex: true,
})
return ps, nil
}
This code compiles but doesn't work.
The error message is datastore: invalid entity type
Making a slice of Property explicitly also does not work:
func (pb *Phonebook) Save() ([]datastore.Property, error) {
ps := []datastore.Property{
{
Name: "Title",
Value: pb.Title,
NoIndex: true,
},
}
cttProps := datastore.Property{
Name: "Contacts",
NoIndex: true,
}
if len(pb.Contacts) > 0 {
props := make([]interface{}, 0, len(pb.Contacts))
for _, contact := range pb.Contacts {
ctt, err := datastore.SaveStruct(contact)
if err != nil {
return nil, err
}
props = append(props, ctt)
}
cttProps.Value = props
}
ps = append(ps, cttProps)
return ps, nil
}
Making a slice of Entity does not work either:
func (pb *Phonebook) Save() ([]datastore.Property, error) {
ps := []datastore.Property{
{
Name: "Title",
Value: pb.Title,
NoIndex: true,
},
}
cttProps := datastore.Property{
Name: "Contacts",
NoIndex: true,
}
if len(pb.Contacts) > 0 {
values := make([]datastore.Entity, len(pb.Contacts))
props := make([]interface{}, 0, len(pb.Contacts))
for _, contact := range pb.Contacts {
ctt, err := datastore.SaveStruct(contact)
if err != nil {
return nil, err
}
values = append(values, datastore.Entity{
Properties: ctt,
})
}
for _, v := range values {
props = append(props, v)
}
cttProps.Value = props
}
ps = append(ps, cttProps)
return ps, nil
}
Both yielded the same error datastore: invalid entity type
Finally I resorted to using JSON. The slice of Contact is converted into a JSON array.
func (pb *Phonebook) Save() ([]datastore.Property, error) {
ps := []datastore.Property{
{
Name: "Title",
Value: pb.Title,
NoIndex: true,
},
}
var values []byte
if len(pb.Contacts) > 0 {
js, err := json.Marshal(pb.Contacts)
if err != nil {
return nil, err
}
values = js
}
ps = append(ps, datastore.Property{
Name: "Contacts",
Value: values,
NoIndex: true,
})
return ps, nil
}
Isn't there a better way of doing this other than using JSON?
I found this document and it mentions src must be a struct pointer.
The only reason you seem to customize the saving of PhoneBook seems to be to avoid saving the Contacts slice if there are no contacts. If so, you can just define your PhoneBook as follows and directly use SaveStruct on the PhoneBook object.
type Phonebook struct {
Contacts []Contact `datastore:"Contacts,noindex,omitempty"`
Title string `datastore:"Title,noindex"`
}

Sorting not working in golang chaincode hyperledger fabric

I'm trying to sort the result in golang chaincode, but the result is random, below is my chaincode sample:
package main
import (
"bytes"
"encoding/json"
"fmt"
"time"
"github.com/hyperledger/fabric/core/chaincode/shim"
"github.com/hyperledger/fabric/protos/peer"
)
type itemStruct struct {
ID string `json:"id"`
Status string `json:"status"`
CreatedAt time.Time `json:"created_at"`
}
func createItem(stub shim.ChaincodeStubInterface, args []string) peer.Response {
if len(args) != 3 {
return shim.Error(fmt.Sprintf("Expecting %v arguments {id, status, created_at}, but got %v", 3, len(args)))
}
itemID := args[0]
if len(itemID) == 0 {
return shim.Error("id field is required")
}
status := args[1]
if len(status) == 0 {
return shim.Error("status field is required")
}
createdAt, err := time.Parse(time.RFC3339, args[2])
if err != nil {
return shim.Error("created_at is not a valid datetime string")
}
item := itemStruct{
ID: itemID,
CreatedAt: createdAt,
}
itemAsJSONBytes, err := json.Marshal(item)
if err != nil {
return shim.Error(err.Error())
}
return shim.Success(itemAsJSONBytes)
}
func getPendingItems(stub shim.ChaincodeStubInterface, args []string) peer.Response {
var bookmark string
if len(args) > 0 && len(args[0]) > 0 {
bookmark = args[0]
}
queryString := `{
"selector": {
"status": "pending"
},
"sort": [
{"created_at": "desc"}
]
}`
result, pagination, err := queryWithPagination(stub, queryString, 20, bookmark)
if err != nil {
return shim.Error(err.Error())
}
return shim.Success(constructResponse(result, pagination).Bytes())
}
func queryWithPagination(stub shim.ChaincodeStubInterface, queryString string, pageSize int32, bookmark string) (map[string]string, string, error) {
var pagination string
iterator, meta, err := stub.GetQueryResultWithPagination(queryString, pageSize, bookmark)
if err != nil {
return nil, pagination, err
}
defer iterator.Close()
result, err := iterateResult(iterator)
if err != nil {
return nil, pagination, err
}
pagination = fmt.Sprintf(`{"count": %v, "next_page_token": "%v"}`, meta.FetchedRecordsCount, meta.Bookmark)
return result, pagination, nil
}
func constructResponse(items map[string]string, pagination string) *bytes.Buffer {
// buffer is a JSON array containing QueryResults
var buffer bytes.Buffer
if len(pagination) > 0 {
buffer.WriteString(`{"data":`)
}
buffer.WriteString(`[`)
bArrayMemberAlreadyWritten := false
for _, val := range items {
// Add a comma before array members, suppress it for the first array member
if bArrayMemberAlreadyWritten == true {
buffer.WriteString(",")
}
buffer.WriteString(val)
bArrayMemberAlreadyWritten = true
}
buffer.WriteString("]")
if len(pagination) > 0 {
buffer.WriteString(`,"pagination":`)
buffer.WriteString(pagination)
buffer.WriteString("}")
}
return &buffer
}
func iterateResult(iterator shim.StateQueryIteratorInterface) (map[string]string, error) {
result := map[string]string{}
for iterator.HasNext() {
queryResponse, err := iterator.Next()
if err != nil {
return nil, err
}
result[queryResponse.Key] = string(queryResponse.Value)
}
return result, nil
}
// SmartContract : Smart contract struct
type SmartContract struct {
}
// Init : This method is called when chaincode is initialized or updated.
func (s *SmartContract) Init(stub shim.ChaincodeStubInterface) peer.Response {
return shim.Success(nil)
}
// Invoke : This method is called when any transaction or query fired
func (s *SmartContract) Invoke(stub shim.ChaincodeStubInterface) peer.Response {
// Retrieve the requested Smart Contract function and arguments
function, args := stub.GetFunctionAndParameters()
// Route to the appropriate handler function to interact with the ledger appropriately
// Transactions
if function == "createItem" {
return createItem(stub, args)
}
// Queries
if function == "getItems" {
return getItems(stub, args)
}
return shim.Error("Invalid function")
}
func main() {
// Create a new Smart Contract
err := shim.Start(new(SmartContract))
if err != nil {
fmt.Printf("Error creating new Smart Contract: %s", err)
}
}
It creates and asset which can have different status based on what's passed, and I have defined one query function which fetches only pending items.
I have applied sort, but the result is still random, can anyone help me here and guide me where I'm going wrong in this?
the sort field has to be present in the selector !
something like:
queryString := `{
"selector": {
"created_at": "$gt": null
"status": "pending"
},
"sort": [
{"created_at": "desc"}
]
}`
or (range)
queryString := `{
"selector": {
"created_at": {
"$gt": "2015-01-01T00:00:00Z",
"$lt": "2019-01-01T00:00:00Z"
},
"status": "pending"
},
"sort": [
{"created_at": "desc"}
]
}`
Per the docs - to use sorting, ensure that:
At least one of the sort fields is included in the selector.
There is an index already defined, with all the sort fields in the same
order.
Each object in the sort array has a single key.
If an object in the sort array does not have a single key, the resulting sort order is implementation specific and might change.
Also, as an fyi Find does not support multiple fields with different sort orders, so the directions must be either all ascending or all descending.
See also CouchDB sort doesn't work

How to access struct with a variable?

I'm new to Go and I'm facing issues in accessing a struct with a variable
I have this function decodeUser. My task is to check whether the keys are present in the request. So this is what I did. I've added a comment where I got the error.
func decodeUser(r *http.Request) (root.User, []string, error) {
var u root.User
if r.Body == nil {
return u, []string{}, errors.New("no request body")
}
decoder := json.NewDecoder(r.Body)
checks := []string{
"UserName",
"FirstName",
"LastName",
"Email",
}
emptyFields := []string{}
for _, check := range checks {
// i'm having problem over here `u[check]` it's showing (invalid
operation: u[check] (type root.User does not support
indexing))
if u[check] == nil {
emptyFields = append(emptyFields, check)
}
}
err := decoder.Decode(&u)
return u, emptyFields, err
}
Just in case I added root.User here's structure for it
type User struct {
ID string
Username string
Password string
FirstName string
LastName string
Email string
PhoneNumber string
PhoneNumberExtension string
DOB time.Time
AboutMe string
}
The problem occurs as it doesn't allow me to access struct by a variable and I can't use this method which is u.check. So basically how should I make u[check] work?
I would suggest you manually check for zero values since it seems that you already know the fields that needs to be non-zero at compile time. However, if that is not the case, here is a simple function (using reflection) that will check for zero values in a struct.
func zeroFields(v interface{}, fields ...string) []string {
val := reflect.Indirect(reflect.ValueOf(v))
if val.Kind() != reflect.Struct {
return nil
}
var zeroes []string
for _, name := range fields {
field := val.FieldByName(name)
if !field.IsValid() {
continue
}
zero := reflect.Zero(field.Type())
// check for zero value
if reflect.DeepEqual(zero.Interface(), field.Interface()) {
zeroes = append(zeroes, name)
}
}
return zeroes
}
func main() {
x := User{
Email: "not#nil",
}
fmt.Println(zeroFields(&x, "ID", "Username", "Email", "Something", "DOB"))
}
Which would output:
[ID Username DOB]
Playground
This is what worked for me
for _, check := range checks {
temp := reflect.Indirect(reflect.ValueOf(&u))
fieldValue := temp.FieldByName(string(check))
if (fieldValue.Type().String() == "string" && fieldValue.Len() == 0) || (fieldValue.Type().String() != "string" && fieldValue.IsNil()) {
fmt.Println("EMPTY->", check)
emptyFields = append(emptyFields, check)
}
}

Resources