Has anybody tried storing result from a query to a map?
I want to able to read data from BQ tables without having the need to define a struct that matches the BQ table schema.
I have tried following https://kylewbanks.com/blog/query-result-to-map-in-golang, but I want to use a RowIterator instead of the approach in this link.
Here's the code I am struggling with:
//Removed error handling for brewity
ctx := context.Background()
client, _ := bigquery.NewClient(ctx, ProjectID)
query := fmt.Sprintf("SELECT * FROM `%s.%s.%s` LIMIT 5;", ProjectID, DatasetId, ResourceName)
queryResult := client.Query(query)
it, _ := queryResult.Read(ctx)
for {
row := make(map[string]bigquery.Value)
err := it.Next(&row)
if err == iterator.Done {
break
}
if err != nil {
fmt.Printf("Error happened")
}}
I am not sure how to proceed after this, I would ideally like to convert the data into a JSON format.
for {
var values []bigquery.Value
err := it.Next(&values)
if err == iterator.Done {
break
}
if err != nil {
// TODO: Handle error.
}
fmt.Println(values)
}
Place rows into slice as you can store a row using anything that implements the ValueLoader interface, or with a slice or map of bigquery.Value
ref: godocs bq
Related
I am new to mongo and mongo go driver. Need to add new field "uri" to my collection with existing data - using mongo go driver. New field needs to be populated with unique values so that unique index can be created on it. the collection uses _id as well, if there is a way we can populate new field based on _id field that will work as well.
I am trying below code, not sure how to populate unique values.
//Step1: update all documents to add new field with unique values
_, err := myColl.UpdateMany(
ctx,
bson.D{},// select all docs in collection
bson.D{
{"$set", bson.D{{"uri", GenerateRandomUniqueString()}}},
},
)
if err != nil {
return err
}
// then next step is to create index on this field:
key := bson.D{{"uri", 1}}
opt := options.Index().SetName("uri-index").SetUnique(true)
model := mongo.IndexModel{Keys: key, Options: opt}
_, err = myColl.Indexes().CreateOne(ctx, model)
if err != nil {
return err
}
Once the index is set up, old records will marked read only, but we can not delete those. New data will have unique 'uri' string value.
Any help is much appreciated.
Using above code fails while unique index creation, as the same value is used for backfill.
I tried this as well:
func BackFillUri(db *mongo.Database) error {
myColl := db.Collection("myColl")
ctx := context.Background()
cursor, err := myColl.Find(ctx, bson.M{})
if err != nil {
return err
}
defer cursor.Close(ctx)
for cursor.Next(ctx) {
var ds bson.M
if err = cursor.Decode(&ds); err != nil {
return err
}
_, err1 := myColl.UpdateOne(
ctx,
bson.D{"_id": ds.ObjectId},
bson.D{
{"$set", bson.D{{"uri", rand.Float64()}}},
},
)
if err1 != nil {
return err1
}
}
return nil
}
But i am getting quite a few errors and not sure if any of the above logic is correct
I finally used below code, hope it helps someone who's new like me :-)
const charset = "abcdefghijklmnopqrstuvwxyz" +
"ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"
func RandomUniqueString(length int) string {
return StringWithCharset(length, charset)
}
var seededRand *rand.Rand = rand.New(
rand.NewSource(time.Now().UnixNano()))
func StringWithCharset(length int, charset string) string {
b := make([]byte, length)
for i := range b {
b[i] = charset[seededRand.Intn(len(charset))]
}
return string(b)
}
// Adds index on uri
func AddUriIndex(db *mongo.Database) error {
mycoll := db.Collection("mycoll")
ctx := context.Background()
//backfill code starts
type attribute struct {
Key string `bson:"key"`
Value interface{} `bson:"value"`
}
type item struct {
ID primitive.ObjectID `bson:"_id"`
ResourceAttributes []attribute `bson:"resourceAttributes,omitempty"`
}
cursor, err := mycoll.Find(ctx, primitive.M{})
if err != nil {
return err
}
defer cursor.Close(ctx)
for cursor.Next(ctx) {
var result item
if err = cursor.Decode(&result); err != nil {
return err
}
//fmt.Println("Found() result:", result)
filter := primitive.M{"_id": result.ID}
update := primitive.M{"$set": primitive.M{"uri": RandomUniqueString(32)}}
if _, err := mycoll.UpdateOne(ctx, filter, update); err != nil {
return err
}
}
//add uri-index starts
key := bson.D{{"uri", 1}}
opt := options.Index().
SetName("uri-index").
SetUnique(true)
model := mongo.IndexModel{Keys: key, Options: opt}
_, err = mycoll.Indexes().CreateOne(ctx, model)
if err != nil {
return err
}
return nil
}
If anyone had tried this before using Go, please get the idea with code, that would be really appreciated.
I wrote few line which is slow
// This is to read the csv file
func usersFileLoader(filename string, channel chan User) {
defer close(channel)
file, err := os.Open(filename)
if err != nil {
panic(err)
}
defer file.Close()
var user User
reader := csv.NewReader(file)
for {
err := Unmarshal(reader, &user)
if err == io.EOF {
break
}
if err != nil {
panic(err)
}
channel <- user
}
}
// This is to insert csv file
func saveUser(channel <-chan User, db *sql.DB) {
stmt, err := db.Prepare(`
INSERT INTO Users( id, name, address) values ( ?, ?, ?)`)
if err != nil {
log.Fatal(err)
}
for usr := range channel {
_, err := stmt.Exec(
usr.ID,
usr.Name,
usr.Address,
)
if err != nil {
log.Fatal(err)
}
}
}
// here is the struct of the user
type User struct {
ID int `csv:"id"`
Name int `csv:"name"`
Address int `csv:"address"`
}
// here is my main func
func main() {
db := DBconnect(ConnectionString(dbConfig()))
channel := make(chan User)
go usersFileLoader("../user.csv", channel)
saveUser(channel, db)
defer db.Close()
}
// This code is working but slow for me.
Share your thought and ideas
I wouldn't attempt to use Go's built in standard library functions for loading a very large CSV file into MySQL (unless, of course, you are simply trying to learn how they work).
For best performance I would simply use MySQL's built in LOAD DATA INFILE functionality.
For example:
result, err := db.Exec("LOAD DATA INFILE ?", filename)
if err != nil {
log.Fatal(err)
}
log.Printf("%d rows inserted\n", result.RowsAffected())
If you haven't used LOAD DATA INFILE before, note carefully the documentation regarding LOCAL. Depending on your server configuration and permissions, you might need to use LOAD DATA LOCAL INFILE instead. (If you intend to use Docker containers, for instance, you will absolutely need to use LOCAL.)
I am creating an application which is communicating with google datastore, to fetch the existing data, and perform add, edit and delete operations on that existing data. I am able to fetch the existing data, and delete the data there. But not getting how to edit/update the data there through api in golang.
Giving the code snippet which I am trying to execute for this :
func EditCustomer(w http.ResponseWriter, r *http.Request){
ctx := context.Background()
params := mux.Vars(r)
customer_id :=params["partner_id"]
projectID := util.MustGetenv("GOOGLE_CLOUD_PROJECT")
client, err := datastore.NewClient(ctx, projectID)
if err != nil {
log.Fatalf("Failed to create client: %v", err)
}
var customer models.Customer
kind := util.MustGetenv("DATA_STORE_KIND")
ds.EditCustomer(client,kind,customer_id,&customer,ctx)
json.NewEncoder(w).Encode(customer)
}
EditCustomer method in dao is as :
func EditCustomer(client *datastore.Client,kind string ,name string,dst interface{},ctx context.Context) {
taskKey := datastore.NameKey(kind, name, nil)
< some methos here to edit and update itin datstorage >
}
Please advise for this. Anybody there who working with api's dev in golang ?
I would update like this:
currentCustomer := &Customer{}
_, err := client.RunInTransaction(ctx, func(tx *datastore.Transaction) error {
if getErr := tx.Get(key, currentCustomer); getErr != nil {
return getErr
}
// edit your object
var putErr error
_, putErr = tx.Put(key, updatedCustomer)
return putErr
})
if err != nil {
return nil, err
}
_, err = client.Put(ctx, key, customer)
I thought I'd try to be a little bit "clever" and abstract some of my boilerplate SQL code (using sqlx -- https://github.com/jmoiron/sqlx). The idea is to feed a code a function pointer to process the result, along with the sql string and args that produce the rows. As it happens, the code works fine provided I strip out the "sqlArgs interface" stuff, but in the "cleverer" format errors with the statement
sql: converting Exec argument $1 type: unsupported type []interface {}, a slice of interface
Here are two versions, the first one that errors, the second that works but without parameterization:
//GetRows (doesn't work)
func GetRows(parseRows func(*sqlx.Rows), sql string, sqlArgs ...interface{}) {
db := sqlx.MustConnect("mysql", ConnString)
defer db.Close()
rows, err := db.Queryx(sql, sqlArgs)
defer rows.Close()
if err != nil {
panic(err)
}
parseRows(rows)
}
//GetRows ... (works, but doesn't allow parameterization)
func GetRows(fp func(*sqlx.Rows), sql string) {
db := sqlx.MustConnect("mysql", ConnString)
defer db.Close()
rows, err := db.Queryx(sql)
defer rows.Close()
if err != nil {
panic(err)
}
fp(rows)
}
The idea is to call the code something like this:
func getUser(userID string) User {
var users []*User
parseRows := func(rows *sqlx.Rows) {
for rows.Next() {
var u User
err := rows.StructScan(&u)
if err != nil {
panic(err)
}
users = append(users, u)
}
}
sql := "SELECT * FROM users WHERE userid = ?;"
sqlutils.GetRows(parseRows, sql, userID)
if len(users) == 1{
return users[0]
}
return User{}
}
I guess my code doesn't actually pass through the userID from call to call, but instead it passes an []interface{}, which the sql package can't handle. I'm not sure about that, however. In any case, is there any way to accomplish this idea? Thanks.
I'm trying to find a way to pass parameters to query, but not quite sure how. The API on the web site looks a little bit outdated?
myQuery := gocb.NewN1qlQuery("SELECT * FROM default")
rows, err := myBucket.ExecuteN1qlQuery(myQuery)
if err != nil {
fmt.Printf("N1QL query error: %s\n", err)
}
var row interface{}
for rows.Next(&row) {
fmt.Printf("Row: %+v\n", row)
}
if err := rows.Close(); err != nil {
fmt.Printf("N1QL query error: %s\n", err)
}
Because, actually ExecuteN1qlQuery takes two params:
func (b *Bucket) ExecuteN1qlQuery(q *N1qlQuery, params interface{}) (ViewResults, error)
I am not sure just how to use it... Like I would like to create a query with placeholders, and pass values to ExecuteN1qlQuery via params. Like with SQL (prepare -> execute). For example something like that:
myQuery := gocb.NewN1qlQuery("SELECT * FROM default where a=? and b=?")
rows, err := myBucket.ExecuteN1qlQuery(myQuery,[]string{"b","c"})
if err != nil {
fmt.Printf("N1QL query error: %s\n", err)
}
var row interface{}
for rows.Next(&row) {
fmt.Printf("Row: %+v\n", row)
}
if err := rows.Close(); err != nil {
fmt.Printf("N1QL query error: %s\n", err)
}
The example you posted for how to do do this is from our developer guide repo on github:
https://github.com/couchbaselabs/devguide-examples/blob/master/go/query-placeholders.go.
Basically, you're using $ which references an interface and a corresponding positional parameter beginning with 1.
For your example it would look something like:
// Setup a new query with a placeholder
myQuery := gocb.NewN1qlQuery("SELECT * FROM default where a=$1 and b=$2")
// Setup an array for parameters
var myParams []interface{}
myParams = append(myParams,"foo")
myParams = append(myParams,"bar")
// Execute Query
rows, err := bucket.ExecuteN1qlQuery(myQuery, myParams)
if err != nil {
fmt.Println("ERROR EXECUTING N1QL QUERY:", err)
}
// Iterate through rows and print output
var row interface{}
for rows.Next(&row) {
fmt.Printf("Results: %+v\n", row)
}
Just found example
myQuery := gocb.NewN1qlQuery("SELECT airportname, city, country FROM `travel-sample` " +
"WHERE type='airport' AND city=$1 ")
// Setup an array for parameters
var myParams []interface{}
myParams = append(myParams, "Reno")
// Execute Query
rows, err := bucket.ExecuteN1qlQuery(myQuery, myParams)