gocql can not unmarshal blob into *[20]uint8 - go

I have a question related to using blob type as partition key.
I use it, as I need to save hash value.
(hash value returns binary data. usually as hexadecimal.)
I tried a select query with gocql, however it failed with following error.
Is there any way to get a successful result for this kind of query?
Your advice highly appreciated!!
-- result
hash_value: [208 61 222 22 16 214 223 135 169 6 25 65 44 237 166 229 50 5 40 221]
/ hash_value: ?=??߇?A,???2(?
/ hash_value: 0xd03dde1610d6df87a90619412ceda6e5320528dd
string
2018/03/22 10:03:17 can not unmarshal blob into *[20]uint8
-- select.go
package main
import (
"fmt"
"log"
"crypto/sha1"
"reflect"
"github.com/gocql/gocql"
)
func main() {
cluster := gocql.NewCluster("10.0.0.1")
cluster.Keyspace = "ks"
cluster.Consistency = gocql.Quorum
cluster.ProtoVersion = 4
cluster.Authenticator = gocql.PasswordAuthenticator{
Username: "cassandra",
Password: "cassandra",
}
session, _ := cluster.CreateSession()
defer session.Close()
text := "text before hashed"
data := []byte(text)
hash_value := sha1.Sum(data)
hexa_string := fmt.Sprintf("0x%x", hash_value)
fmt.Println("hash_value: ", hash_value)
fmt.Println(" / string: ", string(hash_value[:]))
fmt.Println(" / column1: ", hexa_string)
fmt.Println(reflect.TypeOf(hexa_string))
// *** select ***
var column1 int
returned_hash := sha1.Sum(data)
//if err := session.Query(`SELECT hash_value, column1 FROM sample WHERE hash_value= ? LIMIT 1`,
// hexa_string).Consistency(gocql.One).Scan(&returned_hash, &column1); err != nil {
if err := session.Query(`SELECT hash_value, column1 FROM sample WHERE hash_value=0xd03dde1610d6df87a90619412ceda6e5320528dd`).Consistency(gocql.One).Scan(&returned_hash, &column1); err != nil {
//fmt.Println(err)
log.Fatal(err)
}
fmt.Println("comment: ", returned_hash, user_id)
}
-- table definition --
CREATE TABLE IF NOT EXISTS ks.samle (
hash_value blob,
column1 int,
...
PRIMARY KEY((hash_value), column1)
) WITH CLUSTERING ORDER BY (column1 DESC);

I fixed the problem by changing the type of variable: returned_hash.
returned_hash (var to store returned result) should be []byte.
I understands as follows.
marshal: convert the data given in the code to a type cassandra can handle.
unmarshal: convert the data cassandra returned back to a type golang-code can handle.
the original error means latter pattern doesn't go well. so the type of returned_hash must be wrong.
Correct me if I'm wrong. thanks.
package main
import (
"fmt"
"log"
"crypto/sha1"
"reflect"
"github.com/gocql/gocql"
)
func main() {
cluster := gocql.NewCluster("127.0.0.1")
cluster.Keyspace = "browser"
cluster.Consistency = gocql.Quorum
//cluster.ProtoVersion = 4
//cluster.Authenticator = gocql.PasswordAuthenticator{
// Username: "cassandra",
// Password: "cassandra",
//}
session, _ := cluster.CreateSession()
defer session.Close()
text := "text before hashed"
data := []byte(text)
hash_value := sha1.Sum(data)
hexa_string := fmt.Sprintf("0x%x", hash_value)
fmt.Println("hash_value: ", hash_value)
fmt.Println(" / string(hash_value): ", string(hash_value[:]))
fmt.Println(" / hexa(hash_value): ", hexa_string)
fmt.Println(reflect.TypeOf(hexa_string))
// *** select ***
var column1 int
//returned_hash := sha1.Sum(data)
//var returned_hash *[20]uint8
var returned_hash []byte
if err := session.Query(`SELECT hash_value, column1 FROM sample WHERE hash_value=? LIMIT 1`,
hash_value[:]).Consistency(gocql.One).Scan(&returned_hash, &column1); err != nil {
//if err := session.Query(`SELECT hash_value, column1 FROM sample WHERE hash_value=0xd03dde1610d6df87a90619412ceda6e5320528dd`).Consistency(gocql.One).Scan(&returned_hash, &column1); err != nil {
log.Fatal(err)
}
fmt.Printf("Returned: %#x %d \n", returned_hash, column1)
}

Related

Handling decoded data with multiple types using goavro

What I am trying to do?
For every change in the database, I am trying to convert the debezium event into a CSV of the database values for loading into Redshift.
For the below 110 change, i am trying to make a csv file: 110,vck,desc,221.1
mysql> select * from products;
+-----+-------------+---------------------------------------------------------+--------+
| id | name | description | weight |
+-----+-------------+---------------------------------------------------------+--------+
| 110 | vck | desc | 221.1 |
+-----+-------------+---------------------------------------------------------+--------+
Here was my attempt to do it using goavro.
Program
https://play.golang.org/p/A8Wd0sZPUEQ
package main
import (
"fmt"
"encoding/json"
)
func main() {
debeziumEvent := `{"before":null,"after":{"datapipe.inventory.products.Value":{"id":110,"name":"vck","description":{"string":"desc"},"weight":{"double":221.10000610351562}}},"source":{"query":null,"snapshot":{"string":"true"},"server_id":0,"gtid":null,"name":"datapipe","thread":null,"ts_ms":0,"file":"mysql-bin.000049","version":"1.2.1.Final","connector":"mysql","pos":154,"table":{"string":"products"},"row":0,"db":"inventory"},"op":"c","ts_ms":{"long":1597649700266},"transaction":null}`
var data map[string]interface{}
err := json.Unmarshal([]byte(debeziumEvent), &data)
if err != nil {
panic(err)
}
after := data["after"].(map[string]interface{})
csv := make([]interface{}, 0)
for _, v := range after {
for _, v2 := range v.(map[string]interface{}) {
switch stype := v2.(type) {
case map[string]interface{}:
for _, v3 := range v2.(map[string]interface{}) {
csv = append(csv, v3)
}
case string:
csv = append(csv, v2)
case int:
csv = append(csv, v2)
case float64:
csv = append(csv, v2)
default:
fmt.Printf("type %s not handled\n", stype)
panic("unhandled type")
}
}
}
fmt.Println(csv)
}
Is there a way to do this any better? For every data type i would need to have a switch statement here....
Linked GoAVRO issue: https://github.com/linkedin/goavro/issues/217
fmt.Sprintf can be used to convert interfaces to string.
str := fmt.Sprintf("%v", v)
Doing this reduces the case statements to 2:
package main
import (
"fmt"
"encoding/json"
)
func main() {
debeziumEvent := `{"before":null,"after":{"datapipe.inventory.products.Value":{"id":110,"name":"vck","description":{"string":"desc"},"weight":{"double":221.10000610351562}}},"source":{"query":null,"snapshot":{"string":"true"},"server_id":0,"gtid":null,"name":"datapipe","thread":null,"ts_ms":0,"file":"mysql-bin.000049","version":"1.2.1.Final","connector":"mysql","pos":154,"table":{"string":"products"},"row":0,"db":"inventory"},"op":"c","ts_ms":{"long":1597649700266},"transaction":null}`
var data map[string]interface{}
err := json.Unmarshal([]byte(debeziumEvent), &data)
if err != nil {
panic(err)
}
//fmt.Printf("data=%v\n", data)
after := data["after"].(map[string]interface{})
csv := []string{}
for _, v := range after {
for _, v2 := range v.(map[string]interface{}) {
switch v2.(type) {
case map[string]interface{}:
for _, v3 := range v2.(map[string]interface{}) {
csv = append(csv, fmt.Sprintf("%v", v3))
}
default:
csv = append(csv, fmt.Sprintf("%v", v2))
}
}
}
fmt.Println(csv)
}

Byte Array conversion to string

My intent is to read a text file , and print records on console . Sample text file format
abc 123
test zzz
hello world
I am using byte array to initialize my structure .
However when I try to print after splitting the read lines , blank values are printed for the second token which is obtained after the split
package main
import (
"fmt"
"bufio"
"os"
"strings"
"io/ioutil"
)
type person struct {
fName [20]byte
lName [20]byte
}
func main(){
reader := bufio.NewReader(os.Stdin)
fmt.Printf("Enter Name ")
fname, _ := reader.ReadString('\n')
fname = strings.Replace(fname, "\n", "", -1)
fname = strings.Replace(fname, "\r", "", -1)
readFile, err := ioutil.ReadFile(fname)
if err != nil {
panic(err)
}
s := string(readFile)
fileTextLines := strings.Split(s, "\n")
sli := make([]person, len(fileTextLines))
for _, eachline := range fileTextLines {
res1 := strings.Split( eachline, " ")
var fname [20]byte
copy( fname[:], []byte(res1[0] ))
fmt.Println(string(fname[:]))
var lname [20]byte
copy( lname[:], []byte(res1[1] ) )
fmt.Println(string(lname[:]))
p := person{fname ,lname}
sli = append(sli,p)
}
// print the slice
// for _,object := range sli {
// fmt.Printf("First Name: %s Last Name: %s\n", string(object.fName[:]), string(object.lName[:]))
// }
}
With the suggested approach I got the following output
Last Name: 123
Last Name: zzz
First Name: hello Last Name: world
Once you split the first name and last name there is some extra white spaces in the values, so I added strings.TrimSpace and below code is working now, and also you don't need to Initialize sli with fix length, append will automatically increase the size of slice.
package main
import (
"fmt"
"bufio"
"os"
"strings"
"io/ioutil"
)
type person struct {
fName [20]byte
lName [20]byte
}
func main(){
reader := bufio.NewReader(os.Stdin)
fmt.Printf("Enter Name ")
fname, _ := reader.ReadString('\n')
fname = strings.Replace(fname, "\n", "", -1)
fname = strings.Replace(fname, "\r", "", -1)
readFile, err := ioutil.ReadFile(fname)
if err != nil {
panic(err)
}
s := string(readFile)
fileTextLines := strings.Split(s, "\n")
sli := []person{}
for _, eachline := range fileTextLines {
res1 := strings.Split( eachline, " ")
var fname [20]byte
copy( fname[:], []byte(strings.TrimSpace(res1[0])))
fmt.Println(string(fname[:]))
var lname [20]byte
copy( lname[:], []byte(strings.TrimSpace(res1[1])) )
fmt.Println(string(lname[:]))
p := person{fname ,lname}
sli = append(sli,p)
}
// print the slice
for _,object := range sli {
fmt.Printf("First Name: %s Last Name: %s\n", string(object.fName[:]), string(object.lName[:]))
}
}
Just found out that extra with space is carriage return "\r", If you are using windows platform you should use below line to split the text with newline
strings.Split(strings.Replace(windows, "\r\n", "\n", -1), "\n")
Use this code:
readFile, err := ioutil.ReadFile(fname)
if err != nil {
panic(err)
}
fileTextLines := bytes.Split(readFile, []byte("\n"))
sli := make([]person, 0, len(fileTextLines))
for _, eachline := range fileTextLines {
res1 := bytes.Split(eachline, []byte(" "))
if len(res1) < 2 {
continue
}
var p person
copy(p.fName[:], res1[0])
copy(p.lName[:], res1[1])
sli = append(sli, p)
}
for _, object := range sli {
fmt.Printf("First Name: %s Last Name: %s\n", object.fName[:], object.lName[:])
}
This simplifies the code by working with []byte instead of string and handles case where line has only one token.
Working example on the GoLang PlayGround: https://play.golang.org/p/JFcM1uE3Ywm

GORM Count returns 0

Problem Statement
Problem 1
I am getting a 0 when trying to retrieve the count.
The query expression is perfect, by which I assume that the query being built is correct.
However, the result is 0 irrespective of the query.
Problem 2
I am required to specify the db name in the Table clause as Table(dbname.tbname)
This isn't required in the any other queries and is required only when using Count()
What might I be missing?
package main
import (
"github.com/jinzhu/gorm"
_ "github.com/jinzhu/gorm/dialects/mssql"
_ "github.com/jinzhu/gorm/dialects/mysql"
_ "github.com/jinzhu/gorm/dialects/postgres"
_ "github.com/jinzhu/gorm/dialects/sqlite"
)
var db *gorm.DB
var err error
func main() {
db, err = gorm.Open("mysql", "gorm:gorm#tcp(localhost:9910)/gorm?charset=utf8&parseTime=True")
if err != nil {
panic(err)
}
db.LogMode(true)
var count int
var db = "db"
var tb = "tb"
var columnname = "somecol"
var date = "2014-01-02"
err := db.Table(db+"."+tb).Where("? >= ?", columnname, date+" 00:01:00").Count(&count)
if err != nil {
Error.Println(err.Error.Error())
}
fmt.Println("The Count is \n", count)
}
Update 1
The following works.
But this is as per my understanding using the result as *sql.Row and then retrieving the result using scan.
But I don't understand,why ...Count(&count) is giving a run time error?

Find byte offset of a pattern in Golang

We can find the byte offset of a pattern from file by
"grep -ob pattern filename";
However, grep is not utf8 safe.
How do I find byte offset of a pattern in Go? The file is process log, which can be in TB.
This is what I want to get in Go:
$ cat fname
hello world
findme
hello 世界
findme again
...
$ grep -ob findme fname
12:findme
32:findme
FindAllStringIndex(s string, n int) returns byte start/finish indexes (i.e., slices) of all successive matches of the expression:
package main
import "fmt"
import "io/ioutil"
import "regexp"
func main() {
fname := "C:\\Users\\UserName\\go\\src\\so56798431\\fname"
b, err := ioutil.ReadFile(fname)
if err != nil {
panic(err)
}
re, err := regexp.Compile("findme")
if err != nil {
// handle error
}
fmt.Println(re.FindAllStringIndex(string(b), -1))
}
Output:
[[12 18] [32 38]]
Note: I did this on Microsoft Windows, but saved the file in UNIX format (linefeed); if input file saved in Windows format (carriage return & linefeed) the byte offsets would increment to 13 and 35, respectively.
UPDATE: for large files, use bufio.Scanner; for example:
package main
import (
"bufio"
"fmt"
"log"
"os"
"regexp"
)
func main() {
fname, err := os.Open("C:\\Users\\UserName\\go\\src\\so56798431\\fname")
if err != nil {
log.Fatal(err)
}
defer fname.Close()
re, err := regexp.Compile("findme")
if err != nil {
// handle error
}
scanner := bufio.NewScanner(fname)
bytesRead := 0
for scanner.Scan() {
b := scanner.Text()
//fmt.Println(b)
results := re.FindAllStringIndex(b, -1)
for _, result := range results {
fmt.Println(bytesRead + result[0])
}
// account for UNIX EOL marker
bytesRead += len(b) + 1
}
if err := scanner.Err(); err != nil {
log.Fatal(err)
}
}
Output:
12
32

How can I create map with nil value in one field and then marshal it to json?

I need get json string
{"Error": null }
I can do this using this approach
type OKResponse struct {
Error *int `json:"Error"`
}
encoded, err := json.Marshal(OKResponse{})
...
How I can get the same result using map? Or it is not possible?
For example something like this
jsonbody := map[string]int{"Error": nil}
encoded, err := json.Marshal(&jsonbody)
...
For this particular case you may use:
jsonbody := map[string]error{"Error": nil}
But for more generalized you may also use:
jsonbody := map[string]interface{}{"Error": nil}
Both approaches will produce:
{"Error":null}
Since number can't get a nil value you can't do that.
What you can do is to use a pointer type and then it can be done, something like:
package main
import (
"fmt"
"encoding/json"
)
func main() {
jsonbody := map[string]*int{"Error": nil}
encoded, _ := json.Marshal(&jsonbody)
fmt.Printf("%s", encoded)
num := 6
jsonbody = map[string]*int{"Error": &num}
encoded, _ = json.Marshal(&jsonbody)
fmt.Printf("%s", encoded)
}
https://play.golang.org/p/TTpgr7Cy17C
If you change the type to *int, it will do what you want:
type OKResponse struct {
Error *int `json:"Error"`
}
Example:
r1 := OKResponse{}
result1, _ := json.Marshal(r1)
fmt.Printf("result1 = %s\n", string(result1))
errNo := 1
r2 := OKResponse{Error: &errNo}
result2, _ := json.Marshal(r2)
fmt.Printf("result2 = %s\n", string(result2))
Output:
result1 = {"Error":null}
result2 = {"Error":1}
See it in the playground.

Resources