temp1Ctx, temp1Cancer := lib.GetTimeoutCtx(ctx)
pipeline := util.RedisClusterClient.Pipeline()
for _, key := range userIdRedisSlice {
pipeline.HMGet(temp1Ctx, key, userIdRedisFeature...)
}
userProfile, err := pipeline.Exec(temp1Ctx)
if err != nil {
lib.ErrorLogger.Errorf(": %v\n", err)
}
defer temp1Cancer()
// lib.ErrorLogger.Infof(": %v", userProfile)
for _, redisCmd := range userProfile {
//TODO
}
How to get the value from it ? I don't find any document .......
Either retain the concrete command type returned by HMGet
temp1Ctx, temp1Cancer := lib.GetTimeoutCtx(ctx)
pipeline := util.RedisClusterClient.Pipeline()
cmds := []*redis.SliceCmd{}
for _, key := range userIdRedisSlice {
cmds = append(cmds, pipeline.HMGet(temp1Ctx, key, userIdRedisFeature...))
}
if _, err := pipeline.Exec(temp1Ctx); err != nil {
lib.ErrorLogger.Errorf(": %v\n", err)
}
defer temp1Cancer()
for _, c := range cmds {
// use c.Result()
// or use c.Scan
}
or type-assert / type-switch the Cmder to the concrete type.
temp1Ctx, temp1Cancer := lib.GetTimeoutCtx(ctx)
pipeline := util.RedisClusterClient.Pipeline()
for _, key := range userIdRedisSlice {
pipeline.HMGet(temp1Ctx, key, userIdRedisFeature...)
}
userProfile, err := pipeline.Exec(temp1Ctx)
if err != nil {
lib.ErrorLogger.Errorf(": %v\n", err)
}
defer temp1Cancer()
for _, redisCmd := range userProfile {
switch c := redisCmd.(type) {
case *redis.SliceCmd:
// use c.Result()
// or c.Scan()
}
}
Related
For example
func Query(myvarlist []string) {
stmt, err := tx.Prepare("SELECT * FROM mytable WHERE
myvar = $1 AND myvar2 = $2".......)
defer stmt.Close()
if _, err := stmt.Exec(myvarlist....); err != nil {
}
Can we pass in Exec a variable length variable?
You can do something like the this:
func Query() {
db, err := sql.Open("pgx", `postgresql://CONNSTRING`)
if err != nil {
panic(err)
}
queryParamMap := map[string]string{
"id": "6",
"name": "1",
}
// Build up statement and params
cols := make([]string, len(queryParamMap))
args := make([]any, len(queryParamMap))
i := 0
for k, v := range queryParamMap {
cols[i] = fmt.Sprintf("%s = $%d", k, i+1) // WARNING - SQL Injection possible here if col names are not sanitised
args[i] = v
i++
}
// Using Prepare because the question used it but this is only worthwhile if you will run stmt multiple times
stmt, err := db.Prepare(`SELECT id FROM devices WHERE ` + strings.Join(cols, " and "))
if err != nil {
panic(err)
}
defer stmt.Close()
rows, err := stmt.Query(args...)
if err != nil {
panic(err)
}
defer rows.Close()
for rows.Next() {
var id int
if err = rows.Scan(&id); err != nil {
panic(err)
}
fmt.Println("value:", id)
}
// Should check rows.Error() etc but this is just an example...
}
I've put the column names and values into a map because it was not clear where any extra column names would come from in your question but hopefully this provides the info you need.
This example is also using Query rather than Exec (because it's easier to test) but the same approach will work with Exec.
Note: Take a look at squirrel for an example of how to take this a lot further....
I'm trying to get and return all the nodes of NodeClass Variable of my OPC UA Simulation Server starting at the root node and going down all folder nodes and object nodes that have childs. I tried to use browse-example of the gopcua repo but whenever the program gets to attrs, err := n.Attributes(...) for the second time, it returns an EOF error.
I tried to recreate a minimal example:
package main
import (
"context"
"fmt"
"log"
"github.com/gopcua/opcua"
"github.com/gopcua/opcua/id"
"github.com/gopcua/opcua/ua"
)
const (
endpoint string = "opc.tcp://<ServerAddress>"
rootNodeId string = "i=85"
)
func browse(c *opcua.Client, n *opcua.Node) error {
_, err := n.Attributes(ua.AttributeIDNodeClass, ua.AttributeIDDataType)
if err != nil {
return err
}
browseChildren := func(refType uint32) error {
refs, err := n.ReferencedNodes(refType, ua.BrowseDirectionForward, ua.NodeClassAll, true)
if err != nil {
return fmt.Errorf("references: %d: %s", refType, err)
}
fmt.Printf("found %d child refs\n", len(refs))
for _, rn := range refs {
err := browse(c, rn)
if err != nil {
return fmt.Errorf("browse children: %s", err)
}
fmt.Printf("Found a Node: %s\n", rn.ID.String())
}
return nil
}
if err := browseChildren(id.HasChild); err != nil {
return err
}
if err := browseChildren(id.Organizes); err != nil {
return err
}
return nil
}
func main() {
ctx := context.Background()
c := opcua.NewClient(endpoint)
if err := c.Connect(ctx); err != nil {
log.Fatal("Could not connect")
panic(err)
}
defer c.Close()
id, _ := ua.ParseNodeID(rootNodeId)
err := browse(c, c.Node(id))
if err != nil {
log.Fatal(err)
}
}
This is how the Prosys OPC UA Simulation Server looks like:
And this the output I get from the program:
found 0 child refs
found 5 child refs
2022/01/21 14:26:06 browse children: EOF
exit status 1
tl;dr
browseChildren := func(refType uint32) error {
refs, err := n.ReferencedNodes(refType, ua.BrowseDirectionForward, ua.NodeClassAll, true)
if err != nil {
return fmt.Errorf("references: %d: %s", refType, err)
}
fmt.Printf("found %d child refs\n", len(refs))
for _, rn := range refs {
refNodeID := ua.MustParseNodeID(rn.ID.String())
refNode := c.Node(refNodeID) // parse the referenced nodes before usage
err := browse(c, refNode)
if err != nil {
return fmt.Errorf("browse children: %s", err)
}
fmt.Printf("Found a Node: %s\n", refNode.ID.String())
}
return nil
}
Reason
The referenced nodes that are returned by n.ReferencedNodes have an invalid NodeIDType (64). The only supported NodeIDTypes are 0-5. Reparsing them gives them a proper NodeIDType. (Related Gitlab issue: https://github.com/gopcua/opcua/issues/550)
I have a for loop, which inserts data into the 2 different tables. How can I use ants(below package) in this case.
GH Package Ref: https://github.com/panjf2000/ants
for _, row := range rows {
user := User{}
user.Name = row.Name
user.Email = row.Email
err := dm.Insert(&user)
if err != nil {
panic(err)
}
address := Address{}
address.Address1 = row.Address1
address.Address2 = row.Address2
address.PinCode = row.PinCode
address.City = row.City
err := dm.Insert(&address)
if err != nil {
panic(err)
}
}
Something like:
func insertRow() {
// TODO add code to get 'rows'
const workerCount = 10
p, err := NewPool(workerCount)
if err != nil {
panic(err)
}
defer p.Release()
rowChan := make(chan RowType)
var wg sync.WaitGroup
insertRecords := func() {
defer wg.Done()
row <- rowChan
user := User{}
user.Name = row.Name
user.Email = row.Email
err := dm.Insert(&user)
if err != nil {
panic(err)
}
address := Address{}
address.Address1 = row.Address1
address.Address2 = row.Address2
address.PinCode = row.PinCode
address.City = row.City
err := dm.Insert(&address)
if err != nil {
panic(err)
}
}
for _, row := range rows {
wg.Add(1)
_ = ants.Submit(insertRecords)
rowChan <- row
}
wg.Wait()
}
I have a map like this, which I want to save/retrive from redis using redigo:
animals := map[string]bool{
"cat": true,
"dog": false,
"fox": true,
}
The length of the map may vary.
I tried these function:
func SetHash(key string, value map[string]bool) error {
conn := Pool.Get()
defer conn.Close()
_, err := conn.Do("HMSET", key, value)
if err != nil {
return fmt.Errorf("error setting key %s to %s: %v", key, value, err)
}
return err
}
func GetHash(key string) (map[string]bool, error) {
conn := Pool.Get()
defer conn.Close()
val, err := conn.Do("HGETALL", key)
if err != nil {
fmt.Errorf("error setting key %s to %s: %v", key, nil, err)
return nil, err
}
return val, err
}
But can not make GetHash correctly. I've checked the docs examples and it was not helpful. So appreciate your help to have a working example.
HMSET is deprecated, use HSET instead, no effect here though.
The map[string]bool may be flattened with AddFlat() for SetHash().
c.Do("HSET", redis.Args{}.Add("key").AddFlat(value)...)
For GetHash(), use Values(). You may use ScanStruct() to map to a struct or loop through the values to create a map dynamically.
v, err := redis.Values(c.Do("HGETALL", key))
redis.ScanStruct(v, &myStruct);
See example from redigo tests in scan_test.go.
The application is responsible for converting structured types to and from the types understood by Redis.
Flatten the map into a list of arguments:
func SetHash(key string, value map[string]bool) error {
conn := Pool.Get()
defer conn.Close()
// Create arguments: key field value [field value]...
var args = []interface{}{key}
for k, v := range value {
args = append(args, k, v)
}
_, err := conn.Do("HMSET", args...)
if err != nil {
return fmt.Errorf("error setting key %s to %v: %v", key, value, err)
}
return err
}
Convert the returned field value pairs to a map:
func GetHash(key string) (map[string]bool, error) {
conn := Pool.Get()
defer conn.Close()
values, err := redis.Strings(conn.Do("HGETALL", key))
if err != nil {
return nil, err
}
// Loop through [field value]... and parse value as bool.
m := map[string]bool{}
for i := 0; i < len(values); i += 2 {
b, err := strconv.ParseBool(value)
if err != nil {
return nil, errors.New("value not a bool")
}
m[key] = b
}
return m, nil
}
Following some #Sam Whited advance and doing some research on stack, i've rewritten my code see below: This version of the code seems more stable, however, it is having issues where every once and a while the i get a slew of TCP errors as if i'm no closing my requests. I've throttled the requests by adding a sleep. It seems to help a bit.
func main() {
runtime.GOMAXPROCS(maxParallelism())
var file = flag.String("f", "", "Enter new line deliminated text file")
var fileName = flag.String("s", "contact_bot.csv", "Enter new line deliminated text file")
flag.Parse()
if *file != "" {
counter := 0
filters = []string{"info", "ads", "sales", "sale", "info", "media", "mediarelations", "media_relations", "contact", "contacts", "contactus", "contact_us", "contact-us", "about_us", "general", "advertise", "support", "systems", "system"}
emailRE = regexp.MustCompile(`([a-z0-9!#$%&'*+\/=?^_{|}~-]+(?:\.[a-z0-9!#$%&'*+\/=?^_{|}~-]+)*(#|\sat\s)(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?(\.|\sdot\s))+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?)`)
seedUrls, err := readLines(*file)
checkErr(err)
numberOfUrls := len(seedUrls)
usr, err := user.Current()
checkErr(err)
parentPath := filepath.Join(usr.HomeDir, "/Desktop/"+*fileName)
file, err := os.Create(parentPath)
checkErr(err)
defer file.Close()
writer := csv.NewWriter(file)
defer writer.Flush()
var header = []string{"URL", "EMAILS"}
err = writer.Write(header)
checkErr(err)
data = make(chan *HTTPResponse)
go asyncHTTPGets(seedUrls)
loop:
for result := range data {
counter++
emails := findEmails(result.HTML, filters)
fmt.Printf("%s, %s, %s\n", result.URL, emails, strconv.Itoa(numberOfUrls))
var row = []string{result.URL, strings.Join(emails, ",")}
err := writer.Write(row)
// writer.Flush()
checkErr(err)
if counter == len(seedUrls) {
break loop
}
numberOfUrls--
}
}
}
// AsyncHTTPGets ...
func asyncHTTPGets(urls []string) {
counter := 0
for _, url := range urls {
counter++
if counter%10 == 0 {
time.Sleep(1 * time.Second)
}
go func(url string) {
fmt.Printf("Fetching %s \n", url)
resp, err := http.Get(url)
if err != nil {
fmt.Println(err.Error())
data <- &HTTPResponse{url, err.Error()}
return
}
b := resp.Body
buf := new(bytes.Buffer)
buf.ReadFrom(b)
resp.Body.Close()
myHTML := buf.String()
data <- &HTTPResponse{url, myHTML}
}(url)
}
}
func findEmails(html string, filters []string) []string {
emails := emailRE.FindAllString(html, -1)
filteredEmails := []string{}
for _, email := range emails {
if stringInSlice(email, filters) {
if !stringInSlice(email, filteredEmails) {
filteredEmails = append(filteredEmails, email)
}
}
}
sort.Strings(filteredEmails)
return filteredEmails
}
The application will open a large number of sockets and possibly breach file descriptor limits. I suggest limiting the number of concurrent requests to prevent this issue:
var (
requestMu sync.Mutex // protects requestCount
requestCount int // incremented on each request
)
// Create 10 workers. Adjust up or down as needed.
for w := 0; w < 10; w++ {
go func() {
for {
// Increment request count. Exit at end.
requestMu.Lock()
i := requestCount
requestCount++
requestMu.Unlock()
if i >= len(seedUrls) {
return
}
// Fetch the current URL.
myURL := seedUrls[i]
resp, err := http.Get(myUrl)
if err != nil {
fmt.Println(myURL, err.Error(), i)
data <- &HTTPResponse{myURL, err.Error()}
continue
}
// Read body and close.
b, err := ioutil.ReadAll(resp.Body)
resp.Body.Close()
if err != nil {
fmt.Println(myURL, err.Error(), i)
data <- &HTTPResponse{myURL, err.Error()}
continue
}
myHTML := string(b)
data <- &HTTPResponse{myURL, myHTML}
}
}()
}
// Recieve expected number of results
for i := 0; i < len(seedUrls); i++ {
result <- data
emails := findEmails(result.HTML, filters)
fmt.Printf("%s, %s, %d\n", result.URL, emails, i)
var row = []string{result.URL, strings.Join(emails, ",")}
err := writer.Write(row)
writer.Flush()
if err != nil {
panic(err)
}
}