Can I have nested bucket under a nested bucket in boltdb? - go

This is what I have to create nested buckets. It does not return any error but fails at creating nested bucket under another nested bucket.
func CreateNestedBuckets(buckets []string) error {
err := db.Update(func(tx *bolt.Tx) error {
var bkt *bolt.Bucket
var err error
first := true
for _, bucket := range buckets {
log.Error(bucket)
if first == true {
bkt, err = tx.CreateBucketIfNotExists([]byte(bucket))
first = false
} else {
bkt, err = bkt.CreateBucketIfNotExists([]byte(bucket))
}
if err != nil {
log.Error("error creating nested bucket")
return err
}
}
return nil
})
if err != nil {
log.Error("error creating nested bucket!!!")
return err
}
return nil
}

Short answer: yes! You can have nested buckets: https://twitter.com/boltdb/status/454730212010254336
Long answer: your code works fine! Heres some things to check though:
Are you checking the correct bolt database file? The botlt db file will be created in the directory you run your code from, unless you've specified an absolute path.
Does your input actually contain enough elements to create a nested structure?
I've ran your code with the following setup (a couple of small changes but nothing major) and it works fine:
package main
import (
"log"
"os"
"time"
"github.com/boltdb/bolt"
)
var dbname = "test.bdb"
var dbperms os.FileMode = 0770
var options = &bolt.Options{Timeout: 1 * time.Second}
func main() {
var names []string
names = append(names, "bucketOne")
names = append(names, "bucketTwo")
names = append(names, "bucketThree")
if err := CreateNestedBuckets(names); err != nil {
log.Fatal(err)
}
}
// CreateNestedBuckets - Function to create
// nested buckets from an array of Strings
func CreateNestedBuckets(buckets []string) error {
db, dberr := bolt.Open(dbname, dbperms, options)
if dberr != nil {
log.Fatal(dberr)
}
defer db.Close()
err := db.Update(func(tx *bolt.Tx) error {
var bkt *bolt.Bucket
var err error
first := true
for _, bucket := range buckets {
log.Println(bucket)
if first == true {
bkt, err = tx.CreateBucketIfNotExists([]byte(bucket))
first = false
} else {
bkt, err = bkt.CreateBucketIfNotExists([]byte(bucket))
}
if err != nil {
log.Println("error creating nested bucket")
return err
}
}
return nil
})
if err != nil {
log.Println("error creating nested bucket!!!")
return err
}
return nil
}
To test you can cat the file through the strings command:
cat test.bdb | strings
bucketThree
bucketTwo
bucketOne
If you're on Windows, I'm not sure what the equivalent command is, but you can open the file with Notepad and inspect it manually. It won't be pretty, but you should still see the name of your buckets in there.
On another note, you error handling is going to result in very similar messages being printed in succession. Here's a slightly cleaner solution you can use:
// CreateNestedBucketsNew - function to create
// nested buckets from an array of Strings - my implementation
func CreateNestedBucketsNew(buckets []string) (err error) {
err = db.Update(func(tx *bolt.Tx) (err error) {
var bkt *bolt.Bucket
for index, bucket := range buckets {
if index == 0 {
bkt, err = tx.CreateBucketIfNotExists([]byte(bucket))
} else {
bkt, err = bkt.CreateBucketIfNotExists([]byte(bucket))
}
if err != nil {
return fmt.Errorf("Error creating nested bucket [%s]: %v", bucket, err)
}
}
return err
})
return err
}

fste89's demo has some debug;
this right:
package main
import (
"fmt"
"time"
"github.com/boltdb/bolt"
)
func CreateNestedBuckets(fatherTable string, sonTabls []string) error {
db, dberr := bolt.Open("your file path", 0600, &bolt.Options{Timeout: 1 * time.Second})
if dberr != nil {
fmt.Println(dberr)
}
defer db.Close()
err := db.Update(func(tx *bolt.Tx) error {
var bkt *bolt.Bucket
var err error
bkFather, err = tx.CreateBucketIfNotExists([]byte(fatherTable))
for _, ta := range sonTabls {
fmt.Println(ta)
_, err = bkFather.CreateBucketIfNotExists([]byte(ta))
if err != nil {
fmt.Println("error creating nested bucket")
return err
}
}
return nil
})
if err != nil {
fmt.Println("error creating nested bucket!!!")
return err
}
return nil
}
func main() {
t := []string{"cc", "1", "2", "3"}
fmt.Println(CreateNestedBuckets("sb", t))
}
echo:
cc
1
2
3
<nil>
Visible
enter image description here

func CreateNestedBuckets(fatherTable string, sonTables []string) error {
db, dberr := bolt.Open("E:\\OneDrive\\code\\go\\project\\transmission\\static\\localstorage.db", 0600, &bolt.Options{Timeout: 1 * time.Second})
if dberr != nil {
fmt.Println(dberr)
}
defer db.Close()
err := db.Update(func(tx *bolt.Tx) error {
var err error
bkFather, err := tx.CreateBucketIfNotExists([]byte(fatherTable))
for _, ta := range sonTables {
fmt.Println(ta)
_, err = bkFather.CreateBucketIfNotExists([]byte(ta))
if err != nil {
fmt.Println("error creating nested bucket")
return err
}
}
return nil
})
if err != nil {
fmt.Println("error creating nested bucket!!!")
return err
}
return nil
}

Related

rows.Next() halts after some number of rows

Im newbie in Golang, so it may be simple for professionals but I got stuck with no idea what to do next.
I'm making some migration app that extract some data from oracle DB and after some conversion insert it to Postges one-by-one.
The result of native Query in DB console returns about 400k of rows and takes about 13 sec to end.
The data from Oracle extracts with rows.Next() with some strange behavior:
First 25 rows extracted fast enough, then about few sec paused, then new 25 rows until it pauses "forever".
Here is the function:
func GetHrTicketsFromOra() (*sql.Rows, error) {
rows, err := oraDB.Query("select id,STATE_ID,REMEDY_ID,HEADER,CREATE_DATE,TEXT,SOLUTION,SOLUTION_USER_LOGIN,LAST_SOLUTION_DATE from TICKET where SOLUTION_GROUP_ID = 5549")
if err != nil {
println("Error while getting rows from Ora")
return nil, err
}
log.Println("Finished legacy tickets export")
return rows, err
}
And here I export data:
func ConvertRows(rows *sql.Rows, c chan util.ArchTicket, m chan int) error {
log.Println("Conversion start")
defer func(rows *sql.Rows) {
err := rows.Close()
if err != nil {
log.Println("ORA connection closed", err)
return
}
}(rows)
for rows.Next() {
log.Println("Reading the ticket")
ot := util.OraTicket{}
at := util.ArchTicket{}
err := rows.Scan(&ot.ID, &ot.StateId, &ot.RemedyId, &ot.Header, &ot.CreateDate, &ot.Text, &ot.Solution, &ot.SolutionUserLogin, &ot.LastSolutionDate)
if err != nil {
log.Println("Error while reading row", err)
return err
}
at = convertLegTOArch(ot)
c <- at
}
if err := rows.Err(); err != nil {
log.Println("Error while reading row", err)
return err
}
m <- 1
return nil
}
UPD. I use "github.com/sijms/go-ora/v2" driver
UPD2. Seems like the root cause of the problem is in TEXT and SOLUTION fields of the result rows. They are varchar and can be big enough. Deleting them from the direct query changes the time of execution from 13sec to 258ms. But I still have no idea what to do with that.
UPD3.
Minimal reproducible example
package main
import (
"database/sql"
_ "github.com/sijms/go-ora/v2"
"log"
)
var oraDB *sql.DB
var con = "oracle://login:password#ora_db:1521/database"
func InitOraDB(dataSourceName string) error {
var err error
oraDB, err = sql.Open("oracle", dataSourceName)
if err != nil {
return err
}
return oraDB.Ping()
}
func GetHrTicketsFromOra() {
var ot string
rows, err := oraDB.Query("select TEXT from TICKET where SOLUTION_GROUP_ID = 5549")
if err != nil {
println("Error while getting rows from Ora")
}
for rows.Next() {
log.Println("Reading the ticket")
err := rows.Scan(&ot)
if err != nil {
log.Println("Reading failed", err)
}
log.Println("Read:")
}
log.Println("Finished legacy tickets export")
}
func main() {
err := InitOraDB(con)
if err != nil {
log.Println("Error connection Ora")
}
GetHrTicketsFromOra()
}

How to create a reusable code in Golang to read different yamls and put them into different structs types

I have to read let say 2 or 3 or more yamls that are different in structure and have a struct for each of those structures where I want to store them. So far I am creating separate functions for each and it works, but does not look very elegant... I think.
Here are the functions today:
// read the Yaml into struct(s)
type Config struct {...}
type ExecuteQueries struct {...}
func parseYamlConfig(pathYaml string) Config {
myConfig := Config{}
var err error
var yamlFile []byte
if pathYaml == "" {
yamlFile, err = ioutil.ReadFile("./conf/conf.yaml")
} else {
yamlFile, err = ioutil.ReadFile(pathYaml)
}
if err != nil {
log.Fatalf("error: %v", err)
}
err = yaml.Unmarshal([]byte(yamlFile), &myConfig)
if err != nil {
log.Fatalf("error: %v", err)
}
return myConfig
}
func parseYamlConfig2(pathYaml string) ExecuteQueries {
myConfig := ExecuteQueries{}
var err error
var yamlFile []byte
if pathYaml == "" {
yamlFile, err = ioutil.ReadFile("./conf/conf.yaml")
} else {
yamlFile, err = ioutil.ReadFile(pathYaml)
}
if err != nil {
log.Fatalf("error: %v", err)
}
err = yaml.Unmarshal([]byte(yamlFile), &myConfig)
if err != nil {
log.Fatalf("error: %v", err)
}
return myConfig
}
Notice that they are actually different in what they return and what they receive, but the processing of data is very similar. How should this be expressed in a more elegant way?
func unmarshalYAMLFile(path string, v interface{}) error {
if path == "" {
path = "./conf/conf.yaml"
}
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
return yaml.NewDecoder(f).Decode(v)
}
conf1 := Config{}
if err := unmarshalYAMLFile("/path/to/conf.yaml", &conf1); err != nil {
panic(err)
}
conf2 := ExecuteQueries{}
if err := unmarshalYAMLFile("/path/to/conf_2.yaml", &conf2); err != nil {
panic(err)
}

How to gently defer execution of a function that might return an error?

Most cleanup functions, especially those related to the IO operations, return an error, and normally we'd prefer to defer their execution in case if we'd not forget to call them when we're done with acquired resources. For example, at some point in the code we might write something like this:
var r *SomeResource
var err error
if r, err = Open(/* parameters */); err != nil {
return nil, err
}
defer r.Close() // This might return an error
It seems that if Close function returns an error, it'll be ignored. How can we gently process the returned error from such a function?
Using defer with a func() {}() like so.
var r *SomeResource
var err error
if r, err = Open(/* parameters */); err != nil {
return nil, err
}
defer func() {
if err = r.Close(); err != nil {
fmt.Printf("ERROR: %v", err)
}
}()
Fail gracefully with an error. Report the first error. Don't overwrite earlier errors. For example,
package main
import (
"fmt"
"os"
)
func demo() (name string, err error) {
filename := `test.file`
f, err := os.Open(filename)
if err != nil {
return "", err
}
defer func() {
e := f.Close()
if e != nil {
if err == nil {
err = e
}
}
}()
// do someting with the file
name = f.Name()
fi, err := f.Stat()
if err != nil {
return name, err
}
if fi.Size() == 0 {
err = fmt.Errorf("%s: empty file", filename)
return name, err
}
return name, err
}
func main() {
name, err := demo()
fmt.Println(name, err)
}
We can handle this in ways like:
way-1:
func myFn() error {
var err error
if r, err = Open(/* parameters */); err != nil {
return nil, err
}
defer func() {
if cErr = r.Close(); cErr != nil {
err = cErr
}
}()
return err
}
way-2:
func myFn() error {
var err error
if r, err = Open(/* parameters */); err != nil {
return nil, err
}
defer func() {
if cErr = r.Close(); cErr != nil {
// we can log the error
// or
// whatever we want to do
}
}()
return err
}
I have also find a nice blog on this topic, i mean handling error when defer func returns an error. Check here https://blog.learngoprogramming.com/5-gotchas-of-defer-in-go-golang-part-iii-36a1ab3d6ef1.

Copy a folder in go

Is there an easy way to copy a directory in go?
I have the following function:
err = CopyDir("sourceFolder","destinationFolder")
Nothing so far has worked, including libraries such as github.com/cf-guardian/guardian/kernel/fileutils
One important thing to note is that I need to preserve directory structure, including the sourceFolder itself, not simply copy all contents of the folder.
I believe that docker implementation can be considered as complete solution for handling edge cases:
https://github.com/moby/moby/blob/master/daemon/graphdriver/copy/copy.go
There are following good things:
unsupported file type rise error
preserving permissions and ownership
preserving extended attributes
preserving timestamp
but because of a lot of imports your tiny application becomes huge.
I've tried to combine several solutions but use stdlib and for Linux only:
func CopyDirectory(scrDir, dest string) error {
entries, err := os.ReadDir(scrDir)
if err != nil {
return err
}
for _, entry := range entries {
sourcePath := filepath.Join(scrDir, entry.Name())
destPath := filepath.Join(dest, entry.Name())
fileInfo, err := os.Stat(sourcePath)
if err != nil {
return err
}
stat, ok := fileInfo.Sys().(*syscall.Stat_t)
if !ok {
return fmt.Errorf("failed to get raw syscall.Stat_t data for '%s'", sourcePath)
}
switch fileInfo.Mode() & os.ModeType{
case os.ModeDir:
if err := CreateIfNotExists(destPath, 0755); err != nil {
return err
}
if err := CopyDirectory(sourcePath, destPath); err != nil {
return err
}
case os.ModeSymlink:
if err := CopySymLink(sourcePath, destPath); err != nil {
return err
}
default:
if err := Copy(sourcePath, destPath); err != nil {
return err
}
}
if err := os.Lchown(destPath, int(stat.Uid), int(stat.Gid)); err != nil {
return err
}
fInfo, err := entry.Info()
if err != nil {
return err
}
isSymlink := fInfo.Mode()&os.ModeSymlink != 0
if !isSymlink {
if err := os.Chmod(destPath, fInfo.Mode()); err != nil {
return err
}
}
}
return nil
}
func Copy(srcFile, dstFile string) error {
out, err := os.Create(dstFile)
if err != nil {
return err
}
defer out.Close()
in, err := os.Open(srcFile)
defer in.Close()
if err != nil {
return err
}
_, err = io.Copy(out, in)
if err != nil {
return err
}
return nil
}
func Exists(filePath string) bool {
if _, err := os.Stat(filePath); os.IsNotExist(err) {
return false
}
return true
}
func CreateIfNotExists(dir string, perm os.FileMode) error {
if Exists(dir) {
return nil
}
if err := os.MkdirAll(dir, perm); err != nil {
return fmt.Errorf("failed to create directory: '%s', error: '%s'", dir, err.Error())
}
return nil
}
func CopySymLink(source, dest string) error {
link, err := os.Readlink(source)
if err != nil {
return err
}
return os.Symlink(link, dest)
}
This package seems to do exactly what you want to do, give it a try.
From the readme:
err := Copy("your/source/directory", "your/destination/directory")
Not satisfied with the already listed options which include using sketchy libraries, or vastly bloated libraries.
In my case, I opted to do things the old fashioned way. With shell commands!
import (
"os/exec"
)
func main() {
// completely arbitrary paths
oldDir := "/home/arshbot/"
newDir := "/tmp/"
cmd := exec.Command("cp", "--recursive", oldDir, newDir)
cmd.Run()
}
This solution copies a directory recursively, including symbolic links. Trying to be efficient in the actual copy stage using streams.
Also it's fairly easy to handle more of irregular files if needed.
// CopyDir copies the content of src to dst. src should be a full path.
func CopyDir(dst, src string) error {
return filepath.Walk(src, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
// copy to this path
outpath := filepath.Join(dst, strings.TrimPrefix(path, src))
if info.IsDir() {
os.MkdirAll(outpath, info.Mode())
return nil // means recursive
}
// handle irregular files
if !info.Mode().IsRegular() {
switch info.Mode().Type() & os.ModeType {
case os.ModeSymlink:
link, err := os.Readlink(path)
if err != nil {
return err
}
return os.Symlink(link, outpath)
}
return nil
}
// copy contents of regular file efficiently
// open input
in, _ := os.Open(path)
if err != nil {
return err
}
defer in.Close()
// create output
fh, err := os.Create(outpath)
if err != nil {
return err
}
defer fh.Close()
// make it the same
fh.Chmod(info.Mode())
// copy content
_, err = io.Copy(fh, in)
return err
})
}
I've come up with a relatively shorter answer which uses path/filepath's Walk method:
import (
"io/ioutil"
"path/filepath"
"os"
"strings"
)
func copy(source, destination string) error {
var err error = filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
var relPath string = strings.Replace(path, source, "", 1)
if relPath == "" {
return nil
}
if info.IsDir() {
return os.Mkdir(filepath.Join(destination, relPath), 0755)
} else {
var data, err1 = ioutil.ReadFile(filepath.Join(source, relPath))
if err1 != nil {
return err1
}
return ioutil.WriteFile(filepath.Join(destination, relPath), data, 0777)
}
})
return err
}
Also this might be a solution:
available on github.com/floscodes/golang-tools
import (
"fmt"
"io/ioutil"
"os"
)
func CopyDir(src string, dest string) error {
if dest[:len(src)] == src {
return fmt.Errorf("Cannot copy a folder into the folder itself!")
}
f, err := os.Open(src)
if err != nil {
return err
}
file, err := f.Stat()
if err != nil {
return err
}
if !file.IsDir() {
return fmt.Errorf("Source " + file.Name() + " is not a directory!")
}
err = os.Mkdir(dest, 0755)
if err != nil {
return err
}
files, err := ioutil.ReadDir(src)
if err != nil {
return err
}
for _, f := range files {
if f.IsDir() {
err = CopyDir(src+"/"+f.Name(), dest+"/"+f.Name())
if err != nil {
return err
}
}
if !f.IsDir() {
content, err := ioutil.ReadFile(src + "/" + f.Name())
if err != nil {
return err
}
err = ioutil.WriteFile(dest+"/"+f.Name(), content, 0755)
if err != nil {
return err
}
}
}
return nil
}

Need faster way to list all datasets/tables in project

I am creating a utility that needs to be aware of all the datasets/tables that exist in my BigQuery project. My current code for getting this information is as follows (using Go API):
func populateExistingTableMap(service *bigquery.Service, cloudCtx context.Context, projectId string) (map[string]map[string]bool, error) {
tableMap := map[string]map[string]bool{}
call := service.Datasets.List(projectId)
//call.Fields("datasets/datasetReference")
if err := call.Pages(cloudCtx, func(page *bigquery.DatasetList) error {
for _, v := range page.Datasets {
if tableMap[v.DatasetReference.DatasetId] == nil {
tableMap[v.DatasetReference.DatasetId] = map[string]bool{}
}
table_call := service.Tables.List(projectId, v.DatasetReference.DatasetId)
//table_call.Fields("tables/tableReference")
if err := table_call.Pages(cloudCtx, func(page *bigquery.TableList) error {
for _, t := range page.Tables {
tableMap[v.DatasetReference.DatasetId][t.TableReference.TableId] = true
}
return nil
}); err != nil {
return errors.New("Error Parsing Table")
}
}
return nil
}); err != nil {
return tableMap, err
}
return tableMap, nil
}
For a project with about 5000 datasets, each with up to 10 tables, this code takes almost 15 minutes to return. Is there a faster way to iterate through the names of all existing datasets/tables? I have tried using the Fields method to return only the fields I need (you can see those lines commented out above), but that results in only 50 (exactly 50) of my datasets being returned.
Any ideas?
Here is an updated version of my code, with concurrency, that reduced the processing time from about 15 minutes to 3 minutes.
func populateExistingTableMap(service *bigquery.Service, cloudCtx context.Context, projectId string) (map[string]map[string]bool, error) {
tableMap = map[string]map[string]bool{}
call := service.Datasets.List(projectId)
//call.Fields("datasets/datasetReference")
if err := call.Pages(cloudCtx, func(page *bigquery.DatasetList) error {
var wg sync.WaitGroup
wg.Add(len(page.Datasets))
for _, v := range page.Datasets {
if tableMap[v.DatasetReference.DatasetId] == nil {
tableMap[v.DatasetReference.DatasetId] = map[string]bool{}
}
go func(service *bigquery.Service, datasetID string, projectId string) {
defer wg.Done()
table_call := service.Tables.List(projectId, datasetID)
//table_call.Fields("tables/tableReference")
if err := table_call.Pages(cloudCtx, func(page *bigquery.TableList) error {
for _, t := range page.Tables {
tableMap[datasetID][t.TableReference.TableId] = true
}
return nil // NOTE: returning a non-nil error stops pagination.
}); err != nil {
// TODO: Handle error.
fmt.Println(err)
}
}(service, v.DatasetReference.DatasetId, projectId)
}
wg.Wait()
return nil // NOTE: returning a non-nil error stops pagination.
}); err != nil {
return tableMap, err
// TODO: Handle error.
}
return tableMap, nil
}

Resources