I'm trying to bulk remove objects in minio as described here:
objectsCh := make(chan minio.ObjectInfo)
// Send object names that are needed to be removed to objectsCh
go func() {
defer close(objectsCh)
// List all objects from a bucket-name with a matching prefix.
for object := range minioClient.ListObjects(context.Background(), "my-bucketname", "my-prefixname", true, nil) {
if object.Err != nil {
log.Fatalln(object.Err)
}
objectsCh <- object
}
}()
opts := minio.RemoveObjectsOptions{
GovernanceBypass: true,
}
for rErr := range minioClient.RemoveObjects(context.Background(), "my-bucketname", objectsCh, opts) {
fmt.Println("Error detected during deletion: ", rErr)
}
Where I can ListObjects by bucketname and prefixname. However I'm struggling to find an approach where I can ListObjects by for example a slice of object names which I want to remove or any other way. So my question is: how can I properly generate a ListObjects for arbitrary objectNames in a given bucket? Or is there any other way to do remove objects by their names? Thanks.
func DeleteItemInMinio(ctx context.Context, item []string) (string, error) {
minioClient, err := minio.New("test.com", os.Getenv("MINIO_ACCESS_KEY"), os.Getenv("MINIO_SECRET_KEY"), true)
if err != nil {
log.Println(err)
}
for _, val := range item {
err = minioClient.RemoveObject("my-bucketname", val)
if err != nil {
panic(err)
}
}
return "success", nil
}
and call it with :
r.POST("/test/delete", func(c *gin.Context) {
item := []string{"golang.png", "phplogo.jpg"}
execute.DeleteItemInMinio(context.Background(), item)
})
i tried it and it works, in case you still need it
Related
I know this may seem like bad design (and I wish I didn't need to do this), but I need to read a struct that is generated automatically at run time and create a new instance of it.
The struct that I create I only need limited metadata of so that it can be passed to the Gorm Gen FieldRelateModel method (from here) where 'relModel' is the new instance I am hoping to make (see below):
FieldRelateModel = func(relationship field.RelationshipType, fieldName string, relModel interface{}, config *field.RelateConfig) model.CreateFieldOpt {
st := reflect.TypeOf(relModel)
if st.Kind() == reflect.Ptr {
st = st.Elem()
}
fieldType := st.String()
if config == nil {
config = &field.RelateConfig{}
}
if config.JSONTag == "" {
config.JSONTag = ns.ColumnName("", fieldName)
}
return func(*model.Field) *model.Field {
return &model.Field{
Name: fieldName,
Type: config.RelateFieldPrefix(relationship) + fieldType,
JSONTag: config.JSONTag,
GORMTag: config.GORMTag,
NewTag: config.NewTag,
OverwriteTag: config.OverwriteTag,
Relation: field.NewRelationWithModel(relationship, fieldName, fieldType, relModel),
}
}
}
I am able to parse the struct and get its metadata using ast and reflect (see below), but I don't know how I can do the last step so that the 'structInstance' I return is valid when passed as the 'relModel' to the 'FieldRelateModel' function/option.
func StructExtract(filePath string) (any, error) {
file, err := os.Open(filePath)
if err != nil {
fmt.Println(err)
return nil, err
}
defer file.Close()
fset := token.NewFileSet()
f, err := parser.ParseFile(fset, filePath, file, parser.AllErrors)
if err != nil {
fmt.Println(err)
return nil, err
}
var structType *ast.StructType
// Iterate over the top-level declarations in the file
for _, decl := range f.Decls {
// Check if the declaration is a GenDecl (which can contain structs)
genDecl, ok := decl.(*ast.GenDecl)
if !ok {
continue
}
// Iterate over the GenDecl's specs
for _, spec := range genDecl.Specs {
// Check if the spec is a TypeSpec (which can contain structs)
typeSpec, ok := spec.(*ast.TypeSpec)
if !ok {
continue
}
// Check if the TypeSpec's type is a struct
structType, ok = typeSpec.Type.(*ast.StructType)
if !ok {
continue
}
break
}
}
if structType == nil {
fmt.Println("No struct found in file.")
return nil, err
}
structInstance := reflect.New(reflect.TypeOf(structType)).Elem().Interface()
return structInstance, nil
}
Here is a go playground that I thought might make it easier if someone were to help, however, due to the nature of the problem it won't be able to run within the playground (for various reasons).
I'm trying to list all the items in a DynamoDB table with pagination, and here below is my attempt:
const tableName = "RecordingTable"
type Recording struct {
ID string `dynamodbav:"id"`
CreatedAt string `dynamodbav:"createdAt"`
UpdatedAt string `dynamodbav:"updatedAt"`
Duration int `dynamodbav:"duration"`
}
type RecordingRepository struct {
ctx context.Context
svc *dynamodb.Client
}
func NewRecordingRepository(ctx context.Context) (*RecordingRepository, error) {
cfg, err := config.LoadDefaultConfig(ctx)
if err != nil {
return nil, err
}
return &RecordingRepository{ctx, dynamodb.NewFromConfig(cfg)}, nil
}
func (r *RecordingRepository) List(page int, size int) ([]Recording, error) {
size32 := int32(size)
queryInput := &dynamodb.QueryInput{
TableName: aws.String(tableName),
Limit: &size32,
}
recordings := []Recording{}
queryPaginator := dynamodb.NewQueryPaginator(r.svc, queryInput)
for i := 0; queryPaginator.HasMorePages(); i++ {
result, err := queryPaginator.NextPage(r.ctx)
if err != nil {
return nil, err
}
if i == page {
if result.Count > 0 {
for _, v := range result.Items {
recording := Recording{}
if err := attributevalue.UnmarshalMap(v, &recording); err != nil {
return nil, err
}
recordings = append(recordings, recording)
}
}
break
}
}
return recordings, nil
}
When I run the code above, I get the following error message:
api error ValidationException: Either the KeyConditions or KeyConditionExpression parameter must be specified in the request.
But why should I specify a KeyConditionExpression when I want to get all the items? Is there another way to go or a workaround this?
Query does need your keys. It is meant to find specific items in your DynamoDB. To get all items in your DynamoDB, you need to use the Scan operation.
This should be easily fixed in your code.
Instead of QueryInput use ScanInput and instead of NewQueryPaginator use NewScanPaginator.
Just replaced QueryInput with ScanInput and QueryPaginator with ScanPaginator.
I have a map like this, which I want to save/retrive from redis using redigo:
animals := map[string]bool{
"cat": true,
"dog": false,
"fox": true,
}
The length of the map may vary.
I tried these function:
func SetHash(key string, value map[string]bool) error {
conn := Pool.Get()
defer conn.Close()
_, err := conn.Do("HMSET", key, value)
if err != nil {
return fmt.Errorf("error setting key %s to %s: %v", key, value, err)
}
return err
}
func GetHash(key string) (map[string]bool, error) {
conn := Pool.Get()
defer conn.Close()
val, err := conn.Do("HGETALL", key)
if err != nil {
fmt.Errorf("error setting key %s to %s: %v", key, nil, err)
return nil, err
}
return val, err
}
But can not make GetHash correctly. I've checked the docs examples and it was not helpful. So appreciate your help to have a working example.
HMSET is deprecated, use HSET instead, no effect here though.
The map[string]bool may be flattened with AddFlat() for SetHash().
c.Do("HSET", redis.Args{}.Add("key").AddFlat(value)...)
For GetHash(), use Values(). You may use ScanStruct() to map to a struct or loop through the values to create a map dynamically.
v, err := redis.Values(c.Do("HGETALL", key))
redis.ScanStruct(v, &myStruct);
See example from redigo tests in scan_test.go.
The application is responsible for converting structured types to and from the types understood by Redis.
Flatten the map into a list of arguments:
func SetHash(key string, value map[string]bool) error {
conn := Pool.Get()
defer conn.Close()
// Create arguments: key field value [field value]...
var args = []interface{}{key}
for k, v := range value {
args = append(args, k, v)
}
_, err := conn.Do("HMSET", args...)
if err != nil {
return fmt.Errorf("error setting key %s to %v: %v", key, value, err)
}
return err
}
Convert the returned field value pairs to a map:
func GetHash(key string) (map[string]bool, error) {
conn := Pool.Get()
defer conn.Close()
values, err := redis.Strings(conn.Do("HGETALL", key))
if err != nil {
return nil, err
}
// Loop through [field value]... and parse value as bool.
m := map[string]bool{}
for i := 0; i < len(values); i += 2 {
b, err := strconv.ParseBool(value)
if err != nil {
return nil, errors.New("value not a bool")
}
m[key] = b
}
return m, nil
}
How can I get the p.template from firestore, which is a string, into template.ParseFiles function? Is it possible to use the field value in the function to select the correct template file?
type Property struct {
Name string `firestore:"name"`
ApprovedOrigins []interface{} `firestore:"approvedOrigins"`
Template string `firestore:"selected"`
}
As you can see above the firestore field name is selected
func serveHandler(w http.ResponseWriter, r *http.Request, params map[string]string) {
ctx := context.Background()
client, err := firestore.NewClient(ctx, projectId)
if err != nil {
// TODO: Handle error.
log.Println("FIREBASE ERROR:", err)
}
// collection group query in firestore
q := client.CollectionGroup("data").Where("approvedOrigins", "array-contains", r.Host).Limit(1)
// iterate through the document query
iter := q.Documents(ctx)
defer iter.Stop()
for {
doc, err := iter.Next()
if err == iterator.Done {
break
}
if err != nil {
// TODO: Handle error.
log.Println("FIREBASE ERROR:", err)
}
fmt.Println("Database connected...")
var p Property
if err := doc.DataTo(&p); err != nil {
fmt.Println(err)
}
fmt.Println(p.Template) // This is not logging any data/string
t, _ := template.ParseFiles(p.Template + ".html")
fmt.Println(t.Execute(w, p)) //504 error happens here
}
}
I am creating a utility that needs to be aware of all the datasets/tables that exist in my BigQuery project. My current code for getting this information is as follows (using Go API):
func populateExistingTableMap(service *bigquery.Service, cloudCtx context.Context, projectId string) (map[string]map[string]bool, error) {
tableMap := map[string]map[string]bool{}
call := service.Datasets.List(projectId)
//call.Fields("datasets/datasetReference")
if err := call.Pages(cloudCtx, func(page *bigquery.DatasetList) error {
for _, v := range page.Datasets {
if tableMap[v.DatasetReference.DatasetId] == nil {
tableMap[v.DatasetReference.DatasetId] = map[string]bool{}
}
table_call := service.Tables.List(projectId, v.DatasetReference.DatasetId)
//table_call.Fields("tables/tableReference")
if err := table_call.Pages(cloudCtx, func(page *bigquery.TableList) error {
for _, t := range page.Tables {
tableMap[v.DatasetReference.DatasetId][t.TableReference.TableId] = true
}
return nil
}); err != nil {
return errors.New("Error Parsing Table")
}
}
return nil
}); err != nil {
return tableMap, err
}
return tableMap, nil
}
For a project with about 5000 datasets, each with up to 10 tables, this code takes almost 15 minutes to return. Is there a faster way to iterate through the names of all existing datasets/tables? I have tried using the Fields method to return only the fields I need (you can see those lines commented out above), but that results in only 50 (exactly 50) of my datasets being returned.
Any ideas?
Here is an updated version of my code, with concurrency, that reduced the processing time from about 15 minutes to 3 minutes.
func populateExistingTableMap(service *bigquery.Service, cloudCtx context.Context, projectId string) (map[string]map[string]bool, error) {
tableMap = map[string]map[string]bool{}
call := service.Datasets.List(projectId)
//call.Fields("datasets/datasetReference")
if err := call.Pages(cloudCtx, func(page *bigquery.DatasetList) error {
var wg sync.WaitGroup
wg.Add(len(page.Datasets))
for _, v := range page.Datasets {
if tableMap[v.DatasetReference.DatasetId] == nil {
tableMap[v.DatasetReference.DatasetId] = map[string]bool{}
}
go func(service *bigquery.Service, datasetID string, projectId string) {
defer wg.Done()
table_call := service.Tables.List(projectId, datasetID)
//table_call.Fields("tables/tableReference")
if err := table_call.Pages(cloudCtx, func(page *bigquery.TableList) error {
for _, t := range page.Tables {
tableMap[datasetID][t.TableReference.TableId] = true
}
return nil // NOTE: returning a non-nil error stops pagination.
}); err != nil {
// TODO: Handle error.
fmt.Println(err)
}
}(service, v.DatasetReference.DatasetId, projectId)
}
wg.Wait()
return nil // NOTE: returning a non-nil error stops pagination.
}); err != nil {
return tableMap, err
// TODO: Handle error.
}
return tableMap, nil
}