I have a gorm connection that I initially create by passing an AWS authentication token that expires every 15 minutes. The service will be able to connect to the DB for 15 minute. I have some functions for connecting to the db, for creating a new token, and for using a cron library to "refresh" the connection. I'm stuck in knowing how to actually provide the new token to GORM. I though of just doing gorm.Open() from within the cron job, but was told this would be bad because it'd not be thread safe, and would leave the old connections open. I was directed to this but I'm not sure what to do with it:
https://golang.org/pkg/database/sql/driver/#Connector
There is an example here:
https://github.com/aws/aws-sdk-go/issues/3043#issuecomment-581931580
But I'm not sure what I'd replace the driver and connector code in that example for GORM. Anyhow.
Here's the code I have so far. First this connection function that wraps around gorm.Open:
func Connect(conf *config.DbConfig) (*gorm.DB, error) {
host, port, err := net.SplitHostPort(conf.Host)
if err != nil {
return nil, err
}
sslMode := "require"
if conf.DisableTLS {
sslMode = "disable"
}
return gorm.Open(conf.Dialect,
fmt.Sprintf("host=%s port=%s user=%s password=%s dbname=%s sslmode=%s",
host, port, conf.Username, conf.Password, conf.Name, sslMode))
}
This is called in main like so:
//IF WE WANT TO USE AWS TOKEN, REPLACE PW IN CONFIG WITH TOKEN
if conf.DB.UseAwsToken {
authToken, err := field.CreateAuthToken(&conf.DB)
if err != nil {
logger.Fatal(err)
}
conf.DB.Password = authToken
}
//CONNECT TO DB
db, err := field.Connect(&conf.DB)
if err != nil {
logger.Fatal(err)
}
defer db.Close()
//INITIALIZE CRON JOB THAT RUNS IN SEPARATE THREAD TO REFRESH CONNECTION
if conf.DB.UseAwsToken {
processorCron := cron.New()
cronInterval := "#every 10m"
if conf.DB.CronInterval > 0 {
cronInterval = fmt.Sprintf("#every %s", conf.DB.CronInterval)
}
processorCron.AddFunc(cronInterval, func() {
repo.RenewDB(&conf.DB)
})
processorCron.Start()
}
The part I'm stuck with the actual code for the RenewDB function, in the spot with the TODO:
type Repo struct {
db *gorm.DB
}
// FUNCTION
func (r *Repo) RenewDB(dbConf *config.DbConfig) {
var err error
if dbConf.Password, err = CreateAuthToken(dbConf); err != nil {
panic(fmt.Sprintf("failed at createAuthToken: %s\n", err.Error()))
}
//TODO: HOW DO I UPDATE GORM TO USE THE NEW AUTH TOKEN THAT I INSERTED
// INTO dbConf?
}
//AUTH TOKEN GENERATOR
func CreateAuthToken(dbConf *config.DbConfig) (string, error) {
awsSession, err := session.NewSession()
if err != nil {
return "", err
}
authToken, errToken := rdsutils.BuildAuthToken(dbConf.Host, dbConf.AwsRegion, dbConf.Username, awsSession.Config.Credentials)
if errToken != nil {
return "", errToken
}
return authToken, nil
}
Related
Let's assume that I want to make a client, for example, MySQL client (but my question is generic and not just for MySQL client, any client), this is a sample code
func main() {
db, err := sql.Open("mysql", "root:<yourMySQLdatabasepassword>#tcp(127.0.0.1:3306)/test")
if err != nil {
panic(err.Error())
}
defer db.Close()
}
I want to move it into a function so I don't have to initialize it all over the place and here my question begins I can do it in various ways and I want to know which one is the recommended way
Here is functional way
func client() (client *sql.DB, err error) {
client, err = sql.Open("mysql", "root:<yourMySQLdatabasepassword>#tcp(127.0.0.1:3306)/test")
if err != nil {
panic(err.Error())
}
return
}
Global variable way
var client, err = sql.Open("mysql", "root:<yourMySQLdatabasepassword>#tcp(127.0.0.1:3306)/test")
init func way
var Client *sql.DB
func init() {
Client, err := sql.Open("mysql", "root:root#tcp(localhost:3306)/otp")
if err != nil {
panic(err.Error())
}
}
and finally struct way
type MSQL struct {
// Fields...
}
func (m *MSQL) Client() *sql.DB {
client, err := sql.Open("mysql", "root:<yourMySQLdatabasepassword>#tcp(127.0.0.1:3306)/test")
if err != nil {
panic(err.Error())
}
return client
}
// NewMysql().where(...)
I'm really confused here to select which one to follow
Exposing a public variable var Client *sql.DB is error prone as I can do <package>.Client = null from anywhere.
struct way is the way to go, As you can work with multiple connection each to a specific database at any time.
You can also use New() to pass in a config object or other props to customise other db props. like Max Connection.
package database;
type Db struct{
*db sql.DB
}
func New(driver, connection string) *Db {
db, err := sql.Open(driver, connection)
if err != nil {
panic(err)
}
if err = db.Ping(); err != nil {
panic(err)
}
/* configure details
db.SetConnMaxLifetime(conLifeTime)
db.SetMaxOpenConns(maxOpenConns)
db.SetMaxIdleConns(maxIdleConns)
*/
return &Db{db}
}
func (d *Db) Close() error {
return d.db.Close();
}
Its generic, you can pass any driver & connection string for creating a connection to any database.
connection database.New("mysql", "root:<yourMySQLdatabasepassword>#tcp(127.0.0.1:3306)/test")
usage database.Db.Where(...)
close connection database.Db.Close()
I'm trying to implement pgxpool in a new go app. I keep getting a "pool closed" error after attempting a scan into a struct.
The pgx logger into gives me this after connecting. I thought the pgxpool was meant to remain open.
{"level":"info","msg":"closed connection","pid":5499,"time":"2022-02-24T16:36:33+10:30"}
Here is my router code
func router() http.Handler {
var err error
config, err := pgxpool.ParseConfig(os.Getenv("DATABASE_URL"))
if err != nil {
log.Fatalln(err)
}
log.Println(os.Getenv("DATABASE_URL"))
logrusLogger := &logrus.Logger{
Out: os.Stderr,
Formatter: new(logrus.JSONFormatter),
Hooks: make(logrus.LevelHooks),
Level: logrus.InfoLevel,
ExitFunc: os.Exit,
ReportCaller: false,
}
config.ConnConfig.Logger = NewLogger(logrusLogger)
db, err := pgxpool.ConnectConfig(context.Background(), config)
if err != nil {
log.Fatalln(err)
}
defer db.Close()
--- minio connection
rs := newAppResource(db, mc)
Then, in a helper file I setup the resource
type appResource struct {
db *pgxpool.Pool
mc *minio.Client
}
// newAppResource function to pass global var
func newAppResource(db *pgxpool.Pool, mc *minio.Client) *appResource {
return &appResource{
db: db,
mc: mc,
}
}
There "pool closed" error occurs at the end of this code
func (rs *appResource) login(w http.ResponseWriter, r *http.Request) {
var user User
var login Login
d := json.NewDecoder(r.Body)
d.DisallowUnknownFields() // catch unwanted fields
err := d.Decode(&login)
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
if err != nil {
fmt.Println("can't decode JSON", err)
}
if login.Email == "" {
log.Println("empty email")
return
}
log.Println(login.Email)
log.Println(login.Password)
if login.Password == "" {
log.Println("empty password")
return
}
// optional extra check
if d.More() {
http.Error(w, "extraneous data after JSON object", http.StatusBadRequest)
return
}
sqlStatement := "SELECT user_id, password FROM users WHERE active = 'true' AND email = ?"
row := rs.db.QueryRow(context.Background(), sqlStatement, login.Email)
err = row.Scan(&user.UserId, &user.Password)
if err == sql.ErrNoRows {
log.Println("user not found")
http.Error(w, err.Error(), http.StatusUnauthorized)
return
}
if err != nil {
log.Println(err)
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
It appears that you are doing something like the following:
func router() http.Handler {
db, err := pgxpool.ConnectConfig(context.Background(), config)
if err != nil {
log.Fatalln(err)
}
defer db.Close()
return appResource{db: db}
}
The issue with this is that the defer db.Close() runs when the function router() ends and this is before the returned pgxPool.Pool is actually used (the http.Handler returned will be used later when http requests are processed). Attempting to use a closed pgxPool.Pool results in the error you are seeing.
The simplest solution is to simply remove the defer db.Close() however you might also consider calling db.Close() as part of a clean shutdown process (it needs to remain open as long as you are handling requests).
You are using pgxpool which does differ from the standard library; however I believe that the advice given in the standard library docs applies here:
It is rarely necessary to close a DB.
I used the operator-sdk to create a custom resource DatabaseService. Creation a DatabaseService CR should trigger the Reconcile function that would create a secret in the CR namespace after getting it from a third party.
I set the CR as the owner of the secret so that whenever the secret is manually deleted, the reconcile function will trigger again and recreate the secret.
Here is the code:
func (r *DatabaseServiceReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
requeueResult := ctrl.Result{Requeue: true, RequeueAfter: time.Minute}
emptyResult := ctrl.Result{Requeue: false, RequeueAfter: 0}
ds := &operatorsv1alpha1.DatabaseService{}
if err := r.Client.Get(context.Background(), req.NamespacedName, ds); err != nil {
if misc.IsNotFound(err) {
return emptyResult, nil
})
return requeueResult, err
}
secret, err := getSecretFromThirdParty(ds)
if err != nil {
return requeueResult, err
}
if err := controllerutil.SetControllerReference(ds, secret, r.Scheme); err != nil {
logger.Error("failed to set controller reference for the secret", zap.Error(err))
return requeueResult, err
}
if err := r.createOrUpdateSecret(secret, logger); err != nil {
return requeueResult, err
}
return emptyResult, nil
}
func (r *DatabaseServiceReconciler) createOrUpdateSecret(secret *corev1.Secret) error {
if err := r.createNamespaceIfNotExist(secret.Namespace, logger); err != nil {
return err
}
if err := r.Client.Create(context.TODO(), secret); err == nil {
return nil
}
if !apierrors.IsAlreadyExists(err) {
return err
}
if err := r.Client.Update(context.TODO(), secret); err != nil {
return err
}
return nil
}
I am observing that if I set the CR as the owner of the secret before calling createOrUpdateSecret - Reconcile function will be triggered again, because something in the owned object (the secret) has changed.
My Reconcile logic is idempotent so it is not a big problem. However, I've no need for Reconcile to run again after changes to the owned object that took place from inside Reconcile. Right now, every time Reconcile creates / updates a secret it would run again. This behavior seems a bit clunky and results in extra work for the operator and extra calls to the third party.
Is there a way to bypass the re-activation of Reconcile creation / update of owned object from inside Reconcile? Or is it not recommended and I should allow reconcile run repeatedly until nothing is changed?
I got go-sql-driver for Azure CosmosDB from https://github.com/btnguyen2k/gocosmos.
It goes well when i call gocosmos.NewRestClient to get a rest client, CreateDatabase() to create database and CreateCollection() to create collection.
The problem is when i use CreateDocument(), i get response with statuscode 401 and body like this
{"code":"Unauthorized","message":"The input authorization token can't serve the request. Please check that the expected payload is built as per the protocol, and check the key being used. Server used the following payload to sign: 'post\ndocs\ndbs/ToDoList/colls/Items\nmon, 31 may 2021 13:31:44 gmt\n\n'\r\nActivityId: a9bbd729-3495-400f-9d79-ddec3737aa92, Microsoft.Azure.Documents.Common/2.11.0"}
i've tried all the solutions I've seen, but i haven't solved the problem.
I followed this tutorial and with this sample code, I can successfully create database, collection and document. Here's my testing result, can it help you?
// connects to MongoDB
func connect() *mongo.Client {
mongoDBConnectionString := os.Getenv(mongoDBConnectionStringEnvVarName)
if mongoDBConnectionString == "" {
log.Fatal("missing environment variable: ", mongoDBConnectionStringEnvVarName)
}
database = os.Getenv(mongoDBDatabaseEnvVarName)
if database == "" {
log.Fatal("missing environment variable: ", mongoDBDatabaseEnvVarName)
}
collection = os.Getenv(mongoDBCollectionEnvVarName)
if collection == "" {
log.Fatal("missing environment variable: ", mongoDBCollectionEnvVarName)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second*10)
defer cancel()
clientOptions := options.Client().ApplyURI(mongoDBConnectionString).SetDirect(true)
c, err := mongo.NewClient(clientOptions)
err = c.Connect(ctx)
if err != nil {
log.Fatalf("unable to initialize connection %v", err)
}
err = c.Ping(ctx, nil)
if err != nil {
log.Fatalf("unable to connect %v", err)
}
return c
}
// creates a todo
func create(desc string) {
c := connect()
ctx := context.Background()
defer c.Disconnect(ctx)
todoCollection := c.Database(database).Collection(collection)
r, err := todoCollection.InsertOne(ctx, Todo{Description: desc, Status: statusPending})
if err != nil {
log.Fatalf("failed to add todo %v", err)
}
fmt.Println("added todo", r.InsertedID)
}
I am beginner at Go, I had wrote small server to testing and deploy it on heroku platform. I have /logout request, which almost works, but sometimes I see something like this:
PANIC: write tcp 172.17.110.94:36641->10.11.189.195:9951: use of closed network connection
I don't know why it happens, and why sometimes it works perfectly.
My steps:
I send 1st POST request to /token-auth with body then generate token and send as response.
At 2nd I do /logout GET request with that token, and set token to Redis store
Here is full code of my redil_cli.go
package store
import (
"github.com/garyburd/redigo/redis"
)
type RedisCli struct {
conn redis.Conn
}
var instanceRedisCli *RedisCli = nil
func Connect() (conn *RedisCli) {
if instanceRedisCli == nil {
instanceRedisCli = new(RedisCli)
var err error
//this is works!!!
instanceRedisCli.conn, err = redis.Dial("tcp", "lab.redistogo.com:9951")
if err != nil {
panic(err)
}
if _, err := instanceRedisCli.conn.Do("AUTH", "password"); err != nil {
//instanceRedisCli.conn.Close()
panic(err)
}
}
return instanceRedisCli
}
func (redisCli *RedisCli) SetValue(key, value string, expiration ...interface{}) error {
_, err := redisCli.conn.Do("SET", key, value)
if err == nil && expiration != nil {
redisCli.conn.Do("EXPIRE", key, expiration[0])
}
return err
}
func (redisCli *RedisCli) GetValue(key string) (interface{}, error) {
data, err := redisCli.conn.Do("GET", key)
if err != nil{
panic(err)
}
return data, err
}
After that my function that checks Authorization header will panic while trying to do GetValue(key string) method
func (redisCli *RedisCli) GetValue(key string) (interface{}, error) {
data, err := redisCli.conn.Do("GET", key)
if err != nil{
panic(err)
}
return data, err
}
Can anyone point me, what I doing wrong?