I'm trying this neat framework (gorm) but I'm facing some issues with a basic read query.
What I`m doing is basically what the documentation says about how to read from a SQL Server database.
I got a database with a table tests (plural from the struct name Test). There is a single entry in the database (check the image), and I cannot retrieve it...
var server = "localhost"
var port = 1433
var user = "orm"
var password = "123localtest"
var database = "go"
var db *sql.DB
type Test struct {
ID int
Name string
}
func main() {
connectionString := fmt.Sprintf("server=%s;user id=%s;password=%s;port=%d;database=%s",
server, user, password, port, database)
db, err := gorm.Open("mssql", connectionString)
if err != nil {
log.Fatal("Failed to create connection pool. Error: " + err.Error())
}
gorm.DefaultCallback.Create().Remove("mssql:set_identity_insert")
defer db.Close()
tableExists := db.HasTable(&Test{})
if tableExists {
ReadFirstTest(db)
}
}
// ReadFirstTest Gets and prints first Test from dbo.test
func ReadFirstTest(db *gorm.DB) {
var test = Test{}
db.First(&test)
fmt.Printf("%d %s \n", test.ID, test.Name)
db.Where("name = ?", "John").First(&test)
fmt.Printf("%d %s \n", test.ID, test.Name)
}
Can you tell me what is missing here ?
Database state
Related
I'm currently working on an api gateway that forwards requests to an grpc client. The routes use a middleware. Within that middleware the Validate function from the dialed grpc client is used. The function is in the last code snippet at the end.
The gateway is based on a gin router, which, as far as I understand handles each request in a separat go routine. I can send as many requests simultaneously to either to endpoint /protected/1 or to endpoint /protected/2 and they get handled correctly.
The Validate function is having problems handling request sent simultaneously to /protected/1 and /protected/2. Sending two request to both endpoints simultaneously results in that either both, none or only one of the requests is handled correctly. The error message that I receive is actually from the gorm function trying to query the user in the Validate function (ERROR: relation "users" does not exist (SQLSTATE 42P01)). I use a gorm database object that is connected to a postgres database. This behaviour suggests that some how a resource is shared. Using run -race does not give any insights.
So my questions are, why does this setup not work to send requests simultaneously to both endpoints?
I tried to make a working minimal example, but somehow I could nor reproduce the error in a minimal setting, hence I would like to share my code snippets here.
Important snippets of the the Api Gateway implementation
func main() {
c, err := config.LoadConfig()
...
r := gin.Default()
auth.RegisterRoutes(r, &c)
r.Run(":" + c.Port)
}
whereas the routes are defined as
func RegisterRoutes(r *gin.Engine, c *config.Config){
svc := &ServiceClient{
Client: InitServiceClient(c),
}
a := InitAuthMiddleware(svc)
// routes
protected := r.Group("/protected")
protected.Use(a.AuthRequired)
protected.GET("/1", svc.DoStuff)
protected.GET("/2", svc.DoStuff)
}
// dummy handler
func (svc *ServiceClient) DoStuff(ctx *gin.Context) {
handler.DoStuff(ctx, svc.Client)
}
The grpc client is dialed with the following function
type ServiceClient struct {
Client pb.AuthServiceClient
}
func InitServiceClient(c *config.Config) pb.AuthServiceClient {
// using WithInsecure() because no SSL running
cc, err := grpc.Dial(c.AuthSvcUrl, grpc.WithInsecure())
if err != nil {
fmt.Println("Could not connect:", err)
}
return pb.NewAuthServiceClient(cc)
}
and the middleware is implemented in the following way:
type AuthMiddlewareConfig struct {
svc *ServiceClient
}
func InitAuthMiddleware(svc *ServiceClient) AuthMiddlewareConfig {
return AuthMiddlewareConfig{svc}
}
func (c *AuthMiddlewareConfig) AuthRequired(ctx *gin.Context) {
access_token, _ := ctx.Cookie("access_token")
res, err := c.svc.Client.Validate(context.Background(), &pb.ValidateRequest{
Token: access_token,
TokenType: "ACCESS_TOKEN",
Url: ctx.Request.URL.String(),
})
if err != nil || res.Status != http.StatusOK {
ctx.AbortWithStatusJSON(http.StatusUnauthorized, gin.H{"status": http.StatusUnauthorized, "error": res.Error})
return
}
ctx.Set("userId", res.UserId)
ctx.Next()
}
Important snippets of the Auth Service:
func serve(c *conf.Configuration) {
/*
Dial returns:
type Repository struct {
DB *gorm.DB
}
*/
h := storage.Dial(&c.DB)
serviceUri := fmt.Sprintf(":%s", c.API.Port)
lis, err := net.Listen("tcp", serviceUri)
if err != nil {
logrus.Fatal("Failed to listen on: ", err)
}
s := api.Server{
R: h,
//
}
grpcServer := grpc.NewServer()
pb.RegisterAuthServiceServer(grpcServer, &s)
if err := grpcServer.Serve(lis); err != nil {
logrus.Fatalln("Failed to serve:", err)
}
}
and the validate function is implemented by
type Server struct {
R storage.Repository
//
// #https://github.com/grpc/grpc-go/issues/3794:
pb.UnimplementedAuthServiceServer
}
func (s *Server) Validate(ctx context.Context, req *pb.ValidateRequest) (*pb.ValidateResponse, error) {
// token validation stuff works
var user models.User
// causes error sometimes
if result := s.R.DB.Where(&models.User{Id: claims.Id}).First(&user); result.Error != nil {
return &pb.ValidateResponse{
Status: http.StatusNotFound,
Error: "User not found",
}, nil
}
// send response
}
EDIT
For completeness here the function that connects to the postgres client.
func openDb(c *conf.PostgresConfiguration, gormConfig *gorm.Config, database string) *gorm.DB {
connectionString := fmt.Sprintf("postgres://%s:%s#%s:%s/%s", c.User, c.Password, c.Host, c.Port, database)
db, err := gorm.Open(postgres.Open(connectionString), gormConfig)
if err != nil {
logrus.Fatal(fmt.Sprintf("Unable to connect to database '%s' given url: ", database), err)
}
logrus.Info(fmt.Sprintf("Connected to database %s", database))
return db
}
func Dial(c *conf.PostgresConfiguration) Repository {
gormDefaultConfig := &gorm.Config{}
gormAppConfig := &gorm.Config{}
// connect to default database
logrus.Info(fmt.Sprintf("Use default database %s for initialization", c.DefaultDatabase))
defaultDb := openDb(c, gormDefaultConfig, c.DefaultDatabase)
// check if database exists and create it if necessary
var dbexists bool
dbSQL := fmt.Sprintf("SELECT EXISTS (SELECT FROM pg_database WHERE datname = '%s') AS dbexists;", c.AppDatabase)
defaultDb.Raw(dbSQL).Row().Scan(&dbexists)
if !dbexists {
logrus.Info(fmt.Sprintf("Created database %s", c.AppDatabase))
db := defaultDb.Exec(fmt.Sprintf("CREATE DATABASE %s;", c.AppDatabase))
if db.Error != nil {
logrus.Fatal("Unable to create app database: ", db.Error)
}
}
// connect to app databse
appDb := openDb(c, gormAppConfig, c.AppDatabase)
// check if schema exists and create it if necessary
var schemaexists bool
schemaSQL := fmt.Sprintf("SELECT EXISTS(SELECT FROM pg_namespace WHERE nspname = '%s') AS schemaexisits;", c.AppDatabaseSchema)
appDb.Raw(schemaSQL).Row().Scan(&schemaexists)
// create app specfic database, if not already existing
if !schemaexists {
// create service specific schema
db := appDb.Exec(fmt.Sprintf("CREATE SCHEMA %s;", c.AppDatabaseSchema))
if db.Error != nil {
logrus.Fatal(fmt.Sprintf("Unable to create database schema %s", c.AppDatabaseSchema), db.Error)
}
logrus.Info(fmt.Sprintf("Created database schema %s", c.AppDatabaseSchema))
}
db := appDb.Exec(fmt.Sprintf(`set search_path='%s';`, c.AppDatabaseSchema))
logrus.Info(fmt.Sprintf("Use existing database schema %s", c.AppDatabaseSchema))
if db.Error != nil {
logrus.Fatal(fmt.Sprintf("Unable to set search_path for database to schema %s", c.AppDatabaseSchema), db.Error)
}
// migrate table
appDb.AutoMigrate(&models.User{})
return Repository{appDb}
}
Edit2
Changing the connection string in the openDb function to connectionString := fmt.Sprintf("postgres://%s:%s#%s:%s/%s?search_path=%s", c.User, c.Password, c.Host, c.Port, database, c.AppDatabaseSchema) fixes the issue (see how the search_path is part of the url).
Thank you!
Recently I tried to document my code, but I had some trouble using godoc because there's some function that didn't came up when I ran godoc -http:localhost:6060
This is what my code looks like:
type MongoDBInterface interface {
ExecuteTransaction(operation func(mongoClient MongoDBInterface) error) error
Count(tableName string, clause bson.M) (int, error)
Distinct(tableName, fieldName string, clause bson.M) ([]interface{}, error)
InsertOrUpdate(tableName string, clause bson.M, data models.BaseModelInterface) (primitive.ObjectID, error)
InsertOrUpdateFields(tableName string, clause bson.M, data interface{}) (primitive.ObjectID, error)
Insert(tableName string, data models.BaseModelInterface) (primitive.ObjectID, error)
Update(tableName string, clause bson.M, data models.BaseModelInterface) error
UpdateFields(tableName string, clause bson.M, data interface{}) error
FindOne(tableName string, clause, opt bson.M, result interface{}) error
FindMany(tableName string, clause, opt bson.M, result interface{}) error
Truncate(tableName string) error
Delete(tableName string, clause bson.M) error
Aggregate(tableName string, pipelines interface{}, result interface{}) error
EnsureCollections() error
}
type mongoDB struct {
session mongo.Session
db *mongo.Database
ctx context.Context
isTransactionEnabled bool
isConnected bool
connString string
}
// NewMongoDB definition
func NewMongoDB() MongoDBInterface {
mongoClient := new(mongoDB)
mongoClient.ctx = context.Background()
dbHost := os.Getenv("DB_HOST")
if dbHost == "" {
dbHost = "localhost"
}
dbUser := os.Getenv("DB_USERNAME")
if dbUser == "" {
dbUser = "dbadmin"
}
dbPswd := os.Getenv("DB_PASSWORD")
if dbPswd == "" {
dbPswd = "dbpassword"
}
dbName := os.Getenv("DB_NAME")
if dbName == "" {
dbName = "dbname"
}
dbAuth := os.Getenv("DB_AUTH")
if dbAuth == "" {
dbAuth = "admin"
}
dbMode := os.Getenv("DB_MODE")
if dbMode == "" {
dbMode = "admin"
}
// temporary
connString := fmt.Sprintf("mongodb+srv://%s:%s#%s/%s?retryWrites=true&w=majority", dbUser, dbPswd, dbHost, dbName)
mongoClient.connString = connString
return mongoClient
}
// connect to mongodb server
func (s *mongoDB) connect() error {
// get query string from env,
// then parse it to get db name
// connString := os.Getenv("MONGODB_CONN_STRING")
connString := s.connString
log.Println("ConnString =>", connString)
parts := strings.Split(connString, "/")
dbName := strings.Split(parts[len(parts)-1], "?")[0]
// prepare options object for connecting to mongodb
opt := options.Client()
opt.ApplyURI(connString)
// set the timeout info from data defined in the env
// timeout, _ := strconv.Atoi(os.Getenv("MONGODB_TIMEOUT_IN_SECOND"))
timeout := 120
timeoutDuration := time.Duration(timeout) * time.Second
opt.ConnectTimeout = &timeoutDuration
// create client object
client, err := mongo.NewClient(opt)
if err != nil {
log.Println(err.Error())
return err
}
// connect to the db server
err = client.Connect(context.Background())
if err != nil {
log.Println(err.Error())
return err
}
// start new session
session, err := client.StartSession()
if err != nil {
log.Println(err.Error())
return err
}
// store session and db info into props
s.session = session
s.db = client.Database(dbName)
s.isConnected = true
log.Println("Connected to database")
return nil
}
The problem is godoc will never render func (s *mongoDB) connect() error but I need it to be documented, can you guys explain to me what's going on with godoc? Or maybe you can give me some solutions and tips for documenting Go code.
You can refer this doc: https://pkg.go.dev/golang.org/x/tools/cmd/godoc
The presentation mode of web pages served by godoc can be controlled with the "m" URL parameter; it accepts a comma-separated list of flag names as value:
- all show documentation for all declarations, not just the exported ones
- methods show all embedded methods, not just those of unexported anonymous fields
- src show the original source code rather than the extracted documentation
- flat present flat (not indented) directory listings using full paths
For instance, https://golang.org/pkg/math/big/?m=all shows the documentation for all (not just the exported) declarations of package big.
?m=all documents all declaration including the non-exported methods
Given the following data structure which has been created in the database and there is valid data in the rows in the appropriate tables:-
type Deployment struct {
gorm.Model
Name string `gorm:"unique_index:idx_name"`
RestAPIUser string
RestAPIPass string
Servers []Server
model *Model
}
type Server struct {
gorm.Model
DeploymentID uint
Hostname string `gorm:"unique_index:idx_hostname"`
RestPort string
Version string
}
I'm trying to select all Deployments and have GORM automatically fill the Servers for each Deployment.
Unfortunately, it doesn't do this. I've tried several variations of using the Associations() func but I can't seem to get it to work. I seem to have to do this manually:-
func (m *Model) GetDeployments() ([]Deployment, error) {
deployments := []Deployment{}
err := m.db.Find(&deployments).Error
if err != nil {
return nil, err
}
deploymentsWithServers := []Deployment{}
for _, d := range deployments {
servers := []Server{}
err := m.db.Model(&d).Association("Servers").Find(&servers).Error
if err != nil {
return nil, err
}
d.Servers = servers
deploymentsWithServers = append(deploymentsWithServers, d)
}
return deploymentsWithServers, nil
}
Does anyone have any suggestions how I can get GORM to fill the Servers field automatically? Thanks!
Try
m.db.Preload("Servers").Find(&Deployment{})
I am attempting to create a REST API in Go. I have it partially working in that it will return 4 separate json objects like such:
[{"Name":"QA1","Server":"BOT1","Description":"Tools","Apps":""},
{"Name":"QA1","Server":"","Description":"","Apps":"Duo"},
{"Name":"QA1","Server":"","Description":"","Apps":"Git"},
{"Name":"QA1","Server":"","Description":"","Apps":"php"}]
What I want is a single returned object like:
[{"Name":"QA1","Server":"BOT1","Description":"Tools","Apps": "Duo|Git|php"}]
I obviously have the way that I am either making my queries or the structs (or both or something else) not quite correct. I want to make sure I understand how to do this right because I would like to expand on it for other queries and such down the road. I have included the "full" go code below.
To be clear, I'm not simply looking for the solution (though I would of course appreciate that to compare with), but where I've gone wrong in my thinking and what the correct approach would be.
package main
import (
"database/sql"
"encoding/json"
"fmt"
_ "github.com/go-sql-driver/mysql"
"io/ioutil"
"log"
"net/http"
)
// There can be zero or more apps on a volume
type Apps struct {
Name string
}
// Volumes have a name, description, are on a server and have multiple services/apps
type Volume struct {
Name string
Server string
Description string
Services Apps
}
//Handle all requests
func Handler(response http.ResponseWriter, request *http.Request) {
response.Header().Set("Content-type", "text/html")
webpage, err := ioutil.ReadFile("index.html")
if err != nil {
http.Error(response, fmt.Sprintf("home.html file error %v", err), 500)
}
fmt.Fprint(response, string(webpage))
}
// DB Connection
const (
DB_HOST = "mydbhost"
DB_NAME = "mydb"
DB_USER = "mydbuser"
DB_PASS = "mydbpass"
)
// Respond to URLs of the form /api
func APIHandler(response http.ResponseWriter, request *http.Request) {
//Connect to database
dsn := DB_USER + ":" + DB_PASS + "#" + DB_HOST + "/" + DB_NAME + "?charset=utf8"
db, err := sql.Open("mysql", dsn)
if err != nil {
fmt.Println(err.Error())
}
defer db.Close()
// Open doesn't open a connection. Validate DSN data:
err = db.Ping()
if err != nil {
fmt.Println(err.Error())
}
//set mime type to JSON
response.Header().Set("Content-type", "application/json")
result := []*Volume{}
switch request.Method {
case "GET":
srvrnm := request.URL.Query().Get("srvrnm")
appnm := request.URL.Query().Get("appnm")
srvrs, err := db.Prepare("select VOLUMES.name as volnm, SERVERS.name as srvrnm, VOLUMES.description as descr From VOLUMES LEFT JOIN SERVERS ON VOLUMES.server_id = SERVERS.id where SERVERS.name = ?")
if err != nil {
fmt.Print(err)
}
srvcs, err := db.Prepare("select VOLUMES.name as volnm, SUPPRTSVCS.name as app_name From VOLUMES as VOLUMES JOIN HOSTSVCS ON VOLUMES.id = HOSTSVCS.volume_id JOIN SUPPRTSVCS ON SUPPRTSVCS.id = HOSTSVCS.supportsvcs_id where VOLUMES.name = ?")
if err != nil {
fmt.Print(err)
}
// Run the SQL Query to Get Volum & Description From Hostname
srvrrows, err := srvrs.Query(srvrnm)
if err != nil {
fmt.Print(err)
}
for srvrrows.Next() {
var volnm string
var srvrnm string
var descr string
// Scan the First Query
err = srvrrows.Scan(&volnm, &srvrnm, &descr)
if err != nil {
fmt.Println("Error scanning: " + err.Error())
return
}
// Append Slice with results from the scan
result = append(result, &Volume{Name: volnm, Server: srvrnm, Description: descr})
}
// Run the SQL Query for Services/Apps
srvcrows, err := srvcs.Query(appnm)
if err != nil {
fmt.Print(err)
}
for srvcrows.Next() {
var volnm string
var appnm string
// Scan the Second Query
err = srvcrows.Scan(&volnm, &appnm)
if err != nil {
fmt.Println("Error scanning: " + err.Error())
return
}
// Append Slice with results from the scan
result = append(result, &Volume{Name: volnm, Apps: appnm})
}
default:
}
json, err := json.Marshal(result)
if err != nil {
fmt.Println(err)
return
}
fmt.Fprintf(response, string(json))
db.Close()
}
func main() {
port := "1236"
var err string
mux := http.NewServeMux()
mux.Handle("/api", http.HandlerFunc(APIHandler))
mux.Handle("/", http.HandlerFunc(Handler))
// Start listing on a given port with these routes on this server.
log.Print("Listening on port " + port + " ... ")
errs := http.ListenAndServe(":"+port, mux)
if errs != nil {
log.Fatal("ListenAndServe error: ", err)
}
}
From the sounds of it, you want to your result to look like:
[
{"Name":"QA1","Server":"BOT1","Description":"Tools","Apps": ["Duo","Git","php"]
]
Hence you want your Volumes struct to look like:
type Volume struct {
Name string
Server string
Description string
Services []Apps
}
If you want the Apps to actually output Duo|Git|php then you could create a custom type instead of []Apps with a JSON Marshaler implementation. This could simply return json.Marshal(strings.join(names,"|"))
Rather than run two separate queries, it would be more efficient to run a single query that selects the product of volumes & apps together. It is important that this query is sorted by volume so all volume rows are contiguous. Example query output would be:
Name | Server | Desc | App
---- | ------ | ----- | ---
Vol1 | Srv1 | Desc1 | App1
Vol1 | Srv1 | Desc1 | App2
Vol2 | Srv2 | Desc2 | App3
You would then loop over this and detect if you are looking at a new volume. If so, create a new entry in the result. If not, add the App to the list of apps. For example:
var (
volnm string
srvrnm string
descr string
appnm string
v *Volume
result []*Volume
)
for srvrrows.Next() {
if err = srvcrows.Scan(&volnm, &srvrnm, &descr, &appnm);err!=nil {
// Handle error
}
// Add App to current volume if same, otherwise start a new volume
if v!=nil && v.Name == volnm {
v.Services = append(v.Services,Apps{appnm})
} else {
v = &Volume{
Name: volnm,
Server: svrnm,
Description: descr,
Services: []Apps{appnm}}
result = append(result,v)
}
}
// Finished, return result etc...
When taking this approach, you need an appropriate parent record discriminator. I'd just used v.Name == volnm for illustration purposes but this should really be checking the primary key. You can make this an unexported (lowercase) field in the struct if you do not wish to export it through the API.
What I'm trying to accomplish is sharing a pointer of db.sqlx between multiple functions, except for posts saying pass along the pointer, which is fine but how to do that in an interface? I cannot find anything that illustrates the use of this anywhere. Basically what I have is an interface of type Datastore. I also have mysql & pgsql that implements the Datastore type. The interface by itself works fine however the issue is I'm trying to create a single connect function for *sqlx.DB to be shared across all functions within the implemented interface. I think the issue is I've confused myself on how to share the pointer between functions of the interface or even "where" to share it. The main interface looks like below:
var (
storage Datastore
db * sqlx.DB
)
type Datastore interface {
Insert(db *sqlx.DB, table string, item DataItem) bool
CheckEmpty(db *sqlx.DB, table string) bool
FetchAll(db *sqlx.DB, table string) []DataItem
DBInit(db *sqlx.DB)
initDB()
}
Within my implemented interface (simplified mysql example) I have the initDB function which looks like this:
type MySQLDB struct {
config *config.Configuration
}
func (my *MySQLDB) initDB() {
log.Println("Getting DB Connection")
tempdb, err := sqlx.Connect("mysql", my.config.Database.Dsn+"&parseTime=True")
if err != nil {
log.Println(err.Error())
}
db = tempdb
defer db.Close()
}
func (my *MySQLDB) FetchAll(db *sqlx.DB, table string) []DataItem {
dTable := []DataItem{}
query := "SELECT foo, bar FROM " + table + " ORDER BY last_update ASC"
err := db.Select(&dTable, query)
if err != nil{
panic(err)
}
return dTable
}
At this point I know the connection is initially opened but the next time a function is called I get db is closed error. So how do I properly share the db connection between functions, or do I really have to run a connection open in every function?
Don't call defer db.Close() in your initDB function. After that function executed, db will close too! So when you call your method you get the closed error.
Maybe you need to re-desgin your interface, for example:
type Datastore interface {
Insert(table string, item DataItem) bool
CheckEmpty(table string) bool
FetchAll(table string) []DataItem
Close() error // call this method when you want to close the connection
initDB()
}
Your MySQLDB implement now look like:
type MySQLDB struct {
config *config.Configuration
db *sqlx.DB
}
func (my *MySQLDB) initDB() {
log.Println("Getting DB Connection")
tempdb, err := sqlx.Connect("mysql", my.config.Database.Dsn+"&parseTime=True")
if err != nil {
log.Println(err.Error())
}
my.db = tempdb
}
func (my *MySQLDB) Close() error {
return my.db.Close()
}
func (my *MySQLDB) FetchAll(table string) []DataItem {
dTable := []DataItem{}
query := "SELECT foo, bar FROM " + table + " ORDER BY last_update ASC"
err := my.db.Select(&dTable, query)
if err != nil{
panic(err)
}
return dTable
}