Go program memory consumption keeps increasing on Windows Server 2016 - go

Question

This is my Go code. I am using version 1.13
Once the code starts running, the memory consumption continuously increases, never decreasing. I dont think I am creating new variables. I am only reusing global variables for all the storage.
I read online that the GC in Go should kick in every 2 minutes...?
EDIT: I am monitoring the memory consumption of the program on the Windows Task Manager. And the Memory starts at ~5MB, when the program starts to run and increases 0.3~0.5 MB every minute.
package main
import (
"context"
"database/sql"
"encoding/csv"
"fmt"
"log"
"os"
"path/filepath"
"runtime"
"runtime/debug"
"strings"
"time"
_ "github.com/denisenkom/go-mssqldb"
)
var path = os.Args[1]
var table = os.Args[4]
var backupPath = os.Args[2]
var location = os.Args[5]
var server = "localhost"
var port = 1433
var user = "###"
var password = "###"
var database = os.Args[3]
var ctx = context.Background()
var insertQuery = "INSERT INTO [" + database + "].[dbo].[" + table + "] ([meter_id],[time_stamp],[r_voltage],[y_voltage],[b_voltage],[ry_voltage],[yb_voltage],[br_voltage],[r_current],[y_current],[b_current],[kva],[kw],[kvar],[power_factor],[freq],[kwh],[run_hr],[updated_date]) VALUES"
var updateStatusQuery = "UPDATE EMS_Location_Status SET active='100',mailed='N',Updated_Date=getdate(),Error_desc='Data is coming from FTPS' where Location_Id ='" + location + "'"
var db *sql.DB
var connString = fmt.Sprintf("server=%s;user id=%s;password=%s;port=%d;database=%s;",
server, user, password, port, database)
var err error
var files []string
var tempInsertQuery string
var csvFile *os.File
var csvLines [][]string
var i int
var line []string
var point string
var processedFile []string
var fileName string
var iterFile string
var c = 10
func processFile(file string, table string) {
fmt.Println("Processing file...", file)
tempInsertQuery = insertQuery + " "
csvFile, err = os.Open(file)
if err != nil {
for err != nil {
fmt.Println("Encountered error in opening file.. trying again in 10 seconds.")
time.Sleep(10 * time.Second)
csvFile, err = os.Open(file)
}
fmt.Println(err)
}
csvLines, err = csv.NewReader(csvFile).ReadAll()
if err != nil {
fmt.Println(err)
}
for i, line = range csvLines {
if i == 0 {
continue
} else {
tempInsertQuery = tempInsertQuery + "("
for _, point = range line {
tempInsertQuery = tempInsertQuery + "'" + point + "'" + ","
}
tempInsertQuery = tempInsertQuery[0:len(tempInsertQuery)-1] + ", GETDATE()" + "), "
}
}
tempInsertQuery = tempInsertQuery[0 : len(tempInsertQuery)-2]
// Execute query
//fmt.Println(tempInsertQuery)
_, err = db.QueryContext(ctx, tempInsertQuery)
if err != nil {
fmt.Println(err)
return
}
_, err = db.QueryContext(ctx, updateStatusQuery)
if err != nil {
fmt.Println(err)
return
}
csvFile.Close()
fmt.Println("Done processing file ", file)
backupFile(file)
runtime.GC()
debug.FreeOSMemory()
}
func backupFile(file string) {
fmt.Println("Backing up...", file)
processedFile = strings.Split(file, "\\")
fileName = processedFile[len(processedFile)-1]
err = os.Rename(file, backupPath+fileName)
if err != nil {
log.Fatal(err)
}
}
func scanner(scanPath string, table string) {
err = filepath.Walk(scanPath, func(path string, info os.FileInfo, err error) error {
if strings.Contains(path, ".csv") {
files = append(files, path)
}
return nil
})
if err != nil {
panic(err)
}
for _, iterFile = range files {
time.Sleep(2 * time.Second)
processFile(iterFile, table)
}
files = files[:0]
}
func main() {
fmt.Println(connString)
// Create connection pool
db, err = sql.Open("sqlserver", connString)
if err != nil {
fmt.Println("Error creating connection pool: ", err.Error())
}
err = db.PingContext(ctx)
if err != nil {
fmt.Println(err.Error())
return
}
fmt.Printf("Connected to server!\n")
fmt.Printf("Initiating...\n")
for c > 0 {
scanner(path, table)
time.Sleep(1 * time.Second)
}
fmt.Println("Quitting..")
}
Any help would be appreciated! Thanks.

Answer 1

General concept in Go is whatever implements io.Closer (that has a Close() method), when you have such a value which you don't intend to use anymore, you should call its Close() method which in general frees resources which otherwise may not be freed immediately or not at all (ever).
Your db.QueryContext() call is sql.Conn.QueryContext() which returns an *sql.Rows value which has a Close() method. Do call that. Currently you're not even storing the returned sql.Rows.
Do it like this:
var rows *sql.Rows
rows, err = db.QueryContext(ctx, tempInsertQuery)
if err != nil {
fmt.Println(err)
return
}
if err2 := rows.Close(); err2 != nil {
fmt.Println("Error closing rows:", err2
}
Apply this at all uses of Conn.QueryContext().
Whenever makes sense, use defer to call such Close() methods, so it will be executed even if your code panics or have a return statement before calling that "manually".
Such example in your code is csvFile.Close(). There are returns before that, and if one of those branch is executed, the csvFile will not be closed.
Do it like this:
csvFile, err = os.Open(file)
if err != nil {
for err != nil {
fmt.Println("Encountered error in opening file.. trying again in 10 seconds.")
time.Sleep(10 * time.Second)
csvFile, err = os.Open(file)
}
fmt.Println(err)
}
defer func() {
if err2 := csvFile.Close(); err2 != nil {
fmt.Println("Error closing csvFile:", err2
}
}()