Golang function generalization for different packages - go

Question

Imagine these functions needs to be used how can I make this calls generic so that I don't repeat almost the same code.
with "encoding/csv"
func getDataFromCSVFiles(files []string) (error, Data) {
data := Data{}
for _, file := range files {
f, err := os.Open(file)
if err != nil {
panic(err)
return err, data
}
defer f.Close()
r := charmap.ISO8859_1.NewDecoder().Reader(f)
reader := csv.NewReader(r)
for i := 1;;i++ {
rec, err := reader.Read()
if i == 1 {
//Skipping header
continue
}
if err != nil {
if err == io.EOF {
break
}
//TODO log error line and csv filename
log.Fatal(err)
}
addWorkbook(rec, &data)
}
}
return nil, data
}
and with
import fw "github.com/hduplooy/gofixedwidth" which is almost the same except calling fw.NewReader
func getDataFromPRNFiles(files []string) (error, Data) {
data := Data{}
for _, file := range files {
f, err := os.Open(file)
if err != nil {
panic(err)
return err, data
}
defer f.Close()
r := charmap.ISO8859_1.NewDecoder().Reader(f)
reader := fw.NewReader(r)
for i := 1;;i++ {
rec, err := reader.Read()
if i == 1 {
//Skipping header
continue
}
if err != nil {
if err == io.EOF {
break
}
//TODO log error line and csv filename
log.Fatal(err)
}
addWorkbook(rec, &data)
}
}
return nil, data
}

Answer 1

The only apparent difference is:
reader := csv.NewReader(r)
versus:
reader := fw.NewReader(r)
I'm not sure what fw is but presumably both readers implement a common interface:
type StringSliceReader interface {
Read() ([]string, error)
}
So you could pass the openers (csv.NewReader and fw.NewReader) as function arguments:
func getDataFromFiles(files []string, func(r io.Reader) StringArrayReader) (error, Data) {
//...
}
but you'd need to wrap them in little functions to get around the return types:
func newCSVReader(r io.Reader) StringSliceReader {
return csv.NewReader(r)
}
func newFWReader(r io.Reader) StringSliceReader {
return fw.NewReader(r)
}
Also, defer queues up things to execute when the function exits, not on the next iteration of a loop. So if you do this:
for _, file := range files {
f, err := os.Open(file)
if err != nil {
panic(err)
return err, data
}
defer f.Close()
//...
}
and files has a hundred entries then you'll have a hundred open files before any of them are closed. You probably want to move that loop body to a separate function so that you only have one file open at a time.
Furthermore, error is usually the last return value from a function so you should return data, err to be more idiomatic.
The result could look something like this:
type StringSliceReader interface {
Read() ([]string, error)
}
type NewReader func(r io.Reader) StringSliceReader
func newCSVReader(r io.Reader) StringSliceReader {
return csv.NewReader(r)
}
func newFWReader(r io.Reader) StringSliceReader {
return fw.NewReader(r)
}
func getDataFrom(file string, data *Data, newReader NewReader) error {
f, err := os.Open(file)
if err != nil {
return err
}
defer f.Close()
r := charmap.ISO8859_1.NewDecoder().Reader(f)
reader := newReader(r)
for i := 1; ; i++ {
rec, err := reader.Read()
if i == 1 {
continue
}
if err != nil {
if err == io.EOF {
break
}
log.Fatal(err)
}
addWorkbook(rec, data)
}
return nil
}
func getDataFromFiles(files []string, newReader NewReader) (Data, error) {
data := Data{}
for _, file := range files {
err := getDataFrom(file, newReader, &data)
if err != nil {
panic(err)
return data, err
}
}
return data, nil
}
and you could say getDataFromFiles(files, newCSVReader) to read CSVs or getDataFromFiles(files, newFWReader) to read FW files. If you want to read from something else, you'd just need a NewReader function and something that implements the StringSliceReader interface.
You might want to bury/hide the charmap.ISO8859_1.NewDecoder().Reader(f) stuff inside the NewReader functions to make it easier to read non-Latin-1 encoded files. You could also replace newReader NewReader with a map[string]NewReader in getDataFromFiles and choose the NewReader to use based on the file's extension or other format identifier.