binary-to-text encoding that replace a byte into a predefined unique string in golang - go

I am trying to build a binary-to-text encoder with the ability to replace every byte into a predefined unique string and then use the same string set to decode back into binary.
I am able to make the encoder and decoder for simple .txt files but I want to make this workable for .zip files too.
Encoder :
package main
import (
"archive/zip"
"bufio"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"strings"
)
func main() {
// keys.json is a 256 words dictionary for every byte
keysFile, err := os.Open("keys.json")
if err != nil {
log.Printf("unable to read keys.json file , error : %v", err)
return
}
var keys []string
byteValue, _ := ioutil.ReadAll(keysFile)
if err := json.Unmarshal(byteValue, &keys); err != nil {
log.Printf("unable to unmarshal array , error : %v", err)
return
}
Encoder(keys)
}
func listFiles(file *zip.File) ([]byte, error) {
fileToRead, err := file.Open()
if err != nil {
msg := "Failed to open zip %s for reading: %s"
return nil, fmt.Errorf(msg, file.Name, err)
}
b := make([]byte, file.FileInfo().Size())
fileToRead.Read(b)
defer fileToRead.Close()
if err != nil {
return nil, fmt.Errorf("failed to read zip file : %s for reading , error : %s", file.Name, err)
}
return b, nil
}
func Encoder(keys []string) {
read, err := zip.OpenReader("some.zip")
if err != nil {
msg := "Failed to open: %s"
log.Fatalf(msg, err)
}
defer read.Close()
var encodedBytes []byte
f, err := os.Create("result.txt")
w := bufio.NewWriter(f)
defer f.Close()
for _, file := range read.File {
readBytes, err := listFiles(file)
if err != nil {
log.Fatalf("Failed to read %s from zip: %s", file.Name, err)
continue
}
for i, b := range readBytes {
for _, eb := range []byte(keys[b] + " ") {
encodedBytes = append(encodedBytes, eb)
}
}
}
_, err = w.Write(encodedBytes)
if err != nil {
log.Printf("error :%v", err)
return
}
}
Decoder :
func Decoder(keys []string) {
inputFile, err := os.Open("result.txt")
if err != nil {
log.Printf("unable to read file , error : %v", err)
return
}
inputBytes, _ := ioutil.ReadAll(inputFile)
var (
current []byte
decoded []byte
)
for _, c := range inputBytes {
if c != 32 {
current = append(current, c)
} else {
for i, key := range keys {
if string(current) == key {
decoded = append(decoded, byte(i))
break
}
}
current = []byte{}
}
}
// here i want the decoded back into zip file
}
here is a similar one in nodejs.

Two things:
You are dealing with spaces correctly, but not with newlines.
Your decoder loop is wrong. As far as I can tell, it should look like the following:
for i, key := range keys {
if string(current)==key {
decoded=append(decoded,i)
break
}
}
Also, your decoded is an int-array, not a byte-array.

Related

Golang multipart file form request

I'm writing an API client against Mapbox, uploading a batch of svg images to a custom map. The api they provide for this is documented with an example cUrl call that works fine:
curl -F images=#include/mapbox/sprites_dark/aubergine_selected.svg "https://api.mapbox.com/styles/v1/<my_company>/<my_style_id>/sprite?access_token=$MAPBOX_API_KEY" --trace-ascii /dev/stdout
When attemting to do the same from golang I quickly came across that the multiform library is very limited, and wrote some code to make the request as similar to the cUrl request mentioned above.
func createMultipartFormData(fileMap map[string]string) (bytes.Buffer, *multipart.Writer) {
var b bytes.Buffer
var err error
w := multipart.NewWriter(&b)
var fw io.Writer
for fileName, filePath := range fileMap {
h := make(textproto.MIMEHeader)
h.Set("Content-Disposition",
fmt.Sprintf(`form-data; name="%s"; filename="%s"`, "images", fileName))
h.Set("Content-Type", "image/svg+xml")
if fw, err = w.CreatePart(h); err != nil {
fmt.Printf("Error creating form File %v, %v", fileName, err)
continue
}
fileContents, err := ioutil.ReadFile(filePath)
fileContents = bytes.ReplaceAll(fileContents, []byte("\n"), []byte("."))
blockSize := 64
remainder := len(fileContents) % blockSize
iterations := (len(fileContents) - remainder) / blockSize
newBytes := []byte{}
for i := 0; i < iterations; i++ {
start := i * blockSize
end := i*blockSize + blockSize
newBytes = append(newBytes, fileContents[start:end]...)
newBytes = append(newBytes, []byte("\n")...)
}
if remainder > 0 {
newBytes = append(newBytes, fileContents[iterations*blockSize:]...)
newBytes = append(newBytes, []byte("\n")...)
}
if err != nil {
fmt.Printf("Error reading svg file: %v: %v", filePath, err)
continue
}
_, err = fw.Write(newBytes)
if err != nil {
log.Debugf("Could not write file to multipart: %v, %v", fileName, err)
continue
}
}
w.Close()
return b, w
}
Along with setting the headers in the actual request:
bytes, formWriter := createMultipartFormData(filesMap)
req, err := http.NewRequest("Post", fmt.Sprintf("https://api.mapbox.com/styles/v1/%v/%v/sprite?access_token=%v", "my_company", styleID, os.Getenv("MAPBOX_API_KEY")), &bytes)
if err != nil {
return err
}
req.Header.Set("User-Agent", "curl/7.64.1")
req.Header.Set("Accept", "*/*")
req.Header.Set("Content-Length", fmt.Sprintf("%v", len(bytes.Bytes())))
req.Header.Set("Content-Type", formWriter.FormDataContentType())
byts, _ := httputil.DumpRequest(req, true)
fmt.Println(string(byts))
res, err := http.DefaultClient.Do(req)
Even want as far to limit the line length and replicate the encoding used by cUrl but so far no luck. Does anyone with experience know why this works from cUrl but not golang?
Well, I admit that all the parts of the "puzzle" to solve your task can be found on the 'net in abundance, there are two problems with this:
They quite often miss certain interesting details.
Sometimes, they give outright incorrect advice.
So, here's a working solution.
package main
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"mime"
"mime/multipart"
"net/http"
"net/textproto"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
)
func main() {
const (
dst = "https://api.mapbox.com/styles/v1/AcmeInc/Style_001/sprite"
fname = "path/to/a/sprite/image.svg"
token = "an_invalid_token"
)
err := post(dst, fname, token)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
func post(dst, fname, token string) error {
u, err := url.Parse(dst)
if err != nil {
return fmt.Errorf("failed to parse destination url: %w", err)
}
form, err := makeRequestBody(fname)
if err != nil {
return fmt.Errorf("failed to prepare request body: %w", err)
}
q := u.Query()
q.Set("access_token", token)
u.RawQuery = q.Encode()
hdr := make(http.Header)
hdr.Set("Content-Type", form.contentType)
req := http.Request{
Method: "POST",
URL: u,
Header: hdr,
Body: ioutil.NopCloser(form.body),
ContentLength: int64(form.contentLen),
}
resp, err := http.DefaultClient.Do(&req)
if err != nil {
return fmt.Errorf("failed to perform http request: %w", err)
}
defer resp.Body.Close()
_, _ = io.Copy(os.Stdout, resp.Body)
return nil
}
type form struct {
body *bytes.Buffer
contentType string
contentLen int
}
func makeRequestBody(fname string) (form, error) {
ct, err := getImageContentType(fname)
if err != nil {
return form{}, fmt.Errorf(
`failed to get content type for image file "%s": %w`,
fname, err)
}
fd, err := os.Open(fname)
if err != nil {
return form{}, fmt.Errorf("failed to open file to upload: %w", err)
}
defer fd.Close()
stat, err := fd.Stat()
if err != nil {
return form{}, fmt.Errorf("failed to query file info: %w", err)
}
hdr := make(textproto.MIMEHeader)
cd := mime.FormatMediaType("form-data", map[string]string{
"name": "images",
"filename": fname,
})
hdr.Set("Content-Disposition", cd)
hdr.Set("Contnt-Type", ct)
hdr.Set("Content-Length", strconv.FormatInt(stat.Size(), 10))
var buf bytes.Buffer
mw := multipart.NewWriter(&buf)
part, err := mw.CreatePart(hdr)
if err != nil {
return form{}, fmt.Errorf("failed to create new form part: %w", err)
}
n, err := io.Copy(part, fd)
if err != nil {
return form{}, fmt.Errorf("failed to write form part: %w", err)
}
if int64(n) != stat.Size() {
return form{}, fmt.Errorf("file size changed while writing: %s", fd.Name())
}
err = mw.Close()
if err != nil {
return form{}, fmt.Errorf("failed to prepare form: %w", err)
}
return form{
body: &buf,
contentType: mw.FormDataContentType(),
contentLen: buf.Len(),
}, nil
}
var imageContentTypes = map[string]string{
"png": "image/png",
"jpg": "image/jpeg",
"jpeg": "image/jpeg",
"svg": "image/svg+xml",
}
func getImageContentType(fname string) (string, error) {
ext := filepath.Ext(fname)
if ext == "" {
return "", fmt.Errorf("file name has no extension: %s", fname)
}
ext = strings.ToLower(ext[1:])
ct, found := imageContentTypes[ext]
if !found {
return "", fmt.Errorf("unknown file name extension: %s", ext)
}
return ct, nil
}
Some random notes on implementation to help you understands the concepts:
To construct the request's payload (body), we use a bytes.Buffer instance.
It has a nice property in that a pointer to it (*bytes.Buffer) implements both io.Writer and io.Reader and hence can be easily composed with other parts of the Go stdlib which deal with I/O.
When preparing a multipart form for sending, we do not slurp the whole file's contents into memory but instead "pipe" them right into the "mutipart form writer".
We have a lookup table which maps the extension of a file name to submit to its MIME type; I have no idea whether this is needed by the API or not; if it's not really required, the part of the code which prepares a form's field containing a file could be simplified a lot, but cURL send it, so do we.
Just to be curious, What is this for?
fileContents = bytes.ReplaceAll(fileContents, []byte("\n"), []byte("."))
blockSize := 64
remainder := len(fileContents) % blockSize
iterations := (len(fileContents) - remainder) / blockSize
newBytes := []byte{}
for i := 0; i < iterations; i++ {
start := i * blockSize
end := i*blockSize + blockSize
newBytes = append(newBytes, fileContents[start:end]...)
newBytes = append(newBytes, []byte("\n")...)
}
if remainder > 0 {
newBytes = append(newBytes, fileContents[iterations*blockSize:]...)
newBytes = append(newBytes, []byte("\n")...)
}
if err != nil {
fmt.Printf("Error reading svg file: %v: %v", filePath, err)
continue
}
Reading a entire file into memory is rarely a good idea (ioutil.ReadFile).
As #muffin-top says, how about those three lines of code?
for fileName, filePath := range fileMap {
// h := ...
fw, _ := w.CreatePart(h) // TODO: handle error
f, _ := os.Open(filePath) // TODO: handle error
io.Copy(fw, f) // TODO: handle error
f.Close() // TODO: handle error
}

Making a filename truncator in go

I am new to Go, and I've been trying to make a program that truncates filenames after (and including) the character "-" of all files in its directory. It's not working, and I don't know where I'm going wrong.
func changeFilename() {
file, err := os.Open(".")
if err != nil {
log.Fatalf("failed opening directory: %s", err)
}
defer file.Close()
oldNames, _ := file.Readdirnames(0)
var s string
for _, i := range oldNames {
for _, j := range i {
for j != '-' {
k := strconv.QuoteRune(j)
s += k
j++
}
}
err := os.Rename(i, s)
if err != nil {
log.Fatal("failed to rename: %s", err)
}
}
}
I expected filenames in the test directory to be changed when I ran the executable, but they didn't. The program builds without any error messages.
Fixing your logic to correspond to your description of the problem:
package main
import (
"fmt"
"log"
"os"
"path/filepath"
)
// changeFilename truncates filenames after (and including) the character "-"
// of all files in the directory. File extensions are preserved.
func changeFilename(dir string) {
file, err := os.Open(dir)
if err != nil {
log.Fatalf("failed opening directory: %s", err)
}
defer file.Close()
fis, err := file.Readdir(0)
if err != nil {
log.Fatalf("failed reading directory: %s", err)
}
for _, fi := range fis {
if !fi.Mode().IsRegular() {
continue
}
old := fi.Name()
ext := filepath.Ext(old)
for i, r := range old[:len(old)-len(ext)] {
if r == '-' {
new := old[:i] + ext
err := os.Rename(old, new)
if err != nil {
log.Fatalf("failed to rename: %s", err)
}
fmt.Printf("%q %q\n", old, new)
break
}
}
}
}
func main() {
changeFilename(".")
}
Output:
"trunc-ate.typ" "trunc.typ"
"hyp-hen.ext" "hyp.ext"
Your s variable should be moved into the outermost loop since its lifetime should be tied to a single oldName. The innermost loop has no exit condition once a non - character is encountered. That should be an if statement with a corresponding break if a - is found. As for building up the new filename, you ought to use += string(j) instead of quoting the rune.
func changeFilename() {
file, err := os.Open(".")
if err != nil {
log.Fatalf("failed opening directory: %s", err)
}
defer file.Close()
oldNames, _ := file.Readdirnames(0)
for _, i := range oldNames {
var s string
for _, j := range i {
if j != '-' {
s += string(j)
} else {
break
}
}
err := os.Rename(i, s)
if err != nil {
log.Fatal("failed to rename: %s", err)
}
}
}

No output to error file

I'm coding a little Go program.
It reads files in a directory line by line, it only reads lines with a certain prefix, normalizes the data and outputs to one of two files, depending on whether the normalized record has certain number of elements.
Data is being outputted to the Data file, but errors are not being outputted to the Errors file.
Debugging I see no issue.
Any help is much appreciated.
Thanks,
Martin
package main
import (
"bufio"
"fmt"
"io/ioutil"
"log"
"os"
"strings"
)
func main() {
//Output file - Data
if _, err := os.Stat("allData.txt"); os.IsNotExist(err) {
var file, err = os.Create("allData.txt")
if err != nil {
fmt.Println(err)
return
}
defer file.Close()
}
file, err := os.OpenFile("allData.txt", os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
panic(err)
}
w := bufio.NewWriter(file)
//Output file - Errors
if _, err := os.Stat("errorData.txt"); os.IsNotExist(err) {
var fileError, err = os.Create("errorData.txt")
if err != nil {
fmt.Println(err)
return
}
defer fileError.Close()
}
fileError, err := os.OpenFile("errorData.txt", os.O_WRONLY|os.O_APPEND, 0644)
if err != nil {
panic(err)
}
z := bufio.NewWriter(fileError)
//Read Directory
files, err := ioutil.ReadDir("../")
if err != nil {
log.Fatal(err)
}
//Build file path
for _, f := range files {
fName := string(f.Name())
sPath := string("../" + fName)
sFile, err := os.Open(sPath)
if err != nil {
fmt.Println(err)
return
}
//Create scanner
scanner := bufio.NewScanner(sFile)
scanner.Split(bufio.ScanLines)
var lines []string
// This is the buffer now
for scanner.Scan() {
lines = append(lines, scanner.Text())
}
for _, line := range lines {
sRecordC := strings.HasPrefix((line), "DATA:")
if sRecordC {
splitted := strings.Split(line, " ")
splittedNoSpaces := deleteEmpty(splitted)
if len(splittedNoSpaces) == 11 {
splittedString := strings.Join(splittedNoSpaces, " ")
sFinalRecord := string(splittedString + "\r\n")
if _, err = fmt.Fprintf(w, sFinalRecord); err != nil {
}
}
if len(splittedNoSpaces) < 11 {
splitted := strings.Split(line, " ")
splittedNoSpaces := deleteEmpty(splitted)
splittedString := strings.Join(splittedNoSpaces, " ")
sFinalRecord := string(splittedString + "\r\n")
if _, err = fmt.Fprintf(z, sFinalRecord); err != nil {
}
err = fileError.Sync()
if err != nil {
log.Fatal(err)
}
}
}
}
}
err = file.Sync()
if err != nil {
log.Fatal(err)
}
}
//Delete Empty array elements
func deleteEmpty(s []string) []string {
var r []string
for _, str := range s {
if str != "" {
r = append(r, str)
}
}
return r
}
Don't open the file multiple times, and don't check for the file's existence before creating it, just use the os.O_CREATE flag. You're also not deferring the correct os.File.Close call, because it's opened multiple times.
When using a bufio.Writer, you should always call Flush() to ensure that all data has been written to the underlying io.Writer.

how to generate multiple uuid and md5 files in golang

Hi I've generated Md5 and uuid in golang but now I want generate it for multiple files using command line arguments, so what exactly I've to do. This is how I've generated my md5 and uuid:
package main
import (
"crypto/rand"
"crypto/md5"
"fmt"
"io"
"os"
"log"
"text/template"
)
type Data struct {
Uuid string
Md5 string
}
func main() {
uuid, err := newUUID()
if err != nil {
fmt.Printf("error: %v\n", err)
}
fmt.Printf("UUID: %s\n", uuid)
md5 := Getmd5(uuid)
fmt.Printf("Checksum: %s\n",md5)
fillData := Data{uuid, md5}
file, err := os.Create("text.txt")
if err != nil {
return
}
defer file.Close()
templ, err := template.ParseFiles("template.html")
if err !=nil{
log.Fatalln(err)
}
err = templ.Execute(file,fillData)
if err != nil{
log.Fatalln(err)
}
}
// newUUID generates a random UUID according to RFC 4122
func newUUID() (string, error) {
uuid := make([]byte, 16)
n, err := io.ReadFull(rand.Reader, uuid)
if n != len(uuid) || err != nil {
return "", err
}
// variant bits
uuid[8] = uuid[8]&^0xc0 | 0x80
// version 4 (pseudo-random)
uuid[6] = uuid[6]&^0xf0 | 0x40
return fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]), nil
}
func Getmd5(uuid string) (string) {
data := []byte(uuid)
//md5_buffer := fmt.Sprintf("%x", md5.Sum(data))
md5_buffer := md5.Sum(data)
return fmt.Sprintf("{0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x};\n",md5_buffer[0:1],
md5_buffer[1:2],md5_buffer[2:3],md5_buffer[3:4],md5_buffer[4:5],md5_buffer[5:6],md5_buffer[6:7],md5_buffer[7:8],
md5_buffer[8:9],md5_buffer[9:10],md5_buffer[10:11],md5_buffer[11:12],md5_buffer[12:13],md5_buffer[13:14],md5_buffer[14:15],
md5_buffer[15:16])
}
Can anyone help me out?
You can use os.Args to accept command line arguements
os.Args provides access to raw command-line arguments. Note that the first value in this slice is the path to the program, and os.Args[1:] holds the arguments to the program.
Your program will look like this, have a look at createFile and getNumberOfFiles functions and the main
package main
import (
"crypto/md5"
"crypto/rand"
"errors"
"fmt"
"io"
"log"
"os"
"strconv"
"text/template"
)
type Data struct {
Uuid string
Md5 string
}
func createFile(uuid string) {
md5 := Getmd5(uuid)
fmt.Printf("Checksum: %s\n", md5)
fillData := Data{uuid, md5}
file, err := os.Create(uuid + ".txt")
if err != nil {
return
}
defer file.Close()
templ, err := template.ParseFiles("template.html")
if err != nil {
log.Fatalln(err)
}
err = templ.Execute(file, fillData)
if err != nil {
log.Fatalln(err)
}
}
func getNumberOfFiles() (num int, err error) {
if len(os.Args) == 1 {
return 0, errors.New("Not enough arguements")
}
if num, err = strconv.Atoi(os.Args[1]); err != nil {
return
}
return num, nil
}
func main() {
numberOfFiles, err := getNumberOfFiles()
if err != nil {
fmt.Println(err.Error())
}
fmt.Printf("Creating %d files", numberOfFiles)
for i := 0; i < numberOfFiles; i++ {
uuid, err := newUUID()
if err != nil {
fmt.Printf("error: %v\n", err)
}
createFile(uuid)
}
}
// newUUID generates a random UUID according to RFC 4122
func newUUID() (string, error) {
uuid := make([]byte, 16)
n, err := io.ReadFull(rand.Reader, uuid)
if n != len(uuid) || err != nil {
return "", err
}
// variant bits
uuid[8] = uuid[8]&^0xc0 | 0x80
// version 4 (pseudo-random)
uuid[6] = uuid[6]&^0xf0 | 0x40
return fmt.Sprintf("%x-%x-%x-%x-%x", uuid[0:4], uuid[4:6], uuid[6:8], uuid[8:10], uuid[10:]), nil
}
func Getmd5(uuid string) string {
data := []byte(uuid)
//md5_buffer := fmt.Sprintf("%x", md5.Sum(data))
md5_buffer := md5.Sum(data)
return fmt.Sprintf("{0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x,0x%x};\n", md5_buffer[0:1],
md5_buffer[1:2], md5_buffer[2:3], md5_buffer[3:4], md5_buffer[4:5], md5_buffer[5:6], md5_buffer[6:7], md5_buffer[7:8],
md5_buffer[8:9], md5_buffer[9:10], md5_buffer[10:11], md5_buffer[11:12], md5_buffer[12:13], md5_buffer[13:14], md5_buffer[14:15],
md5_buffer[15:16])
}

Reading CSV file in Go

Here is a code snippet that reads CSV file:
func parseLocation(file string) (map[string]Point, error) {
f, err := os.Open(file)
defer f.Close()
if err != nil {
return nil, err
}
lines, err := csv.NewReader(f).ReadAll()
if err != nil {
return nil, err
}
locations := make(map[string]Point)
for _, line := range lines {
name := line[0]
lat, laterr := strconv.ParseFloat(line[1], 64)
if laterr != nil {
return nil, laterr
}
lon, lonerr := strconv.ParseFloat(line[2], 64)
if lonerr != nil {
return nil, lonerr
}
locations[name] = Point{lat, lon}
}
return locations, nil
}
Is there a way to improve readability of this code? if and nil noise.
Go now has a csv package for this. Its is encoding/csv. You can find the docs here: https://golang.org/pkg/encoding/csv/
There are a couple of good examples in the docs. Here is a helper method I created to read a csv file and returns its records.
package main
import (
"encoding/csv"
"fmt"
"log"
"os"
)
func readCsvFile(filePath string) [][]string {
f, err := os.Open(filePath)
if err != nil {
log.Fatal("Unable to read input file " + filePath, err)
}
defer f.Close()
csvReader := csv.NewReader(f)
records, err := csvReader.ReadAll()
if err != nil {
log.Fatal("Unable to parse file as CSV for " + filePath, err)
}
return records
}
func main() {
records := readCsvFile("../tasks.csv")
fmt.Println(records)
}
Go is a very verbose language, however you could use something like this:
// predeclare err
func parseLocation(file string) (locations map[string]*Point, err error) {
f, err := os.Open(file)
if err != nil {
return nil, err
}
defer f.Close() // this needs to be after the err check
lines, err := csv.NewReader(f).ReadAll()
if err != nil {
return nil, err
}
//already defined in declaration, no need for :=
locations = make(map[string]*Point, len(lines))
var lat, lon float64 //predeclare lat, lon
for _, line := range lines {
// shorter, cleaner and since we already have lat and err declared, we can do this.
if lat, err = strconv.ParseFloat(line[1], 64); err != nil {
return nil, err
}
if lon, err = strconv.ParseFloat(line[2], 64); err != nil {
return nil, err
}
locations[line[0]] = &Point{lat, lon}
}
return locations, nil
}
//edit
A more efficient and proper version was posted by #Dustin in the comments, I'm adding it here for completeness sake:
func parseLocation(file string) (map[string]*Point, error) {
f, err := os.Open(file)
if err != nil {
return nil, err
}
defer f.Close()
csvr := csv.NewReader(f)
locations := map[string]*Point{}
for {
row, err := csvr.Read()
if err != nil {
if err == io.EOF {
err = nil
}
return locations, err
}
p := &Point{}
if p.lat, err = strconv.ParseFloat(row[1], 64); err != nil {
return nil, err
}
if p.lon, err = strconv.ParseFloat(row[2], 64); err != nil {
return nil, err
}
locations[row[0]] = p
}
}
playground
I basically copied my answer from here: https://www.dotnetperls.com/csv-go. For me, this was a better answer than what I found on stackoverflow.
import (
"bufio"
"encoding/csv"
"os"
"fmt"
"io"
)
func ReadCsvFile(filePath string) {
// Load a csv file.
f, _ := os.Open(filePath)
// Create a new reader.
r := csv.NewReader(f)
for {
record, err := r.Read()
// Stop at EOF.
if err == io.EOF {
break
}
if err != nil {
panic(err)
}
// Display record.
// ... Display record length.
// ... Display all individual elements of the slice.
fmt.Println(record)
fmt.Println(len(record))
for value := range record {
fmt.Printf(" %v\n", record[value])
}
}
}
I also dislike the verbosity of the default Reader, so I made a new type that is
similar to bufio#Scanner:
package main
import "encoding/csv"
import "io"
type Scanner struct {
Reader *csv.Reader
Head map[string]int
Row []string
}
func NewScanner(o io.Reader) Scanner {
csv_o := csv.NewReader(o)
a, e := csv_o.Read()
if e != nil {
return Scanner{}
}
m := map[string]int{}
for n, s := range a {
m[s] = n
}
return Scanner{Reader: csv_o, Head: m}
}
func (o *Scanner) Scan() bool {
a, e := o.Reader.Read()
o.Row = a
return e == nil
}
func (o Scanner) Text(s string) string {
return o.Row[o.Head[s]]
}
Example:
package main
import "strings"
func main() {
s := `Month,Day
January,Sunday
February,Monday`
o := NewScanner(strings.NewReader(s))
for o.Scan() {
println(o.Text("Month"), o.Text("Day"))
}
}
https://golang.org/pkg/encoding/csv
You can also read contents of a directory to load all the CSV files. And then read all those CSV files 1 by 1 with goroutines
csv file:
101,300.00,11000901,1155686400
102,250.99,11000902,1432339200
main.go file:
const sourcePath string = "./source"
func main() {
dir, _ := os.Open(sourcePath)
files, _ := dir.Readdir(-1)
for _, file := range files {
fmt.Println("SINGLE FILE: ")
fmt.Println(file.Name())
filePath := sourcePath + "/" + file.Name()
f, _ := os.Open(filePath)
defer f.Close()
// os.Remove(filePath)
//func
go func(file io.Reader) {
records, _ := csv.NewReader(file).ReadAll()
for _, row := range records {
fmt.Println(row)
}
}(f)
time.Sleep(10 * time.Millisecond)// give some time to GO routines for execute
}
}
And the OUTPUT will be:
$ go run main.go
SINGLE FILE:
batch01.csv
[101 300.00 11000901 1155686400]
[102 250.99 11000902 1432339200]
----------------- -------------- ---------------------- -------
---------------- ------------------- ----------- --------------
Below example with the Invoice struct
func main() {
dir, _ := os.Open(sourcePath)
files, _ := dir.Readdir(-1)
for _, file := range files {
fmt.Println("SINGLE FILE: ")
fmt.Println(file.Name())
filePath := sourcePath + "/" + file.Name()
f, _ := os.Open(filePath)
defer f.Close()
go func(file io.Reader) {
records, _ := csv.NewReader(file).ReadAll()
for _, row := range records {
invoice := new(Invoice)
invoice.InvoiceNumber = row[0]
invoice.Amount, _ = strconv.ParseFloat(row[1], 64)
invoice.OrderID, _ = strconv.Atoi(row[2])
unixTime, _ := strconv.ParseInt(row[3], 10, 64)
invoice.Date = time.Unix(unixTime, 0)
fmt.Printf("Received invoice `%v` for $ %.2f \n", invoice.InvoiceNumber, invoice.Amount)
}
}(f)
time.Sleep(10 * time.Millisecond)
}
}
type Invoice struct {
InvoiceNumber string
Amount float64
OrderID int
Date time.Time
}

Resources