I am pretty new in Go. I have next code in Python, where I am getting all test cases in prepare_tests function. I need the same thing in Go.
from collections import namedtuple
from typing import List
TestCase = namedtuple("TestCase", ["number_of_purchased_items", "prices"])
def prepare_tests() -> List[TestCase]:
tests = []
cases_amount = int(input())
for _ in range(cases_amount):
number_of_purchased_items = int(input())
prices = [int(price) for price in input().split(maxsplit=number_of_purchased_items + 1)]
test_case = TestCase(number_of_purchased_items, prices)
tests.append(test_case)
return tests
Input( where 2 - test_cases amount, then 6 - numOfPurchasedItems, 2 2 2 3 3 3 - list
2
6
2 2 2 3 3 3
7
1 1 2 2 3 3 5
Output
[TestCase(number_of_purchased_items=6, prices=[2, 2, 2, 3, 3, 3]), TestCase(number_of_purchased_items=7, prices=[1, 1, 2, 2, 3, 3, 5])]
I tried to write the same in Go:
package main
import (
"bufio"
"fmt"
"os"
"strconv"
"strings"
)
type testCase struct {
numberOfPurchasedItems int
prices []int
}
func prepareTests() []testCase {
var tests []testCase
scanner := bufio.NewScanner(os.Stdin)
scanner.Split(bufio.ScanWords)
scanner.Scan()
casesAmount, err := strconv.Atoi(scanner.Text())
if err != nil {
fmt.Println("Error:", err)
return tests
}
for i := 0; i < casesAmount; i++ {
scanner.Scan()
numberOfPurchasedItems, err := strconv.Atoi(scanner.Text())
if err != nil {
fmt.Println("Error:", err)
return tests
}
scanner.Scan()
pricesStr := scanner.Text()
var prices []int
for _, priceStr := range strings.Split(pricesStr, " ") {
price, err := strconv.Atoi(priceStr)
if err != nil {
fmt.Println("Error:", err)
return tests
}
prices = append(prices, price)
}
tests = append(tests, testCase{numberOfPurchasedItems, prices})
}
return tests
}
Output in Go:
[{6 [2]} {2 [2]}]
you don`t need
scanner.Split(bufio.ScanWords)
so just delete that code
type testCase struct {
numberOfPurchasedItems int
prices []int
}
func prepareTests() []testCase {
var tests []testCase
scanner := bufio.NewScanner(os.Stdin)
//scanner.Split(bufio.ScanWords)
scanner.Scan()
casesAmount, err := strconv.Atoi(scanner.Text())
if err != nil {
fmt.Println("Error:", err)
return tests
}
for i := 0; i < casesAmount; i++ {
scanner.Scan()
numberOfPurchasedItems, err := strconv.Atoi(scanner.Text())
if err != nil {
fmt.Println("Error:", err)
return tests
}
scanner.Scan()
pricesStr := scanner.Text()
var prices []int
for _, priceStr := range strings.Split(pricesStr, " ") {
price, err := strconv.Atoi(priceStr)
if err != nil {
fmt.Println("Error:", err)
return tests
}
prices = append(prices, price)
}
tests = append(tests, testCase{numberOfPurchasedItems, prices})
}
return tests
}
Related
For the below type of inputs in golang coding interviews, what is the best way to get the input?
Input:
3
hello elloh
test estt
tier riet
I found two methods:
Method 1:
reader := bufio.NewReader(os.Stdin)
var lines []string
for {
line,err := reader.ReadString('\n') //this reads only one read
if err != nil {
log.Fatal(err)
}
if len(strings.TrimSpace(line)) == 0 {
break
}
line_s := strings.Split(line, " ")
lines = append(lines, line_s...)
}
Method 2:
bytes, err := ioutil.ReadAll(os.Stdin)
fmt.Println(len(bytes))
if err == nil {
input := strings.Split(string(bytes), "\n")
count, _ := strconv.Atoi(input[0])
fmt.Println(input)
var lines []string
for i := 1; i < count; i++ {
line := strings.Split(input[i], " ")
lines = append(lines, line...)
}
fmt.Println(lines)
}
But not sure how to end getting input from stdin in Method2.
Please suggest the best method to get input.
Use bufio.Scanner to read input. Use a function to encapsulate complexity and implementation details. For example,
package main
import (
"bufio"
"fmt"
"os"
"strconv"
"strings"
)
func readData(s *bufio.Scanner) ([][]string, error) {
var data [][]string
if !s.Scan() {
return nil, s.Err()
}
nLine, err := strconv.Atoi(strings.TrimSpace(s.Text()))
if err != nil {
return nil, err
}
for ; nLine > 0 && s.Scan(); nLine-- {
data = append(data, strings.Fields(s.Text()))
}
if err := s.Err(); err != nil {
return nil, err
}
if nLine != 0 {
err := fmt.Errorf("missing %d lines of data", nLine)
return nil, err
}
return data, nil
}
func main() {
s := bufio.NewScanner(os.Stdin)
data, err := readData(s)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fmt.Println(len(data))
for _, datum := range data {
fmt.Println(datum)
}
}
https://go.dev/play/p/0Xwp3-hwGyK
3
hello elloh
test estt
tier riet
3
[hello elloh]
[test estt]
[tier riet]
I have an awkward csv file and I need to skip the first row to read it.
I'm doing this easily with python/pandas
df = pd.read_csv(filename, skiprows=1)
but I don't know how to do it in Go.
package main
import (
"encoding/csv"
"fmt"
"log"
"os"
)
type mwericsson struct {
id string
name string
region string
}
func main() {
rows := readSample()
fmt.Println(rows)
//appendSum(rows)
//writeChanges(rows)
}
func readSample() [][]string {
f, err := os.Open("D:/in/20190629/PM_IG30014_15_201906290015_01.csv")
if err != nil {
log.Fatal(err)
}
rows, err := csv.NewReader(f).ReadAll()
f.Close()
if err != nil {
log.Fatal(err)
}
return rows
}
Error:
2019/07/01 12:38:40 record on line 2: wrong number of fields
PM_IG30014_15_201906290015_01.csv:
PTN Ethernet-Port RMON Performance,PORT_BW_UTILIZATION,2019-06-29 20:00:00,33366
DeviceID,DeviceName,ResourceName,CollectionTime,GranularityPeriod,PORT_RX_BW_UTILIZATION,PORT_TX_BW_UTILIZATION,RXGOODFULLFRAMESPEED,TXGOODFULLFRAMESPEED,PORT_RX_BW_UTILIZATION_MAX,PORT_TX_BW_UTILIZATION_MAX
3174659,H1095,H1095-11-ISM6-1(to ZJBSC-V1),2019-06-29 20:00:00,15,22.08,4.59,,,30.13,6.98
3174659,H1095,H1095-14-ISM6-1(to T6147-V),2019-06-29 20:00:00,15,2.11,10.92,,,4.43,22.45
skip the first row when reading a csv file
For example,
package main
import (
"bufio"
"encoding/csv"
"fmt"
"io"
"os"
)
func readSample(rs io.ReadSeeker) ([][]string, error) {
// Skip first row (line)
row1, err := bufio.NewReader(rs).ReadSlice('\n')
if err != nil {
return nil, err
}
_, err = rs.Seek(int64(len(row1)), io.SeekStart)
if err != nil {
return nil, err
}
// Read remaining rows
r := csv.NewReader(rs)
rows, err := r.ReadAll()
if err != nil {
return nil, err
}
return rows, nil
}
func main() {
f, err := os.Open("sample.csv")
if err != nil {
panic(err)
}
defer f.Close()
rows, err := readSample(f)
if err != nil {
panic(err)
}
fmt.Println(rows)
}
Output:
$ cat sample.csv
one,two,three,four
1,2,3
4,5,6
$ go run sample.go
[[1 2 3] [4 5 6]]
$
$ cat sample.csv
PTN Ethernet-Port RMON Performance,PORT_BW_UTILIZATION,2019-06-29 20:00:00,33366
DeviceID,DeviceName,ResourceName,CollectionTime,GranularityPeriod,PORT_RX_BW_UTILIZATION,PORT_TX_BW_UTILIZATION,RXGOODFULLFRAMESPEED,TXGOODFULLFRAMESPEED,PORT_RX_BW_UTILIZATION_MAX,PORT_TX_BW_UTILIZATION_MAX
3174659,H1095,H1095-11-ISM6-1(to ZJBSC-V1),2019-06-29 20:00:00,15,22.08,4.59,,,30.13,6.98
3174659,H1095,H1095-14-ISM6-1(to T6147-V),2019-06-29 20:00:00,15,2.11,10.92,,,4.43,22.45
$ go run sample.go
[[DeviceID DeviceName ResourceName CollectionTime GranularityPeriod PORT_RX_BW_UTILIZATION PORT_TX_BW_UTILIZATION RXGOODFULLFRAMESPEED TXGOODFULLFRAMESPEED PORT_RX_BW_UTILIZATION_MAX PORT_TX_BW_UTILIZATION_MAX] [3174659 H1095 H1095-11-ISM6-1(to ZJBSC-V1) 2019-06-29 20:00:00 15 22.08 4.59 30.13 6.98] [3174659 H1095 H1095-14-ISM6-1(to T6147-V) 2019-06-29 20:00:00 15 2.11 10.92 4.43 22.45]]
$
Simply call Reader.Read() to read a line, then proceed to read the rest with Reader.ReadAll().
See this example:
src := "one,two,three\n1,2,3\n4,5,6"
r := csv.NewReader(strings.NewReader(src))
if _, err := r.Read(); err != nil {
panic(err)
}
records, err := r.ReadAll()
if err != nil {
panic(err)
}
fmt.Println(records)
Output (try it on the Go Playground):
[[1 2 3] [4 5 6]]
while it was informative to learn about io.ReadSeeker, I think a simpler way to skip the first line/row (often times the header) of a csv is to use the slice functionality as follows:
func readCsv(filename string) [][]string {
f, err := os.Open(filename)
if err != nil {
log.Fatal(err)
}
defer f.Close()
records := [][]string{}
r := csv.NewReader(f)
for {
record, err := r.Read()
if err == io.EOF {
break
}
if err != nil {
log.Fatal(err)
}
records = append(records, record)
}
return records[1:] // skip the header
}
we can just use bufio.ReadBytes('\n') and pass bufio as Reader to csv.NewReader
func readSample(reader io.Reader) ([][]string, error) {
// if reader is bufio, we don't need to NewReader againg
buf, ok := (reader).(*bufio.Reader)
if !ok {
buf = bufio.NewReader(reader)
}
_, err := buf.ReadBytes('\n')
if err != nil {
return nil, err
}
rows, err := csv.NewReader(buf).ReadAll()
if err != nil {
return nil, err
}
return rows, nil
}
doc: https://developers.google.com/ad-exchange/rtb/response-guide/decrypt-price#sample_code
There is no official "Double Click Crypto" example code in golang, so I try to implement by myself. But I can't pass the test in doc. Please help me!
skU7Ax_NL5pPAFyKdkfZjZz2-VhIN8bjj1rVFOaJ_5o= // Encryption key (e_key)
arO23ykdNqUQ5LEoQ0FVmPkBd7xB5CO89PDZlSjpFxo= // Integrity key (i_key)
WEp8wQAAAABnFd5EkB2k1wJeFcAj-Z_JVOeGzA // 100 CPI micros
WEp8sQAAAACwF6CtLJrXSRFBM8UiTTIyngN-og // 1900 CPI micros
WEp8nQAAAAADG-y45xxIC1tMWuTjzmDW6HtroQ // 2700 CPI micros
Here is my code:
package main
// https://developers.google.com/ad-exchange/rtb/response-guide/decrypt-price?hl=zh-CN
import (
"bytes"
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"encoding/binary"
"fmt"
)
var base64Codec = base64.URLEncoding.WithPadding(base64.NoPadding)
func safeXORBytes(dst, a, b []byte) int {
n := len(a)
if len(b) < n {
n = len(b)
}
for i := 0; i < n; i++ {
dst[i] = a[i] ^ b[i]
}
return n
}
func EncryptPrice(eKey []byte, iKey []byte, price uint64, iv []byte) (finalMessage string, err error) {
if len(iv) != 16 {
err = fmt.Errorf("len(iv) = %d != 16", len(iv))
return
}
h1 := hmac.New(sha1.New, eKey)
h1.Write(iv)
pad := h1.Sum(nil)[:8]
priceBytes := make([]byte, 8)
binary.BigEndian.PutUint64(priceBytes, price)
encPrice := make([]byte, 8)
n := safeXORBytes(encPrice, priceBytes, pad)
if n != 8 {
err = fmt.Errorf("safeXORBytes n != %d", n)
return
}
h2 := hmac.New(sha1.New, iKey)
h2.Write(priceBytes)
h2.Write(iv)
signature := h2.Sum(nil)[:4]
finalMessage = base64Codec.EncodeToString(append(append(iv, encPrice...), signature...))
return
}
func DecryptPrice(eKey []byte, iKey []byte, finalMessage string) (price uint64, err error) {
finalMessageBytes, err := base64Codec.DecodeString(finalMessage)
if err != nil {
return
}
if len(finalMessageBytes) != 28 {
err = fmt.Errorf("len(finalMessageBytes) = %d != 28", len(finalMessageBytes))
return
}
iv := finalMessageBytes[:16]
encPrice := finalMessageBytes[16:24]
signature := finalMessageBytes[24:]
h1 := hmac.New(sha1.New, eKey)
h1.Write(iv)
pad := h1.Sum(nil)[:8]
priceBytes := make([]byte, 8)
n := safeXORBytes(priceBytes, encPrice, pad)
if n != 8 {
err = fmt.Errorf("safeXORBytes n != %d", n)
return
}
h2 := hmac.New(sha1.New, iKey)
h2.Write(priceBytes)
h2.Write(iv)
confSignature := h2.Sum(nil)[:4]
if bytes.Compare(confSignature, signature) != 0 {
err = fmt.Errorf("sinature mismatch: confSignature = %s, sinature = %s", confSignature, signature)
return
}
price = binary.BigEndian.Uint64(priceBytes)
return
}
package main
import (
"github.com/stretchr/testify/assert"
"testing"
)
func TestEncryptPrice(t *testing.T) {
eKeyBase64Encoded := "skU7Ax_NL5pPAFyKdkfZjZz2-VhIN8bjj1rVFOaJ_5o"
eKey, err := base64Codec.DecodeString(eKeyBase64Encoded)
assert.Nil(t, err)
iKeyBase64Encoded := "arO23ykdNqUQ5LEoQ0FVmPkBd7xB5CO89PDZlSjpFxo"
iKey, err := base64Codec.DecodeString(iKeyBase64Encoded)
assert.Nil(t, err)
finalMessage, err := EncryptPrice(eKey, iKey, uint64(100), []byte{88, 74, 124, 193, 0, 0, 0, 0, 103, 21, 222, 68, 144, 29, 164, 215})
assert.Nil(t, err)
assert.Equal(t, "WEp8wQAAAABnFd5EkB2k1wJeFcAj-Z_JVOeGzA", finalMessage)
}
func TestDecryptPrice(t *testing.T) {
eKeyBase64Encoded := "skU7Ax_NL5pPAFyKdkfZjZz2-VhIN8bjj1rVFOaJ_5o"
eKey, err := base64Codec.DecodeString(eKeyBase64Encoded)
assert.Nil(t, err)
iKeyBase64Encoded := "arO23ykdNqUQ5LEoQ0FVmPkBd7xB5CO89PDZlSjpFxo"
iKey, err := base64Codec.DecodeString(iKeyBase64Encoded)
assert.Nil(t, err)
price, err := DecryptPrice(eKey, iKey, "WEp8wQAAAABnFd5EkB2k1wJeFcAj-Z_JVOeGzA")
assert.Nil(t, err)
assert.Equal(t, uint64(100), price)
}
test result:
--- FAIL: TestEncryptPrice (0.00s)
Error Trace: price_test.go:19
Error: Not equal:
expected: "WEp8wQAAAABnFd5EkB2k1wJeFcAj-Z_JVOeGzA"
actual: "WEp8wQAAAABnFd5EkB2k1wf7jAcG7gZ9tPUnAA"
--- FAIL: TestDecryptPrice (0.00s)
Error Trace: price_test.go:32
Error: Expected nil, but got: &errors.errorString{s:"sinature mismatch: confSignature = \xbe\xaa\xf6h, sinature = T\xe7\x86\xcc"}
Error Trace: price_test.go:33
Error: Not equal:
expected: 0x64
actual: 0x0
FAIL
this code can pass the test in https://github.com/google/openrtb-doubleclick/blob/0.9.0/doubleclick-core/src/test/java/com/google/doubleclick/crypto/DoubleClickCryptoTest.java
, so I guess that the encrypted price in the doc are not excrypted by the e_key and i_key in the doc.
I met the same problem today. Finally, I found the bug was that 'ekey' and 'ikey' did not need to base64Codec.DecodeString, just use []byte(eKeyBase64Encoded).
eKeyBase64Encoded := "skU7Ax_NL5pPAFyKdkfZjZz2-VhIN8bjj1rVFOaJ_5o"
eKey := []byte(eKeyBase64Encoded)
assert.Nil(t, err)
iKeyBase64Encoded := "arO23ykdNqUQ5LEoQ0FVmPkBd7xB5CO89PDZlSjpFxo"
iKey :=[]byte(iKeyBase64Encoded)
assert.Nil(t, err)
price, err := DecryptPrice(eKey, iKey, "WEp8wQAAAABnFd5EkB2k1wJeFcAj-Z_JVOeGzA")
....
The encrypted price that are listed in this document https://developers.google.com/authorized-buyers/rtb/response-guide/decrypt-price have been encoded using the keys they showed above the prices. The examples are correct. I wrote a small Go package that provides a Decrypt function, I might add an encrypt one soon: https://github.com/matipan/doubleclick. Below I leave an example and code snippets for easy access.
First, to be able to parse the keys we need to respect the web-safe base64 decoding, which means we need to use base64.URLEncoding! to decode the keys:
// ParseKeys parses the base64 web-safe encoded keys as explained in Google's documentation:
// https://developers.google.com/authorized-buyers/rtb/response-guide/decrypt-price
func ParseKeys(ic, ec []byte) (icKey []byte, ecKey []byte, err error) {
icKey = make([]byte, base64.URLEncoding.DecodedLen(len([]byte(ic))))
n, err := base64.URLEncoding.Decode(icKey, []byte(ic))
if err != nil {
return nil, nil, fmt.Errorf("%w: could not decode price integrity key", err)
}
icKey = icKey[:n]
ecKey = make([]byte, base64.URLEncoding.DecodedLen(len([]byte(ec))))
n, err = base64.URLEncoding.Decode(ecKey, []byte(ec))
if err != nil {
return nil, nil, fmt.Errorf("%w: could not decode price encryption key", err)
}
ecKey = ecKey[:n]
return icKey, ecKey, nil
}
We can call this code and decrypt the keys they show:
icKey, ecKey, err = ParseKeys([]byte("arO23ykdNqUQ5LEoQ0FVmPkBd7xB5CO89PDZlSjpFxo="), []byte("skU7Ax_NL5pPAFyKdkfZjZz2-VhIN8bjj1rVFOaJ_5o="))
The function to decrypt the price with a bit more error handling is:
// ErrInvalidPrice is the error returned when the price parsed
// by DecryptPrice is not correct.
var ErrInvalidPrice = errors.New("price is invalid")
// DecryptPrice decrypts the price with google's doubleclick cryptography encoding.
// encPrice is an unpadded web-safe base64 encoded string according to RFC 3548.
// https://developers.google.com/authorized-buyers/rtb/response-guide/decrypt-price
func DecryptPrice(icKey, ecKey, encPrice []byte) (uint64, error) {
if len(icKey) == 0 || len(ecKey) == 0 {
return 0, errors.New("encryption and integrity keys are required")
}
if len(encPrice) != 38 {
return 0, fmt.Errorf("%w: invalid length, expected 28 got %d", ErrInvalidPrice, len(encPrice))
}
dprice := make([]byte, base64.RawURLEncoding.DecodedLen(len(encPrice)))
n, err := base64.RawURLEncoding.Decode(dprice, encPrice)
if err != nil {
return 0, fmt.Errorf("%w: invalid base64 string", err)
}
dprice = dprice[:n]
if len(dprice) != 28 {
return 0, fmt.Errorf("%w: invalid decoded price length. Expected 28 got %d", ErrInvalidPrice, len(dprice))
}
// encrypted price is composed of parts of fixed lenth. We break it up according to:
// {initialization_vector (16 bytes)}{encrypted_price (8 bytes)}{integrity (4 bytes)}
iv, p, sig := dprice[0:16], dprice[16:24], dprice[24:]
h := hmac.New(sha1.New, ecKey)
n, err = h.Write(iv)
if err != nil || n != len(iv) {
return 0, fmt.Errorf("%w: could not write hmac hash for iv. err=%s, n=%d, len(iv)=%d", ErrInvalidPrice, err, n, len(iv))
}
pricePad := h.Sum(nil)
price := safeXORBytes(p, pricePad)
if price == nil {
return 0, fmt.Errorf("%w: price xor price_pad failed", ErrInvalidPrice)
}
h = hmac.New(sha1.New, icKey)
n, err = h.Write(price)
if err != nil || n != len(price) {
return 0, fmt.Errorf("%w: could not write hmac hash for price. err=%s, n=%d, len(price)=%d", ErrInvalidPrice, err, n, len(price))
}
n, err = h.Write(iv)
if err != nil || n != len(iv) {
return 0, fmt.Errorf("%w: could not write hmac hash for iv. err=%s, n=%d, len(iv)=%d", ErrInvalidPrice, err, n, len(iv))
}
confSig := h.Sum(nil)[:4]
if bytes.Compare(confSig, sig) != 0 {
return 0, fmt.Errorf("%w: integrity of price is not valid", ErrInvalidPrice)
}
return binary.BigEndian.Uint64(price), nil
}
func safeXORBytes(a, b []byte) []byte {
n := len(a)
if len(b) < n {
n = len(b)
}
if n == 0 {
return nil
}
dst := make([]byte, n)
for i := 0; i < n; i++ {
dst[i] = a[i] ^ b[i]
}
return dst
}
We can now call this code using the example of 1900 micros: YWJjMTIzZGVmNDU2Z2hpN7fhCuPemC32prpWWw:
price, err := DecryptPrice(icKey, ecKey, []byte("YWJjMTIzZGVmNDU2Z2hpN7fhCuPemC32prpWWw"))
And price will be 1900
I am get leveldb's all key-val to a map[string][]byte, but it is not running as my expection.
code is as below
package main
import (
"fmt"
"strconv"
"github.com/syndtr/goleveldb/leveldb"
)
func main() {
db, err := leveldb.OpenFile("db", nil)
if err != nil {
panic(err)
}
defer db.Close()
for i := 0; i < 10; i++ {
err := db.Put([]byte("key"+strconv.Itoa(i)), []byte("value"+strconv.Itoa(i)), nil)
if err != nil {
panic(err)
}
}
snap, err := db.GetSnapshot()
if err != nil {
panic(err)
}
if snap == nil {
panic("snap shot is nil")
}
data := make(map[string][]byte)
iter := snap.NewIterator(nil, nil)
for iter.Next() {
Key := iter.Key()
Value := iter.Value()
data[string(Key)] = Value
}
iter.Release()
if iter.Error() != nil {
panic(iter.Error())
}
for k, v := range data {
fmt.Println(string(k) + ":" + string(v))
}
}
but the result is below
key3:value9
key6:value9
key7:value9
key8:value9
key1:value9
key2:value9
key4:value9
key5:value9
key9:value9
key0:value9
rather not key0:value0
Problem is with casting around types (byte[] to string, etc.).
You are trying to print string values. To avoid unnecessary casting apply the following modifications:
Change data initialization into data := make(map[string]string)
Assign values into data with `data[string(Key)] = string(Value) (by the way, don't use capitalization for variables you aren't intend to export)
Print data's values with fmt.Println(k + ":" + v))
This should produce the following result:
key0:value0
key1:value1
key7:value7
key2:value2
key3:value3
key4:value4
key5:value5
key6:value6
key8:value8
key9:value9
Here is a code snippet that reads CSV file:
func parseLocation(file string) (map[string]Point, error) {
f, err := os.Open(file)
defer f.Close()
if err != nil {
return nil, err
}
lines, err := csv.NewReader(f).ReadAll()
if err != nil {
return nil, err
}
locations := make(map[string]Point)
for _, line := range lines {
name := line[0]
lat, laterr := strconv.ParseFloat(line[1], 64)
if laterr != nil {
return nil, laterr
}
lon, lonerr := strconv.ParseFloat(line[2], 64)
if lonerr != nil {
return nil, lonerr
}
locations[name] = Point{lat, lon}
}
return locations, nil
}
Is there a way to improve readability of this code? if and nil noise.
Go now has a csv package for this. Its is encoding/csv. You can find the docs here: https://golang.org/pkg/encoding/csv/
There are a couple of good examples in the docs. Here is a helper method I created to read a csv file and returns its records.
package main
import (
"encoding/csv"
"fmt"
"log"
"os"
)
func readCsvFile(filePath string) [][]string {
f, err := os.Open(filePath)
if err != nil {
log.Fatal("Unable to read input file " + filePath, err)
}
defer f.Close()
csvReader := csv.NewReader(f)
records, err := csvReader.ReadAll()
if err != nil {
log.Fatal("Unable to parse file as CSV for " + filePath, err)
}
return records
}
func main() {
records := readCsvFile("../tasks.csv")
fmt.Println(records)
}
Go is a very verbose language, however you could use something like this:
// predeclare err
func parseLocation(file string) (locations map[string]*Point, err error) {
f, err := os.Open(file)
if err != nil {
return nil, err
}
defer f.Close() // this needs to be after the err check
lines, err := csv.NewReader(f).ReadAll()
if err != nil {
return nil, err
}
//already defined in declaration, no need for :=
locations = make(map[string]*Point, len(lines))
var lat, lon float64 //predeclare lat, lon
for _, line := range lines {
// shorter, cleaner and since we already have lat and err declared, we can do this.
if lat, err = strconv.ParseFloat(line[1], 64); err != nil {
return nil, err
}
if lon, err = strconv.ParseFloat(line[2], 64); err != nil {
return nil, err
}
locations[line[0]] = &Point{lat, lon}
}
return locations, nil
}
//edit
A more efficient and proper version was posted by #Dustin in the comments, I'm adding it here for completeness sake:
func parseLocation(file string) (map[string]*Point, error) {
f, err := os.Open(file)
if err != nil {
return nil, err
}
defer f.Close()
csvr := csv.NewReader(f)
locations := map[string]*Point{}
for {
row, err := csvr.Read()
if err != nil {
if err == io.EOF {
err = nil
}
return locations, err
}
p := &Point{}
if p.lat, err = strconv.ParseFloat(row[1], 64); err != nil {
return nil, err
}
if p.lon, err = strconv.ParseFloat(row[2], 64); err != nil {
return nil, err
}
locations[row[0]] = p
}
}
playground
I basically copied my answer from here: https://www.dotnetperls.com/csv-go. For me, this was a better answer than what I found on stackoverflow.
import (
"bufio"
"encoding/csv"
"os"
"fmt"
"io"
)
func ReadCsvFile(filePath string) {
// Load a csv file.
f, _ := os.Open(filePath)
// Create a new reader.
r := csv.NewReader(f)
for {
record, err := r.Read()
// Stop at EOF.
if err == io.EOF {
break
}
if err != nil {
panic(err)
}
// Display record.
// ... Display record length.
// ... Display all individual elements of the slice.
fmt.Println(record)
fmt.Println(len(record))
for value := range record {
fmt.Printf(" %v\n", record[value])
}
}
}
I also dislike the verbosity of the default Reader, so I made a new type that is
similar to bufio#Scanner:
package main
import "encoding/csv"
import "io"
type Scanner struct {
Reader *csv.Reader
Head map[string]int
Row []string
}
func NewScanner(o io.Reader) Scanner {
csv_o := csv.NewReader(o)
a, e := csv_o.Read()
if e != nil {
return Scanner{}
}
m := map[string]int{}
for n, s := range a {
m[s] = n
}
return Scanner{Reader: csv_o, Head: m}
}
func (o *Scanner) Scan() bool {
a, e := o.Reader.Read()
o.Row = a
return e == nil
}
func (o Scanner) Text(s string) string {
return o.Row[o.Head[s]]
}
Example:
package main
import "strings"
func main() {
s := `Month,Day
January,Sunday
February,Monday`
o := NewScanner(strings.NewReader(s))
for o.Scan() {
println(o.Text("Month"), o.Text("Day"))
}
}
https://golang.org/pkg/encoding/csv
You can also read contents of a directory to load all the CSV files. And then read all those CSV files 1 by 1 with goroutines
csv file:
101,300.00,11000901,1155686400
102,250.99,11000902,1432339200
main.go file:
const sourcePath string = "./source"
func main() {
dir, _ := os.Open(sourcePath)
files, _ := dir.Readdir(-1)
for _, file := range files {
fmt.Println("SINGLE FILE: ")
fmt.Println(file.Name())
filePath := sourcePath + "/" + file.Name()
f, _ := os.Open(filePath)
defer f.Close()
// os.Remove(filePath)
//func
go func(file io.Reader) {
records, _ := csv.NewReader(file).ReadAll()
for _, row := range records {
fmt.Println(row)
}
}(f)
time.Sleep(10 * time.Millisecond)// give some time to GO routines for execute
}
}
And the OUTPUT will be:
$ go run main.go
SINGLE FILE:
batch01.csv
[101 300.00 11000901 1155686400]
[102 250.99 11000902 1432339200]
----------------- -------------- ---------------------- -------
---------------- ------------------- ----------- --------------
Below example with the Invoice struct
func main() {
dir, _ := os.Open(sourcePath)
files, _ := dir.Readdir(-1)
for _, file := range files {
fmt.Println("SINGLE FILE: ")
fmt.Println(file.Name())
filePath := sourcePath + "/" + file.Name()
f, _ := os.Open(filePath)
defer f.Close()
go func(file io.Reader) {
records, _ := csv.NewReader(file).ReadAll()
for _, row := range records {
invoice := new(Invoice)
invoice.InvoiceNumber = row[0]
invoice.Amount, _ = strconv.ParseFloat(row[1], 64)
invoice.OrderID, _ = strconv.Atoi(row[2])
unixTime, _ := strconv.ParseInt(row[3], 10, 64)
invoice.Date = time.Unix(unixTime, 0)
fmt.Printf("Received invoice `%v` for $ %.2f \n", invoice.InvoiceNumber, invoice.Amount)
}
}(f)
time.Sleep(10 * time.Millisecond)
}
}
type Invoice struct {
InvoiceNumber string
Amount float64
OrderID int
Date time.Time
}