How to get this specific shape for my data - go

In Go, I'm using this function bars, err := custplotter.NewCandlesticks(data)
from here:
https://github.com/pplcc/plotext/tree/master/custplotter
It's expecting this shape for data:
[{2 16435 16458 16435 16446 1} {3 16446 16458 16435.04 16455 1} .....]
But my code below is creating my data in this shape instead:
[[2 16435 16458 16435 16446 1] [3 16446 16458 16435.04 16455 1] .....]
Which gives me this error message:
cannot use data (type [ ][ ]string) as type custplotter.TOHLCVer in argument to custplotter.NewCandlesticks:
[ ][ ]string does not implement custplotter.TOHLCVer (missing Len method)
I believe the problem is the data shape. How can i change my code to create the required data shape (with { } instead of [ ]) ?
//read excel file******************************************
xlsx, err := excelize.OpenFile("/media/Snaps/test snaps.xlsm")
if err != nil {
fmt.Println(err)
return
}
//read all rows into df
df := xlsx.GetRows("ticker_2")
//get only TOHLCV columns and 60 rows
df3 := make([][]string, 60) // create slice for 60 rows
idx := 0
for _, row := range df[1:61] { // read 60 rows
df3row := make([]string, 6) // create slice for 6 columns
copy(df3row, row[28:34]) // copy desired columns to new row slice
df3[idx] = df3row
idx++
}
All examples of slices i found in Go litterature uses only [ [ ], [ ] ]

as per https://github.com/pplcc/plotext/blob/68ab3c6e05c34baf5af21c9f5c3341f527a110ac/examples/tohlcvexampledata.go#L42
It seems that what you need is a custplotter.TOHLCVs which is just a slice of a struct of float64.
https://github.com/pplcc/plotext/blob/master/custplotter/tohlcv.go:
type TOHLCVer interface {
// Len returns the number of time, open, high, low, close, volume tuples.
Len() int
// TOHLCV returns an time, open, high, low, close, volume tuple.
TOHLCV(int) (float64, float64, float64, float64, float64, float64)
}
// TOHLCVs implements the TOHLCVer interface using a slice.
type TOHLCVs []struct{ T, O, H, L, C, V float64 }
so basically your solution could resemble this:
df3 := make(TOHLCVs, 60) // create slice for 60 rows
idx := 0
for _, row := range df[1:61] { // read 60 rows
df3[idx].T, err = strconv.ParseFloat(row[28], 64)
df3[idx].O, err = strconv.ParseFloat(row[29], 64)
df3[idx].H, err = strconv.ParseFloat(row[30], 64)
df3[idx].L, err = strconv.ParseFloat(row[31], 64)
df3[idx].C, err = strconv.ParseFloat(row[32], 64)
df3[idx].V, err = strconv.ParseFloat(row[33], 64)
idx++
}
Or you could just implement the TOHLCVer interface too :)
type SlicesOfTOHLCV [][6]float64
func (s SlicesOfTOHLCV) Len() int {
return len(s)
}
func (s SlicesOfTOHLCV) TOHLCV(i int) (float64, float64, float64, float64, float64) {
return s[i][0], s[i][1], s[i][2], s[i][3], s[i][4], s[i][5]
}
mySlice := make(SlicesOfTOHLCV, 60)
i := 0
for _, row := range df[1:61] {
mySlice[i] = [6]float64{}
for j := 0; j < 6; j ++ {
mySlice[i][j], err = strconv.ParseFloat(row[28+j], 64)
if err != nil {
panic(err)
}
}
i ++
}

Related

Go copy bytes into struct fields with reflection

How can I iterate over a byte slice and assign them to the fields of a struct?
type s struct {
f1 []byte
f2 []byte
f3 []byte
}
func S s {
x := s{}
x.f1 = make([]byte, 4)
x.f1 = make([]byte, 2)
x.f1 = make([]byte, 2)
return x
}
func main() {
data := []byte{83, 117, 110, 83, 0, 1, 0, 65}
Z := S()
//pesudo code from here
i:= 0
for field in Z {
field = data[i:len(field)]
i += len(field)
}
Expecting:
f1 = [83,117,110,83]
f2 = [0,1]
f3 = [0,65]
I've done this in C/C++ before but I can't figure out how to do it in Go. I need the assigning function to be generic as I'm going to have several different structs some of which may not exist in the stream.
Ideally I want to pass in the initialized struct and my code would iterate over the struct fields filling them in.
Leverage the reflection code in the binary/encoding package.
Step 1: Declare the fields as arrays instead of slices.
type S struct {
F1 [4]byte
F2 [2]byte
F3 [2]byte
}
Step 2: Decode the data to the struct using binary.Read
var s S
data := []byte{83, 117, 110, 83, 0, 1, 0, 65}
err := binary.Read(bytes.NewReader(data), binary.LittleEndian, &s)
if err != nil {
log.Fatal(err)
}
Step 3: Done!
fmt.Print(s) // prints {[83 117 110 83] [0 1] [0 65]}
https://go.dev/play/p/H-e8Lusw0RC
You can use reflect.Copy. Like the built-in copy, it copies data into the destination up to its length. Make sure the fields you need to set are exported.
func main() {
data := []byte{83, 117, 110, 83, 0, 1, 0, 65}
z := S{
F1: make([]byte, 4),
F2: make([]byte, 2),
F3: make([]byte, 2),
}
SetBytes(&z, data)
fmt.Println(z) // {[83 117 110 83] [0 1] [0 65]}
}
func SetBytes(dst any, data []byte) {
v := reflect.ValueOf(dst)
if v.Kind() != reflect.Ptr {
panic("dst must be addressable")
}
v = v.Elem()
j := 0
for i := 0; i < v.NumField(); i++ {
field := v.Field(i)
if field.Kind() != reflect.Slice {
continue
}
j += reflect.Copy(v.Field(i), reflect.ValueOf(data[j:]))
}
}
Since data is assumed to be always []byte, you can subslice it directly.
Alternatively, you can use reflect.Value#Slice:
d := reflect.ValueOf(data)
// and later
j += reflect.Copy(v.Field(i), d.Slice(j, d.Len()))
Playground: https://go.dev/play/p/o1MR1qrW5pL

Modifying receiver with a method on value?

package matrix
import (
"errors"
"strconv"
"strings"
)
// Matrix matrix inteface
type Matrix interface {
Rows() [][]int
Cols() [][]int
Set(r, c, val int) bool
}
// matrix implements the interface Matrix
type matrix struct {
data [][]int
rows int
cols int
}
// New returns a valid matrix created from the input
func New(input string) (Matrix, error) {
var m matrix
rows := strings.Split(input, "\n")
for r, row := range rows {
rowElements := strings.Fields(row)
switch {
case r == 0:
m.rows, m.cols = len(rows), len(rowElements)
matrix, err := allocateMemory(m.rows, m.cols)
if err != nil {
return invalidMatrix()
}
m.data = matrix
case len(rowElements) != m.cols:
return invalidMatrix()
}
for c, element := range rowElements {
element, err := strconv.Atoi(element)
if err != nil {
return invalidMatrix()
}
m.data[r][c] = element
}
}
return m, nil
}
// invalidMatrix returns the error indicating the
// provided matrix is invalid
func invalidMatrix() (Matrix, error) {
return nil, errors.New("invalid matrix")
}
// allocateMemory allocates a 2D slice of int having size RxC
func allocateMemory(R, C int) ([][]int, error) {
if R < 1 || C < 1 {
return nil, errors.New("invalid matrix")
}
matrix := make([][]int, R)
for r := range matrix {
matrix[r] = make([]int, C)
}
return matrix, nil
}
// Set sets the given value at (r,c) in the matrix,
// if (r,c) belongs to the matrix.
func (m matrix) Set(r, c, val int) bool {
switch {
case r < 0 || c < 0:
return false
case r >= m.rows || c >= m.cols:
return false
default:
m.data[r][c] = val
return true
}
}
// order defines the order the matrix to export
// two useful values are columnMajor and rowMajor
type order int
const (
columnMajor order = iota
rowMajor
)
// Cols returns columns of the matrix.
func (m matrix) Cols() [][]int {
return m.export(columnMajor)
}
// Rows returns rows of the matrix.
func (m matrix) Rows() [][]int {
return m.export(rowMajor)
}
// export return the matrix in the required order;
// either columnMajor or rowMajor.
func (m matrix) export(o order) [][]int {
var matrix [][]int
var err error
switch o {
case columnMajor:
matrix, err = allocateMemory(m.cols, m.rows)
if err != nil {
return nil
}
for r, row := range m.data {
for c, element := range row {
matrix[c][r] = element
}
}
case rowMajor:
matrix, err = allocateMemory(m.rows, m.cols)
if err != nil {
return nil
}
for r, row := range m.data {
copy(matrix[r], row)
}
}
return matrix
}
I am having a hard time understanding why the method Set() is able to modify the data of the struct. I had an understanding that methods defined on values cannot do that. I have tried to compare it with another problem where I cannot modify the content of receiver but in this case it just works. A test file for this code is available at test file. Any idea what I am missing?
The reason Set can modify the contents of the slice is that the slice is a reference value. Your other example (in the comment) attempts to assign the field holding the slice, and this won't work - because it's working on a copy. See this code sample:
package main
import (
"fmt"
)
type Holder struct {
s []int
v []int
}
func (h Holder) Set() {
// This will successfully modify the `s` slice's contents
h.s[0] = 99
// This will assign a new slice to a copy of the v field,
// so it won't affect the actual value on which this
// method is invoked.
h.v = []int{1, 2, 3}
}
func main() {
var h Holder
h.s = []int{10, 20, 30}
h.v = []int{40, 50, 60}
fmt.Println("before Set:", h)
h.Set()
fmt.Println("after Set:", h)
}
You can run it on the playground, and it prints:
before Set: {[10 20 30] [40 50 60]}
after Set: {[99 20 30] [40 50 60]}
What happens here is that even though Set gets a copy of h, and hence h.s is a copy too, but both copies point to the same underlying slice, so the contents can be modified. Read this post for all the details.
A slice value contains (ptr, len, cap) where ptr is a pointer to the slice's underlying array. The Set method modifies the slice's underlying array by dereferencing the pointer. The slice value, stored in the field, is not modified.
The Go Language blog post on slices describes the slice memory layout in more detail.

how should I change parsing method or correct random method

I would like to generate a random 17 digits number string and parse it to uint64 by golang, here is my code:
const msgMaxValue = 100000000000000000
s := fmt.Sprintf("%17v", rand.New(rand.NewSource(time.Now().UnixNano())).Int63n(msgMaxValue))
log.Info("s:", s)
msgId, err := strconv.Atoi(s)
the error is: invalid syntax, I found there is a space in random value but don't know why, how should I fix it, thank u
have fixed it by "crypto/rand", below is code:
func CreateRandomNumber(len int) (string, error) {
var numbers = []byte{1, 2, 3, 4, 5, 7, 8, 9}
var container string
length := bytes.NewReader(numbers).Len()
for i := 0; i < len; i++ {
random, err := rand.Int(rand.Reader, big.NewInt(int64(length)))
if err != nil {
log.Error("random error:", err)
return "", err
}
container += fmt.Sprintf("%d", numbers[random.Int64()])
}
return container, nil
}
then strconv.Atoi to get a int type number is okay

Implementing a gradient descent

I'm trying to implement a gradient descent in Go. My goal is to predict the cost of a car from it's mileage.
Here is my data set:
km,price
240000,3650
139800,3800
150500,4400
185530,4450
176000,5250
114800,5350
166800,5800
89000,5990
144500,5999
84000,6200
82029,6390
63060,6390
74000,6600
97500,6800
67000,6800
76025,6900
48235,6900
93000,6990
60949,7490
65674,7555
54000,7990
68500,7990
22899,7990
61789,8290
I've tried various approaches, like normalizing the data set, not normalizing it, leaving thetas as is, denormalizing thetas... But I cannot get the correct result.
My maths must be off somewhere, but I cannot figure out where.
The result I'm trying to get should be approximately t0 = 8500, t1 = -0.02
My implementation is the following:
package main
import (
"encoding/csv"
"fmt"
"log"
"math"
"os"
"strconv"
)
const (
dataFile = "data.csv"
iterations = 20000
learningRate = 0.1
)
type dataSet [][]float64
var minKm, maxKm, minPrice, maxPrice float64
func (d dataSet) getExtremes(column int) (float64, float64) {
min := math.Inf(1)
max := math.Inf(-1)
for _, row := range d {
item := row[column]
if item > max {
max = item
}
if item < min {
min = item
}
}
return min, max
}
func normalizeItem(item, min, max float64) float64 {
return (item - min) / (max - min)
}
func (d *dataSet) normalize() {
minKm, maxKm = d.getExtremes(0)
minPrice, maxPrice = d.getExtremes(1)
for _, row := range *d {
row[0], row[1] = normalizeItem(row[0], minKm, maxKm), normalizeItem(row[1], minPrice, maxPrice)
}
}
func processEntry(entry []string) []float64 {
if len(entry) != 2 {
log.Fatalln("expected two fields")
}
km, err := strconv.ParseFloat(entry[0], 64)
if err != nil {
log.Fatalln(err)
}
price, err := strconv.ParseFloat(entry[1], 64)
if err != nil {
log.Fatalln(err)
}
return []float64{km, price}
}
func getData() dataSet {
file, err := os.Open(dataFile)
if err != nil {
log.Fatalln(err)
}
reader := csv.NewReader(file)
entries, err := reader.ReadAll()
if err != nil {
log.Fatalln(err)
}
entries = entries[1:]
data := make(dataSet, len(entries))
for k, entry := range entries {
data[k] = processEntry(entry)
}
return data
}
func outputResult(theta0, theta1 float64) {
file, err := os.OpenFile("weights.csv", os.O_WRONLY, 0644)
if err != nil {
log.Fatalln(err)
}
defer file.Close()
file.Truncate(0)
file.Seek(0, 0)
file.WriteString(fmt.Sprintf("theta0,%.6f\ntheta1,%.6f\n", theta0, theta1))
}
func estimatePrice(theta0, theta1, mileage float64) float64 {
return theta0 + theta1*mileage
}
func (d dataSet) computeThetas(theta0, theta1 float64) (float64, float64) {
dataSize := float64(len(d))
t0sum, t1sum := 0.0, 0.0
for _, it := range d {
mileage := it[0]
price := it[1]
err := estimatePrice(theta0, theta1, mileage) - price
t0sum += err
t1sum += err * mileage
}
return theta0 - (t0sum / dataSize * learningRate), theta1 - (t1sum / dataSize * learningRate)
}
func denormalize(theta, min, max float64) float64 {
return theta*(max-min) + min
}
func main() {
data := getData()
data.normalize()
theta0, theta1 := 0.0, 0.0
for k := 0; k < iterations; k++ {
theta0, theta1 = data.computeThetas(theta0, theta1)
}
theta0 = denormalize(theta0, minKm, maxKm)
theta1 = denormalize(theta1, minPrice, maxPrice)
outputResult(theta0, theta1)
}
What should I fix in order to properly implement a gradient descent?
Linear Regression is really simple:
// yi = alpha + beta*xi + ei
func linearRegression(x, y []float64) (float64, float64) {
EX := expected(x)
EY := expected(y)
EXY := expectedXY(x, y)
EXX := expectedXY(x, x)
covariance := EXY - EX*EY
variance := EXX - EX*EX
beta := covariance / variance
alpha := EY - beta*EX
return alpha, beta
}
Try it here, Output:
8499.599649933218 -0.021448963591702314 396270.87871142407
Code:
package main
import (
"encoding/csv"
"fmt"
"strconv"
"strings"
)
func main() {
x, y := readXY(`data.csv`)
alpha, beta := linearRegression(x, y)
fmt.Println(alpha, beta, -alpha/beta) // 8499.599649933218 -0.021448963591702314 396270.87871142407
}
// https://en.wikipedia.org/wiki/Ordinary_least_squares#Simple_linear_regression_model
// yi = alpha + beta*xi + ei
func linearRegression(x, y []float64) (float64, float64) {
EX := expected(x)
EY := expected(y)
EXY := expectedXY(x, y)
EXX := expectedXY(x, x)
covariance := EXY - EX*EY
variance := EXX - EX*EX
beta := covariance / variance
alpha := EY - beta*EX
return alpha, beta
}
// E[X]
func expected(x []float64) float64 {
sum := 0.0
for _, v := range x {
sum += v
}
return sum / float64(len(x))
}
// E[XY]
func expectedXY(x, y []float64) float64 {
sum := 0.0
for i, v := range x {
sum += v * y[i]
}
return sum / float64(len(x))
}
func readXY(filename string) ([]float64, []float64) {
// file, err := os.Open(filename)
// if err != nil {
// panic(err)
// }
// defer file.Close()
file := strings.NewReader(data)
reader := csv.NewReader(file)
records, err := reader.ReadAll()
if err != nil {
panic(err)
}
records = records[1:]
size := len(records)
x := make([]float64, size)
y := make([]float64, size)
for i, v := range records {
val, err := strconv.ParseFloat(v[0], 64)
if err != nil {
panic(err)
}
x[i] = val
val, err = strconv.ParseFloat(v[1], 64)
if err != nil {
panic(err)
}
y[i] = val
}
return x, y
}
var data = `km,price
240000,3650
139800,3800
150500,4400
185530,4450
176000,5250
114800,5350
166800,5800
89000,5990
144500,5999
84000,6200
82029,6390
63060,6390
74000,6600
97500,6800
67000,6800
76025,6900
48235,6900
93000,6990
60949,7490
65674,7555
54000,7990
68500,7990
22899,7990
61789,8290`
Gradient descent is based on the observation that if the multi-variable function F(x) is defined and differentiable in a neighborhood of a point a , then F(x) decreases fastest if one goes from a in the direction of the negative gradient of F at a,-∇F(a), for example:
// F(x)
f := func(x float64) float64 {
return alpha + beta*x // write your target function here
}
Derivative function:
h := 0.000001
// Derivative function ∇F(x)
df := func(x float64) float64 {
return (f(x+h) - f(x-h)) / (2 * h) // write your target function derivative here
}
Search:
minimunAt := 1.0 // We start the search here
gamma := 0.01 // Step size multiplier
precision := 0.0000001 // Desired precision of result
max := 100000 // Maximum number of iterations
currentX := 0.0
step := 0.0
for i := 0; i < max; i++ {
currentX = minimunAt
minimunAt = currentX - gamma*df(currentX)
step = minimunAt - currentX
if math.Abs(step) <= precision {
break
}
}
fmt.Printf("Minimum at %.8f value: %v\n", minimunAt, f(minimunAt))

How to make fmt.Scanln() read into a slice of integers

I have a line containing 3 numbers that I want to read from stdin with fmt.Scanln() but this code won't work:
X := make([]int, 3)
fmt.Scanln(X...)
fmt.Printf("%v\n", X)
I get this error message:
cannot use X (type []int) as type []interface {} in function argument
I don't get it.
Idiomatic Go would be:
func read(n int) ([]int, error) {
in := make([]int, n)
for i := range in {
_, err := fmt.Scan(&in[i])
if err != nil {
return in[:i], err
}
}
return in, nil
}
interface{} means nothing. Please don't use it if you don't have to.
For example,
package main
import "fmt"
func intScanln(n int) ([]int, error) {
x := make([]int, n)
y := make([]interface{}, len(x))
for i := range x {
y[i] = &x[i]
}
n, err := fmt.Scanln(y...)
x = x[:n]
return x, err
}
func main() {
x, err := intScanln(3)
if err != nil {
fmt.Println(err)
return
}
fmt.Printf("%v\n", x)
}
Input:
1 2 3
Output:
[1 2 3]
I think the the correct version should be
X := make([]int, 3)
fmt.Scanln(&X[0], &X[1], &X[2])
fmt.Printf("%v\n", X)
This error message occurs b/c there's no reasonable way to convert []int to []interface{}. Note, this is in reference to a slice. So the syntax your using is correct, but fmt.Scanln expects []interface{}. This has implications outside of pkg fmt.
The reason I've seen given for this is due to Go giving you control over memory layout so it currently has no reasonable way to do the slice conversion. This means you'll need to do the conversion manually before passing it to a function expecting the slice of a given type. For example:
package main
import (
"fmt"
)
func main() {
x := make([]int, 3)
y := make([]interface{}, 3)
y[0] = x[0]
y[1] = x[1]
y[2] = x[2]
fmt.Println(y...)
}
Or something a little more general:
x := make([]int, 3)
y := make([]interface{}, len(x))
for i, v := range x {
y[i] = v
}
fmt.Println(y...)
Regarding your specific issue, see the following:
x := make([]*int, 3)
for i := range x {
x[i] = new(int)
}
y := make([]interface{}, 3)
for i, v := range x {
y[i] = v
}
if _, err := fmt.Scanln(y...); err != nil {
fmt.Println("Scanln err: ", err)
}
for _, v := range y {
val := v.(*int)
fmt.Println(*val)
}
I saw in a comment you said the lines can have different lengths. In that case
you can implement your own fmt.Scanner:
package main
import (
"bytes"
"fmt"
)
type slice struct {
tok []int
}
func (s *slice) Scan(state fmt.ScanState, verb rune) error {
tok, err := state.Token(false, func(r rune) bool { return r != '\n' })
if err != nil { return err }
if _, _, err := state.ReadRune(); err != nil {
if len(tok) == 0 {
panic(err)
}
}
b := bytes.NewReader(tok)
for {
var d int
_, err := fmt.Fscan(b, &d)
if err != nil { break }
s.tok = append(s.tok, d)
}
return nil
}
func main() {
var s slice
fmt.Scan(&s)
fmt.Println(s.tok)
}
https://golang.org/pkg/fmt#Scanner

Resources