read csv into [][]byte - go
I experienced some weird behavior when I was reading a csv file into 2 dimensional byte slice. The first 42 rows are fine and after that it seems like extra line ending are put into the data which messes up things:
first row in the first 42 times:
row 0: 504921600000000000,truck_0,South,Trish,H-2,v2.3,1500,150,12,52.31854,4.72037,124,0,221,0,25
first row after I appended 43 rows:
row 0: 504921600000000000,truck_49,South,Andy,F-150,v2.0,2000,200,15,38.9349,179.94282,289,0,269,0
row 1: 25
minimal code to reproduce the problem:
package main
import (
"bufio"
"log"
"os"
)
type fileDataSource struct {
scanner *bufio.Scanner
}
type batch struct {
rows [][]byte
}
func (b *batch) Len() uint {
return uint(len(b.rows))
}
func (b *batch) Append(row []byte) {
b.rows = append(b.rows, row)
for index, row := range b.rows {
log.Printf("row %d: %s\n", index, string(row))
}
if len(b.rows) > 43 {
log.Fatalf("asdf")
}
}
type factory struct{}
func (f *factory) New() *batch {
return &batch{rows: make([][]byte, 0)}
}
func main() {
file, _ := os.Open("/tmp/data1.csv")
scanner := bufio.NewScanner(bufio.NewReaderSize(file, 4<<20))
b := batch{}
for scanner.Scan() {
b.Append(scanner.Bytes())
}
}
csv I used:
504921600000000000,truck_0,South,Trish,H-2,v2.3,1500,150,12,52.31854,4.72037,124,0,221,0,25
504921600000000000,truck_1,South,Albert,F-150,v1.5,2000,200,15,72.45258,68.83761,255,0,181,0,25
504921600000000000,truck_2,North,Derek,F-150,v1.5,2000,200,15,24.5208,28.09377,428,0,304,0,25
504921600000000000,truck_3,East,Albert,F-150,v2.0,2000,200,15,18.11037,98.65573,387,0,192,0,25
504921600000000000,truck_4,West,Andy,G-2000,v1.5,5000,300,19,81.93919,56.12266,236,0,335,0,25
504921600000000000,truck_5,East,Seth,F-150,v2.0,2000,200,15,5.00552,114.50557,89,0,187,0,25
504921600000000000,truck_6,East,Trish,G-2000,v1.0,5000,300,19,41.59689,57.90174,395,0,150,0,25
504921600000000000,truck_8,South,Seth,G-2000,v1.0,5000,300,19,21.89157,44.58919,411,0,232,0,25
504921600000000000,truck_9,South,Andy,H-2,v2.3,1500,150,12,15.67271,112.4023,402,0,75,0,25
504921600000000000,truck_10,North,Albert,F-150,v2.3,2000,200,15,35.05682,36.20513,359,0,68,0,25
504921600000000000,truck_7,East,Andy,H-2,v2.0,1500,150,12,7.74826,14.96075,105,0,323,0,25
504921600000000000,truck_11,South,Derek,F-150,v1.0,2000,200,15,87.9924,134.71544,293,0,133,0,25
504921600000000000,truck_14,North,Albert,H-2,v1.0,1500,150,12,66.68217,105.76965,222,0,252,0,25
504921600000000000,truck_18,West,Trish,F-150,v2.0,2000,200,15,67.15164,153.56165,252,0,240,0,25
504921600000000000,truck_20,North,Rodney,G-2000,v2.0,5000,300,19,38.88807,65.86698,104,0,44,0,25
504921600000000000,truck_21,East,Derek,G-2000,v2.0,5000,300,19,81.87812,167.8083,345,0,327,0,25
504921600000000000,truck_22,West,Albert,G-2000,v1.5,5000,300,19,39.9433,16.0241,449,0,42,0,25
504921600000000000,truck_23,South,Andy,F-150,v2.0,2000,200,15,73.28358,98.05159,198,0,276,0,25
504921600000000000,truck_24,West,Rodney,G-2000,v2.3,5000,300,19,22.19262,0.27462,223,0,318,0,25
504921600000000000,truck_25,North,Trish,F-150,v2.0,2000,200,15,17.26704,16.91226,461,0,183,0,25
504921600000000000,truck_26,South,Seth,F-150,v1.5,2000,200,15,45.65327,144.60354,58,0,182,0,25
504921600000000000,truck_12,East,Trish,G-2000,v1.0,5000,300,19,36.03928,113.87118,39,0,294,0,25
504921600000000000,truck_13,West,Derek,H-2,v1.0,1500,150,12,14.07479,110.77267,152,0,69,0,25
504921600000000000,truck_27,West,Seth,G-2000,v1.5,5000,300,19,79.55971,97.86182,252,0,345,0,25
504921600000000000,truck_28,West,Rodney,G-2000,v1.5,5000,300,19,60.33457,4.62029,74,0,199,0,25
504921600000000000,truck_16,South,Albert,G-2000,v1.5,5000,300,19,51.16438,121.32451,455,0,290,0,25
504921600000000000,truck_19,West,Derek,G-2000,v1.5,5000,300,19,19.69355,139.493,451,0,300,0,25
504921600000000000,truck_31,North,Albert,G-2000,v1.0,5000,300,19,0.75251,116.83474,455,0,49,0,25
504921600000000000,truck_32,West,Seth,F-150,v2.0,2000,200,15,4.07566,164.43909,297,0,277,0,25
504921600000000000,truck_33,West,Rodney,G-2000,v1.5,5000,300,19,89.19448,10.47499,407,0,169,0,25
504921600000000000,truck_34,West,Rodney,G-2000,v2.0,5000,300,19,73.7383,10.79582,488,0,170,0,25
504921600000000000,truck_35,West,Seth,G-2000,v2.3,5000,300,19,60.02428,2.51011,480,0,307,0,25
504921600000000000,truck_36,North,Andy,G-2000,v1.0,5000,300,19,87.52877,45.07308,161,0,128,0,25
504921600000000000,truck_38,West,Andy,H-2,v2.3,,150,12,63.54604,119.82031,282,0,325,0,25
504921600000000000,truck_39,East,Derek,G-2000,v1.5,5000,300,19,33.83548,3.90996,294,0,123,0,25
504921600000000000,truck_40,West,Albert,H-2,v2.0,1500,150,12,32.32773,118.43138,276,0,316,0,25
504921600000000000,truck_41,East,Rodney,F-150,v1.0,2000,200,15,68.85572,173.23123,478,0,207,0,25
504921600000000000,truck_42,West,Trish,F-150,v2.0,2000,200,15,38.45195,171.2884,113,0,180,0,25
504921600000000000,truck_43,East,Derek,H-2,v2.0,1500,150,12,52.90189,49.76966,295,0,195,0,25
504921600000000000,truck_44,South,Seth,H-2,v1.0,1500,150,12,32.33297,3.89306,396,0,320,0,25
504921600000000000,truck_30,East,Andy,G-2000,v1.5,5000,300,19,29.62198,83.73482,291,0,267,0,25
504921600000000000,truck_46,West,Seth,H-2,v2.3,1500,150,12,26.07966,118.49629,321,,267,0,25
504921600000000000,truck_37,South,Andy,G-2000,v2.0,5000,300,19,57.90077,77.20136,77,0,179,0,25
504921600000000000,truck_49,South,Andy,F-150,v2.0,2000,200,15,38.9349,179.94282,289,0,269,0,25
504921600000000000,truck_53,West,Seth,G-2000,v2.3,5000,300,19,25.02,157.45082,272,0,5,0,25
504921600000000000,truck_54,North,Andy,H-2,v2.0,1500,150,12,87.62736,106.0376,360,0,66,0,25
504921600000000000,truck_55,East,Albert,G-2000,v1.0,5000,300,19,78.56605,71.16225,295,0,150,0,25
504921600000000000,truck_56,North,Derek,F-150,v2.0,2000,200,15,23.51619,123.22682,71,0,209,0,25
504921600000000000,truck_57,South,Rodney,F-150,v2.3,2000,200,15,26.07996,159.92716,454,0,22,0,25
504921600000000000,truck_58,South,Derek,F-150,v2.0,2000,200,15,84.79333,79.23813,175,0,246,0,25
504921600000000000,truck_59,East,Andy,H-2,v2.0,1500,150,12,8.7621,82.48318,82,0,55,0,25
504921600000000000,truck_45,East,Trish,G-2000,v1.0,5000,300,19,17.48624,100.78121,306,0,193,0,25
504921600000000000,truck_47,South,Derek,G-2000,v1.5,5000,300,19,41.62173,110.80422,111,0,78,0,25
504921600000000000,truck_48,East,Trish,G-2000,v1.5,5000,300,19,63.90773,141.50555,53,0,,0,25
504921600000000000,truck_50,East,Andy,H-2,v2.3,1500,150,12,45.44111,172.39833,219,0,88,0,25
504921600000000000,truck_51,East,Rodney,F-150,v2.3,2000,200,15,89.03645,91.57675,457,0,337,0,25
504921600000000000,truck_52,West,Derek,G-2000,v1.0,5000,300,19,89.0133,97.8037,23,0,168,0,25
504921600000000000,truck_61,East,Albert,G-2000,v2.3,5000,300,19,75.91676,167.78366,462,0,60,0,25
504921600000000000,truck_62,East,Derek,H-2,v1.5,1500,150,12,54.61668,103.21398,231,0,143,0,25
504921600000000000,truck_63,South,Rodney,H-2,v2.0,1500,150,12,37.13702,149.25546,46,0,118,0,25
504921600000000000,truck_64,South,Albert,G-2000,v2.0,5000,300,19,45.04214,10.73002,447,0,253,0,25
504921600000000000,truck_60,South,Derek,H-2,v1.5,1500,150,12,57.99184,33.45994,310,0,93,0,25
504921600000000000,truck_67,South,Seth,H-2,v1.0,1500,150,12,4.62985,155.01707,308,0,22,0,25
504921600000000000,truck_68,West,Rodney,F-150,v1.5,2000,200,15,16.90741,123.03863,303,0,43,0,25
504921600000000000,truck_69,East,Derek,H-2,v2.3,1500,150,12,79.88424,120.79121,407,0,138,0,25
504921600000000000,truck_70,North,Albert,H-2,v2.0,1500,150,12,77.87592,164.70924,270,0,21,0,25
504921600000000000,truck_71,West,Seth,G-2000,v2.3,5000,300,19,72.75635,78.0365,391,0,32,0,25
504921600000000000,truck_73,North,Seth,F-150,v1.5,2000,200,15,37.67468,91.09732,489,0,103,0,25
504921600000000000,truck_74,North,Trish,H-2,v1.0,1500,150,12,41.4456,158.13897,206,0,79,0,25
504921600000000000,truck_75,South,Andy,F-150,v1.5,2000,200,15,4.11709,175.65994,378,0,176,0,25
504921600000000000,truck_66,South,Seth,G-2000,v2.0,5000,300,19,42.24286,151.8978,227,0,67,0,25
504921600000000000,truck_72,South,Andy,G-2000,v2.3,5000,300,19,82.46228,2.44504,487,0,39,0,25
504921600000000000,truck_76,South,Rodney,F-150,v2.3,2000,200,15,71.62798,121.89842,283,0,164,0,25
504921600000000000,truck_78,South,Seth,F-150,v2.0,2000,200,15,13.96218,39.04615,433,0,326,0,25
504921600000000000,truck_79,South,Andy,G-2000,v2.0,5000,300,19,56.54137,,46,0,127,0,25
504921600000000000,truck_81,West,Rodney,G-2000,v2.3,5000,300,19,59.42624,115.59744,68,0,296,0,25
504921600000000000,truck_83,South,Albert,F-150,v2.0,2000,200,15,49.20261,115.98262,449,0,132,0,25
504921600000000000,truck_84,West,Derek,H-2,v1.0,1500,150,12,70.16476,59.05399,301,0,134,0,25
504921600000000000,truck_85,West,Derek,G-2000,v1.0,5000,300,19,11.75251,142.86513,358,0,339,0,25
504921600000000000,truck_86,West,Rodney,G-2000,v1.0,5000,300,19,30.92821,127.53274,367,0,162,0,25
504921600000000000,truck_87,West,Rodney,H-2,v2.0,1500,150,12,32.86913,155.7666,122,0,337,0,25
504921600000000000,truck_88,West,Andy,G-2000,v1.5,5000,300,19,60.03367,9.5707,204,0,333,0,25
504921600000000000,truck_80,East,Andy,G-2000,v2.3,5000,300,,46.13937,137.42962,295,0,290,0,25
504921600000000000,truck_91,East,Derek,F-150,v2.0,2000,200,15,7.13401,52.78885,100,0,147,0,25
504921600000000000,truck_93,North,Derek,G-2000,v2.0,5000,300,19,11.46065,20.57173,242,0,148,0,25
504921600000000000,truck_94,North,Derek,F-150,v1.0,2000,200,15,59.53287,26.98247,427,0,341,0,25
504921600000000000,truck_95,East,Albert,G-2000,v2.0,5000,300,19,37.31513,134.40078,383,0,121,0,25
504921600000000000,truck_96,East,Albert,G-2000,v1.5,5000,300,19,15.78803,146.68255,348,0,189,0,25
504921600000000000,truck_97,South,Seth,F-150,v1.0,2000,200,15,14.08559,18.49763,369,0,34,0,25
504921600000000000,truck_98,South,Albert,G-2000,v1.5,5000,300,19,15.1474,71.85194,89,0,238,0,25
504921600000000000,truck_77,East,Trish,F-150,v2.0,2000,200,15,80.5734,17.68311,389,0,218,0,25
504921600000000000,truck_82,West,Derek,H-2,v2.0,1500,150,12,57.00976,90.13642,102,0,296,0,25
504921600000000000,truck_92,North,Derek,H-2,v1.0,1500,150,12,54.40335,153.5809,123,0,150,0,25
504921600000000000,truck_99,West,Trish,G-2000,v1.5,5000,300,19,62.73061,26.1884,309,0,202,0,25
504921610000000000,truck_1,South,Albert,F-150,v1.5,2000,200,15,72.45157,68.83919,259,0,180,2,27.5
504921610000000000,truck_2,North,Derek,F-150,v1.5,2000,200,15,24.5195,28.09369,434,6,302,0,22.1
504921610000000000,truck_3,East,Albert,F-150,v2.0,2000,200,15,18.107,98.66002,390,,190,0,21.2
504921610000000000,truck_4,West,Andy,G-2000,v1.5,5000,300,19,81.9438,56.12717,244,8,334,2,27.6
504921610000000000,truck_5,East,Seth,F-150,v2.0,2000,200,15,5.00695,114.50676,92,7,183,2,28.5
504921610000000000,truck_6,East,Trish,G-2000,v1.0,5000,300,19,41.59389,57.90166,403,0,149,0,22.7
504921610000000000,truck_7,East,Andy,H-2,v2.0,1500,150,12,7.74392,14.95756,,0,320,0,28.2
504921610000000000,truck_12,East,Trish,G-2000,v1.0,5000,300,19,36.03979,113.8752,34,0,293,1,26.3
504921610000000000,truck_13,West,Derek,H-2,v1.0,1500,150,12,14.07315,110.77235,150,0,72,,21.9
504921610000000000,truck_14,North,Albert,H-2,v1.0,1500,150,12,,105.76727,218,5,253,1,21.9
504921610000000000,truck_15,South,Albert,H-2,v1.5,1500,150,12,6.78254,166.86685,5,0,110,0,26.3
504921610000000000,truck_16,South,Albert,G-2000,v1.5,5000,300,19,51.16405,121.32556,445,0,294,3,29.9
504921610000000000,truck_17,West,Derek,H-2,v1.5,1500,150,12,8.12913,56.57343,9,0,6,4,29
504921610000000000,truck_18,West,Trish,F-150,v2.0,2000,200,15,67.15167,153.56094,260,1,239,1,23.3
504921610000000000,truck_19,West,Derek,G-2000,v1.5,5000,300,19,19.69456,139.49545,448,4,298,0,29.9
504921610000000000,truck_20,North,Rodney,G-2000,v2.0,5000,300,19,38.88968,65.86504,103,0,41,1,23.6
504921610000000000,truck_21,East,Derek,G-2000,v2.0,5000,300,19,81.88232,167.81287,345,0,326,0,20.8
504921610000000000,truck_0,South,Trish,H-2,v2.3,1500,150,12,52.32335,4.71786,128,9,225,0,25.8
504921610000000000,truck_22,West,Albert,G-2000,v1.5,5000,300,19,39.94345,16.02353,440,1,45,0,27.8
504921610000000000,truck_8,South,Seth,G-2000,v1.0,5000,300,19,21.89464,44.58628,402,0,234,0,20.3
504921610000000000,truck_23,South,Andy,F-150,v2.0,2000,200,15,73.28131,98.05635,201,7,277,0,25.3
504921610000000000,truck_24,West,Rodney,G-2000,v2.3,5000,300,19,22.19506,0.27702,217,0,321,2,29.5
504921610000000000,truck_9,South,Andy,H-2,v2.3,1500,150,12,,112.40429,402,9,75,4,29.5
504921610000000000,truck_26,South,Seth,F-150,v1.5,2000,200,15,45.65798,144.60844,59,1,183,0,21.7
504921610000000000,truck_27,West,Seth,G-2000,v1.5,5000,300,19,79.55699,97.86561,255,7,348,2,20.2
504921610000000000,truck_25,North,Trish,F-150,v2.0,2000,200,15,17.26506,16.91691,453,8,186,0,24.3
504921610000000000,truck_28,West,Rodney,G-2000,v1.5,5000,300,19,60.33272,4.61578,84,3,198,0,23.1
504921610000000000,truck_29,East,Rodney,G-2000,v2.0,5000,300,19,80.30331,146.54254,340,5,118,0,25.6
504921610000000000,truck_30,East,Andy,G-2000,v1.5,5000,300,19,29.62434,83.73246,300,0,270,4,22.3
504921610000000000,truck_33,West,Rodney,G-2000,v1.5,5000,300,19,89.19593,10.47733,403,8,170,0,29.6
504921610000000000,truck_36,North,Andy,G-2000,v1.0,5000,300,19,87.53087,45.07276,163,0,132,1,27.6
I expected the rows [][]byte contain the csv data row by row
As already suggested, you really should look to use encoding/csv.
That said, the reason for your issue is explained in the godoc above the Bytes() function:
// Bytes returns the most recent token generated by a call to Scan.
// The underlying array may point to data that will be overwritten
// by a subsequent call to Scan. It does no allocation.
func (s *Scanner) Bytes() []byte {
return s.token
}
So the returned byte slice may be modified by subsequent calls to Scan(). To avoid this, you'd need to make a copy of the byte slice, e.g.
for scanner.Scan() {
row := scanner.Bytes()
bs := make([]byte, len(row))
copy(bs, row)
b.Append(bs)
}
You need to create a copy of the data returned by Bytes.
https://pkg.go.dev/bufio#go1.19.3#Scanner.Bytes
Bytes returns the most recent token generated by a call to Scan. The underlying array may point to data that will be overwritten by a subsequent call to Scan. It does no allocation.
for scanner.Scan() {
row := make([]byte, len(scanner.Bytes()))
copy(row, scanner.Bytes())
b.Append(row)
}
https://go.dev/play/p/Lqot-wOXiwh
Related
Append to golang slice passed as empty interface
How to append to empty interface (that has been verified to be a *[]struct)?
func main() {
var mySlice []myStruct // myStruct can be any struct (dynamic)
decode(&mySlice, "...")
}
func decode(dest interface{}, src string) {
// assume dest has been verified to be *[]struct
var modelType reflect.Type = getStructType(dest)
rows, fields := getRows(src)
for _, row := range rows {
// create new struct of type modelType and assign all fields
model := reflect.New(modelType)
for field := fields {
fieldValue := getRowValue(row, field)
model.Elem().FieldByName(field).Set(fieldValue)
}
castedModelRow := model.Elem().Interface()
// append model to dest; how to do this?
// dest = append(dest, castedModelRow)
}
}
Things I've tried:
This simply panics: reflect: call of reflect.Append on ptr Value (as we pass &mySlice instead of mySlice)
dest = reflect.Append(reflect.ValueOf(dest), reflect.ValueOf(castedModelRow))
This works but doesn't set the value back to dest... in main func, len(mySlice) remains 0 after decode function is called.
func decode(dest interface{}, src string) {
...
result := reflect.MakeSlice(reflect.SliceOf(modelType), rowCount, rowCount)
for _, row : range rows {
...
result = reflect.Append(result, reflect.ValueOf(castedModelRow))
}
dest = reflect.ValueOf(result)
}
Here's how to fix the second decode function shown in the question. The statement
dest = reflect.ValueOf(result)
modifies local variable dest, not the caller's value. Use the following statement to modify the caller's slice:
reflect.ValueOf(dest).Elem().Set(result)
The code in the question appends decoded elements after the elements created in reflect.MakeSlice. The resulting slice has len(rows) zero values followed by len(rows) decoded values. Fix by changing
result = reflect.Append(result, reflect.ValueOf(castedModelRow))
to:
result.Index(i).Set(model)
Here's the update version of the second decode function in the question:
func decode(dest interface{}, src string) {
var modelType reflect.Type = getStructType(dest)
rows, fields := getRows(src)
result := reflect.MakeSlice(reflect.SliceOf(modelType), len(rows), len(rows))
for i, row := range rows {
model := reflect.New(modelType).Elem()
for _, field := range fields {
fieldValue := getRowValue(row, field)
model.FieldByName(field).Set(fieldValue)
}
result.Index(i).Set(model)
}
reflect.ValueOf(dest).Elem().Set(result)
}
Run it on the Playground.
You were very close with your original solution. You had to de-reference the pointer before calling the append operation. This solution would be helpful if your dest already had some existing elements and you don't want to lose them by creating a newSlice.
tempDest := reflect.ValueOf(dest).Elem()
tempDest = reflect.Append(tempDest, reflect.ValueOf(model.Interface()))
Similar to how #I Love Reflection pointed out, you finally need to set the new slice back to the pointer.
reflect.ValueOf(dest).Elem().Set(tempDest)
Overall Decode:
var modelType reflect.Type = getStructType(dest)
rows, fields := getRows(src)
tempDest := reflect.ValueOf(dest).Elem()
for _, row := range rows {
model := reflect.New(modelType).Elem()
for _, field := range fields {
fieldValue := getRowValue(row, field)
model.FieldByName(field).Set(fieldValue)
}
tempDest = reflect.Append(tempDest, reflect.ValueOf(model.Interface()))
}
reflect.ValueOf(dest).Elem().Set(tempDest)
Why [capacity]string assert to []string will be failed in Golang?
I am using Golang1.14.
Here is the test code.
package main
import "time"
func checkData(data interface{}) {
if _, ok := data.([]string); ok {
println("Assert true.")
} else {
println("Assert false.")
}
}
func main() {
var years [20]string
for i := 0; i < 20; i++ {
years[i] = string(time.Now().Year() - 10 + i)
}
checkData(years)
foods := []string{"Fruit", "Grass", "Fish", "Meat"}
checkData(foods)
}
The output is:
Assert false.
Assert true.
I am new to Golang and really confusing that [20]string is not a []string.Can someone tell me why?Thanks.
[20]string is an array. It is a type that contains 20 strings, and if you pass it as an interface{}, you can recover it using intf.([20]string).
[]string is a slice. It has a backing array, but it is essentially a view over an array. You assertion checks if the interface is a slice, so this one works.
Arrays and slices are different things in Go. An array is a data type with a fixed size. For instance:
func f(arr [10]int) {...}
You can only call f with an int array of size 10. When you do call it, the array will be passes as value, so the function will get a copy of the array, all 10 members of it. But:
func f(arr []int) {...}
You can call f with any size of slice. A slice contains a reference to its underlying array, so an array copy will not take place here. You cannot call thisf` with an array.
golang get char* as return value from dll
I'm using golang to call a Dll function like char* fn(), the dll is not written by myself and I cannot change it. Here's my code:
package main
import (
"fmt"
"syscall"
"unsafe"
)
func main() {
dll := syscall.MustLoadDLL("my.dll")
fn := dll.MustFindProc("fn")
r, _, _ := fn.Call()
p := (*byte)(unsafe.Pointer(r))
// define a slice to fill with the p string
data := make([]byte, 0)
// loop until find '\0'
for *p != 0 {
data = append(data, *p) // append 1 byte
r += unsafe.Sizeof(byte(0)) // move r to next byte
p = (*byte)(unsafe.Pointer(r)) // get the byte value
}
name := string(data) // convert to Golang string
fmt.Println(name)
}
I have some questions:
Is there any better way of doing this? There're hundred of dll functions like this, I'll have to write the loop for all functions.
For very-long-string like 100k+ bytes, will append() cause performance issue?
Solved. the unsafe.Pointer(r) causes linter govet shows warning possible misuse of unsafe.Pointer, but the code runs fine, how to avoid this warning? Solution: This can be solved by adding -unsafeptr=false to govet command line, for vim-ale, add let g:ale_go_govet_options = '-unsafeptr=false'.
Casting uintptr as upointer is haram. You must read the rules: https://golang.org/pkg/unsafe/#Pointer But there's hacky way, that shouldn't produce warning: //go:linkname gostringn runtime.gostringn func gostringn(p uintptr, l int) string //go:linkname findnull runtime.findnull //go:nosplit func findnull(s uintptr) int // .... name := gostringn(r, findnull(r)) Functions takes pointer, but we link them from runtime as uintptr because they have same sizeof. Might work in theory. But is also frowned upon. Getting back to your code, as JimB said, you could do it one line with: name := C.GoString((*C.char)(unsafe.Pointer(r)))
I got the following solution by tracking the os.Args of the go source code, But I am based on go1.17. If you are in another version, you can read the source code to solve it.
func UintPtrToString(r uintptr) string {
p := (*uint16)(unsafe.Pointer(r))
if p == nil {
return ""
}
n, end, add := 0, unsafe.Pointer(p), unsafe.Sizeof(*p)
for *(*uint16)(end) != 0 {
end = unsafe.Add(end, add)
n++
}
return string(utf16.Decode(unsafe.Slice(p, n)))
}
Expanding a slice's size to prevent slice bounds out of range error
I have written the following:
func main() {
//inside main
fileInputBytes, err := ioutil.ReadFile("/tmp/test")
byteSize2 := len(fileInputBytes)
var inputFileByteSlice = fileInputBytes[0:]
var numberOfIndexes = math.Floor(float64(byteSize / indexingOffset))
for i := 1; i <= int(numberOfIndexes); i++ {
// adding i to the indexer insures that we use lookahed to ignore previously inserted indexing values
var v int = (i * indexingOffset) + i
Insert(&inputFileByteSlice, v+i, indexingByteValue)
fmt.Println(i)
}
}
//outside main
//variation of https://blog.golang.org/slices with pointers and such
func Insert(slice *[]byte, index int, value byte) {
// Grow the slice by one element.
(*slice) = (*slice)[0 : len(*slice)+1]
// Use copy to move the upper part of the slice out of the way and open a hole.
copy((*slice)[index+1:], (*slice)[index:])
// Store the new value.
(*slice)[index] = value
// Return the result.
}
The slice bounds out of range error is getting on my nerves. The length of the slice grows outside of the size and overflows, the reason I don't understand is that I thought the call to 'grow' the slice by one element(before copy) will dynamically allocate more space. Since that is not the case, can anyone offer me a better suggestion?
First of all, a slice is already a reference type. So you don't need to pass its pointer around if you are not going to change its capacity. So your main can be simplified as:
func main() {
fileInputBytes, err := ioutil.ReadFile("/tmp/test")
byteSize2 := len(fileInputBytes)
// No need to use pointer to slice. If you want a brand new slice
// that does not affect the original slice values, use copy()
inputFileByteArray := fileInputBytes
var numberOfIndexes = math.Floor(float64(byteSize / indexingOffset))
for i := 1; i <= int(numberOfIndexes); i++ {
var v int = (i * indexingOffset) + i
// Insert needs to return the newly updated slice reference
// which should be assigned in each iteration.
inputFileByteArray = Insert(inputFileByteArray, v+i, indexingByteValue)
fmt.Println(i)
}
}
Then, the Insert function can be simplified simply by using append along with copy and returning the newly created slice:
func Insert(slice []byte, index int, value byte) []byte {
if index >= len(slice) {
// add to the end of slice in case of index >= len(slice)
return append(slice, value)
}
tmp := make([]byte, len(slice[:index + 1]))
copy(tmp, slice[:index])
tmp[index] = value
return append(tmp, slice[index:]...)
}
This may not be the best implementation but it is simple enough. Example usage at: https://play.golang.org/p/Nuq4RX9XQD
Your function only works if the slice happens to have enough initial capacity. If you need more capacity, you can only "grow" the slice using the append function. You can still use the *[]byte pointer argument to modify the slice in place like so:
func Insert(slice *[]byte, index int, value byte) {
*slice = append(*slice, 0)
copy((*slice)[index+1:], (*slice)[index:])
(*slice)[index] = value
}
However, it's more customary to return a new slice value, and reassign it each time. This gives you a similar function signature to the builtin append.
func Insert(slice []byte, index int, value byte) []byte {
slice = append(slice, 0)
copy(slice[index+1:], slice[index:])
slice[index] = value
return slice
}
Why Golang object property of byte array will be wiped when assigned to another variable
We need to wipe out some variables after use. But it seems really weird when it's assigned with a []byte field in a struct.
Why this assignment of []byte is not a copy but a pointer?
What should I do to keep the value in struct a.bs, but wipe out the b as local variable?
http://play.golang.org/p/MT_wAHj2OM
package main
import "fmt"
type so struct {
bs []byte
}
func zeroes(n int) []byte {
return make([]byte, n)
}
func wipeBytes(b []byte) {
copy(b, zeroes(len(b)))
}
func main() {
a := so{bs: []byte{0x01, 0x02}}
b := a.bs
wipeBytes(b)
fmt.Println(b) //b == []byte{}
fmt.Println(a.bs) //a.bs == []byte{}
}
Slices are inherently reference-y things. Assigning one doesn't copy its contents. You can think of a slice value as being a "slice head" structure, which contains a pointer to the slice's underlying array, and the offset and length of the slice within the array. It's this structure that's copied when you copy the slice, not any of the values in the array. You can do b := make([]byte, len(a.bs))) copy(b, a.bs) to make b a new slice and copy a.bs's contents into it. Then nothing you do to one will have any effect on the other.
When declaring/creating the 'array' ([]byte{0x01, 0x02}), you're not specifying a length ([2]byte{0x01, 0x02}), which means that it's a slice instead of an array. And slices objects internally contains a pointer to it's content.
func ObjectAssign(target interface{}, object interface{}) {
// object atributes values in target atributes values
// using pattern matching (https://golang.org/pkg/reflect/#Value.FieldByName)
// https://stackoverflow.com/questions/35590190/how-to-use-the-spread-operator-in-golang
t := reflect.ValueOf(target).Elem()
o := reflect.ValueOf(object).Elem()
for i := 0; i < o.NumField(); i++ {
for j := 0; j < t.NumField(); j++ {
if t.Field(j).Name() == o.Field(i).Name() {
t.Field(j).Set(o.Field(i))
}
}
}
}