I would like to read a slice of strings representing hexadecimal numbers, and decode them to a slice of byte slices ([]string --> [][]byte). This is my code so far:
func (self *algo_t) decode_args(args []string) ([][]byte, error) {
var data [][]byte
for i := uint32(0); i < self.num_args; i++ {
data = make([][]byte, self.num_args)
tmp, err := hex.DecodeString(args[i])
fmt.Printf("i = %d\ttmp = %x\n", i, tmp)
data[i] = make([]byte, len(tmp))
copy(data[i], tmp)
if err != nil {
fmt.Fprintf(os.Stderr, "Error decoding hex string %s: %s\n", args[i], err.Error())
return nil, err
}
}
fmt.Printf("line 69\tdata[0] = %x\tdata[1] = %x\tdata[2] = %x\n",data[0], data[1], data[2])
return data, nil
}
calling this code and passing args = []string{"010203","040506","070809"} yields the following output:
i = 0 tmp = 010203
i = 1 tmp = 040506
i = 3 tmp = 070809
line 69 data[0] = data[1] = data[2] = 070809
Presumably the function returns [][]byte{[]byte{}, []byte{}, []byte{0x07, 0x08, 0x09}}.
I understand that this is because of the pointer behavior of Go; what is the best practice for doing a deep copy of this kind?
For example,
package main
import (
"encoding/hex"
"fmt"
)
// Decode hex []string to [][]byte
func decode(s []string) ([][]byte, error) {
b := make([][]byte, len(s))
for i, ss := range s {
h, err := hex.DecodeString(ss)
if err != nil {
err = fmt.Errorf(
"Error decoding hex string %s: %s\n",
ss, err.Error(),
)
return nil, err
}
b[i] = h
}
return b, nil
}
func main() {
s := []string{"010203", "040506", "070809"}
fmt.Println(s)
b, err := decode(s)
if err != nil {
fmt.Println(err)
} else {
fmt.Println(b)
}
s = []string{"ABCDEF", "012345", "09AF"}
fmt.Println(s)
b, err = decode(s)
if err != nil {
fmt.Println(err)
} else {
fmt.Println(b)
}
s = []string{"01", "123XYZ"}
fmt.Println(s)
b, err = decode(s)
if err != nil {
fmt.Println(err)
} else {
fmt.Println(b)
}
}
Output:
[010203 040506 070809]
[[1 2 3] [4 5 6] [7 8 9]]
[ABCDEF 012345 09AF]
[[171 205 239] [1 35 69] [9 175]]
[01 123XYZ]
Error decoding hex string 123XYZ: encoding/hex: invalid byte: U+0058 'X'
There is a package built specifically to handle deep copy: http://godoc.org/code.google.com/p/rog-go/exp/deepcopy
You can look at the source here: https://code.google.com/p/rog-go/source/browse/exp/deepcopy/deepcopy.go. It covers copying slices and pointers, so it should cover your case.
Related
I have 2 yaml files that I've exported to 2 different map variables. My yaml files have this example of format:
file1.yaml
a: b
c: d
e:
- f
- g
x:
- y
file2.yaml
m: n
c: d
x:
- y
- z
I've unmarshalled the yaml files into structs. My goal is to now compare the 2 maps, and only output the difference found in file2.yaml. So far I have something simple:
type ParametersYaml struct {
Parameters Params `yaml:"parameters""`
}
type Params map[string]interface{}
var pParam ParametersYaml
var sParam ParametersYaml
func Extractor(primaryYaml, secondaryYaml string) error {
primaryOpenedFile, err := fileOpener(primaryYaml)
secondaryOpenedFile, err := fileOpener(secondaryYaml)
err = yamlParser(primaryOpenedFile, &pParam)
err = yamlParser(secondaryOpenedFile, &sParam)
if err != nil {
return err
}
compare(pParam, sParam)
return err
}
func fileOpener(file string) ([]byte, error) {
f, err := ioutil.ReadFile(file)
return f, err
}
func yamlParser(fileObject []byte, param *ParametersYaml) error {
err := yaml.Unmarshal(fileObject, ¶m)
return err
}
func compare(pMarshalled, sMarshalled ParametersYaml) {
for s, sv := range sMarshalled.Parameters {
for p, pv := range pMarshalled.Parameters {
if s == p && sv == pv {
break
} else if s == p && sv != pv {
fmt.Printf("%s: %s,\n", s, sv)
break
} else if _, ok := pMarshalled.Parameters[s]; !ok {
fmt.Printf("%s: %s,\n", s, sv)
break
}
}
}
}
But it can't compare values with lists since the type is of interface{}.
There exists https://github.com/sters/yaml-diff which does the same thing you're intending to achieve.
I'm having some issue with writing []float32 data in stream order to a io.Writer buffer in Go.
I have the following 16 bit per pixel image noise16.jpeg that I open, and read as follows in my test:
func TestNewFITSImageFrom2DDataWriteFloatData(t *testing.T) {
f, err := os.Open("../../images/noise16.jpeg")
if err != nil {
t.Errorf("Error opening image: %s", err)
}
defer f.Close()
img, err := jpeg.Decode(f)
if err != nil {
t.Errorf("Error decoding image: %s", err)
}
bounds := img.Bounds()
ex := make([][]uint32, bounds.Dx())
for x := 0; x < bounds.Dx(); x++ {
col := make([]uint32, bounds.Dy())
ex[x] = col
}
for j := 0; j < bounds.Dy(); j++ {
for i := 0; i < bounds.Dx(); i++ {
r, g, b, _ := img.At(i, j).RGBA()
lum := 0.299*float64(r) + 0.587*float64(g) + 0.114*float64(b)
ex[i][j] = uint32(lum)
}
}
var fit = NewFITSImageFrom2DData(ex, 16, 2, int32(bounds.Dx()), int32(bounds.Dy()))
var w io.Writer
err = writeFloat32Array(w, fit.Data, true)
if err != nil {
t.Errorf("Error writing float32 array: %s", err)
}
}
I essentially open the image, and read in each pixel to an "exposure" array (the []float32 data). I then pass the data to the following method, with an instantiated writer:
// Writes FITS binary body data in network byte order.
// Optionally replaces NaNs with zeros for compatibility with other software
func writeFloat32Array(w io.Writer, data []float32, replaceNaNs bool) error {
bufLen := 16 * 1024
buff := make([]byte, bufLen)
// Process the data in chunks of BUFFER_LENGTH bytes:
for block := 0; block < len(data); block += (bufLen >> 2) {
size := len(data) - block
if size > (bufLen >> 2) {
size = (bufLen >> 2)
}
for offset := 0; offset < size; offset++ {
d := data[block+offset]
if replaceNaNs && math.IsNaN(float64(d)) {
d = 0
}
val := math.Float32bits(d)
buff[(offset<<2)+0] = byte(val >> 24)
buff[(offset<<2)+1] = byte(val >> 16)
buff[(offset<<2)+2] = byte(val >> 8)
buff[(offset<<2)+3] = byte(val)
}
_, err := w.Write(buff[:(size << 2)])
if err != nil {
return err
}
}
// Complete the last partial block, for strictly FITS compliant software
bytesWritten := len(data) << 2
lastPartialBlock := bytesWritten % 2880
if lastPartialBlock != 0 {
sb := strings.Builder{}
for i := lastPartialBlock; i < 2880; i++ {
sb.WriteRune(' ')
}
_, err := w.Write([]byte(sb.String()))
if err != nil {
fmt.Println(err)
return err
}
}
return nil
}
My testing is coming back with a Panic error:
panic: runtime error: invalid memory address or nil pointer dereference [recovered]
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0x18 pc=0x119101f]
Which signifies that this line is throwing an error from inside the writeFloat32Array function in the first iteration of the for block := 0; block < len(data); block += (bufLen >> 2) loop:
_, err := w.Write(buff[:(size << 2)])
Could anyone shed some light on what I have possibly done wrong? My instincts are telling me something around bufLen is incorrect, this should probably be the number of bytes in the image, rather than 16384 bytes?
However, when I set a very large number ... I still see the same error?
I have also tried as #jdizzle suggested, and encoded with the binary package but I am still seeing the panic error:
// Writes FITS binary body data in network byte order.
// Optionally replaces NaNs with zeros for compatibility with other software
func writeFloat32Array(w io.Writer, data []float32, replaceNaNs bool) error {
buf := new(bytes.Buffer)
err := binary.Write(buf, binary.BigEndian, data)
if err != nil {
return err
}
_, err = w.Write(buf.Bytes())
if err != nil {
return err
}
return nil
}
I have the following input, where on the first line is N - count of numbers, and on the second line N numbers, separated by space:
5
2 1 0 3 4
In Python I can read numbers without specifying its count (N):
_ = input()
numbers = list(map(int, input().split()))
How can I do the same in Go? Or I have to know exactly how many numbers are?
You can iterate through a file line-by-line using bufio, and the strings module can split a string into a slice. So that gets us something like:
package main
import (
"bufio"
"fmt"
"os"
"strconv"
"strings"
)
func main() {
readFile, err := os.Open("data.txt")
defer readFile.Close()
if err != nil {
fmt.Println(err)
}
fileScanner := bufio.NewScanner(readFile)
fileScanner.Split(bufio.ScanLines)
for fileScanner.Scan() {
// get next line from the file
line := fileScanner.Text()
// split it into a list of space-delimited tokens
chars := strings.Split(line, " ")
// create an slice of ints the same length as
// the chars slice
ints := make([]int, len(chars))
for i, s := range chars {
// convert string to int
val, err := strconv.Atoi(s)
if err != nil {
panic(err)
}
// update the corresponding position in the
// ints slice
ints[i] = val
}
fmt.Printf("%v\n", ints)
}
}
Which given your sample data will output:
[5]
[2 1 0 3 4]
Since you know the delimiter and you only have 2 lines, this is also a more compact solution:
package main
import (
"fmt"
"os"
"regexp"
"strconv"
"strings"
)
func main() {
parts, err := readRaw("data.txt")
if err != nil {
panic(err)
}
n, nums, err := toNumbers(parts)
if err != nil {
panic(err)
}
fmt.Printf("%d: %v\n", n, nums)
}
// readRaw reads the file in input and returns the numbers inside as a slice of strings
func readRaw(fn string) ([]string, error) {
b, err := os.ReadFile(fn)
if err != nil {
return nil, err
}
return regexp.MustCompile(`\s`).Split(strings.TrimSpace(string(b)), -1), nil
}
// toNumbers plays with the input string to return the data as a slice of int
func toNumbers(parts []string) (int, []int, error) {
n, err := strconv.Atoi(parts[0])
if err != nil {
return 0, nil, err
}
nums := make([]int, 0)
for _, p := range parts[1:] {
num, err := strconv.Atoi(p)
if err != nil {
return n, nums, err
}
nums = append(nums, num)
}
return n, nums, nil
}
The output out be:
5: [2 1 0 3 4]
The code below reads its values from this file:
2 3\n
1.0 2.0 3.0\n
-1.0 -2.0 -3.0\n
And should print:
[ {1 2 3}, {-1 -2 -3} ]
But instead I get this:
[{2 [31 2 3]} {0 []}] strconv.ParseFloat: parsing "3.0-1.0": invalid syntax
It seems that the reader.ReadLine() stays at the same location. Is there a simpler way to scan lines, then values inside each line?
package main
import (
"bufio"
"bytes"
"fmt"
"os"
"strconv"
"strings"
)
type Example struct {
classLabel int
attributes []float64
}
func NewExample(classLabel int, attributes []float64) *Example {
return &Example{classLabel, attributes}
}
func readFile(path string) ([]Example, error) {
var (
result []Example
err error
file *os.File
part []byte
size int
attributeNum int
)
if file, err = os.Open(path); err != nil {
return result, err
}
defer file.Close()
reader := bufio.NewReader(file)
buffer := bytes.NewBuffer(make([]byte, 0))
if part, _, err = reader.ReadLine(); err != nil {
return result, err
}
buffer.Write(part)
newLine := buffer.String()
fmt.Println("newLine=" + newLine)
r := strings.NewReader(newLine)
scanner := bufio.NewScanner(r)
scanner.Split(bufio.ScanWords)
if scanner.Scan() {
size, err = strconv.Atoi(scanner.Text())
if err != nil {
return result, err
}
}
fmt.Println("size=" + strconv.Itoa(size))
if scanner.Scan() {
attributeNum, err = strconv.Atoi(scanner.Text())
if err != nil {
return result, err
}
}
fmt.Println("attributeNum=" + strconv.Itoa(attributeNum))
result = make([]Example, size)
var classLabel int
var attributes []float64
for k := 0; k < size; k++ {
if part, _, err = reader.ReadLine(); err != nil {
return result, err
}
buffer.Write(part)
newLine := buffer.String()
fmt.Println("newLine=" + newLine)
r := strings.NewReader(newLine)
scanner := bufio.NewScanner(r)
scanner.Split(bufio.ScanWords)
if scanner.Scan() {
classLabel, err = strconv.Atoi(scanner.Text())
if err != nil {
return result, err
}
}
fmt.Println("classLabel=" + strconv.Itoa(classLabel))
for i := 0; i < attributeNum; i++ {
var attribute float64
if scanner.Scan() {
attribute, err = strconv.ParseFloat(scanner.Text(), 64)
if err != nil {
return result, err
}
attributes = append(attributes, attribute)
fmt.Println("attribute=" + strconv.FormatFloat(attribute, 'f', -1, 64))
}
}
result[k] = *NewExample(classLabel, attributes)
}
return result, scanner.Err()
}
func main() {
example, err := readFile("test.txt")
fmt.Println(example, err)
}
When you do this inside the for loop:
buffer.Write(part)
newLine := buffer.String()
fmt.Println("newLine=" + newLine)
The next line gets appended to buffer.
That is,
before the loop begins, buffer contains 2 3,
and then after reading 1.0 2.0 3.0,
it gets appended to buffer,
so the content becomes 2 31.0 2.0 3.0,
which you store in newLine.
That's where things start to go sideways.
You probably want to clear the buffer before reading each new line:
buffer.Reset()
buffer.Write(part)
newLine := buffer.String()
fmt.Println("newLine=" + newLine)
But then you will have further problems still, here:
if scanner.Scan() {
classLabel, err = strconv.Atoi(scanner.Text())
if err != nil {
return result, err
}
}
Since the line contains 1.0 2.0 3.0, the strconf.Atoi is going to fail.
I don't understand the purpose of this snippet,
perhaps you can delete it (or comment out).
With the above fixed, you will still have one more problem, on this line:
attributes = append(attributes, attribute)
Since attributes is never reset, it keeps growing.
That is, after the first line, it will contain 1 2 3,
and after the second line it will contain 1 2 3 -1 -2 -3.
You could correct that by moving the declaration of attributes without the outer loop, like this:
var attributes []float64
for i := 0; i < attributeNum; i++ {
var attribute float64
if scanner.Scan() {
attribute, err = strconv.ParseFloat(scanner.Text(), 64)
if err != nil {
return result, err
}
attributes = append(attributes, attribute)
fmt.Println("attribute=" + strconv.FormatFloat(attribute, 'f', -1, 64))
}
}
I have a line containing 3 numbers that I want to read from stdin with fmt.Scanln() but this code won't work:
X := make([]int, 3)
fmt.Scanln(X...)
fmt.Printf("%v\n", X)
I get this error message:
cannot use X (type []int) as type []interface {} in function argument
I don't get it.
Idiomatic Go would be:
func read(n int) ([]int, error) {
in := make([]int, n)
for i := range in {
_, err := fmt.Scan(&in[i])
if err != nil {
return in[:i], err
}
}
return in, nil
}
interface{} means nothing. Please don't use it if you don't have to.
For example,
package main
import "fmt"
func intScanln(n int) ([]int, error) {
x := make([]int, n)
y := make([]interface{}, len(x))
for i := range x {
y[i] = &x[i]
}
n, err := fmt.Scanln(y...)
x = x[:n]
return x, err
}
func main() {
x, err := intScanln(3)
if err != nil {
fmt.Println(err)
return
}
fmt.Printf("%v\n", x)
}
Input:
1 2 3
Output:
[1 2 3]
I think the the correct version should be
X := make([]int, 3)
fmt.Scanln(&X[0], &X[1], &X[2])
fmt.Printf("%v\n", X)
This error message occurs b/c there's no reasonable way to convert []int to []interface{}. Note, this is in reference to a slice. So the syntax your using is correct, but fmt.Scanln expects []interface{}. This has implications outside of pkg fmt.
The reason I've seen given for this is due to Go giving you control over memory layout so it currently has no reasonable way to do the slice conversion. This means you'll need to do the conversion manually before passing it to a function expecting the slice of a given type. For example:
package main
import (
"fmt"
)
func main() {
x := make([]int, 3)
y := make([]interface{}, 3)
y[0] = x[0]
y[1] = x[1]
y[2] = x[2]
fmt.Println(y...)
}
Or something a little more general:
x := make([]int, 3)
y := make([]interface{}, len(x))
for i, v := range x {
y[i] = v
}
fmt.Println(y...)
Regarding your specific issue, see the following:
x := make([]*int, 3)
for i := range x {
x[i] = new(int)
}
y := make([]interface{}, 3)
for i, v := range x {
y[i] = v
}
if _, err := fmt.Scanln(y...); err != nil {
fmt.Println("Scanln err: ", err)
}
for _, v := range y {
val := v.(*int)
fmt.Println(*val)
}
I saw in a comment you said the lines can have different lengths. In that case
you can implement your own fmt.Scanner:
package main
import (
"bytes"
"fmt"
)
type slice struct {
tok []int
}
func (s *slice) Scan(state fmt.ScanState, verb rune) error {
tok, err := state.Token(false, func(r rune) bool { return r != '\n' })
if err != nil { return err }
if _, _, err := state.ReadRune(); err != nil {
if len(tok) == 0 {
panic(err)
}
}
b := bytes.NewReader(tok)
for {
var d int
_, err := fmt.Fscan(b, &d)
if err != nil { break }
s.tok = append(s.tok, d)
}
return nil
}
func main() {
var s slice
fmt.Scan(&s)
fmt.Println(s.tok)
}
https://golang.org/pkg/fmt#Scanner