The source code is here
I commented on what I understand
type mmapper struct {
sync.Mutex
active map[*byte][]byte // active mappings; key is last byte in mapping
mmap func(addr, length uintptr, prot, flags, fd int, offset int64) (uintptr, error)
munmap func(addr uintptr, length uintptr) error
}
func (m *mmapper) Mmap(fd int, offset int64, length int, prot int, flags int) (data []byte, err error) {
if length <= 0 {
return nil, EINVAL
}
// Map the requested memory using a operating-system dependent function
addr, errno := m.mmap(0, uintptr(length), prot, flags, fd, offset)
if errno != nil {
return nil, errno
}
// Create slice memory layout
var sl = struct {
addr uintptr
len int
cap int
}{addr, length, length}
// Cast it to byte slice
b := *(*[]byte)(unsafe.Pointer(&sl))
// ??
p := &b[cap(b)-1]
m.Lock()
defer m.Unlock()
m.active[p] = b
return b, nil
}
func (m *mmapper) Munmap(data []byte) (err error) {
if len(data) == 0 || len(data) != cap(data) {
return EINVAL
}
// Check if the slice is valid ?
p := &data[cap(data)-1]
m.Lock()
defer m.Unlock()
b := m.active[p]
if b == nil || &b[0] != &data[0] {
return EINVAL
}
// Unmap the memory and update m.
if errno := m.munmap(uintptr(unsafe.Pointer(&b[0])), uintptr(len(b))); errno != nil {
return errno
}
delete(m.active, p)
return nil
}
So my question is just about the field mmapper.active, I don't understand what it is for.
I have readed about some issue with uintptr and garbage collector, maybe is for avoid them ?
Or maybe it's just for validate the slice when Munmap is call ?
It might be a slice of a slice, then its backed pointer will be different from the original. This map allows us to check whether this slice was sliced.
b := []byte{1,2,3,4}
fmt.Println(&b[0]) // 0x40e020
b = b[1:]
fmt.Println(&b[0]) // 0x40e021
https://play.golang.org/p/WJogLiMOfj7
Related
I'm working on encoding and decoding files in golang. I specifically do need the 2D array that I'm using, this is just test code to show the point. I'm not entirely sure what I'm doing wrong, I'm attempting to convert the file into a list of uint32 numbers and then take those numbers and convert them back to a file. The problem is that when I do it the file looks fine but the checksum doesn't line up. I suspect that I'm doing something wrong in the conversion to uint32. I have to do the switch/case because I have no way of knowing how many bytes I'll read for sure at the end of a given file.
package main
import (
"bufio"
"encoding/binary"
"fmt"
"io"
"os"
)
const (
headerSeq = 8
body = 24
)
type part struct {
Seq int
Data uint32
}
func main() {
f, err := os.Open("speech.pdf")
if err != nil {
panic(err)
}
defer f.Close()
reader := bufio.NewReader(f)
b := make([]byte, 4)
o := make([][]byte, 0)
var value uint32
for {
n, err := reader.Read(b)
if err != nil {
if err != io.EOF {
panic(err)
}
}
if n == 0 {
break
}
fmt.Printf("len array %d\n", len(b))
fmt.Printf("len n %d\n", n)
switch n {
case 1:
value = uint32(b[0])
case 2:
value = uint32(uint32(b[1]) | uint32(b[0])<<8)
case 3:
value = uint32(uint32(b[2]) | uint32(b[1])<<8 | uint32(b[0])<<16)
case 4:
value = uint32(uint32(b[3]) | uint32(b[2])<<8 | uint32(b[1])<<16 | uint32(b[0])<<24)
}
fmt.Println(value)
bs := make([]byte, 4)
binary.BigEndian.PutUint32(bs, value)
o = append(o, bs)
}
fo, err := os.OpenFile("test.pdf", os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
if err != nil {
panic(err)
}
defer fo.Close()
for _, ba := range o {
_, err := fo.Write(ba)
if err != nil {
panic(err)
}
}
}
So, you want to write and read arrays of varying length in a file.
import "encoding/binary"
// You need a consistent byte order for reading and writing multi-byte data types
const order = binary.LittleEndian
var dataToWrite = []byte{ ... ... ... }
var err error
// To write a recoverable array of varying length
var w io.Writer
// First, encode the length of data that will be written
err = binary.Write(w, order, int64(len(dataToWrite)))
// Check error
err = binary.Write(w, order, dataToWrite)
// Check error
// To read a variable length array
var r io.Reader
var dataLen int64
// First, we need to know the length of data to be read
err = binary.Read(r, order, &dataLen)
// Check error
// Allocate a slice to hold the expected amount of data
dataReadIn := make([]byte, dataLen)
err = binary.Read(r, order, dataReadIn)
// Check error
This pattern works not just with byte, but any other fixed size data type. See binary.Write for specifics about the encoding.
If the size of encoded data is a big concern, you can save some bytes by storing the array length as a varint with binary.PutVarint and binary.ReadVarint
I want to take data from DB and write to excel
let's say I have a struct like:
type user struct {
ID int64
Name string
Age int
}
I can get a pointer to slice of user type form DB &[]user{}
but I want to convert that slice to a 2D slice of string [][]string{}
and here's my code try to do such job:
func toStrings(slice interface{}) [][]string {
switch reflect.TypeOf(slice).Elem().Kind() {
case reflect.Slice:
ret := [][]string{}
val := reflect.ValueOf(slice).Elem()
for i := 0; i < val.Len(); i++ {
tempSlice := []string{}
tempV := reflect.ValueOf(val.Index(i))
for j := 0; j < tempV.NumField(); j++ {
tempSlice = append(tempSlice, tempV.Field(j).String())
}
ret = append(ret, tempSlice)
}
return ret
}
return nil
}
But from the code above all I get is a slice like [<*reflect.rtype Value> <unsafe.Pointer Value> <reflect.flag Value>]
where I do it wrong?
my codes in golang playground
sorry, I found where I do it wrong, I got tempV wrong
func toStrings(slice interface{}) [][]string {
switch reflect.TypeOf(slice).Elem().Kind() {
case reflect.Slice:
ret := [][]string{}
val := reflect.ValueOf(slice).Elem()
for i := 0; i < val.Len(); i++ {
tempSlice := []string{}
// tempV should be:
tempV := val.Index(i)
// instead of reflect.ValueOf(val.Index(i))
for j := 0; j < tempV.NumField(); j++ {
tempSlice = append(tempSlice, tempV.Field(j).String())
}
ret = append(ret, tempSlice)
}
return ret
}
return nil
}
There are two problems in the code in the question. The first problem is the slice element is doubled wrapped by a a reflect.Value in the expression reflect.Value(val.Index(i)). Fix by removing the extra call to reflect.Value. The second problem is that the reflect.Value String method does not convert the underlying value to its string representation. Use fmt.Sprint (or one of its friends) to do that.
Try this:
func toStrings(slice interface{}) [][]string {
// Get reflect value for slice. Use Indirect to
// handle slice argument and pointer to slice
// argument.
v := reflect.Indirect(reflect.ValueOf(slice))
if v.Kind() != reflect.Slice {
return nil
}
var result [][]string
// For each element...
for i := 0; i < v.Len(); i++ {
// Get reflect value for slice element (a struct). Use
// Indirect to handle slice of struct and slice of
// pointer to struct.
e := reflect.Indirect(v.Index(i))
if e.Kind() != reflect.Struct {
return nil
}
// Convert fields to string and append.
var element []string
for i := 0; i < e.NumField(); i++ {
// Use fmt.Sprint to convert arbitrary Go value
// to a string.
element = append(element, fmt.Sprint(e.Field(i).Interface()))
}
result = append(result, element)
}
return result
}
Run it on the playground.
Maybe I have a simple way to resolve the problem, golang playground here
I used encoding/json to convert to json data, then convert it to map[string]interface{}.
func toStrings2(slice interface{}) [][]string {
jsonData, _ := json.Marshal(slice)
var out []map[string]interface{}
_ = json.Unmarshal(jsonData, &out)
var fields []string
if len(out) > 0 {
for k := range out[0] {
fields = append(fields, k)
}
}
var ret [][]string
for _, row := range out {
var r []string
for _, k := range fields {
r = append(r, fmt.Sprint(row[k]))
}
ret = append(ret, r)
}
return ret
}
Notice:
With the help of #CeriseLimón, I known that the code in this answer can't handle large values for User.ID.
I'm puzzled by what this line of code does from the ioutil package. It appears to compare the same value twice but casts it twice on one side. Any insights would be greatly appreciated!
int64(int(capacity)) == capacity
from this function
func readAll(r io.Reader, capacity int64) (b []byte, err error) {
var buf bytes.Buffer
// If the buffer overflows, we will get bytes.ErrTooLarge.
// Return that as an error. Any other panic remains.
defer func() {
e := recover()
if e == nil {
return
}
if panicErr, ok := e.(error); ok && panicErr == bytes.ErrTooLarge {
err = panicErr
} else {
panic(e)
}
}()
if int64(int(capacity)) == capacity {
buf.Grow(int(capacity))
}
_, err = buf.ReadFrom(r)
return buf.Bytes(), err
}
Converting Tim Cooper's comment into an answer.
bytes.Buffer.Grow takes in an int and capacity is int64.
func (b *Buffer) Grow(n int)
Grow grows the buffer's capacity, if necessary, to guarantee space for
another n bytes. After Grow(n), at least n bytes can be written to the
buffer without another allocation.
As mentioned in GoDoc, Grow is used as an optimisation to prevent further allocations.
int64(int(capacity)) == capacity
makes sure that capacity is within the range of int values so that the optimisation can be applied.
How do I use bufio.ScanWords and bufio.ScanLines functions to count words and lines?
I tried:
fmt.Println(bufio.ScanWords([]byte("Good day everyone"), false))
Prints:
5 [103 111 111 100] <nil>
Not sure what that means?
To count words:
input := "Spicy jalapeno pastrami ut ham turducken.\n Lorem sed ullamco, leberkas sint short loin strip steak ut shoulder shankle porchetta venison prosciutto turducken swine.\n Deserunt kevin frankfurter tongue aliqua incididunt tri-tip shank nostrud.\n"
scanner := bufio.NewScanner(strings.NewReader(input))
// Set the split function for the scanning operation.
scanner.Split(bufio.ScanWords)
// Count the words.
count := 0
for scanner.Scan() {
count++
}
if err := scanner.Err(); err != nil {
fmt.Fprintln(os.Stderr, "reading input:", err)
}
fmt.Printf("%d\n", count)
To count lines:
input := "Spicy jalapeno pastrami ut ham turducken.\n Lorem sed ullamco, leberkas sint short loin strip steak ut shoulder shankle porchetta venison prosciutto turducken swine.\n Deserunt kevin frankfurter tongue aliqua incididunt tri-tip shank nostrud.\n"
scanner := bufio.NewScanner(strings.NewReader(input))
// Set the split function for the scanning operation.
scanner.Split(bufio.ScanLines)
// Count the lines.
count := 0
for scanner.Scan() {
count++
}
if err := scanner.Err(); err != nil {
fmt.Fprintln(os.Stderr, "reading input:", err)
}
fmt.Printf("%d\n", count)
This is an exercise in book The Go Programming Language Exercise 7.1
This is an extension of #repler solution:
package main
import (
"bufio"
"fmt"
"os"
"strings"
)
type byteCounter int
type wordCounter int
type lineCounter int
func main() {
var c byteCounter
c.Write([]byte("Hello This is a line"))
fmt.Println("Byte Counter ", c)
var w wordCounter
w.Write([]byte("Hello This is a line"))
fmt.Println("Word Counter ", w)
var l lineCounter
l.Write([]byte("Hello \nThis \n is \na line\n.\n.\n"))
fmt.Println("Length ", l)
}
func (c *byteCounter) Write(p []byte) (int, error) {
*c += byteCounter(len(p))
return len(p), nil
}
func (w *wordCounter) Write(p []byte) (int, error) {
count := retCount(p, bufio.ScanWords)
*w += wordCounter(count)
return count, nil
}
func (l *lineCounter) Write(p []byte) (int, error) {
count := retCount(p, bufio.ScanLines)
*l += lineCounter(count)
return count, nil
}
func retCount(p []byte, fn bufio.SplitFunc) (count int) {
s := string(p)
scanner := bufio.NewScanner(strings.NewReader(s))
scanner.Split(fn)
count = 0
for scanner.Scan() {
count++
}
if err := scanner.Err(); err != nil {
fmt.Fprintln(os.Stderr, "reading input:", err)
}
return
}
This is an exercise in book The Go Programming Language Exercise 7.1
This is my solution:
package main
import (
"bufio"
"fmt"
)
// WordCounter count words
type WordCounter int
// LineCounter count Lines
type LineCounter int
type scanFunc func(p []byte, EOF bool) (advance int, token []byte, err error)
func scanBytes(p []byte, fn scanFunc) (cnt int) {
for true {
advance, token, _ := fn(p, true)
if len(token) == 0 {
break
}
p = p[advance:]
cnt++
}
return cnt
}
func (c *WordCounter) Write(p []byte) (int, error) {
cnt := scanBytes(p, bufio.ScanWords)
*c += WordCounter(cnt)
return cnt, nil
}
func (c WordCounter) String() string {
return fmt.Sprintf("contains %d words", c)
}
func (c *LineCounter) Write(p []byte) (int, error) {
cnt := scanBytes(p, bufio.ScanLines)
*c += LineCounter(cnt)
return cnt, nil
}
func (c LineCounter) String() string {
return fmt.Sprintf("contains %d lines", c)
}
func main() {
var c WordCounter
fmt.Println(c)
fmt.Fprintf(&c, "This is an sentence.")
fmt.Println(c)
c = 0
fmt.Fprintf(&c, "This")
fmt.Println(c)
var l LineCounter
fmt.Println(l)
fmt.Fprintf(&l, `This is another
line`)
fmt.Println(l)
l = 0
fmt.Fprintf(&l, "This is another\nline")
fmt.Println(l)
fmt.Fprintf(&l, "This is one line")
fmt.Println(l)
}
bufio.ScanWords and bufio.ScanLines (as well as bufio.ScanBytes and bufio.ScanRunes) are split functions: they provide a bufio.Scanner with the strategy to tokenize its input data – how the process of scanning should split the data. The split function for a bufio.Scanner is bufio.ScanLines by default but can be changed through the method bufio.Scanner.Split.
These split functions are of type SplitFunc:
type SplitFunc func(data []byte, atEOF bool) (advance int, token []byte, err error)
Usually, you won't need to call any of these functions directly; instead, bufio.Scanner will. However, you might need to create your own split function for implementing a custom tokenization strategy. So, let's have a look at its parameters:
data: remaining data not processed yet.
atEOF: whether or not the caller has reached EOF and therefore has no more new data to provide in the next call.
advance: number of bytes the caller must advance the input data for the next call.
token: the token to return to the caller as a result of the splitting performed.
To gain further understanding, let's see bufio.ScanBytes implementation:
func ScanBytes(data []byte, atEOF bool) (advance int, token []byte, err error) {
if atEOF && len(data) == 0 {
return 0, nil, nil
}
return 1, data[0:1], nil
}
As long as data isn't empty, it returns a token byte to the caller (data[0:1]) and tells the caller to advance the input data by one byte.
I have the following code to double the slice.
func doubleSlice(s []int) []int {
t := make([]int, len(s), (cap(s) + 1) * 2 )
for i := range s {
t[i] = s[i]
}
return t
}
I want to make the func to double any type of slice. And I need to know the element type first.
func showInterfaceItem(s interface{}) interface{} {
if reflect.TypeOf(s).Kind() != reflect.Slice {
fmt.Println("The interface is not a slice.")
return
}
var t interface{}
newLen := reflect.ValueOf(s).Len()
newCap := (cap(reflect.ValueOf(s).Cap()) + 1) * 2
t = make([]reflect.TypeOf(s), newLen, newCap)
return t
}
The reflect.TypeOf(s) return the type of interface{}, not the type of element. How can I get the element type of slice interface?
You can use reflect.TypeOf(s).Elem()
to get the type of element of slice.
package main
import (
"fmt"
"reflect"
)
func doubleSlice(s interface{}) interface{} {
if reflect.TypeOf(s).Kind() != reflect.Slice {
fmt.Println("The interface is not a slice.")
return nil
}
v := reflect.ValueOf(s)
newLen := v.Len()
newCap := (v.Cap() + 1) * 2
typ := reflect.TypeOf(s).Elem()
t := reflect.MakeSlice(reflect.SliceOf(typ), newLen, newCap)
reflect.Copy(t, v)
return t.Interface()
}
func main() {
xs := doubleSlice([]string{"foo", "bar"}).([]string)
fmt.Println("data =", xs, "len =", len(xs), "cap =", cap(xs))
ys := doubleSlice([]int{3, 1, 4}).([]int)
fmt.Println("data =", ys, "len =", len(ys), "cap =", cap(ys))
}
The output will be:
data = [foo bar] len = 2 cap = 6
data = [3 1 4] len = 3 cap = 8
Check it in: Go Playground
This is doable in golang and takes me whole day to discover the pattern.
Firstly, we want to get a pointer of slice to make gorm happy, which is has type "*[]Obj". To achieve that in golang, we can create a make wrapper like so:
func makeWrapper(cap uint) interface{} {
arr:= make([]Sth, 0, cap)
return &arr
}
Notice that, we can't directly reference the maked value, which might be the book keeping data need to have a stack space to store.
//Not working example
func makeWrapper(cap uint) interface{} {
return &(make([]Sth, 0, cap))
}
And as the answer before, the reflect.MakeSlice(reflect.SliceOf(typ), 0, capacity).Interface() returns interface{[]Sth}. (the typ here is refer to reflect.TypeOf(Sth{}), which equiv to typ == reflect.TypeOf(v))
Thus we need to create a return object of *[]Sth and the value inside is a slice []Sth with capacity. After understanding the objective, we can have this code:
package main
import (
"reflect"
)
type Sth struct {
a, b string
}
func main() {
af:= createSlice(Sth{})
arr := makeWrapper(10).(*[]Sth)
println(reflect.TypeOf(arr).String())
// equiv to makeWrapper, but we do it via reflection
arr = af(10).(*[]Sth)
println(reflect.TypeOf(arr).String())
}
func makeWrapper(cap uint) interface{} {
arr:= make([]Sth, 0, cap)
return &arr
}
func createSlice(v interface{}) func(int) interface{} {
var typ reflect.Type
if reflect.ValueOf(v).Kind() == reflect.Ptr {
typ = reflect.ValueOf(v).Elem().Type()
} else if reflect.ValueOf(v).Kind() == reflect.Struct {
typ = reflect.TypeOf(v)
} else {
panic("only support instance of struct or pointer of that instance")
}
return func(capacity int) interface{}{
// create the outer object saves our slice
outerObj:=reflect.New(reflect.SliceOf(typ))
// create the slice and save it to return
outerObj.Elem().Set(reflect.MakeSlice(reflect.SliceOf(typ), 0, capacity))
// retrive the interface of outer object
return outerObj.Interface()
}
}