gob encoding to/decoding from *os.File not working - go

When I use a file pointer f *os.File I get a empty map back
func decode(f *os.File, b map[string]interface{}) error {
err := gob.NewDecoder(f).Decode(&b)
fmt.Printf("%+v\n", b)
return err
}
func encode(f *os.File, b map[string]interface{}) error {
bb := map[string]interface{}{
"X": 1,
"Greeting": "hello",
}
err := gob.NewEncoder(f).Encode(bb)
f.Sync()
//fmt.Println(buf.Bytes())
return err
}
prints map[]
If I replace it with a global buffer pointer buf *bytes.Buffer it works
func decode(f *os.File, b map[string]interface{}) error {
err := gob.NewDecoder(buf).Decode(&b)
fmt.Printf("%+v\n", b)
return err
}
func encode(f *os.File, b map[string]interface{}) error {
bb := map[string]interface{}{
"X": 1,
"Greeting": "hello",
}
err := gob.NewEncoder(buf).Encode(bb)
return err
}
prints map[Greeting:hello X:1]
f is opened in main like this
var (
buf *bytes.Buffer
f *os.File
memcache map[string]interface{}
)
func main() {
var err error
f, err = os.Create("_memcache.txt")
if err != nil {
log.Print(err.Error())
}
defer f.Close()
memcache = make(map[string]interface{})
buf = new(bytes.Buffer)
gob.Register(map[string]interface{}{})
err = write()
if err != nil {
log.Print(err.Error())
}
err = read()
if err != nil {
log.Print(err.Error())
}
}
func read() (err error) {
err = decode(f, memcache)
if err != nil && err != io.EOF {
return err
}
}
func write() (err error) {
err = encode(f, memcache)
if err != nil {
return err
}
}

The solution is already mentioned in the comments, so the update method main() is following:
func main() {
var err error
f, err = os.Create("_memcache.txt")
if err != nil {
log.Print(err.Error())
}
defer f.Close()
memcache = make(map[string]interface{})
buf = new(bytes.Buffer)
gob.Register(map[string]interface{}{})
err = write()
if err != nil {
log.Print(err.Error())
}
// after write the file pointer is at the end of the file,
// so it should be moved:
f.Seek(0, 0)
// alternatively, open the file in read()
err = read()
if err != nil {
log.Print(err.Error())
}
}

Related

How to gently defer execution of a function that might return an error?

Most cleanup functions, especially those related to the IO operations, return an error, and normally we'd prefer to defer their execution in case if we'd not forget to call them when we're done with acquired resources. For example, at some point in the code we might write something like this:
var r *SomeResource
var err error
if r, err = Open(/* parameters */); err != nil {
return nil, err
}
defer r.Close() // This might return an error
It seems that if Close function returns an error, it'll be ignored. How can we gently process the returned error from such a function?
Using defer with a func() {}() like so.
var r *SomeResource
var err error
if r, err = Open(/* parameters */); err != nil {
return nil, err
}
defer func() {
if err = r.Close(); err != nil {
fmt.Printf("ERROR: %v", err)
}
}()
Fail gracefully with an error. Report the first error. Don't overwrite earlier errors. For example,
package main
import (
"fmt"
"os"
)
func demo() (name string, err error) {
filename := `test.file`
f, err := os.Open(filename)
if err != nil {
return "", err
}
defer func() {
e := f.Close()
if e != nil {
if err == nil {
err = e
}
}
}()
// do someting with the file
name = f.Name()
fi, err := f.Stat()
if err != nil {
return name, err
}
if fi.Size() == 0 {
err = fmt.Errorf("%s: empty file", filename)
return name, err
}
return name, err
}
func main() {
name, err := demo()
fmt.Println(name, err)
}
We can handle this in ways like:
way-1:
func myFn() error {
var err error
if r, err = Open(/* parameters */); err != nil {
return nil, err
}
defer func() {
if cErr = r.Close(); cErr != nil {
err = cErr
}
}()
return err
}
way-2:
func myFn() error {
var err error
if r, err = Open(/* parameters */); err != nil {
return nil, err
}
defer func() {
if cErr = r.Close(); cErr != nil {
// we can log the error
// or
// whatever we want to do
}
}()
return err
}
I have also find a nice blog on this topic, i mean handling error when defer func returns an error. Check here https://blog.learngoprogramming.com/5-gotchas-of-defer-in-go-golang-part-iii-36a1ab3d6ef1.

Golang unexpected EOF

Here's my code, I'm new to Go.
I tried googling the issue, but I can't quite put my finger on it.
I think it has something to do with the Read() method.
package main
import (
...
)
type compressor struct {
content []byte
}
func (r *compressor) compress() []byte {
...
}
func (r *compressor) decompress() []byte {
var buffer bytes.Buffer
dc := flate.NewReader(&buffer)
_, err := dc.Read(r.content)
if err != nil {
if err != io.EOF {
log.Fatal(err)
}
}
return buffer.Bytes()
}
func main() {
fileName := os.Args[1]
fmt.Println(os.Args)
contents, err := ioutil.ReadFile(fileName)
if err != nil {
log.Fatal(err)
}
fmt.Print("Uncompressed data: ")
fmt.Println(len(contents))
comp := compressor{contents}
buffer := comp.decompress()
fmt.Print("Uncompressed data: ")
fmt.Println(len(comp.decompress()))
err = ioutil.WriteFile(fileName+".decjc", buffer, 0644)
if err != nil {
log.Fatal(err)
}
}
Here's the output
dylan#skynet:~/Documents/EXP/jc$ ./jc data.txt.jc
[./jc data.txt.jc]
Uncompressed data: 2364480
2018/06/29 21:41:35 unexpected EOF
After doing a trace on the particular code in question I have come to the following answer.
/src/bytes/reader.go 70
func (r *Reader) ReadByte() (byte, error) {
...
if r.i >= int64(len(r.s)) {
return 0, io.EOF
}
....
}
There are four functions in bytes/reader that can return io.EOF, and zero functions that can return io.ErrUnexpectedEOF. The four functions that can return io.EOF are:
Read(b []byte)
ReadAt(b []byte, off int64)
ReadByte()
ReadRune()
/src/compress/flate/inflate.go 698
func (f *decompressor) moreBits() error {
c, err := f.r.ReadByte()
if err != nil {
return noEOF(err)
}
...
}
Of the four functions that can return io.EOF, only one function in flate/inflate.go calls any of them: moreBits() calls ReadByte()
/src/compress/flate/inflate.go 690
func noEOF(e error) error {
if e == io.EOF {
return io.ErrUnexpectedEOF
}
...
}
When moreBits() receives an error it calls noEOF(), which checks if it had received an io.EOF. If this was the case then io.ErrUnexpectedEOF is returned backed. Everything seems to be working as intended, and it appears that it is the user's responsibility to be on the look out for this particular case. A suggested edit to the code above to handle what appears to be defined behavior is:
func (r *compressor) decompress() []byte {
dc := flate.NewReader(bytes.NewReader(r.content))
defer dc.Close()
rb, err := ioutil.ReadAll(dc)
if err != nil {
if err != io.EOF && err != io.ErrUnexpectedEOF {
log.Fatalf("Err %v\n read %v", err, rb)
}
}
return rb
}
This was checked under go1.12.9
You got the in and outputs mixed up.
flate.NewReader takes the compressed input as an io.Reader and it returns a io.ReadCloser that can be used to get the uncompressed output:
func (r *compressor) decompress() []byte {
dc := flate.NewReader(bytes.NewReader(r.content))
defer dc.Close()
rb, err := ioutil.ReadAll(dc)
if err != nil {
if err != io.EOF {
log.Fatalf("Err %v\n read %v", err, rb)
}
}
return rb
}

Create an io.ReaderAt from io.Reader

Is there an implementation of io.ReaderAt that can be created from an implementation of io.Reader without first being read into a []byte or string?
Something like the below. Note bytes.Reader implements the ReadAt(...) method/function: https://golang.org/pkg/bytes/#Reader.ReadAt. So the line bytes.NewReader is esssentially what you are looking for.
Getting a bytes.Reader:
var ioReader io.Reader
...
buff := bytes.NewBuffer([]byte{})
size, err := io.Copy(buff, ioReader)
if err != nil {
return err
}
reader := bytes.NewReader(buff.Bytes())
// Do something with `reader`
Yes, this is possible. As mentioned in my comment above, the implementation is limited in that you cannot seek backward nor can you re-read a section that has already been read.
Here is a example implementation:
type unbufferedReaderAt struct {
R io.Reader
N int64
}
func NewUnbufferedReaderAt(r io.Reader) io.ReaderAt {
return &unbufferedReaderAt{R: r}
}
func (u *unbufferedReaderAt) ReadAt(p []byte, off int64) (n int, err error) {
if off < u.N {
return 0, errors.New("invalid offset")
}
diff := off - u.N
written, err := io.CopyN(ioutil.Discard, u.R, diff)
u.N += written
if err != nil {
return 0, err
}
n, err = u.R.Read(p)
u.N += int64(n)
return
}
Example usage:
s := strings.NewReader("hello world")
var b [5]byte
ura := NewUnbufferedReaderAt(s)
if _, err := ura.ReadAt(b[:], 0); err != nil {
panic(err)
}
fmt.Printf("%s\n", b[:]) // prints "hello"
/*
if _, err := ura.ReadAt(b[:], 0); err != nil {
panic(err) // panics
}
fmt.Printf("%s\n", b[:])
*/
if _, err := ura.ReadAt(b[:], 6); err != nil {
panic(err)
}
fmt.Printf("%s\n", b[:]) // prints "world"

Trying to test write file from goroutines in Go

Well, part of my code was working without a method approach, I'm trying to test
append text to a file and reading from goroutines, but I'm stuck here trying to
write it.
What is wrong? the file is created, but I can't append text to it, maybe something obvious, but seems I'm blind, maybe I'm failing understanding some language concepts...
package main
import (
"bufio"
"fmt"
"os"
"sync"
"time"
)
var w sync.WaitGroup
type Buffer struct {
F *os.File
}
func (buff *Buffer) Open(pathName string) (err error) {
buff.F, err = os.OpenFile(pathName, os.O_APPEND|os.O_CREATE, 0666)
if err != nil {
return
}
fmt.Println("Open() ok")
return nil
}
func (buff *Buffer) Close() (err error) {
err = buff.F.Close()
if err != nil {
return
}
fmt.Println("Close() ok")
return nil
}
func (buff *Buffer) Push(data string) (err error) {
w := bufio.NewWriter(buff.F)
_, err = fmt.Fprintf(w, "data=%s", data)
if err != nil {
return
}
w.Flush()
fmt.Println("Push() ok")
return nil
}
func checkErr(err error) {
if err != nil {
fmt.Println(err.Error())
}
}
func worker() {
var err error
buffer := new(Buffer)
err = buffer.Open("test")
checkErr(err)
err = buffer.Push("data\n")
checkErr(err)
time.Sleep(5 * time.Second)
err = buffer.Close()
checkErr(err)
w.Done()
}
func main() {
w.Add(2)
go worker()
go worker()
w.Wait()
}
Thanks
Open the file like this:
buff.F, err = os.OpenFile(pathName, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
The write flag is required to write to the file.
You missed the write error because the return from bufio Flush is ignored. Change Push to:
func (buff *Buffer) Push(data string) (err error) {
w := bufio.NewWriter(buff.F)
_, err = fmt.Fprintf(w, "data=%s", data)
if err != nil {
return
}
err = w.Flush()
if err != nil {
return err
}
fmt.Println("Push() ok")
return nil
}
To cleanly append data without intermixing with other pushes, the data must be written with a single call to the file Write method. Use a bytes.Buffer instead of a bufio.Writer to ensure a single call to the file Write method:
func (buff *Buffer) Push(data string) (err error) {
var b bytes.Buffer
_, err = fmt.Fprintf(&b, "data=%s", data)
if err != nil {
return
}
_, err := buff.F.Write(b.Bytes())
if err != nil {
return err
}
fmt.Println("Push() ok")
return nil
}

pass interface pointer and assignment value

I want to write a file cache in Go. I am using gob encoding, and saving to a file, but my get function has some problem:
package main
import (
"encoding/gob"
"fmt"
"os"
)
var (
file = "tmp.txt"
)
type Data struct {
Expire int64
D interface{}
}
type User struct {
Id int
Name string
}
func main() {
user := User{
Id: 1,
Name: "lei",
}
err := set(file, user, 10)
if err != nil {
fmt.Println(err)
return
}
user = User{}
err = get(file, &user)
if err != nil {
fmt.Println(err)
return
}
//user not change.
fmt.Println(user)
}
func set(file string, v interface{}, expire int64) error {
f, err := os.OpenFile(file, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600)
if err != nil {
return err
}
defer f.Close()
//wrapper data
//save v in data.D
data := Data{
Expire: expire,
D: v,
}
gob.Register(v)
enc := gob.NewEncoder(f)
err = enc.Encode(data)
if err != nil {
return err
}
return nil
}
func get(file string, v interface{}) error {
f, err := os.OpenFile(file, os.O_RDONLY, 0600)
if err != nil {
return err
}
defer f.Close()
var data Data
dec := gob.NewDecoder(f)
err = dec.Decode(&data)
if err != nil {
return err
}
//get v
v = data.D
fmt.Println(v)
return nil
}
The get function passes interface type and I want to change the value, but not change.
http://play.golang.org/p/wV7rBH028o
In order to insert an unknown value into v of type interface{}, you need to use reflection. This is somewhat involved, but if you want to support this in full, you can see how its done by walking through the decoding process in some of the encoding packages (json, gob).
To get you started, here's a basic version of your get function using reflection. This skips a number of checks, and will only decode something that was encoded as a pointer.
func get(file string, v interface{}) error {
f, err := os.OpenFile(file, os.O_RDONLY, 0600)
if err != nil {
return err
}
defer f.Close()
rv := reflect.ValueOf(v)
if rv.Kind() != reflect.Ptr || rv.IsNil() {
panic("need a non nil pointer")
}
var data Data
dec := gob.NewDecoder(f)
err = dec.Decode(&data)
if err != nil {
return err
}
dv := reflect.ValueOf(data.D)
if dv.Kind() != reflect.Ptr {
panic("didn't decode a pointer")
}
rv.Elem().Set(dv.Elem())
return nil
}
I would actually suggest an easier way to handle this in your own code, which is to have the Get function return an interface{}. Since you will know what the possible types are at that point, you can use a type switch to assert the correct value.
An alternative approach is to return directly the value from the file:
func get(file string) (interface{}, error) {
f, err := os.OpenFile(file, os.O_RDONLY, 0600)
if err != nil {
return nil, err
}
defer f.Close()
var data Data
dec := gob.NewDecoder(f)
err = dec.Decode(&data)
if err != nil {
return nil,err
}
fmt.Println(data.D)
return data.D,nil
}
full working example: http://play.golang.org/p/178U_LVC5y

Resources