Here's my code, I'm new to Go.
I tried googling the issue, but I can't quite put my finger on it.
I think it has something to do with the Read() method.
package main
import (
...
)
type compressor struct {
content []byte
}
func (r *compressor) compress() []byte {
...
}
func (r *compressor) decompress() []byte {
var buffer bytes.Buffer
dc := flate.NewReader(&buffer)
_, err := dc.Read(r.content)
if err != nil {
if err != io.EOF {
log.Fatal(err)
}
}
return buffer.Bytes()
}
func main() {
fileName := os.Args[1]
fmt.Println(os.Args)
contents, err := ioutil.ReadFile(fileName)
if err != nil {
log.Fatal(err)
}
fmt.Print("Uncompressed data: ")
fmt.Println(len(contents))
comp := compressor{contents}
buffer := comp.decompress()
fmt.Print("Uncompressed data: ")
fmt.Println(len(comp.decompress()))
err = ioutil.WriteFile(fileName+".decjc", buffer, 0644)
if err != nil {
log.Fatal(err)
}
}
Here's the output
dylan#skynet:~/Documents/EXP/jc$ ./jc data.txt.jc
[./jc data.txt.jc]
Uncompressed data: 2364480
2018/06/29 21:41:35 unexpected EOF
After doing a trace on the particular code in question I have come to the following answer.
/src/bytes/reader.go 70
func (r *Reader) ReadByte() (byte, error) {
...
if r.i >= int64(len(r.s)) {
return 0, io.EOF
}
....
}
There are four functions in bytes/reader that can return io.EOF, and zero functions that can return io.ErrUnexpectedEOF. The four functions that can return io.EOF are:
Read(b []byte)
ReadAt(b []byte, off int64)
ReadByte()
ReadRune()
/src/compress/flate/inflate.go 698
func (f *decompressor) moreBits() error {
c, err := f.r.ReadByte()
if err != nil {
return noEOF(err)
}
...
}
Of the four functions that can return io.EOF, only one function in flate/inflate.go calls any of them: moreBits() calls ReadByte()
/src/compress/flate/inflate.go 690
func noEOF(e error) error {
if e == io.EOF {
return io.ErrUnexpectedEOF
}
...
}
When moreBits() receives an error it calls noEOF(), which checks if it had received an io.EOF. If this was the case then io.ErrUnexpectedEOF is returned backed. Everything seems to be working as intended, and it appears that it is the user's responsibility to be on the look out for this particular case. A suggested edit to the code above to handle what appears to be defined behavior is:
func (r *compressor) decompress() []byte {
dc := flate.NewReader(bytes.NewReader(r.content))
defer dc.Close()
rb, err := ioutil.ReadAll(dc)
if err != nil {
if err != io.EOF && err != io.ErrUnexpectedEOF {
log.Fatalf("Err %v\n read %v", err, rb)
}
}
return rb
}
This was checked under go1.12.9
You got the in and outputs mixed up.
flate.NewReader takes the compressed input as an io.Reader and it returns a io.ReadCloser that can be used to get the uncompressed output:
func (r *compressor) decompress() []byte {
dc := flate.NewReader(bytes.NewReader(r.content))
defer dc.Close()
rb, err := ioutil.ReadAll(dc)
if err != nil {
if err != io.EOF {
log.Fatalf("Err %v\n read %v", err, rb)
}
}
return rb
}
Related
I want to achieve exactly opposite of the solution given here, zipping a slice of byte into another slice of byte -
Convert zipped []byte to unzip []byte golang code
Something like -
func ZipBytes(unippedBytes []byte) ([]byte, error) {
// ...
}
[I am going to upload that zipped file as multipart form data for a POST request]
You can compress directly into memory using a bytes.Buffer.
The following example uses compress/zlib since it is the opposite of the example given in the question. Depending on your use case you could easily change it to compress/gzip as well (very similar APIs).
package data_test
import (
"bytes"
"compress/zlib"
"io"
"testing"
)
func compress(buf []byte) ([]byte, error) {
var out bytes.Buffer
w := zlib.NewWriter(&out)
if _, err := w.Write(buf); err != nil {
return nil, err
}
if err := w.Close(); err != nil {
return nil, err
}
return out.Bytes(), nil
}
func decompress(buf []byte) (_ []byte, e error) {
r, err := zlib.NewReader(bytes.NewReader(buf))
if err != nil {
return nil, err
}
defer func() {
if err := r.Close(); e == nil {
e = err
}
}()
return io.ReadAll(r)
}
func TestRoundtrip(t *testing.T) {
want := []byte("test data")
zdata, err := compress(want)
if err != nil {
t.Fatalf("compress: %v", err)
}
got, err := decompress(zdata)
if err != nil {
t.Fatalf("decompress: %v", err)
}
if !bytes.Equal(want, got) {
t.Errorf("roundtrip: got = %q; want = %q", got, want)
}
}
I'm currently trying to figure out the Ethereum code and have learned how to send transactions to the blockchain using the client module.Here is an example of a contract call function:
func (ec *Client) CallContract(ctx context.Context, msg ethereum.CallMsg, blockNumber *big.Int) ([]byte, error) {
var hex hexutil.Bytes
err := ec.c.CallContext(ctx, &hex, "eth_call", toCallArg(msg), toBlockNumArg(blockNumber))
if err != nil {
return nil, err
}
return hex, nil
}
, where CallContext defined as:`
func (c *Client) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error {
if result != nil && reflect.TypeOf(result).Kind() != reflect.Ptr {
return fmt.Errorf("call result parameter must be pointer or nil interface: %v", result)
}
msg, err := c.newMessage(method, args...)
if err != nil {
return err
}
op := &requestOp{ids: []json.RawMessage{msg.ID}, resp: make(chan *jsonrpcMessage, 1)}
if c.isHTTP {
err = c.sendHTTP(ctx, op, msg)
} else {
err = c.send(ctx, op, msg)
}
if err != nil {
return err
}
// dispatch has accepted the request and will close the channel when it quits.
switch resp, err := op.wait(ctx, c); {
case err != nil:
return err
case resp.Error != nil:
return resp.Error
case len(resp.Result) == 0:
return ErrNoResult
default:
return json.Unmarshal(resp.Result, &result)
}`
And my question is: Where is the handler for these messages implemented in go - ethereum?
For example:
switch msg.name:
case "eth_call": ...
case "eth_sendTx": ...
...
Imagine these functions needs to be used how can I make this calls generic so that I don't repeat almost the same code.
with "encoding/csv"
func getDataFromCSVFiles(files []string) (error, Data) {
data := Data{}
for _, file := range files {
f, err := os.Open(file)
if err != nil {
panic(err)
return err, data
}
defer f.Close()
r := charmap.ISO8859_1.NewDecoder().Reader(f)
reader := csv.NewReader(r)
for i := 1;;i++ {
rec, err := reader.Read()
if i == 1 {
//Skipping header
continue
}
if err != nil {
if err == io.EOF {
break
}
//TODO log error line and csv filename
log.Fatal(err)
}
addWorkbook(rec, &data)
}
}
return nil, data
}
and with
import fw "github.com/hduplooy/gofixedwidth" which is almost the same except calling fw.NewReader
func getDataFromPRNFiles(files []string) (error, Data) {
data := Data{}
for _, file := range files {
f, err := os.Open(file)
if err != nil {
panic(err)
return err, data
}
defer f.Close()
r := charmap.ISO8859_1.NewDecoder().Reader(f)
reader := fw.NewReader(r)
for i := 1;;i++ {
rec, err := reader.Read()
if i == 1 {
//Skipping header
continue
}
if err != nil {
if err == io.EOF {
break
}
//TODO log error line and csv filename
log.Fatal(err)
}
addWorkbook(rec, &data)
}
}
return nil, data
}
The only apparent difference is:
reader := csv.NewReader(r)
versus:
reader := fw.NewReader(r)
I'm not sure what fw is but presumably both readers implement a common interface:
type StringSliceReader interface {
Read() ([]string, error)
}
So you could pass the openers (csv.NewReader and fw.NewReader) as function arguments:
func getDataFromFiles(files []string, func(r io.Reader) StringArrayReader) (error, Data) {
//...
}
but you'd need to wrap them in little functions to get around the return types:
func newCSVReader(r io.Reader) StringSliceReader {
return csv.NewReader(r)
}
func newFWReader(r io.Reader) StringSliceReader {
return fw.NewReader(r)
}
Also, defer queues up things to execute when the function exits, not on the next iteration of a loop. So if you do this:
for _, file := range files {
f, err := os.Open(file)
if err != nil {
panic(err)
return err, data
}
defer f.Close()
//...
}
and files has a hundred entries then you'll have a hundred open files before any of them are closed. You probably want to move that loop body to a separate function so that you only have one file open at a time.
Furthermore, error is usually the last return value from a function so you should return data, err to be more idiomatic.
The result could look something like this:
type StringSliceReader interface {
Read() ([]string, error)
}
type NewReader func(r io.Reader) StringSliceReader
func newCSVReader(r io.Reader) StringSliceReader {
return csv.NewReader(r)
}
func newFWReader(r io.Reader) StringSliceReader {
return fw.NewReader(r)
}
func getDataFrom(file string, data *Data, newReader NewReader) error {
f, err := os.Open(file)
if err != nil {
return err
}
defer f.Close()
r := charmap.ISO8859_1.NewDecoder().Reader(f)
reader := newReader(r)
for i := 1; ; i++ {
rec, err := reader.Read()
if i == 1 {
continue
}
if err != nil {
if err == io.EOF {
break
}
log.Fatal(err)
}
addWorkbook(rec, data)
}
return nil
}
func getDataFromFiles(files []string, newReader NewReader) (Data, error) {
data := Data{}
for _, file := range files {
err := getDataFrom(file, newReader, &data)
if err != nil {
panic(err)
return data, err
}
}
return data, nil
}
and you could say getDataFromFiles(files, newCSVReader) to read CSVs or getDataFromFiles(files, newFWReader) to read FW files. If you want to read from something else, you'd just need a NewReader function and something that implements the StringSliceReader interface.
You might want to bury/hide the charmap.ISO8859_1.NewDecoder().Reader(f) stuff inside the NewReader functions to make it easier to read non-Latin-1 encoded files. You could also replace newReader NewReader with a map[string]NewReader in getDataFromFiles and choose the NewReader to use based on the file's extension or other format identifier.
Is there an implementation of io.ReaderAt that can be created from an implementation of io.Reader without first being read into a []byte or string?
Something like the below. Note bytes.Reader implements the ReadAt(...) method/function: https://golang.org/pkg/bytes/#Reader.ReadAt. So the line bytes.NewReader is esssentially what you are looking for.
Getting a bytes.Reader:
var ioReader io.Reader
...
buff := bytes.NewBuffer([]byte{})
size, err := io.Copy(buff, ioReader)
if err != nil {
return err
}
reader := bytes.NewReader(buff.Bytes())
// Do something with `reader`
Yes, this is possible. As mentioned in my comment above, the implementation is limited in that you cannot seek backward nor can you re-read a section that has already been read.
Here is a example implementation:
type unbufferedReaderAt struct {
R io.Reader
N int64
}
func NewUnbufferedReaderAt(r io.Reader) io.ReaderAt {
return &unbufferedReaderAt{R: r}
}
func (u *unbufferedReaderAt) ReadAt(p []byte, off int64) (n int, err error) {
if off < u.N {
return 0, errors.New("invalid offset")
}
diff := off - u.N
written, err := io.CopyN(ioutil.Discard, u.R, diff)
u.N += written
if err != nil {
return 0, err
}
n, err = u.R.Read(p)
u.N += int64(n)
return
}
Example usage:
s := strings.NewReader("hello world")
var b [5]byte
ura := NewUnbufferedReaderAt(s)
if _, err := ura.ReadAt(b[:], 0); err != nil {
panic(err)
}
fmt.Printf("%s\n", b[:]) // prints "hello"
/*
if _, err := ura.ReadAt(b[:], 0); err != nil {
panic(err) // panics
}
fmt.Printf("%s\n", b[:])
*/
if _, err := ura.ReadAt(b[:], 6); err != nil {
panic(err)
}
fmt.Printf("%s\n", b[:]) // prints "world"
Well, part of my code was working without a method approach, I'm trying to test
append text to a file and reading from goroutines, but I'm stuck here trying to
write it.
What is wrong? the file is created, but I can't append text to it, maybe something obvious, but seems I'm blind, maybe I'm failing understanding some language concepts...
package main
import (
"bufio"
"fmt"
"os"
"sync"
"time"
)
var w sync.WaitGroup
type Buffer struct {
F *os.File
}
func (buff *Buffer) Open(pathName string) (err error) {
buff.F, err = os.OpenFile(pathName, os.O_APPEND|os.O_CREATE, 0666)
if err != nil {
return
}
fmt.Println("Open() ok")
return nil
}
func (buff *Buffer) Close() (err error) {
err = buff.F.Close()
if err != nil {
return
}
fmt.Println("Close() ok")
return nil
}
func (buff *Buffer) Push(data string) (err error) {
w := bufio.NewWriter(buff.F)
_, err = fmt.Fprintf(w, "data=%s", data)
if err != nil {
return
}
w.Flush()
fmt.Println("Push() ok")
return nil
}
func checkErr(err error) {
if err != nil {
fmt.Println(err.Error())
}
}
func worker() {
var err error
buffer := new(Buffer)
err = buffer.Open("test")
checkErr(err)
err = buffer.Push("data\n")
checkErr(err)
time.Sleep(5 * time.Second)
err = buffer.Close()
checkErr(err)
w.Done()
}
func main() {
w.Add(2)
go worker()
go worker()
w.Wait()
}
Thanks
Open the file like this:
buff.F, err = os.OpenFile(pathName, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
The write flag is required to write to the file.
You missed the write error because the return from bufio Flush is ignored. Change Push to:
func (buff *Buffer) Push(data string) (err error) {
w := bufio.NewWriter(buff.F)
_, err = fmt.Fprintf(w, "data=%s", data)
if err != nil {
return
}
err = w.Flush()
if err != nil {
return err
}
fmt.Println("Push() ok")
return nil
}
To cleanly append data without intermixing with other pushes, the data must be written with a single call to the file Write method. Use a bytes.Buffer instead of a bufio.Writer to ensure a single call to the file Write method:
func (buff *Buffer) Push(data string) (err error) {
var b bytes.Buffer
_, err = fmt.Fprintf(&b, "data=%s", data)
if err != nil {
return
}
_, err := buff.F.Write(b.Bytes())
if err != nil {
return err
}
fmt.Println("Push() ok")
return nil
}