I have a binary package of data, but length of the package is a first 4 bytes of it. Is there a correct way to read length and then read whole package with net.Conn in GoLang?
Try using this:
package main
import (
"encoding/binary"
"io"
)
func ReadPacket(r io.Reader) ([]byte, error) {
lenB := make([]byte, 4)
if _, err := r.Read(lenB); err != nil {
return nil, err
}
//you can use BigEndian depending on the proto
l := binary.LittleEndian.Uint32(lenB)
packet := make([]byte, l)
_, err := r.Read(packet)
return packet, err
}
Related
I want to generate 64-bit random integer using the secure crypto/rand package. I found this online:
package main
import (
"crypto/rand"
"encoding/base64"
)
// GenerateRandomBytes returns securely generated random bytes.
// It will return an error if the system's secure random
// number generator fails to function correctly, in which
// case the caller should not continue.
func GenerateRandomBytes(n int) ([]byte, error) {
b := make([]byte, n)
_, err := rand.Read(b)
// Note that err == nil only if we read len(b) bytes.
if err != nil {
return nil, err
}
return b, nil
}
But it seems to generate random bytes instead. I want a random 64-bit int. Namely, I want something like var i uint64 = rand(). Any ideas how to achieve this?
You can generate a random number with crypto.Rand, and then convert those bytes to an int64 using the binary package:
func randint64() (int64, error) {
var b [8]byte
if _, err := rand.Read(b[:]); err != nil {
return 0, err
}
return int64(binary.LittleEndian.Uint64(b[:])), nil
}
https://play.golang.org/p/2Q8tvttqbJ (result is cached)
If you look at the source code for LittleEndian.Uint64, you can see it's simply performing a few bit operations on the data; something that you could implemented for yourself.
You also use rand.Int in crypto/rand package
func randint64() (int64, error) {
val, err := rand.Int(rand.Reader, big.NewInt(int64(math.MaxInt64)))
if err != nil {
return 0, err
}
return val.Int64(), nil
}
https://play.golang.org/p/fqoQxpmjOSu
I'm trying to figure out how the best way to write a binary file in Go that corresponds with the following Python:
import struct
f = open('tst.bin', 'wb')
fmt = 'iih'
f.write(struct.pack(fmt,4, 185765, 1020))
f.close()
I have been tinkering with some of the examples I've seen on Github.com and a few other sources but I can't seem to get anything working correctly. What is the idiomatic way to do this sort of thing in Go?
Here is how I am accomplishing this now (Golang):
package main
import (
"fmt"
"os"
"encoding/binary"
)
func main() {
fp, err := os.Create("tst.bin")
if err != nil {
panic(err)
}
defer fp.Close()
aBuf := make([]byte, 4)
bBuf := make([]byte, 4)
cBuf := make([]byte, 2)
binary.LittleEndian.PutUint32(aBuf, 4)
binary.LittleEndian.PutUint32(bBuf, 185765)
binary.LittleEndian.PutUint16(cBuf, 1020)
binary.Write(fp, binary.LittleEndian, aBuf)
binary.Write(fp, binary.LittleEndian, bBuf)
binary.Write(fp, binary.LittleEndian, cBuf)
}
Verified with this Python:
import numpy as np
data = np.fromfile('tst.bin', dtype='i4,i4,i2')
print(data)
As I mentioned in my question, I'm not sure this is THE idiomatic way to do this but after a fair bit of tinkering this does solve the problem I was having. Posting so there is at least one answer in case someone else comes looking:
package main
import (
"fmt"
"os"
"encoding/binary"
)
func main() {
fp, err := os.Create("tst.bin")
if err != nil {
panic(err)
}
defer fp.Close()
aBuf := make([]byte, 4)
bBuf := make([]byte, 4)
cBuf := make([]byte, 2)
binary.LittleEndian.PutUint32(aBuf, 4)
binary.LittleEndian.PutUint32(bBuf, 185765)
binary.LittleEndian.PutUint16(cBuf, 1020)
binary.Write(fp, binary.LittleEndian, aBuf)
binary.Write(fp, binary.LittleEndian, bBuf)
binary.Write(fp, binary.LittleEndian, cBuf)
}
Slightly cleaning this up to make it more brief I'll be doing something like this:
func main() {
fp, err := os.Create("tst.bin")
if err != nil {
panic(err)
}
defer fp.Close()
lineBuf := make([]byte, 10) //sized large enough to hold my two 4 byte ints and one 2 byte int
binary.LittleEndian.PutUint32(lineBuf[0:4], 4) //packing into 4 bytes
binary.LittleEndian.PutUint32(lineBuf[4:8], 185765)
binary.LittleEndian.PutUint16(lineBuf[8:10], 1020) //packing into 2 bytes
binary.Write(fp, binary.LittleEndian, lineBuf)
}
After a little more tinkering and some feedback on another question I was able to come up with this (seems much cleaner but will post feedback on performance after testing with generating some large files):
package main
import (
"os"
"encoding/binary"
)
type binData struct {
A int32
B int32
C int16
}
func main() {
fp, err := os.Create("tst.bin")
if err != nil {
panic(err)
}
defer fp.Close()
bd := binData{A:4, B:185765, C:1020}
binary.Write(fp, binary.LittleEndian, &bd)
}
Say I would like to generate a secure random int between 0 and 27 using:
func Int(rand io.Reader, max *big.Int) (n *big.Int, err error)
in the "crypto/rand" package.
How would I do that?
I do not really understand how this works, why does it not return one of the built in Go ints instead of pointer to some big.Int type?
EDIT:
Would this be considered secure enough for tokens?
func getToken(length int) string {
token := ""
codeAlphabet := "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
codeAlphabet += "abcdefghijklmnopqrstuvwxyz"
codeAlphabet += "0123456789"
for i := 0; i < length; i++ {
token += string(codeAlphabet[cryptoRandSecure(int64(len(codeAlphabet)))])
}
return token
}
func cryptoRandSecure(max int64) int64 {
nBig, err := rand.Int(rand.Reader, big.NewInt(max))
if err != nil {
log.Println(err)
}
return nBig.Int64()
}
func main() {
fmt.Println(getToken(32))
}
This would output something like this:
qZDbuPwNQGrgVmZCU9A7FUWbp8eIfn0Z
EwZVoQ5D5SEfdhiRsDfH6dU6tAovILCZ
cOqzODVP0GwbiNBwtmqLA78rFgV9d3VT
Here is some working code :
package main
import (
"fmt"
"crypto/rand"
"math/big"
)
func main() {
nBig, err := rand.Int(rand.Reader, big.NewInt(27))
if err != nil {
panic(err)
}
n := nBig.Int64()
fmt.Printf("Here is a random %T in [0,27) : %d\n", n, n)
}
But to generate a random token, I'd do something like this :
package main
import (
"crypto/rand"
"encoding/base32"
"fmt"
)
func main() {
token := getToken(10)
fmt.Println("Here is a random token : ", token)
}
func getToken(length int) string {
randomBytes := make([]byte, 32)
_, err := rand.Read(randomBytes)
if err != nil {
panic(err)
}
return base32.StdEncoding.EncodeToString(randomBytes)[:length]
}
If you're generating secure tokens for session IDs, OAuth Bearer tokens, CSRF or similar: you want to generate a token of (ideally) 256 bits (32 bytes) or no less than 192 bits (24 bytes).
A token with values between (0-27) can be brute-forced in less than a second and could not be considered secure.
e.g.
package main
import (
"crypto/rand"
"encoding/base64"
)
// GenerateRandomBytes returns securely generated random bytes.
// It will return an error if the system's secure random
// number generator fails to function correctly, in which
// case the caller should not continue.
func GenerateRandomBytes(n int) ([]byte, error) {
b := make([]byte, n)
_, err := rand.Read(b)
// Note that err == nil only if we read len(b) bytes.
if err != nil {
return nil, err
}
return b, nil
}
// GenerateRandomString returns a URL-safe, base64 encoded
// securely generated random string.
func GenerateRandomString(s int) (string, error) {
b, err := GenerateRandomBytes(s)
return base64.URLEncoding.EncodeToString(b), err
}
func main() {
// Example: this will give us a 44 byte, base64 encoded output
token, err := GenerateRandomString(32)
if err != nil {
// Serve an appropriately vague error to the
// user, but log the details internally.
}
}
The base64 output is safe for headers, HTTP forms, JSON bodies, etc.
If you need an integer it may help to explain your use-case, as it would be odd for a system to require tokens as ints.
If you only need a small number (i.e. [0, 255]), you could just read a byte out of the package's Reader:
b := []byte{0}
if _, err := rand.Reader.Read(b); err != nil {
panic(err)
}
n := b[0]
fmt.Println(n)
Playground: http://play.golang.org/p/4VO52LiEVh (the example won't work there, I don't know if it's working as intended or it's a playground bug).
I am trying to read a buffered stream of signed 16 bit integers (wav format), but the bufio.Read method only accepts an array of bytes. My question is a 2-parter:
Can I preformat the byte stream into a buffered int16 array?
If I can't, whats the best way of post-processing the byte array into int16 array? My initial thought is to use tmp arrays and keep pushing/processing them, but I was curious if there was a more idiomatic way of doing this?
package main
import (
"bufio"
"io"
"log"
"os/exec"
)
func main() {
app := "someapp"
cmd := exec.Command(app)
stdout, err := cmd.StdoutPipe()
r := bufio.NewReader(stdout)
if err != nil {
log.Fatal(err)
}
if err := cmd.Start(); err != nil {
log.Fatal(err)
}
//"someapp" outputs signed 16bit integers (little endian))
buf := make([]byte, 0, 4*1024)
for {
n, err := r.Read(buf[:cap(buf)]) //r.Read only accepts type []byte
buf = buf[:n]
if n == 0 {
if err == nil {
continue
}
if err == io.EOF {
break
}
log.Fatal(err)
}
log.Printf("%x\n", buf)
//process buf here
if err != nil && err != io.EOF {
log.Fatal(err)
}
}
}
When working with IO, you always work with []bytes, there's no way to substitute that with []int16, or pre-format that as int16s, it's always a stream of bytes.
You can look at the encoding/binary package to decode this stream.
// to get the first uint16 as i
i := binary.LittleEndian.Uint16(buf[:2])
You can then iterate through the buf as needed.
You can also use binary.Read to read directly from the io.Reader.
var i uint16
for {
err := binary.Read(r, binary.LittleEndian, &i)
if err != nil {
log.Println(err)
break
}
fmt.Println(i)
}
It may worth noting the simplicity of what needs to be done. Each uint16 is created via:
func (littleEndian) Uint16(b []byte) uint16 {
return uint16(b[0]) | uint16(b[1])<<8
}
You can use encoding/binary.Read to fill an []int16 directly from your reader, although technically the answer to your first question is still no (check the source of binary.Read, it reads the data to a []byte first).
I am trying to parse a file that annoying consists of many separately zipped segments. I have parsed these segments one at a time into a slice of bytes and I want to uncompress them as I go.
Here is my current code that does the decompressing, which doesn't work. from and to are just set at the top as an example, in reality they are set by the code. data is the byte array containing the entire file. I don't want to seek it while it's on disk because its location on another server, so it's only realistic for me to load the entire file to []byte first and then parse it.
from, to := 0, 1000;
b := bytes.NewReader(data[from:from+to])
z, err := zlib.NewReader(b)
CheckErr(err)
defer z.Close()
p := make([]byte,0,1024)
z.Read(p)
fmt.Println(string(p))
So how is it so massively difficult just to unzip a slice of bytes? Anyway...
The problem appears to with how I am reading it out. Where it says z.Read, that doesn't seem to do anything.
How can I read the entire thing in one go into a slice of bytes?
Here's an outline for you. Note: In Go, CHECK FOR ERRORS!
package main
import (
"bytes"
"compress/zlib"
"fmt"
"io/ioutil"
)
func readSegment(data []byte, from, to int) ([]byte, error) {
b := bytes.NewReader(data[from : from+to])
z, err := zlib.NewReader(b)
if err != nil {
return nil, err
}
defer z.Close()
p, err := ioutil.ReadAll(z)
if err != nil {
return nil, err
}
return p, nil
}
func main() {
from, to := 0, 1000
data := make([]byte, from+to)
// ** parse input segments into data **
p, err := readSegment(data, from, to)
if err != nil {
fmt.Println(err)
return
}
fmt.Println(string(p))
}
Use ReadAll(r io.Reader) ([]byte, error) from the io/ioutil package.
p, err := ioutil.ReadAll(b)
fmt.Println(string(p))
Read only reads up to the length of the given slice (1024 bytes in your case).
To read in chunks of 1024 bytes:
p := make([]byte,1024)
for {
numBytes, err := l.Read(p)
if err == io.EOF {
// you are done, numBytes might be less than len(p)
break
}
// do what you want with p
}
If you are getting the data from a webserver, you might even do
import (
"net/http"
"io/ioutil"
)
...
resp, errGet := http.Get("http://example.com/somefile")
// do error handling
z, errZ := zlib.NewReader(resp.Body)
// do error handling
resp.Body.Close()
p, err := ioutil.ReadAll(b)
// do error handling
since resp.Body happens to be an io.Reader as most io related types.