doc: https://developers.google.com/ad-exchange/rtb/response-guide/decrypt-price#sample_code
There is no official "Double Click Crypto" example code in golang, so I try to implement by myself. But I can't pass the test in doc. Please help me!
skU7Ax_NL5pPAFyKdkfZjZz2-VhIN8bjj1rVFOaJ_5o= // Encryption key (e_key)
arO23ykdNqUQ5LEoQ0FVmPkBd7xB5CO89PDZlSjpFxo= // Integrity key (i_key)
WEp8wQAAAABnFd5EkB2k1wJeFcAj-Z_JVOeGzA // 100 CPI micros
WEp8sQAAAACwF6CtLJrXSRFBM8UiTTIyngN-og // 1900 CPI micros
WEp8nQAAAAADG-y45xxIC1tMWuTjzmDW6HtroQ // 2700 CPI micros
Here is my code:
package main
// https://developers.google.com/ad-exchange/rtb/response-guide/decrypt-price?hl=zh-CN
import (
"bytes"
"crypto/hmac"
"crypto/sha1"
"encoding/base64"
"encoding/binary"
"fmt"
)
var base64Codec = base64.URLEncoding.WithPadding(base64.NoPadding)
func safeXORBytes(dst, a, b []byte) int {
n := len(a)
if len(b) < n {
n = len(b)
}
for i := 0; i < n; i++ {
dst[i] = a[i] ^ b[i]
}
return n
}
func EncryptPrice(eKey []byte, iKey []byte, price uint64, iv []byte) (finalMessage string, err error) {
if len(iv) != 16 {
err = fmt.Errorf("len(iv) = %d != 16", len(iv))
return
}
h1 := hmac.New(sha1.New, eKey)
h1.Write(iv)
pad := h1.Sum(nil)[:8]
priceBytes := make([]byte, 8)
binary.BigEndian.PutUint64(priceBytes, price)
encPrice := make([]byte, 8)
n := safeXORBytes(encPrice, priceBytes, pad)
if n != 8 {
err = fmt.Errorf("safeXORBytes n != %d", n)
return
}
h2 := hmac.New(sha1.New, iKey)
h2.Write(priceBytes)
h2.Write(iv)
signature := h2.Sum(nil)[:4]
finalMessage = base64Codec.EncodeToString(append(append(iv, encPrice...), signature...))
return
}
func DecryptPrice(eKey []byte, iKey []byte, finalMessage string) (price uint64, err error) {
finalMessageBytes, err := base64Codec.DecodeString(finalMessage)
if err != nil {
return
}
if len(finalMessageBytes) != 28 {
err = fmt.Errorf("len(finalMessageBytes) = %d != 28", len(finalMessageBytes))
return
}
iv := finalMessageBytes[:16]
encPrice := finalMessageBytes[16:24]
signature := finalMessageBytes[24:]
h1 := hmac.New(sha1.New, eKey)
h1.Write(iv)
pad := h1.Sum(nil)[:8]
priceBytes := make([]byte, 8)
n := safeXORBytes(priceBytes, encPrice, pad)
if n != 8 {
err = fmt.Errorf("safeXORBytes n != %d", n)
return
}
h2 := hmac.New(sha1.New, iKey)
h2.Write(priceBytes)
h2.Write(iv)
confSignature := h2.Sum(nil)[:4]
if bytes.Compare(confSignature, signature) != 0 {
err = fmt.Errorf("sinature mismatch: confSignature = %s, sinature = %s", confSignature, signature)
return
}
price = binary.BigEndian.Uint64(priceBytes)
return
}
package main
import (
"github.com/stretchr/testify/assert"
"testing"
)
func TestEncryptPrice(t *testing.T) {
eKeyBase64Encoded := "skU7Ax_NL5pPAFyKdkfZjZz2-VhIN8bjj1rVFOaJ_5o"
eKey, err := base64Codec.DecodeString(eKeyBase64Encoded)
assert.Nil(t, err)
iKeyBase64Encoded := "arO23ykdNqUQ5LEoQ0FVmPkBd7xB5CO89PDZlSjpFxo"
iKey, err := base64Codec.DecodeString(iKeyBase64Encoded)
assert.Nil(t, err)
finalMessage, err := EncryptPrice(eKey, iKey, uint64(100), []byte{88, 74, 124, 193, 0, 0, 0, 0, 103, 21, 222, 68, 144, 29, 164, 215})
assert.Nil(t, err)
assert.Equal(t, "WEp8wQAAAABnFd5EkB2k1wJeFcAj-Z_JVOeGzA", finalMessage)
}
func TestDecryptPrice(t *testing.T) {
eKeyBase64Encoded := "skU7Ax_NL5pPAFyKdkfZjZz2-VhIN8bjj1rVFOaJ_5o"
eKey, err := base64Codec.DecodeString(eKeyBase64Encoded)
assert.Nil(t, err)
iKeyBase64Encoded := "arO23ykdNqUQ5LEoQ0FVmPkBd7xB5CO89PDZlSjpFxo"
iKey, err := base64Codec.DecodeString(iKeyBase64Encoded)
assert.Nil(t, err)
price, err := DecryptPrice(eKey, iKey, "WEp8wQAAAABnFd5EkB2k1wJeFcAj-Z_JVOeGzA")
assert.Nil(t, err)
assert.Equal(t, uint64(100), price)
}
test result:
--- FAIL: TestEncryptPrice (0.00s)
Error Trace: price_test.go:19
Error: Not equal:
expected: "WEp8wQAAAABnFd5EkB2k1wJeFcAj-Z_JVOeGzA"
actual: "WEp8wQAAAABnFd5EkB2k1wf7jAcG7gZ9tPUnAA"
--- FAIL: TestDecryptPrice (0.00s)
Error Trace: price_test.go:32
Error: Expected nil, but got: &errors.errorString{s:"sinature mismatch: confSignature = \xbe\xaa\xf6h, sinature = T\xe7\x86\xcc"}
Error Trace: price_test.go:33
Error: Not equal:
expected: 0x64
actual: 0x0
FAIL
this code can pass the test in https://github.com/google/openrtb-doubleclick/blob/0.9.0/doubleclick-core/src/test/java/com/google/doubleclick/crypto/DoubleClickCryptoTest.java
, so I guess that the encrypted price in the doc are not excrypted by the e_key and i_key in the doc.
I met the same problem today. Finally, I found the bug was that 'ekey' and 'ikey' did not need to base64Codec.DecodeString, just use []byte(eKeyBase64Encoded).
eKeyBase64Encoded := "skU7Ax_NL5pPAFyKdkfZjZz2-VhIN8bjj1rVFOaJ_5o"
eKey := []byte(eKeyBase64Encoded)
assert.Nil(t, err)
iKeyBase64Encoded := "arO23ykdNqUQ5LEoQ0FVmPkBd7xB5CO89PDZlSjpFxo"
iKey :=[]byte(iKeyBase64Encoded)
assert.Nil(t, err)
price, err := DecryptPrice(eKey, iKey, "WEp8wQAAAABnFd5EkB2k1wJeFcAj-Z_JVOeGzA")
....
The encrypted price that are listed in this document https://developers.google.com/authorized-buyers/rtb/response-guide/decrypt-price have been encoded using the keys they showed above the prices. The examples are correct. I wrote a small Go package that provides a Decrypt function, I might add an encrypt one soon: https://github.com/matipan/doubleclick. Below I leave an example and code snippets for easy access.
First, to be able to parse the keys we need to respect the web-safe base64 decoding, which means we need to use base64.URLEncoding! to decode the keys:
// ParseKeys parses the base64 web-safe encoded keys as explained in Google's documentation:
// https://developers.google.com/authorized-buyers/rtb/response-guide/decrypt-price
func ParseKeys(ic, ec []byte) (icKey []byte, ecKey []byte, err error) {
icKey = make([]byte, base64.URLEncoding.DecodedLen(len([]byte(ic))))
n, err := base64.URLEncoding.Decode(icKey, []byte(ic))
if err != nil {
return nil, nil, fmt.Errorf("%w: could not decode price integrity key", err)
}
icKey = icKey[:n]
ecKey = make([]byte, base64.URLEncoding.DecodedLen(len([]byte(ec))))
n, err = base64.URLEncoding.Decode(ecKey, []byte(ec))
if err != nil {
return nil, nil, fmt.Errorf("%w: could not decode price encryption key", err)
}
ecKey = ecKey[:n]
return icKey, ecKey, nil
}
We can call this code and decrypt the keys they show:
icKey, ecKey, err = ParseKeys([]byte("arO23ykdNqUQ5LEoQ0FVmPkBd7xB5CO89PDZlSjpFxo="), []byte("skU7Ax_NL5pPAFyKdkfZjZz2-VhIN8bjj1rVFOaJ_5o="))
The function to decrypt the price with a bit more error handling is:
// ErrInvalidPrice is the error returned when the price parsed
// by DecryptPrice is not correct.
var ErrInvalidPrice = errors.New("price is invalid")
// DecryptPrice decrypts the price with google's doubleclick cryptography encoding.
// encPrice is an unpadded web-safe base64 encoded string according to RFC 3548.
// https://developers.google.com/authorized-buyers/rtb/response-guide/decrypt-price
func DecryptPrice(icKey, ecKey, encPrice []byte) (uint64, error) {
if len(icKey) == 0 || len(ecKey) == 0 {
return 0, errors.New("encryption and integrity keys are required")
}
if len(encPrice) != 38 {
return 0, fmt.Errorf("%w: invalid length, expected 28 got %d", ErrInvalidPrice, len(encPrice))
}
dprice := make([]byte, base64.RawURLEncoding.DecodedLen(len(encPrice)))
n, err := base64.RawURLEncoding.Decode(dprice, encPrice)
if err != nil {
return 0, fmt.Errorf("%w: invalid base64 string", err)
}
dprice = dprice[:n]
if len(dprice) != 28 {
return 0, fmt.Errorf("%w: invalid decoded price length. Expected 28 got %d", ErrInvalidPrice, len(dprice))
}
// encrypted price is composed of parts of fixed lenth. We break it up according to:
// {initialization_vector (16 bytes)}{encrypted_price (8 bytes)}{integrity (4 bytes)}
iv, p, sig := dprice[0:16], dprice[16:24], dprice[24:]
h := hmac.New(sha1.New, ecKey)
n, err = h.Write(iv)
if err != nil || n != len(iv) {
return 0, fmt.Errorf("%w: could not write hmac hash for iv. err=%s, n=%d, len(iv)=%d", ErrInvalidPrice, err, n, len(iv))
}
pricePad := h.Sum(nil)
price := safeXORBytes(p, pricePad)
if price == nil {
return 0, fmt.Errorf("%w: price xor price_pad failed", ErrInvalidPrice)
}
h = hmac.New(sha1.New, icKey)
n, err = h.Write(price)
if err != nil || n != len(price) {
return 0, fmt.Errorf("%w: could not write hmac hash for price. err=%s, n=%d, len(price)=%d", ErrInvalidPrice, err, n, len(price))
}
n, err = h.Write(iv)
if err != nil || n != len(iv) {
return 0, fmt.Errorf("%w: could not write hmac hash for iv. err=%s, n=%d, len(iv)=%d", ErrInvalidPrice, err, n, len(iv))
}
confSig := h.Sum(nil)[:4]
if bytes.Compare(confSig, sig) != 0 {
return 0, fmt.Errorf("%w: integrity of price is not valid", ErrInvalidPrice)
}
return binary.BigEndian.Uint64(price), nil
}
func safeXORBytes(a, b []byte) []byte {
n := len(a)
if len(b) < n {
n = len(b)
}
if n == 0 {
return nil
}
dst := make([]byte, n)
for i := 0; i < n; i++ {
dst[i] = a[i] ^ b[i]
}
return dst
}
We can now call this code using the example of 1900 micros: YWJjMTIzZGVmNDU2Z2hpN7fhCuPemC32prpWWw:
price, err := DecryptPrice(icKey, ecKey, []byte("YWJjMTIzZGVmNDU2Z2hpN7fhCuPemC32prpWWw"))
And price will be 1900
Related
I'm looking for encrypt string in my Go application and decrypt the encoded string with Crypto-js.
I have been trying for hours without success, trying many of the solutions offered by Stackoverflow, github or gist.
If anyone has the solution they would save me from a certain nervous breakdown lol
My Go encrypt code:
func EncryptBody() (encodedmess string, err error) {
key := []byte("6368616e676520746869732070617373")
// Create the AES cipher
block, err := aes.NewCipher(key)
if err != nil {
panic(err)
}
plaintext, _ := pkcs7Pad([]byte("exampletext"), block.BlockSize())
// The IV needs to be unique, but not secure. Therefore it's common to
// include it at the beginning of the ciphertext.
ciphertext := make([]byte, aes.BlockSize+len(plaintext))
iv := ciphertext[:aes.BlockSize]
if _, err := io.ReadFull(rand.Reader, iv); err != nil {
panic(err)
}
bm := cipher.NewCBCEncrypter(block, iv)
bm.CryptBlocks(ciphertext[aes.BlockSize:], plaintext)
return fmt.Sprintf("%x", ciphertext), nil
}
My pkcs7Pad function :
func pkcs7Pad(b []byte, blocksize int) ([]byte, error) {
if blocksize <= 0 {
return nil, errors.New("invalid blocksize")
}
if b == nil || len(b) == 0 {
return nil, errors.New("invalid PKCS7 data (empty or not padded)")
}
n := blocksize - (len(b) % blocksize)
pb := make([]byte, len(b)+n)
copy(pb, b)
copy(pb[len(b):], bytes.Repeat([]byte{byte(n)}, n))
return pb, nil
}
My Crypto-JS decrypt code :
public decryptData() {
const data = "3633fbef042b01da5fc4b69d8f038a83130994a898137bb0386604cf2c1cbbe6"
const key = "6368616e676520746869732070617373"
const decrypted = CryptoJS.AES.decrypt(data, key, {
mode: CryptoJS.mode.CBC,
padding: CryptoJS.pad.Pkcs7
})
console.log("Result : " + decrypted.toString(CryptoJS.enc.Hex))
return decrypted.toString(CryptoJS.enc.Hex)
}
Thanks to #Topaco for your help !
Solution :
Go code :
func EncryptBody(data string) (encodedmess string, err error) {
key := []byte("6368616e676520746869732070617373")
// Create the AES cipher
block, err := aes.NewCipher(key)
if err != nil {
panic(err)
}
plaintext, _ := pkcs7Pad([]byte(data), block.BlockSize())
// The IV needs to be unique, but not secure. Therefore it's common to
// include it at the beginning of the ciphertext.
ciphertext := make([]byte, aes.BlockSize+len(plaintext))
iv := ciphertext[:aes.BlockSize]
if _, err := io.ReadFull(rand.Reader, iv); err != nil {
panic(err)
}
bm := cipher.NewCBCEncrypter(block, iv)
bm.CryptBlocks(ciphertext[aes.BlockSize:], plaintext)
return fmt.Sprintf("%x", ciphertext), nil
}
NodeJS Code :
protected decryptData(data: string) {
const iv = CryptoJS.enc.Hex.parse(data.substr(0,32))
const ct = CryptoJS.enc.Hex.parse(data.substr(32))
const key = CryptoJS.enc.Utf8.parse("6368616e676520746869732070617373")
// #ts-ignore !!!!!!!! IMPORTANT IF YOU USE TYPESCRIPT COMPILER
const decrypted = CryptoJS.AES.decrypt({ciphertext: ct}, key, {
mode: CryptoJS.mode.CBC,
iv: iv
})
console.log("Result : " + decrypted.toString(CryptoJS.enc.Utf8))
return decrypted.toString(CryptoJS.enc.Utf8)
}
I successfully write Byte array to port but failed to read from it. function "read"
return "read /dev/tty.usbserial1: interrupted system call" error, I running it on macOS sierra 10.12.6, use "github.com/tarm/serial", anyone met the same problem?
here is the code:
func TestTarmSerialDataRecvAndSend(t *testing.T) {
c := &serial.Config{
Name: "/dev/cu.usbserial1",
Baud: 9600,
}
s, err := serial.OpenPort(c)
if err != nil {
fmt.Println(err)
return
}
for i := 0; i < 10; i++ {
n, err := s.Write([]byte("test"))
if err != nil {
fmt.Println(err)
return
}
fmt.Printf("send byte : %v", n)
}
fmt.Println("send over")
buf := make([]byte, 128)
n, err := s.Read(buf)
if err != nil {
fmt.Println(err)
return
}
fmt.Printf("%q", buf[:n])
}
func sendCommand(p *serial.Port, command string, waitForOk bool) string {
log.Println("--- SendCommand: ", command)
var status string = ""
p.Flush()
_, err := p.Write([]byte(command))
if err != nil {
log.Fatal(err)
}
buf := make([]byte, 32)
var loop int = 1
if waitForOk {
loop = 10
}
for i := 0; i < loop; i++ {
// ignoring error as EOF raises error on Linux
n, _ := p.Read(buf)
if n > 0 {
status = string(buf[:n])
log.Printf("SendCommand: rcvd %d bytes: %s\n", n, status)
if strings.HasSuffix(status, "OK\r\n") || strings.HasSuffix(status, "ERROR\r\n") {
break
}
}
}
return status
}
the solutions is here: https://golang.hotexamples.com/examples/github.com.tarm.serial/Port/Read/golang-port-read-method-examples.html
I want get increment id with file, code as below:
// get increment id
func GetID() uint64 {
appIdLock.Lock()
defer appIdLock.Unlock()
f, err := os.OpenFile(idPath, os.O_RDWR, 0666)
if err != nil {
return 0
}
defer f.Close()
// Read
bufferTemp := make([]byte, 16)
bufferResult := make([]byte, 0)
for {
n, _ := f.Read(bufferTemp)
if n > 0 {
bufferResult = append(bufferResult, bufferTemp[:n]...)
} else {
break
}
}
if len(bufferResult) == 0 {
return 0
}
s := common.StringToUint64(string(bufferResult))
s += 1
// Write (how to cover?)
f.WriteString(strconv.FormatUint(s, 10))
return s
}
f.WriteString function was append, example, my file content: 123, run GetID() I hope my file content is: 124, But result was: 123124
Without changing much of your code, here is the solution that will work and do what you want to do. No loops or more than a single byte slice.
func GetID() uint64 {
appIdLock.Lock()
defer appIdLock.Unlock()
// Added + os.O_CREATE to create the file if it doesn't exist.
f, err := os.OpenFile(idPath, os.O_RDWR + os.O_CREATE, 0666)
if err != nil {
fmt.Println(err)
return 0
}
defer f.Close()
// Know file content beforehand so I allocate a suitable bytes slice.
fileStat, err := f.Stat()
if err != nil {
fmt.Println(err)
return 0
}
buffer := make([]byte, fileStat.Size())
_, err = f.Read(buffer)
if err != nil {
fmt.Println(err)
return 0
}
s, _ := strconv.ParseUint(string(buffer), 10, 64)
s += 1
// The Magic is here ~ Writes bytes at 0 index of the file.
_, err = f.WriteAt([]byte(strconv.FormatUint(s, 10)), 0)
if err != nil {
fmt.Println(err)
return 0
}
return s
}
Hope it helps!
I want to use gob to encode and decode object, I do it like this:
type transProp struct {
a []int
b []float64
}
func (p transProp) MarshalBinary() ([]byte, error) {
// A simple encoding: plain text.
var b bytes.Buffer
fmt.Fprintln(&b, p.a, p.b)
return b.Bytes(), nil
}
// UnmarshalBinary modifies the receiver so it must take a pointer receiver.
func (p *transProp) UnmarshalBinary(data []byte) error {
// A simple encoding: plain text.
b := bytes.NewBuffer(data)
_, err := fmt.Fscanln(b, &p.a, &p.b)
return err
}
func TestGobEncode(t *testing.T) {
p := transProp{
a: []int{3, 4, 5},
b: []float64{1.0, 2.0},
}
var network bytes.Buffer
enc := gob.NewEncoder(&network)
err := enc.Encode(p)
if err != nil {
log.Fatal("encode:", err)
}
dec := gob.NewDecoder(&network)
var p1 transProp
err = dec.Decode(&p1)
if err != nil {
log.Fatal("decode:", err)
}
fmt.Println(p1)
}
But, this report error like this:
decode:can't scan type: *[]int
If you can change the visibility of transProp fields to public, e.g.
type transProp struct {
A []int
B []float64
}
then you don't need to implement custom binary marshaller/unmarshaller. You can use default gob decoder/encoder.
However, if you can't, there are many options.
The simplest one, define another wrapper struct that export related fields, wrap transProp, then use default gob encoder/decoder, e.g.
type wrapTransProp struct {
A []int
B []float64
}
func (p transProp) MarshalBinary() ([]byte, error) {
//Wrap struct
w := wrapTransProp{p.a, p.b}
//use default gob encoder
var buf bytes.Buffer
enc := gob.NewEncoder(&buf)
if err := enc.Encode(w); err != nil {
return nil, err
}
return buf.Bytes(), nil
}
func (p *transProp) UnmarshalBinary(data []byte) error {
w := wrapTransProp{}
//Use default gob decoder
reader := bytes.NewReader(data)
dec := gob.NewDecoder(reader)
if err := dec.Decode(&w); err != nil {
return err
}
p.a = w.A
p.b = w.B
return nil
}
Custom marshaller/unmarshaller with custom data layout. There are many implementation possibilities. Several considerations:
Byte order, little/big endian?
Packet/stream layout?
An example implementation of big-endian with stream format:
// Big-Endian
// Size : 4, 4, 1, n, 4, n
// Types : uint32, uint32, uint8, []int, uint32, []float64
// Data : #numbytes, #nInt, #intSize, p.a, #nFloat, p.b
The challenge is in how to represent int since it's architecture dependent. Sample implementation will be:
func (p transProp) MarshalBinary() ([]byte, error) {
//Get sizeof int (in bits) from strconv package
szInt := strconv.IntSize / 8
nInt := len(p.a)
nFloat := len(p.b)
nStream := 4 + 4 + 1 + nInt*szInt + 4 + nFloat*8
stream := make([]byte, nStream)
pos := 0
//total number of bytes
binary.BigEndian.PutUint32(stream, uint32(nStream))
pos += 4
//num of items in p.a
binary.BigEndian.PutUint32(stream[pos:], uint32(nInt))
pos += 4
//int size
stream[pos] = uint8(szInt)
pos++
//items in a
switch szInt {
case 1:
for _, v := range p.a {
stream[pos] = uint8(v)
pos++
}
case 2: //16-bit
for _, v := range p.a {
binary.BigEndian.PutUint16(stream[pos:], uint16(v))
pos += 2
}
case 4: //32-bit
for _, v := range p.a {
binary.BigEndian.PutUint32(stream[pos:], uint32(v))
pos += 4
}
case 8: //64-bit
for _, v := range p.a {
binary.BigEndian.PutUint64(stream[pos:], uint64(v))
pos += 8
}
}
//number of items in p.b
binary.BigEndian.PutUint32(stream[pos:], uint32(nFloat))
pos += 4
//items in b
s := stream[pos:pos] //slice len=0, capacity=nFloat
buf := bytes.NewBuffer(s)
if err := binary.Write(buf, binary.BigEndian, p.b); err != nil {
return nil, err
}
return stream, nil
}
func (p *transProp) UnmarshalBinary(data []byte) error {
buf := bytes.NewBuffer(data)
var intSize uint8
var k, nBytes, nInt, nFloat uint32
//num bytes
if err := binary.Read(buf, binary.BigEndian, &nBytes); err != nil {
return err
}
if len(data) < int(nBytes) {
return errors.New("len(data) < #Bytes")
}
//num of int items
if err := binary.Read(buf, binary.BigEndian, &nInt); err != nil {
return err
}
//int size
if err := binary.Read(buf, binary.BigEndian, &intSize); err != nil {
return err
}
//read int into p.a
pos := 0
stream := buf.Bytes()
p.a = make([]int, nInt)
switch intSize {
case 1:
for pos = 0; pos < int(nInt); pos++ {
p.a[pos] = int(stream[pos])
}
case 2:
for k = 0; k < nInt; k++ {
p.a[k] = int(binary.BigEndian.Uint16(stream[pos:]))
pos += 2
}
case 4:
for k = 0; k < nInt; k++ {
p.a[k] = int(binary.BigEndian.Uint32(stream[pos:]))
pos += 4
}
case 8:
for k = 0; k < nInt; k++ {
p.a[k] = int(binary.BigEndian.Uint64(stream[pos:]))
pos += 8
}
}
//advance buffer
buf.Next(pos)
//num of float64 items
if err := binary.Read(buf, binary.BigEndian, &nFloat); err != nil {
return err
}
//items in b
p.b = make([]float64, nFloat)
if err := binary.Read(buf, binary.BigEndian, p.b); err != nil {
return err
}
return nil
}
I have some a Go client for a custom protocol. The protocol is lz4-compressed JSON-RPC with a four byte header giving the length of the compressed JSON.
func ReceiveMessage(conn net.Conn) ([]byte, error) {
start := time.Now()
bodyLen := 0
body := make([]byte, 0, 4096)
buf := make([]byte, 0, 256)
for bodyLen == 0 || len(body) < bodyLen {
if len(body) > 4 {
header := body[:4]
body = body[:4]
bodyLen = int(unpack(header))
}
n, err := conn.Read(buf[:])
if err != nil {
if err != io.EOF {
return body, err
}
}
body = append(body, buf[0:n]...)
now := time.Now()
if now.Sub(start) > time.Duration(readTimeout) * time.Millisecond {
return body, fmt.Errorf("Timed-out while reading from socket.")
}
time.Sleep(time.Duration(1) * time.Millisecond)
}
return lz4.Decode(nil, body)
}
The client:
func main() {
address := os.Args[1]
msg := []byte(os.Args[2])
fmt.Printf("Sending %s to %s\n", msg, address)
conn, err := net.Dial(address)
if err != nil {
fmt.Printf("%v\n", err)
return
}
// Another library call
_, err = SendMessage(conn, []byte(msg))
if err != nil {
fmt.Printf("%v\n", err)
return
}
response, err := ReceiveMessage(conn)
conn.Close()
if err != nil {
fmt.Printf("%v\n", err)
return
}
fmt.Printf("Response: %s\n", response)
}
When I call it, I get no response and it just times out. (If I do not explicitly ignore the EOF, it returns there with io.EOF error.) I have another library for this written in Python that also works against the same endpoint with the same payload. Do you see anything immediately?
[JimB just beat me to an answer but here goes anyway.]
The root issue is that you did body = body[:4]
when you wanted body = body[4:].
The former keeps only the first four header bytes
while the latter tosses
the four header bytes just decoded.
Here is a self contained version with some debug logs
that works.
It has some of the other changes I mentioned.
(I guessed at various things that you didn't include, like the lz4 package used, the timeout, unpack, etc.)
package main
import (
"encoding/binary"
"errors"
"fmt"
"io"
"log"
"net"
"time"
"github.com/bkaradzic/go-lz4"
)
const readTimeout = 30 * time.Second // XXX guess
func ReceiveMessage(conn net.Conn) ([]byte, error) {
bodyLen := 0
body := make([]byte, 0, 4096)
var buf [256]byte
conn.SetDeadline(time.Now().Add(readTimeout))
defer conn.SetDeadline(time.Time{}) // disable deadline
for bodyLen == 0 || len(body) < bodyLen {
if bodyLen == 0 && len(body) >= 4 {
bodyLen = int(unpack(body[:4]))
body = body[4:]
if bodyLen <= 0 {
return nil, errors.New("invalid body length")
}
log.Println("read bodyLen:", bodyLen)
continue
}
n, err := conn.Read(buf[:])
body = append(body, buf[:n]...)
log.Printf("appended %d bytes, len(body) now %d", n, len(body))
// Note, this is checked *after* handing any n bytes.
// An io.Reader is allowed to return data with an error.
if err != nil {
if err != io.EOF {
return nil, err
}
break
}
}
if len(body) != bodyLen {
return nil, fmt.Errorf("got %d bytes, expected %d",
len(body), bodyLen)
}
return lz4.Decode(nil, body)
}
const address = ":5678"
var msg = []byte(`{"foo":"bar"}`)
func main() {
//address := os.Args[1]
//msg := []byte(os.Args[2])
fmt.Printf("Sending %s to %s\n", msg, address)
conn, err := net.Dial("tcp", address)
if err != nil {
fmt.Printf("%v\n", err)
return
}
// Another library call
_, err = SendMessage(conn, msg)
if err != nil {
fmt.Printf("%v\n", err)
return
}
response, err := ReceiveMessage(conn)
conn.Close()
if err != nil {
fmt.Printf("%v\n", err)
return
}
fmt.Printf("Response: %s\n", response)
}
// a guess at what your `unpack` does
func unpack(b []byte) uint32 {
return binary.LittleEndian.Uint32(b)
}
func SendMessage(net.Conn, []byte) (int, error) {
// stub
return 0, nil
}
func init() {
// start a simple test server in the same process as a go-routine.
ln, err := net.Listen("tcp", address)
if err != nil {
log.Fatal(err)
}
go func() {
defer ln.Close()
for {
conn, err := ln.Accept()
if err != nil {
log.Fatalln("accept:", err)
}
go Serve(conn)
}
}()
}
func Serve(c net.Conn) {
defer c.Close()
// skip readding the initial request/message and just respond
const response = `{"somefield": "someval"}`
// normally (de)compression in Go is done streaming via
// an io.Reader or io.Writer but we need the final length.
data, err := lz4.Encode(nil, []byte(response))
if err != nil {
log.Println("lz4 encode:", err)
return
}
log.Println("sending len:", len(data))
if err = binary.Write(c, binary.LittleEndian, uint32(len(data))); err != nil {
log.Println("writing len:", err)
return
}
log.Println("sending data")
if _, err = c.Write(data); err != nil {
log.Println("writing compressed response:", err)
return
}
log.Println("Serve done, closing connection")
}
Playground (but not runnable there).
You have a number of issues with the server code. Without a full reproducing case, it's hard to tell if these will fix everything.
for bodyLen == 0 || len(body) < bodyLen {
if len(body) > 4 {
header := body[:4]
body = body[:4]
bodyLen = int(unpack(header))
}
every iteration, if len(body) > 4, you slice body back to the first 4 bytes. Body might never get to be >= bodyLen.
n, err := conn.Read(buf[:])
You don't need to re-slice buf here, use conn.Read(buf)
if err != nil {
if err != io.EOF {
return body, err
}
}
io.EOF is the end of the stream, and you need to handle it. Note that n might still be > 0 when you get an EOF. Check after processing the body for io.EOF or you could loop indefinitely.
body = append(body, buf[0:n]...)
now := time.Now()
if now.Sub(start) > time.Duration(readTimeout) * time.Millisecond {
return body, fmt.Errorf("Timed-out while reading from socket.")
you would be better off using conn.SetReadDeadline before each read, so a stalled Read could be interrupted.