What does crypto/cipher-XORKeyStream do to the src []byte? - go

I'm doing AES encryption using Go, I found that the source bytes changed after encryption. Seems that XORKeyStream function does the change if cap(source) > len(source), what it exactly does to the src []byte?
go version go1.12.5 darwin/amd64
func main() {
byte1 := []byte("123abc")
fmt.Println("content1:", byte1, "len1:", len(byte1), "cap1:", cap(byte1)) // content1: [49 50 51 97 98 99] len1: 6 cap1: 6
buf := bytes.NewBuffer([]byte("123abc"))
byte2, _ := ioutil.ReadAll(buf)
fmt.Println("content2:", byte2, "len2:", len(byte2), "cap2:", cap(byte2)) // content2: [49 50 51 97 98 99] len2: 6 cap2: 1536
_, _, _, err := crypt.AESEnc(byte1)
if err != nil {
log.Fatal(err)
}
fmt.Println("content1:", byte1, "len1:", len(byte1), "cap1:", cap(byte1)) // content1: [49 50 51 97 98 99] len1: 6 cap1: 6
_, _, _, err = crypt.AESEnc(byte2)
if err != nil {
log.Fatal(err)
}
fmt.Println("content2:", byte2, "len2:", len(byte2), "cap2:", cap(byte2)) // content2: [132 200 7 200 195 8] len2: 6 cap2: 1536
}
func AESEnc(data []byte) ([]byte, []byte, string, error) {
key := make([]byte, 16)
iv := make([]byte, 16)
_, err := io.ReadFull(rand.Reader, key)
if err != nil {
return nil, nil, "", err
}
_, err = io.ReadFull(rand.Reader, iv)
if err != nil {
return nil, nil, "", err
}
block, err := aes.NewCipher(key)
if err != nil {
return nil, nil, "", err
}
pdata := pckspadding(data, block.BlockSize())
stream := cipher.NewCFBEncrypter(block, iv)
stream.XORKeyStream(pdata, pdata)
return key, iv, base64.StdEncoding.EncodeToString(pdata), nil
}
func pckspadding(ciphertext []byte, blockSize int) []byte {
padding := blockSize - len(ciphertext)%blockSize
padtext := bytes.Repeat([]byte{byte(padding)}, padding)
return append(ciphertext, padtext...)
}
byte2 changes after encryption, what happened?

I'm not familiar a crypto/cypher-XORKeyStream but I can tell you what XOR does to bits if that is helpfull. I have some Electronic experience and here is the truth table to an XOR gate:
Inputs X and Y represent two bits. The output Z is the result of XOR-ing X and Y.
In English you would say to yourself "Inputs, either one or the other but not both" results in an output of "True".
Don't know how much help this will be or how to apply it to more than two input bits with a crypto/cypher-XORKeyStream.
But here would be an example:
X = 00110001010
Y = 11111111111
Z = 11001110101
Good Luck!

Related

How to implement TripleDES DESede/ECB/PKCS5Padding encryption with 32-bytes key for golang?

I really need to implement 3DES/ECB/PKCS5Padding for Golang, which I found for implementation like https://gist.github.com/cuixin/10612934 but my secret key length is 32 bytes and is not working. I need your help, guys. Thank you very much
3DES
The 3DES algorithm uses a 24-byte length key to divide the key into 3 sub-keys of 8 bytes: K1, K2, K3. Use these 3 keys to encrypt and decrypt plaintext, as follows:
E (K,d), D (k,d), respectively, uses the key k to encrypt or decrypt data d, returning the encrypted or decrypted data.
3DES Encryption Process:
E(k3,D(k2,E(k1,d)))
The meaning is: will clear text D first uses K1 to encrypt, obtains the ciphertext D1, to D1 again uses K2 to do the decryption processing, obtains the ciphertext D2; then the D2 uses the K3 to do the encryption processing, obtains the final cipher text.
The 3DES decryption process is the opposite of encryption:
D(k1,E(k2,D(k3,d)))
The ciphertext d is decrypted with K3, the ciphertext is D1, the D1 is encrypted with K2, the cipher is D2, and the D2 is decrypted with K1, and the final plaintext is obtained.
You can try this code witch is compatiable with your gist code:
package main
import (
"bytes"
"crypto/des"
"errors"
"fmt"
"log"
)
//ECB PKCS5Padding
func PKCS5Padding(ciphertext []byte, blockSize int) []byte {
padding := blockSize - len(ciphertext)%blockSize
padtext := bytes.Repeat([]byte{byte(padding)}, padding)
return append(ciphertext, padtext...)
}
//ECB PKCS5Unpadding
func PKCS5Unpadding(origData []byte) []byte {
length := len(origData)
unpadding := int(origData[length-1])
return origData[:(length - unpadding)]
}
//Des encryption
func encrypt(origData, key []byte) ([]byte, error) {
if len(origData) < 1 || len(key) < 1 {
return nil, errors.New("wrong data or key")
}
block, err := des.NewCipher(key)
if err != nil {
return nil, err
}
bs := block.BlockSize()
if len(origData)%bs != 0 {
return nil, errors.New("wrong padding")
}
out := make([]byte, len(origData))
dst := out
for len(origData) > 0 {
block.Encrypt(dst, origData[:bs])
origData = origData[bs:]
dst = dst[bs:]
}
return out, nil
}
//Des Decrypt
func decrypt(crypted, key []byte) ([]byte, error) {
if len(crypted) < 1 || len(key) < 1 {
return nil, errors.New("wrong data or key")
}
block, err := des.NewCipher(key)
if err != nil {
return nil, err
}
out := make([]byte, len(crypted))
dst := out
bs := block.BlockSize()
if len(crypted)%bs != 0 {
return nil, errors.New("wrong crypted size")
}
for len(crypted) > 0 {
block.Decrypt(dst, crypted[:bs])
crypted = crypted[bs:]
dst = dst[bs:]
}
return out, nil
}
//[golang ECB 3DES Encrypt]
func TripleEcbDesEncrypt(origData, key []byte) ([]byte, error) {
tkey := make([]byte, 24, 24)
copy(tkey, key)
k1 := tkey[:8]
k2 := tkey[8:16]
k3 := tkey[16:]
block, err := des.NewCipher(k1)
if err != nil {
return nil, err
}
bs := block.BlockSize()
origData = PKCS5Padding(origData, bs)
buf1, err := encrypt(origData, k1)
if err != nil {
return nil, err
}
buf2, err := decrypt(buf1, k2)
if err != nil {
return nil, err
}
out, err := encrypt(buf2, k3)
if err != nil {
return nil, err
}
return out, nil
}
//[golang ECB 3DES Decrypt]
func TripleEcbDesDecrypt(crypted, key []byte) ([]byte, error) {
tkey := make([]byte, 24, 24)
copy(tkey, key)
k1 := tkey[:8]
k2 := tkey[8:16]
k3 := tkey[16:]
buf1, err := decrypt(crypted, k3)
if err != nil {
return nil, err
}
buf2, err := encrypt(buf1, k2)
if err != nil {
return nil, err
}
out, err := decrypt(buf2, k1)
if err != nil {
return nil, err
}
out = PKCS5Unpadding(out)
return out, nil
}
func main() {
key2 := []byte("5e8487e65e8487e65e8487e6")
origtext2 := []byte("hello world123563332")
erytext2, err := TripleEcbDesEncrypt(origtext2, key2)
if err != nil {
log.Fatal(err)
}
fmt.Printf("%v\n", origtext2)
destext2, err2 := TripleEcbDesDecrypt(erytext2, key2)
if err2 != nil {
log.Fatal(err2)
}
fmt.Println(string(destext2))
fmt.Println(len(origtext2), len(string(destext2)))
fmt.Println(string(origtext2) == string(destext2))
}

What does the error "binary.Write: invalid type" mean?

The code shown below, I create a struct type and want to encode it to binary.
But it show binary.Write: invalid type main.Stu error, I had read some similar code like this,but I can't find why my code doesn't work?
type Stu struct {
Name string
Age int
Id int
}
func main() {
s := &Stu{
Name: "Leo",
Age: 21,
Id: 1,
}
buf := new(bytes.Buffer)
err := binary.Write(buf, binary.BigEndian, s)
if err != nil{
fmt.Println(err)
}
fmt.Printf("%q\n", buf)
}
In short: encoding/binary cannot be used to encode arbitrary values that have non-fixed size. int and string are such examples. Quoting from binary.Write():
Write writes the binary representation of data into w. Data must be a fixed-size value or a slice of fixed-size values, or a pointer to such data.
Note that if you remove the string field and change int fields to int32, it'll work:
type Stu struct {
Age int32
Id int32
}
func main() {
s := &Stu{
Age: 21,
Id: 1,
}
buf := new(bytes.Buffer)
err := binary.Write(buf, binary.BigEndian, s)
if err != nil {
fmt.Println(err)
}
fmt.Printf("%q\n", buf)
}
Which outputs (try it on the Go Playground):
"\x00\x00\x00\x15\x00\x00\x00\x01"
As the doc suggests, to encode complex structures, use encoding/gob.
Example of encoding and decoding using encoding/gob:
buf := new(bytes.Buffer)
enc := gob.NewEncoder(buf)
if err := enc.Encode(s); err != nil {
fmt.Println(err)
}
fmt.Printf("%v\n", buf.Bytes())
dec := gob.NewDecoder(buf)
var s2 *Stu
if err := dec.Decode(&s2); err != nil {
fmt.Println(err)
}
fmt.Printf("%+v\n", s2)
Which outputs (try it on the Go Playground):
[41 255 129 3 1 1 3 83 116 117 1 255 130 0 1 3 1 4 78 97 109 101 1 12 0 1 3 65 103 101 1 4 0 1 2 73 100 1 4 0 0 0 12 255 130 1 3 76 101 111 1 42 1 2 0]
&{Name:Leo Age:21 Id:1}

reader.ReadLine() doesn't advance after a scanner.Scan() call

The code below reads its values from this file:
2 3\n
1.0 2.0 3.0\n
-1.0 -2.0 -3.0\n
And should print:
[ {1 2 3}, {-1 -2 -3} ]
But instead I get this:
[{2 [31 2 3]} {0 []}] strconv.ParseFloat: parsing "3.0-1.0": invalid syntax
It seems that the reader.ReadLine() stays at the same location. Is there a simpler way to scan lines, then values inside each line?
package main
import (
"bufio"
"bytes"
"fmt"
"os"
"strconv"
"strings"
)
type Example struct {
classLabel int
attributes []float64
}
func NewExample(classLabel int, attributes []float64) *Example {
return &Example{classLabel, attributes}
}
func readFile(path string) ([]Example, error) {
var (
result []Example
err error
file *os.File
part []byte
size int
attributeNum int
)
if file, err = os.Open(path); err != nil {
return result, err
}
defer file.Close()
reader := bufio.NewReader(file)
buffer := bytes.NewBuffer(make([]byte, 0))
if part, _, err = reader.ReadLine(); err != nil {
return result, err
}
buffer.Write(part)
newLine := buffer.String()
fmt.Println("newLine=" + newLine)
r := strings.NewReader(newLine)
scanner := bufio.NewScanner(r)
scanner.Split(bufio.ScanWords)
if scanner.Scan() {
size, err = strconv.Atoi(scanner.Text())
if err != nil {
return result, err
}
}
fmt.Println("size=" + strconv.Itoa(size))
if scanner.Scan() {
attributeNum, err = strconv.Atoi(scanner.Text())
if err != nil {
return result, err
}
}
fmt.Println("attributeNum=" + strconv.Itoa(attributeNum))
result = make([]Example, size)
var classLabel int
var attributes []float64
for k := 0; k < size; k++ {
if part, _, err = reader.ReadLine(); err != nil {
return result, err
}
buffer.Write(part)
newLine := buffer.String()
fmt.Println("newLine=" + newLine)
r := strings.NewReader(newLine)
scanner := bufio.NewScanner(r)
scanner.Split(bufio.ScanWords)
if scanner.Scan() {
classLabel, err = strconv.Atoi(scanner.Text())
if err != nil {
return result, err
}
}
fmt.Println("classLabel=" + strconv.Itoa(classLabel))
for i := 0; i < attributeNum; i++ {
var attribute float64
if scanner.Scan() {
attribute, err = strconv.ParseFloat(scanner.Text(), 64)
if err != nil {
return result, err
}
attributes = append(attributes, attribute)
fmt.Println("attribute=" + strconv.FormatFloat(attribute, 'f', -1, 64))
}
}
result[k] = *NewExample(classLabel, attributes)
}
return result, scanner.Err()
}
func main() {
example, err := readFile("test.txt")
fmt.Println(example, err)
}
When you do this inside the for loop:
buffer.Write(part)
newLine := buffer.String()
fmt.Println("newLine=" + newLine)
The next line gets appended to buffer.
That is,
before the loop begins, buffer contains 2 3,
and then after reading 1.0 2.0 3.0,
it gets appended to buffer,
so the content becomes 2 31.0 2.0 3.0,
which you store in newLine.
That's where things start to go sideways.
You probably want to clear the buffer before reading each new line:
buffer.Reset()
buffer.Write(part)
newLine := buffer.String()
fmt.Println("newLine=" + newLine)
But then you will have further problems still, here:
if scanner.Scan() {
classLabel, err = strconv.Atoi(scanner.Text())
if err != nil {
return result, err
}
}
Since the line contains 1.0 2.0 3.0, the strconf.Atoi is going to fail.
I don't understand the purpose of this snippet,
perhaps you can delete it (or comment out).
With the above fixed, you will still have one more problem, on this line:
attributes = append(attributes, attribute)
Since attributes is never reset, it keeps growing.
That is, after the first line, it will contain 1 2 3,
and after the second line it will contain 1 2 3 -1 -2 -3.
You could correct that by moving the declaration of attributes without the outer loop, like this:
var attributes []float64
for i := 0; i < attributeNum; i++ {
var attribute float64
if scanner.Scan() {
attribute, err = strconv.ParseFloat(scanner.Text(), 64)
if err != nil {
return result, err
}
attributes = append(attributes, attribute)
fmt.Println("attribute=" + strconv.FormatFloat(attribute, 'f', -1, 64))
}
}

Go: deep copy slices

I would like to read a slice of strings representing hexadecimal numbers, and decode them to a slice of byte slices ([]string --> [][]byte). This is my code so far:
func (self *algo_t) decode_args(args []string) ([][]byte, error) {
var data [][]byte
for i := uint32(0); i < self.num_args; i++ {
data = make([][]byte, self.num_args)
tmp, err := hex.DecodeString(args[i])
fmt.Printf("i = %d\ttmp = %x\n", i, tmp)
data[i] = make([]byte, len(tmp))
copy(data[i], tmp)
if err != nil {
fmt.Fprintf(os.Stderr, "Error decoding hex string %s: %s\n", args[i], err.Error())
return nil, err
}
}
fmt.Printf("line 69\tdata[0] = %x\tdata[1] = %x\tdata[2] = %x\n",data[0], data[1], data[2])
return data, nil
}
calling this code and passing args = []string{"010203","040506","070809"} yields the following output:
i = 0 tmp = 010203
i = 1 tmp = 040506
i = 3 tmp = 070809
line 69 data[0] = data[1] = data[2] = 070809
Presumably the function returns [][]byte{[]byte{}, []byte{}, []byte{0x07, 0x08, 0x09}}.
I understand that this is because of the pointer behavior of Go; what is the best practice for doing a deep copy of this kind?
For example,
package main
import (
"encoding/hex"
"fmt"
)
// Decode hex []string to [][]byte
func decode(s []string) ([][]byte, error) {
b := make([][]byte, len(s))
for i, ss := range s {
h, err := hex.DecodeString(ss)
if err != nil {
err = fmt.Errorf(
"Error decoding hex string %s: %s\n",
ss, err.Error(),
)
return nil, err
}
b[i] = h
}
return b, nil
}
func main() {
s := []string{"010203", "040506", "070809"}
fmt.Println(s)
b, err := decode(s)
if err != nil {
fmt.Println(err)
} else {
fmt.Println(b)
}
s = []string{"ABCDEF", "012345", "09AF"}
fmt.Println(s)
b, err = decode(s)
if err != nil {
fmt.Println(err)
} else {
fmt.Println(b)
}
s = []string{"01", "123XYZ"}
fmt.Println(s)
b, err = decode(s)
if err != nil {
fmt.Println(err)
} else {
fmt.Println(b)
}
}
Output:
[010203 040506 070809]
[[1 2 3] [4 5 6] [7 8 9]]
[ABCDEF 012345 09AF]
[[171 205 239] [1 35 69] [9 175]]
[01 123XYZ]
Error decoding hex string 123XYZ: encoding/hex: invalid byte: U+0058 'X'
There is a package built specifically to handle deep copy: http://godoc.org/code.google.com/p/rog-go/exp/deepcopy
You can look at the source here: https://code.google.com/p/rog-go/source/browse/exp/deepcopy/deepcopy.go. It covers copying slices and pointers, so it should cover your case.

Go byte to integer encoding with RPMs

I am trying to create a go program that can read and create RPM files without the need of librpm and rpmbuild. Most of the reason for this is to get a better understanding of programming in go.
I am parsing an RPM based off the following: https://github.com/jordansissel/fpm/wiki/rpm-internals
I am looking at the header and trying to parse the number of tags + the length, and I have the following code
fi, err := os.Open("golang-1.1-2.fc19.i686.rpm")
...
// header
head := make([]byte, 16)
// read a chunk
_, err = fi.Read(head)
if err != nil && err != io.EOF { panic(err) }
fmt.Printf("Magic number %s\n", head[:8])
tags, read := binary.Varint(head[8:12])
fmt.Printf("Tag Count: %d\n", tags)
fmt.Printf("Read %d\n", read)
length, read := binary.Varint(head[12:16])
fmt.Printf("Length : %d\n", length)
fmt.Printf("Read %d\n", read)
I get back the following:
Magic number ���
Tag Count: 0
Read 1
Length : 0
Read 1
I printed out the slice and I see this:
Tag bytes: [0 0 0 7]
Length bytes: [0 0 4 132]
I then tried just doing this:
length, read = binary.Varint([]byte{4, 132})
which returns length as 2 and read 1.
Based off what I am reading, the tag and length should be "4 byte 'tag count'", so how would I get the four bytes as one number?
EDIT:
Based off the feedback from #nick-craig-wood and #james-henstridge below is my following prototype code that does what Im looking for:
package main
import (
"io"
"os"
"fmt"
"encoding/binary"
"bytes"
)
type Header struct {
// begin with the 8-byte header magic value: 8D AD E8 01 00 00 00 00
Magic uint64
// 4 byte 'tag count'
Count uint32
// 4 byte 'data length'
Length uint32
}
func main() {
// open input file
fi, err := os.Open("golang-1.1-2.fc19.i686.rpm")
if err != nil { panic(err) }
// close fi on exit and check for its returned error
defer func() {
if err := fi.Close(); err != nil {
panic(err)
}
}()
// ignore lead
fi.Seek(96, 0)
// header
head := make([]byte, 16)
// read a chunk
_, err = fi.Read(head)
if err != nil && err != io.EOF { panic(err) }
fmt.Printf("Magic number %s\n", head[:8])
tags := binary.BigEndian.Uint32(head[8:12])
fmt.Printf("Count Count: %d\n", tags)
length := binary.BigEndian.Uint32(head[12:16])
fmt.Printf("Length : %d\n", length)
// read it as a struct
buf := bytes.NewBuffer(head)
header := Header{}
err = binary.Read(buf, binary.BigEndian, &header)
if err != nil {
fmt.Println("binary.Read failed:", err)
}
fmt.Printf("header = %#v\n", header)
fmt.Printf("Count bytes: %d\n", header.Count)
fmt.Printf("Length bytes: %d\n", header.Length)
}
Firstly don't use Varint - it doesn't do what you think it does!
Decode like this into a go structure is the most convenient way
package main
import (
"bytes"
"encoding/binary"
"fmt"
)
type Header struct {
// begin with the 8-byte header magic value: 8D AD E8 01 00 00 00 00
Magic uint64
// 4 byte 'tag count'
Count uint32
// 4 byte 'data length'
Length uint32
}
var data = []byte{0x8D, 0xAD, 0xE8, 0x01, 0, 0, 0, 0, 0, 0, 0, 7, 0, 0, 4, 132}
func main() {
buf := bytes.NewBuffer(data)
header := Header{}
err := binary.Read(buf, binary.BigEndian, &header)
if err != nil {
fmt.Println("binary.Read failed:", err)
}
fmt.Printf("header = %#v\n", header)
}
Prints
header = main.Header{Magic:0x8dade80100000000, Count:0x7, Length:0x484}
Playground link
The data you are reading doesn't look like it is in Go's variable length integer encoding.
Instead, you probably want binary.BigEndian.Uint32():
tags := binary.BigEndian.Uint32(head[8:12])
length := binary.BigEndian.Uint32(head[12:16])

Resources