Convert from and to messagepack - go

So I'm trying to get the string representation of a JSON message in Golang. I just want to receive the messagepack encoded JSON, modify some values and send it back.
I haven't found an easy way to do it. Most of the times, I can't know in advance what is the structure of the JSON (apart from having JSON structure), so want I want to do is to receive the binary message. Decode it as a JSON, print it as a string to the standard output, modify the content, convert it to MessagePack again and send it back.
I've been looking at these two packages, but I don't know how to properly use them for a simple task like that:
https://godoc.org/github.com/vmihailenco/msgpack
https://godoc.org/github.com/ugorji/go/codec
So I will receive something like this:
DF 00 00 00 01 A7 6D 65 73 73 61 67 65 A3 48 69 21
I want to print this:
{"message": "Hi!"}
Modify the "Hi!":
{"message": "Hello Sir!"}
Send it as messagepack:
DF 00 00 00 01 A7 6D 65 73 73 61 67 65 AA 48 65 6C 6C 6F 20 53 69 72 21
Current Python code I'm trying to port to Golang:
def decode_msgpack(jsonData):
packedStuff = 0
for key in jsonData.keys():
if type(jsonData[key]) is bytes:
packedStuff += 1
try:
jsonData[key] = umsgpack.unpackb(jsonData[key])
except umsgpack.InvalidStringException:
try:
jsonData[key] = umsgpack.unpackb(jsonData[key], allow_invalid_utf8=True)
except umsgpack.InsufficientDataException:
print("[!] InsufficientDataException")
jsonData[key] = base64.b64encode(jsonData[key]).decode('utf-8')
else:
jsonData[key] = base64.b64encode(jsonData[key]).decode('utf-8')
if packedStuff > 0:
return decode_msgpack(jsonData)
else:
return jsonData

Using the codec library and assuming that {"message": "Hi"} is a map, the code would look something like this.
package main
import (
"fmt"
"github.com/ugorji/go/codec"
)
func main() {
var data []byte
original := map[string]string{"message": "Hi!"}
enc := codec.NewEncoderBytes(&data, new(codec.MsgpackHandle))
if err := enc.Encode(&original); err != nil {
panic(err)
}
fmt.Printf("Encoded: ")
for _, b := range data {
fmt.Printf("%X ", b)
}
fmt.Printf("\n")
decoded := make(map[string]string)
dec := codec.NewDecoderBytes(data, new(codec.MsgpackHandle))
if err := dec.Decode(&decoded); err != nil {
panic(err)
}
fmt.Printf("Decoded: %v\n", decoded)
decoded["message"] = "Hello Sir!"
/* reinitialize the encoder */
enc = codec.NewEncoderBytes(&data, new(codec.MsgpackHandle))
if err := enc.Encode(&decoded); err != nil {
panic(err)
}
fmt.Printf("Encoded: ")
for _, b := range data {
fmt.Printf("%X ", b)
}
fmt.Printf("\n")
}
That said, if you get {"message": "Hi"} as a JSON string, you can use codec to decode the JSON into a map, update the map and then re-encode it as msgpack.

The best way is to first decode it, make your changes via Go structs and then re-encode it.
data := []byte(`{"message": "Hi!"}`)
var p map[string]interface{}
// Decode into Struct
if err := json.Unmarshal(data, &p); err != nil {
// TODO: handle err
}
// Modify contents
p["message"] = "Hello Sir!"
// Encode from struct
newData, err := json.Marshal(p)
if err != nil {
// TODO: Handle err
}
fmt.Println(string(newData))

Related

Convert zipped []byte to unzip []byte golang code

I have []byte of zip file. I have to unzip it without creating a new file, and get a []byte of that unzipped file. Please help me to do that.
I am making an API call and the response I get is the []byte in zipped format - I am trying to unzip it - and use it's content for creating a new zip file. So unzip - rezip.
Language: Golang
Code I've used:
func UnzipBytes(zippedBytes []byte) ([]byte, error) {
reader := bytes.NewReader(zippedBytes)
zipReader, err := zlib.NewReader(reader)
if err != nil {
return nil, err
}
defer zipReader.Close()
p, err := ioutil.ReadAll(zipReader)
if err != nil {
return nil, err
}
return p, nil
}
I get an error saying "zlib: invalid header"
The code that was initially used to zip the []byte
buffer := new(bytes.Buffer)
zipWriter := zip.NewWriter(buffer)
zipFile, err := zipWriter.Create(file.name)
_, err = zipFile.Write(file.content)
Hex dump of the []byte - the zippedBytes
00059350 78 b4 5b 0d 2b 81 c2 87 35 76 1b 11 4a ec 07 d1 |x.[.+...5v..J...|
00059360 76 77 a2 e1 3b d9 12 e2 51 d4 c5 bd 4b 2f 09 da |vw..;...Q...K/..|
00059370 f7 21 c7 26 73 1f 8e da f0 ff a3 52 f6 e2 00 e6 |.!.&s......R....|
You used zip.Writer to compress the data. You must close it by calling its Writer.Close() method. And you must use zip.Reader to read it, and use Reader.Open() with the same name you used when compressed it (file.name).
This is how it could look like:
func UnzipBytes(name string, zippedBytes []byte) ([]byte, error) {
reader := bytes.NewReader(zippedBytes)
zipReader, err := zip.NewReader(reader, int64(len(zippedBytes)))
if err != nil {
return nil, err
}
f, err := zipReader.Open(name)
if err != nil {
panic(err)
}
p, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
return p, nil
}
Testing it:
filename := "test.txt"
filecontent := []byte("line1\nline2")
buffer := new(bytes.Buffer)
zipWriter := zip.NewWriter(buffer)
zipFile, err := zipWriter.Create(filename)
if err != nil {
panic(err)
}
if _, err = zipFile.Write(filecontent); err != nil {
panic(err)
}
if err = zipWriter.Close(); err != nil {
panic(err)
}
decoded, err := UnzipBytes(filename, buffer.Bytes())
fmt.Println(err)
fmt.Println(string(decoded))
This will output (try it on the Go Playground):
<nil>
line1
line2
If you don't know the name when decompressing, you may see all files in the Reader.Files header field. You may choose to open the first file:
func UnzipBytes(zippedBytes []byte) ([]byte, error) {
reader := bytes.NewReader(zippedBytes)
zipReader, err := zip.NewReader(reader, int64(len(zippedBytes)))
if err != nil {
return nil, err
}
if len(zipReader.File) == 0 {
return nil, nil // No file to open / extract
}
f, err := zipReader.File[0].Open()
if err != nil {
panic(err)
}
p, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
return p, nil
}
This outputs the same. Try this one on the Go Playground.

json.Marshal returning weird values [closed]

Closed. This question is not reproducible or was caused by typos. It is not currently accepting answers.
This question was caused by a typo or a problem that can no longer be reproduced. While similar questions may be on-topic here, this one was resolved in a way less likely to help future readers.
Closed 1 year ago.
Improve this question
I want array of json from sql rows. when i try to do marshal on each struct after scanning each row, its returning weird values like [123 34 105 100 34 ..]
type Org struct {
Id int `json:"id"`
Name string `json:"name"`
}
res, err := db.Query("select id,name from organization")
if err != nil {
// fmt.Print("err in query")
panic(err)
}
// var orgArray []Org
defer res.Close()
for res.Next() {
var org Org
fmt.Println(&org.Id, &org.Name, "PRINT ADDRESS BEFORE SCAN")
// 0xc0001c0648 0xc0001c0650 PRINT ADDRESS BEFORE SCAN
err = res.Scan(&org.Id, &org.Name)
fmt.Println(org.Id, org.Name, org, "PRINT VALUES AFTER SCAN")
// 1535 TestOrg {1535 TestOrg} PRINT VALUES AFTER SCAN
b, err := json.Marshal(org)
if err != nil {
panic(err)
}
fmt.Println(b)
//[123 34 105 100 34 58 49 53 51 55 44 34 110 97 109 101 34 58 34 98 114 97 110 100 32 69 104 71 74 89 34 125]
}
whats the problem here?
json.Marshal response is bytes array, convert to string before printing
package main
import (
"encoding/json"
"fmt"
)
type Abc struct {
A string `json:"a"`
B string `json:"b"`
}
func main() {
d := Abc{A: "aaa", B: "bbbb"}
a, _ := json.Marshal(d)
fmt.Println(string(a))
}
json.Marshal returns a byte array - []byte.
The Println prints b out as such. The array of integers (byte values) you see is how byte arrays are printed out in Go.
Use string(b) to print a string. fmt.Println(string(b)), etc.

convert the shell output to float64?

i am using shell with golang to access apache log file and get some data. first i used to write output to the file directly and it was working, but now i need to get the output and use it in the program directly. and also i need to convert it to float64. i tried converting it into a string and then to float64, but it is not working?
func Mem_usage_data(j int) (Mem_predict float64, err error) {
awkPart := fmt.Sprintf("awk '{print $%d/1024}'", j)
out1, err := exec.Command("bash", "-c", "tail -n 1 /var/log/apache2/access.log| "+awkPart+" ").Output()
fmt.Println("memory usage is", out1)
s1 := string(out1)
v1, err1 := strconv.ParseFloat(s1, 64)
if err1 != nil {
fmt.Println(err)
}
if err != nil {
fmt.Println(err)
}
return v1, err
}
when i print the out1 i get something like this [48 46 49 50 48 49 49 55 10]. CAn you please help how to get the exact output in out1 and how to convert it into float64?
The conversion fails probably because there is some whitespace or newline character on s1. Try to trim it first before doing the conversion. Use strings.TrimSpace() to achieve that.
v1, err1 := strconv.ParseFloat(strings.TrimSpace(s1), 64)

Send ISO8583 Message Golang

I'm trying to build a ISO8583 Client using a Golang,
when using java, i don't have any problem when creating client.
But when trying creating a client using golang (i'm just starting learning golang btw), i can't send a message to the server. Can someone help me, why i can't send a message?
I've tried to send a SIGN IN Message, The client and server already connected, but the message i send not readable by the server.
This My Code
package main
import (
"bufio"
"fmt"
"net"
"os"
"time"
"github.com/ideazxy/iso8583"
)
type ISOSignIn struct {
Bit3 *iso8583.Numeric `field:"3" length:"6" encode:"bcd"`
Bit7 *iso8583.Numeric `field:"7" length:"10" encode:"ascii`
Bit11 *iso8583.Numeric `field:"11" length:"6" encode:"rbcd`
Bit32 *iso8583.Llnumeric `field:"32" length:"11" encode:"ascii`
Bit70 *iso8583.Numeric `field:"70" length:"3" encode:"ascii`
}
func main() {
testIso()
}
func testIso() {
data := ISOSignIn{
Bit3: iso8583.NewNumeric("001111"),
Bit7: iso8583.NewNumeric("0913110004"),
Bit11: iso8583.NewNumeric("000001"),
Bit32: iso8583.NewLlnumeric("9999"), // Client ID
Bit70: iso8583.NewNumeric("301"), // Login Code
}
msg := iso8583.NewMessage("0800", data)
msg.MtiEncode = iso8583.BCD
b, err := msg.Bytes()
if err != nil {
fmt.Println(err.Error())
}
fmt.Printf("% x\n", b)
tcpClientNew(b)
}
func tcpClientNew(b []byte) {
tcpAddr, err := net.ResolveTCPAddr("tcp", "192.168.100.5:12346")
if err != nil {
println("ResolveTCPAddr failed:", err.Error())
os.Exit(1)
}
conn, err := net.DialTCP("tcp", nil, tcpAddr)
if err != nil {
println("Dial failed:", err.Error())
os.Exit(1)
}
timeoutDuration := 30 * time.Second
_, err = conn.Write(b)
if err != nil {
println("Write to server failed:", err.Error())
os.Exit(1)
}
conn.SetReadDeadline(time.Now().Add(timeoutDuration))
bufReader := bufio.NewReader(conn)
resp, _ := bufReader.ReadByte()
fmt.Print("Message from server: " + string(resp))
conn.Close()
}
Server Already Connected
<log realm="Server-A.server.session/192.168.100.1:32218" at="Mon Jan 07 09:37:15.747 WIB 2019">
<session-start/>
</log>
<log realm="channel/192.168.100.1:32218" at="Mon Jan 07 09:37:19.034 WIB 2019" lifespan="3287ms">
<receive>
<peer-disconnect/>
</receive>
</log>
<log realm="Server-A.server.session/192.168.100.1:32218" at="Mon Jan 07 09:37:19.035 WIB 2019">
<session-end/>
</log>
Output from Client Terminal :
GOROOT=/Users/ivanaribanilia/Applications/go
GOPATH=/Users/ivanaribanilia/Project/Workspace-Github/Project-Go/pclient
/Users/ivanaribanilia/Applications/go/bin/go build -i -o /Users/ivanaribanilia/Project/Workspace-Github/Project-Go/pclient/build/pclient /Users/ivanaribanilia/Project/Workspace-Github/Project-Go/pclient/src/github.com/ivanj4u/pclient/main.go
/Users/ivanaribanilia/Project/Workspace-Github/Project-Go/pclient/build/pclient
08 00 22 20 00 01 00 00 00 00 00 11 11 30 39 31 33 31 31 30 30 30 34 30 30 30 30 30 31 30 34 31 31 31 34
Message from server:
Process finished with exit code 0
I expect a response from the server, so i can develop other message like INQUIRY or PAYMENT.
Thank you
ReadByte reads and returns a single byte. If no byte is available,
returns an error.
seems that what you read from server is only one byte, which is a white-space char.
The server and client should make a protocol when to close the connection. Thus, if server don't close the conn actively, client should read all bytes from server and close the connection. Like this:
recvBuf := make([]byte, 1024)
n, err := bufReader.Read(recvBuf)
for err == nil {
println("Recv data from server:", string(recvBuf[:n]))
n, err = bufReader.Read(recvBuf)
}
if err != io.EOF {
println("recv from server failed, err:", err)
}
conn.Close()
Or if the protocol defines the client should close the connection when received a certain byte, client can use ReadBytes() and close the connection actively.
func (b *Reader) ReadBytes(delim byte) ([]byte, error)
ReadBytes reads until the first occurrence of delim in the input,
returning a slice containing the data up to and including the
delimiter. If ReadBytes encounters an error before finding a
delimiter, it returns the data read before the error and the error
itself (often io.EOF). ReadBytes returns err != nil if and only if the
returned data does not end in delim.

Suitable struct type for unmarshal of geojson

I want to unmarshal a geojson string into a suitable struct type.
I have three different geojson strings that I want to unmarshal into the same struct:
var jsonBlobPointString = []byte(`{"Type":"Point", "Coordinates":[1.1,2.0]}`)
var jsonBlobLineString = []byte(`{"Type":"LineString", "Coordinates":[[1.1,2.0],[3.0,6.3]]}`)
var jsonBlobPolygonString = []byte(`{"Type":"Polygon", "Coordinates":[[[1.1,2.0],[3.0,6.3],[5.1,7.0],[1.1,2.0]]]}`)
I came up with a struct type that I´m not totally happy with:
type GeojsonType struct {
Type string
Coordinates interface{}
}
See this link for complete example:
http://play.golang.org/p/Bt-51BX__A
I would rather not use interface{} for Coordinates.
I would instead use somehting that give me some validation for example Coordinates [] float64 for Point
and Coordinates[][] float64 for LineString.
Is it possible to create a struct type so that Point, LineString and Polygon all can be represented in Coordinates without using interface?
What you want is to create 3 different types of object from the same json dictionary.
As far as I know that isn't possible, however you can use the RawMessage type to delay the json decoding and use a bit of pre-processing like this
package main
import (
"encoding/json"
"fmt"
)
type Point struct {
Coordinates []float64
}
type Line struct {
Points [][]float64
}
type Polygon struct {
Lines [][][]float64
}
type GeojsonType struct {
Type string
Coordinates json.RawMessage
Point Point
Line Line
Polygon Polygon
}
var jsonBlob = []byte(`[
{"Type":"Point", "Coordinates":[1.1,2.0]},
{"Type":"LineString", "Coordinates":[[1.1,2.0],[3.0,6.3]]},
{"Type":"Polygon", "Coordinates":[[[1.1,2.0],[3.0,6.3],[5.1,7.0],[1.1,2.0]]]}
]`)
func main() {
var geojsonPoints []GeojsonType
err := json.Unmarshal(jsonBlob, &geojsonPoints)
if err != nil {
fmt.Println("error:", err)
}
// Postprocess the coordinates
for i := range geojsonPoints {
t := &geojsonPoints[i]
switch t.Type {
case "Point":
err = json.Unmarshal(t.Coordinates, &t.Point.Coordinates)
case "LineString":
err = json.Unmarshal(t.Coordinates, &t.Line.Points)
case "Polygon":
err = json.Unmarshal(t.Coordinates, &t.Polygon.Lines)
default:
panic("Unknown type")
}
if err != nil {
fmt.Printf("Failed to convert %s: %s", t.Type, err)
}
fmt.Printf("%+v\n", t)
}
}
Which prints
&{Type:Point Coordinates:[91 49 46 49 44 50 46 48 93] Point:{Coordinates:[1.1 2]} Line:{Points:[]} Polygon:{Lines:[]}}
&{Type:LineString Coordinates:[91 91 49 46 49 44 50 46 48 93 44 91 51 46 48 44 54 46 51 93 93] Point:{Coordinates:[]} Line:{Points:[[1.1 2] [3 6.3]]} Polygon:{Lines:[]}}
&{Type:Polygon Coordinates:[91 91 91 49 46 49 44 50 46 48 93 44 91 51 46 48 44 54 46 51 93 44 91 53 46 49 44 55 46 48 93 44 91 49 46 49 44 50 46 48 93 93 93] Point:{Coordinates:[]} Line:{Points:[]} Polygon:{Lines:[[[1.1 2] [3 6.3] [5.1 7] [1.1 2]]]}}
Based on Nick Craig-Wood answer I built the following Marshal/UnMarshal functions
package geojson
//https://stackoverflow.com/questions/15719532/suitable-struct-type-for-unmarshal-of-geojson
import (
"encoding/json"
)
type Point struct {
Coordinates []float64
}
type Line struct {
Points [][]float64
}
type Polygon struct {
Lines [][][]float64
}
type Geojson struct {
Type string `json:"type"`
Coordinates json.RawMessage `json:"coordinates"`
Point Point `json:"-"`
Line Line `json:"-"`
Polygon Polygon `json:"-"`
}
func (g *Geojson) UnmarshalJSON(b []byte) error {
type Alias Geojson
aux := (*Alias)(g)
err := json.Unmarshal(b, &aux)
if err != nil {
return err
}
switch g.Type {
case "Point":
err = json.Unmarshal(g.Coordinates, &g.Point.Coordinates)
case "LineString":
err = json.Unmarshal(g.Coordinates, &g.Line.Points)
case "Polygon":
err = json.Unmarshal(g.Coordinates, &g.Polygon.Lines)
}
g.Coordinates = []byte(nil)
return err
}
func (g Geojson) MarshalJSON() ([]byte, error) {
var raw json.RawMessage
var err error
switch g.Type {
case "Point":
raw, err = json.Marshal(&g.Point.Coordinates)
case "LineString":
raw, err = json.Marshal(&g.Line.Points)
case "Polygon":
raw, err = json.Marshal(&g.Polygon.Lines)
}
if err != nil {
return nil, err
}
g.Coordinates = raw
type Alias Geojson
aux := (*Alias)(&g)
return json.Marshal(aux)
}

Resources