Failed Unmarshal for xxx invalid character '{' after top-level value - go

Failed Unmarshal for LP name invalid character '{' after top-level value
package main
import (...)
type hostInfo struct {
Name string
}
var b bytes.Buffer
var c bytes.Buffer
var tmphostInfo hostInfo
var result map[string]interface{}
func main() {
keypath := os.Args[1]
hostiplist := []string {"192.168.1.150","192.168.1.151","192.168.1.152","192.168.1.153"}
port := "22"
key,err := ioutil.ReadFile(keypath)
if err != nil {
log.Fatalf("Unable to read private key: %v", err)
}
signer,err:=ssh.ParsePrivateKey(key)
if err != nil {
log.Fatalf("Unable to parse private key: %v",err)
}
var conn *ssh.Client
config := &ssh.ClientConfig{
User: "support",
Auth: []ssh.AuthMethod{ssh.PublicKeys(signer)},
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
Timeout: 5*time.Second,
}
for _,hostip := range hostiplist {
conn,err = ssh.Dial("tcp",net.JoinHostPort(hostip,port),config)
if err != nil {
log.Fatalf("unable to connect: %v",err)
}
defer conn.Close()
session1,err:=conn.NewSession()
if err != nil {
log.Fatalf("unable to connect: %v",err)
}
defer session1.Close()
session1.Stdout = &b
if err := session1.Run("some mongo query"); err != nil {
log.Fatal("Failed to run:sess1 " + err.Error())
}
tmphostInfo=hostInfo{}
fmt.Printf("tmphostInfo: %v , ip : %s",tmphostInfo.Name,hostip)
err1 := json.Unmarshal([]byte(""),&tmphostInfo)
if err1 != nil {
log.Fatalf("Failed Unmarshal for LP name %v for IP %s",err1,hostip)
}
session2,err:=conn.NewSession()
session2.Stdout = &c
if err := session2.Run("another mongo query"); err != nil {
log.Fatal("Failed to run:sess2 " + err.Error())
}
err2 := json.Unmarshal([]byte(c.String()),&result)
if err2 != nil {
log.Fatalf("Failed Unmarshal for Customer name %v",err2)
}
//fmt.Println(c.String())
custmap := result["customer"].(map[string]interface{})
//fmt.Println(birds)
var LPname string
for key, value := range custmap {
// Each value is an interface{} type, that is type asserted as a string
//fmt.Println(key, value.(string))
if key == "name" {
LPname=value.(string)
}
}
fmt.Println(hostip,tmphostInfo.Name,LPname)
f, err := os.OpenFile("/tmp/supportip.csv", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
log.Fatal(err)
}
if _, err := f.Write([]byte(hostip+","+tmphostInfo.Name+","+LPname+"\n")); err != nil {
log.Fatal(err)
}
if err := f.Close(); err != nil {
log.Fatal(err)
}
}
}
go run scan.go /path/to/key results in
tmphostInfo: {} , ip : 192.168.1.150 10.45.9.141 servername1 DFIDorg
tmphostInfo: {} , ip : 192.168.1.151 2021/12/10 15:07:36 Failed Unmarshal for LP name invalid character '{' after top-level value for 192.168.1.151
exit status 1
The unmarshal for both queries result for first array element "192.168.1.150" is successful but fails on second and rest of items
The json string for first query looks like below
{ "name" : "john doe" }
result of second query looks like
{
"customer" : {
"address1" : "ktm",
"secret_key" : "12237918f0d441d25fb",
"address2" : "",
"name" : "KUKL Limited",
"phone" : "12345"
}
}

I had to reset the bytes.Buffer variable right after the for loop start, the issue was that the result pulled from ssh execution had been appended to buffer which could not be deserialized
for _,hostip := range hostiplist {
b.Reset()
c.Reset()

Related

How to unmarshal/parse a request using Go correctly?

How to parse a curl request response in Golang correctly? I have tried the following where I send a request to an api and its response is:
{
"Certificates": [
{
"Name": "some-name.here",
.......
}
],
"DataRange": "Certificates 1 - 1",
"TotalCount": 1
}
Now I want to use the Name in the Certificates in a string variable. i.e match. Before even I get to the looping through of the response, I get the error: json: cannot unmarshal object into Go value of type []program.Item. This error is coming from json.Unmarshal function where I pass my []bytes and the struct to use the bytes. Am I doing this correctly?
type Item struct {
Certificates []CertificatesInfo
}
type CertificatesInfo struct {
Name string
}
func main() {
url := .....
req, err := http.NewRequest("GET", url, nil)
resp, err := http.DefaultClient.Do(req)
if err != nil {
// handle err
continue
}
defer resp.Body.Close()
bodyBytes, err := io.ReadAll(resp.Body)
if err != nil {
continue
}
var ItemInfo []Item
if err := json.Unmarshal(bodyBytes, &ItemInfo); err != nil {
return nil, fmt.Errorf("failed to parse %v", err)
}
for _, infos := range ItemInfo {
for _, names := range infos.Certificates {
infraId:= names.Name
match:= display_name
}
}
}
There's a mismatch between your sample JSON response data (a single object containing a list of certificates), and your code, which is expecting a list of objects, each of which contain a list of certificates).
Assuming that your JSON example is correct, this bit:
var ItemInfo []Item
if err := json.Unmarshal(bodyBytes, &ItemInfo); err != nil {
. . .
should probably be
var itemInfo Item
if err := json.Unmarshal(bodyBytes, &ItemInfo); err != nil {
. . .
Works on my machine:
https://go.dev/play/p/8FqIif1zzsQ
package main
import (
"encoding/json"
"fmt"
)
func main() {
resp := Item{}
err := json.Unmarshal([]byte(data), &resp)
if err != nil {
fmt.Print(err)
panic(err)
}
fmt.Printf("resp.DataRange: %s\n", resp.DataRange)
fmt.Printf("resp.TotalCount: %d\n", resp.TotalCount)
fmt.Printf("len(resp.Certificates: %d\n", len(resp.Certificates))
for i, c := range resp.Certificates {
fmt.Printf("resp.Certificates[%d].Name: %s\n", i, c.Name)
}
}
type Item struct {
Certificates []CertificatesInfo
DataRange string
TotalCount int
}
type CertificatesInfo struct {
Name string
}
const data = `
{
"Certificates": [
{ "Name": "alpha" },
{ "Name": "bravo" },
{ "Name": "charlie" }
],
"DataRange": "Certificates 1 - 1",
"TotalCount": 3
}
`

How to create a reusable code in Golang to read different yamls and put them into different structs types

I have to read let say 2 or 3 or more yamls that are different in structure and have a struct for each of those structures where I want to store them. So far I am creating separate functions for each and it works, but does not look very elegant... I think.
Here are the functions today:
// read the Yaml into struct(s)
type Config struct {...}
type ExecuteQueries struct {...}
func parseYamlConfig(pathYaml string) Config {
myConfig := Config{}
var err error
var yamlFile []byte
if pathYaml == "" {
yamlFile, err = ioutil.ReadFile("./conf/conf.yaml")
} else {
yamlFile, err = ioutil.ReadFile(pathYaml)
}
if err != nil {
log.Fatalf("error: %v", err)
}
err = yaml.Unmarshal([]byte(yamlFile), &myConfig)
if err != nil {
log.Fatalf("error: %v", err)
}
return myConfig
}
func parseYamlConfig2(pathYaml string) ExecuteQueries {
myConfig := ExecuteQueries{}
var err error
var yamlFile []byte
if pathYaml == "" {
yamlFile, err = ioutil.ReadFile("./conf/conf.yaml")
} else {
yamlFile, err = ioutil.ReadFile(pathYaml)
}
if err != nil {
log.Fatalf("error: %v", err)
}
err = yaml.Unmarshal([]byte(yamlFile), &myConfig)
if err != nil {
log.Fatalf("error: %v", err)
}
return myConfig
}
Notice that they are actually different in what they return and what they receive, but the processing of data is very similar. How should this be expressed in a more elegant way?
func unmarshalYAMLFile(path string, v interface{}) error {
if path == "" {
path = "./conf/conf.yaml"
}
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
return yaml.NewDecoder(f).Decode(v)
}
conf1 := Config{}
if err := unmarshalYAMLFile("/path/to/conf.yaml", &conf1); err != nil {
panic(err)
}
conf2 := ExecuteQueries{}
if err := unmarshalYAMLFile("/path/to/conf_2.yaml", &conf2); err != nil {
panic(err)
}

I have got panic: runtime error: invalid memory address or nil pointer dereference in Golang

panic: runtime error: invalid memory address or nil pointer dereference
when i add
statusCOD, err := c.route.FindStatusCODByOrigin(ctx, req.Origin)
if err != nil {
if err != sql.ErrNoRows {
ddlogger.Log(ctx, span, "Find status cod destination and cod origin", err)
errChan <- err
return
}
}
if statusCOD != nil {
IsCODOrigin = statusCOD.IsCODOrigin
IsCODDestination = statusCOD.IsCODDestination
}
in this func
for i, v := range detailShipments {
var dtPackage repo.PackageBaseModel
go func(idx int, vShipment repo.ShipmentDetailBaseModel, dataShipmentNew repo.ShipmentCODCreateModel) {
defer wg1.Done()
randomID := commonString.RandomWithCustomCharList(c.config.ShipmentCODIDRandom, c.config.ShipmentIDCharlist)
shipmentID := fmt.Sprintf("%s%s", prefix, randomID)
dataShipmentNew.ShipmentBaseModel.ShipmentID = strings.ToUpper(shipmentID)
dataShipmentNew.ShipmentDetailBaseModel = vShipment
var commodityName string
sCategory, err := c.shipmentCategoryRepo.FindOneShipmentCategoryByID(ctx, vShipment.ShipmentCategoryID.Int64)
if err != err && err != sql.ErrNoRows {
ddlogger.Log(ctx, span, "shipmentService-CreateShipmentCOD "+shipmentID, " Failed shipmentCategoryRepo.FindOneShipmentCategoryByID", err)
} else {
if sCategory != nil {
commodityName = sCategory.CommodityName.String
}
}
req := &repo.TariffRequestModelV2{
Origin: form.DetailSender.Origin,
Destination: vShipment.Destination,
Weight: vShipment.GrossWeight / 1000,
Commodity: commodityName,
GoodsValue: int64(vShipment.GoodValues.Float64),
IsInsurance: vShipment.IsInsurance.Bool,
Product: vShipment.ProductType,
Width: vShipment.DimensionWidth.Float64,
Height: vShipment.DimensionHeight.Float64,
Length: vShipment.DimensionLength.Float64,
IsCOD: true,
CODValue: vShipment.CODValue.Float64,
}
tariff, err := c.coreSystemRepo.CheckTariffV3(ctx, req)
if err != nil {
ddlogger.Log(ctx, span, "[shipmentService-CreateShipmentCOD [coreSystemRepo.CheckTariffV3]]", err)
errChan <- err
return
}
statusCOD, err := c.route.FindStatusCODByOrigin(ctx, req.Origin)
if err != nil {
if err != sql.ErrNoRows {
ddlogger.Log(ctx, span, "Find status cod destination and cod origin", err)
errChan <- err
return
}
}
if statusCOD != nil {
IsCODOrigin = statusCOD.IsCODOrigin
IsCODDestination = statusCOD.IsCODDestination
}
................
}
wg1.Wait()
close(shipmentChan)
close(errChan)
close(amountChan)
It looks like you just created the pointer, and it doesn't point to any memory spaces, you can check all the variables in your new code to find it. e.g. ctx

Go - How can I make sure there aren't empty fields being sent in this code via gRPC?

I am writing some code to send logs with gRPC to a server. The problem I'm having is that the logs are different and not all have every field, but . I'm wondering how I can set only the fields they do have inside logpb.Log short of creating several different types of logs in the proto file?
This is all one methods, but the formatting on StackOverflow isn't with it.
The code:
func (*Client) SendWithgRPC(entry *log.Entry) error {
conn, err := grpc.Dial("localhost:50051", grpc.WithInsecure())
if err != nil {
log.Fatalf("Could not connect: %v", err)
}
defer conn.Close()
c := logpb.NewLogServiceClient(conn)
stream, err := c.SendLogs(context.Background())
if err != nil {
log.Fatalf("Error while calling SendLogs: %v", err)
}
logFields := entry.Data
category := ""
debug_id := ""
request_id := ""
tipi := ""
uri := ""
var user_id int32 = 0
ip := ""
if logFields["category"] != nil {
category = logFields["category"].(string)
}
if logFields["debug_id"] != nil {
debug_id = logFields["debug_id"].(string)
}
if logFields["request_id"] != nil {
request_id = logFields["request_id"].(string)
}
if logFields["type"] != nil {
tipi = logFields["type"].(string)
}
if logFields["uri"] != nil {
uri = logFields["uri"].(string)
}
if logFields["user_id"] != nil {
user_id = int32(logFields["user_id"].(int))
}
if logFields["ip"] != nil {
ip = logFields["ip"].(string)
}
logEntry := &logpb.Log{
Time: entry.Time.String(),
Level: entry.Level.String(),
Msg: entry.Message,
Category: category,
DebugId: debug_id,
Ip: ip,
RequestId: request_id,
Type: tipi,
Uri: uri,
Id: user_id,
}
logs := []*logpb.Log{logEntry}
logarray := []*logpb.SendLogsRequest{
&logpb.SendLogsRequest{
Logs: logs,
},
}
fmt.Print(logarray)
for _, req := range logarray {
stream.Send(req)
}
_, err = stream.CloseAndRecv()
if err != nil {
fmt.Printf("Error with response: %v", err)
}
return nil
}
You can use something like https://github.com/go-playground/validator
It offers validation using tags. There is a lot more to this lib apart from basic validation. Do check it out.
Sample example https://github.com/go-playground/validator/blob/master/_examples/simple/main.go
As easy as it gets :
package main
import (
"fmt"
"github.com/go-playground/validator/v10"
)
type Name struct {
First string `validate:"required"`
Last string
}
func main() {
validName := Name{First: "John"} // Last name is empty and is not required
invalidName := Name{Last: "Doe"} // first name is empty but is required
validate := validator.New()
err := validate.Struct(invalidName)
if err != nil {
fmt.Println("invalid name struct caused error")
}
err = validate.Struct(validName)
if err != nil {
fmt.Println("valid name struct caused error")
}
fmt.Println("GoodBye")
}
The output of above :
invalid name struct caused error
GoodBye
working code on go-playground : https://play.golang.org/p/S4Yj1yr16t2

Upload a file from aws s3 to linux server directly with private key

There are some larger size files in AWS S3. I need to copy those files to a remote server.
I've tried with the below code, but I'm not able to transfer the file. Is this right way or I need to do in any other ways.
package main
import (
"fmt"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/pkg/errors"
"github.com/tmc/scp"
"golang.org/x/crypto/ssh"
"io"
"io/ioutil"
"devtools/objectstorage"
"log"
"os"
)
func main() {
var osSecretKey = "secret-key"
var osAccessKeyID = "access-key"
var osRegion = "region"
var osUrl = "url"
var osBucketName = "bucket"
accpusr := "username"
accphost := "server"
sshport := "22"
scpDestinationPath := "/destination/path"
s3Config := objectstorage.S3Config{
AccessKeyID: osAccessKeyID,
SecretAccessKey: osSecretKey,
Region: osRegion,
Bucket: osBucketName,
URL: osUrl,
}
var s3FileName = "sample.txt"
var storage, err = objectstorage.New(s3Config)
if err != nil {
log.Fatal("failed to create s3 session", err)
}
dlObj, err := storage.Download(s3FileName)
if err != nil {
log.Fatal("failed to download s3 file", err)
}
log.Println("downloaded object")
err = Send(accpusr, accphost, sshport, scpDestinationPath, s3FileName, *dlObj.ContentLength, dlObj.Body)
if err != nil {
log.Fatal("failed to copy file", err)
}
log.Println("file copied successfully")
}
func Send(remoteUser, remoteHost, remoteSSHPort, remoteFolder, remoteFileName string, size int64, r io.Reader) error {
session, err := connect(remoteUser, remoteHost, remoteSSHPort)
if err != nil {
return err
}
defer session.Close()
err = scp.Copy(size, os.FileMode.Perm(0770), remoteFileName, r, remoteFolder, session)
if err != nil {
errMsg := fmt.Sprintf("Error sending file: %s", err.Error())
return errors.New(errMsg)
}
return nil
}
func connect(remoteUser string, remoteHost string, remoteSSHPort string) (*ssh.Session, error) {
//retrieve public key from private key
key, err := retrieveKey()
if err != nil {
return nil, err
}
var sshConfig ssh.Config
sshConfig.SetDefaults()
sshConfig.Ciphers = append(sshConfig.Ciphers, "aes256-cbc")
// Define the Client Config as:
config := &ssh.ClientConfig{
Config: sshConfig,
User: remoteUser,
Auth: []ssh.AuthMethod{
ssh.PublicKeys(key),
},
HostKeyCallback: ssh.InsecureIgnoreHostKey(),
}
client, err := ssh.Dial("tcp", fmt.Sprintf("%s:%s", remoteHost, remoteSSHPort), config)
if err != nil {
errMsg := fmt.Sprintf("Error connecting to client: %s", err.Error())
return nil, errors.New(errMsg)
}
session, err := client.NewSession()
if err != nil {
errMsg := fmt.Sprintf("Error creating session: %s", err.Error())
return nil, errors.New(errMsg)
}
return session, nil
}
func retrieveKey() (key ssh.Signer, err error) {
scpCert := ""
buf, err := ioutil.ReadFile(fmt.Sprintf("%s", scpCert))
if err != nil {
errMsg := fmt.Sprintf("Error retrieving key: %s", err.Error())
return nil, errors.New(errMsg)
}
key, err = ssh.ParsePrivateKey(buf)
if err != nil {
errMsg := fmt.Sprintf("Error parsing key: %s", err.Error())
return nil, errors.New(errMsg)
}
return key, nil
}
objectstorage.New will return aws s3 session.
storage.Download will return s3.GetObjectOutput.

Resources