Reading JSON from a link - go

I want to return the string returned from this link, and build a JSON object based on the returned data, I wrote the below code that is reading the url and getting the string correctly, but it fails in building the JSON object, and do not return anything!
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
)
type Product struct {
ProductId string `json:"id"`
ProductName string `json:"description"`
}
func main() {
url := "https://script.googleusercontent.com/macros/echo?user_content_key=WBSJPDNSN6X1FCYeXsR6TDaDval0vdvmSoMmXFhGbt5sfK0ia80Dp7kPD27GLpZbYz8vrwfDiUecI2oGMjEtgfL5o8Da25T1m5_BxDlH2jW0nuo2oDemN9CCS2h10ox_1xSncGQajx_ryfhECjZEnGb6k9xaGtOX6M1tIiG811CRpk9nXl8ZKS7UJTno1dvQXMe1kqfAj8WxsSkLor-EqzOmbnRGq-tk&lib=M0B6GXYh0EOYMkP7qr1Xy9xw8GuJxFqGH"
resp, err := http.Get(url)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
htmlData, err := ioutil.ReadAll(resp.Body) //<--- here!
if err != nil {
fmt.Println(err)
os.Exit(1)
}
// print out
fmt.Println(string(htmlData))
var product Product
json.Unmarshal([]byte(string(htmlData)), &product)
fmt.Printf("ID: %s, Description: %s", product.ProductId, product.ProductName)
}
Output:
{"user":[{"ProductId":1,"ProductName":"Helmet"},{"ProductId":2,"ProductName":"Glove"},{"ProductId":3,"ProductName":"Detecttor"}]}
ID: , Description:

1- The unmarshal target struct must match the data,
type Data struct {
User []Product `json:"user"`
}
2- The fields types should be matching, so ProductID should be uint
3- The json OP got in output should match the tags used, json:"id" should be replaced by json:"ProductId" same for description
type Product struct {
ProductId uint `json:"ProductId"`
ProductName string `json:"ProductName"`
}
So, a working code is:
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
)
type Data struct {
User []Product `json:"user"`
}
type Product struct {
ProductId uint `json:"ProductId"`
ProductName string `json:"ProductName"`
}
func main() {
url := "https://script.googleusercontent.com/macros/echo?user_content_key=WBSJPDNSN6X1FCYeXsR6TDaDval0vdvmSoMmXFhGbt5sfK0ia80Dp7kPD27GLpZbYz8vrwfDiUecI2oGMjEtgfL5o8Da25T1m5_BxDlH2jW0nuo2oDemN9CCS2h10ox_1xSncGQajx_ryfhECjZEnGb6k9xaGtOX6M1tIiG811CRpk9nXl8ZKS7UJTno1dvQXMe1kqfAj8WxsSkLor-EqzOmbnRGq-tk&lib=M0B6GXYh0EOYMkP7qr1Xy9xw8GuJxFqGH"
resp, err := http.Get(url)
if err != nil {
log.Fatal(err)
}
defer resp.Body.Close()
htmlData, err := ioutil.ReadAll(resp.Body)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
// print out
fmt.Println(string(htmlData))
//var product Product
var data Data
json.Unmarshal([]byte(string(htmlData)), &data)
fmt.Println(data)
fmt.Println(data.User)
fmt.Println(data.User[0])
fmt.Printf("id: %v, description: %s", data.User[0].ProductId, data.User[0].ProductName)
}

Related

Golang RCP call hanging

I have a back-end app and a front-end, both in Go. I am trying to make them talk via rpc.
back-end main.go
package main
import (
"fmt"
"log"
"net"
"net/http"
"net/rpc"
"time"
)
type Application struct {
config struct {
server struct {
port int
network string
}
}
}
type MusicProject struct {
Id string
CreatedTime time.Time
LastEditedTime time.Time
Title string
Year int
Status string
Description string
ChoirRollup string
}
func main() {
var app Application
app.config.server.port = 5002
app.config.server.network = "tcp"
if err := rpc.Register(new(Application)); err != nil {
log.Fatal(err)
}
rpc.HandleHTTP()
// start the rpc server
log.Println("Starting server port", app.config.server.port)
l, err := net.Listen("tcp", fmt.Sprintf(":%v", app.config.server.port))
if err != nil {
log.Fatal(err)
}
if err = http.Serve(l, nil); err != nil {
log.Fatal(err)
}
}
func (app *Application) GetMusicProjectById(id string, model *MusicProject) error {
musicProject := MusicProject{
Id: id,
CreatedTime: time.Now(),
LastEditedTime: time.Now(),
Title: "Test Project",
Year: 2020,
Status: "Completed",
Description: "Short project Description",
ChoirRollup: "Best Choir",
}
model = &musicProject
return nil
}
front-end rpc call
package main
import (
"log"
"net/rpc"
"time"
)
type MusicProject struct {
Id string
CreatedTime time.Time
LastEditedTime time.Time
Title string
Year int
Status string
Description string
ChoirRollup string
}
func main() {
dial, err := rpc.Dial("tcp", "localhost:5002")
if err != nil {
log.Println(err)
}
projectID := "some_id_123"
var musicProject MusicProject
err = dial.Call("Application.GetMusicProjectById", projectID, &musicProject)
if err != nil {
log.Println(err)
}
log.Println(musicProject)
}
Once the client method is call, the call hanged indefinite with out sending back any error, so quite difficult to debug.
Do you have any suggestion for debugging? Do you see anything wrong in my code?
Thank you for your help!
Your server is serving HTTP:
if err = http.Serve(l, nil); err != nil {
log.Fatal(err)
}
But your client is using straight TCP (HTTP runs over TCP but adds another layer):
err = dial.Call("Application.GetMusicProjectById", projectID, &musicProject)
So to fix your issue you need to change one side or the other. For example change the server to:
rpc.Accept(l)
(alternatively you could use rpc.DialHTTP in the client)
There are a couple of other issues with your code (the main one being the way you set the response in the back-end). The following works for me (have not put much effort into making it tidy or testing!).
server
package main
import (
"fmt"
"log"
"net"
"net/rpc"
"time"
)
type MusicProject struct {
Id string
CreatedTime time.Time
LastEditedTime time.Time
Title string
Year int
Status string
Description string
ChoirRollup string
}
type Application struct {
}
func (app *Application) GetMusicProjectById(id string, model *MusicProject) error {
*model = MusicProject{
Id: id,
CreatedTime: time.Now(),
LastEditedTime: time.Now(),
Title: "Test Project",
Year: 2020,
Status: "Completed",
Description: "Short project Description",
ChoirRollup: "Best Choir",
}
return nil
}
func main() {
const port = 5002
if err := rpc.Register(new(Application)); err != nil {
log.Fatal(err)
}
// start the rpc server
log.Println("Starting server on port:", port)
l, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
if err != nil {
log.Fatal(err)
}
rpc.Accept(l)
}
client
package main
import (
"log"
"net/rpc"
"time"
)
type MusicProject struct {
Id string
CreatedTime time.Time
LastEditedTime time.Time
Title string
Year int
Status string
Description string
ChoirRollup string
}
func main() {
dial, err := rpc.Dial("tcp", "localhost:5002")
if err != nil {
log.Fatal(err)
}
projectID := "some_id_123"
var musicProject MusicProject
err = dial.Call("Application.GetMusicProjectById", projectID, &musicProject)
if err != nil {
log.Fatal("error:", err)
}
log.Println(musicProject)
}

Cast value from JSON when unmarshalling to struct

I have this JSON API that returns the delivery date as a UNIX timestamp. I would prefer using Time in the rest of the application. The following works but doesn't feel very Go like.
type Delivery struct {
Time string `json:"time"`
}
func timestampToTime(s string) time.Time {
i, _ := strconv.ParseInt(s, 10, 64)
returns time.Unix(i, 0)
}
fmt.Println(timestampToTime(Delivery.Time))
// 2019-02-17 11:55:00 +0100 CET
Is there a way to cast an incoming value in a struct?
You can do something very similar to the custom JSON Unmarshal method described here:
http://choly.ca/post/go-json-marshalling/
Assuming that the JSON contains a string in your case this would look like this:
package main
import (
"encoding/json"
"fmt"
"os"
"strconv"
"time"
)
const input = `{"time": "946684799"}`
type Delivery struct {
Time time.Time `json:"time"`
}
func (d *Delivery) UnmarshalJSON(data []byte) error {
type Alias Delivery
aux := &struct {
Time string `json:"time"`
*Alias
}{
Alias: (*Alias)(d),
}
if err := json.Unmarshal(data, &aux); err != nil {
return err
}
i, err := strconv.ParseInt(aux.Time, 10, 64)
if err != nil {
return err
}
d.Time = time.Unix(i, 0)
return nil
}
func main() {
var delivery Delivery
err := json.Unmarshal([]byte(input), &delivery)
if err != nil {
fmt.Println(err.Error())
os.Exit(1)
}
fmt.Println(delivery.Time)
}
https://play.golang.org/p/mdOmUO2EDIR

How to convert the dynamically produced array object data into JSON format string in golang?

On the data retrieval the data is in the form of array object like this:
[{1 fruits Apple Apple is my favorite fruit.} {2 colors Red Red color is always charming.} {3 flowers Lotus It is one of the most beautiful flowers in this world.}]
how will I change it in JSON. I just need to break the array object braces [].
I have tried to Marshal it. But It gives me like:
[{"id":1,"category":"fruits","name":"Apple","description":"Apple is my favorite fruit."},{"id":2,"category":"colors","name":"Red","description":"Red color is always charming."},{"id":3,"category":"flowers","name":"Lotus","description":"It is one of the most beautiful flowers in this world."}]
The code I'd tried
Struct
type Item struct {
Id int `json:"id"`
Category string `json:"category"`
Name string `json:"name"`
Description string `json:"description"`
}
type Items []Item
Here the function for retrieving data
func GetData(productQuery interface{}) (result Items, err error) {
mongoSession := ConnectDb()
sessionCopy := mongoSession.Copy()
defer sessionCopy.Close()
getCollection := mongoSession.DB("custom").C("custom")
err = getCollection.Find(productQuery).Select(bson.M{"password": 0}).All(&result) //.Skip(skip).Limit(limit)
if err != nil {
return result, err
}
return result, nil
}
/*
*
* Retrieve the data used by main function
*
*
*/
func retrieve(c *gin.Context) {
conditions := bson.M{}
data, err :=GetData(conditions)
if err != nil {
fmt.Println("There is somthing wrong")
}
fmt.Println("--------------------")
fmt.Println(data)
fmt.Println("--------------------")
arrange(data)
return
}
func arrange(data Items) {
pagesJson, err := json.Marshal(data)
if err != nil {
log.Fatal("Cannot encode to JSON ", err)
}
fmt.Println(string(pagesJson))
}
I want to make the output like
{"id": 1,"category": "fruits","name": "Apple","description": "Apple is my favorite fruit."} {"id": 2,"category": "colors","name": "Red",description": "Red color is always charming."} {"id": 3,"category": "flowers","name": "Lotus","description": "It is one of the most beautiful flowers in this world."}
can anyone help me I tried it lot of times but don't take any success.
Simply encode each element of your slice individually:
package main
import (
"encoding/json"
"log"
"os"
)
type T struct {
ID int `json:"id"`
Category string `json:"category"`
Name string `json:"name"`
Description string `json:"description"`
}
func main() {
var data []T
enc := json.NewEncoder(os.Stdout)
for _, x := range data {
if err := enc.Encode(x); err != nil {
log.Fatal(err)
}
}
}
This code will works
package main
import (
"bytes"
"encoding/json"
"fmt"
"log"
"strings"
)
type Item struct {
Id int `json:"id"`
Category string `json:"category"`
Name string `json:"name"`
Description string `json:"description"`
}
type Items []Item
var myJson = []byte(`[{
"id":1,
"category":"fruits",
"name":"Apple",
"description":"Apple is my favorite fruit."
},
{
"id":2,
"category":"colors",
"name":"Red",
"description":"Red color is always charming."
},
{
"id":3,
"category":"flowers",
"name":"Lotus",
"description":"It is one of the most beautiful flowers in this world."
}]`)
func main() {
var items Items
err := json.Unmarshal(myJson, &items)
if err != nil {
log.Fatal(err)
}
s, err := getMyString(items)
if err != nil {
log.Fatal(err)
}
fmt.Println(s)
}
func getMyString(items Items) (string, error) {
var buffer bytes.Buffer
var err error
var b []byte
for _, item := range items {
b, err = json.Marshal(item)
if err != nil {
return "", err
}
buffer.WriteString(string(b) + " ")
}
s := strings.TrimSpace(buffer.String())
return s, nil
}

Using a singleton database class with separate models and services packages

My question is similar to How to create singleton DB class in GoLang but I'm having trouble getting it to work with separate models and services packages.
project/lib/database/mysql.go:
package database
import (
"fmt"
_ "github.com/go-sql-driver/mysql"
"github.com/jinzhu/gorm"
)
type Manager struct {
*gorm.DB
}
var Mgr *Manager
func init() {
dsn := MysqlConnectionString("parseTime=true")
tablePrefix := "demo"
var err error
gorm.DefaultTableNameHandler = func(db *gorm.DB, defaultTableName string) string {
return fmt.Sprintf("%v_%v", tablePrefix, defaultTableName)
}
db, err := gorm.Open("mysql", dsn)
if err != nil {
panic(err)
}
Mgr = &Manager{db}
}
project/lib/models/retailer_keys.go
package models
import (
"fmt"
"project/lib/database"
"time"
)
type RetailerKeysInterface interface {
RetailerKeys() ([]*RetailerKey, error)
}
type DB struct {
database.Manager
}
type RetailerKey struct {
Id int `json:"id"`
RetailerId int `json:"retailer_id"`
Key string `json:"key"`
Enabled *bool `json:"enabled"`
CreatedAt *time.Time `json:"created_at"`
UpdatedAt *time.Time `json:"updated_at"`
}
func (db *DB) RetailerKeys() ([]*RetailerKey, error) {
var keys []*RetailerKey
if err := db.Find(&keys).Error; err != nil {
return nil, err
}
return keys, nil
}
project/lib/services/retailer_keys.go
import (
"fmt"
"strings"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/apigateway"
"gopkg.in/volatiletech/null.v6"
"project/lib/models"
"project/lib/services/api_keys"
)
func GetKeys() ([]*models.RetailerKey, error) {
var q models.RetailerKeysInterface
keys, err := q.RetailerKeys()
if err != nil {
return nil, err
}
return keys, nil
}
func CreateKey(id int) (models.RetailerKey, error) {
...
}
Then be able to use it in my main package like:
package main
import (
"context"
"encoding/json"
// "reflect"
"fmt"
"github.com/aws/aws-lambda-go/events"
"github.com/aws/aws-lambda-go/lambda"
_ "project/lib/config"
"project/lib/services"
)
func Handler(ctx context.Context, request events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) {
statusCode := 200
keys, err := services.GetKeys()
if err != nil {
statusCode = 400
}
body, _ := json.Marshal(keys)
return events.APIGatewayProxyResponse{
Body: string(body),
StatusCode: statusCode,
}, nil
}
...
I'd like to be able to embed the relevant subset of the Manager type in my models.
EDIT:
Edited the question/code based on feedback in comments.
This gives me an error: runtime error: invalid memory address or nil pointer dereference.
I was definitely misunderstanding interfaces in go. After going through A Tour of Go it started to become clearer how it all fits together.
This is that I ended up doing for anyone that is going through the same thing. I'll leave the original question up so you can see the differences.
project/lib/database/mysql.go:
package database
import (
"fmt"
"log"
_ "github.com/go-sql-driver/mysql" // Needed for gorm
"github.com/jinzhu/gorm"
)
var Manager *gorm.DB
func init() {
var err error
dsn := MysqlConnectionString("parseTime=true")
tablePrefix := "qcommerce"
gorm.DefaultTableNameHandler = func(db *gorm.DB, defaultTableName string) string {
return fmt.Sprintf("%v_%v", tablePrefix, defaultTableName)
}
Manager, err = gorm.Open("mysql", dsn)
if err != nil {
log.Fatal(err)
}
if err := Manager.DB().Ping(); err != nil {
log.Fatal(err)
}
}
project/lib/models/retailer_keys.go
package models
import (
"project/lib/database"
"time"
)
type QRetailerKeys interface {
Insert() error
Find() error
}
type RetailerKey struct {
ID int `json:"id"`
RetailerID int `json:"retailer_id"`
Retailer Retailer `json:"retailer"`
Key string `json:"key"`
Enabled bool `json:"enabled" gorm:"DEFAULT:true"`
CreatedAt *time.Time `json:"created_at"`
UpdatedAt *time.Time `json:"updated_at"`
}
// RetailerKeys returns a slice of all keys in table
func RetailerKeys() ([]*RetailerKey, error) {
var keys []*RetailerKey
if err := database.Manager.Find(&keys).Error; err != nil {
return nil, err
}
return keys, nil
}
func (r *RetailerKey) Find() error {
...
}
// Create a new key
func (r *RetailerKey) Create() error {
return database.Manager.Create(&r).Error
}
project/lib/services/retailer_keys.go
package services
import (
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/session"
// "github.com/aws/aws-sdk-go/service/apigateway"
"partners.drinks.com/lib/models"
"partners.drinks.com/lib/services/api_keys"
)
func sessionBuilder() *session.Session {
config := &aws.Config{
Region: aws.String("us-west-2"),
}
session := session.Must(session.NewSession(config))
return session
}
func GetKeys() ([]*models.RetailerKey, error) {
keys, err := models.RetailerKeys()
if err != nil {
return nil, err
}
return keys, nil
}
func CreateKey(id int) (models.RetailerKey, error) {
apikeys := &api_keys.ApiKeyBuilder{}
base64Key := apikeys.GenUUID().GenKey().Base64
var key = models.RetailerKey{
RetailerID: id,
Key: base64Key,
Enabled: func(b bool)
}
if err := key.Create(); err != nil {
return models.RetailerKey{}, err
}
...
return key, nil
}
I use it like:
package main
import (
"context"
"encoding/json"
"github.com/aws/aws-lambda-go/events"
"github.com/aws/aws-lambda-go/lambda"
_ "partners.drinks.com/lib/config"
"partners.drinks.com/lib/services"
)
func Handler(ctx context.Context, request events.APIGatewayProxyRequest) (events.APIGatewayProxyResponse, error) {
statusCode := 200
keys, err := services.GetKeys()
if err != nil {
statusCode = 400
}
body, _ := json.Marshal(keys)
return events.APIGatewayProxyResponse{
Body: string(body),
StatusCode: statusCode,
}, nil
}
...
Thanks to #mkopriva for the linked resources in the comments.

gin-gonic json conversion output throws empty array

for the following golang program, i'm not able to get json output using gin-gonic library, if i do fmt.Print variable shows values, but when i convert to c.JSON (inventory) out it shows empty array, what is wrong with my code?
package main
import (
"database/sql"
"os"
"github.com/gin-gonic/gin"
_ "github.com/go-sql-driver/mysql"
)
func index(c *gin.Context) {
hostname, err := os.Hostname()
checkErr(err)
c.String(200, "v3 "+hostname)
}
func healthz(c *gin.Context) {
c.String(200, "OK")
}
type InventoryItem struct {
id int
productID string
productCost int
productAvailabilty int
productSubcat string
}
func checkErr(err error) {
if err != nil {
panic(err)
}
}
/******************* MAIN Function **************/
func main() {
app := gin.Default()
app.GET("/", index)
app.GET("/healthz", healthz)
app.GET("/inventory", fetch)
app.Run(":8000")
}
/******************* End MAIN Function **************/
func fetch(c *gin.Context) {
var (
invt InventoryItem
inventory []InventoryItem
)
connStr := os.Getenv("sql_user") + ":" + os.Getenv("sql_password") + "#tcp(" + os.Getenv("sql_host") + ":3306)/" + os.Getenv("sql_db")
db, err := sql.Open("mysql", connStr)
checkErr(err)
defer db.Close()
rows, err := db.Query("SELECT id,product_id as productID,product_cost as productCost,product_availabilty as productAvailabilty,product_subcat as productSubcat FROM inventory;")
for rows.Next() {
err = rows.Scan(&invt.id, &invt.productID, &invt.productCost, &invt.productAvailabilty, &invt.productSubcat)
checkErr(err)
inventory = append(inventory, invt)
}
checkErr(err)
defer rows.Close()
//fmt.Print(inventory[0].productAvailabilty)
c.JSON(200, inventory)
}
Issue is InventoryItem struct fields have to be exported -
type InventoryItem struct {
ID int `json:"id"`
ProductID string `json:"product_id"`
ProductCost int `json:"product_cost"`
ProductAvailabilty int `json:"product_availability"`
ProductSubcat string `json:"product_subact"`
}
Read more about exported and un-exported fields.

Resources