I'm trying to get information from a google api, but the response body appears to be empty. it just outputs {} to the console. Not sure where I went wrong as I used the docs to get the payload information for the request: https://developers.google.com/safe-browsing/v4/lookup-api
package main
import (
"fmt"
"encoding/json"
"io/ioutil"
"net/http"
"strings"
)
type payload struct {
Client client `json:"client"`
ThreatInfo threatInfo `json:"threatInfo"`
}
type client struct {
ClientId string `json:"clientId"`
ClientVersion string `json:"clientVersion"`
}
type threatInfo struct {
ThreatTypes []string `json:"threatTypes"`
PlatformTypes []string `json:"platformTypes"`
ThreatEntryTypes []string `json:"threatEntryTypes"`
ThreatEntries []entry `json:"threatEntries"`
}
type entry struct {
URL string `json:"url"`
}
func checkURLs(urls []string) {
// populate entries
var entries = []entry{}
for _, url := range urls {
entries = append(entries, entry{URL: url})
}
data := payload {
Client: client{
ClientId: "myapp",
ClientVersion: "0.0.1",
},
ThreatInfo: threatInfo{
ThreatTypes: []string{"MALWARE", "SOCIAL_ENGINEERING", "POTENTIALLY_HARMFUL_APPLICATION"},
PlatformTypes: []string{"ANY_PLATFORM"},
ThreatEntryTypes: []string{"URL"},
ThreatEntries: entries,
},
}
jsonBytes, _ := json.Marshal(data)
key := "*"
api := fmt.Sprintf("https://safebrowsing.googleapis.com/v4/threatMatches:find?key=%s", key)
req, _ := http.NewRequest("POST", api, strings.NewReader(string(jsonBytes)))
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
fmt.Println(res) // 200 OK
fmt.Println(err) // nil
fmt.Println(string(body)) // {}
}
func main() {
checkURLs([]string{"http://www.urltocheck1.org/", "http://www.urltocheck2.org/"})
}
EDIT
I found a go package by google to do most of the heavy lifting, and yet, still an empty response. I should add I managed to get my hands on some urls that do contain malware, and IS detected via googles transparency report url search: https://transparencyreport.google.com/safe-browsing/search
So why is it empty for me when there should be results?
package main
import (
"fmt"
"github.com/google/safebrowsing"
)
func checkURLs(urls []string) {
sb, err := safebrowsing.NewSafeBrowser(safebrowsing.Config{
ID: "myapp",
Version: "0.0.1",
APIKey: "*",
})
if err != nil {
fmt.Println(err)
return
}
threats, err := sb.LookupURLs(urls)
if err != nil {
fmt.Println(err)
return
}
fmt.Println(threats)
}
func main() {
checkURLs([]string{"http://www.urltocheck1.org/", "http://www.urltocheck2.org/"})
}
I think this was stated in the docs
Note: If there are no matches (that is, if none of the URLs specified in the request are found on any of the lists specified in a request), the HTTP POST response simply returns an empty object in the response body.
Related
I'm trying to develop a Terraform provider but I have a problem of the first request body. Here is the code:
type Body struct {
id string
}
func resourceServerCreate(d *schema.ResourceData, m interface{}) error {
key := d.Get("key").(string)
token := d.Get("token").(string)
workspace_name := d.Get("workspace_name").(string)
board_name := d.Get("board_name").(string)
resp, err := http.Post("https://api.trello.com/1/organizations?key="+key+"&token="+token+"&displayName="+workspace_name,"application/json",nil)
if err != nil {
log.Fatalln(err)
}
defer resp.Body.Close()
//lettura body.
body := new(Body)
json.NewDecoder(resp.Body).Decode(body)
log.Println("[ORCA MADONNA] il log funzia "+body.id)
d.Set("board_id",body.id)
resp1, err1 := http.Post("https://api.trello.com/1/boards?key="+key+"&token="+token+"&idOrganization="+body.id+"&=&name="+board_name,"application/json",nil)
if err1 != nil {
log.Fatalln(resp1)
}
defer resp1.Body.Close()
d.SetId(board_name)
return resourceServerRead(d, m)
}
In the log is empty, but the second call have it and work fine. How is it possible?
Go doesn't force you to check error responses, therefore it's easy to make silly mistakes. Had you checked the return value from Decode(), you would have immediately discovered a problem.
err := json.NewDecoder(resp.Body).Decode(body)
if err != nil {
log.Fatal("Decode error: ", err)
}
Decode error: json: Unmarshal(non-pointer main.Body)
So your most immediate fix is to use & to pass a pointer to Decode():
json.NewDecoder(resp.Body).Decode(&body)
Also of note, some programming editors will highlight this mistake for you:
Here's a working demonstration, including a corrected Body structure as described at json.Marshal(struct) returns “{}”:
package main
import (
"bytes"
"encoding/json"
"fmt"
"log"
"net/http"
"time"
)
type JSON = map[string]interface{}
type JSONArray = []interface{}
func ErrFatal(err error, msg string) {
if err != nil {
log.Fatal(msg+": ", err)
}
}
func handleTestRequest(w http.ResponseWriter, req *http.Request) {
w.Write(([]byte)("{\"id\":\"yourid\"}"))
}
func launchTestServer() {
http.HandleFunc("/", handleTestRequest)
go http.ListenAndServe(":8080", nil)
time.Sleep(1 * time.Second) // allow server to get started
}
// Medium: "Don’t use Go’s default HTTP client (in production)"
var restClient = &http.Client{
Timeout: time.Second * 10,
}
func DoREST(method, url string, headers, payload JSON) *http.Response {
requestPayload, err := json.Marshal(payload)
ErrFatal(err, "json.Marshal(payload")
request, err := http.NewRequest(method, url, bytes.NewBuffer(requestPayload))
ErrFatal(err, "NewRequest "+method+" "+url)
for k, v := range headers {
request.Header.Add(k, v.(string))
}
response, err := restClient.Do(request)
ErrFatal(err, "DoRest client.Do")
return response
}
type Body struct {
Id string `json:"id"`
}
func clientDemo() {
response := DoREST("POST", "http://localhost:8080", JSON{}, JSON{})
defer response.Body.Close()
var body Body
err := json.NewDecoder(response.Body).Decode(&body)
ErrFatal(err, "Decode")
fmt.Printf("Body: %#v\n", body)
}
func main() {
launchTestServer()
for i := 0; i < 5; i++ {
clientDemo()
}
}
I am trying to do a simple golang with gin post and get request, every other thing works just fine, apart from the part that the values that are supposed to be in the struct variables are empty, the example is bellow if i didnt explain well
my code(main)
package main
import (
//"fmt"
"github.com/cosimmichael/assessment/app/db_client"
"github.com/cosimmichael/assessment/app/controllers"
"github.com/gin-gonic/gin"
// you need to import go mod init for this parkage to work
// "github.com/cosimmichael/assessment/app/strutil"
// "github.com/cosimmichael/assessment/app/routers"
// "net/http"
)
func main(){
db_client.InitialiseDBConnection()
r := gin.Default()
r.POST("api/v1/products/create", controller.CreateProducts)
r.GET("api/v1/products/{product_id}/show", controller.GetPosts)
if err := r.Run(":3000"); err != nil {
panic(err.Error())
}
// router.HandleRoutes()
// fmt.Println("Server Starting.. # port :3000")
// http.ListenAndServe(":3000", nil)
}
my code (controller)
package controller
import (
"net/http"
"github.com/gin-gonic/gin"
"github.com/cosimmichael/assessment/app/db_client"
// "fmt"
)
type Post struct {
id int64 `json: "id"`
title *string `json: "title"`
description *string `json: "description"`
}
func CreateProducts(c *gin.Context) {
var reqBody Post
if err := c.ShouldBindJSON(&reqBody); err != nil {
c.JSON(http.StatusUnprocessableEntity, gin.H{
"error": true,
"message": "Invalid request body",
})
return
}
res, err := db_client.DBClient.Exec("INSERT INTO products (title, description) VALUES (?, ?);",
reqBody.title,//"testing",
reqBody.description,//"Just testing something",
)
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{
"error": true,
"message": "Invalid request body2",
})
return
}
id, err := res.LastInsertId()
if err != nil {
c.JSON(http.StatusInternalServerError, gin.H{
"error": true,
"message": "Invalid request body3",
})
return
}
c.JSON(http.StatusCreated, gin.H{
"error": false,
"id": id,
})
}
func GetPosts(c *gin.Context){
var posts []Post
rows, err := db_client.DBClient.Query("SELECT id, title, description FROM products;")
if err != nil {
c.JSON(http.StatusUnprocessableEntity, gin.H{
"error": true,
"message": "Invalid request body",
})
return
}
for rows.Next(){
var singlePost Post
if err := rows.Scan(&singlePost.id, &singlePost.title, &singlePost.description); err != nil {
c.JSON(http.StatusUnprocessableEntity, gin.H{
"error": true,
"message": "Invalid request body",
})
return
}
posts = append(posts, singlePost)
}
c.JSON(http.StatusOK, rows)
}
my code db_client
package db_client
import (
"database/sql"
//"time"
_ "github.com/go-sql-driver/mysql"
)
var DBClient *sql.DB
func InitialiseDBConnection(){
//[username[:password]#][protocol[(address)]]/dbname[?param1=value1&...¶mN=valueN]
db, err := sql.Open("mysql","root:2580#tcp(localhost:3306)/grabit?parseTime=true")
if err != nil {
panic(err.Error())
}
err = db.Ping()
if err != nil {
panic(err.Error())
}
DBClient = db
}
now when I use postman insert new row, it inserts an empty row with only id, no title nor description, when i try fetching, i get an empty array, please what is the problem, i am new to golang
you need to capitalise the first character of values inside struct field.
For Example:
type Book struct {
ID uint `json:"id" gorm:"primary_key"`
Title string `json:"title"`
Author string `json:"author"`
}
Need to use a capitalise letter because if you don't use it you can only see in the same package.
Capitalise letter = see in all package
Normal letter = see only in same package (for example: controller only here)
Using Structs
If a field or method name starts with a capital letter, the member is exported and is accessible outside of the package.
If a field or method starts with a lowercase letter, the member is unexported and does not have accessibility outside of the package.
Note: The Inorder to do the operations like Marshalling Un-marshalling etc in golang json package you need to have field names should start with uppercase letters. Because it uses reflection inside to process.
So I am new to go and I currently try to build a little REST-API using chi (and I love it). Yesterday I run into a problem, that I cannot quite understand.
In my little test-project I have a main.go file which contains the main function with router instantiation, adding middlewares and starting the server:
func main() {
router := chi.NewRouter()
// Middleware
router.Use(middleware.RequestID)
router.Use(middleware.RealIP)
router.Use(middleware.Logger)
router.Use(middleware.Recoverer)
// Routes
router.Post("/login", users.Login)
router.Post("/register", users.Register)
router.With(users.LoginRequired).Route("/users", func(r chi.Router) {
r.Get("/{user_id}", users.GetUser)
})
// Start Server
port := ":8080"
log.Printf("Server starting at port %v\n", port)
log.Fatal(http.ListenAndServe(port, router))
}
First the problem didn't exist because I defined all the handler functions within my main.go file and the GetUser-function worked as expected and returned a user from my "Database" (array with 3 users):
func GetUser(w http.ResponseWriter, r *http.Request) {
uID := chi.URLParam(r, "user_id") // Problem when not in main -> uID = ""
id, err := strconv.Atoi(uID)
if err != nil {
log.Printf("Error while parsing int: %v\n", err)
// TODO: return error 400
}
user := DataBase[id-1]
response, err := json.Marshal(user)
if err != nil {
log.Printf("Error while marshalling user: %v\n", err)
}
w.Write(response)
}
As soon as I moved this function out of the main.go file into another package called users the chi.URLParam function returns an empty string and cannot find the URLParam anymore. I read it has something to do with the context, but I cannot wrap my head around that I have to place functions inside the main-file if I want to use the chi functions.
Am I missing something?
UPDATE
As requested I removed everything except the GetUser function. My main.go file currently looks like this:
package main
import (
"encoding/json"
"log"
"net/http"
"strconv"
"github.com/MyUserName/MyProjectName/internals/users"
"github.com/go-chi/chi/v5"
)
func GetUser(w http.ResponseWriter, r *http.Request) {
id, err := strconv.Atoi(chi.URLParam(r, "user_id"))
if err != nil {
log.Printf("Error while parsing int: %v\n", err)
// TODO: return error 400
}
log.Printf("ID=%v, Current Database=%v\n", id, users.DataBase)
user := users.DataBase[id-1]
response, err := json.Marshal(user)
if err != nil {
log.Printf("Error while marshalling user: %v\n", err)
}
w.Write(response)
}
func main() {
router := chi.NewRouter()
// Routes
router.Get("/users/{user_id}", GetUser)
// Start Server
port := ":8080"
log.Printf("Server starting at port %v\n", port)
log.Fatal(http.ListenAndServe(port, router))
}
and my users package looks like this:
package users
import (
"encoding/json"
"log"
"net/http"
"strconv"
"github.com/MyUserName/MyProjectName/internals/models"
"github.com/go-chi/chi"
)
var (
DataBase = make([]models.User, 0)
)
func GetUser(w http.ResponseWriter, r *http.Request) {
id, err := strconv.Atoi(chi.URLParam(r, "user_id"))
if err != nil {
log.Printf("Error while parsing int: %v\n", err)
// TODO: return error 400
}
log.Printf("ID=%v, Current Database=%v\n", id, DataBase)
user := DataBase[id-1]
response, err := json.Marshal(user)
if err != nil {
log.Printf("Error while marshalling user: %v\n", err)
}
w.Write(response)
}
func init() {
initUser := []models.User{
{
ID: 1,
UserName: "john",
Password: "doe",
},
{
ID: 2,
UserName: "max",
Password: "mustermann",
},
{
ID: 3,
UserName: "jane",
Password: "doe",
},
}
for _, user := range initUser {
DataBase = append(DataBase, user)
}
log.Println("Initializing Database")
}
When I use the function from the users package it does not work and is still an empty string, if I use the function from the main.go file it works.
UPDATE
So apparently I am to stupid to import the same packages twice. In my main file I used "github.com/go-chi/chi/v5" and in my users package I used "github.com/go-chi/chi". Using the same resolved the issue, thanks a lot
Adding answer because the comments just saved me!
Check that all files in your go solution have the same version of chi in use. If you're using VSCode it may import a different version than you expect. In my code I had one file with
import(
"github.com/go-chi/chi"
)
and in the other
import(
"github.com/go-chi/chi/v5"
)
This meant that when I was calling into middleware function to extract URLParams the context was not finding a value.
TL;DR
Check that all files use same version of Chi!
So I currently have a function that will take in a string APIKey to check it against my MongoDB collection. If nothing is found (not authenticated), it returns false - if a user is found, it returns true. My problem, however, is I'm unsure how to integrate this with a Gin POST route. Here is my code:
import (
"context"
"fmt"
"log"
"os"
"github.com/gin-gonic/gin"
_ "github.com/joho/godotenv/autoload"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/bson/primitive"
"go.mongodb.org/mongo-driver/mongo"
"go.mongodb.org/mongo-driver/mongo/options"
)
type User struct {
Name string
APIKey string
}
func validateAPIKey(users *mongo.Collection, APIKey string) bool {
var user User
filter := bson.D{primitive.E{Key: "APIKey", Value: APIKey}}
if err := users.FindOne(context.TODO(), filter).Decode(&user); err != nil {
fmt.Printf("Found 0 results for API Key: %s\n", APIKey)
return false
}
fmt.Printf("Found: %s\n", user.Name)
return true
}
func handleUpload(c *gin.Context) {
}
func main() {
r := gin.Default()
api := r.Group("/api")
v1 := api.Group("/v1")
v1.POST("/upload", handleUpload)
mongoURI := os.Getenv("MONGO_URI")
mongoOptions := options.Client().ApplyURI(mongoURI)
client, err := mongo.Connect(context.TODO(), mongoOptions)
if err != nil {
log.Fatal(err, "Unable to access MongoDB server, exiting...")
}
defer client.Disconnect(context.TODO())
// users := client.Database("sharex_api").Collection("authorized_users") // commented out when testing to ignore unused warnings
r.Run(":8085")
}
The validateAPIKey function works exactly as intended if tested alone, I am just unsure how I would run this function for a specific endpoint (in this case, /api/v1/upload) and pass in the users collection.
After a bit of searching, I found a resolution. I changed my validateAPIKey function to return git.HandlerFunc. Here's the code:
func validateAPIKey(users *mongo.Collection) gin.HandlerFunc {
return func(c *gin.Context) {
var user authorizedUser
APIKey := c.Request.Header.Get("X-API-Key")
filter := bson.D{primitive.E{Key: "APIKey", Value: APIKey}}
if err := users.FindOne(context.TODO(), filter).Decode(&user); err != nil {
fmt.Printf("Found 0 results for API Key: %s\n", APIKey)
c.JSON(http.StatusUnauthorized, gin.H{"status": 401, "message": "Authentication failed"})
return
}
return
}
}
For the route, I have the following:
v1.POST("/upload", validateAPIKey(users), handleUpload)
I am new to Go so hopefully I'm making myself clear with this issue I'm having. My problem is that I am trying to iterate over an array of structs but I keep running into an index out of range issue. For the purposes of this problem, I have already verified that my array is not empty but that it in fact does contain at least one Services struct and file_content is the string that contains my valid JSON
Here is the snippet of code that represents the problem I'm having:
type service_config struct {
Services []struct {
Name string
Command string
Request map[string]interface{}
}
}
var ServiceConf = service_config{}
err_json := json.Unmarshal(file_content, &ServiceConf)
for _, s := range ServiceConf.Services {
log.Println(s)
}
So every time I run my code I get:
2014/03/14 18:19:53 http: panic serving [::1]:65448: runtime error: index out of range
{
"services" : [
{
"name": "translation",
"command": "to german",
"request": {
"key": "XXX",
"url": "https://www.googleapis.com/language/translate/v2?"
}
}
]
}
If you're interested in the complete source file:
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"net/url"
"os"
)
type SlackResponse struct {
token string
team_id string
channel_id string
channel_name string
timestamp string
user_id string
user_name string
text string
}
type service_config struct {
Services []struct {
Name string
Command string
Request map[string]interface{}
}
}
var ServiceConf = service_config{}
func main() {
content, err_read := ioutil.ReadFile("config.ini")
if err_read != nil {
log.Println("Could not read config")
return
}
log.Println(string(content))
err_json := json.Unmarshal(content, &ServiceConf)
if err_json != nil {
log.Println(err_json)
}
http.HandleFunc("/", handler)
http.ListenAndServe(":"+os.Getenv("PORT"), nil)
}
func handler(w http.ResponseWriter, r *http.Request) {
slack_response := SlackResponse{
r.FormValue("token"),
r.FormValue("team_id"),
r.FormValue("channel_id"),
r.FormValue("channel_name"),
r.FormValue("timestamp"),
r.FormValue("user_id"),
r.FormValue("user_name"),
r.FormValue("text"),
}
// log.Println(ServiceConf.Services[0].Request["key"])
// loop through services to find command phrases
for _, s := range ServiceConf.Services {
log.Println(s)
}
if slack_response.user_name == "slackbot" {
return
}
// fmt.Fprintf(w, "{ \"text\": \"Master %s! You said: '%s'\" }", slack_response.user_name, slack_response.text)
content, err := getContent("https://www.googleapis.com/language/translate/v2?key=&source=en&target=de&q=" + url.QueryEscape(slack_response.text))
if err != nil {
fmt.Fprintf(w, "{ \"text\": \"Huh?!\" }")
} else {
type trans struct {
Data struct {
Translations []struct {
TranslatedText string `json:"translatedText"`
} `json:"translations"`
} `json:"data"`
}
f := trans{}
err := json.Unmarshal(content, &f)
if err != nil {
log.Println(err)
}
fmt.Fprintf(w, "{ \"text\": \"Translated to German you said: '%s'\" }", f.Data.Translations[0].TranslatedText)
}
}
// array of bytes if retrieved successfully.
func getContent(url string) ([]byte, error) {
// Build the request
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, err
}
// Send the request via a client
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
// Defer the closing of the body
defer resp.Body.Close()
// Read the content into a byte array
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
// At this point we're done - simply return the bytes
return body, nil
}
Here is the stack trace:
2014/03/21 23:21:29 http: panic serving [::1]:59508: runtime error: index out of range
goroutine 3 [running]:
net/http.func·009()
/usr/local/Cellar/go/1.2/libexec/src/pkg/net/http/server.go:1093 +0xae
runtime.panic(0x215f80, 0x4b6537)
/usr/local/Cellar/go/1.2/libexec/src/pkg/runtime/panic.c:248 +0x106
main.handler(0x5a85e8, 0xc21000f6e0, 0xc210037dd0)
/Users/et/src/go/src/github.com/etdebruin/gojacques/main.go:100 +0x81b
net/http.HandlerFunc.ServeHTTP(0x2cbc60, 0x5a85e8, 0xc21000f6e0, 0xc210037dd0)
/usr/local/Cellar/go/1.2/libexec/src/pkg/net/http/server.go:1220 +0x40
net/http.(*ServeMux).ServeHTTP(0xc21001e5d0, 0x5a85e8, 0xc21000f6e0, 0xc210037dd0)
/usr/local/Cellar/go/1.2/libexec/src/pkg/net/http/server.go:1496 +0x163
net/http.serverHandler.ServeHTTP(0xc21001f500, 0x5a85e8, 0xc21000f6e0, 0xc210037dd0)
/usr/local/Cellar/go/1.2/libexec/src/pkg/net/http/server.go:1597 +0x16e
net/http.(*conn).serve(0xc210058300)
/usr/local/Cellar/go/1.2/libexec/src/pkg/net/http/server.go:1167 +0x7b7
created by net/http.(*Server).Serve
/usr/local/Cellar/go/1.2/libexec/src/pkg/net/http/server.go:1644 +0x28b
The error comes from this line
fmt.Fprintf(w, "{ \"text\": \"Translated to German you said: '%s'\" }",
f.Data.Translations[0].TranslatedText)
So you didn't get any Translations back - that array is empty.
You might want to check resp.Status to see if an error was returned. This isn't returned as an error - you need to check it yourself.