I'm trying to port over something I had written in Node, the request looks like this in Node(JS):
function _initialConnection(user, pass, callback) {
let opts = {
url: config.loginURL,
headers: {
"User-Agent": "niantic"
}
};
request.get(opts, (err, resp, body) => {
if (err) return callback(err, null);
console.log(resp.headers);
let data;
try {
data = JSON.parse(body);
} catch(err) {
return callback(err, null);
}
return callback(null, user, pass, data);
});
}
function _postConnection(user, pass, data, callback) {
let opts = {
url: config.loginURL,
form: {
'lt': data.lt,
'execution': data.execution,
'_eventId': 'submit',
'username': user,
'password': pass
},
headers: {
'User-Agent': 'niantic'
}
};
request.post(opts, (err, resp, body) => {
if (err) return callback(err, null);
let parsedBody;
if (body) {
try {
parsedBody = JSON.parse(body)
if (('errors' in parsedBody) && parsedBody.errors.length > 0) {
return callback(
new Error('Error Logging In: ' + paredBody.errors[0]),
null
)
}
} catch(err) {
return callback(err, null);
}
}
console.log(resp.headers)
let ticket = resp.headers['location'].split('ticket=')[1];
callback(null, ticket);
});
}
If I console.log(resp.headers) I can see a location header.
I have tried to recreate this in Go the best way I could which I ended up with:
// Initiate HTTP Client / Cookie JAR
jar, err := cookiejar.New(nil)
if err != nil {
return "", fmt.Errorf("Failed to create new cookiejar for client")
}
newClient := &http.Client{Jar: jar, Timeout: 5 * time.Second}
// First Request
req, err := http.NewRequest("GET", loginURL, nil)
if err != nil {
return "", fmt.Errorf("Failed to authenticate with Google\n Details: \n\n Username: %s\n Password: %s\n AuthType: %s\n", details.Username, details.Password, details.AuthType)
}
req.Header.Set("User-Agent", "niantic")
resp, err := newClient.Do(req)
if err != nil {
return "", fmt.Errorf("Failed to send intial handshake: %v", err)
}
respJSON := make(map[string]string)
err = json.NewDecoder(resp.Body).Decode(&respJSON)
if err != nil {
return "", fmt.Errorf("Failed to decode JSON Body: %v", err)
}
resp.Body.Close()
// Second Request
form := url.Values{}
form.Add("lt", respJSON["lt"])
form.Add("execution", respJSON["execution"])
form.Add("_eventId", "submit")
form.Add("username", details.Username)
form.Add("password", details.Password)
req, err = http.NewRequest("POST", loginURL, strings.NewReader(form.Encode()))
if err != nil {
return "", fmt.Errorf("Failed to send second request authing with PTC: %v", err)
}
req.Header.Set("User-Agent", "niantic")
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
resp, err = newClient.Do(req)
if err != nil {
return "", fmt.Errorf("Failed to send second request authing with PTC: %v", err)
}
log.Println(resp.Location())
ticket := resp.Header.Get("Location")
if strings.Contains(ticket, "ticket") {
ticket = strings.Split(ticket, "ticket=")[1]
} else {
return "", fmt.Errorf("Failed could not get the Ticket from the second request\n")
}
resp.Body.Close()
But when I log.Println(resp.Location()) I get <nil>
I'm really not sure what the differences are here (I've tried with and without the Content-Type header but for some reason I just can NOT get the Location header that I'm looking for.
I really can't see a discrepancy between the Node request, vs the Go request but any help would be great as I have been beating my head off the wall for the last day. Thanks.
You can get that url by checking resp.Request.URL after the resp, err = newClient.Do(req). There isn't a really simple way to ignore redirects, you can do a manual call through a http.RoundTripper, or you can set a CheckRedirect function on the client, but it's not ideal.
Related
I am trying to convert a nodejs app to Go. Here I am trying to upload a file to B2. But I am getting Post "https://pod-XX.backblaze.com/b2api/v2/b2_upload_file/WERTGVWGTE/cSEREf": net/http: HTTP/1.x transport connection broken: http: ContentLength=3312 with Body length 0 . Here is my code:
// open file
file, err := os.Open(location)
if err != nil {
log.Fatal(err)
return "", err
}
defer file.Close()
// create sha1 hash of file
hash := sha1.New()
if _, err := io.Copy(hash, file); err != nil {
log.Fatal(err)
return "", err
}
sha1Sum := hex.EncodeToString(hash.Sum(nil))
// http client
client := http.Client{}
req, _ := http.NewRequest("POST", b2.UploadUrl, file)
contentLength, _ := file.Stat()
req.ContentLength = contentLength.Size() // without this, its throwing map[code:bad_request message:Missing header: Content-Length status:400]
req.Header.Set("Content-Type", "application/octet-stream")
req.Header.Set("Authorization", b2.AuthorizationToken)
req.Header.Set("X-Bz-File-Name", name)
req.Header.Set("X-Bz-Content-Sha1", sha1Sum)
res , errr := client.Do(req)
if errr != nil {
log.Fatalln(errr)
return "", errr
}
Just for reference here is the nodejs code that is perfectly working:
const file = await fs.readFile(location);
const sha1 = crypto.createHash('sha1').update(file).digest("hex");
const config = {
headers: {
'Authorization': auth_token,
'X-Bz-File-Name': final_name,
'Content-Type': 'b2/x-auto',
'X-Bz-Content-Sha1': sha1
}
};
let response = await axios.post(upload_url, file, config);
return response.data['fileId'];
You have consumed file here:
if _, err := io.Copy(hash, file); err != nil {
log.Fatal(err)
return "", err
}
Use func (*File) Seek or load file to memory.
I have the following function to create a server side HTTPOnly with gofiber framework using the v2 version "github.com/gofiber/fiber/v2"
func Signin(c *fiber.Ctx) error {
type SigninData struct {
Email string `json:"email" xml:"email" form:"email"`
Password string `json:"password" xml:"password" form:"password"`
}
data := SigninData{}
if err := c.BodyParser(&data); err != nil {
return err
}
var user models.User
findUser := database.DB.Where("email = ?", data.Email).First(&user)
if findUser == nil {
c.Status(fiber.StatusBadRequest)
return c.JSON(fiber.Map{
"message": "Account not found",
})
}
if err := user.ComparePassword(data.Password); err != nil {
c.Status(fiber.StatusBadRequest)
return c.JSON(fiber.Map{
"message": "Invalid credentials",
})
}
isSuperuser := database.DB.Where("email = ? AND is_superuser = ?", data.Email, true).First(&user).Error
var scope string
if errors.Is(isSuperuser, gorm.ErrRecordNotFound) {
scope = "user"
} else {
scope = "admin"
}
token, err := middlewares.CreateTokens(user.Email, scope)
if err != nil {
c.Status(fiber.StatusBadRequest)
return c.JSON(fiber.Map{
"message": "Could not generate session tokens",
})
}
saveErr := middlewares.RedisStoreTokens(user.Email, token)
if saveErr != nil {
c.Status(fiber.StatusBadRequest)
return c.JSON(fiber.Map{
"message": "Could not save session to redis",
})
}
tokens := map[string]string{
"access_token": token.AccessToken,
"refresh_token": token.RefreshToken,
}
cookie := fiber.Cookie{
Name: "access_token",
Value: tokens["access_token"],
Expires: time.Now().Add(time.Hour * 24),
HTTPOnly: true,
Secure: true,
}
c.Cookie(&cookie)
return c.JSON(fiber.Map{
"access_token": tokens["access_token"],
"refresh_token": tokens["refresh_token"],
"token_type": "bearer",
})
}
Here is what it returned on signin
{
"access_token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NfdXVpZCI6ImFlMmQ4MDlhLTNhZDgtNDgwNS1iMjZlLWUyYWMwNTYyMjZhZiIsImF1dGhvcml6ZWQiOnRydWUsImV4cCI6MTY0MTE4NTg5MCwicGVybWlzc2lvbiI6InVzZXIiLCJzdWIiOiJ0ZXN0OEBleGFtcGxlLmNvbSJ9.cXzkNoDb1XKmt_quQ4ONvDcXfmPrBjt4umG38a1xwqA",
"refresh_token": "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJyZWZyZXNoX3V1aWQiOiJhZTJkODA5YS0zYWQ4LTQ4MDUtYjI2ZS1lMmFjMDU2MjI2YWYrK3Rlc3Q4QGV4YW1wbGUuY29tIn0._6zOG65GmnwbWnpKaQb2LxuPIhKZGCzg9P62xoBds8U",
"token_type": "bearer"
}
cookie access_token is created with the value of eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NfdXVpZCI6ImFlMmQ4MDlhLTNhZDgtNDgwNS1iMjZlLWUyYWMwNTYyMjZhZiIsImF1dGhvcml6ZWQiOnRydWUsImV4cCI6MTY0MTE4NTg5MCwicGVybWlzc2lvbiI6InVzZXIiLCJzdWIiOiJ0ZXN0OEBleGFtcGxlLmNvbSJ9.cXzkNoDb1XKmt_quQ4ONvDcXfmPrBjt4umG38a1xwqA
and if one checks the payload data of the cookie one gets the following
{
"access_uuid": "ae2d809a-3ad8-4805-b26e-e2ac056226af",
"authorized": true,
"exp": 1641185890,
"permission": "user",
"sub": "test8#example.com"
}
So now i want another function that will pull and be able to grab all of these payload data within the cookie so i can use it within the app
Here is a function i have that is supposed to grab those data but things are not working and gofiber does not log any error so difficult to even troubleshoot
type ClaimsWithScope struct {
jwt.RegisteredClaims
Scope string `json:"permissions"`
}
type AccessDetails struct {
AccessUuid string `json:"access_uuid"`
Email string `json:"email"`
}
type AccessDetailsClaims struct {
jwt.RegisteredClaims
Scope string `json:"permissions"`
AccessUuid string `json:"access_uuid"`
Authorized string `json:"authorized"`
}
...
...
...
func GetAccessDetails(c *fiber.Ctx) (*AccessDetails, error) {
ad := &AccessDetails{}
cookie := c.Cookies("access_token")
var err error
token, err := jwt.ParseWithClaims(cookie, &AccessDetailsClaims{}, func(token *jwt.Token) (interface{}, error) {
return []byte(SecretKey), nil
})
if err != nil {
return nil, err
}
payload := token.Claims.(*AccessDetailsClaims)
ad.Email = payload.Subject
ad.AccessUuid = payload.AccessUuid
return ad, nil
}
what am i doing wrong here? ad should be able to return the full payload data like those created from the signin function like this
{
"access_uuid": "ae2d809a-3ad8-4805-b26e-e2ac056226af",
"authorized": true,
"exp": 1641185890,
"permission": "user",
"sub": "test8#example.com"
}
so i can then be able to grab whatever data i need from it
finally figured it out
func GetAccessDetails(c *fiber.Ctx) (*AccessDetails, error) {
ad := &AccessDetails{}
cookie := c.Cookies("access_token")
var err error
token, err := jwt.Parse(cookie, func(token *jwt.Token) (interface{}, error) {
return []byte(os.Getenv("ACCESS_SECRET")), nil
})
if err != nil {
return nil, err
}
payload := token.Claims.(jwt.MapClaims)
ad.Email = payload["sub"].(string)
ad.AccessUuid = payload["access_uuid"].(string)
return ad, nil
}
so since i used mapClaims to create the token, so i can grab it with this
token, err := jwt.Parse(cookie, func(token *jwt.Token) (interface{}, error) {
return []byte(os.Getenv("ACCESS_SECRET")), nil
})
and then assigns the elements of ad := &AccessDetails{} as follows
payload := token.Claims.(jwt.MapClaims)
ad.Email = payload["sub"].(string)
ad.AccessUuid = payload["access_uuid"].(string)
I'm following this official Docs: https://developers.facebook.com/docs/messenger-platform/reference/attachment-upload-api/ to try to send a message with video attachment.
What I've tried:
//...
api.BaseRoutes.Conversation.Handle("/video", api.ApiSessionRequired(sendVideo)).Methods("POST")
//...
func sendVideo(c *Context, w http.ResponseWriter, r *http.Request) {
c.RequirePageId().RequireConversationId()
if c.Err != nil {
return
}
conversation, getErr := c.App.GetConversation(c.Params.ConversationId)
if getErr != nil {
c.Err = getErr
return
}
if err := r.ParseMultipartForm(25*1024*1024); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
// detect file header
f, _, err := r.FormFile("files")
// Create a buffer to store the header of the file in
fileHeader := make([]byte, 512)
// Copy the headers into the FileHeader buffer
if _, err := f.Read(fileHeader); err != nil {
}
// set position back to start.
if _, err := f.Seek(0, 0); err != nil {
}
_fileType := http.DetectContentType(fileHeader)
m := r.MultipartForm
fileArray, ok := m.File["files"]
if !ok {
c.Err = model.NewAppError("uploadPlugin", "api.plugin.upload.no_file.app_error", nil, "", http.StatusBadRequest)
return
}
if len(fileArray) <= 0 {
c.Err = model.NewAppError("uploadPlugin", "api.plugin.upload.array.app_error", nil, "", http.StatusBadRequest)
return
}
file, err := fileArray[0].Open()
if err != nil {
c.Err = model.NewAppError("uploadPlugin", "api.plugin.upload.file.app_error", nil, "", http.StatusBadRequest)
return
}
defer file.Close()
// build a form body
body := &bytes.Buffer{}
writer := multipart.NewWriter(body)
// add form fields
writer.WriteField("message", "{\"attachment\":{\"type\":\"video\", \"payload\":{\"is_reusable\":true}}}\")
//fileWriter, err := CreateFormFile(writer, "filedata", fileArray[0].Filename)
// add a form file to the body
fileWriter, err := writer.CreateFormFile("filedata", fileArray[0].Filename)
if err != nil {
c.Err = model.NewAppError("upload_video", "upload_video.error", nil, "", http.StatusBadRequest)
return
}
// copy the file into the fileWriter
_, err = io.Copy(fileWriter, file)
if err != nil {
c.Err = model.NewAppError("upload_video", "upload_video.error", nil, "", http.StatusBadRequest)
return
}
// Close the body writer
writer.Close()
reqUrl := "https://graph.facebook.com/v10.0/me/message_attachments"
token := c.App.Session().GetPageToken(c.Params.PageId)
reqUrl += "?access_token=" + token
var netTransport = &http.Transport{
Dial: (&net.Dialer{
Timeout: 120 * time.Second,
}).Dial,
TLSHandshakeTimeout: 120 * time.Second,
ResponseHeaderTimeout: 120 * time.Second, // This will fixed the i/o timeout error
}
client := &http.Client{
Timeout: time.Second * 120,
Transport: netTransport,
}
req, _ := http.NewRequest("POST", reqUrl, body)
req.Header.Set("Authorization", "Bearer "+token)
req.Header.Set("Content-Type", writer.FormDataContentType())
resp, err1 := client.Do(req)
if err1 != nil {
c.Err = model.NewAppError("send_video", err1.Error(), nil, "", http.StatusBadRequest)
return
} else {
defer resp.Body.Close()
var bodyBytes []byte
bodyBytes, _ = ioutil.ReadAll(resp.Body)
resp.Body = ioutil.NopCloser(bytes.NewBuffer(bodyBytes))
if resp.StatusCode != http.StatusOK {
fbErr := facebookgraph.FacebookErrorFromJson(resp.Body)
fmt.Println("__ERROR___", fbErr)
c.Err = model.NewAppErrorFromFacebookError("send_video", fbErr)
return
}
// Do what ever we want with attachment_id result
}
ReturnStatusOK(w)
}
But always failed with error from Facebook:
{
"error": {
"message": "(#100) Upload attachment failure.",
"type": "OAuthException",
"code": 100,
"error_subcode": 2018047,
"fbtrace_id": "A2hkvhTQlmA98XmcrPvSy8O"
}
}
The error subcode is: 2018047 according to Facebook docs:
Upload attachment failure. A common way to trigger this error is that the provided media type does not match type of file provided int the URL
I also try via cURL and everything's OK:
curl \
-F 'message={"attachment":{"type":"video", "payload":{"is_reusable":true}}}' \
-F 'filedata=#/home/cong/Downloads/123.mp4;type=video/mp4' \
"https://graph.facebook.com/v10.0/me/message_attachments?access_token=EAAUxUcj3C64BADxxsm70hZCXTMO0eQHmSp..."
{"attachment_id":"382840319882695"}%
If I change "video" to "file", upload is success:
writer.WriteField("message", "{\"attachment\":{\"type\":\"file\", \"payload\":{\"is_reusable\":true}}}\")
But Facebook will send video as "file attachment" (can not view as video, must download to view). That's not what I want.
Can any one tell me how to fix this problem?
Thank you very much!
You shouldnt create write like this
fileWriter, err := writer.CreateFormFile("filedata", fileArray[0].Filename)
Because it will be created with header "Content-Type": "application/octet-stream" and facebook will send error file type. Replace this line like this:
h := make(textproto.MIMEHeader)
h.Set("Content-Disposition", fmt.Sprintf(`form-data; name="%s"; filename="%s"`,"filedata", fileArray[0].Filename)))
h.Set("Content-Type", "video/mp4")
fileWriter, err := writer.CreatePart(h)
Use CreatePart(header) and try again
I am using colly framework for scraping the website. Am trying to login the Evernote account for scraping some things. But I can't go through it. I used "username" and "password" titles for giving the credentials. Is this the right way ?.
Thank you in advance.
package main
import (
"log"
"github.com/gocolly/colly"
)
func main() {
// create a new collector
c := colly.NewCollector()
// authenticate
err := c.Post("https://www.evernote.com/Login.action",
map[string]string{"username":
"XXXXXX#XXX.com", "password": "*********"})
if err != nil {
log.Fatal("Error : ",err)
}
// attach callbacks after login
c.OnResponse(func(r *colly.Response) {
log.Println("response received", r.StatusCode)
})
// start scraping
c.Visit("https://www.evernote.com/")
}
You should try to mimic the browser behavior, take a look at this implementation, I've added comments on each step:
package evernote
import (
"bytes"
"errors"
"fmt"
"io/ioutil"
"net/http"
"net/http/cookiejar"
"net/url"
"regexp"
"strings"
)
const (
evernoteLoginURL = "https://www.evernote.com/Login.action"
)
var (
evernoteJSParamsExpr = regexp.MustCompile(`document.getElementById\("(.*)"\).value = "(.*)"`)
evernoteRedirectExpr = regexp.MustCompile(`Redirecting to <a href="(.*)">`)
errNoMatches = errors.New("No matches")
errRedirectURL = errors.New("Redirect URL not found")
)
// EvernoteClient wraps all methods required to interact with the website.
type EvernoteClient struct {
Username string
Password string
httpClient *http.Client
// These parameters persist during the login process:
hpts string
hptsh string
}
// NewEvernoteClient initializes a new Evernote client.
func NewEvernoteClient(username, password string) *EvernoteClient {
// Allocate a new cookie jar to mimic the browser behavior:
cookieJar, _ := cookiejar.New(nil)
// Fill up basic data:
c := &EvernoteClient{
Username: username,
Password: password,
}
// When initializing the http.Client, copy default values from http.DefaultClient
// Pass a pointer to the cookie jar that was created earlier:
c.httpClient = &http.Client{
Transport: http.DefaultTransport,
CheckRedirect: http.DefaultClient.CheckRedirect,
Jar: cookieJar,
Timeout: http.DefaultClient.Timeout,
}
return c
}
func (e *EvernoteClient) extractJSParams(body []byte) (err error) {
matches := evernoteJSParamsExpr.FindAllSubmatch(body, -1)
if len(matches) == 0 {
return errNoMatches
}
for _, submatches := range matches {
if len(submatches) < 3 {
err = errNoMatches
break
}
key := submatches[1]
val := submatches[2]
if bytes.Compare(key, hptsKey) == 0 {
e.hpts = string(val)
}
if bytes.Compare(key, hptshKey) == 0 {
e.hptsh = string(val)
}
}
return nil
}
// Login handles the login action.
func (e *EvernoteClient) Login() error {
// First step: fetch the login page as a browser visitor would do:
res, err := e.httpClient.Get(evernoteLoginURL)
if err != nil {
return err
}
if res.Body == nil {
return errors.New("No response body")
}
body, err := ioutil.ReadAll(res.Body)
if err != nil {
return err
}
err = e.extractJSParams(body)
if err != nil {
return err
}
// Second step: we have extracted the "hpts" and "hptsh" parameters
// We send a request using only the username and setting "evaluateUsername":
values := &url.Values{}
values.Set("username", e.Username)
values.Set("evaluateUsername", "")
values.Set("analyticsLoginOrigin", "login_action")
values.Set("clipperFlow", "false")
values.Set("showSwitchService", "true")
values.Set("hpts", e.hpts)
values.Set("hptsh", e.hptsh)
rawValues := values.Encode()
req, err := http.NewRequest(http.MethodPost, evernoteLoginURL, bytes.NewBufferString(rawValues))
if err != nil {
return err
}
req.Header.Set("Accept", "application/json")
req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
req.Header.Set("x-requested-with", "XMLHttpRequest")
req.Header.Set("referer", evernoteLoginURL)
res, err = e.httpClient.Do(req)
if err != nil {
return err
}
body, err = ioutil.ReadAll(res.Body)
if err != nil {
return err
}
bodyStr := string(body)
if !strings.Contains(bodyStr, `"usePasswordAuth":true`) {
return errors.New("Password auth not enabled")
}
// Third step: do the final request, append password to form data:
values.Del("evaluateUsername")
values.Set("password", e.Password)
values.Set("login", "Sign in")
rawValues = values.Encode()
req, err = http.NewRequest(http.MethodPost, evernoteLoginURL, bytes.NewBufferString(rawValues))
if err != nil {
return err
}
req.Header.Set("Accept", "text/html")
req.Header.Set("Content-Type", "application/x-www-form-urlencoded; charset=UTF-8")
req.Header.Set("x-requested-with", "XMLHttpRequest")
req.Header.Set("referer", evernoteLoginURL)
res, err = e.httpClient.Do(req)
if err != nil {
return err
}
// Check the body in order to find the redirect URL:
body, err = ioutil.ReadAll(res.Body)
if err != nil {
return err
}
bodyStr = string(body)
matches := evernoteRedirectExpr.FindAllStringSubmatch(bodyStr, -1)
if len(matches) == 0 {
return errRedirectURL
}
m := matches[0]
if len(m) < 2 {
return errRedirectURL
}
redirectURL := m[1]
fmt.Println("Login is ok, redirect URL:", redirectURL)
return nil
}
After you successfully get the redirect URL, you should be able to send authenticated requests as long as you keep using the HTTP client that was used for the login process, the cookie jar plays a very important role here.
To call this code use:
func main() {
evernoteClient := NewEvernoteClient("user#company", "password")
err := evernoteClient.Login()
if err != nil {
panic(err)
}
}
These days I was working on send message via websoket,using Beego framework.
but meet the wrong message http: multiple response.WriteHeader calls
Where is the problem?
Any tips would be great!
func (this *WsController) Get() {
fmt.Println("connected")
handler(this.Ctx.ResponseWriter, this.Ctx.Request, this);
conn, err := upgrader.Upgrade(this.Ctx.ResponseWriter, this.Ctx.Request, nil)
if _, ok := err.(websocket.HandshakeError); ok {
http.Error(this.Ctx.ResponseWriter, "Not a websocket handshake", 400)
return
} else if err != nil {
return
}
fmt.Println("connected")
connection := consumer.New(beego.AppConfig.String("LoggregatorAddress"), &tls.Config{InsecureSkipVerify: true}, nil)
fmt.Println("===== Tailing messages")
msgChan, err := connection.Tail(this.Ctx.Input.Param(":appGuid"), this.Ctx.Input.Param(":token"))
if err != nil {
fmt.Printf("===== Error tailing: %v\n", err)
} else {
for msg := range msgChan {
// if closeRealTimeLogFlag{
// consumer.Close()
// break
// }
if err = conn.WriteMessage(websocket.TextMessage, msg.Message); err != nil {
fmt.Println(err)
}
fmt.Printf("%v \n", msg)
}
}
}
because you write more than statusCode