I want to read gzip compressed file from blob storage. I know an URL of storage, offset and length on needed file. So the most simple way to download the file id to use http.client & gzip.Reader
My code is
package main
import (
"bytes"
"compress/gzip"
"fmt"
"io/ioutil"
"net/http"
"os"
)
func main() {
req, _ := http.NewRequest("GET", "https://commoncrawl.s3.amazonaws.com/crawl-data/CC-MAIN-2021-10/segments/1614178385529.97/warc/CC-MAIN-20210308205020-20210308235020-00200.warc.gz", nil)
req.Header.Add("Range", "bytes=842089698-842207846")
var client http.Client
resp, err := client.Do(req)
if err != nil {
fmt.Println(err)
os.Exit(0)
}
// for key, value := range resp.Header {
// fmt.Printf("%s: %s\n", key, value)
// }
// fmt.Printf("length: %v\n", resp.ContentLength)
option := 2
if option == 1 {
// --- Option 1 ----
reader, _ := gzip.NewReader(resp.Body)
body, err := ioutil.ReadAll(reader)
if err != nil {
fmt.Println(err)
os.Exit(0)
}
fmt.Println(string(body))
} else {
//--- Option 2 ----
body, _ := ioutil.ReadAll(resp.Body)
gr, _ := gzip.NewReader(bytes.NewBuffer(body))
body, _ = ioutil.ReadAll(gr)
fmt.Println(string(body))
}
}
If I set option variable to 2 - everything works fine, but if use option #1 programm ends with a error "unexpected EOF". Why does it happen?
Related
I am using Pinata to upload files in Go to a private gateway using its submarine feature; while it works well, I am wondering how to pin a directory instead. The documentation says it is "identically to pinning a file, with the main difference being an array of files and need to provide a relative file path for each file in the directory". How can I tweak my current Go code to work for a directory.
package ipfs
import (
"bytes"
"fmt"
"io"
"io/ioutil"
"mime/multipart"
"net/http"
"os"
"path/filepath"
)
func PinFile(filePath string, fileName string) {
//set pinata url for file pinning
url := "https://managed.mypinata.cloud/api/v1/content"
//init method and payload
method := "POST"
payload := &bytes.Buffer{}
writer := multipart.NewWriter(payload)
file, errFile1 := os.Open(filePath)
defer file.Close()
part1, errFile1 := writer.CreateFormFile("files", filepath.Base(filePath))
_, errFile1 = io.Copy(part1, file)
if errFile1 != nil {
fmt.Println(errFile1)
return
}
//set additional pinata option
_ = writer.WriteField("name", "fileName")
_ = writer.WriteField("metadata", "{\"keyvalues\": { \"app\": \"...\" }}")
_ = writer.WriteField("wrapWithDirectory", "false")
_ = writer.WriteField("pinToIPFS", "false")
//close writer if error
err := writer.Close()
if err != nil {
fmt.Println(err)
return
}
client := &http.Client{}
//set new request
req, err := http.NewRequest(method, url, payload)
if err != nil {
fmt.Println(err)
return
}
// set Submarine Key
req.Header.Add("x-api-key", env.GetEnv["SUBMARINE_KEY"])
//add content type to request header
req.Header.Set("Content-Type", writer.FormDataContentType())
res, err := client.Do(req)
if err != nil {
fmt.Println(err)
return
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
fmt.Println(err)
return
}
fmt.Println(string(body))
}
I am trying to implement a python code from the JIRA REST API examples:
https://developer.atlassian.com/cloud/jira/platform/rest/v3/api-group-jql/#api-rest-api-3-jql-parse-post
My python code (which works as expected):
import requests
from requests.auth import HTTPBasicAuth
import json
url = "https://my-url.com/rest/api/2/search"
auth = HTTPBasicAuth("user1", "pwd1")
headers = {
"Accept": "application/json",
"Content-Type": "application/json"
}
payload = json.dumps( {
"jql": "my-query-string"
}
response = requests.request("POST", url, data=payload, headers=headers, auth=auth, verify=False)
print(json.dumps(json.loads(response.text), sort_keys=True, indent=4, separators=(",", ": ")))
I'm trying to transform this to a golang code as below:
package main
import (
"io/ioutil"
"fmt"
"log"
"time"
"net/http"
"net/url"
}
func main() {
timeout := time.Duration(500 * time.Second)
client := http.Client{
Timeout: timeout,
}
req, err := http.NewRequest("POST", "https://my-url.com/rest/api/2/search", nil)
if err != nil {
log.Fatalln(err)
}
req.SetBasicAuth("user1", "pwd1")
req.Header.Set("Content-Type", "application/json")
q := url.Values{}
q.Add("jql", "my-query-string")
req.URL.RawQuery = q.Encode()
fmt.Println(req.URL.String())
resp, err := client.Do(req)
if err != nil {
log.Fatalln(err)
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatalln(err)
}
log.Println(string(data))
The code builds with no issues. When I run the go code, I get this error:
2021/04/17 19:36:31 {"errorMessages":["No content to map to Object due to end of input"]}
I have 2 questions :
a. How can I fix the above error ?
b. I also want to include concurrency in the same code, i.e the same POST request will actually be executed for 5 different query strings (concurrently) and fetch the results, how can i achieve that ?
For POST requests you need to send the data as json. Note that in Go setting a request's Content-Type header does not automagically convert whatever you give it to the specified type.
An example sending json.
package main
import (
"strings"
"net/http"
"io/ioutil"
"fmt"
)
func main() {
body := strings.NewReader(`{"jql": "project = HSP"}`)
req, err := http.NewRequest("POST", "https://your-domain.atlassian.com/rest/api/2/search", body)
if err != nil {
panic(err)
}
req.SetBasicAuth("email#example.com", "<api_token>")
req.Header.Set("Accept", "application/json")
req.Header.Set("Content-Type", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
out, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
fmt.Println(string(out))
}
If you want to use query parameters you should use the endpoint with the GET method.
package main
import (
"net/http"
"net/url"
"io/ioutil"
"fmt"
)
func main() {
query := url.Values{"jql": {"project = HSP"}}
req, err := http.NewRequest("GET", "https://your-domain.atlassian.com/rest/api/2/search?" + query.Encode(), nil)
if err != nil {
panic(err)
}
req.SetBasicAuth("email#example.com", "<api_token>")
req.Header.Set("Accept", "application/json")
resp, err := http.DefaultClient.Do(req)
if err != nil {
panic(err)
}
defer resp.Body.Close()
out, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err)
}
fmt.Println(string(out))
}
Implemented an logic in go to fetch the information from given URL,The problem is response of net/http is empty.
package main
import (
"fmt"
"io/ioutil"
"net/http"
)
func main() {
resp, err := http.Get("https://azure.microsoft.com/en-us/")
if err != nil {
// handle error
}
body, err := ioutil.ReadAll(resp.Body)
bodyString := string(body)
fmt.Print(bodyString)
fmt.Printf("%v %v", body, err)
}
Output: its returning empty slice instead of returning HTML content
[]byte{} <nil>
I'm using Go version 1.14.3.
It seems that's working when you set the User-Agent header :
package main
import (
"fmt"
"io/ioutil"
"net/http"
)
func main() {
client := &http.Client{}
req, err := http.NewRequest("GET", "https://azure.microsoft.com/en-us/", nil)
req.Header.Add("User-Agent", "Mozilla")
resp, err := client.Do(req)
if err != nil {
fmt.Println(err)
}
body, err := ioutil.ReadAll(resp.Body)
bodyString := string(body)
fmt.Print(bodyString)
}
I've just recently started trying to learn Go and I'm trying to write a small server/client application for sending a csv file, from a server to a client. I'm running into an invalid type error when trying to encode a struct into BigEndian binary. My struct seems to already be in a binary format, I'm unsure why I get the following error:
Error writing binary buffer to big endian binary.Write: invalid type *main.DataPack
Also, I'd like to keep the TCP connection open after sending the file, that's why I'm not using io.Copy.
Currently I'm triggering the handling of the file upload through by sending a '\x00' byte:
// Server
package main
import (
"path/filepath"
"fmt"
"io/ioutil"
"net"
"os"
"encoding/binary"
"bytes"
)
type DataPack struct {
Length int64
Contents []byte
}
func main() {
absPath, _ := filepath.Abs("./progs.csv")
data, _ := ioutil.ReadFile(absPath)
fmt.Println("%s",data)
tel, err := net.Listen("tcp", "0.0.0.0:23")
if err != nil {
fmt.Println(err)
return
}
for {
conn, err := tel.Accept()
if err != nil {
break
}
fmt.Println("above filehandler")
go fileHandler(conn)
}
}
func fileHandler(conn net.Conn) {
buf := make([]byte, 0)
buf = append(buf, '\x00')
conn.Write(buf)
absPath, _ := filepath.Abs("./progs.csv")
file, err := os.Open(absPath)
defer file.Close()
fi, err := file.Stat()
if err != nil {
fmt.Println("Fatal error reading file: ", err)
}
fmt.Println("This is the length of the file: ", fi.Size())
data := &DataPack{Length: fi.Size()} // , Contents: }
data.Contents = make([]byte, data.Length)
n, err := file.Read(data.Contents)
if err != nil {
fmt.Println("error reading contents into struct: ", err)
}
fmt.Println("tried to read file contents: ", n)
fmt.Println("DataPack: %+v", data)
buf1 := new(bytes.Buffer)
err = binary.Write(buf1, binary.BigEndian, &data)
if err != nil {
fmt.Println("Error writing binary buffer to big endian ", err)
}
conn.Write(buf1.Bytes())
}
Here is the client
package main
import (
"log"
"fmt"
"net"
"strings"
"strconv"
"bufio"
)
const (
host = "127.0.0.1"
port = 23
)
func main() {
addr := strings.Join([]string{host, strconv.Itoa(port)}, ":")
client := NewClient()
var err error
client.socket, err = net.Dial("tcp", addr)
if err != nil {
log.Println("error setting up socket: %s", err)
}
for {
m := bufio.NewReader(client.socket)
b, err := m.ReadByte()
if err != nil {
fmt.Println("here is the error: ", err)
}
if b == '\x00'{
fmt.Println("about to receive a file!!!")
b, _ := m.ReadByte()
fmt.Println("just got another byte ", b )
}
}
log.Printf("Over")
}
why you get the error
Visit https://golang.org/pkg/encoding/binary/#Write
Write writes the binary representation of data into w. Data must be a fixed-size value or a slice of fixed-size values, or a pointer to such data.
type DataPack struct {
Length int64
Contents []byte
}
Contents isn't a fixed-size value, so you got the invalid type error.
how to solve it
go binary
json
others
I have the following code which is suppose to download file by splitting it into multiple parts. But right now it only works on images, when I try downloading other files like tar files the output is an invalid file.
UPDATED:
Used os.WriteAt instead of os.Write and removed os.O_APPEND file mode.
package main
import (
"errors"
"flag"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"strconv"
)
var file_url string
var workers int
var filename string
func init() {
flag.StringVar(&file_url, "url", "", "URL of the file to download")
flag.StringVar(&filename, "filename", "", "Name of downloaded file")
flag.IntVar(&workers, "workers", 2, "Number of download workers")
}
func get_headers(url string) (map[string]string, error) {
headers := make(map[string]string)
resp, err := http.Head(url)
if err != nil {
return headers, err
}
if resp.StatusCode != 200 {
return headers, errors.New(resp.Status)
}
for key, val := range resp.Header {
headers[key] = val[0]
}
return headers, err
}
func download_chunk(url string, out string, start int, stop int) {
client := new(http.Client)
req, _ := http.NewRequest("GET", url, nil)
req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", start, stop))
resp, _ := client.Do(req)
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Fatalln(err)
return
}
file, err := os.OpenFile(out, os.O_WRONLY, 0600)
if err != nil {
if file, err = os.Create(out); err != nil {
log.Fatalln(err)
return
}
}
defer file.Close()
if _, err := file.WriteAt(body, int64(start)); err != nil {
log.Fatalln(err)
return
}
fmt.Println(fmt.Sprintf("Range %d-%d: %d", start, stop, resp.ContentLength))
}
func main() {
flag.Parse()
headers, err := get_headers(file_url)
if err != nil {
fmt.Println(err)
} else {
length, _ := strconv.Atoi(headers["Content-Length"])
bytes_chunk := length / workers
fmt.Println("file length: ", length)
for i := 0; i < workers; i++ {
start := i * bytes_chunk
stop := start + (bytes_chunk - 1)
go download_chunk(file_url, filename, start, stop)
}
var input string
fmt.Scanln(&input)
}
}
Basically, it just reads the length of the file, divides it with the number of workers then each file downloads using HTTP's Range header, after downloading it seeks to a position in the file where that chunk is written.
If you really ignore many errors like seen above then your code is not supposed to work reliably for any file type.
However, I guess I can see on problem in your code. I think that mixing O_APPEND and seek is probably a mistake (Seek should be ignored with this mode). I suggest to use (*os.File).WriteAt instead.
IIRC, O_APPEND forces any write to happen at the [current] end of file. However, your download_chunk function instances for file parts can be executing in unpredictable order, thus "reordering" the file parts. The result is then a corrupted file.
1.the sequence of the go routine is not sure。
eg. the execute result maybe as follows:
...
file length:20902
Range 10451-20901:10451
Range 0-10450：10451
...
so the chunks can't just append.
2.when write chunk datas must have a sys.Mutex
(my english is poor,please forget it)