I have []byte of zip file. I have to unzip it without creating a new file, and get a []byte of that unzipped file. Please help me to do that.
I am making an API call and the response I get is the []byte in zipped format - I am trying to unzip it - and use it's content for creating a new zip file. So unzip - rezip.
Language: Golang
Code I've used:
func UnzipBytes(zippedBytes []byte) ([]byte, error) {
reader := bytes.NewReader(zippedBytes)
zipReader, err := zlib.NewReader(reader)
if err != nil {
return nil, err
}
defer zipReader.Close()
p, err := ioutil.ReadAll(zipReader)
if err != nil {
return nil, err
}
return p, nil
}
I get an error saying "zlib: invalid header"
The code that was initially used to zip the []byte
buffer := new(bytes.Buffer)
zipWriter := zip.NewWriter(buffer)
zipFile, err := zipWriter.Create(file.name)
_, err = zipFile.Write(file.content)
Hex dump of the []byte - the zippedBytes
00059350 78 b4 5b 0d 2b 81 c2 87 35 76 1b 11 4a ec 07 d1 |x.[.+...5v..J...|
00059360 76 77 a2 e1 3b d9 12 e2 51 d4 c5 bd 4b 2f 09 da |vw..;...Q...K/..|
00059370 f7 21 c7 26 73 1f 8e da f0 ff a3 52 f6 e2 00 e6 |.!.&s......R....|
You used zip.Writer to compress the data. You must close it by calling its Writer.Close() method. And you must use zip.Reader to read it, and use Reader.Open() with the same name you used when compressed it (file.name).
This is how it could look like:
func UnzipBytes(name string, zippedBytes []byte) ([]byte, error) {
reader := bytes.NewReader(zippedBytes)
zipReader, err := zip.NewReader(reader, int64(len(zippedBytes)))
if err != nil {
return nil, err
}
f, err := zipReader.Open(name)
if err != nil {
panic(err)
}
p, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
return p, nil
}
Testing it:
filename := "test.txt"
filecontent := []byte("line1\nline2")
buffer := new(bytes.Buffer)
zipWriter := zip.NewWriter(buffer)
zipFile, err := zipWriter.Create(filename)
if err != nil {
panic(err)
}
if _, err = zipFile.Write(filecontent); err != nil {
panic(err)
}
if err = zipWriter.Close(); err != nil {
panic(err)
}
decoded, err := UnzipBytes(filename, buffer.Bytes())
fmt.Println(err)
fmt.Println(string(decoded))
This will output (try it on the Go Playground):
<nil>
line1
line2
If you don't know the name when decompressing, you may see all files in the Reader.Files header field. You may choose to open the first file:
func UnzipBytes(zippedBytes []byte) ([]byte, error) {
reader := bytes.NewReader(zippedBytes)
zipReader, err := zip.NewReader(reader, int64(len(zippedBytes)))
if err != nil {
return nil, err
}
if len(zipReader.File) == 0 {
return nil, nil // No file to open / extract
}
f, err := zipReader.File[0].Open()
if err != nil {
panic(err)
}
p, err := ioutil.ReadAll(f)
if err != nil {
return nil, err
}
return p, nil
}
This outputs the same. Try this one on the Go Playground.
Related
I am trying to read in an mrt (with .bz2 file extension) from archive.routeviews.org namely file - http://archive.routeviews.org/route-views.chile/bgpdata/2022.05/UPDATES/updates.20220501.0000.bz2.
I have found some code online that parses it using three different packages - FGBGP, go-mrt, goBGP. Here is the code:
package main
import (
"bufio"
"bytes"
"fmt"
"log"
"os"
mrt1 "github.com/cloudflare/fgbgp/mrt"
mrt2 "github.com/kaorimatz/go-mrt"
mrt3 "github.com/osrg/gobgp/pkg/packet/mrt"
)
func main() {
data, err := os.ReadFile("updates.20220501.0000")
if err != nil {
log.Fatal(err)
}
// or paste bytes instead data := []byte{}
fmt.Println("FGBGP")
rdr := bytes.NewBuffer(data)
r, err := mrt1.DecodeSingle(rdr)
for r != nil && err == nil {
fmt.Println(r)
r, err = mrt1.DecodeSingle(rdr)
}
fmt.Println("Go-mrt")
rdr2 := mrt2.NewReader(bytes.NewBuffer(data))
r2, err := rdr2.Next()
for r2 != nil && err == nil {
fmt.Println(r2)
r2, err = rdr2.Next()
}
fmt.Println("GoBGP")
sc := bufio.NewScanner(bytes.NewBuffer(data))
sc.Split(mrt3.SplitMrt)
for {
b := sc.Scan()
if !b {
break
}
mrtb := sc.Bytes()
hdr, err := mrt3.NewMRTHeader(0, mrt3.BGP4MP, mrt3.RIB_IPV4_UNICAST, 0)
if err != nil {
fmt.Println(err)
break
}
hdr.DecodeFromBytes(mrtb)
r3, err := mrt3.ParseMRTBody(hdr, mrtb[mrt3.MRT_COMMON_HEADER_LEN:])
if err != nil {
fmt.Println(err)
}
fmt.Println(r3)
}
}
When this is run, FGBGP does not input anything, go-mrt does seem to output the lines but each line seems to be missing things I would like to see such as AS PATH etc. And goBGP which was the one I was most interested in, throws an unsupported type error.
Here is an example output for go-mrt and goBGP:
go-mrt: &{{{0 63786960781 <nil>} 17 4} 27678 6447 0 1 200.16.114.34 200.16.114.60 0x140003f5900}
goBGP: <nil> unsupported type: 17
I'm trying to download and decrypt HLS streams by using io.ReadFull to process the data in chunks to conserve memory:
Irrelevant parts of code has been left out for simplicity.
func main() {
f, _ := os.Create(out.ts)
for _, v := range mediaPlaylist {
resp, _ := http.Get(v.URI)
for {
r, err := decryptHLS(key, iv, resp.Body)
if err != nil && err == io.EOF {
break
else if err != nil && err != io.ErrUnexpectedEOF {
panic(err)
}
io.Copy(f, r)
}
}
}
func decryptHLS(key []byte, iv []byte, r io.Reader) (io.Reader, error) {
block, _ := aes.NewCipher(key)
buf := make([]byte, 8192)
mode := cipher.NewCBCDecrypter(block, iv)
n, err := io.ReadFull(r, buf)
if err != nil && err != io.ErrUnexpectedEOF {
return nil, err
}
mode.CryptBlocks(buf, buf)
return bytes.NewReader(buf[:n]), err
}
At first this seems to work as file size is correct and no errors during download,
but the video is corrupted. Not completely as the file is still recognized as a video, but image and sound is distorted.
If I change the code to use ioutil.ReadAll instead, the final video files will no longer be corrupted:
func main() {
f, _ := os.Create(out.ts)
for _, v := range mediaPlaylist {
resp, _ := http.Get(v.URI)
segment, _ := ioutil.ReadAll(resp.Body)
r, _ := decryptHLS(key, iv, &segment)
io.Copy(f, r)
}
}
func decryptHLS(key []byte, iv []byte, s *[]byte) io.Reader {
block, _ := aes.NewCipher(key)
mode := cipher.NewCBCDecrypter(block, iv)
mode.CryptBlocks(*s, *s)
return bytes.NewReader(*s)
}
Any ideas why it works correctly when reading the entire segment into memory, and not when using io.ReadFull and processing it in chunks?
Internally, CBCDecrypter makes a copy of your iv, so subsequent blocks start with the initial IV rather than the one that's been mutated by previous decryptions.
Create the decrypter once, and you should be able to keep re-using it to decrypt block by block (assuming the block size is a multiple of the block size expected by this crypto algorithm).
I have a empty file called a.txt, I want to output a value(int) to it in a loop, and overwrite last content in file a.txt. For example,
// open a file
f, err := os.Open("test.txt")
if err != nil {
log.Fatal(err)
}
defer f.Close()
// another file
af, err := os.OpenFile("a.txt", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
log.Fatal(err)
}
defer af.Close()
b := []byte{}
scanner := bufio.NewScanner(f)
for scanner.Scan() {
b = append(b, scanner.Bytes()...)
// how to output len(b) into a.txt?
}
You can also try:
os.OpenFile with custom flags to truncate file, as shown below
package main
import (
"log"
"os"
)
func main() {
f, err := os.OpenFile("notes.txt", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
if err != nil {
log.Fatal(err)
}
if err := f.Close(); err != nil {
log.Fatal(err)
}
}
Just use the truncate method and write again to file starting at the begining.
err = f.Truncate(0)
_, err = f.Seek(0, 0)
_, err = fmt.Fprintf(f, "%d", len(b))
Use os.Create() instead:
f, err := os.Create("test.txt")
From the func's doc:
Create creates or truncates the named file. If the file already exists, it is truncated. If the file does not exist, it is created ...
i am trying to call a shell program using golang (os/exec) but the output i am getting is in bytes and i need to convert it into float64 but it is showing error?
error: cannot convert out (type []byte) to type float64
func Cpu_usage_data() (cpu_predict float64, err error) {
out,err1 := exec.Command("/bin/sh","data_cpu.sh").Output()
if err1 != nil {
fmt.Println(err1.Error())
}
return float64(out), err1
}
data_cpu.sh is:
top -b n 1 | egrep -w 'apache2|mysqld|php' | awk '{cpu += $9}END{print cpu/NR}'
Use bytes.Buffer and strconv.ParseFloat.
func Cpu_usage_data() (cpu_predict float64, err error) {
cmd := exec.Command("/bin/sh", "data_cpu.sh")
var out bytes.Buffer
cmd.Stdout = &out
err = cmd.Run()
if err != nil {
fmt.Println(err.Error())
}
cpu_predict, err = strconv.ParseFloat(out.String(), 64)
if err != nil {
fmt.Println(err.Error())
}
return
}
i am trying to write to to a file. i read the whole content of the file and now i want to change the content of the file based on some word that i have got from the file. but when i check, the content of the file, it is still the same and it has not change. this is what i used
if strings.Contains(string(read), sam) {
fmt.Println("this file contain that word")
temp := strings.ToUpper(sam)
fmt.Println(temp)
err := ioutil.WriteFile(fi.Name(), []byte(temp), 0644)
} else {
fmt.Println(" the word is not in the file")
}
Considering that your call to ioutil.WriteFile() is consistent with what is used in "Go by Example: Writing Files", this should work.
But that Go by example article check the err just after the write call.
You check the err outside the scope of your test:
if matched {
read, err := ioutil.ReadFile(path)
//fmt.Println(string(read))
fmt.Println(" This is the name of the file", fi.Name())
if strings.Contains(string(read), sam) {
fmt.Println("this file contain that word")
Value := strings.ToUpper(sam)
fmt.Println(Value)
err = ioutil.WriteFile(fi.Name(), []byte(Value), 0644)
} else {
fmt.Println(" the word is not in the file")
}
check(err) <===== too late
}
The err you are testing is the one you got when reading the file (ioutil.ReadFile), because of blocks and scope.
You need to check the error right after the Write call
err = ioutil.WriteFile(fi.Name(), []byte(Value), 0644)
check(err) <===== too late
Since WriteFile overwrite the all file, you could strings.Replace() to replace your word by its upper case equivalent:
r := string(read)
r = strings.Replace(r, sam, strings.ToUpper(sam), -1)
err := ioutil.WriteFile(fi.Name(), []byte(r), 0644)
For a replace which is case insensitive, use a regexp as in "How do I do a case insensitive regular expression in Go?".
The, use func (*Regexp) ReplaceAllString:
re := regexp.MustCompile("(?i)\\b"+sam+"\\b")
r = re.ReplaceAllString(r, strings.ToUpper(sam))
err := ioutil.WriteFile(fi.Name(), []byte(r), 0644)
Note the \b: word boundary to find the any word starting and ending with sam content (instead of finding substrings containing sam content).
If you want to replace substrings, simply drop the \b:
re := regexp.MustCompile("(?i)"+sam)
It's not clear what you want to do. My best guess is something like this:
package main
import (
"bytes"
"errors"
"fmt"
"io/ioutil"
"os"
)
func UpdateWord(filename string, data, word []byte) (int, error) {
n := 0
f, err := os.OpenFile(filename, os.O_WRONLY, 0644)
if err != nil {
return n, err
}
uWord := bytes.ToUpper(word)
if len(word) < len(uWord) {
err := errors.New("Upper case longer than lower case:" + string(word))
return n, err
}
if len(word) > len(uWord) {
uWord = append(uWord, bytes.Repeat([]byte{' '}, len(word))...)[:len(word)]
}
off := int64(0)
for {
i := bytes.Index(data[off:], word)
if i < 0 {
break
}
off += int64(i)
_, err = f.WriteAt(uWord, off)
if err != nil {
return n, err
}
n++
off += int64(len(word))
}
f.Close()
if err != nil {
return n, err
}
return n, nil
}
func main() {
// Test file
filename := `ltoucase.txt`
// Create test file
lcase := []byte(`update a bc def ghij update klmno pqrstu update vwxyz update`)
perm := os.FileMode(0644)
err := ioutil.WriteFile(filename, lcase, perm)
if err != nil {
fmt.Println(err)
return
}
// Read test file
data, err := ioutil.ReadFile(filename)
if err != nil {
fmt.Println(err)
return
}
fmt.Println(string(data))
// Update word in test file
word := []byte("update")
n, err := UpdateWord(filename, data, word)
if err != nil {
fmt.Println(n, err)
return
}
fmt.Println(filename, string(word), n)
data, err = ioutil.ReadFile(filename)
if err != nil {
fmt.Println(err)
return
}
fmt.Println(string(data))
}
Output:
update a bc def ghij update klmno pqrstu update vwxyz update
ltoucase.txt update 4
UPDATE a bc def ghij UPDATE klmno pqrstu UPDATE vwxyz UPDATE