Extracting a tar file from a .tgz file - go

i have some issue with the package archive/tar.
Using the tarReader.Next() he recognize a file that should be considered as a file as a directory. The issue with that is the fact that it's a .tar file in it and i need it as a file to process it as well.
The fragment of code that recognize the file
header, err := tarReader.Next()
fmt.Println(header.Typeflag)
if err == io.EOF {
break
}
if err != nil {
fmt.Println(err)
os.Exit(1)
}
name := header.Name
fmt.Println(name)
Files:
file.tgz
-file.tar
--other files
Here is the full function
package main
import (
"archive/tar"
"compress/gzip"
"flag"
"fmt"
"io"
"io/ioutil"
"os"
)
func main() {
// get the arguments from the command line
flag.Int("n", 4, "an integer")
flag.Parse()
sourceFile := flag.Arg(0)
if sourceFile == "" {
fmt.Println("Dude, you didn't pass in a tar file!")
os.Exit(1)
}
fmt.Println("arg 1: ", flag.Arg(0))
processFile(sourceFile)
}
// ReadPackedFile is a function to unpack a tar.gz
func ReadPackedFile(filepath string) {
if filepath == "" {
panic("Empty input!")
}
processFile(filepath)
}
func processFile(srcFile string) {
f, err := os.Open(srcFile)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
defer f.Close()
gzf, err := gzip.NewReader(f)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
tarReader := tar.NewReader(gzf)
// defer io.Copy(os.Stdout, tarReader)
for true {
header, err := tarReader.Next()
fmt.Println(header.Typeflag)
if err == io.EOF {
break
}
if err != nil {
fmt.Println(err)
os.Exit(1)
}
name := header.Name
fmt.Println(name)
switch header.Typeflag {
case tar.TypeDir: // = directory
fmt.Println("Directory:", name)
os.Mkdir(name, 0755)
case tar.TypeReg: // = regular file
fmt.Println("Regular file:", name)
data := make([]byte, header.Size)
_, err := tarReader.Read(data)
if err != nil {
panic("Error reading file!!! PANIC!!!!!!")
}
ioutil.WriteFile(name, data, 0755)
default:
fmt.Printf("%s : %c %s %s\n",
"Yikes! Unable to figure out type",
header.Typeflag,
"in file",
name,
)
}
}
}
The error given by the console
PS C:\Users\ErikT\Workspace\Misc\pen\htmltag> go run .\main.go .\dragontail-12.14.1.tgz
arg 1: .\dragontail-12.14.1.tgz
53
/
Directory: /
53
12.14.1/
Directory: 12.14.1/
53
12.14.1/css/
Directory: 12.14.1/css/
48
12.14.1/css/view.css
Regular file: 12.14.1/css/view.css
panic: Error reading file!!! PANIC!!!!!!
goroutine 1 [running]:
main.processFile({0xc000014108?, 0xc000006018?})
C:/Users/ErikT/Workspace/Misc/pen/htmltag/main.go:83 +0x60c
main.main()
C:/Users/ErikT/Workspace/Misc/pen/htmltag/main.go:27 +0x1a7
exit status 2

Related

How to work with tar command using golang exec.Cmd

I have a tar.gz file and i need to unpack it using golang.
I've tried libs like "archive/tar" but they gave me error: archive/tar: invalid tar header. Now my idea was to use exec to run tar command and unpack tarball, but it always exits with code 2.
My code:
func unpack(tarName string) error {
path, _ := os.Getwd()
//err := Untar(path+"/"+tarName, path+"/")
fmt.Printf(path + "/" + tarName)
cmd := exec.Command("tar", "-xfv", path+"/"+tarName)
cmd.Stdout = os.Stdout
err := cmd.Run()
return err
}
If you are trying to compress tar.gz file you need first to decompress gzip.
import (
"archive/tar"
"compress/gzip"
"fmt"
"io"
"log"
"os"
)
func ExtractTarGz(gzipStream io.Reader) {
uncompressedStream, err := gzip.NewReader(gzipStream)
if err != nil {
log.Fatal("ExtractTarGz: NewReader failed")
}
tarReader := tar.NewReader(uncompressedStream)
for true {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
log.Fatalf("ExtractTarGz: Next() failed: %s", err.Error())
}
switch header.Typeflag {
case tar.TypeDir:
if err := os.Mkdir(header.Name, 0755); err != nil {
log.Fatalf("ExtractTarGz: Mkdir() failed: %s", err.Error())
}
case tar.TypeReg:
outFile, err := os.Create(header.Name)
if err != nil {
log.Fatalf("ExtractTarGz: Create() failed: %s", err.Error())
}
if _, err := io.Copy(outFile, tarReader); err != nil {
log.Fatalf("ExtractTarGz: Copy() failed: %s", err.Error())
}
outFile.Close()
default:
log.Fatalf(
"ExtractTarGz: uknown type: %s in %s",
header.Typeflag,
header.Name)
}
}
}
func main() {
r, err := os.Open("./file.tar.gz")
if err != nil {
fmt.Println("error")
}
ExtractTarGz(r)
}

How to compress a file to .zip without directory folder in Go

There're examples about compressing a file to .zip in Go. However, the file they generate include the directory folder. When I decompress the .zip file, there will be a new folder.
So, how can I compress a file to .zip without getting the directory folder included?
An example:
https://golangcode.com/create-zip-files-in-go/
package main
import (
"archive/zip"
"fmt"
"io"
"os"
)
func main() {
// List of Files to Zip
files := []string{"example.csv", "data.csv"}
output := "done.zip"
if err := ZipFiles(output, files); err != nil {
panic(err)
}
fmt.Println("Zipped File:", output)
}
// ZipFiles compresses one or many files into a single zip archive file.
// Param 1: filename is the output zip file's name.
// Param 2: files is a list of files to add to the zip.
func ZipFiles(filename string, files []string) error {
newZipFile, err := os.Create(filename)
if err != nil {
return err
}
defer newZipFile.Close()
zipWriter := zip.NewWriter(newZipFile)
defer zipWriter.Close()
// Add files to zip
for _, file := range files {
if err = AddFileToZip(zipWriter, file); err != nil {
return err
}
}
return nil
}
func AddFileToZip(zipWriter *zip.Writer, filename string) error {
fileToZip, err := os.Open(filename)
if err != nil {
return err
}
defer fileToZip.Close()
// Get the file information
info, err := fileToZip.Stat()
if err != nil {
return err
}
header, err := zip.FileInfoHeader(info)
if err != nil {
return err
}
// Using FileInfoHeader() above only uses the basename of the file. If we want
// to preserve the folder structure we can overwrite this with the full path.
header.Name = filename
// Change to deflate to gain better compression
// see http://golang.org/pkg/archive/zip/#pkg-constants
header.Method = zip.Deflate
writer, err := zipWriter.CreateHeader(header)
if err != nil {
return err
}
_, err = io.Copy(writer, fileToZip)
return err
}
Just use a base name of the file in the zip header.
header.Name = filepath.Base(filename)
^^^^^^^^^^^^^^
Here is a version that does the same thing
package main
import (
"archive/zip"
"io"
"log"
"os"
"path/filepath"
)
func createFlatZip(w io.Writer, files ...string) error {
z := zip.NewWriter(w)
for _, file := range files {
src, err := os.Open(file)
if err != nil {
return err
}
info, err := src.Stat()
if err != nil {
return err
}
hdr, err := zip.FileInfoHeader(info)
if err != nil {
return err
}
hdr.Name = filepath.Base(file) // Write only the base name in the header
dst, err := z.CreateHeader(hdr)
if err != nil {
return err
}
_, err = io.Copy(dst, src)
if err != nil {
return err
}
src.Close()
}
return z.Close()
}
func main() {
if len(os.Args) < 3 {
log.Fatalf("archive name and at least one file are required")
}
a, err := os.Create(os.Args[1])
if err != nil {
log.Fatal(err)
}
defer a.Close()
err = createFlatZip(a, os.Args[2:]...)
if err != nil {
log.Fatal(err)
}
}
Results:
~/src/gozip
➜ go build
~/src/gozip
➜ mkdir test && echo 1 > test/1.txt # create a test file in a subfolder
~/src/gozip
➜ ./gozip 1.zip test/1.txt
~/src/gozip
➜ unzip -l 1.zip
Archive: 1.zip
Length Date Time Name
--------- ---------- ----- ----
2 08-15-2019 01:29 1.txt
--------- -------
2 1 file

zip a folder of files with golang creates a broken zip

how do I zip files in a folder with sub-directories correctly.
I have a local folder with following structure:
folder/hello/
folder/hello/world/
folder/hello/world/helloword.txt
folder/index.txt
this is my code:
package main
import (
"archive/zip"
"fmt"
"io"
"os"
"path/filepath"
)
func main() {
files, err := listFiles("./folder")
if err != nil {
panic(err)
}
zipMe(files, "test.zip")
for _, f := range files {
fmt.Println(f)
}
fmt.Println("Done!")
}
func listFiles(root string) ([]string, error) {
var files []string
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
files = append(files, path)
return nil
})
if err != nil {
return nil, err
}
return files, nil
}
func zipMe(filepaths []string, target string) error {
flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC
file, err := os.OpenFile(target, flags, 0644)
if err != nil {
return fmt.Errorf("Failed to open zip for writing: %s", err)
}
defer file.Close()
zipw := zip.NewWriter(file)
defer zipw.Close()
for _, filename := range filepaths {
if err := addFileToZip(filename, zipw); err != nil {
return fmt.Errorf("Failed to add file %s to zip: %s", filename, err)
}
}
return nil
}
func addFileToZip(filename string, zipw *zip.Writer) error {
file, err := os.Open(filename)
if err != nil {
return fmt.Errorf("Error opening file %s: %s", filename, err)
}
defer file.Close()
wr, err := zipw.Create(filename)
if err != nil {
return fmt.Errorf("Error adding file; '%s' to zip : %s", filename, err)
}
if _, err := io.Copy(wr, file); err != nil {
return fmt.Errorf("Error writing %s to zip: %s", filename, err)
}
return nil
}
This creates a broken zip which can not be extracted (I am running on mac os but this should not make a difference).
I also tried several other examples from stackoverflow and links found through google, but I do always get a broken zip. I get a zip with 135 bytes when I extract it I get 1 binary file with 0 bytes).
It would be great if someone could help me to find out what I am missing here.
Thx
You need to list and zip the files, not the directories. Simply make this adjustment to ignore directories in your listFiles function.
if !info.IsDir() {
files = append(files, path)
}

Move a file to a different drive with Go

I'm trying to move a file from my C-drive to my H-drive using os.Replace().
The code looks as follows:
func MoveFile(source string, destination string) {
err := os.Rename(source, destination)
if err != nil {
fmt.Println(err)
}
}
However, when I run the code I get the following error:
rename C:\old\path\to\file.txt H:\new\path\to\file.txt: The system cannot move the file to a different disk drive.
I found this issue on GitHub that specifies the problem but it appears that they will not change this function to allow it to move file on different disk drives.
I already searched for other possibilities to move files, but found nothing in the standard documentation or the internet.
So, what should I do now to be able to move files on different disk drives?
As the comment said, you'll need to create a new file on the other disk, copy the contents, and then remove the original. It's straightforward using os.Create, io.Copy, and os.Remove:
import (
"fmt"
"io"
"os"
)
func MoveFile(sourcePath, destPath string) error {
inputFile, err := os.Open(sourcePath)
if err != nil {
return fmt.Errorf("Couldn't open source file: %s", err)
}
outputFile, err := os.Create(destPath)
if err != nil {
inputFile.Close()
return fmt.Errorf("Couldn't open dest file: %s", err)
}
defer outputFile.Close()
_, err = io.Copy(outputFile, inputFile)
inputFile.Close()
if err != nil {
return fmt.Errorf("Writing to output file failed: %s", err)
}
// The copy was successful, so now delete the original file
err = os.Remove(sourcePath)
if err != nil {
return fmt.Errorf("Failed removing original file: %s", err)
}
return nil
}
You need to make sure that you handle all cases on both Linux and Windows. For example, for any size file,
package main
import (
"flag"
"fmt"
"io"
"os"
)
func MoveFile(source, destination string) (err error) {
src, err := os.Open(source)
if err != nil {
return err
}
defer src.Close()
fi, err := src.Stat()
if err != nil {
return err
}
flag := os.O_WRONLY | os.O_CREATE | os.O_TRUNC
perm := fi.Mode() & os.ModePerm
dst, err := os.OpenFile(destination, flag, perm)
if err != nil {
return err
}
defer dst.Close()
_, err = io.Copy(dst, src)
if err != nil {
dst.Close()
os.Remove(destination)
return err
}
err = dst.Close()
if err != nil {
return err
}
err = src.Close()
if err != nil {
return err
}
err = os.Remove(source)
if err != nil {
return err
}
return nil
}
func main() {
var src, dst string
flag.StringVar(&src, "src", "", "source file")
flag.StringVar(&dst, "dst", "", "destination file")
flag.Parse()
if src == "" || dst == "" {
flag.Usage()
os.Exit(1)
}
err := MoveFile(src, dst)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fmt.Printf("moved %q to %q\n", src, dst)
}
Output (Linux):
$ cp move.file src.file && go build movefile.go && ./movefile -src=src.file -dst=dst.file
moved "src.file" to "dst.file"
$
Output (Windows):
>copy /Y move.file src.file && go build movefile.go && movefile -src=src.file -dst=dst.file
moved "src.file" to "dst.file"
>
This solution Moves the file and preserves permissions:
func MoveFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return fmt.Errorf("Couldn't open source file: %s", err)
}
out, err := os.Create(dst)
if err != nil {
in.Close()
return fmt.Errorf("Couldn't open dest file: %s", err)
}
defer out.Close()
_, err = io.Copy(out, in)
in.Close()
if err != nil {
return fmt.Errorf("Writing to output file failed: %s", err)
}
err = out.Sync()
if err != nil {
return fmt.Errorf("Sync error: %s", err)
}
si, err := os.Stat(src)
if err != nil {
return fmt.Errorf("Stat error: %s", err)
}
err = os.Chmod(dst, si.Mode())
if err != nil {
return fmt.Errorf("Chmod error: %s", err)
}
err = os.Remove(src)
if err != nil {
return fmt.Errorf("Failed removing original file: %s", err)
}
return nil
}
If only want to Copy the file without remove the original:
func CopyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return fmt.Errorf("Couldn't open source file: %s", err)
}
out, err := os.Create(dst)
if err != nil {
in.Close()
return fmt.Errorf("Couldn't open dest file: %s", err)
}
defer out.Close()
_, err = io.Copy(out, in)
in.Close()
if err != nil {
return fmt.Errorf("Writing to output file failed: %s", err)
}
err = out.Sync()
if err != nil {
return fmt.Errorf("Sync error: %s", err)
}
si, err := os.Stat(src)
if err != nil {
return fmt.Errorf("Stat error: %s", err)
}
err = os.Chmod(dst, si.Mode())
if err != nil {
return fmt.Errorf("Chmod error: %s", err)
}
return nil
}
Maybe you can use a magic approach, just using the syscall.MoveFile as follows.
func main() {
oldpath := "D:\\black.txt"
newpath := "E:\\black-new.txt"
from, _ := syscall.UTF16PtrFromString(oldpath)
to, _ := syscall.UTF16PtrFromString(newpath)
fmt.Println(*from, *to)
err := syscall.MoveFile(from, to)
if err != nil {
panic(err)
}
}
the program works.
if you want a cross-platform compatibility program, you can implement your own MoveFile.
func MoveFile(src string, dst string) error {
if runtime.GOOS == "windows" {
from, _ := syscall.UTF16PtrFromString(src)
to, _ := syscall.UTF16PtrFromString(dst)
return syscall.MoveFile(from, to)
} else {
return os.Rename(src, dst)
}
}

Unable to delete an unzipped folder using golang

I wrote code that unzips a file in a particular location then copies the contents of the folder to outside where the folder is unzipped then it removes the folder.
This is the Code I wrote:
package main
import (
"os"
"flag"
"fmt"
"io"
"path/filepath"
"os/exec"
"archive/zip"
"time"
)
func RemoveContents(dir string) error {
d, err := os.Open(dir)
if err != nil {
return err
}
names, err := d.Readdirnames(-1)
if err != nil {
return err
}
for _, name := range names {
err = os.RemoveAll(filepath.Join(dir, name))
if err != nil {
return err
}
}
d.Close()
return nil
}
func CopyFile(source string, dest string) (err error) {
sourcefile, err := os.Open(source)
if err != nil {
return err
}
defer sourcefile.Close()
destfile, err := os.Create(dest)
if err != nil {
return err
}
defer destfile.Close()
_, err = io.Copy(destfile, sourcefile)
if err == nil {
sourceinfo, err := os.Stat(source)
if err != nil {
err = os.Chmod(dest, sourceinfo.Mode())
}
}
return
}
func CopyDir(source string, dest string) (err error) {
// get properties of source dir
sourceinfo, err := os.Stat(source)
if err != nil {
return err
}
// create dest dir
err = os.MkdirAll(dest, sourceinfo.Mode())
if err != nil {
return err
}
directory, _ := os.Open(source)
objects, err := directory.Readdir(-1)
for _, obj := range objects {
sourcefilepointer := source + "/" + obj.Name()
destinationfilepointer := dest + "/" + obj.Name()
if obj.IsDir() {
// create sub-directories - recursively
err = CopyDir(sourcefilepointer, destinationfilepointer)
if err != nil {
fmt.Println(err)
}
} else {
// perform copy
err = CopyFile(sourcefilepointer, destinationfilepointer)
if err != nil {
fmt.Println(err)
}
}
}
return
}
func main() {
flag.Parse() // get the source and destination directory
source_dir := flag.Arg(0) // get the source directory from 1st argument
dest_dir := flag.Arg(1) // get the destination directory from the 2nd argument
os.MkdirAll("E:\\go\\copyDirectory\\myFile.zip",0777)
zipFilePath := "E:\\go\\copyDirectory\\myFile.zip"
tempWrkDir := "E:\\go\\copyDirectory\\"
//Read zip file and get path handle.
fileHandleReader, err := zip.OpenReader(zipFilePath)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
//open zip file and read all the folder and files inside
for _, fileReadHandler := range fileHandleReader.Reader.File {
//read the file or folder handle inside zip
fileOpenHandle, err := fileReadHandler.Open()
if err != nil {
fmt.Println(err)
os.Exit(1)
}
defer fileOpenHandle.Close()
targetUnZipPath := filepath.Join(tempWrkDir, fileReadHandler.Name)
if fileReadHandler.FileInfo().IsDir() {
os.MkdirAll(targetUnZipPath, fileReadHandler.Mode())
//fmt.Println("Creating directory", path)
}else {
// create new dummy file to copy original file.
newTempFileHandle, err := os.OpenFile(targetUnZipPath, os.O_WRONLY|os.O_CREATE, fileReadHandler.Mode())
if err != nil {
fmt.Println(err)
os.Exit(1)
}
defer newTempFileHandle.Close()
//copying original file to dummy file.
if _, err = io.Copy(newTempFileHandle, fileOpenHandle); err != nil {
fmt.Println(err)
os.Exit(1)
}
}
}
time.Sleep(1000*time.Millisecond)
fmt.Println("Source :" + source_dir)
// check if the source dir exist
src, err := os.Stat(source_dir)
if err != nil {
panic(err)
}
if !src.IsDir() {
fmt.Println("Source is not a directory")
os.Exit(1)
}
// create the destination directory
fmt.Println("Destination :"+ dest_dir)
/*_, err = os.Open(dest_dir)
if !os.IsNotExist(err) {
fmt.Println("Destination directory already exists. Abort!")
os.Exit(1)
}*/
err = CopyDir(source_dir, dest_dir)
if err != nil {
fmt.Println(err)
} else {
fmt.Println("Directory copied")
}
err = RemoveContents("./myFiles")
if err != nil {
fmt.Println("ERRR:::",err)
}
//time.Sleep(10000*time.Millisecond)
}
The problem is that everything works fine except for deleting the folder. The folder has only one file in it. The location of the file is as follows:
E:\go\copyDirectory\myfile\mytextfile.txt
The Location of the zip file is as follows:
E:\go\copyDirectory\myfile.zip
The zip file has only one text file. The File inside the zip file is as follows:
E:\go\copyDirectory\myfile.zip\myfile\mytextfile.txt
The error I get is:
ERRR::: remove myfile\mytextfile.txt: The process cannot
access the file because it is being used by another process.
Thanks in advance.
You aren't closing the file. This:
defer newTempFileHandle.Close()
Is run when main finishes, which is after:
err = RemoveContents("./myFiles")
You can wrap that bit of code in an unnamed function:
func() {
//read the file or folder handle inside zip
fileOpenHandle, err := fileReadHandler.Open()
if err != nil {
fmt.Println(err)
os.Exit(1)
}
defer fileOpenHandle.Close()
targetUnZipPath := filepath.Join(tempWrkDir, fileReadHandler.Name)
if fileReadHandler.FileInfo().IsDir() {
os.MkdirAll(targetUnZipPath, fileReadHandler.Mode())
//fmt.Println("Creating directory", path)
} else {
// create new dummy file to copy original file.
newTempFileHandle, err := os.OpenFile(targetUnZipPath, os.O_WRONLY|os.O_CREATE, fileReadHandler.Mode())
if err != nil {
fmt.Println(err)
os.Exit(1)
}
defer newTempFileHandle.Close()
//copying original file to dummy file.
if _, err = io.Copy(newTempFileHandle, fileOpenHandle); err != nil {
fmt.Println(err)
os.Exit(1)
}
}
}()
And then your defer will happen before you try and remove the files. I would recommend pulling this out into a named function though.

Resources