how do I zip files in a folder with sub-directories correctly.
I have a local folder with following structure:
folder/hello/
folder/hello/world/
folder/hello/world/helloword.txt
folder/index.txt
this is my code:
package main
import (
"archive/zip"
"fmt"
"io"
"os"
"path/filepath"
)
func main() {
files, err := listFiles("./folder")
if err != nil {
panic(err)
}
zipMe(files, "test.zip")
for _, f := range files {
fmt.Println(f)
}
fmt.Println("Done!")
}
func listFiles(root string) ([]string, error) {
var files []string
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
files = append(files, path)
return nil
})
if err != nil {
return nil, err
}
return files, nil
}
func zipMe(filepaths []string, target string) error {
flags := os.O_WRONLY | os.O_CREATE | os.O_TRUNC
file, err := os.OpenFile(target, flags, 0644)
if err != nil {
return fmt.Errorf("Failed to open zip for writing: %s", err)
}
defer file.Close()
zipw := zip.NewWriter(file)
defer zipw.Close()
for _, filename := range filepaths {
if err := addFileToZip(filename, zipw); err != nil {
return fmt.Errorf("Failed to add file %s to zip: %s", filename, err)
}
}
return nil
}
func addFileToZip(filename string, zipw *zip.Writer) error {
file, err := os.Open(filename)
if err != nil {
return fmt.Errorf("Error opening file %s: %s", filename, err)
}
defer file.Close()
wr, err := zipw.Create(filename)
if err != nil {
return fmt.Errorf("Error adding file; '%s' to zip : %s", filename, err)
}
if _, err := io.Copy(wr, file); err != nil {
return fmt.Errorf("Error writing %s to zip: %s", filename, err)
}
return nil
}
This creates a broken zip which can not be extracted (I am running on mac os but this should not make a difference).
I also tried several other examples from stackoverflow and links found through google, but I do always get a broken zip. I get a zip with 135 bytes when I extract it I get 1 binary file with 0 bytes).
It would be great if someone could help me to find out what I am missing here.
Thx
You need to list and zip the files, not the directories. Simply make this adjustment to ignore directories in your listFiles function.
if !info.IsDir() {
files = append(files, path)
}
Related
I wrote a function that makes a .zip archive from a directory (and subdir-s, files in it)
func ZipDirectory(backupName string) error {
file, err := os.Create(backupName)
if err != nil {
return err
}
defer file.Close()
w := zip.NewWriter(file)
defer w.Close()
walker := func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
file, err := os.Open(path)
if err != nil {
return fmt.Errorf("error opening file %q: %w", path, err)
}
defer file.Close()
f, err := w.Create(path)
if err != nil {
return fmt.Errorf("error creating file %q: %w", path, err)
}
_, err = io.Copy(f, file)
if err != nil {
return fmt.Errorf("error copying file %q: %w", path, err)
}
return nil
}
err = filepath.Walk(config.FilePath, walker)
if err != nil {
return err
}
return nil
}
but if directory weights more then 3GB it creates empty archive which weights exactly as it should be.
it looks like this:
help, don't understand this mystic result....
Big zip archives shows empty after archivation because you don't have installed software for unzipiing it (like 7Zip)
After installation and openning with 7Zip everything works
I am trying to create a .tar.gz file from folder that contains multiple files / folders. Once the .tar.gz file gets created, while extracting, the files are not not properly extracted. Mostly I think its because of large names or path exceeding some n characters, because same thing works when the filename/path is small. I referred this https://github.com/golang/go/issues/17630 and tried to add below code but it did not help.
header.Uid = 0
header.Gid = 0
I am using simple code seen below to create .tar.gz. The approach is, I create a temp folder, do some processing on the files and from that temp path, I create the .tar.gz file hence in the path below I am using pre-defined temp folder path.
package main
import (
"archive/tar"
"compress/gzip"
"fmt"
"io"
"log"
"os"
fp "path/filepath"
)
func main() {
// Create output file
out, err := os.Create("output.tar.gz")
if err != nil {
log.Fatalln("Error writing archive:", err)
}
defer out.Close()
// Create the archive and write the output to the "out" Writer
tmpDir := "C:/Users/USERNAME~1/AppData/Local/Temp/temp-241232063"
err = createArchive1(tmpDir, out)
if err != nil {
log.Fatalln("Error creating archive:", err)
}
fmt.Println("Archive created successfully")
}
func createArchive1(path string, targetFile *os.File) error {
gw := gzip.NewWriter(targetFile)
defer gw.Close()
tw := tar.NewWriter(gw)
defer tw.Close()
// walk through every file in the folder
err := fp.Walk(path, func(filePath string, info os.FileInfo, err error) error {
// ensure the src actually exists before trying to tar it
if _, err := os.Stat(filePath); err != nil {
return err
}
if err != nil {
return err
}
if info.IsDir() {
return nil
}
file, err := os.Open(filePath)
if err != nil {
return err
}
defer file.Close()
// generate tar header
header, err := tar.FileInfoHeader(info, info.Name())
header.Uid = 0
header.Gid = 0
if err != nil {
return err
}
header.Name = filePath //strings.TrimPrefix(filePath, fmt.Sprintf("%s/", fp.Dir(path))) //info.Name()
// write header
if err := tw.WriteHeader(header); err != nil {
return err
}
if _, err := io.Copy(tw, file); err != nil {
return err
}
return nil
})
return err
}
Please let me know what wrong I am doing.
There're examples about compressing a file to .zip in Go. However, the file they generate include the directory folder. When I decompress the .zip file, there will be a new folder.
So, how can I compress a file to .zip without getting the directory folder included?
An example:
https://golangcode.com/create-zip-files-in-go/
package main
import (
"archive/zip"
"fmt"
"io"
"os"
)
func main() {
// List of Files to Zip
files := []string{"example.csv", "data.csv"}
output := "done.zip"
if err := ZipFiles(output, files); err != nil {
panic(err)
}
fmt.Println("Zipped File:", output)
}
// ZipFiles compresses one or many files into a single zip archive file.
// Param 1: filename is the output zip file's name.
// Param 2: files is a list of files to add to the zip.
func ZipFiles(filename string, files []string) error {
newZipFile, err := os.Create(filename)
if err != nil {
return err
}
defer newZipFile.Close()
zipWriter := zip.NewWriter(newZipFile)
defer zipWriter.Close()
// Add files to zip
for _, file := range files {
if err = AddFileToZip(zipWriter, file); err != nil {
return err
}
}
return nil
}
func AddFileToZip(zipWriter *zip.Writer, filename string) error {
fileToZip, err := os.Open(filename)
if err != nil {
return err
}
defer fileToZip.Close()
// Get the file information
info, err := fileToZip.Stat()
if err != nil {
return err
}
header, err := zip.FileInfoHeader(info)
if err != nil {
return err
}
// Using FileInfoHeader() above only uses the basename of the file. If we want
// to preserve the folder structure we can overwrite this with the full path.
header.Name = filename
// Change to deflate to gain better compression
// see http://golang.org/pkg/archive/zip/#pkg-constants
header.Method = zip.Deflate
writer, err := zipWriter.CreateHeader(header)
if err != nil {
return err
}
_, err = io.Copy(writer, fileToZip)
return err
}
Just use a base name of the file in the zip header.
header.Name = filepath.Base(filename)
^^^^^^^^^^^^^^
Here is a version that does the same thing
package main
import (
"archive/zip"
"io"
"log"
"os"
"path/filepath"
)
func createFlatZip(w io.Writer, files ...string) error {
z := zip.NewWriter(w)
for _, file := range files {
src, err := os.Open(file)
if err != nil {
return err
}
info, err := src.Stat()
if err != nil {
return err
}
hdr, err := zip.FileInfoHeader(info)
if err != nil {
return err
}
hdr.Name = filepath.Base(file) // Write only the base name in the header
dst, err := z.CreateHeader(hdr)
if err != nil {
return err
}
_, err = io.Copy(dst, src)
if err != nil {
return err
}
src.Close()
}
return z.Close()
}
func main() {
if len(os.Args) < 3 {
log.Fatalf("archive name and at least one file are required")
}
a, err := os.Create(os.Args[1])
if err != nil {
log.Fatal(err)
}
defer a.Close()
err = createFlatZip(a, os.Args[2:]...)
if err != nil {
log.Fatal(err)
}
}
Results:
~/src/gozip
➜ go build
~/src/gozip
➜ mkdir test && echo 1 > test/1.txt # create a test file in a subfolder
~/src/gozip
➜ ./gozip 1.zip test/1.txt
~/src/gozip
➜ unzip -l 1.zip
Archive: 1.zip
Length Date Time Name
--------- ---------- ----- ----
2 08-15-2019 01:29 1.txt
--------- -------
2 1 file
I'm trying to move a file from my C-drive to my H-drive using os.Replace().
The code looks as follows:
func MoveFile(source string, destination string) {
err := os.Rename(source, destination)
if err != nil {
fmt.Println(err)
}
}
However, when I run the code I get the following error:
rename C:\old\path\to\file.txt H:\new\path\to\file.txt: The system cannot move the file to a different disk drive.
I found this issue on GitHub that specifies the problem but it appears that they will not change this function to allow it to move file on different disk drives.
I already searched for other possibilities to move files, but found nothing in the standard documentation or the internet.
So, what should I do now to be able to move files on different disk drives?
As the comment said, you'll need to create a new file on the other disk, copy the contents, and then remove the original. It's straightforward using os.Create, io.Copy, and os.Remove:
import (
"fmt"
"io"
"os"
)
func MoveFile(sourcePath, destPath string) error {
inputFile, err := os.Open(sourcePath)
if err != nil {
return fmt.Errorf("Couldn't open source file: %s", err)
}
outputFile, err := os.Create(destPath)
if err != nil {
inputFile.Close()
return fmt.Errorf("Couldn't open dest file: %s", err)
}
defer outputFile.Close()
_, err = io.Copy(outputFile, inputFile)
inputFile.Close()
if err != nil {
return fmt.Errorf("Writing to output file failed: %s", err)
}
// The copy was successful, so now delete the original file
err = os.Remove(sourcePath)
if err != nil {
return fmt.Errorf("Failed removing original file: %s", err)
}
return nil
}
You need to make sure that you handle all cases on both Linux and Windows. For example, for any size file,
package main
import (
"flag"
"fmt"
"io"
"os"
)
func MoveFile(source, destination string) (err error) {
src, err := os.Open(source)
if err != nil {
return err
}
defer src.Close()
fi, err := src.Stat()
if err != nil {
return err
}
flag := os.O_WRONLY | os.O_CREATE | os.O_TRUNC
perm := fi.Mode() & os.ModePerm
dst, err := os.OpenFile(destination, flag, perm)
if err != nil {
return err
}
defer dst.Close()
_, err = io.Copy(dst, src)
if err != nil {
dst.Close()
os.Remove(destination)
return err
}
err = dst.Close()
if err != nil {
return err
}
err = src.Close()
if err != nil {
return err
}
err = os.Remove(source)
if err != nil {
return err
}
return nil
}
func main() {
var src, dst string
flag.StringVar(&src, "src", "", "source file")
flag.StringVar(&dst, "dst", "", "destination file")
flag.Parse()
if src == "" || dst == "" {
flag.Usage()
os.Exit(1)
}
err := MoveFile(src, dst)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fmt.Printf("moved %q to %q\n", src, dst)
}
Output (Linux):
$ cp move.file src.file && go build movefile.go && ./movefile -src=src.file -dst=dst.file
moved "src.file" to "dst.file"
$
Output (Windows):
>copy /Y move.file src.file && go build movefile.go && movefile -src=src.file -dst=dst.file
moved "src.file" to "dst.file"
>
This solution Moves the file and preserves permissions:
func MoveFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return fmt.Errorf("Couldn't open source file: %s", err)
}
out, err := os.Create(dst)
if err != nil {
in.Close()
return fmt.Errorf("Couldn't open dest file: %s", err)
}
defer out.Close()
_, err = io.Copy(out, in)
in.Close()
if err != nil {
return fmt.Errorf("Writing to output file failed: %s", err)
}
err = out.Sync()
if err != nil {
return fmt.Errorf("Sync error: %s", err)
}
si, err := os.Stat(src)
if err != nil {
return fmt.Errorf("Stat error: %s", err)
}
err = os.Chmod(dst, si.Mode())
if err != nil {
return fmt.Errorf("Chmod error: %s", err)
}
err = os.Remove(src)
if err != nil {
return fmt.Errorf("Failed removing original file: %s", err)
}
return nil
}
If only want to Copy the file without remove the original:
func CopyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return fmt.Errorf("Couldn't open source file: %s", err)
}
out, err := os.Create(dst)
if err != nil {
in.Close()
return fmt.Errorf("Couldn't open dest file: %s", err)
}
defer out.Close()
_, err = io.Copy(out, in)
in.Close()
if err != nil {
return fmt.Errorf("Writing to output file failed: %s", err)
}
err = out.Sync()
if err != nil {
return fmt.Errorf("Sync error: %s", err)
}
si, err := os.Stat(src)
if err != nil {
return fmt.Errorf("Stat error: %s", err)
}
err = os.Chmod(dst, si.Mode())
if err != nil {
return fmt.Errorf("Chmod error: %s", err)
}
return nil
}
Maybe you can use a magic approach, just using the syscall.MoveFile as follows.
func main() {
oldpath := "D:\\black.txt"
newpath := "E:\\black-new.txt"
from, _ := syscall.UTF16PtrFromString(oldpath)
to, _ := syscall.UTF16PtrFromString(newpath)
fmt.Println(*from, *to)
err := syscall.MoveFile(from, to)
if err != nil {
panic(err)
}
}
the program works.
if you want a cross-platform compatibility program, you can implement your own MoveFile.
func MoveFile(src string, dst string) error {
if runtime.GOOS == "windows" {
from, _ := syscall.UTF16PtrFromString(src)
to, _ := syscall.UTF16PtrFromString(dst)
return syscall.MoveFile(from, to)
} else {
return os.Rename(src, dst)
}
}
I wrote code that unzips a file in a particular location then copies the contents of the folder to outside where the folder is unzipped then it removes the folder.
This is the Code I wrote:
package main
import (
"os"
"flag"
"fmt"
"io"
"path/filepath"
"os/exec"
"archive/zip"
"time"
)
func RemoveContents(dir string) error {
d, err := os.Open(dir)
if err != nil {
return err
}
names, err := d.Readdirnames(-1)
if err != nil {
return err
}
for _, name := range names {
err = os.RemoveAll(filepath.Join(dir, name))
if err != nil {
return err
}
}
d.Close()
return nil
}
func CopyFile(source string, dest string) (err error) {
sourcefile, err := os.Open(source)
if err != nil {
return err
}
defer sourcefile.Close()
destfile, err := os.Create(dest)
if err != nil {
return err
}
defer destfile.Close()
_, err = io.Copy(destfile, sourcefile)
if err == nil {
sourceinfo, err := os.Stat(source)
if err != nil {
err = os.Chmod(dest, sourceinfo.Mode())
}
}
return
}
func CopyDir(source string, dest string) (err error) {
// get properties of source dir
sourceinfo, err := os.Stat(source)
if err != nil {
return err
}
// create dest dir
err = os.MkdirAll(dest, sourceinfo.Mode())
if err != nil {
return err
}
directory, _ := os.Open(source)
objects, err := directory.Readdir(-1)
for _, obj := range objects {
sourcefilepointer := source + "/" + obj.Name()
destinationfilepointer := dest + "/" + obj.Name()
if obj.IsDir() {
// create sub-directories - recursively
err = CopyDir(sourcefilepointer, destinationfilepointer)
if err != nil {
fmt.Println(err)
}
} else {
// perform copy
err = CopyFile(sourcefilepointer, destinationfilepointer)
if err != nil {
fmt.Println(err)
}
}
}
return
}
func main() {
flag.Parse() // get the source and destination directory
source_dir := flag.Arg(0) // get the source directory from 1st argument
dest_dir := flag.Arg(1) // get the destination directory from the 2nd argument
os.MkdirAll("E:\\go\\copyDirectory\\myFile.zip",0777)
zipFilePath := "E:\\go\\copyDirectory\\myFile.zip"
tempWrkDir := "E:\\go\\copyDirectory\\"
//Read zip file and get path handle.
fileHandleReader, err := zip.OpenReader(zipFilePath)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
//open zip file and read all the folder and files inside
for _, fileReadHandler := range fileHandleReader.Reader.File {
//read the file or folder handle inside zip
fileOpenHandle, err := fileReadHandler.Open()
if err != nil {
fmt.Println(err)
os.Exit(1)
}
defer fileOpenHandle.Close()
targetUnZipPath := filepath.Join(tempWrkDir, fileReadHandler.Name)
if fileReadHandler.FileInfo().IsDir() {
os.MkdirAll(targetUnZipPath, fileReadHandler.Mode())
//fmt.Println("Creating directory", path)
}else {
// create new dummy file to copy original file.
newTempFileHandle, err := os.OpenFile(targetUnZipPath, os.O_WRONLY|os.O_CREATE, fileReadHandler.Mode())
if err != nil {
fmt.Println(err)
os.Exit(1)
}
defer newTempFileHandle.Close()
//copying original file to dummy file.
if _, err = io.Copy(newTempFileHandle, fileOpenHandle); err != nil {
fmt.Println(err)
os.Exit(1)
}
}
}
time.Sleep(1000*time.Millisecond)
fmt.Println("Source :" + source_dir)
// check if the source dir exist
src, err := os.Stat(source_dir)
if err != nil {
panic(err)
}
if !src.IsDir() {
fmt.Println("Source is not a directory")
os.Exit(1)
}
// create the destination directory
fmt.Println("Destination :"+ dest_dir)
/*_, err = os.Open(dest_dir)
if !os.IsNotExist(err) {
fmt.Println("Destination directory already exists. Abort!")
os.Exit(1)
}*/
err = CopyDir(source_dir, dest_dir)
if err != nil {
fmt.Println(err)
} else {
fmt.Println("Directory copied")
}
err = RemoveContents("./myFiles")
if err != nil {
fmt.Println("ERRR:::",err)
}
//time.Sleep(10000*time.Millisecond)
}
The problem is that everything works fine except for deleting the folder. The folder has only one file in it. The location of the file is as follows:
E:\go\copyDirectory\myfile\mytextfile.txt
The Location of the zip file is as follows:
E:\go\copyDirectory\myfile.zip
The zip file has only one text file. The File inside the zip file is as follows:
E:\go\copyDirectory\myfile.zip\myfile\mytextfile.txt
The error I get is:
ERRR::: remove myfile\mytextfile.txt: The process cannot
access the file because it is being used by another process.
Thanks in advance.
You aren't closing the file. This:
defer newTempFileHandle.Close()
Is run when main finishes, which is after:
err = RemoveContents("./myFiles")
You can wrap that bit of code in an unnamed function:
func() {
//read the file or folder handle inside zip
fileOpenHandle, err := fileReadHandler.Open()
if err != nil {
fmt.Println(err)
os.Exit(1)
}
defer fileOpenHandle.Close()
targetUnZipPath := filepath.Join(tempWrkDir, fileReadHandler.Name)
if fileReadHandler.FileInfo().IsDir() {
os.MkdirAll(targetUnZipPath, fileReadHandler.Mode())
//fmt.Println("Creating directory", path)
} else {
// create new dummy file to copy original file.
newTempFileHandle, err := os.OpenFile(targetUnZipPath, os.O_WRONLY|os.O_CREATE, fileReadHandler.Mode())
if err != nil {
fmt.Println(err)
os.Exit(1)
}
defer newTempFileHandle.Close()
//copying original file to dummy file.
if _, err = io.Copy(newTempFileHandle, fileOpenHandle); err != nil {
fmt.Println(err)
os.Exit(1)
}
}
}()
And then your defer will happen before you try and remove the files. I would recommend pulling this out into a named function though.