I would like to decompress tar-gz file by golang.
err := DecompressTarGz('xxx.tar.gz', '/Users/foobarbuzz/')
Use compress/gzip in combination with archive/tar or use os/exec to call tar and gzip directly if you don't like to implement all of that in Go.
I was looking for an answer to this one too. Implemented something that should work.
func Decompress(targetdir string, reader io.ReadCloser) error {
gzReader, err := gzip.NewReader(reader)
if err != nil {
return err
}
defer gzReader.Close()
tarReader := tar.NewReader(gzReader)
for {
header, err := tarReader.Next()
if err == io.EOF {
break
} else if err != nil {
return err
}
target := path.Join(targetdir, header.Name)
switch header.Typeflag {
case tar.TypeDir:
err = os.MkdirAll(target, os.FileMode(header.Mode))
if err != nil {
return err
}
setAttrs(target, header)
break
case tar.TypeReg:
w, err := os.Create(target)
if err != nil {
return err
}
_, err = io.Copy(w, tarReader)
if err != nil {
return err
}
w.Close()
setAttrs(target, header)
break
default:
log.Printf("unsupported type: %v", header.Typeflag)
break
}
}
return nil
}
func setAttrs(target string, header *tar.Header) {
os.Chmod(target, os.FileMode(header.Mode))
os.Chtimes(target, header.AccessTime, header.ModTime)
}
Related
I have a tar.gz file that I received via HTTP request. Inside this tar is a folder, which has a file i'm interested in. I am unsure what parameter to give as root to WalkDir, most attempts end in an ENOENT error. My code:
func untarFiles(tarBytes io.ReadCloser) (*os.File, error) {
var dbFile os.File
gzipReader, err := gzip.NewReader(tarBytes)
if err != nil {
return nil, err
}
defer gzipReader.Close()
tarReader := tar.NewReader(gzipReader)
for {
header, err := tarReader.Next()
switch {
case err == io.EOF:
break
case header.FileInfo().IsDir():
err = filepath.WalkDir(header.Name, func(path string, info fs.DirEntry, err error) error {
if err != nil {
return err
}
if info.IsDir() {
return nil
}
if !strings.HasSuffix(path, "my_suffix") {
return nil
}
dirFile, err := os.Open(path)
if err != nil {
return err
}
defer dirFile.Close()
dbFile = *dirFile
return nil
})
break
}
break
}
return &dbFile, nil
I've tried header.Name, header.FileInfo().Name(). Do I have to save this as a temporary directory and delete it later?
Thanks!
I'm trying to list directory contents on a share from Linux to Windows using Go.
So far I've managed to Create/Remove new files inside a share with full Read/Write permissions.
Go module: https://godoc.org/github.com/hirochachacha/go-smb2#Client
Functions:
func connect_client(host string, share string, session map[string]string) *smb2.Client {
//Checks for a connection on port
conn, err := net.Dial("tcp", host+":445")
if err != nil {
panic(err)
}
//smb auth
d := &smb2.Dialer{
Initiator: &smb2.NTLMInitiator{
User: session["Username"],
Password: session["Password"],
Domain: session["Domain"],
},
}
//Returns a client session
client, err := d.Dial(conn)
if err != nil {
fmt.Println("Connection failed")
client.Logoff()
} else {
fmt.Println("Connection Succeeded")
}
return client
}
func check_write(host string, client *smb2.Client) {
file := "asdasdas.txt"
fs, err := client.Mount(host)
if err != nil {
fmt.Println(err)
os.Exit(0)
}
defer fs.Umount()
share := strings.Split(host, `\\`)
f, err := fs.Create(file)
if err != nil {
fmt.Println("You do not have write permissions on directory:%s ! \n", strings.Split(share[1], `\`)[1])
os.Exit(0)
}
defer fs.Remove(file)
defer f.Close()
fmt.Printf("You have write permissions to directory: %s \n", strings.Split(share[1], `\`)[1]))
}
func list_all(client *smb2.Client, host string) {
fs, err := client.Mount(host)
if err != nil {
fmt.Println(err)
os.Exit(0)
}
defer fs.Umount()
_, err = fs.Open(`Test.txt`)
if err != nil {
fmt.Println(err)
os.Exit(0)
}
}
func main() {
host, share, action, session := get_flags()
client := connect_client(host, share, session)
full_host := `\\` + host + `\` + share
//File create
if action == "check_write" {
check_write(full_host, client)
}
if action == "list_files" {
list_all(client, full_host)
}
}
In the function list_all() everything works, but when I am trying to access \\192.168.1.19\Sharing only..
When I input just a host with directory name it seas it can not list the directory path because it can not find the object specified.
I can't understand how I can get pointer used for *RemoteFile in order to use the functions:
f.Readdir()
f.Name()
etc....
So far I managed to use *RemoteFileSystem only for all other actions but I want to list all contents of the directory..
Help would be much appreciated!
Edit:
If it wasn't clear enough, in order to use functions like:
f.Readdir()
f.Name()
I need to get a pointer for *RemoteFile, this is my main issue
https://godoc.org/github.com/hirochachacha/go-smb2#RemoteFileSystem.Open
Use Open on a RemoteFileSystem and either a directory name, or empty string for the directory at the root of the filesystem.
e.g.
client, err := d.Dial(conn)
if err != nil {
return err
}
rfs, err := client.Mount("jrwren")
if err != nil {
return err
}
// cat the NOTES file.
f, err := rfs.Open("NOTES")
if err != nil {
return err
}
defer f.Close()
io.Copy(os.Stdout, f)
// List all the files
dir, err := rfs.Open("")
if err != nil {
return err
}
fis, err := dir.Readdir(10)
if err != nil {
return err
}
for i := range fis {
fmt.Println(fis[i].Name())
}
func list_all(client *smb2.Client, host string) {
fs, err := client.Mount(host)
if err != nil {
fmt.Println(err)
os.Exit(0)
}
dir, err := fs.Open("")
if err != nil {
fmt.Println(err)
os.Exit(0)
}
fis, err := dir.Readdir(-1)
if err != nil {
fmt.Println(err)
os.Exit(0)
}
for i := range fis {
fmt.Println(fis[i].Name())
}
}
This would be the answer thank you!
I have a function in which I take in a base64 string and get the content of it (PDF or JPEG).
I read in the base64 content, convert it to bytes and decode it into the file that it is.
I then create a file where I will output the decoded file (JPEG or PDF).
Then I write the bytes to it.
Then I call my GetFileContentType on it and it returns to me an empty string.
If I run the functions separately, as in I first the first function to create the decoded file, and end it. And then call the second function to get the content type, it works and returns it as JPEG or PDF.
What am I doing wrong here?
And is there a better way to do this?
func ConvertToJPEGBase64(
src string,
dst string,
) error {
b, err := ioutil.ReadFile(src)
if err != nil {
return err
}
str := string(b)
byteArray, err := base64.StdEncoding.DecodeString(str)
if err != nil {
return err
}
f, err := os.Create(dst)
if err != nil {
return err
}
if _, err := f.Write(byteArray); err != nil {
return err
}
f.Sync()
filetype, err := client.GetFileContentType(f)
if err != nil {
return err
}
if strings.Contains(filetype, "jpeg") {
// do something
} else {
// do something else
}
return nil
}
// GetFileContentType tells us the type of file
func GetFileContentType(out *os.File) (string, error) {
// Only the first 512 bytes are used to sniff the content type.
buffer := make([]byte, 512)
_, err := out.Read(buffer)
if err != nil {
return "", err
}
contentType := http.DetectContentType(buffer)
return contentType, nil
}
The problem is that GetFileContentType reads from the end of the file. Fix this be seeking back to the beginning of the file before calling calling GetFileContentType:
if _, err := f.Seek(io.SeekStart, 0); err != nil {
return err
}
A better fix is to use the file data that's already in memory. This simplifies the code to the point where there's no need for the GetFileContentType function.
func ConvertToJPEGBase64(
src string,
dst string,
) error {
b, err := ioutil.ReadFile(src)
if err != nil {
return err
}
str := string(b)
byteArray, err := base64.StdEncoding.DecodeString(str)
if err != nil {
return err
}
f, err := os.Create(dst)
if err != nil {
return err
}
defer f.Close() // <-- Close the file on return.
if _, err := f.Write(byteArray); err != nil {
return err
}
fileType := http.DetectContentType(byteArray) // <-- use data in memory
if strings.Contains(fileType, "jpeg") {
// do something
} else {
// do something else
}
return nil
}
More code can be eliminated by using ioutil.WriteFile:
func ConvertToJPEGBase64(src, dst string) error {
b, err := ioutil.ReadFile(src)
if err != nil {
return err
}
byteArray, err := base64.StdEncoding.DecodeString(string(b))
if err != nil {
return err
}
if err := ioutil.WriteFile(dst, byteArray, 0666); err != nil {
return err
}
fileType := http.DetectContentType(byteArray)
if strings.Contains(fileType, "jpeg") {
// do something
} else {
// do something else
}
return nil
}
Is there an easy way to copy a directory in go?
I have the following function:
err = CopyDir("sourceFolder","destinationFolder")
Nothing so far has worked, including libraries such as github.com/cf-guardian/guardian/kernel/fileutils
One important thing to note is that I need to preserve directory structure, including the sourceFolder itself, not simply copy all contents of the folder.
I believe that docker implementation can be considered as complete solution for handling edge cases:
https://github.com/moby/moby/blob/master/daemon/graphdriver/copy/copy.go
There are following good things:
unsupported file type rise error
preserving permissions and ownership
preserving extended attributes
preserving timestamp
but because of a lot of imports your tiny application becomes huge.
I've tried to combine several solutions but use stdlib and for Linux only:
func CopyDirectory(scrDir, dest string) error {
entries, err := os.ReadDir(scrDir)
if err != nil {
return err
}
for _, entry := range entries {
sourcePath := filepath.Join(scrDir, entry.Name())
destPath := filepath.Join(dest, entry.Name())
fileInfo, err := os.Stat(sourcePath)
if err != nil {
return err
}
stat, ok := fileInfo.Sys().(*syscall.Stat_t)
if !ok {
return fmt.Errorf("failed to get raw syscall.Stat_t data for '%s'", sourcePath)
}
switch fileInfo.Mode() & os.ModeType{
case os.ModeDir:
if err := CreateIfNotExists(destPath, 0755); err != nil {
return err
}
if err := CopyDirectory(sourcePath, destPath); err != nil {
return err
}
case os.ModeSymlink:
if err := CopySymLink(sourcePath, destPath); err != nil {
return err
}
default:
if err := Copy(sourcePath, destPath); err != nil {
return err
}
}
if err := os.Lchown(destPath, int(stat.Uid), int(stat.Gid)); err != nil {
return err
}
fInfo, err := entry.Info()
if err != nil {
return err
}
isSymlink := fInfo.Mode()&os.ModeSymlink != 0
if !isSymlink {
if err := os.Chmod(destPath, fInfo.Mode()); err != nil {
return err
}
}
}
return nil
}
func Copy(srcFile, dstFile string) error {
out, err := os.Create(dstFile)
if err != nil {
return err
}
defer out.Close()
in, err := os.Open(srcFile)
defer in.Close()
if err != nil {
return err
}
_, err = io.Copy(out, in)
if err != nil {
return err
}
return nil
}
func Exists(filePath string) bool {
if _, err := os.Stat(filePath); os.IsNotExist(err) {
return false
}
return true
}
func CreateIfNotExists(dir string, perm os.FileMode) error {
if Exists(dir) {
return nil
}
if err := os.MkdirAll(dir, perm); err != nil {
return fmt.Errorf("failed to create directory: '%s', error: '%s'", dir, err.Error())
}
return nil
}
func CopySymLink(source, dest string) error {
link, err := os.Readlink(source)
if err != nil {
return err
}
return os.Symlink(link, dest)
}
This package seems to do exactly what you want to do, give it a try.
From the readme:
err := Copy("your/source/directory", "your/destination/directory")
Not satisfied with the already listed options which include using sketchy libraries, or vastly bloated libraries.
In my case, I opted to do things the old fashioned way. With shell commands!
import (
"os/exec"
)
func main() {
// completely arbitrary paths
oldDir := "/home/arshbot/"
newDir := "/tmp/"
cmd := exec.Command("cp", "--recursive", oldDir, newDir)
cmd.Run()
}
This solution copies a directory recursively, including symbolic links. Trying to be efficient in the actual copy stage using streams.
Also it's fairly easy to handle more of irregular files if needed.
// CopyDir copies the content of src to dst. src should be a full path.
func CopyDir(dst, src string) error {
return filepath.Walk(src, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
// copy to this path
outpath := filepath.Join(dst, strings.TrimPrefix(path, src))
if info.IsDir() {
os.MkdirAll(outpath, info.Mode())
return nil // means recursive
}
// handle irregular files
if !info.Mode().IsRegular() {
switch info.Mode().Type() & os.ModeType {
case os.ModeSymlink:
link, err := os.Readlink(path)
if err != nil {
return err
}
return os.Symlink(link, outpath)
}
return nil
}
// copy contents of regular file efficiently
// open input
in, _ := os.Open(path)
if err != nil {
return err
}
defer in.Close()
// create output
fh, err := os.Create(outpath)
if err != nil {
return err
}
defer fh.Close()
// make it the same
fh.Chmod(info.Mode())
// copy content
_, err = io.Copy(fh, in)
return err
})
}
I've come up with a relatively shorter answer which uses path/filepath's Walk method:
import (
"io/ioutil"
"path/filepath"
"os"
"strings"
)
func copy(source, destination string) error {
var err error = filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
var relPath string = strings.Replace(path, source, "", 1)
if relPath == "" {
return nil
}
if info.IsDir() {
return os.Mkdir(filepath.Join(destination, relPath), 0755)
} else {
var data, err1 = ioutil.ReadFile(filepath.Join(source, relPath))
if err1 != nil {
return err1
}
return ioutil.WriteFile(filepath.Join(destination, relPath), data, 0777)
}
})
return err
}
Also this might be a solution:
available on github.com/floscodes/golang-tools
import (
"fmt"
"io/ioutil"
"os"
)
func CopyDir(src string, dest string) error {
if dest[:len(src)] == src {
return fmt.Errorf("Cannot copy a folder into the folder itself!")
}
f, err := os.Open(src)
if err != nil {
return err
}
file, err := f.Stat()
if err != nil {
return err
}
if !file.IsDir() {
return fmt.Errorf("Source " + file.Name() + " is not a directory!")
}
err = os.Mkdir(dest, 0755)
if err != nil {
return err
}
files, err := ioutil.ReadDir(src)
if err != nil {
return err
}
for _, f := range files {
if f.IsDir() {
err = CopyDir(src+"/"+f.Name(), dest+"/"+f.Name())
if err != nil {
return err
}
}
if !f.IsDir() {
content, err := ioutil.ReadFile(src + "/" + f.Name())
if err != nil {
return err
}
err = ioutil.WriteFile(dest+"/"+f.Name(), content, 0755)
if err != nil {
return err
}
}
}
return nil
}
I'm trying to move a file from my C-drive to my H-drive using os.Replace().
The code looks as follows:
func MoveFile(source string, destination string) {
err := os.Rename(source, destination)
if err != nil {
fmt.Println(err)
}
}
However, when I run the code I get the following error:
rename C:\old\path\to\file.txt H:\new\path\to\file.txt: The system cannot move the file to a different disk drive.
I found this issue on GitHub that specifies the problem but it appears that they will not change this function to allow it to move file on different disk drives.
I already searched for other possibilities to move files, but found nothing in the standard documentation or the internet.
So, what should I do now to be able to move files on different disk drives?
As the comment said, you'll need to create a new file on the other disk, copy the contents, and then remove the original. It's straightforward using os.Create, io.Copy, and os.Remove:
import (
"fmt"
"io"
"os"
)
func MoveFile(sourcePath, destPath string) error {
inputFile, err := os.Open(sourcePath)
if err != nil {
return fmt.Errorf("Couldn't open source file: %s", err)
}
outputFile, err := os.Create(destPath)
if err != nil {
inputFile.Close()
return fmt.Errorf("Couldn't open dest file: %s", err)
}
defer outputFile.Close()
_, err = io.Copy(outputFile, inputFile)
inputFile.Close()
if err != nil {
return fmt.Errorf("Writing to output file failed: %s", err)
}
// The copy was successful, so now delete the original file
err = os.Remove(sourcePath)
if err != nil {
return fmt.Errorf("Failed removing original file: %s", err)
}
return nil
}
You need to make sure that you handle all cases on both Linux and Windows. For example, for any size file,
package main
import (
"flag"
"fmt"
"io"
"os"
)
func MoveFile(source, destination string) (err error) {
src, err := os.Open(source)
if err != nil {
return err
}
defer src.Close()
fi, err := src.Stat()
if err != nil {
return err
}
flag := os.O_WRONLY | os.O_CREATE | os.O_TRUNC
perm := fi.Mode() & os.ModePerm
dst, err := os.OpenFile(destination, flag, perm)
if err != nil {
return err
}
defer dst.Close()
_, err = io.Copy(dst, src)
if err != nil {
dst.Close()
os.Remove(destination)
return err
}
err = dst.Close()
if err != nil {
return err
}
err = src.Close()
if err != nil {
return err
}
err = os.Remove(source)
if err != nil {
return err
}
return nil
}
func main() {
var src, dst string
flag.StringVar(&src, "src", "", "source file")
flag.StringVar(&dst, "dst", "", "destination file")
flag.Parse()
if src == "" || dst == "" {
flag.Usage()
os.Exit(1)
}
err := MoveFile(src, dst)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fmt.Printf("moved %q to %q\n", src, dst)
}
Output (Linux):
$ cp move.file src.file && go build movefile.go && ./movefile -src=src.file -dst=dst.file
moved "src.file" to "dst.file"
$
Output (Windows):
>copy /Y move.file src.file && go build movefile.go && movefile -src=src.file -dst=dst.file
moved "src.file" to "dst.file"
>
This solution Moves the file and preserves permissions:
func MoveFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return fmt.Errorf("Couldn't open source file: %s", err)
}
out, err := os.Create(dst)
if err != nil {
in.Close()
return fmt.Errorf("Couldn't open dest file: %s", err)
}
defer out.Close()
_, err = io.Copy(out, in)
in.Close()
if err != nil {
return fmt.Errorf("Writing to output file failed: %s", err)
}
err = out.Sync()
if err != nil {
return fmt.Errorf("Sync error: %s", err)
}
si, err := os.Stat(src)
if err != nil {
return fmt.Errorf("Stat error: %s", err)
}
err = os.Chmod(dst, si.Mode())
if err != nil {
return fmt.Errorf("Chmod error: %s", err)
}
err = os.Remove(src)
if err != nil {
return fmt.Errorf("Failed removing original file: %s", err)
}
return nil
}
If only want to Copy the file without remove the original:
func CopyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return fmt.Errorf("Couldn't open source file: %s", err)
}
out, err := os.Create(dst)
if err != nil {
in.Close()
return fmt.Errorf("Couldn't open dest file: %s", err)
}
defer out.Close()
_, err = io.Copy(out, in)
in.Close()
if err != nil {
return fmt.Errorf("Writing to output file failed: %s", err)
}
err = out.Sync()
if err != nil {
return fmt.Errorf("Sync error: %s", err)
}
si, err := os.Stat(src)
if err != nil {
return fmt.Errorf("Stat error: %s", err)
}
err = os.Chmod(dst, si.Mode())
if err != nil {
return fmt.Errorf("Chmod error: %s", err)
}
return nil
}
Maybe you can use a magic approach, just using the syscall.MoveFile as follows.
func main() {
oldpath := "D:\\black.txt"
newpath := "E:\\black-new.txt"
from, _ := syscall.UTF16PtrFromString(oldpath)
to, _ := syscall.UTF16PtrFromString(newpath)
fmt.Println(*from, *to)
err := syscall.MoveFile(from, to)
if err != nil {
panic(err)
}
}
the program works.
if you want a cross-platform compatibility program, you can implement your own MoveFile.
func MoveFile(src string, dst string) error {
if runtime.GOOS == "windows" {
from, _ := syscall.UTF16PtrFromString(src)
to, _ := syscall.UTF16PtrFromString(dst)
return syscall.MoveFile(from, to)
} else {
return os.Rename(src, dst)
}
}