How to work with tar command using golang exec.Cmd - go

I have a tar.gz file and i need to unpack it using golang.
I've tried libs like "archive/tar" but they gave me error: archive/tar: invalid tar header. Now my idea was to use exec to run tar command and unpack tarball, but it always exits with code 2.
My code:
func unpack(tarName string) error {
path, _ := os.Getwd()
//err := Untar(path+"/"+tarName, path+"/")
fmt.Printf(path + "/" + tarName)
cmd := exec.Command("tar", "-xfv", path+"/"+tarName)
cmd.Stdout = os.Stdout
err := cmd.Run()
return err
}

If you are trying to compress tar.gz file you need first to decompress gzip.
import (
"archive/tar"
"compress/gzip"
"fmt"
"io"
"log"
"os"
)
func ExtractTarGz(gzipStream io.Reader) {
uncompressedStream, err := gzip.NewReader(gzipStream)
if err != nil {
log.Fatal("ExtractTarGz: NewReader failed")
}
tarReader := tar.NewReader(uncompressedStream)
for true {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
log.Fatalf("ExtractTarGz: Next() failed: %s", err.Error())
}
switch header.Typeflag {
case tar.TypeDir:
if err := os.Mkdir(header.Name, 0755); err != nil {
log.Fatalf("ExtractTarGz: Mkdir() failed: %s", err.Error())
}
case tar.TypeReg:
outFile, err := os.Create(header.Name)
if err != nil {
log.Fatalf("ExtractTarGz: Create() failed: %s", err.Error())
}
if _, err := io.Copy(outFile, tarReader); err != nil {
log.Fatalf("ExtractTarGz: Copy() failed: %s", err.Error())
}
outFile.Close()
default:
log.Fatalf(
"ExtractTarGz: uknown type: %s in %s",
header.Typeflag,
header.Name)
}
}
}
func main() {
r, err := os.Open("./file.tar.gz")
if err != nil {
fmt.Println("error")
}
ExtractTarGz(r)
}

Related

Extracting a tar file from a .tgz file

i have some issue with the package archive/tar.
Using the tarReader.Next() he recognize a file that should be considered as a file as a directory. The issue with that is the fact that it's a .tar file in it and i need it as a file to process it as well.
The fragment of code that recognize the file
header, err := tarReader.Next()
fmt.Println(header.Typeflag)
if err == io.EOF {
break
}
if err != nil {
fmt.Println(err)
os.Exit(1)
}
name := header.Name
fmt.Println(name)
Files:
file.tgz
-file.tar
--other files
Here is the full function
package main
import (
"archive/tar"
"compress/gzip"
"flag"
"fmt"
"io"
"io/ioutil"
"os"
)
func main() {
// get the arguments from the command line
flag.Int("n", 4, "an integer")
flag.Parse()
sourceFile := flag.Arg(0)
if sourceFile == "" {
fmt.Println("Dude, you didn't pass in a tar file!")
os.Exit(1)
}
fmt.Println("arg 1: ", flag.Arg(0))
processFile(sourceFile)
}
// ReadPackedFile is a function to unpack a tar.gz
func ReadPackedFile(filepath string) {
if filepath == "" {
panic("Empty input!")
}
processFile(filepath)
}
func processFile(srcFile string) {
f, err := os.Open(srcFile)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
defer f.Close()
gzf, err := gzip.NewReader(f)
if err != nil {
fmt.Println(err)
os.Exit(1)
}
tarReader := tar.NewReader(gzf)
// defer io.Copy(os.Stdout, tarReader)
for true {
header, err := tarReader.Next()
fmt.Println(header.Typeflag)
if err == io.EOF {
break
}
if err != nil {
fmt.Println(err)
os.Exit(1)
}
name := header.Name
fmt.Println(name)
switch header.Typeflag {
case tar.TypeDir: // = directory
fmt.Println("Directory:", name)
os.Mkdir(name, 0755)
case tar.TypeReg: // = regular file
fmt.Println("Regular file:", name)
data := make([]byte, header.Size)
_, err := tarReader.Read(data)
if err != nil {
panic("Error reading file!!! PANIC!!!!!!")
}
ioutil.WriteFile(name, data, 0755)
default:
fmt.Printf("%s : %c %s %s\n",
"Yikes! Unable to figure out type",
header.Typeflag,
"in file",
name,
)
}
}
}
The error given by the console
PS C:\Users\ErikT\Workspace\Misc\pen\htmltag> go run .\main.go .\dragontail-12.14.1.tgz
arg 1: .\dragontail-12.14.1.tgz
53
/
Directory: /
53
12.14.1/
Directory: 12.14.1/
53
12.14.1/css/
Directory: 12.14.1/css/
48
12.14.1/css/view.css
Regular file: 12.14.1/css/view.css
panic: Error reading file!!! PANIC!!!!!!
goroutine 1 [running]:
main.processFile({0xc000014108?, 0xc000006018?})
C:/Users/ErikT/Workspace/Misc/pen/htmltag/main.go:83 +0x60c
main.main()
C:/Users/ErikT/Workspace/Misc/pen/htmltag/main.go:27 +0x1a7
exit status 2

Convert protobuf serialized messages to JSON without precompiling Go code

I want to convert protobuf serialized messages into a human readable JSON format. The major problem I face is that I need to do this without compiling the proto descriptor into Go code beforehand. I have access to the .proto files at runtime, but not at compile time.
I had the impression that the new Protobuf API v2 (https://github.com/protocolbuffers/protobuf-go) supports dynamic deserialization (see package types/dynamicpb), but I couldn't figure out how to use it apparently:
func readDynamically(in []byte) {
// How do I load the required descriptor (for NewMessage()) from my `addressbook.proto` file?)
descriptor := ??
msg := dynamicpb.NewMessage(descriptor)
err := protojson.Unmarshal(in, msg)
if err != nil {
panic(err)
}
}
Above code is annotated with my problem: How can I get the required descriptor for the dynamicpb.NewMessage() from a .proto file?
Should work like this with the dynamicpb package.
func readDynamically(in []byte) {
registry, err := createProtoRegistry(".", "addressbook.proto")
if err != nil {
panic(err)
}
desc, err := registry.FindFileByPath("addressbook.proto")
if err != nil {
panic(err)
}
fd := desc.Messages()
addressBook := fd.ByName("AddressBook")
msg := dynamicpb.NewMessage(addressBook)
err = proto.Unmarshal(in, msg)
jsonBytes, err := protojson.Marshal(msg)
if err != nil {
panic(err)
}
fmt.Println(string(jsonBytes))
if err != nil {
panic(err)
}
}
func createProtoRegistry(srcDir string, filename string) (*protoregistry.Files, error) {
// Create descriptors using the protoc binary.
// Imported dependencies are included so that the descriptors are self-contained.
tmpFile := filename + "-tmp.pb"
cmd := exec.Command("./protoc/protoc",
"--include_imports",
"--descriptor_set_out=" + tmpFile,
"-I"+srcDir,
path.Join(srcDir, filename))
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
err := cmd.Run()
if err != nil {
return nil, err
}
defer os.Remove(tmpFile)
marshalledDescriptorSet, err := ioutil.ReadFile(tmpFile)
if err != nil {
return nil, err
}
descriptorSet := descriptorpb.FileDescriptorSet{}
err = proto.Unmarshal(marshalledDescriptorSet, &descriptorSet)
if err != nil {
return nil, err
}
files, err := protodesc.NewFiles(&descriptorSet)
if err != nil {
return nil, err
}
return files, nil
}
This question is kind of interesting. I have done some works on protobuf plugs. As far as i can tell, additional cli is needed because we don't want to "reinvent the wheel".
Step one, we need protoc to translate ".proto" file to some format so we can get "protoreflect.MessageDescriptor" easily.
This plug is to get raw bytes which protoc sends to other plugs as input.
package main
import (
"fmt"
"io/ioutil"
"os"
)
func main() {
if len(os.Args) == 2 && os.Args[1] == "--version" {
// fmt.Fprintf(os.Stderr, "%v %v\n", filepath.Base(os.Args[0]), version.String())
os.Exit(0)
}
in, err := ioutil.ReadAll(os.Stdin)
if err != nil {
fmt.Printf("error: %v", err)
return
}
ioutil.WriteFile("./out.pb", in, 0755)
}
build and rename it as protoc-gen-raw, then generate protoc --raw_out=./pb ./server.proto, you will get out.pb. Forget your ".proto" file from now on, and put this "out.pb" where you intend to put ".proto". And what we get is official support with this .pb file.
Step 2: Deserialize a protobuf serialized message into JSON.
package main
import (
"fmt"
"io/ioutil"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/compiler/protogen"
"google.golang.org/protobuf/encoding/protojson"
"google.golang.org/protobuf/types/dynamicpb"
"google.golang.org/protobuf/types/pluginpb"
)
func main() {
in, err := ioutil.ReadFile("./out.pb")
if err != nil {
fmt.Printf("failed to read proto file: %v", err)
return
}
req := &pluginpb.CodeGeneratorRequest{}
if err := proto.Unmarshal(in, req); err != nil {
fmt.Printf("failed to unmarshal proto: %v", err)
return
}
gen, err := protogen.Options{}.New(req)
if err != nil {
fmt.Printf("failed to create new plugin: %v", err)
return
}
// serialize protobuf message "ServerConfig"
data := &ServerConfig{
GameType: 1,
ServerId: 105,
Host: "host.host.host",
Port: 10024,
}
raw, err := data.Marshal()
if err != nil {
fmt.Printf("failed to marshal protobuf: %v", err)
return
}
for _, f := range gen.Files {
for _, m := range f.Messages {
// "ServerConfig" is the message name of the serialized message
if m.GoIdent.GoName == "ServerConfig" {
// m.Desc is MessageDescriptor
msg := dynamicpb.NewMessage(m.Desc)
// unmarshal []byte into proto message
err := proto.Unmarshal(raw, msg)
if err != nil {
fmt.Printf("failed to Unmarshal protobuf data: %v", err)
return
}
// marshal message into json
jsondata, err := protojson.Marshal(msg)
if err != nil {
fmt.Printf("failed to Marshal to json: %v", err)
return
}
fmt.Printf("out: %v", string(jsondata))
}
}
}
}
// the output is:
// out: {"gameType":1, "serverId":105, "host":"host.host.host", "port":10024}

Move a file to a different drive with Go

I'm trying to move a file from my C-drive to my H-drive using os.Replace().
The code looks as follows:
func MoveFile(source string, destination string) {
err := os.Rename(source, destination)
if err != nil {
fmt.Println(err)
}
}
However, when I run the code I get the following error:
rename C:\old\path\to\file.txt H:\new\path\to\file.txt: The system cannot move the file to a different disk drive.
I found this issue on GitHub that specifies the problem but it appears that they will not change this function to allow it to move file on different disk drives.
I already searched for other possibilities to move files, but found nothing in the standard documentation or the internet.
So, what should I do now to be able to move files on different disk drives?
As the comment said, you'll need to create a new file on the other disk, copy the contents, and then remove the original. It's straightforward using os.Create, io.Copy, and os.Remove:
import (
"fmt"
"io"
"os"
)
func MoveFile(sourcePath, destPath string) error {
inputFile, err := os.Open(sourcePath)
if err != nil {
return fmt.Errorf("Couldn't open source file: %s", err)
}
outputFile, err := os.Create(destPath)
if err != nil {
inputFile.Close()
return fmt.Errorf("Couldn't open dest file: %s", err)
}
defer outputFile.Close()
_, err = io.Copy(outputFile, inputFile)
inputFile.Close()
if err != nil {
return fmt.Errorf("Writing to output file failed: %s", err)
}
// The copy was successful, so now delete the original file
err = os.Remove(sourcePath)
if err != nil {
return fmt.Errorf("Failed removing original file: %s", err)
}
return nil
}
You need to make sure that you handle all cases on both Linux and Windows. For example, for any size file,
package main
import (
"flag"
"fmt"
"io"
"os"
)
func MoveFile(source, destination string) (err error) {
src, err := os.Open(source)
if err != nil {
return err
}
defer src.Close()
fi, err := src.Stat()
if err != nil {
return err
}
flag := os.O_WRONLY | os.O_CREATE | os.O_TRUNC
perm := fi.Mode() & os.ModePerm
dst, err := os.OpenFile(destination, flag, perm)
if err != nil {
return err
}
defer dst.Close()
_, err = io.Copy(dst, src)
if err != nil {
dst.Close()
os.Remove(destination)
return err
}
err = dst.Close()
if err != nil {
return err
}
err = src.Close()
if err != nil {
return err
}
err = os.Remove(source)
if err != nil {
return err
}
return nil
}
func main() {
var src, dst string
flag.StringVar(&src, "src", "", "source file")
flag.StringVar(&dst, "dst", "", "destination file")
flag.Parse()
if src == "" || dst == "" {
flag.Usage()
os.Exit(1)
}
err := MoveFile(src, dst)
if err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
fmt.Printf("moved %q to %q\n", src, dst)
}
Output (Linux):
$ cp move.file src.file && go build movefile.go && ./movefile -src=src.file -dst=dst.file
moved "src.file" to "dst.file"
$
Output (Windows):
>copy /Y move.file src.file && go build movefile.go && movefile -src=src.file -dst=dst.file
moved "src.file" to "dst.file"
>
This solution Moves the file and preserves permissions:
func MoveFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return fmt.Errorf("Couldn't open source file: %s", err)
}
out, err := os.Create(dst)
if err != nil {
in.Close()
return fmt.Errorf("Couldn't open dest file: %s", err)
}
defer out.Close()
_, err = io.Copy(out, in)
in.Close()
if err != nil {
return fmt.Errorf("Writing to output file failed: %s", err)
}
err = out.Sync()
if err != nil {
return fmt.Errorf("Sync error: %s", err)
}
si, err := os.Stat(src)
if err != nil {
return fmt.Errorf("Stat error: %s", err)
}
err = os.Chmod(dst, si.Mode())
if err != nil {
return fmt.Errorf("Chmod error: %s", err)
}
err = os.Remove(src)
if err != nil {
return fmt.Errorf("Failed removing original file: %s", err)
}
return nil
}
If only want to Copy the file without remove the original:
func CopyFile(src, dst string) error {
in, err := os.Open(src)
if err != nil {
return fmt.Errorf("Couldn't open source file: %s", err)
}
out, err := os.Create(dst)
if err != nil {
in.Close()
return fmt.Errorf("Couldn't open dest file: %s", err)
}
defer out.Close()
_, err = io.Copy(out, in)
in.Close()
if err != nil {
return fmt.Errorf("Writing to output file failed: %s", err)
}
err = out.Sync()
if err != nil {
return fmt.Errorf("Sync error: %s", err)
}
si, err := os.Stat(src)
if err != nil {
return fmt.Errorf("Stat error: %s", err)
}
err = os.Chmod(dst, si.Mode())
if err != nil {
return fmt.Errorf("Chmod error: %s", err)
}
return nil
}
Maybe you can use a magic approach, just using the syscall.MoveFile as follows.
func main() {
oldpath := "D:\\black.txt"
newpath := "E:\\black-new.txt"
from, _ := syscall.UTF16PtrFromString(oldpath)
to, _ := syscall.UTF16PtrFromString(newpath)
fmt.Println(*from, *to)
err := syscall.MoveFile(from, to)
if err != nil {
panic(err)
}
}
the program works.
if you want a cross-platform compatibility program, you can implement your own MoveFile.
func MoveFile(src string, dst string) error {
if runtime.GOOS == "windows" {
from, _ := syscall.UTF16PtrFromString(src)
to, _ := syscall.UTF16PtrFromString(dst)
return syscall.MoveFile(from, to)
} else {
return os.Rename(src, dst)
}
}

Golang streams and readers

I am writing a simple script to get download unzip the tar.gz file and then remove it. Whenever I try to remove it I get an error:
The process cannot access the file because it is being used by another process.
I assume the error is in how I pass the file to the extractTarGz function, but I am not sure.
Here is the code:
package main
import (
"archive/tar"
"compress/gzip"
"io"
"log"
"os"
)
func main() {
f, err := os.Open("file.tar.gz")
if err != nil {
panic(err)
}
defer f.Close()
extractTarGz(f)
err = os.Remove("file.tar.gz")
}
func extractTarGz(gzipStream io.Reader) {
uncompressedStream, err := gzip.NewReader(gzipStream)
if err != nil {
log.Fatal("ExtractTarGz: NewReader failed")
}
tarReader := tar.NewReader(uncompressedStream)
for true {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
log.Fatalf("ExtractTarGz: Next() failed: %s", err.Error())
}
switch header.Typeflag {
case tar.TypeDir:
if err := os.Mkdir(header.Name, 0755); err != nil {
log.Fatalf("ExtractTarGz: Mkdir() failed: %s", err.Error())
}
case tar.TypeReg:
outFile, err := os.Create(header.Name)
if err != nil {
log.Fatalf("ExtractTarGz: Create() failed: %s", err.Error())
}
defer outFile.Close()
if _, err := io.Copy(outFile, tarReader); err != nil {
log.Fatalf("ExtractTarGz: Copy() failed: %s", err.Error())
}
default:
log.Fatalf(
"ExtractTarGz: uknown type: %s in %s",
header.Typeflag,
header.Name)
}
}
}
You should first close the file, and then attempt to remove it. Since you close it using defer, that will / would be called after the os.Remove() call.
Try it like this:
name := "file.tar.gz"
defer func() {
if err = os.Remove(name); err != nil {
log.Printf("Failed to remove %s: %v", name, err)
}
}()
f, err := os.Open(name)
if err != nil {
panic(err)
}
defer f.Close()
extractTarGz(f)
Deferred functions are executed in LIFO (last-in-first-out) order, so first f.Close() will be called, and then the other which tries to remove the file. Quoting from Spec: Deferred statements:
...deferred functions are invoked immediately before the surrounding function returns, in the reverse order they were deferred.
f, err := os.Open("file.tar.gz")
if err != nil {
panic(err)
}
defer f.Close()
extractTarGz(f)
err = os.Remove("file.tar.gz")
At the very least, you need to close the file before you removeit.
err = f.Close()
if err != nil {
panic(err)
}
err = os.Remove("file.tar.gz")
defer f.Close() won't run until the end of the function.

Read stderr after process finished

I call imagemagick's convert command with some data I have in memory (from html form upload/web server). This works fine, but I'd like to get the error output of convert in case of an error. How can I do that?
This is my code:
package main
import (
"bytes"
"io"
"io/ioutil"
"log"
"os/exec"
"path/filepath"
)
func runImagemagick(data []byte, destfilename string) error {
data_buf := bytes.NewBuffer(data)
cmd := exec.Command("convert", "-", destfilename)
stdin, err := cmd.StdinPipe()
if err != nil {
return err
}
err = cmd.Start()
if err != nil {
return err
}
_, err = io.Copy(stdin, data_buf)
if err != nil {
return err
}
stdin.Close()
err = cmd.Wait()
if err != nil {
return err
}
return nil
}
func main() {
data, err := ioutil.ReadFile("source.gif")
if err != nil {
log.Fatal(err)
}
err = runImagemagick(data, filepath.Join("/tmp", "abc", "dest.png"))
if err != nil {
log.Fatal(err)
}
}
Now the artificial problem is that the directory /tmp/abc/ does not exist. Normally convert would give me this result:
$ convert - /tmp/abc/foo.png < source.gif
convert: unable to open image `/tmp/abc/foo.png': No such file or directory # error/blob.c/OpenBlob/2617.
convert: WriteBlob Failed `/tmp/abc/foo.png' # error/png.c/MagickPNGErrorHandler/1755.
but I don't "see" this error message within my small program. How can I get the error message and show it to my user?
(And another sub-question is: can you give me an advice if this code looks OK? Are there any obvious flaws in it?)
Pipe stdout and stderr too. For example,
package main
import (
"bytes"
"io"
"io/ioutil"
"log"
"os/exec"
"path/filepath"
)
func runImagemagick(data []byte, destfilename string) error {
cmd := exec.Command("convert", "-", destfilename)
stdin, err := cmd.StdinPipe()
if err != nil {
return err
}
stdout, err := cmd.StdoutPipe()
if err != nil {
return err
}
stderr, err := cmd.StderrPipe()
if err != nil {
return err
}
err = cmd.Start()
if err != nil {
return err
}
_, err = io.Copy(stdin, bytes.NewBuffer(data))
if err != nil {
return err
}
stdin.Close()
outData, err := ioutil.ReadAll(stdout)
if err != nil {
return err
}
if len(outData) > 0 {
log.Print(string(outData))
}
errData, err := ioutil.ReadAll(stderr)
if err != nil {
return err
}
if len(errData) > 0 {
log.Print(string(errData))
}
err = cmd.Wait()
if err != nil {
return err
}
return nil
}
func main() {
data, err := ioutil.ReadFile("source.gif")
if err != nil {
log.Fatal(err)
}
err = runImagemagick(data, filepath.Join("/tmp", "abc", "dest.png"))
if err != nil {
log.Fatal(err)
}
}
Output:
2013/03/03 15:02:20 convert.im6: unable to open image `/tmp/abc/dest-0.png': No such file or directory # error/blob.c/OpenBlob/2638.
convert.im6: WriteBlob Failed `/tmp/abc/dest-0.png' # error/png.c/MagickPNGErrorHandler/1728.
2013/03/03 15:02:20 exit status 1
exit status 1
There's no need to use pipes because bytes.Buffer implements the io.Writer interface and so it can be used just fine to collect the program's output:
func runImagemagick(data []byte, destfilename string) error {
cmd := exec.Command("convert", "-", destfilename)
var stdout, stderr bytes.Buffer
cmd.Stdout = &stdout
cmd.Stderr = &stderr
err := cmd.Run()
if err != nil {
if ee, ok := err.(*exec.ExitError); ok {
return &imagemagickError{ee, stdout.Bytes(), stderr.Bytes()}
} else {
return err
}
}
if stderr.Len() > 0 {
return errors.New(fmt.Sprintf("imagemagick wrote to stderr: %s", stderr.Bytes()))
}
if stdout.Len() > 0 {
log.Print(stdout.Bytes())
}
return nil
}

Resources