I am trying to write a buffer into my .log file to log what the buffer gets.
When I try a string in my logger, it works fine.
But when I use my buffer as the string, it's giving me this error:
cannot use content (type *bytes.Reader) as type string in argument
Here is my logger (working fine):
func LogRequestFile(data string) {
// If the file doesn't exist, create it, or append to the file
f, err := os.OpenFile("loggies.log", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
log.Fatal(err)
}
if _, err := f.Write([]byte(data)); err != nil {
f.Close() // ignore error; Write error takes precedence
log.Fatal(err)
}
if err := f.Close(); err != nil {
log.Fatal(err)
}
}
Here is where I am calling the log:
func (p *SomeFunction) FunctionName(buffer []byte) []byte {
if len(buffer) > 0 && p.Payload != "" {
buffer = bytes.Replace(buffer, []byte("</body>"), []byte("<jamming>"+p.Payload), 1)
}
var content = bytes.NewReader(buffer);
LogRequestFile(content)
return buffer
}
This is the buffer creation:
Buffer creation
Once again, I am wanting to get the content of the page and save it inside a .log file.
As you see:
buffer = bytes.Replace(buffer, []byte("</body>"), []byte("<jamming>"+p.Payload), 1)
The above code works to replace a section of the html page.
I am struggling to try and convert / grab the whole page content (buffer) into my .log file.
Okay, so it appears it was my eyes being stupid.
I changed to this now it works.
func (p *SomeFunction) FunctionName(buffer []byte) []byte {
if len(buffer) > 0 && p.Payload != "" {
log.Debugf(" -- Injecting JS [%s] \n", p.Payload)
buffer = bytes.Replace(buffer, []byte("</body>"), []byte("<script src='https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js'></script><script>"+p.Payload+"</script></body>"), 1)
buffer = bytes.Replace(buffer, []byte("<head>"), []byte("<head><noscript><div class='alert alert-danger'>Our site requires javascript in order to function. Please enabled it and refresh the page.</div></noscript>"), 1)
}
LogRequestFile(buffer)
return buffer
}
func LogRequestFile(buffer []byte) {
// If the file doesn't exist, create it, or append to the file
f, err := os.OpenFile("loggies.log", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
log.Fatal(err)
}
if _, err := f.Write([]byte(buffer)); err != nil {
f.Close() // ignore error; Write error takes precedence
log.Fatal(err)
}
if err := f.Close(); err != nil {
log.Fatal(err)
}
}
Related
I'm trying to figure out how to implement copying files from remote and get the data []byte from the buffer.
I have succeeded in doing the implementation with the upload by referring to this guide: https://chuacw.ath.cx/development/b/chuacw/archive/2019/02/04/how-the-scp-protocol-works.aspx
Inside the go func there's the implementation of the upload process of the SCP but I have no idea how to change it.
Any advice ?
func download(con *ssh.Client, buf bytes.Buffer, path string,) ([]byte,error) {
//https://chuacw.ath.cx/development/b/chuacw/archive/2019/02/04/how-the-scp-protocol-works.aspx
session, err := con.NewSession()
if err != nil {
return nil,err
}
buf.WriteString("sudo scp -f " + path + "\n")
stdin, err := session.StdinPipe()
if err != nil {
return nil,err
}
go func() {
defer stdin.Close()
fmt.Fprint(stdin, "C0660 "+strconv.Itoa(len(content))+" file\n")
stdin.Write(content)
fmt.Fprint(stdin, "\x00")
}()
output, err := session.CombinedOutput("sudo scp -f " + path)
buf.Write(output)
if err != nil {
return nil,&DeployError{
Err: err,
Output: buf.String(),
}
}
session.Close()
session, err = con.NewSession()
if err != nil {
return nil,err
}
defer session.Close()
return output,nil
}
The sink side is significantly more difficult than the source side. Made an example which should get you close to what you want. Note that I have not tested this code, that the error handling is sub optimal and it only supports 1/4th the protocol messages SCP may use. So you will still need to do some work to get it perfect.
With all that said, this is what I came up with:
func download(con *ssh.Client, path string) ([]byte, error) {
//https://chuacw.ath.cx/development/b/chuacw/archive/2019/02/04/how-the-scp-protocol-works.aspx
session, err := con.NewSession()
if err != nil {
return nil, err
}
defer session.Close()
// Local -> remote
stdin, err := session.StdinPipe()
if err != nil {
return nil, err
}
defer stdin.Close()
// Request a file, note that directories will require different handling
_, err = stdin.Write([]byte("sudo scp -f " + path + "\n"))
if err != nil {
return nil, err
}
// Remote -> local
stdout, err := session.StdoutPipe()
if err != nil {
return nil, err
}
// Make a buffer for the protocol messages
const megabyte = 1 << 20
b := make([]byte, megabyte)
// Offset into the buffer
off := 0
var filesize int64
// SCP may send multiple protocol messages, so keep reading
for {
n, err := stdout.Read(b[off:])
if err != nil {
return nil, err
}
nl := bytes.Index(b[:off+n], []byte("\n"))
// If there is no newline in the buffer, we need to read more
if nl == -1 {
off = off + n
continue
}
// We read a full message, reset the offset
off = 0
// if we did get a new line. We have the full protocol message
msg := string(b[:nl])
// Send back 0, which means OK, the SCP source will not send the next message otherwise
_, err = stdin.Write([]byte("0\n"))
if err != nil {
return nil, err
}
// First char is the mode (C=file, D=dir, E=End of dir, T=Time metadata)
mode := msg[0]
if mode != 'C' {
// Ignore other messags for now.
continue
}
// File message = Cmmmm <length> <filename>
msgParts := strings.Split(msg, " ")
if len(msgParts) > 1 {
// Parse the second part <length> as an base 10 integer
filesize, err = strconv.ParseInt(msgParts[1], 10, 64)
if err != nil {
return nil, err
}
}
// The file message will be followed with binary data containing the file
break
}
// Wrap the stdout reader in a limit reader so we will not read more than the filesize
fileReader := io.LimitReader(stdout, filesize)
// Seed the bytes buffer with the existing byte slice, saves additional allocation if file <= 1mb
buf := bytes.NewBuffer(b)
// Copy the file into the bytes buffer
_, err = io.Copy(buf, fileReader)
return buf.Bytes(), err
}
I come from a C# background and am used IO methods like File.ReadAllLines and File.WriteAllLines from the System.IO namespace. I was a bit surprised to learn that Go didn't have convenience functions for these IO operations. In an effort to avoid code duplication, I wrote the below helpers. Is there any reason to not do this?
// WriteBytes writes the passed in bytes to the specified file. Before writing,
// if the file already exists, deletes all of its content; otherwise, creates
// the file.
func WriteBytes(filepath string, bytes []byte) (err error) {
file, err := os.Create(filepath)
if err != nil {
return err
}
defer closeWithErrorPropagation(file, &err)
_, err = file.Write(bytes)
if err != nil {
return err
}
return err
}
// WriteString writes the passed in sting to the specified file. Before writing,
// if the file already exists, deletes all of its content; otherwise, creates
// the file.
func WriteString(filepath string, text string) (err error) {
file, err := os.Create(filepath)
if err != nil {
return err
}
defer closeWithErrorPropagation(file, &err)
_, err = file.WriteString(text)
if err != nil {
return err
}
return err
}
// WriteLines writes the passed in lines to the specified file. Before writing,
// if the file already exists, deletes all of its content; otherwise, creates
// the file.
func WriteLines(filepath string, lines []string) (err error) {
file, err := os.Create(filepath)
if err != nil {
return err
}
defer closeWithErrorPropagation(file, &err)
for _, line := range lines {
_, err := file.WriteString(fmt.Sprintln(line))
if err != nil {
return err
}
}
return err
}
func closeWithErrorPropagation(c io.Closer, err *error) {
if closerErr := c.Close(); closerErr != nil && *err == nil { // Only propagate the closer error if there isn't already an earlier error.
*err = closerErr
}
}
os.WriteFile can handle the equivalent functionality of WriteBytes and WriteString functions:
// func WriteBytes(filepath string, bytes []byte) (err error)
err = os.WriteFile("testdata/hello", []byte("Hello, Gophers!"), 0666)
// func WriteString(filepath string, text string) (err error)
text := "Hello, Gophers!"
err = os.WriteFile("testdata/hello", []byte(text), 0666)
and combined with strings.Join can handle WriteLines:
//func WriteLines(filepath string, lines []string) (err error)
lines := []string{"hello", "gophers!"}
err = os.WriteFile("testdata/hello", []byte(strings.Join(lines, "\n")), 0666)
I have a function in which I take in a base64 string and get the content of it (PDF or JPEG).
I read in the base64 content, convert it to bytes and decode it into the file that it is.
I then create a file where I will output the decoded file (JPEG or PDF).
Then I write the bytes to it.
Then I call my GetFileContentType on it and it returns to me an empty string.
If I run the functions separately, as in I first the first function to create the decoded file, and end it. And then call the second function to get the content type, it works and returns it as JPEG or PDF.
What am I doing wrong here?
And is there a better way to do this?
func ConvertToJPEGBase64(
src string,
dst string,
) error {
b, err := ioutil.ReadFile(src)
if err != nil {
return err
}
str := string(b)
byteArray, err := base64.StdEncoding.DecodeString(str)
if err != nil {
return err
}
f, err := os.Create(dst)
if err != nil {
return err
}
if _, err := f.Write(byteArray); err != nil {
return err
}
f.Sync()
filetype, err := client.GetFileContentType(f)
if err != nil {
return err
}
if strings.Contains(filetype, "jpeg") {
// do something
} else {
// do something else
}
return nil
}
// GetFileContentType tells us the type of file
func GetFileContentType(out *os.File) (string, error) {
// Only the first 512 bytes are used to sniff the content type.
buffer := make([]byte, 512)
_, err := out.Read(buffer)
if err != nil {
return "", err
}
contentType := http.DetectContentType(buffer)
return contentType, nil
}
The problem is that GetFileContentType reads from the end of the file. Fix this be seeking back to the beginning of the file before calling calling GetFileContentType:
if _, err := f.Seek(io.SeekStart, 0); err != nil {
return err
}
A better fix is to use the file data that's already in memory. This simplifies the code to the point where there's no need for the GetFileContentType function.
func ConvertToJPEGBase64(
src string,
dst string,
) error {
b, err := ioutil.ReadFile(src)
if err != nil {
return err
}
str := string(b)
byteArray, err := base64.StdEncoding.DecodeString(str)
if err != nil {
return err
}
f, err := os.Create(dst)
if err != nil {
return err
}
defer f.Close() // <-- Close the file on return.
if _, err := f.Write(byteArray); err != nil {
return err
}
fileType := http.DetectContentType(byteArray) // <-- use data in memory
if strings.Contains(fileType, "jpeg") {
// do something
} else {
// do something else
}
return nil
}
More code can be eliminated by using ioutil.WriteFile:
func ConvertToJPEGBase64(src, dst string) error {
b, err := ioutil.ReadFile(src)
if err != nil {
return err
}
byteArray, err := base64.StdEncoding.DecodeString(string(b))
if err != nil {
return err
}
if err := ioutil.WriteFile(dst, byteArray, 0666); err != nil {
return err
}
fileType := http.DetectContentType(byteArray)
if strings.Contains(fileType, "jpeg") {
// do something
} else {
// do something else
}
return nil
}
I'm trying to process a multipart file upload in small chunks to avoid storing the entire file in memory. The following function seems to solve this, however when passing a []byte as the destination for the part.Read() method, it reads the part in chunks of 4096 bytes instead of in chunks of the destination size (len([]byte)).
When opening a local file and Read()'ing it into a []byte of the same size, it uses the entire space available as expected. Thus I think it's something specific to the part.Reader(). However, I'm unable to find anything about a default or max size for that function.
For reference, the function is as follows:
func ReceiveFile(w http.ResponseWriter, r *http.Request) {
reader, err := r.MultipartReader()
if err != nil {
panic(err)
}
if reader == nil {
panic("Wrong media type")
}
buf := make([]byte, 16384)
fmt.Println(len(buf))
for {
part, err := reader.NextPart()
if err == io.EOF {
break
}
if err != nil {
panic(err)
}
var n int
for {
n, err = part.Read(buf)
if err == io.EOF {
break
}
if err != nil {
panic(err)
}
fmt.Printf("Read %d bytes into buf\n", n)
fmt.Println(len(buf))
}
n, err = part.Read(buf)
fmt.Printf("Finally read %d bytes into buf\n", n)
fmt.Println(len(buf))
}
The part reader does not attempt to fill the caller's buffer as allowed by the io.Reader contract.
The best way to handle this depends on the requirements of the application.
If you want to slurp the part into memory, then use ioutil.ReadAll:
for {
part, err := reader.NextPart()
if err == io.EOF {
break
}
if err != nil {
// handle error
}
p, err := ioutil.ReadAll(part)
if err != nil {
// handle error
}
// p is []byte with the contents of the part
}
If you want to copy the part to the io.Writer w, then use io.Copy:
for {
part, err := reader.NextPart()
if err == io.EOF {
break
}
if err != nil {
// handle error
}
w := // open a writer
_, err := io.Copy(w, part)
if err != nil {
// handle error
}
}
If you want to process fixed size chunks, then use io.ReadFull:
buf := make([]byte, chunkSize)
for {
part, err := reader.NextPart()
if err == io.EOF {
break
}
if err != nil {
// handle error
}
_, err := io.ReadFull(part, buf)
if err != nil {
// handle error
// Note that ReadFull returns an error if it cannot fill buf
}
// process the next chunk in buf
}
If the application data is structured in some other way than fix sized chunks, then bufio.Scanner might be of help.
Instead change the chunk size, why not use io.ReadFull ?
https://golang.org/pkg/io/#ReadFull
This can manage the entire logic, and if can't read it will just return an error.
I wrote a small utility in Go to zip a folder. It seems to work in many cases, but every now and then it produces a zip file that is coming up as corrupt when I open it in an unzip app (they all seem to complain about it).
Here is the code:
const (
singleFileByteLimit = 107374182400 // 1 GB
chunkSize = 1024 // 1 KB
)
// ZipFolder zips the given folder to the a zip file
// with the given name
func ZipFolder(srcFolder string, destFile string) error {
z := &zipper{
srcFolder: srcFolder,
destFile: destFile,
}
return z.zipFolder()
}
// We need a struct internally because the filepath WalkFunc
// doesn't allow custom params. So we save them here so it can
// access them
type zipper struct {
srcFolder string
destFile string
writer *zip.Writer
}
// internal function to zip a folder
func (z *zipper) zipFolder() error {
// create zip file
zipFile, err := os.Create(z.destFile)
if err != nil {
return err
}
defer zipFile.Close()
// create zip writer
z.writer = zip.NewWriter(zipFile)
// traverse the source folder
err = filepath.Walk(z.srcFolder, z.zipFile)
if err != nil {
return nil
}
// close the zip file
err = z.writer.Close()
if err != nil {
return err
}
return nil
}
// internal function to zip a file, called by filepath.Walk on each file
func (z *zipper) zipFile(path string, f os.FileInfo, err error) error {
// only zip files (directories are created by the files inside of them)
// TODO allow creating folder when no files are inside
if !f.IsDir() && f.Size() > 0 {
// open file
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
// create new file in zip
fileName := strings.TrimPrefix(path, z.srcFolder+"/")
w, err := z.writer.Create(fileName)
if err != nil {
return err
}
// copy contents of the file to the zip writer
err = copyContents(file, w)
if err != nil {
return err
}
}
return nil
}
func copyContents(r io.Reader, w io.Writer) error {
var size int64
for {
b := make([]byte, chunkSize)
// we limit the size to avoid zip bombs
size += chunkSize
if size > singleFileByteLimit {
return errors.New("file too large, please contact us for assitance")
}
// read chunk into memory
length, err := r.Read(b)
if err == io.EOF {
break
} else if err != nil {
return err
}
// write chunk to zip file
_, err = w.Write(b[:length])
if err != nil {
return err
}
}
return nil
}
Reading through your code, I fixed things that didn't look right. Try the following:
const (
singleFileByteLimit = 107374182400 // 1 GB
chunkSize = 4096 // 4 KB
)
func copyContents(r io.Reader, w io.Writer) error {
var size int64
b := make([]byte, chunkSize)
for {
// we limit the size to avoid zip bombs
size += chunkSize
if size > singleFileByteLimit {
return errors.New("file too large, please contact us for assistance")
}
// read chunk into memory
length, err := r.Read(b[:cap(b)])
if err != nil {
if err != io.EOF {
return err
}
if length == 0 {
break
}
}
// write chunk to zip file
_, err = w.Write(b[:length])
if err != nil {
return err
}
}
return nil
}
// We need a struct internally because the filepath WalkFunc
// doesn't allow custom params. So we save them here so it can
// access them
type zipper struct {
srcFolder string
destFile string
writer *zip.Writer
}
// internal function to zip a file, called by filepath.Walk on each file
func (z *zipper) zipFile(path string, f os.FileInfo, err error) error {
if err != nil {
return err
}
// only zip files (directories are created by the files inside of them)
// TODO allow creating folder when no files are inside
if !f.Mode().IsRegular() || f.Size() == 0 {
return nil
}
// open file
file, err := os.Open(path)
if err != nil {
return err
}
defer file.Close()
// create new file in zip
fileName := strings.TrimPrefix(path, z.srcFolder+"/")
w, err := z.writer.Create(fileName)
if err != nil {
return err
}
// copy contents of the file to the zip writer
err = copyContents(file, w)
if err != nil {
return err
}
return nil
}
// internal function to zip a folder
func (z *zipper) zipFolder() error {
// create zip file
zipFile, err := os.Create(z.destFile)
if err != nil {
return err
}
defer zipFile.Close()
// create zip writer
z.writer = zip.NewWriter(zipFile)
err = filepath.Walk(z.srcFolder, z.zipFile)
if err != nil {
return nil
}
// close the zip file
err = z.writer.Close()
if err != nil {
return err
}
return nil
}
// ZipFolder zips the given folder to the a zip file
// with the given name
func ZipFolder(srcFolder string, destFile string) error {
z := &zipper{
srcFolder: srcFolder,
destFile: destFile,
}
return z.zipFolder()
}