i am trying to write to to a file. i read the whole content of the file and now i want to change the content of the file based on some word that i have got from the file. but when i check, the content of the file, it is still the same and it has not change. this is what i used
if strings.Contains(string(read), sam) {
fmt.Println("this file contain that word")
temp := strings.ToUpper(sam)
fmt.Println(temp)
err := ioutil.WriteFile(fi.Name(), []byte(temp), 0644)
} else {
fmt.Println(" the word is not in the file")
}
Considering that your call to ioutil.WriteFile() is consistent with what is used in "Go by Example: Writing Files", this should work.
But that Go by example article check the err just after the write call.
You check the err outside the scope of your test:
if matched {
read, err := ioutil.ReadFile(path)
//fmt.Println(string(read))
fmt.Println(" This is the name of the file", fi.Name())
if strings.Contains(string(read), sam) {
fmt.Println("this file contain that word")
Value := strings.ToUpper(sam)
fmt.Println(Value)
err = ioutil.WriteFile(fi.Name(), []byte(Value), 0644)
} else {
fmt.Println(" the word is not in the file")
}
check(err) <===== too late
}
The err you are testing is the one you got when reading the file (ioutil.ReadFile), because of blocks and scope.
You need to check the error right after the Write call
err = ioutil.WriteFile(fi.Name(), []byte(Value), 0644)
check(err) <===== too late
Since WriteFile overwrite the all file, you could strings.Replace() to replace your word by its upper case equivalent:
r := string(read)
r = strings.Replace(r, sam, strings.ToUpper(sam), -1)
err := ioutil.WriteFile(fi.Name(), []byte(r), 0644)
For a replace which is case insensitive, use a regexp as in "How do I do a case insensitive regular expression in Go?".
The, use func (*Regexp) ReplaceAllString:
re := regexp.MustCompile("(?i)\\b"+sam+"\\b")
r = re.ReplaceAllString(r, strings.ToUpper(sam))
err := ioutil.WriteFile(fi.Name(), []byte(r), 0644)
Note the \b: word boundary to find the any word starting and ending with sam content (instead of finding substrings containing sam content).
If you want to replace substrings, simply drop the \b:
re := regexp.MustCompile("(?i)"+sam)
It's not clear what you want to do. My best guess is something like this:
package main
import (
"bytes"
"errors"
"fmt"
"io/ioutil"
"os"
)
func UpdateWord(filename string, data, word []byte) (int, error) {
n := 0
f, err := os.OpenFile(filename, os.O_WRONLY, 0644)
if err != nil {
return n, err
}
uWord := bytes.ToUpper(word)
if len(word) < len(uWord) {
err := errors.New("Upper case longer than lower case:" + string(word))
return n, err
}
if len(word) > len(uWord) {
uWord = append(uWord, bytes.Repeat([]byte{' '}, len(word))...)[:len(word)]
}
off := int64(0)
for {
i := bytes.Index(data[off:], word)
if i < 0 {
break
}
off += int64(i)
_, err = f.WriteAt(uWord, off)
if err != nil {
return n, err
}
n++
off += int64(len(word))
}
f.Close()
if err != nil {
return n, err
}
return n, nil
}
func main() {
// Test file
filename := `ltoucase.txt`
// Create test file
lcase := []byte(`update a bc def ghij update klmno pqrstu update vwxyz update`)
perm := os.FileMode(0644)
err := ioutil.WriteFile(filename, lcase, perm)
if err != nil {
fmt.Println(err)
return
}
// Read test file
data, err := ioutil.ReadFile(filename)
if err != nil {
fmt.Println(err)
return
}
fmt.Println(string(data))
// Update word in test file
word := []byte("update")
n, err := UpdateWord(filename, data, word)
if err != nil {
fmt.Println(n, err)
return
}
fmt.Println(filename, string(word), n)
data, err = ioutil.ReadFile(filename)
if err != nil {
fmt.Println(err)
return
}
fmt.Println(string(data))
}
Output:
update a bc def ghij update klmno pqrstu update vwxyz update
ltoucase.txt update 4
UPDATE a bc def ghij UPDATE klmno pqrstu UPDATE vwxyz UPDATE
Related
I'm trying to implement a function to ignore a line containing a pattern from a long text file (ASCII guaranteed) in Go
The functions I have below withoutIgnore and withIgnore, both take a filename argument input and return a *byte.Buffer, which can be subsequently used to write to a io.Writer.
The withIgnore function takes an additional argument pattern to exclude the line containing the pattern from the file. The function works, but with benchmarking, found it to be 5x slower than withoutIgnore. Is there a way it could be improved?
package main
import (
"bufio"
"bytes"
"io"
"log"
"os"
)
func withoutIgnore(f string) (*bytes.Buffer, error) {
rfd, err := os.Open(f)
if err != nil {
log.Fatal(err)
}
defer func() {
if err := rfd.Close(); err != nil {
log.Fatal(err)
}
}()
inputBuffer := make([]byte, 1048576)
var bytesRead int
var bs []byte
opBuffer := bytes.NewBuffer(bs)
for {
bytesRead, err = rfd.Read(inputBuffer)
if err == io.EOF {
return opBuffer, nil
}
if err != nil {
return nil, nil
}
_, err = opBuffer.Write(inputBuffer[:bytesRead])
if err != nil {
return nil, err
}
}
return opBuffer, nil
}
func withIgnore(f, pattern string) (*bytes.Buffer, error) {
rfd, err := os.Open(f)
if err != nil {
log.Fatal(err)
}
defer func() {
if err := rfd.Close(); err != nil {
log.Fatal(err)
}
}()
scanner := bufio.NewScanner(rfd)
var bs []byte
buffer := bytes.NewBuffer(bs)
for scanner.Scan() {
if !bytes.Contains(scanner.Bytes(), []byte(pattern)) {
_, err := buffer.WriteString(scanner.Text() + "\n")
if err != nil {
return nil, nil
}
}
}
return buffer, nil
}
func main() {
// buff, err := withoutIgnore("base64dump.log")
buff, err := withIgnore("base64dump.log", "AUDIT")
if err != nil {
log.Fatal(err)
}
_, err = buff.WriteTo(os.Stdout)
if err != nil {
log.Fatal(err)
}
}
Benchmark test
package main
import "testing"
func BenchmarkTestWithoutIgnore(b *testing.B) {
for i := 0; i < b.N; i++ {
_, err := withoutIgnore("base64dump.log")
if err != nil {
b.Fatal(err)
}
}
}
func BenchmarkTestWithIgnore(b *testing.B) {
for i := 0; i < b.N; i++ {
_, err := withIgnore("base64dump.log", "AUDIT")
if err != nil {
b.Fatal(err)
}
}
}
and the "base64dump.log" can be generated in the command line using
base64 /dev/urandom | head -c 10000000 > base64dump.log
Since ASCII is guaranteed, one can work directly at byte level.
Still if one checks each byte for line breaks when reading the input and then searches for the pattern again within the line, operations are applied to each byte.
If, on the other hand, one reads chunks of the input and performs an optimized search for the pattern in the text, not even examining each input byte, one minimizes the operations per input byte.
For example, there is the Boyer-Moore string search algorithm. Go's built-in bytes.Index function is also optimized. The achieved speed depends of course on the input data and the actual pattern. For the input as specified in the question, `bytes.Index turned out to be significantly more performant when measured.
Procedure
read in a chunk, where the chunk size should be significantly longer than the maximum line length, a value >= 64KB should probably be good, in the test 1MB was used as in the question.
a chunk usually doesn't end at a linefeed, so search from the end of the chunk to the next linefeed, limit the search to this slice and remember the remaining data for the next pass
the last chunk does not necessarily end in a linefeed
with the help of the performant GO function bytes.Index you can find the places where the pattern occurs in the chunk
from the found location one searches for the preceding and the following linefeed
then the block is output up to the corresponding beginning of the line
and the search is continued from the end of the line where the pattern occurred
if the search does not find another location, the rest is output
read the next chunk and apply the described steps again until the end of the file is reached
Noteworthy
A read operation may return less data than the chunk size, so it makes sense to repeat the read operation until the chunk size data has been read.
Benchmark
Optimized code is often significantly more complicated, but the performance is also significantly better, as we will see in a moment.
BenchmarkTestWithoutIgnore-8 270 4137267 ns/op
BenchmarkTestWithIgnore-8 54 22403931 ns/op
BenchmarkTestFilter-8 150 7947454 ns/op
Here, the optimized code BenchmarkTestFilter-8 is only about 1.9x slower than the operation without filtering while the BenchmarkTestWithIgnore-8 method is 5.4x slower than the comparison value without filtering.
Looked at another way: the optimized code is 2.8 times faster than the unoptimized one.
Code
Of course, here is the code for your own tests:
func filterFile(f, pattern string) (*bytes.Buffer, error) {
rfd, err := os.Open(f)
if err != nil {
log.Fatal(err)
}
defer func() {
if err := rfd.Close(); err != nil {
log.Fatal(err)
}
}()
reader := bufio.NewReader(rfd)
return filter(reader, []byte(pattern), 1024*1024)
}
// chunkSize must be larger than the longest line
// a reasonable size is probably >= 64K
func filter(reader io.Reader, pattern []byte, chunkSize int) (*bytes.Buffer, error) {
var bs []byte
buffer := bytes.NewBuffer(bs)
chunk := make([]byte, chunkSize)
var remaining []byte
for lastChunk := false; !lastChunk; {
n, err := readChunk(reader, chunk, remaining, chunkSize)
if err != nil {
if err == io.EOF {
lastChunk = true
} else {
return nil, err
}
}
remaining = remaining[:0]
if !lastChunk {
for i := n - 1; i > 0; i-- {
if chunk[i] == '\n' {
remaining = append(remaining, chunk[i+1:n]...)
n = i + 1
break
}
}
}
s := 0
for s < n {
hit := bytes.Index(chunk[s:n], pattern)
if hit < 0 {
break
}
hit += s
startOfLine := hit
for ; startOfLine > 0; startOfLine-- {
if chunk[startOfLine] == '\n' {
startOfLine++
break
}
}
endOfLine := hit + len(pattern)
for ; endOfLine < n; endOfLine++ {
if chunk[endOfLine] == '\n' {
break
}
}
endOfLine++
_, err = buffer.Write(chunk[s:startOfLine])
if err != nil {
return nil, err
}
s = endOfLine
}
if s < n {
_, err = buffer.Write(chunk[s:n])
if err != nil {
return nil, err
}
}
}
return buffer, nil
}
func readChunk(reader io.Reader, chunk, remaining []byte, chunkSize int) (int, error) {
copy(chunk, remaining)
r := len(remaining)
for r < chunkSize {
n, err := reader.Read(chunk[r:])
r += n
if err != nil {
return r, err
}
}
return r, nil
}
And the benchmark part might look something like this:
func BenchmarkTestFilter(b *testing.B) {
for i := 0; i < b.N; i++ {
_, err := filterFile("base64dump.log", "AUDIT")
if err != nil {
b.Fatal(err)
}
}
}
The filter function was split and the actual job is done in func filter(reader io.Reader, pattern []byte, chunkSize int) (*bytes.Buffer, error).
By injecting a reader and a chunkSize, the creation of unit tests is already prepared or contemplated, which is missing here, but is definitely recommended when dealing with indexes.
However, the main point here was to find a way to significantly improve it in terms of performance.
I am trying to read in an mrt (with .bz2 file extension) from archive.routeviews.org namely file - http://archive.routeviews.org/route-views.chile/bgpdata/2022.05/UPDATES/updates.20220501.0000.bz2.
I have found some code online that parses it using three different packages - FGBGP, go-mrt, goBGP. Here is the code:
package main
import (
"bufio"
"bytes"
"fmt"
"log"
"os"
mrt1 "github.com/cloudflare/fgbgp/mrt"
mrt2 "github.com/kaorimatz/go-mrt"
mrt3 "github.com/osrg/gobgp/pkg/packet/mrt"
)
func main() {
data, err := os.ReadFile("updates.20220501.0000")
if err != nil {
log.Fatal(err)
}
// or paste bytes instead data := []byte{}
fmt.Println("FGBGP")
rdr := bytes.NewBuffer(data)
r, err := mrt1.DecodeSingle(rdr)
for r != nil && err == nil {
fmt.Println(r)
r, err = mrt1.DecodeSingle(rdr)
}
fmt.Println("Go-mrt")
rdr2 := mrt2.NewReader(bytes.NewBuffer(data))
r2, err := rdr2.Next()
for r2 != nil && err == nil {
fmt.Println(r2)
r2, err = rdr2.Next()
}
fmt.Println("GoBGP")
sc := bufio.NewScanner(bytes.NewBuffer(data))
sc.Split(mrt3.SplitMrt)
for {
b := sc.Scan()
if !b {
break
}
mrtb := sc.Bytes()
hdr, err := mrt3.NewMRTHeader(0, mrt3.BGP4MP, mrt3.RIB_IPV4_UNICAST, 0)
if err != nil {
fmt.Println(err)
break
}
hdr.DecodeFromBytes(mrtb)
r3, err := mrt3.ParseMRTBody(hdr, mrtb[mrt3.MRT_COMMON_HEADER_LEN:])
if err != nil {
fmt.Println(err)
}
fmt.Println(r3)
}
}
When this is run, FGBGP does not input anything, go-mrt does seem to output the lines but each line seems to be missing things I would like to see such as AS PATH etc. And goBGP which was the one I was most interested in, throws an unsupported type error.
Here is an example output for go-mrt and goBGP:
go-mrt: &{{{0 63786960781 <nil>} 17 4} 27678 6447 0 1 200.16.114.34 200.16.114.60 0x140003f5900}
goBGP: <nil> unsupported type: 17
It seems that image.Decode (line 24) is returning an error when decoding the image. The image "test2.png" exists in the directory of the file, and is a function PNG image. For reference, this code is supposed to create a new image of the same resolution filled with one color randomly selected from test2.png. Any help here would be appreciated.
The code:
package main
import (
"fmt"
"image"
"image/color"
_ "image/jpeg"
"image/png"
"math/rand"
"os"
)
func main() {
file, err := os.Open("test2.png")
if err != nil {
fmt.Println("0 ", err)
return
}
imageInConfig, _, err := image.DecodeConfig(file)
if err != nil {
fmt.Println("1 ", err)
return
}
imageIn, fileType, err := image.Decode(file)
fmt.Println("File type: ", fileType)
if err != nil {
fmt.Println("2 ", err)
return
}
outputWidth := imageInConfig.Width
outputHeight := imageInConfig.Height
imageOut := image.NewRGBA(image.Rectangle{image.Point{0, 0}, image.Point{outputWidth, outputHeight}})
r, g, b, a := imageIn.At(int(rand.Intn(imageInConfig.Width)), int(rand.Intn(imageInConfig.Height))).RGBA()
pixelColor := color.RGBA{uint8(r), uint8(g), uint8(b), uint8(a)}
for x := 0; x < outputWidth; x++ {
for y := 0; y < outputHeight; y++ {
imageOut.Set(x, y, pixelColor)
}
}
fileCreated, _ := os.Create("test2out.png")
png.Encode(fileCreated, imageOut)
}
The error outputted is:
image: unknown format
You are trying to read from the same io.reader(file) twice. The first time in
image.DecodeConfig(file)
and the second time in
image.Decode(file)
The second time you try to read from the same io.reader, you will get EOF
When Read encounters an error or end-of-file condition after successfully reading n > 0 bytes, it returns the number of bytes read. It may return the (non-nil) error from the same call or return the error (and n == 0) from a subsequent call. An instance of this general case is that a Reader returning a non-zero number of bytes at the end of the input stream may return either err == EOF or err == nil. The next Read should return 0, EOF.
Read more about it here https://golang.org/pkg/io/#Reader
Something quick and simple you can do is to open the file twice
file, err := os.Open("test2.png")
if err != nil {
fmt.Println("0 ", err)
return
}
imageInConfig, _, err := image.DecodeConfig(file)
if err != nil {
fmt.Println("1 ", err)
return
}
file2, err := os.Open("test2.png")
if err != nil {
fmt.Println("3 ", err)
return
}
imageIn, _, err := image.Decode(file2)
if err != nil {
fmt.Println("4 ", err)
return
}
Alternatively, you can try reading multiple times from the same file.
How to read multiple times from same io.Reader
I have a empty file called a.txt, I want to output a value(int) to it in a loop, and overwrite last content in file a.txt. For example,
// open a file
f, err := os.Open("test.txt")
if err != nil {
log.Fatal(err)
}
defer f.Close()
// another file
af, err := os.OpenFile("a.txt", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
if err != nil {
log.Fatal(err)
}
defer af.Close()
b := []byte{}
scanner := bufio.NewScanner(f)
for scanner.Scan() {
b = append(b, scanner.Bytes()...)
// how to output len(b) into a.txt?
}
You can also try:
os.OpenFile with custom flags to truncate file, as shown below
package main
import (
"log"
"os"
)
func main() {
f, err := os.OpenFile("notes.txt", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
if err != nil {
log.Fatal(err)
}
if err := f.Close(); err != nil {
log.Fatal(err)
}
}
Just use the truncate method and write again to file starting at the begining.
err = f.Truncate(0)
_, err = f.Seek(0, 0)
_, err = fmt.Fprintf(f, "%d", len(b))
Use os.Create() instead:
f, err := os.Create("test.txt")
From the func's doc:
Create creates or truncates the named file. If the file already exists, it is truncated. If the file does not exist, it is created ...
I have a flat file that has 339276 line of text in it for a size of 62.1 MB. I am attempting to read in all the lines, parse them based on some conditions I have and then insert them into a database.
I originally attempted to use a bufio.Scan() loop and bufio.Text() to get the line but I was running out of buffer space. I switched to using bufio.ReadLine/ReadString/ReadByte (I tried each) and had the same problem with each. I didn't have enough buffer space.
I tried using read and setting the buffer size but as the document says it actually a const that can be made smaller but never bigger that 64*1024 bytes. I then tried to use File.ReadAt where I set the starting postilion and moved it along as I brought in each section to no avail. I have looked at the following examples and explanations (not an exhaustive list):
Read text file into string array (and write)
How to Read last lines from a big file with Go every 10 secs
reading file line by line in go
How do I read in an entire file (either line by line or the whole thing at once) into a slice so I can then go do things to the lines?
Here is some code that I have tried:
file, err := os.Open(feedFolder + value)
handleError(err)
defer file.Close()
// fileInfo, _ := file.Stat()
var linesInFile []string
r := bufio.NewReader(file)
for {
path, err := r.ReadLine("\n") // 0x0A separator = newline
linesInFile = append(linesInFile, path)
if err == io.EOF {
fmt.Printf("End Of File: %s", err)
break
} else if err != nil {
handleError(err) // if you return error
}
}
fmt.Println("Last Line: ", linesInFile[len(linesInFile)-1])
Here is something else I tried:
var fileSize int64 = fileInfo.Size()
fmt.Printf("File Size: %d\t", fileSize)
var bufferSize int64 = 1024 * 60
bytes := make([]byte, bufferSize)
var fullFile []byte
var start int64 = 0
var interationCounter int64 = 1
var currentErr error = nil
for currentErr != io.EOF {
_, currentErr = file.ReadAt(bytes, st)
fullFile = append(fullFile, bytes...)
start = (bufferSize * interationCounter) + 1
interationCounter++
}
fmt.Printf("Err: %s\n", currentErr)
fmt.Printf("fullFile Size: %s\n", len(fullFile))
fmt.Printf("Start: %d", start)
var currentLine []string
for _, value := range fullFile {
if string(value) != "\n" {
currentLine = append(currentLine, string(value))
} else {
singleLine := strings.Join(currentLine, "")
linesInFile = append(linesInFile, singleLine)
currentLine = nil
}
}
I am at a loss. Either I don't understand exactly how the buffer works or I don't understand something else. Thanks for reading.
bufio.Scan() and bufio.Text() in a loop perfectly works for me on a files with much larger size, so I suppose you have lines exceeded buffer capacity. Then
check your line ending
and which Go version you use path, err :=r.ReadLine("\n") // 0x0A separator = newline? Looks like func (b *bufio.Reader) ReadLine() (line []byte, isPrefix bool, err error) has return value isPrefix specifically for your use case
http://golang.org/pkg/bufio/#Reader.ReadLine
It's not clear that it's necessary to read in all the lines before parsing them and inserting them into a database. Try to avoid that.
You have a small file: "a flat file that has 339276 line of text in it for a size of 62.1 MB." For example,
package main
import (
"bytes"
"fmt"
"io"
"io/ioutil"
)
func readLines(filename string) ([]string, error) {
var lines []string
file, err := ioutil.ReadFile(filename)
if err != nil {
return lines, err
}
buf := bytes.NewBuffer(file)
for {
line, err := buf.ReadString('\n')
if len(line) == 0 {
if err != nil {
if err == io.EOF {
break
}
return lines, err
}
}
lines = append(lines, line)
if err != nil && err != io.EOF {
return lines, err
}
}
return lines, nil
}
func main() {
// a flat file that has 339276 lines of text in it for a size of 62.1 MB
filename := "flat.file"
lines, err := readLines(filename)
fmt.Println(len(lines))
if err != nil {
fmt.Println(err)
return
}
}
It seems to me this variant of readLines is shorter and faster than suggested peterSO
func readLines(filename string) (map[int]string, error) {
lines := make(map[int]string)
data, err := ioutil.ReadFile(filename)
if err != nil {
return nil, err
}
for n, line := range strings.Split(string(data), "\n") {
lines[n] = line
}
return lines, nil
}
package main
import (
"fmt"
"os"
"log"
"bufio"
)
func main() {
FileName := "assets/file.txt"
file, err := os.Open(FileName)
if err != nil {
log.Fatal(err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
fmt.Println(scanner.Text())
}
}