I use the library (http://github.com/fsnotify/fsnotify) to monitor the file system.
I m trying to adjust the repository example to match my requirements, but when i do so the program is not working anymore.
I commented the done channel within the ExampleNewWatcher function
done := make(chan bool)
<-done
As a result, now when i run the example, this channel does not output anything anymore.
event, ok := <-watcher.Events
Complete code:
package main
import (
"github.com/fsnotify/fsnotify"
"log"
"os"
"strconv"
"time"
)
func ExampleNewWatcher() {
watcher, err := fsnotify.NewWatcher()
if err != nil {
log.Fatal(err)
}
defer watcher.Close()
done := make(chan bool) // if i commen this line the `panic` not norking
go func() {
for {
select {
case event, ok := <-watcher.Events:
if !ok {
return
}
log.Println("event:", event)
if event.Op&fsnotify.Write == fsnotify.Write {
log.Println("modified file:", event.Name)
}
panic("just for test") // new output this line
case err, ok := <-watcher.Errors:
if !ok {
return
}
log.Println("error:", err)
}
}
}()
err = watcher.Add("/tmp/foo")
if err != nil {
log.Fatal(err)
}
<-done // comment this
}
func check(err error) {
if err != nil {
panic(err)
}
}
func main() {
// create the test file
var err error
file, err := os.Create("/tmp/foo")
check(err)
_, err = file.Write([]byte("hello world"))
check(err)
stopchan := make(chan struct{})
// test change file
go func() {
for i := 0; i < 10; i++ {
file1, err := os.OpenFile("/tmp/foo", os.O_RDWR, 0644)
check(err)
d := strconv.Itoa(i) + "hello world"
_, err = file1.Write([]byte(d))
check(err)
err = file1.Close()
check(err)
time.Sleep(2 * time.Second) // wait the context writed to the file
}
}()
ExampleNewWatcher() // monitor file
stopchan <- struct{}{}
}
Related
I read this and this and this but none of them solving my issue..
I'm trying to read 2 files async, so I wrote the below:
//readlines.go
package main
import (
"bufio"
"os"
)
// readLines reads a whole file into memory
// and returns a slice of its lines.
func readLines(path string) ([]string, error) {
file, err := os.Open(path)
if err != nil {
return nil, err
}
defer file.Close()
var lines []string
scanner := bufio.NewScanner(file)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}
return lines, scanner.Err()
}
And calling it as:
package main
import (
"fmt"
"os"
"github.com/gocarina/gocsv"
)
func (s *stocks) Read() {
fmt.Println("Reading")
stockFile, err := os.OpenFile("current_invenory.csv", os.O_RDWR|os.O_CREATE, os.ModePerm)
if err != nil {
panic(err)
}
defer stockFile.Close()
stocks := []systemStock{}
if err := gocsv.UnmarshalFile(stockFile, &stocks); err != nil { // Load stocks from file
panic(err)
}
*s = stocks
}
package main
import (
"fmt"
"os"
"github.com/gocarina/gocsv"
)
func (t *transactions) Read() {
fmt.Println("Reading")
trxFile, err := os.OpenFile("current_transactions.csv", os.O_RDWR|os.O_CREATE, os.ModePerm)
if err != nil {
panic(err)
}
defer trxFile.Close()
trx := []systemTransactions{}
if err := gocsv.UnmarshalFile(trxFile, &trx); err != nil { // Load stocks from file
panic(err)
}
*t = trx
}
The above working very fine with:
stock := stocks{}
trx := transactions{}
stock.Read()
trx.Read()
for _, s := range stock {
fmt.Println("Hello", s.Code)
}
But give the error fatal error: all goroutines are asleep - deadlock! when I tried to read them as:
cs, ct := readData()
for _, s := range cs {
fmt.Println("Hello", s.Code)
}
for _, t := range ct {
fmt.Println("Hello trx of ", t.Code)
}
Using
import "sync"
//func readData(cs chan stocks, ct chan transactions) (stocks, transactions) {
func readData() (stocks, transactions) {
var wg sync.WaitGroup
defer wg.Done()
stock := stocks{}
trx := transactions{}
wg.Add(1)
go stock.Read()
wg.Add(1)
go trx.Read()
wg.Wait()
return stock, trx
}
So the error is related for something wrong I made (or do not understand) in the last block~
To run the Read methods for stocks and transactions concurrently, these methods need to have a way of signaling when they are finished executing. This can be done in a lot of ways, but here are two which require the least modifications to your code.
Solution 1
Use the sync.WaitGroup package. With this package, the Read methods should execute wg.Done() statement when they are done with executing. It should look something like this:
func (s *stocks) Read(wg *sync.WaitGroup) {
defer wg.Done()
fmt.Println("Reading")
stockFile, err := os.OpenFile("current_invenory.csv", os.O_RDWR|os.O_CREATE, os.ModePerm)
if err != nil {
panic(err)
}
defer stockFile.Close()
stocks := []systemStock{}
if err := gocsv.UnmarshalFile(stockFile, &stocks); err != nil { // Load stocks from file
panic(err)
}
*s = stocks
}
func (t *transactions) Read(wg *sync.WaitGroup) {
defer wg.Done()
fmt.Println("Reading")
trxFile, err := os.OpenFile("current_transactions.csv", os.O_RDWR|os.O_CREATE, os.ModePerm)
if err != nil {
panic(err)
}
defer trxFile.Close()
trx := []systemTransactions{}
if err := gocsv.UnmarshalFile(trxFile, &trx); err != nil { // Load stocks from file
panic(err)
}
*t = trx
}
func readData() (stocks, transactions) {
var wg sync.WaitGroup
wg.Add(2)
stock := stocks{}
trx := transactions{}
go stock.Read(&wg)
go trx.Read(&wg)
wg.Wait()
return stock, trx
}
Solution 2
This approach uses the golang.org/x/sync/errgroup package. In this case, you do not need to handle the synchronization and signaling yourself, but functions that are added with errgroup.Go method need to have a strict func() error signature. Your code should look like this:
func (s *stocks) Read() error {
fmt.Println("Reading")
stockFile, err := os.OpenFile("current_invenory.csv", os.O_RDWR|os.O_CREATE, os.ModePerm)
if err != nil {
return err
}
defer stockFile.Close()
stocks := []systemStock{}
if err := gocsv.UnmarshalFile(stockFile, &stocks); err != nil { // Load stocks from file
return err
}
*s = stocks
return nil
}
func (t *transactions) Read() error {
fmt.Println("Reading")
trxFile, err := os.OpenFile("current_transactions.csv", os.O_RDWR|os.O_CREATE, os.ModePerm)
if err != nil {
return err
}
defer trxFile.Close()
trx := []systemTransactions{}
if err := gocsv.UnmarshalFile(trxFile, &trx); err != nil { // Load stocks from file
return err
}
*t = trx
return nil
}
func readData() (stocks, transactions) {
g, _ := errgroup.WithContext(context.Background())
stock := stocks{}
trx := transactions{}
g.Go(stock.Read)
g.Go(trx.Read)
if err:= g.Wait(); err != nil {
panic(err)
}
return stock, trx
}
Solution 3
You’re (correctly) adding 1 to the wait group when you start reading from each CSV, bringing the wait group’s internal counter to 2, but wg.Wait() will wait until that counter goes down to zero and you don’t have any calls to wg.Done() to do that. I recommend changing go stock.Read() to:
go func() {
defer wg Done()
stock.Read()
}()
So, the full working code be:
func readData() (stocks, transactions) {
var wg sync.WaitGroup
stock := stocks{}
trx := transactions{}
wg.Add(1)
go func() {
defer wg.Done()
stock.Read()
}()
wg.Add(1)
go func() {
defer wg.Done()
trx.Read()
}()
wg.Wait()
return stock, trx
}
The following snippet validates a phone number and write the details to CSV.
func Parse(phone Input, output *PhoneNumber) error {
var n PhoneNumber
num, _ := phonenumbers.Parse(phone.Number, phone.Prefix)
n.PhoneNumber = phonenumbers.Format(num, phonenumbers.E164)
n.CountryCode = num.GetCountryCode()
n.PhoneType = phonenumbers.GetNumberType(num)
n.NetworkName, _ = phonenumbers.GetCarrierForNumber(num, "EN")
n.Region = phonenumbers.GetRegionCodeForNumber(num)
*output = n
return nil
}
func createFile(path string) {
// detect if file exists
var _, err = os.Stat(path)
// create file if not exists
if os.IsNotExist(err) {
var file, err = os.Create(path)
if err != nil {
return
}
defer file.Close()
}
}
func worker(ctx context.Context, dst chan string, src chan []string) {
for {
select {
case dataArray, ok := <-src: // you must check for readable state of the channel.
if !ok {
return
}
go processNumber(dataArray[0])
case <-ctx.Done(): // if the context is cancelled, quit.
return
}
}
}
func processNumber(number string) {
num, e := phonenumbers.Parse(number, "")
if e != nil {
return
}
region := phonenumbers.GetRegionCodeForNumber(num)
carrier, _ := phonenumbers.GetCarrierForNumber(num, "EN")
path := "sample_all.csv"
createFile(path)
var csvFile, _ = os.OpenFile(path, os.O_APPEND|os.O_WRONLY, os.ModeAppend)
csvwriter := csv.NewWriter(csvFile)
_ = csvwriter.Write([]string{phonenumbers.Format(num, phonenumbers.E164), fmt.Sprintf("%v", num.GetCountryCode()), fmt.Sprintf("%v", phonenumbers.GetNumberType(num)), carrier, region})
defer csvFile.Close()
csvwriter.Flush()
}
func ParseFile(phone Input, output *PhoneNumber) error {
// create a context
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// that cancels at ctrl+C
go onSignal(os.Interrupt, cancel)
numberOfWorkers := 2
start := time.Now()
csvfile, err := os.Open(phone.File)
if err != nil {
log.Fatal(err)
}
defer csvfile.Close()
reader := csv.NewReader(csvfile)
// create the pair of input/output channels for the controller=>workers com.
src := make(chan []string)
out := make(chan string)
// use a waitgroup to manage synchronization
var wg sync.WaitGroup
// declare the workers
for i := 0; i < numberOfWorkers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
worker(ctx, out, src)
}()
}
// read the csv and write it to src
go func() {
for {
record, err := reader.Read()
if err == io.EOF {
break
} else if err != nil {
log.Fatal(err)
}
src <- record // you might select on ctx.Done().
}
close(src) // close src to signal workers that no more job are incoming.
}()
// wait for worker group to finish and close out
go func() {
wg.Wait() // wait for writers to quit.
close(out) // when you close(out) it breaks the below loop.
}()
// drain the output
for res := range out {
fmt.Println(res)
}
fmt.Printf("\n%2fs", time.Since(start).Seconds())
return nil
}
In processNumber function, if I skip writing to CSV, the process of verifying number completes 6 seconds but writing one record at a time on CSV stretch the time consumption to 15s.
How can I optimize the code?
Can I chunk the records and write them in chunks instead of writing one row at a time?
Do work directly in worker goroutine instead of firing off goroutine per task.
Open file output file once. Flush output file once.
func worker(ctx context.Context, dst chan []string, src chan []string) {
for {
select {
case dataArray, ok := <-src: // you must check for readable state of the channel.
if !ok {
return
}
dst <- processNumber(dataArray[0])
case <-ctx.Done(): // if the context is cancelled, quit.
return
}
}
}
func processNumber(number string) []string {
num, e := phonenumbers.Parse(number, "")
if e != nil {
return
}
region := phonenumbers.GetRegionCodeForNumber(num)
carrier, _ := phonenumbers.GetCarrierForNumber(num, "EN")
return []string{phonenumbers.Format(num, phonenumbers.E164), fmt.Sprintf("%v", num.GetCountryCode()), fmt.Sprintf("%v", phonenumbers.GetNumberType(num)), carrier, region}
}
func ParseFile(phone Input, output *PhoneNumber) error {
// create a context
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// that cancels at ctrl+C
go onSignal(os.Interrupt, cancel)
numberOfWorkers := 2
start := time.Now()
csvfile, err := os.Open(phone.File)
if err != nil {
log.Fatal(err)
}
defer csvfile.Close()
reader := csv.NewReader(csvfile)
// create the pair of input/output channels for the controller=>workers com.
src := make(chan []string)
out := make(chan string)
// use a waitgroup to manage synchronization
var wg sync.WaitGroup
// declare the workers
for i := 0; i < numberOfWorkers; i++ {
wg.Add(1)
go func() {
defer wg.Done()
worker(ctx, out, src)
}()
}
// read the csv and write it to src
go func() {
for {
record, err := reader.Read()
if err == io.EOF {
break
} else if err != nil {
log.Fatal(err)
}
src <- record // you might select on ctx.Done().
}
close(src) // close src to signal workers that no more job are incoming.
}()
// wait for worker group to finish and close out
go func() {
wg.Wait() // wait for writers to quit.
close(out) // when you close(out) it breaks the below loop.
}()
path := "sample_all.csv"
file, err := os.Create(path)
if err != nil {
return err
}
defer file.Close()
csvwriter := csv.NewWriter(csvFile)
// drain the output
for res := range out {
csvwriter.Write(res)
}
csvwriter.Flush()
fmt.Printf("\n%2fs", time.Since(start).Seconds())
return nil
}
I am trying to explore Go concurrency. Here Grabber() prints and writes the result of the execution. The program prints the expected result, but does not write it to urls.txt. Can anyone explain to me what i am missing here?
main.go
package main
import (
"bufio"
"fmt"
"io/ioutil"
"log"
"net/http"
"os"
"regexp"
"strings"
"sync"
)
var wg sync.WaitGroup
var mt sync.Mutex
// Final Literation
func main() {
file, err := os.Open("ip.txt")
if err != nil {
log.Fatal(err)
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
go Grabber(scanner.Text())
wg.Add(1)
}
wg.Wait()
if err := scanner.Err(); err != nil {
log.Fatal(err)
}
}
// stringInArray do If string in list return true false otherwise.
func stringInArray(a string, list []string) bool {
for _, b := range list {
if b == a {
return true
}
}
return false
}
// Grabber Do Search the bing and collect array of sitelist
func Grabber(ip string) {
defer wg.Done()
var output []string
outfile, err := os.Create("urls.txt")
if err != nil {
log.Fatal(err)
}
defer outfile.Close()
if ip == "" {
}
page := 1
for page < 251 {
client := &http.Client{}
req, err := http.NewRequest(
http.MethodGet,
fmt.Sprintf(
"http://www.bing.com/search?q=ip:%s+&count=50&first=1",
ip,
),
nil,
)
if err != nil {
}
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 6.1; rv:57.0) Gecko/20100101 Firefox/57.0")
res, err := client.Do(req)
if err != nil {
fmt.Println("Invalid Request")
}
defer res.Body.Close()
body, err := ioutil.ReadAll(res.Body)
if err != nil {
fmt.Println("Couldn't Read")
}
re := regexp.MustCompile(`<h2><a href="(.*?)"`)
links := re.FindAllString(string(body), -1)
if links != nil {
for l := range links {
o := strings.Split(links[l], `"`)
d := strings.Split(o[1], "/")
s := d[0] + "//" + d[2]
if !stringInArray(s, output) {
output = append(output, s)
}
}
}
page = page + 50
}
for _, links := range output {
fmt.Println(links)
fmt.Fprintln(outfile, links)
}
}
Ip.txt as input
103.253.145.129
103.253.146.125
103.253.146.239
103.253.147.72
146.185.176.79
146.185.176.45
146.185.179.250
146.185.180.35
146.185.180.185
146.185.180.113
146.185.181.51
146.185.183.107
146.185.183.202
146.185.183.248
146.185.183.219
146.185.184.69
146.185.185.169
git repo URLGrabber
You are calling create in each goroutine, which will truncate the file. Instead, create the file outside, and serialize the writes to it using another goroutine:
outfile, err := os.Create("urls.txt")
results:=make(chan []string)
go func() {
for output:=range results {
for _, links := range output {
fmt.Fprintln(outfile, links)
}
}
}()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
go Grabber(scanner.Text(), results)
wg.Add(1)
}
wg.Wait()
close(results)
When you get the results in Grabber, instead of writing it to the file, write it to the channel:
results<-output
for _, links := range output {
fmt.Println(links)
}
every one,I am new to golang.I wanna get the data from log file generated by my application.cuz roll-back mechanism, I met some problem.For instance,my target log file is chats.log,it will be renamed to chats.log.2018xxx and a new chats.log will be created.so my go routine that read log file will fail to work.
so I need detect the change and shutdown the previous go routine and then establish the new go routine.
I looked for modules that can help me,and I found
func ExampleNewWatcher(fn string, createnoti chan string, wg sync.WaitGroup) {
wg.Add(1)
defer wg.Done()
watcher, err := fsnotify.NewWatcher()
if err != nil {
log.Fatal(err)
}
defer watcher.Close()
done := make(chan bool)
go func() {
for {
select {
case event := <-watcher.Events:
if event.Op == fsnotify.Create && event.Name==fn{
createnoti <- "has been created"
}
case err := <-watcher.Errors:
log.Println("error:", err)
}
}
}()
err = watcher.Add("./")
if err != nil {
log.Fatal(err)
}
<-done
}
I use fsnotify to detech the change,and make sure the event of file is my log file,and then send some message to a channel.
this is my worker go routine:
func tailer(fn string,isfollow bool, outchan chan string, done <-chan interface{},wg sync.WaitGroup) error {
wg.Add(1)
defer wg.Done()
_, err := os.Stat(fn)
if err != nil{
panic(err)
}
t, err := tail.TailFile(fn, tail.Config{Follow:isfollow})
if err != nil{
panic(err)
}
defer t.Stop()
for line := range t.Lines{
select{
case outchan <- line.Text:
case <- done:
return nil
}
}
return nil
}
I using tail module to read the log file,and I add a done channel to it to shutdown the cycle(I don't know whether I put it in the right way)
And I will send every log content to a channel to consuming it.
So here is the question:how should I put it together?
ps: Actually,I can use some tool to do this job.like apache-flume,but all of those tools need dependency.
Thank you a lot!
Here is a complete example that reloads and rereads the file as it changes or gets deleted and recreated:
package main
import (
"github.com/fsnotify/fsnotify"
"io/ioutil"
"log"
)
const filename = "myfile.txt"
func ReadFile(filename string) string {
data, err := ioutil.ReadFile(filename)
if err != nil {
log.Println(err)
}
return string(data)
}
func main() {
watcher, err := fsnotify.NewWatcher()
if err != nil {
log.Fatal(err)
}
defer watcher.Close()
err = watcher.Add("./")
if err != nil {
log.Fatal(err)
}
for {
select {
case event := <-watcher.Events:
if event.Op == fsnotify.Create && event.Name == filename {
log.Println(ReadFile(filename))
}
case err := <-watcher.Errors:
log.Println("error:", err)
}
}
}
Note this doesn't require goroutines, channels or a WaitGroup. Better to keep things simple and reserve those for when they're actually needed.
I have a closure where I declare and define local variables:
func writer_factory() func() *net.TCPConn {
response_port := "localhost:8000"
tcpAddr_res, err := net.ResolveTCPAddr("tcp4", response_port)
checkError(err)
var response_writer *net.TCPConn
checkError(err)
return func() *net.TCPConn {
if response_writer == nil{
response_writer, err = net.DialTCP("tcp", nil, tcpAddr_res)
checkError(err)
}
return response_writer
}
}
Now my question is, if I call this writer_factory multiple times will I get a memory leak?
So specifically, will I have a memory leak using writer_factory in this program:
package main
import (
"fmt"
"net"
"os"
"strings"
// "io/ioutil"
)
//fmt.Printf("messages are (1) %q\n", messages)
func main() {
end_of_message_terminator := "||"
beginning_of_next_message := ""
request := make([]byte, 512)
service_port := ":7777"
tcpAddr, err := net.ResolveTCPAddr("tcp4", service_port)
checkError(err)
listener, err := net.ListenTCP("tcp", tcpAddr)
checkError(err)
for {
response_writer := writer_factory()
conn, err := listener.Accept()
if err != nil {
continue
}
read_len, err := conn.Read(request)
if read_len == 0 {
continue
}
request_string := string(request[:read_len])
messages := strings.Split(request_string, end_of_message_terminator)
messages[0] = beginning_of_next_message + messages[0]
if messages[len(messages) - 1] != "" {
beginning_of_next_message = messages[len(messages) - 1]
messages[len(messages) - 1] = ""
}
if len(messages) == 1 {
continue
}
rw := response_writer()
join_channel := make(chan struct{})
for i := 0; i < len(messages); i++ {
go func(i int, rw *net.TCPConn){
respond_to_message(messages[i], rw)
join_channel <- struct{}{}
}(i, rw)
}
go func (){
for i := 0; i < len(messages); i++ {
<- join_channel
}
rw.Close()
}()
conn.Close()
}
}
func writer_factory() func() *net.TCPConn {
response_port := "localhost:8000"
tcpAddr_res, err := net.ResolveTCPAddr("tcp4", response_port)
checkError(err)
var response_writer *net.TCPConn
checkError(err)
return func() *net.TCPConn {
if response_writer == nil{
response_writer, err = net.DialTCP("tcp", nil, tcpAddr_res)
checkError(err)
}
return response_writer
}
}
func respond_to_message(message string, response_writer *net.TCPConn){
message_parameters := strings.Split(message, "|")
//response_writer.Write([]byte("asti de chris"))
for i := range message_parameters {
param_parts := strings.Split(message_parameters[i], "=")
param_name := param_parts[0]
//param_value := param_parts[1]
response_writer.Write([]byte(param_name))
//fmt.Println(string(result))
}
}
func checkError(err error) {
if err != nil {
fmt.Fprintf(os.Stderr, "Fatal error: %s", err.Error())
os.Exit(1)
}
}
To exercise this code launch the program, then launch this program netcat -l -p 8000 and then this program printf "asti||" | netcat localhost 7777
You can definitely improve performance by using one connection or (if one connection is not thread save) use a sync.Pool. That would save you memory allocations (and with it garbage collection) and also time wasted on opening all those tcp connections.