Replacement for *os.File of Go default log - go

I'm trying to write a wrapper for Go's built-in logger.
This is to have compatibility.
package main
import (
"log"
"os"
)
var(
mylog *log.Logger
)
func main() {
mylog = log.New(os.Stdout, "", 0)
mylog.Printf("test")
}
Instead of using os.Stdout, I want to create one something. Similar to os.Stdout but prints with prefix like below.
package main
import(
"log"
"mylibrary"
)
var(
mylog *log.Logger
)
func main() {
mylog = log.New(mylibrary.Prefix, "", 0)
mylog.Printf("test")
}
Basically, I still want to have *log.Logger while having custom log. Can someone give me a hint how I can make this works?
Currently, I'm using following to do that. But I bet there's a better way.
func NewIoWriter(f func(string)) *io.PipeWriter {
r, w := io.Pipe()
go func() {
scanner := bufio.NewScanner(r)
for scanner.Scan() {
f(scanner.Text())
}
if err := scanner.Err(); err != nil {
f(err.Error())
}
r.Close()
}()
runtime.SetFinalizer(w, (*io.PipeWriter).Close)
return w
}
What would be the better way to make it work?
Thank you

How about something like this:
type myLogWriter struct {
logFunc func(string)
line string
}
func (w *myLogWriter) Write(b []byte) (int, error) {
l := len(b)
for len(b) != 0 {
i := bytes.Index(b, []byte{'\n'})
if i == -1 {
w.line += string(b)
break
} else {
w.logFunc(w.line + string(b[:i]))
b = b[i+1:]
w.line = ""
}
}
return l, nil
}
func NewLogWriter(f func(string)) *myLogWriter {
return &myLogWriter{
logFunc: f,
}
}
See https://play.golang.org/p/L6PG1gCK1er.

Related

log printf and Write connection

I cannot for the life of me from below golang code, why when you call write function at the bottom,
func write(message string) {
log.Printf("%v\n", message)
}
why log.Printf calls below method
func (fl fileLog) Write(data []byte) (int, error ) {
fmt.Println("does this ever get called?")
f, err := os.OpenFile(string(fl), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
if err != nil {
return 0, err
}
defer f.Close()
return f.Write(data)
}
Logger's printf definition is below
func (*Logger) Printf ΒΆ
func (l *Logger) Printf(format string, v ...interface{})
-- Full Code below, someone please explain to me why this is so please--
package main
import (
"fmt"
stlog "log"
"os"
)
var log *stlog.Logger
type fileLog string
func (fl fileLog) Write(data []byte) (int, error ) {
fmt.Println("does this ever get called?")
f, err := os.OpenFile(string(fl), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0600)
if err != nil {
return 0, err
}
defer f.Close()
return f.Write(data)
}
func registerHandlers() {
var msgRaw = "how are you"
write(msgRaw)
}
func run(destination string) {
log = stlog.New(fileLog(destination), "", stlog.LstdFlags)
}
func main() {
fmt.Println("here we go")
run("../test.log")
registerHandlers()
}
func write(message string) {
log.Printf("%v\n", message)
}
The logger writes the log messages to an io.Writer, which is an interface defined as:
type Writer interface {
Write([]byte) (int,error)
}
The fileLog type implements the io.Writer interface, and you set it as the output of the new logger. So whenever the logger tries to write a log, it calls the Write function of its writer, which is fileLog.Write.

Golang segment stdin stream to file

instead of writing a pipe to a huge file i want to segment the stream in chunks on signal USR1. i think i got the basics working but the app just hangs and nothing happens, any clues or best practices when handling with an uncontrollable input stream and byte perfect segmentation?
package main
import (
"bufio"
"fmt"
"io"
"os"
"os/signal"
"syscall"
"time"
)
var done bool
func handle(c chan os.Signal) {
for {
sig := <-c
switch sig {
case syscall.SIGUSR1:
fmt.Println("###Sink temporarily_closed###")
done = true
case syscall.SIGUSR2:
fmt.Println("###Sink closed###")
done = true
case syscall.SIGHUP:
fmt.Println("###Sink running###")
}
}
}
func check(e error) {
if e != nil {
panic(e)
}
}
func main() {
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGUSR1, syscall.SIGUSR2, syscall.SIGHUP)
go handle(c)
reader := bufio.NewReaderSize(os.Stdin,1024*10)
for true {
if done {
file, err := os.Create("./temp.file")
check(err)
writer := bufio.NewWriter(file)
written, err := io.Copy(writer,reader)
check(err)
fmt.Println(written)
writer.Flush()
file.Close()
reader.Reset(os.Stdin)
done = false
}
time.Sleep(time.Millisecond)
}
}
So you need to io.CopyN(dst, src, 4096) in the loop and rotate the file once in a while. See example. I made rotation by size but it is easy to add signal handling.
package main
import (
"fmt"
"io"
"log"
"os"
"time"
)
var count int
var f *os.File
func rotate() *os.File {
if f != nil {
if err := f.Close(); err != nil {
log.Fatal(err)
}
}
fname := fmt.Sprintf("./dump-%d.bin", count)
count++
f, err := os.Create(fname)
if err != nil {
log.Fatal(err)
}
log.Println("rotated:", fname)
return f
}
func main() {
var n, written int
reader := os.Stdin
for {
if written == 0 || written >= 4096*10 {
f = rotate()
written = 0
}
n, err := io.CopyN(f, reader, 4096)
if err != nil {
log.Fatal(err)
}
written += n
log.Println("written:", written)
time.Sleep(time.Millisecond * 500)
}
}

How to get string input with ReadLine()?

I want to get string from stdio with func (b *Reader) ReadLine() (line []byte, isPrefix bool, err error) but my code doent work correctly.
I'm leaning about golang. I want to know about how to get string from standard input with ReadLine()
I know, fmt.Scan or Scanner help me, but I want to use ReadLine()
package main
import (
"bufio"
"fmt"
"io"
"os"
"strconv"
)
var sc = bufio.NewScanner(os.Stdin)
var rd = bufio.NewReaderSize(os.Stdin, 1000000)
func nextInt() int {
sc.Scan()
i, e := strconv.Atoi(sc.Text())
if e != nil {
panic(e)
}
return i
}
func nextLine() string {
buf := make([]byte, 0, 1000000)
for {
line, isPrefix, err := rd.ReadLine()
if err == io.EOF {
break
} else if err != nil {
panic(err)
}
buf = append(buf, line...)
if !isPrefix {
break
}
}
return string(buf)
}
func main() {
var s string
var a int
s = nextLine()
a = nextInt()
fmt.Println(s)
fmt.Println(a)
}
Result
$ ./a.out
test # input
334 # input
test
334
$ cat in.txt
test
334
$ ./a.out < in.txt
panic: strconv.Atoi: parsing "": invalid syntax
goroutine 1 [running]:
main.nextInt(0xc042056088)
I expect the two output should have been same,
but when I use redirection, it didn't work and get different output.
Get rid of the scanner (you already said you prefer ReadLine()) and change your nextInt() function to call nextLine() like this:
func nextInt() int {
i, e := strconv.Atoi(nextLine())
if e != nil {
panic(e)
}
return i
}
(BTW It's not a good idea to panic on bad user input but I assume this is just a test and you wouldn't do that for production code :)
You could try not using a scanner, perhaps like this
Readline will get the number for you, just convert it
package main
import (
"bufio"
"fmt"
"io"
"os"
"strconv"
)
var rd = bufio.NewReaderSize(os.Stdin, 1000000)
func nextLine() string {
buf := make([]byte, 0, 1000000)
for {
line, isPrefix, err := rd.ReadLine()
if err == io.EOF {
break
} else if err != nil {
panic(err)
}
buf = append(buf, line...)
if !isPrefix {
break
}
}
return string(buf)
}
func main() {
var s string
var a int
s = nextLine()
fmt.Println(s)
s = nextLine()
a, e := strconv.Atoi(s)
if e != nil {
panic(e)
}
fmt.Println(a)
}

Golang - how this func is called

I'm working with this sample code but cannot understand how this function is called and which parameter is belong.
go func(r []string) {
processData(r)
ch <- r
}(record)
function closures:
Go functions may be closures. A closure is a function value that
references variables from outside its body. The function may access
and assign to the referenced variables; in this sense the function is
"bound" to the variables.
To Understand this:
go func(r []string) {
processData(r)
ch <- r
}(record)
let's first declare this function:
func routine(r []string) {
processData(r)
ch <- r
}
and this global variable:
var ch = make(chan []string)
Now you may call it:
go routine(record)
this calls function named routine with input parameter named record as a goroutine.
And see: https://gobyexample.com/goroutines
Try it on The Go Playground:
package main
import (
"encoding/csv"
"flag"
"fmt"
"io"
"os"
"strings"
"time"
)
func routine(r []string) {
processData(r)
ch <- r
}
var ch = make(chan []string)
func main() {
start := time.Now()
flag.Parse()
fmt.Print(strings.Join(flag.Args(), "\n"))
if *filename == "REQUIRED" {
return
}
csvfile, err := os.Open(*filename)
if err != nil {
fmt.Println(err)
return
}
defer csvfile.Close()
reader := csv.NewReader(csvfile)
i := 0
for {
record, err := reader.Read()
if err == io.EOF {
break
} else if err != nil {
fmt.Println(err)
return
}
i++
go routine(record)
fmt.Printf("go %d %s\n", i, record)
}
for ; i >= 0; i-- {
fmt.Printf("<- %d %s\n", i, <-ch)
}
fmt.Printf("\n%2fs", time.Since(start).Seconds())
}
func processData([]string) {
time.Sleep(10 * time.Millisecond)
}
var filename = flag.String("f", "REQUIRED", "source CSV file")
var numChannels = flag.Int("c", 4, "num of parallel channels")
//var bufferedChannels = flag.Bool("b", false, "enable buffered channels")

inconsistent results using golang channels

I a task written in Go to get a unique list from a bunch of text files. I put in some parallelization using channels and am having inconsistent results now - a variance of 5 records output/not output each time with the same input files.
The am testing it with go run process.go | wc -l on Fedora x86_64, go1.1.2, 8 core amd.
The code is:
package main
import (
"fmt"
"os"
"io"
"encoding/csv"
"regexp"
"log"
)
var (
cleanRe *regexp.Regexp = regexp.MustCompile("[^0-9]+")
comma rune ='\t'
fieldsPerRecord=-1
)
func clean(s string) string {
clean:=cleanRe.ReplaceAllLiteralString(s,"")
if len(clean)<6 {return ""}
return clean
}
func uniqueChannel(inputChan chan []string, controlChan chan string) {
defer func(){controlChan<-"Input digester."}()
uniq:=make(map[string]map[string]bool)
i:=0
for record:= range inputChan {
i++
id,v:=record[0],record[1]
if uniq[id]==nil {
uniq[id]=make(map[string]bool)
} else if !uniq[id][v] {
uniq[id][v]=true
fmt.Println(id,string(comma),v)
}
}
log.Println("digest ", i)
}
func processFile(fileName string, outputChan chan []string, controlChan chan string) {
defer func(){controlChan<-fileName}()
f,err:=os.Open(fileName)
if err!=nil{log.Fatal(err)}
r:=csv.NewReader(f)
r.FieldsPerRecord = fieldsPerRecord
r.Comma = comma
// Process the records
i:=0
for record,err:=r.Read();err!=io.EOF;record,err=r.Read() {
if err!=nil{continue}
id:=record[0]
for _,v:=range record[1:] {
if cleanV:=clean(v);cleanV!=""{
i++
outputChan<-[]string{id,cleanV}
}
}
}
log.Println(fileName,i)
}
func main() {
inputs:=[]string{}
recordChan:=make(chan []string,100)
processesLeft:=len(inputs)+1
controlChan:=make(chan string,processesLeft)
// Ingest the inputs
for _,fName:=range inputs {
go processFile(fName,recordChan,controlChan)
}
// This is the loop to ensure it's all unique
go uniqueChannel(recordChan,controlChan)
// Make sure all the channels close up
for processesLeft>0 {
if processesLeft==1{
close(recordChan)
}
c:=<-controlChan
log.Println(c)
processesLeft--
}
close(controlChan)
}
It seems like it closes the channel before it's empty and quite. Without the closing mechanism I was getting deadlocks - I'm out of ideas.
You could ditch the control channel and use a sync.WaitGroup:
package main
import (
"encoding/csv"
"fmt"
"io"
"log"
"os"
"regexp"
"sync"
)
var (
cleanRe *regexp.Regexp = regexp.MustCompile("[^0-9]+")
comma rune = '\t'
fieldsPerRecord = -1
)
func clean(s string) string {
clean := cleanRe.ReplaceAllLiteralString(s, "")
if len(clean) < 6 {
return ""
}
return clean
}
func uniqueChannel(inputChan chan []string) {
uniq := make(map[string]map[string]bool)
i := 0
for record := range inputChan {
i++
id, v := record[0], record[1]
if uniq[id] == nil {
uniq[id] = make(map[string]bool)
} else if !uniq[id][v] {
uniq[id][v] = true
fmt.Println(id, string(comma), v)
}
}
log.Println("digest ", i)
}
func processFile(fileName string, outputChan chan []string) {
f, err := os.Open(fileName)
if err != nil {
log.Fatal(err)
}
r := csv.NewReader(f)
r.FieldsPerRecord = fieldsPerRecord
r.Comma = comma
// Process the records
for record, err := r.Read(); err != io.EOF; record, err = r.Read() {
if err != nil {
continue
}
id := record[0]
for _, v := range record[1:] {
if cleanV := clean(v); cleanV != "" {
outputChan <- []string{id, cleanV}
}
}
}
}
func main() {
inputs := []string{"ex.tsv"}
recordChan := make(chan []string)
var wg sync.WaitGroup
// Ingest the inputs
for _, fName := range inputs {
wg.Add(1)
go func() {
processFile(fName, recordChan)
wg.Done()
}()
}
go func() {
wg.Wait()
close(recordChan)
}()
// This is the loop to ensure it's all unique
uniqueChannel(recordChan)
}

Resources