How to write log into .log files in golang? - go

I'am new use go language, still learn and need help. How can I write my application logs in go language into ".log" files similiar like PHP phalcon framework ?
I have searching google and get go language tutorials, but there is no simple understanding example. I have try some example but the logs text doesn't write continously. Here's the example that I used to learn for logging log, and I think it's completly wrong.
package main
import (
"bufio"
"fmt"
"io/ioutil"
"os"
"time"
)
func check(e error) {
if e != nil {
panic(e)
}
}
func main() {
now := time.Now()
dt := now.Format("2006-01-02")
dt2 := now.Format("2006-01-02 15:04:05")
// To start, here's how to dump a string (or just
// bytes) into a file.
d1 := []byte("hello\ngo11\n" + dt2)
err := ioutil.WriteFile("/Users/my/Documents/work/src/logs/log-"+dt+".log", d1, 0644)
check(err)
// For more granular writes, open a file for writing.
f, err := os.Create("/Users/my/Documents/work/src/logs/log1.log")
check(err)
// It's idiomatic to defer a `Close` immediately
// after opening a file.
defer f.Close()
// You can `Write` byte slices as you'd expect.
d2 := []byte{115, 111, 109, 101, 10}
n2, err := f.Write(d2)
check(err)
fmt.Printf("wrote %d bytes\n", n2)
// A `WriteString` is also available.
n3, err := f.WriteString("writes\n" + dt)
fmt.Printf("wrote %d bytes\n", n3)
// Issue a `Sync` to flush writes to stable storage.
f.Sync()
// `bufio` provides buffered writers in addition
// to the buffered readers we saw earlier.
w := bufio.NewWriter(f)
n4, err := w.WriteString("buffered\n")
fmt.Printf("wrote %d bytes\n", n4)
// Use `Flush` to ensure all buffered operations have
// been applied to the underlying writer.
w.Flush()
}

You're trying so many different things it's hard to tell what you're aiming for specifically, but if you simply want to write logs to a file, here is an example:
package main
import (
"log"
"os"
)
func main() {
//create your file with desired read/write permissions
f, err := os.OpenFile("filename", os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0644)
if err != nil {
log.Fatal(err)
}
//defer to close when you're done with it, not because you think it's idiomatic!
defer f.Close()
//set output of logs to f
log.SetOutput(f)
//test case
log.Println("check to make sure it works")
}

I recommend taking a look at Lumberjack. It handles rolling logs with several file conditions quite nicely. I'm using it for a streaming network sensor.

I have a small project that uses the log.Logger as following:
package main
import (
"errors"
"log"
"os"
)
const (
logNone = iota
logInfo
logWarning
logError
logVerbose
logDebug
)
type myFileLogger struct {
logger *log.Logger
logFile *os.File
logLevel int
}
func newFileLogger() *myFileLogger {
return &myFileLogger{
logger: nil,
logFile: nil,
logLevel: logNone,
}
}
func (myLogger *myFileLogger) startLog(level int, file string) error {
f, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_APPEND, os.ModePerm)
if err != nil {
return err
}
myLogger.logger = log.New(f, "", 0)
myLogger.logLevel = level
myLogger.logFile = f
return nil
}
func (myLogger *myFileLogger) stopLog() error {
if myLogger.logFile != nil {
return myLogger.logFile.Close()
}
return nil
}
// You can add a log of auxiliary functions here to make the log more easier
func (myLogger *myFileLogger) log(level int, msg string) error {
if myLogger.logger == nil {
return errors.New("myFileLogger is not initialized correctly")
}
if level >= myLogger.logLevel {
myLogger.logger.Print(msg) // maybe you want to include the loglevel here, modify it as you want
}
return nil
}
func main() {
logger := newFileLogger()
if err := logger.startLog(logError, "myLogFile.log"); err != nil {
panic(err.Error())
}
defer func() {
logger.stopLog()
}()
logger.log(logInfo, "Info level log msg\n") // this will be ignored
logger.log(logError, "Error: error message\n") // this should included in the log file
}

import (
"os/exec"
)
func main() {
// check error here...
exec.Command("/bin/sh", "-c", "echo "+err.Error()+" >> log.log").Run()
}

I have my logger do this, here is an example.
Send message sends error to my IRC and sendSMS sends error to my phone.
in my config file I have:
var ErrorFile = "error.log"
type errorLog struct {
}
func (e errorLog) Write(p []byte) (n int, err error) {
fmt.Println("Error: " + string(p))
if config.Verbose == 0 {
file, _ := os.OpenFile(config.ErrorFile, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0666)
file.WriteString(string(p))
sendMSG(string(p))
sendSMS(string(p))
// Close the file when the surrounding function exists
defer file.Close()
}
return n, err
}
// Logger is a helpper method to print out a more useful error message
var Logger = log.New(errorLog{}, "", log.Lmicroseconds|log.Lshortfile)

Related

Writing to file from cmd output

I am trying to write a small code in Go that will collect and save stats from IPFS.
So my Go code will execute IPFS command and save its output in .txt file and keep updating that .txt file.
I am having trouble doing that.
This is my code:
package main
import (
"fmt"
"io"
"log"
"os"
"os/exec"
"time"
)
func ipfsCommand() (ipfsOutput string) {
// output and error
out, err := exec.Command("ipfs","stats","bitswap","--human").Output()
// if there are errors, print/log them
if err != nil {
log.Printf("error!")
log.Fatal(err)
} else {
log.Printf("no error, printing output")
fmt.Printf("%s", out)
}
return
}
func writeToFile(message string) error {
f, err := os.Create("outputTest2_2.txt")
if err != nil {
return err
}
defer f.Close()
l, err := io.WriteString(f, message)
if err != nil {
fmt.Println(err)
f.Close()
return err
}
fmt.Println(l, "bytes written successfully")
return f.Sync()
}
func main() {
// get current time
currentTime := time.Now()
fmt.Println("YYYY.MM.DD : ", currentTime.Format("2006.01.02 15:04:05"))
writeToFile(currentTime)
// get output from ipfs command
msg := ipfsCommand()
// write the output to file
writeToFile(msg)
fmt.Println("file written!!!")
/* // write to file many times
for i:=0;i<3;i++{
// get output from ipfs command
msg := ipfsCommand()
// write the output to file
writeToFile(msg)
}*/
}
When the above code is run, this is the error:
# command-line-arguments
.\test2.go:49:13: cannot use currentTime (type time.Time) as type string in argument to writeToFile
Again, I want to get output from IPFS and save it to .txt file along with current time. I want to do this in loop because I want to save output from IPFS over a long period of time.
I tried to fix your script as is, but it just has too many issues. Here is a
rewrite, maybe you can use it as a new starting point:
package main
import (
"os"
"os/exec"
"time"
)
func main() {
f, err := os.Create("outputTest2_2.txt")
if err != nil {
panic(err)
}
defer f.Close()
currentTime, err := time.Now().MarshalText()
if err != nil {
panic(err)
}
f.Write(append(currentTime, '\n'))
msg, err := exec.Command("go", "env").Output()
if err != nil {
panic(err)
}
f.Write(msg)
}

How to do websocket portforward with filter or save (in array) some forwarded packet in golang?

i found a ready source code on github for portforwarding in golang
here is the code
package main
import (
"io"
"log"
"net"
)
var localServerHost = "localhost:1020"
var remoteServerHost = "192.168.10.1:1020"
func main() {
ln, err := net.Listen("tcp", localServerHost)
if err != nil {
log.Fatal(err)
}
log.Println("Port forwarding server up and listening on ",
localServerHost)
for {
conn, err := ln.Accept()
if err != nil {
log.Fatal(err)
}
go handleConnection(conn)
}
}
func forward(src, dest net.Conn) {
defer src.Close()
defer dest.Close()
io.Copy(src, dest)
}
func handleConnection(c net.Conn) {
log.Println("Connection from : ", c.RemoteAddr())
remote, err := net.Dial("tcp", remoteServerHost)
if err != nil {
log.Fatal(err)
}
log.Println("Connected to ", remoteServerHost)
go forward(c, remote)
go forward(remote, c)
}
i dont have any idea now how can i put flag on user connections, so then i can read data form user connection and save or filter some of packets depend on connection flag
its a wrong way ?
Edit: This is an answer original question about forwarding and filtering TCP. #ermya has since changed the question to ask about WebSockets. This answer is not relevant to what is now a very different question.
You can filter the stream by interposing a reader or writer in the io.Copy operation. Here's how to interpose a reader:
type myFilter struct {
r io.Reader
}
func (f myFilter) Read(p []byte) (int, error) {
n, err := f.r.Read(p)
// Do something with p[:n]. As an example, the following for loop shows how
// to uppercase ASCII letters. Replace this for loop with the filtering of
// your choice.
for i, b := range p[:n] {
if 'a' <= b && b <= 'z' {
p[i] = b - ('a' - 'A')
}
}
return n, err
}
Filter data from net.Conn by replacing the call to go forward(remote, c) with:
go func() {
defer remote.Close()
defer c.Close()
io.Copy(remote, myFilter{c}) // interpose the filter here
}()
As an aside, the naming in forward is confusing because src is actually the destination and dest is the source. Use this:
func forward(dest, src net.Conn) {
defer src.Close()
defer dest.Close()
io.Copy(dest, src)
}

How to read a text file? [duplicate]

This question already has answers here:
How can I read a whole file into a string variable
(7 answers)
Closed 4 years ago.
I'm trying to read "file.txt" and put the contents into a variable using Golang. Here is what I've tried...
package main
import (
"fmt"
"os"
"log"
)
func main() {
file, err := os.Open("file.txt")
if err != nil {
log.Fatal(err)
}
fmt.Print(file)
}
The file gets read successfully and the return from os.Open returns a type of *os.File
It depends on what you are trying to do.
file, err := os.Open("file.txt")
fmt.print(file)
The reason it outputs &{0xc082016240}, is because you are printing the pointer value of a file-descriptor (*os.File), not file-content. To obtain file-content, you may READ from a file-descriptor.
To read all file content(in bytes) to memory, ioutil.ReadAll
package main
import (
"fmt"
"io/ioutil"
"os"
"log"
)
func main() {
file, err := os.Open("file.txt")
if err != nil {
log.Fatal(err)
}
defer func() {
if err = file.Close(); err != nil {
log.Fatal(err)
}
}()
b, err := ioutil.ReadAll(file)
fmt.Print(b)
}
But sometimes, if the file size is big, it might be more memory-efficient to just read in chunks: buffer-size, hence you could use the implementation of io.Reader.Read from *os.File
func main() {
file, err := os.Open("file.txt")
if err != nil {
log.Fatal(err)
}
defer func() {
if err = file.Close(); err != nil {
log.Fatal(err)
}
}()
buf := make([]byte, 32*1024) // define your buffer size here.
for {
n, err := file.Read(buf)
if n > 0 {
fmt.Print(buf[:n]) // your read buffer.
}
if err == io.EOF {
break
}
if err != nil {
log.Printf("read %d bytes: %v", n, err)
break
}
}
}
Otherwise, you could also use the standard util package: bufio, try Scanner. A Scanner reads your file in tokens: separator.
By default, scanner advances the token by newline (of course you can customise how scanner should tokenise your file, learn from here the bufio test).
package main
import (
"fmt"
"os"
"log"
"bufio"
)
func main() {
file, err := os.Open("file.txt")
if err != nil {
log.Fatal(err)
}
defer func() {
if err = file.Close(); err != nil {
log.Fatal(err)
}
}()
scanner := bufio.NewScanner(file)
for scanner.Scan() { // internally, it advances token based on sperator
fmt.Println(scanner.Text()) // token in unicode-char
fmt.Println(scanner.Bytes()) // token in bytes
}
}
Lastly, I would also like to reference you to this awesome site: go-lang file cheatsheet. It encompassed pretty much everything related to working with files in go-lang, hope you'll find it useful.

Proper pattern to encapsulate log setup in golang

When trying to move log setup code into a separate function I ran into inability to hide the destination file object from the main function. In the following INCORRECT simplified example the attempt is made to setup log writing to both Stderr and a file via a single function call:
package main
import (
"io"
"log"
"os"
)
func SetupLogging() {
logFile, err := os.OpenFile("test.log", os.O_APPEND|os.O_CREATE, 0666)
if err != nil {
log.Panicln(err)
}
defer logFile.Close()
log.SetOutput(io.MultiWriter(os.Stderr, logFile))
}
func main() {
SetupLogging()
log.Println("Test message")
}
Clearly is does not work because defer closes the log file at the end of the SetupLogging function.
A working example below adds extra code and IMHO looses some clarity if repeated in a larger application as a pattern:
package main
import (
"io"
"log"
"os"
)
func SetupLogging() *os.File {
logFile, err := os.OpenFile("test.log", os.O_APPEND|os.O_CREATE, 0666)
if err != nil {
log.Panicln(err)
}
log.SetOutput(io.MultiWriter(os.Stderr, logFile))
return logFile
}
func main() {
logf := SetupLogging()
defer logf.Close()
log.Println("Test message")
}
Is there a different way to fully encapsulate open file management into a function, yet still nicely release the handle?
I have now successfully used the below approach for about a year in multiple projects. The idea is to return a function from the setup call. That resulting function contains the destruction logic. Here is an example:
package main
import (
"fmt"
"io"
"log"
"os"
)
func LogSetupAndDestruct() func() {
logFile, err := os.OpenFile("test.log", os.O_CREATE|os.O_APPEND|os.O_RDWR, 0666)
if err != nil {
log.Panicln(err)
}
log.SetOutput(io.MultiWriter(os.Stderr, logFile))
return func() {
e := logFile.Close()
if e != nil {
fmt.Fprintf(os.Stderr, "Problem closing the log file: %s\n", e)
}
}
}
func main() {
defer LogSetupAndDestruct()()
log.Println("Test message")
}
It is using a closure around the cleanup logic being deferred.
A somewhat more elaborate public example of using this approach is in the Viper code: here is the return from a test initializer, and here it is used to encapsulate the cleanup logic and objects
The proper way of doing this is passing the handle in main to SetupLogging:
func SetupLogging(lf *os.File) {
log.SetOutput(io.MultiWriter(os.Stderr, logFile))
log.Println("Started")
}
func main() {
logFile, err := os.OpenFile("test.log", os.O_APPEND|os.O_CREATE, 0666)
if err != nil {
log.Panicln(err)
}
defer logFile.Close()
SetupLogging(logFile)
log.Println("Test message")
}
Another option is to use runtime.SetFinalizer, but it's not always guaranteed to run before main exits.
func SetupLogging() {
logFile, err := os.OpenFile("test.log", os.O_APPEND|os.O_CREATE, 0666)
if err != nil {
log.Panicln(err)
}
runtime.SetFinalizer(logFile, func(h *os.File) {
h.Close()
})
log.SetOutput(io.MultiWriter(os.Stderr, logFile))
}
You can do this using channels, here is my approach
type InfoLog struct {
InfoChan chan string
CloseChan chan struct{} //empty signal
log *log.Logger
file *os.File
}
func NewInfoLog(file *os.File) *InfoLog {
return &InfoLog{
InfoChan: make(chan string),
CloseChan: make(chan struct{}),
log: log.New(file, "TAG", log.Ldate|log.Ltime),
file: file,
}
}
func (i *InfoLog) listen() {
for {
select {
case infoMsg := <-i.InfoChan:
i.log.Println(infoMsg)
case <-i.CloseChan:
i.file.Close()
close(i.InfoChan)
}
}
}
then in main
func main() {
infoLog := NewInfoLog(ANY_OPEN_FILE_HERE)
go infoLog.listen()
infoLog.InfoChan <- "msg"
infoLog.InfoChan <- "msg"
infoLog.InfoChan <- "msg"
infoLog.CloseChan <- struct{}{}
// exits normaly
}
you can see an asynchronous log system i have made for a complete example: https://github.com/sescobb27/ciudad-gourmet/blob/master/services/log_service.go
in case where multiple "teardown" processes are needed, great solution to this is using google context package (https://blog.golang.org/context). advantage is that you can teardown all currently executing procedures with single context. smth like this:
package main
import (
"fmt"
"io"
"log"
"os"
"golang.org/x/net/context"
)
func LogSetup(ctx context.Context) error {
logFile, err := os.OpenFile("test.log", os.O_CREATE|os.O_APPEND|os.O_RDWR, 0666)
if err != nil {
return err
}
log.SetOutput(io.MultiWriter(os.Stderr, logFile))
// here we could f.ex. execute:
// sendLogOutputToExternalService(ctx)
// and it could have it's own teardown procedure
// which would be called on main context's expiration
go func() {
for _ = range ctx.Done() {
err := logFile.Close()
if err = nil {
fmt.Fprintf(os.Stderr, "Problem closing the log file: %s\n", e)
}
}()
return nil
}
func main() {
var stopAll func()
mainContext, stopAll = context.WithCancel(context.Background())
defer stopAll()
err := LogSetup(mainContext)
if err!=nil {
log.Fatal("error while initializing logging")
}
log.Println("Test message")
}

Restricting to Single Instance of Executable with Golang

I need to only allow one instance of my Golang executable at a time. I'm not sure how to use a Global Mutex to make sure no other instances are running.
This would be running on a Windows Machine.
I know this topic is a bit old, but I needed it recently on Windows and I'll post here how I did it in case someone else needs.
Thx to #VonC for pointing me in the right direction.
var (
kernel32 = syscall.NewLazyDLL("kernel32.dll")
procCreateMutex = kernel32.NewProc("CreateMutexW")
)
func CreateMutex(name string) (uintptr, error) {
ret, _, err := procCreateMutex.Call(
0,
0,
uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(name))),
)
switch int(err.(syscall.Errno)) {
case 0:
return ret, nil
default:
return ret, err
}
}
// mutexName starting with "Global\" will work across all user sessions
_, err := CreateMutex("SomeMutexName")
I created a lib with a more complete example: https://github.com/rodolfoag/gow32
Thx!
There doesn't seem to be a cross-platform solution (beside writing a file, and looking for that file at start time).
On Windows, this thread reports
the recommended approach (and the one that has worked great for me) is to use the CreateSemaphore function.
If the name you specify starts with "Global\", then the semaphore is unique for the entire system and a second attempt to open it will fail.
This is a kernel32 call, which has some wrapper in Go available.
kostix adds in the comments:
look at the Go source code around the pkg\syscall hierarchy -- it contains a good wealth of examples on how to call out to DLLs on Windows using syscalls (and that's how you access anything in Windows API).
That would be syscall/dll_windows.go. (And here is a gist)
The odbc package by brainman is another example of direct API calls on Windows -- possibly easier to digest.
Like api/zapi_windows.go.
You could use sockets, simple to use and will work on everything really.
package main
import (
"fmt"
"net"
"os"
"strings"
)
const (
INSTANCE_PORT = 9292
)
func main() {
listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", INSTANCE_PORT))
if err != nil {
if strings.Index(err.Error(), "in use") != -1 {
//optionally send command line arguments to the other instance
fmt.Fprintln(os.Stderr, "Already running.")
return
} else {
panic(err)
}
}
for {
conn, err := listener.Accept()
if err != nil {
println("Error accept:", err.Error())
return
}
go do_something_with(conn)
}
}
You could adapt the code from tendo's python library source
what they do is
for windows :
creating a file made of the executable absolute path (well it's a library, so in your case, you can just define an identifier, to prevent you from "i put the executable in 2 places")
For windows: trying first to remove the file if existing, and if not creating the file with os.O_CREAT | os.O_EXCL | os.O_RDWR
For POSIX compatible systems: trying first to remove the file if existing and if not creating the file and acquiring a lock on it using fcntl.LOCK_EX | fcntl.LOCK_NB
any failure mean the program is already running
and then you can use a defer action to remove the lock (on posix system) and delete the file
Go permit you to create both version wit a build comment to tell which file to compile depending on your OS so you have
for unix system
// +build !windows
package main
import (
"os"
"syscall"
)
func create_lock_file(filename string) (*os.File, error) {
file, err := os.OpenFile(filename, os.O_WRONLY, 0666)
if err != nil {
return nil, err
}
err = syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB)
if err != nil {
return nil, err
}
return file, nil
}
for windows:
// +build !windows
package main
import (
"os"
)
func create_lock_file(filename string) (*os.File, error) {
if _, err := os.Stat(filename); err == nil {
err = os.Remove(filename)
if err != nil {
return nil, err
}
}
return os.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666)
}
and a test
package main
import (
"fmt"
"time"
)
func main() {
_, err := create_lock_file("plop.lock")
if err != nil {
fmt.Println("error ", err.Error())
}
time.Sleep(10 * time.Second)
fmt.Println("end ")
}
I've started a library out of it that you can find here
Improvements to this answer. (I am unsure if this answer will distort the original meaning, so I have written a new answer.)
Features:
deprecated: StringToUTF16Ptr is deprecated. Use UTF16PtrFromString instead.
Add the CloseHandle so that you can cancel the CreateMutexW.
package _test
import (
"syscall"
"testing"
"unsafe"
)
var (
kernel32 = syscall.NewLazyDLL("kernel32.dll")
procCreateMutexW = kernel32.NewProc("CreateMutexW")
procCloseHandle = kernel32.NewProc("CloseHandle")
)
// https://learn.microsoft.com/en-us/windows/win32/api/synchapi/nf-synchapi-createmutexW#return-value
func CreateMutexW(proc *syscall.LazyProc, name string) (uintptr, error) {
if proc.Name != "CreateMutexW" {
panic("proc.Name != CreateMutexW")
}
lpName, _ := syscall.UTF16PtrFromString(name) // LPCWSTR
if handleID, _, err := proc.Call(
0,
0,
uintptr(unsafe.Pointer(lpName)),
); err.(syscall.Errno) == 0 {
return handleID, nil
} else {
return handleID, err
}
}
// https://learn.microsoft.com/en-us/windows/win32/api/handleapi/nf-handleapi-closehandle?redirectedfrom=MSDN
func CloseHandle(proc *syscall.LazyProc, handle uintptr) error {
if proc.Name != "CloseHandle" {
panic("proc.Name != CloseHandle")
}
val, _, err := proc.Call(handle)
if val == 0 {
return err
}
return nil
}
func TestCreateMutexW(t *testing.T) {
handle, err := CreateMutexW(procCreateMutexW, "hello world")
if err != nil {
t.Fatalf(err.Error())
}
_, err = CreateMutexW(procCreateMutexW, "hello world")
if err == nil || err != syscall.ERROR_ALREADY_EXISTS {
t.Error("should panic")
}
if err = CloseHandle(procCloseHandle, handle); err != nil {
t.Error(err)
}
// We can create again since we have closed.
handle, _ = CreateMutexW(procCreateMutexW, "hello world")
if err = CloseHandle(procCloseHandle, handle); err != nil {
t.Error(err)
}
}

Resources