I have the following script that generates random sequences and hashes them on the cpu with several threads in go.
package main
import(
"fmt"
"crypto/sha256"
"encoding/hex"
"math/rand"
"time"
"log"
"os"
)
func randChr(i int)(string){
i = i + 65
if i>90 {
i = i - 43
}
return string(i)
}
func randStr(random *rand.Rand, length int)(string){
result := ""
for len(result)<length{
result = result + randChr(random.Intn(36))
}
return result
}
func HashPass(data []byte) (bool,[32]byte){
hash := sha256.Sum256(data)
s := hex.EncodeToString(hash[:])
pass := true
for i := 0; i<7; i++ {
if s[i] != s[i+1]{
pass = false
break;
}
}
return pass,hash
}
func getAPassingHash()(string){
randSource := rand.NewSource(time.Now().UnixNano())
random := rand.New(randSource)
passes := false
s := ""
for !passes {
s=randStr(random,64)
passes,_ = HashPass([]byte(s))
}
return(s)
}
func worker(ch chan string){
ch <- getAPassingHash()
}
func timer(ch chan string,wait time.Duration){
time.Sleep(wait)
ch <- "End"
}
func append(fn string,conts string){
f, err := os.OpenFile(fn,
os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {
log.Println(err)
}
defer f.Close()
if _, err := f.WriteString(conts); err != nil {
log.Println(err)
}
}
func main(){
ch := make(chan string)
go timer(ch,6*time.Hour)
for i:=0;i<9;i++{
time.Sleep(time.Second)
go worker(ch)
}
for true{
result := <-ch
if result == "End"{
break;
}
go worker(ch)
fmt.Println(result)
hash := sha256.Sum256([]byte(result))
fmt.Println(hex.EncodeToString(hash[:]))
fmt.Println()
append("hashes.txt","\n"+result+"\n"+hex.EncodeToString(hash[:])+"\n")
}
fmt.Println("done")
}
For some reason the script hangs up occasionally until I click on the command prompt and hit enter. I'm not sure where it is getting stuck but I can see that my system CPU utilization goes down and I know I have results somehow blocking out the program, I hit enter and results print and cpu usage spikes back up. I know this might be hard to replicate but I would really appreciate any suggestions.
It doesn't hang, it waits for the value of passes to be true
func getAPassingHash()(string){
randSource := rand.NewSource(time.Now().UnixNano())
random := rand.New(randSource)
passes := false
s := ""
for !passes { // wait for value is true
s=randStr(random,64)
passes,_ = HashPass([]byte(s))
}
return(s)
}
Related
I want to have the limited number of goroutines that make some computation (func worker(), it makes some computation and places the result in a channel). Also a have another channel, that has "jobs" for my workers. As a result I can see that all jobs were computed correctly, but after computation executions stucks.
package main
import (
"bufio"
"fmt"
"os"
"net/http"
"io/ioutil"
"strings"
"time"
)
func worker(id int, urls <- chan string, results chan<- int) {
var data string
for url := range urls {
fmt.Println("worker", id, "started job", url)
if (strings.HasPrefix(url, "http") || strings.HasPrefix(url, "https")) {
resp, err := http.Get(url)
if err != nil {
fmt.Println(err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
fmt.Println(err)
}
data = string(body)
} else {
body, err := ioutil.ReadFile(url)
if err != nil {
fmt.Println(err)
}
data = string(body)
}
number := strings.Count(data, "Go")
fmt.Println("worker", id, "finished job", url, "Number of Go is", number)
results <- number
}
return
}
func main() {
final_result := 0
maxNbConcurrentGoroutines := 5
numJobs := 0
urls := make(chan string)
results := make(chan int)
scanner := bufio.NewScanner(os.Stdin)
start := time.Now()
for w := 1; w <= maxNbConcurrentGoroutines; w++ {
go worker(w, urls, results)
}
for scanner.Scan() {
url := (scanner.Text())
urls <- url
numJobs += 1
}
close(urls)
for num := range results {
final_result += num
}
t := time.Now()
elapsed := t.Sub(start)
for i := 1; i <= numJobs; i++ {
one_result := <- results
final_result += one_result
}
fmt.Println("Number = ", final_result)
fmt.Println("Time = ", elapsed)
if err := scanner.Err(); err != nil {
fmt.Fprintln(os.Stderr, "error:", err)
os.Exit(1)
}
}
I tried to use https://gobyexample.com/worker-pools to extract all the values from results channel, but was not succeed. What should I do to have it unstacked and gone further. Here is an example of how to run it:
echo -e 'https://golang.org\n/etc/passwd\nhttps://golang.org\nhttps://golang.org' | go run 1.go
Your program doesn't return because it waits the closed status of results channel.
In https://gobyexample.com/worker-pools the loop for getting results is different:
for a := 1; a <= numJobs; a++ {
<-results
}
If you want to use for num := range results you need close(results) and determine when to call it.
You can view another example using WaitGroup at https://gobyexample.com/waitgroups
I have three commands to run, but I'd like to make sure the two first are running before running the third one.
Currently, it does run A and B then C.
I run A and B in goroutines
I communicate their name through chan if there's no stderr
the main functions pushes the names received through chan into a slice
once the slice contains all names of module A and B it starts C
Some context
I'm in the process of learning goroutines and chan as a hobbyist. It's not clear to me how to output exec.Command("foo", "bar").Run() in a reliable way while it's running. It's not clear either how to handle errors received by each process through chan.
The reason why I need A and B to run before C is because A and B are graphql microservices, C needs them to run in order to get their schemas through HTTP and start doing some graphql federation (f.k.a. graphql stitching)
Inconsistencies
With my current approach, I will know if A and B are running only if they print something I guess.
I don't like that each subsequent stdout will hit an if statement, just to know if the process is running.
My error handling is not as clean as I'd like it to be.
Question
How could I have a more reliable way to ensure that A and B are running, event if they don't print anything and that they did not throw errors?
package main
import (
"bufio"
"fmt"
"log"
"os/exec"
"reflect"
"sort"
"strings"
"sync"
)
var wg sync.WaitGroup
var modulesToRun = []string{"micro-post", "micro-hello"}
func main() {
// Send multiple values to chan
// https://stackoverflow.com/a/50857250/9077800
c := make(chan func() (string, error))
go runModule([]string{"go", "run", "micro-post"}, c) // PROCESS A
go runModule([]string{"go", "run", "micro-hello"}, c) // PROCESS B
modulesRunning := []string{}
for {
msg, err := (<-c)()
if err != nil {
log.Fatalln(err)
}
if strings.HasPrefix(msg, "micro-") && err == nil {
modulesRunning = append(modulesRunning, msg)
if CompareUnorderedSlices(modulesToRun, modulesRunning) {
go runModule([]string{"go", "run", "micro-federation"}, c) // PROCESS C
}
}
}
}
func runModule(commandArgs []string, o chan func() (string, error)) {
cmd := exec.Command(commandArgs[0], commandArgs[1], commandArgs[2]+"/main.go")
// Less verbose solution to stream output with io?
// var stdBuffer bytes.Buffer
// mw := io.MultiWriter(os.Stdout, &stdBuffer)
// cmd.Stdout = mw
// cmd.Stderr = mw
c := make(chan struct{})
wg.Add(1)
// Stream command output
// https://stackoverflow.com/a/38870609/9077800
go func(cmd *exec.Cmd, c chan struct{}) {
defer wg.Done()
stdout, err := cmd.StdoutPipe()
if err != nil {
close(o)
panic(err)
}
stderr, err := cmd.StderrPipe()
if err != nil {
close(o)
panic(err)
}
<-c
outScanner := bufio.NewScanner(stdout)
for outScanner.Scan() {
m := outScanner.Text()
fmt.Println(commandArgs[2]+":", m)
o <- (func() (string, error) { return commandArgs[2], nil })
}
errScanner := bufio.NewScanner(stderr)
for errScanner.Scan() {
m := errScanner.Text()
fmt.Println(commandArgs[2]+":", m)
o <- (func() (string, error) { return "bad", nil })
}
}(cmd, c)
c <- struct{}{}
cmd.Start()
wg.Wait()
close(o)
}
// CompareUnorderedSlices orders slices before comparing them
func CompareUnorderedSlices(a, b []string) bool {
if len(a) != len(b) {
return false
}
sort.Strings(a)
sort.Strings(b)
return reflect.DeepEqual(a, b)
}
About process management
Starting the process is the action of calling the binary path with its arguments.
It will fail if the bin path is not found, or some malformed arguments syntax is provided.
As a consequence you might start a process with success, but receive an exit error because somehow its execution fails.
Those details are important to figure out if you need only to startup the process to consider the operation as successful or dig further its state and/or output.
In your code it appears you wait for the first line of stderr to be printed to consider it as started, without any consideration to the content being printed.
It resemble more to a kind of sleeping time to ensure the process has initialized.
Consider that starting the binary happens much faster in comparison to the execution of its bootstrap sequence.
About the code, your exit rules are unclear. What is keeping main from exiting ?
In the current code it will exit before C is executed when A and B has started (not anylising other cases)
Your implementation of job concurrency in main is not standard. It is missing the loop to collect results, quit and close(chan).
The chan signature is awkward, i would rather use a struct {Module string, Err error}
The runModule function is buggy. It might close(o) while another routine might attempt to write it. If starts fails, you are not returning any error signal.
A somewhat solution might look like this, consider it as being opinniated and depending the binary run other strategies can/should be implemented to detect error over the standard FDs.
package main
import (
"bufio"
"fmt"
"log"
"os"
"os/exec"
"strings"
"sync"
"time"
)
type cmd struct {
Module string
Cmd string
Args []string
Err error
}
func main() {
torun := []cmd{
cmd{
Module: "A",
Cmd: "ping",
Args: []string{"8.8.8.8"},
},
cmd{
Module: "B",
Cmd: "ping",
// Args: []string{"8.8.8.8.9"},
Args: []string{"8.8.8.8"},
},
}
var wg sync.WaitGroup // use a waitgroup to ensure all concurrent jobs are done
wg.Add(len(torun))
out := make(chan cmd) // a channel to output cmd status
go func() {
wg.Wait() //wait for the group to finish
close(out) // then close the signal channel
}()
// start the commands
for _, c := range torun {
// go runCmd(c, out, &wg)
go runCmdAndWaitForSomeOutput(c, out, &wg)
}
// loop over the chan to collect errors
// it ends when wg.Wait unfreeze and closes out
for c := range out {
if c.Err != nil {
log.Fatalf("%v %v has failed with %v", c.Cmd, c.Args, c.Err)
}
}
// here all commands started you can proceed further to run the last command
fmt.Println("all done")
os.Exit(0)
}
func runCmd(o cmd, out chan cmd, wg *sync.WaitGroup) {
defer wg.Done()
cmd := exec.Command(o.Cmd, o.Args...)
if err := cmd.Start(); err != nil {
o.Err = err // save err
out <- o // signal completion error
return // return to unfreeze the waitgroup wg
}
go cmd.Wait() // dont wait for command completion,
// consider its done once the program started with success.
// out <- o // useless as main look ups only for error
}
func runCmdAndWaitForSomeOutput(o cmd, out chan cmd, wg *sync.WaitGroup) {
defer wg.Done()
cmd := exec.Command(o.Cmd, o.Args...)
stdout, err := cmd.StdoutPipe()
if err != nil {
o.Err = err // save err
out <- o // signal completion
return // return to unfreeze the waitgroup wg
}
stderr, err := cmd.StderrPipe()
if err != nil {
o.Err = err
out <- o
return
}
if err := cmd.Start(); err != nil {
o.Err = err
out <- o
return
}
go cmd.Wait() // dont wait for command completion
// build a concurrent fd's scanner
outScan := make(chan error) // to signal errors detected on the fd
var wg2 sync.WaitGroup
wg2.Add(2) // the number of fds being watched
go func() {
defer wg2.Done()
sc := bufio.NewScanner(stdout)
for sc.Scan() {
line := sc.Text()
if strings.Contains(line, "icmp_seq") { // the OK marker
return // quit asap to unfreeze wg2
} else if strings.Contains(line, "not known") { // the nOK marker, if any...
outScan <- fmt.Errorf("%v", line)
return // quit to unfreeze wg2
}
}
}()
go func() {
defer wg2.Done()
sc := bufio.NewScanner(stderr)
for sc.Scan() {
line := sc.Text()
if strings.Contains(line, "icmp_seq") { // the OK marker
return // quit asap to unfreeze wg2
} else if strings.Contains(line, "not known") { // the nOK marker, if any...
outScan <- fmt.Errorf("%v", line) // signal error
return // quit to unfreeze wg2
}
}
}()
go func() {
wg2.Wait() // consider that if the program does not output anything,
// or never prints ok/nok, this will block forever
close(outScan) // close the chan so the next loop is finite
}()
// - simple timeout less loop
// for err := range outScan {
// if err != nil {
// o.Err = err // save the execution error
// out <- o // signal the cmd
// return // qui to unfreeze the wait group wg
// }
// }
// - more complex version with timeout
timeout := time.After(time.Second * 3)
for {
select {
case err, ok := <-outScan:
if !ok { // if !ok, outScan is closed and we should quit the loop
return
}
if err != nil {
o.Err = err // save the execution error
out <- o // signal the cmd
return // quit to unfreeze the wait group wg
}
case <-timeout:
o.Err = fmt.Errorf("timed out...%v", timeout) // save the execution error
out <- o // signal the cmd
return // quit to unfreeze the wait group wg
}
}
// exit and unfreeze the wait group wg
}
How can I read a file in Go and skip the first line / headers?
In Python I know I could do
counter = 0
with open("my_file_path", "r") as fo:
try:
next(fo)
except:
pass
for _ in fo:
counter = counter + 1
This is my Go application
package main
import (
"bufio"
"flag"
"os"
)
func readFile(fileLocation string) int {
fileOpen, _ := os.Open(fileLocation)
defer fileOpen.Close()
fileScanner := bufio.NewScanner(fileOpen)
counter := 0
for fileScanner.Scan() {
//fmt.Println(fileScanner.Text())
counter = counter + 1
}
return counter
}
func main() {
fileLocation := flag.String("file_location", "default value", "file path to count lines")
flag.Parse()
counted := readFile(*fileLocation)
println(counted)
}
I will be reading a huge file and don't want to be evaluating each line if the index is 0.
How about to move to the next token before the loop
scanner := bufio.NewScanner(file)
scanner.Scan() // this moves to the next token
for scanner.Scan() {
fmt.Println(scanner.Text())
}
file
1
2
3
output
2
3
https://play.golang.org/p/I2w50zFdcg0
For example,
package main
import (
"bufio"
"fmt"
"os"
)
func readFile(filename string) (int, error) {
f, err := os.Open(filename)
if err != nil {
return 0, err
}
defer f.Close()
count := 0
s := bufio.NewScanner(f)
if s.Scan() {
for s.Scan() {
count++
}
}
if err := s.Err(); err != nil {
return 0, err
}
return count, nil
}
func main() {
filename := `test.file`
count, err := readFile(filename)
if err != nil {
fmt.Fprintln(os.Stderr, err)
return
}
fmt.Println(count)
}
Output:
$ cat test.file
1234567890
abc
$ go run count.go
1
$
you can try something like this
func readFile(fileLocation string) int {
fileOpen, _ := os.Open(fileLocation)
defer fileOpen.Close()
fileScanner := bufio.NewScanner(fileOpen)
counter := 0
for fileScanner.Scan() {
// read first line and ignore
fileScanner.Text()
break
}
for fileScanner.Scan() {
// read remaining lines and process
txt := fileScanner.Text()
counter = counter + 1
// do something with text
}
return counter
}
Edit:
func readFile(fileLocation string) int {
fileOpen, _ := os.Open(fileLocation)
defer fileOpen.Close()
fileScanner := bufio.NewScanner(fileOpen)
counter := 0
if fileScanner.Scan() {
// read first line and ignore
fileScanner.Text()
}
for fileScanner.Scan() {
// read remaining lines and process
txt := fileScanner.Text()
// do something with text
counter = counter + 1
}
return counter
}
package main
import (
"encoding/json"
"fmt"
"/something/models"
"os"
"path/filepath"
"runtime"
)
func WriteDeviceToFile(d chan *models.Device, fileName string) {
_, b, _, _ := runtime.Caller(0)
basepath := filepath.Dir(b)
filePath := basepath + "/dataFile/" + fileName
var f *os.File
var err error
f, _ = os.OpenFile(filePath, os.O_APPEND|os.O_WRONLY, 0600)
defer f.Close()
for device := range d {
deviceB, err := json.Marshal(device)
fmt.Println(string(deviceB))
if err == nil {
if _, err = f.WriteString(string(deviceB)); err != nil {
panic(err)
}
} else {
panic(err)
}
}
}
func main() {
deviceChan := make(chan *models.Device)
go WriteDeviceToFile(deviceChan, "notalive.txt")
d := models.NewDevice("12346", "")
deviceChan <- d
d = models.NewDevice("abcd", "")
deviceChan <- d
close(deviceChan)
}
This only works with at least two devices sent to channel. With only one device in deviceChan, the function does not receive anything. Is the channel gone before the WriteDeviceToFile gets to it?
The program exits when main returns. Nothing prevents main from exiting before the files are written
I need to read a file of integers into an array. I have it working with this:
package main
import (
"fmt"
"io"
"os"
)
func readFile(filePath string) (numbers []int) {
fd, err := os.Open(filePath)
if err != nil {
panic(fmt.Sprintf("open %s: %v", filePath, err))
}
var line int
for {
_, err := fmt.Fscanf(fd, "%d\n", &line)
if err != nil {
fmt.Println(err)
if err == io.EOF {
return
}
panic(fmt.Sprintf("Scan Failed %s: %v", filePath, err))
}
numbers = append(numbers, line)
}
return
}
func main() {
numbers := readFile("numbers.txt")
fmt.Println(len(numbers))
}
The file numbers.txt is just:
1
2
3
...
ReadFile() seems too long (maybe because of the error handing).
Is there a shorter / more Go idiomatic way to load a file?
Using a bufio.Scanner makes things nice. I've also used an io.Reader rather than taking a filename. Often that's a good technique, since it allows the code to be used on any file-like object and not just a file on disk. Here it's "reading" from a string.
package main
import (
"bufio"
"fmt"
"io"
"strconv"
"strings"
)
// ReadInts reads whitespace-separated ints from r. If there's an error, it
// returns the ints successfully read so far as well as the error value.
func ReadInts(r io.Reader) ([]int, error) {
scanner := bufio.NewScanner(r)
scanner.Split(bufio.ScanWords)
var result []int
for scanner.Scan() {
x, err := strconv.Atoi(scanner.Text())
if err != nil {
return result, err
}
result = append(result, x)
}
return result, scanner.Err()
}
func main() {
tf := "1\n2\n3\n4\n5\n6"
ints, err := ReadInts(strings.NewReader(tf))
fmt.Println(ints, err)
}
I would do it like this:
package main
import (
"fmt"
"io/ioutil"
"strconv"
"strings"
)
// It would be better for such a function to return error, instead of handling
// it on their own.
func readFile(fname string) (nums []int, err error) {
b, err := ioutil.ReadFile(fname)
if err != nil { return nil, err }
lines := strings.Split(string(b), "\n")
// Assign cap to avoid resize on every append.
nums = make([]int, 0, len(lines))
for _, l := range lines {
// Empty line occurs at the end of the file when we use Split.
if len(l) == 0 { continue }
// Atoi better suits the job when we know exactly what we're dealing
// with. Scanf is the more general option.
n, err := strconv.Atoi(l)
if err != nil { return nil, err }
nums = append(nums, n)
}
return nums, nil
}
func main() {
nums, err := readFile("numbers.txt")
if err != nil { panic(err) }
fmt.Println(len(nums))
}
Your solution with fmt.Fscanf is fine. There are certainly a number of other ways to do though, depending on your situation. Mostafa's technique is one I use a lot (although I might allocate the result all at once with make. oops! scratch that. He did.) but for ultimate control you should learn bufio.ReadLine. See go readline -> string for some example code.