Golang - embed.FS pass argument - go

I need to modify fragment of code which needs to take argument instead of use hardcoded path in go:embed
My code:
package main
import (
"embed"
"log"
"github.com/golang-migrate/migrate/v4/source/iofs"
)
// Instead of use hardcoded `migrations/*.sql` I need to pass
// `path` to `migrations/*.sql` as a argument to `migrate` function
//go:embed migrations/*.sql
var fs embed.FS
func main() {
const path = "migrations/*.sql"
if err := migrate(path); err != nil {
log.Fatal(err)
}
log.Println("done")
}
func migrate(path string) error {
d, err := iofs.New(fs, "migrations")
if err != nil {
return err
}
// rest of the code...
return nil
}
I already tried to use os.DirFS however result of this function only contains data with dir parameter without actual files from folder migrations/*.sql

I'm not entirely sure I understand the question. You've created an embedded filesystem that contains multiple files (everything matching *.sql in the migrations directory). You can load files from this embedded filesystem by name.
Let's assume that your migrations directory looks like:
migrations/
├── bar.sql
└── foo.sql
Where foo.sql contains this is foo.sql, and similarly for bar.sql.
If we write code like this:
func main() {
const path = "migrations/foo.sql"
if err := migrate(path); err != nil {
log.Fatal(err)
}
log.Println("done")
}
func migrate(path string) error {
data, err := fs.ReadFile(path)
if err != nil {
panic(err)
}
fmt.Println(string(data))
// rest of the code...
return nil
}
You'd get the output:
this is foo.sql
If you want to iterate over files in your embedded filesystem, you can use the ReadDir method:
func main() {
const path = "migrations"
if err := migrate(path); err != nil {
log.Fatal(err)
}
log.Println("done")
}
func migrate(path string) error {
items, err := fs.ReadDir(path)
if err != nil {
panic(err)
}
for _, ent := range items {
fmt.Println(ent.Name())
}
// rest of the code...
return nil
}
This would print:
bar.sql
foo.sql
Does that help?

Related

Generate .pb file without using protoc in golang

I'm trying to generate .pb.go file using service.proto as file input in Go.
Is there a way to do it without using protoc binary (like directly using package github.com/golang/protobuf/protoc-gen-go)?
If you have a detail.proto like this:
message AppDetails {
optional string version = 4;
}
You can parse it into a message like this:
package main
import (
"fmt"
"github.com/golang/protobuf/proto"
"github.com/jhump/protoreflect/desc/protoparse"
"github.com/jhump/protoreflect/dynamic"
)
func parse(file, msg string) (*dynamic.Message, error) {
var p protoparse.Parser
fd, err := p.ParseFiles(file)
if err != nil {
return nil, err
}
md := fd[0].FindMessage(msg)
return dynamic.NewMessage(md), nil
}
func main() {
b := []byte("\"\vhello world")
m, err := parse("detail.proto", "AppDetails")
if err != nil {
panic(err)
}
if err := proto.Unmarshal(b, m); err != nil {
panic(err)
}
fmt.Println(m) // version:"hello world"
}
However you may notice, this package is still using the old Protobuf V1. I did
find a Pull Request for V2:
https://github.com/jhump/protoreflect/pull/354

I am trying to traverse the file structure in Breadth first fashion. I am getting segmentation violation

I know about ioutil.ReadDir and os.filePath but none of them traverse the directory in Breadth first fashion.
My approach is to call ioutil.ReadDir and append all the contents of the root dir into a slice. Then I am iterating over the contents and checking if it IsDir[] and calling the function recursively if true.
package main
import (
"io/ioutil"
"os"
)
var files []string
var path string
func appendFiles(root string) {
fileInfo, err := ioutil.ReadDir(root)
if err != nil {
return
}
for _, file := range fileInfo {
files = append(files, file.Name())
}
for _, file := range fileInfo {
fileStat, _ := os.Stat(file.Name())
if fileStat.Mode().IsDir() {
// path = path + "/" + file.Name()
appendFiles(file.Name())
}
}
}
func main() {
appendFiles(".")
}
The problem is that os.Stat() may return an error which you omit. When that happens, fileStat may be nil, so calling fileStat.Mode() in the next line panics.
And the reason os.Stat() fails is because file.Name() is relative to root, file.Name() by itself has little chance to exist, it must be joined with root. If os.Stat() is called with a file name that doesn't exist, it returns a nil file info and a non-nil error.
You may use filepath.Join() to construct a valid path for files that os.Stat() will work with. And it would be better to handle errors, e.g. return them, which you can inspect in main().
func appendFiles(root string) error {
fileInfo, err := ioutil.ReadDir(root)
if err != nil {
return fmt.Errorf("ReadDir error: %w", err)
}
for _, file := range fileInfo {
files = append(files, filepath.Join(root, file.Name()))
}
for _, file := range fileInfo {
fullName := filepath.Join(root, file.Name())
fileStat, err := os.Stat(fullName)
if err != nil {
return fmt.Errorf("Stat error: %w", err)
}
if fileStat.Mode().IsDir() {
if err := appendFiles(fullName); err != nil {
return fmt.Errorf("appendFiles error: %w", err)
}
}
}
return nil
}
func main() {
if err := appendFiles("."); err != nil {
fmt.Println(err)
}
}

os.Create and os.Open not working with gomobile and react native

package component
import (
"encoding/json"
"io/ioutil"
"os"
)
type LastComponent struct {
Name string
}
const fname = "componentfiles"
func Persist(comp string) string {
lcomp := LastComponent{Name: comp}
b, err := json.Marshal(lcomp)
if err != nil {
return "err-MARSHAL"
}
file, err := os.Create(fname)
if err != nil {
return "err-CREATE-FILE"
}
defer file.Close()
_, err = file.Write(b)
if err != nil {
return "err-FILE-WRITE-PROB"
}
return ""
}
func Component() string {
f, err := os.Open(fname)
if err != nil {
return "err-FILE-NOT-OPEN"
}
defer f.Close()
b, err := ioutil.ReadAll(f)
if err != nil {
return ""
}
var v LastComponent
json.Unmarshal(b, v)
return v.Name
}
}
The code above works fine and so does the javascript side of code. I keep receiving err-CREATE-FILE inside my javascript. So os.Create and os.Open are not working as expected.
Although it is an internal storage, permissions are not required, but I also turned on the permissions in manifest file, but with no avail.
What could be the correct way to Open and Create files in android using gomobile when using along side React Native?
Update:
In adb logcat, I keep getting this all over the place
E/Vold ( 276): Failed to find mounted volume for /storage/sdcard1/Android/data/com.gotest/cache/
So you should have some success if you pass this in as a parameter - something like the following is working for me:
go:
func Component(folderPath string) string {
f, err := os.Open(path.Join(folderPath, fname))
...
Java:
Component.Component(getApplicationContext().getFilesDir().getAbsolutePath())
Alternatively, you could use something like getExternalStorageDirectory().getAbsolutePath(). They key is that you need to get somewhere storagewise that is writable by your process/user.

Golang: Parse all templates in directory and subdirectories?

This is my directory structure:
app/
template/
layout/
base.tmpl
index.tmpl
template.ParseGlob("*/*.tmpl") parses index.tmpl but not base.tmpl in the layout subdirectory. Is there a way to parse all templates recursively?
Not without implementing your own function to do it, I've been using something like this
func ParseTemplates() *template.Template {
templ := template.New("")
err := filepath.Walk("./views", func(path string, info os.FileInfo, err error) error {
if strings.Contains(path, ".html") {
_, err = templ.ParseFiles(path)
if err != nil {
log.Println(err)
}
}
return err
})
if err != nil {
panic(err)
}
return templ
}
This will parse all your templates then you can render them by calling their names e.g.
template.ExecuteTemplate(w, "home", nil)
if it is not deeply nested (if you know names of sub-directories beforehand) you can just do this:
t := template.Must(template.ParseGlob("template/*.tmpl"))
template.Must(t.ParseGlob("template/layout/*.tmpl"))
Then for each sub-directory do the same as for 'layout'
Datsik's answer has the drawback that there are name collision issues when several directories contain many templates. If two templates in different directories have the same filename it won't work properly: only the second of them will be usable.
This is caused by the implementation of template.ParseFiles, so we can solve it by avoiding template.ParseFiles. Here's a modified walk algorithm that does this by using template.Parse directly instead.
func findAndParseTemplates(rootDir string, funcMap template.FuncMap) (*template.Template, error) {
cleanRoot := filepath.Clean(rootDir)
pfx := len(cleanRoot)+1
root := template.New("")
err := filepath.Walk(cleanRoot, func(path string, info os.FileInfo, e1 error) error {
if !info.IsDir() && strings.HasSuffix(path, ".html") {
if e1 != nil {
return e1
}
b, e2 := ioutil.ReadFile(path)
if e2 != nil {
return e2
}
name := path[pfx:]
t := root.New(name).Funcs(funcMap)
_, e2 = t.Parse(string(b))
if e2 != nil {
return e2
}
}
return nil
})
return root, err
}
This will parse all your templates then you can render them by calling their names e.g.
template.ExecuteTemplate(w, "a/home.html", nil)
You can load multiple subdirectories this way. Here we ignore if subdirectories do not exist. But we want to make sure that the first directory can be loaded.
func ParseTemplates() (*template.Template, error) {
templateBuilder := template.New("")
if t, _ := templateBuilder.ParseGlob("/*/*/*/*/*.tmpl"); t != nil {
templateBuilder = t
}
if t, _ := templateBuilder.ParseGlob("/*/*/*/*.tmpl"); t != nil {
templateBuilder = t
}
if t, _ := templateBuilder.ParseGlob("/*/*/*.tmpl"); t != nil {
templateBuilder = t
}
if t, _ := templateBuilder.ParseGlob("/*/*.tmpl"); t != nil {
templateBuilder = t
}
return templateBuilder.ParseGlob("/*.tmpl")
}
I made a package that solves exactly this problem.
https://github.com/karelbilek/template-parse-recursive
package main
import (
"html/template"
"os"
recurparse "github.com/karelbilek/template-parse-recursive"
)
func main() {
t, err := recurparse.HTMLParse(
template.New("templates"),
"path/to/templates",
"*.html",
)
if err != nil {
panic(err)
}
templateUnder := t.Lookup("subdir/subdir/template.html")
templateUnder.Execute(os.Stdout, nil)
}
With the answer of pete911 I could create this function to parse all the html templates on multiple subdirectories.
// parse.go
func ParseHtmlTemplates() *template.Template {
var directories []string
var filenames []string
// Root directory of template files
root := "./templates"
// Get all directories on /templates and check if there's repeated files
err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
if !info.IsDir() {
// Is file
filename := info.Name()
hasRepeatedFiles := contains(filenames, filename)
if hasRepeatedFiles {
return fmt.Errorf("You can't have repeated template files: %s", filename)
}
filenames = append(filenames, filename)
} else {
// Is directory
directories = append(directories, path)
}
return nil
})
if err != nil {
fmt.Println(err)
os.Exit(1)
}
// Create a template for parsing all directories
tmpl := template.Must(template.ParseGlob(root + "/*.html"))
// Parse all directories (minus the root directory)
for _, path := range directories[1:] {
pattern := path + "/*.html"
template.Must(tmpl.ParseGlob(pattern))
}
return tmpl
}
// contains private method
func contains(filenames []string, filename string) bool {
for _, f := range filenames {
if f == filename {
return true
}
}
return false
}
// main.go
func main() {
tmpl = ParseHtmlTemplates()
if err := tmpl.Execute(os.Stdout, ""); err != nil {
panic(err)
}
}

Specify names for parsed templates

I am trying to dynamically parse files using walk in a folder and I want to be able to set the path of the file "path/file.html". But my issue is if I have a file in a folder "path/folder/files.html" I can't do it because when I ExecuteTemplate the file name will be the same "files.html". Is it possible to name each template as I ParseFiles?
Im ok with doing a file one at a time if trying to do them all at once wont work.
// Parse file and send to responsewriter
func View(w http.ResponseWriter, path string) {
temp, err := template.ParseFiles("application/views/"+path+".html")
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
} else {
temp.ExecuteTemplate(w, path, nil)
}
}
Walk the filesystem using filepath.Walk and a consumer method that will create templates with the full file paths as names:
package main
import (
"fmt"
"html/template"
"os"
"path/filepath"
)
func consumer(p string, i os.FileInfo, e error) error {
t := template.New(p)
fmt.Println(t.Name())
return nil
}
func main() {
filepath.Walk("/path/to/template/root", filepath.WalkFunc(consumer))
}
You can try template.Lookup, the whole process looks like:
var (
templates *template.Template
)
func loadTemplate() {
funcMap := template.FuncMap{
"safe":func(s string) template.HTML {
return template.HTML(s)
},
}
var err error
templates, err = utils.BuildTemplate("/theme/path/", funcMap)
if err != nil {
log.Printf("Can't read template file %v,", err)
}
}
func homeHandler(w http.ResponseWriter, r *http.Request) {
//lookup the theme your want to use
templ = templates.Lookup("theme.html")
err := templ.Execute(w, data)
if err != nil {
log.Println(err)
}
}
func main() {
loadTemplate()
}
BuildTemplate looks like:
func BuildTemplate(dir string, funcMap template.FuncMap) (*template.Template, error) {
fs, err := ioutil.ReadDir(dir)
if err != nil {
fmt.Printf("Can't read template folder: %s\n", dir)
return nil, err
}
files := make([]string, len(fs))
for i, f := range (fs) {
files[i] = path.Join(dir, f.Name())
}
return template.Must(template.New("Template").Funcs(funcMap).ParseFiles(files...)), nil
}

Resources