Default HTTP file server - modifications of the content - go

How can I compose the default Go HTTP file server (serve if exists, show file listing otherwise) with additional HTML?
Sample http.go with default file server:
package main
import "net/http"
func main() {
http.Handle("/", http.FileServer(http.Dir(".")))
http.ListenAndServe(":8090", nil)
}
Loading the default page (http://localhost:8090) gives something like:
<pre>LICENSE
README.md
studio.jpg
</pre>
I found it is declared at fs.go.
I want to keep that section, but with my own header and footer (preferably without copying the dirList function and making small changes):
<title>My files</title>
<pre>LICENSE
README.md
studio.jpg
</pre>
<p>And that's all, folks!</p>

Based on this answer, you can implement own FileSystem for a FileServer
This implementation is very buggy at best, and you should probably never ever use it, but it should show you how the FileSystem interface can be implemented for arbitrary 'files'.
type InMemoryFS map[string]http.File
type InMemoryFile struct {
at int64
Name string
data []byte
fs InMemoryFS
}
func NewFile(name string, data []byte) *InMemoryFile {
return &InMemoryFile{at: 0,
Name: name,
data: data,
fs: make(InMemoryFS)}
}
// Implements the http.File interface
func (f *InMemoryFile) Close() error {
return nil
}
func (f *InMemoryFile) Stat() (os.FileInfo, error) {
return &InMemoryFileInfo{f}, nil
}
func (f *InMemoryFile) Readdir(count int) ([]os.FileInfo, error) {
return nil, nil
}
func (f *InMemoryFile) Read(b []byte) (int, error) {
i := 0
for f.at < int64(len(f.data)) && i < len(b) {
b[i] = f.data[f.at]
i++
f.at++
}
return i, nil
}
func (f *InMemoryFile) Seek(offset int64, whence int) (int64, error) {
switch whence {
case 0:
f.at = offset
case 1:
f.at += offset
case 2:
f.at = int64(len(f.data)) + offset
}
return f.at, nil
}
type InMemoryFileInfo struct {
file *InMemoryFile
}
// Implements os.FileInfo
func (s *InMemoryFileInfo) Name() string { return s.file.Name }
func (s *InMemoryFileInfo) Size() int64 { return int64(len(s.file.data)) }
func (s *InMemoryFileInfo) Mode() os.FileMode { return os.ModeTemporary }
func (s *InMemoryFileInfo) ModTime() time.Time { return time.Time{} }
func (s *InMemoryFileInfo) IsDir() bool { return false }
func (s *InMemoryFileInfo) Sys() interface{} { return nil }
// CustomFsDecorator: is `http.FileSystem` decorator
type CustomFsDecorator struct {
http.FileSystem
}
func (fs CustomFsDecorator) Open(name string) (http.File, error) {
file, err := fs.FileSystem.Open(name)
if err != nil {
return nil, err
}
info, err := file.Stat()
if err != nil {
return nil, err
}
if info.IsDir() {
return file, nil
}
b, err := io.ReadAll(file)
if err != nil {
return nil, err
}
buf := new(bytes.Buffer)
// add header's lines
_, err = buf.Write([]byte("<title>My files</title>\n"))
if err != nil {
return nil, err
}
_, err = buf.Write(b)
if err != nil {
return nil, err
}
// add footer's lines
_, err = buf.Write([]byte("\n<p>And that's all, folks!</p>"))
if err != nil {
return nil, err
}
return NewFile(info.Name(), buf.Bytes()), nil
}
func Test(t *testing.T) {
cfsys := CustomFsDecorator{FileSystem: http.Dir("./static")}
fsys := http.FileServer(cfsys)
req := httptest.NewRequest(http.MethodGet, "/some.html", nil)
w := httptest.NewRecorder()
fsys.ServeHTTP(w, req)
res := w.Result()
defer func() {
_ = res.Body.Close()
}()
data, err := io.ReadAll(res.Body)
if err != nil {
t.Errorf("expected error to be nil got %v", err)
}
fmt.Println(string(data))
}
👇🏻
<title>My files</title>
<pre>LICENSE
README.md
studio.jpg
</pre>
<p>And that's all, folks!</p>
PLAYGROUND

Related

Mocking open in go…

I have been following this answer trying to mock open. I have got exactly no where.
This is the test code I have:
func (m mockedFS) Open(name string) (file, error) {
if m.reportErrOpen {
return nil, errors.New("Fake failure")
}
mockedFile := mockIORead{}
mockedFile.On("ReadAll", mock.AnythingOfType("[]uint8")).Return(0, fmt.Errorf("error reading"))
mockedFile.On("Read", mock.AnythingOfType("[]byte")).Return(0, errors.New("NON"))
return mockedFile, nil
}
type mockIORead struct {
mock.Mock
reportErr bool // Tells if this mocked FS should return error in our tests
reportSize int64 // Tells what size should Stat() report in our test
}
func (m mockIORead) Read(b []byte) (n int, err error) {
if m.reportErr {
return 0, errors.New("A fake failure")
}
s := "Fear the old blood"
copy(b[:], s)
return 0, nil
}
func (m mockIORead) Close() error {
return nil
}
func (m mockIORead) ReadAt([]byte, int64) (int, error) {
return 0, nil
}
func (m mockIORead) Seek(int64, int) (int64, error) {
return 0, nil
}
func (m mockIORead) Stat() (os.FileInfo, error) {
if m.reportErr {
return nil, os.ErrNotExist
}
return mockedFileInfo{size: m.reportSize}, nil
}
func TestOok(t *testing.T) {
oldFs := fs
// Create and "install" mocked fs:
mfs := &mockedFS{}
fs = mfs
// Make sure fs is restored after this test:
defer func() {
fs = oldFs
}()
mfs.reportErr = false
mfs.reportErrOpen = false
token, err := Ook("fake")
assert.NotNil(t, err)
assert.Equal(t, "Fear the old blood", token)
}
And this is the code under test:
func Ook(name string) (string, error) {
_, err := fs.Stat(name)
if err != nil {
return "", nil
}
file, err := fs.Open(name)
if err != nil {
return "", errors.Wrap(err, "Cannot open token file")
}
defer file.Close()
_, err = ioutil.ReadAll(file)
fmt.Print("PING\n")
if err != nil {
return "", errors.Wrap(err, "Could not read token")
}
return "Fear the old blood", nil
//return string(token), nil
}
What the hell am I doing wrong?
The first error is that your mockIORead.Read() returns wrong values. It must return the number of read bytes (bytes written to the slice argument) (e.g. what copy() would return).
Next, mockIORead.Read() must be stateful! Reader.Read() might be called several times, there is no guarantee the passed slice can accommodate all the data you want to return (via the passed b slice).
So mockIORead must store the data you want to return, and it must remember how much of them has been delivered so far, so the next Read() call can continue from there.
An easy implementation of this is to utilize bytes.Buffer:
type mockIORead struct {
mock.Mock
reportErr bool // Tells if this mocked FS should return error in our tests
reportSize int64 // Tells what size should Stat() report in our test
content *bytes.Buffer
}
When returning such a mockIORead, initialize content with the content you wish to return:
func (m mockedFS) Open(name string) (file, error) {
if m.reportErrOpen {
return nil, errors.New("Fake failure")
}
mockedFile := mockIORead{
content: bytes.NewBufferString("Fear the old blood"),
}
return mockedFile, nil
}
And thanks to the available bytes.Buffer.Read() method, the mockIORead.Read() implementation can be as simple as this:
func (m mockIORead) Read(b []byte) (n int, err error) {
if m.reportErr {
return 0, errors.New("A fake failure")
}
return m.content.Read(b)
}
The Ook() function itself should not try to "stat" as you haven't mocked it (and so calling the original os.Stat() will likely yield an error for the "fake" file name used in the test):
func Ook(name string) (string, error) {
file, err := fs.Open(name)
if err != nil {
return "", errors.Wrap(err, "Cannot open token file")
}
defer file.Close()
token, err := ioutil.ReadAll(file)
fmt.Print("PING\n")
if err != nil {
return "", errors.Wrap(err, "Could not read token")
}
return string(token), nil
}
And the testing code:
func TestOok(t *testing.T) {
oldFs := fs
// Create and "install" mocked fs:
mfs := &mockedFS{}
fs = mfs
// Make sure fs is restored after this test:
defer func() {
fs = oldFs
}()
mfs.reportErr = false
mfs.reportErrOpen = false
token, err := Ook("fake")
assert.Nil(t, err)
assert.Equal(t, "Fear the old blood", token)
}
Which yields a successful ("OK") test.

How to extend go-yaml to support custom tags

I have spent some time reading the code and docs of go-yaml, but I have not found any way to do this, except forking the project..
I want to extend the YAML unmarshaller so that it can accept a custom YAML tag (!include <file> in this case), which in turn would allow me to add support for including files. This is easily implemented with other YAML libraries, like in this answer.
Is there any way to accomplish this, using the public interface of the library (or another yaml library)?
Yes, this is possible (since v3). You can load the whole YAML file into a yaml.Node and then walk over the structure. The trick is that yaml.Node is an intermediate representation which you can only access if you define an unmarshaler.
For example:
package main
import (
"errors"
"fmt"
"io/ioutil"
"gopkg.in/yaml.v3"
)
// used for loading included files
type Fragment struct {
content *yaml.Node
}
func (f *Fragment) UnmarshalYAML(value *yaml.Node) error {
var err error
// process includes in fragments
f.content, err = resolveIncludes(value)
return err
}
type IncludeProcessor struct {
target interface{}
}
func (i *IncludeProcessor) UnmarshalYAML(value *yaml.Node) error {
resolved, err := resolveIncludes(value)
if err != nil {
return err
}
return resolved.Decode(i.target)
}
func resolveIncludes(node *yaml.Node) (*yaml.Node, error) {
if node.Tag == "!include" {
if node.Kind != yaml.ScalarNode {
return nil, errors.New("!include on a non-scalar node")
}
file, err := ioutil.ReadFile(node.Value)
if err != nil {
return nil, err
}
var f Fragment
err = yaml.Unmarshal(file, &f)
return f.content, err
}
if node.Kind == yaml.SequenceNode || node.Kind == yaml.MappingNode {
var err error
for i := range node.Content {
node.Content[i], err = resolveIncludes(node.Content[i])
if err != nil {
return nil, err
}
}
}
return node, nil
}
type MyStructure struct {
// this structure holds the values you want to load after processing
// includes, e.g.
Num int
}
func main() {
var s MyStructure
yaml.Unmarshal([]byte("!include foo.yaml"), &IncludeProcessor{&s})
fmt.Printf("Num: %v", s.Num)
}
Code prints Num: 42 when a file foo.yaml exists with the content num: 42.
Modified #flyx's original code a little to make it modular for adding custom resolvers.
package main
import (
"errors"
"fmt"
"io/ioutil"
"os"
"gopkg.in/yaml.v3"
)
var tagResolvers = make(map[string]func(*yaml.Node) (*yaml.Node, error))
type Fragment struct {
content *yaml.Node
}
func (f *Fragment) UnmarshalYAML(value *yaml.Node) error {
var err error
// process includes in fragments
f.content, err = resolveTags(value)
return err
}
type CustomTagProcessor struct {
target interface{}
}
func (i *CustomTagProcessor) UnmarshalYAML(value *yaml.Node) error {
resolved, err := resolveTags(value)
if err != nil {
return err
}
return resolved.Decode(i.target)
}
func resolveTags(node *yaml.Node) (*yaml.Node, error) {
for tag, fn := range tagResolvers {
if node.Tag == tag {
return fn(node)
}
}
if node.Kind == yaml.SequenceNode || node.Kind == yaml.MappingNode {
var err error
for i := range node.Content {
node.Content[i], err = resolveTags(node.Content[i])
if err != nil {
return nil, err
}
}
}
return node, nil
}
func resolveIncludes(node *yaml.Node) (*yaml.Node, error) {
if node.Kind != yaml.ScalarNode {
return nil, errors.New("!include on a non-scalar node")
}
file, err := ioutil.ReadFile(node.Value)
if err != nil {
return nil, err
}
var f Fragment
err = yaml.Unmarshal(file, &f)
return f.content, err
}
func resolveGetValueFromEnv(node *yaml.Node) (*yaml.Node, error) {
if node.Kind != yaml.ScalarNode {
return nil, errors.New("!getValueFromEnv on a non-scalar node")
}
value := os.Getenv(node.Value)
if value == "" {
return nil, fmt.Errorf("environment variable %v not set", node.Value)
}
var f Fragment
err := yaml.Unmarshal([]byte(value), &f)
return f.content, err
}
func AddResolvers(tag string, fn func(*yaml.Node) (*yaml.Node, error)) {
tagResolvers[tag] = fn
}
func main() {
// Register custom tag resolvers
AddResolvers("!include", resolveIncludes)
AddResolvers("!getValueFromEnv", resolveGetValueFromEnv)
type MyStructure struct {
// this structure holds the values you want to load after processing
// includes, e.g.
Num int
}
var s MyStructure
os.Setenv("FOO", `{"num": 42}`)
err := yaml.Unmarshal([]byte("!getValueFromEnv FOO"), &CustomTagProcessor{&s})
if err != nil {
panic("Error encountered during unmarshalling")
}
fmt.Printf("\nNum: %v", s.Num)
err = yaml.Unmarshal([]byte("!include foo.yaml"), &CustomTagProcessor{&s})
if err != nil {
panic("Error encountered during unmarshalling")
}
fmt.Printf("\nNum: %v", s.Num)
}

How to 'disable directory listing' and 'custom 404 page' handle same time

I'm using gorilla mux for routing in my http server
https://github.com/gorilla/mux
This is my code for disable directory listing
type justFilesFilesystem struct {
fs http.FileSystem
// readDirBatchSize - configuration parameter for Readdir func
readDirBatchSize int
}
func (fs justFilesFilesystem) Open(name string) (http.File, error) {
f, err := fs.fs.Open(name)
if err != nil {
return nil, err
}
return neuteredStatFile{File: f, readDirBatchSize: fs.readDirBatchSize}, nil
}
type neuteredStatFile struct {
http.File
readDirBatchSize int
}
func (e neuteredStatFile) Stat() (os.FileInfo, error) {
s, err := e.File.Stat()
if err != nil {
return nil, err
}
if s.IsDir() {
LOOP:
for {
fl, err := e.File.Readdir(e.readDirBatchSize)
switch err {
case io.EOF:
break LOOP
case nil:
for _, f := range fl {
if f.Name() == "" {
return s, err
}
}
default:
return nil, err
}
}
return nil, os.ErrNotExist
}
return s, err
}
and this is my main func
mux := mux.NewRouter()
// This line why not work?
mux.NotFoundHandler = http.HandlerFunc(NotFound)
mux.HandleFunc("/index",HandleIndex)
fs := justFilesFilesystem{fs: http.Dir("assets"), readDirBatchSize: 0}
staticFileHandler := http.StripPrefix("", http.FileServer(fs))
mux.PathPrefix("").Handler(staticFileHandler).Methods("GET")
and finally my 404 handle func
func NotFound(w http.ResponseWriter, r *http.Request) {
fmt.Fprint(w, "404")
}
The question is how to handle custom 404 page error if file not found and also disable directory listing in the same time.

How do I make this Go code more DRY?

I'm implementing a Go wrapper for a REST API. It basically parses JSON and should return the appropriate struct type. I find myself doing a lot of this:
// GetBlueprintDetails returns details about a blueprint
func (c *Client) GetBlueprintDetails(projectID string, blueprintID string) (*BlueprintDetails, *APIError) {
path := fmt.Sprintf("projects/%s/blueprints/%s", projectID, blueprintID)
res, err := c.Request("GET", path, nil, nil)
if err != nil {
return nil, err
}
var ret BlueprintDetails
e := json.Unmarshal(res.Body, &ret)
if e != nil {
return nil, &APIError{Error: &e}
}
return &ret, nil
}
// GetProjects returns a list of projects for the user
func (c *Client) GetProjects() (*[]Project, *APIError) {
res, err := c.Request("GET", "projects", nil, nil)
if err != nil {
return nil, err
}
var ret []Project
e := json.Unmarshal(res.Body, &ret)
if e != nil {
return nil, &APIError{Error: &e}
}
return &ret, nil
}
The only difference between the two functions is the type of the unmarshaled struct basically. I know there are no generic in Go, but there has to be a pattern to make this more DRY.
Any ideas?
You may create a MakeRequest function that does the http request part and unmarshal the json to struct
Here is how you may do it, have a look at the MakeRequest function
// GetBlueprintDetails returns details about a blueprint
func (c *Client) GetBlueprintDetails(projectID string, blueprintID string) (*BlueprintDetails, *APIError) {
path := fmt.Sprintf("projects/%s/blueprints/%s", projectID, blueprintID)
bluePrintDetails = new(BlueprintDetails)
err := c.MakeRequest("GET", path, bluePrintDetails)
return bluePrintDetails, err
}
// GetProjects returns a list of projects for the user
func (c *Client) GetProjects() (*[]Project, *APIError) {
projects = make([]Project, 0)
err := c.MakeRequest("GET", "project", &projects)
return &projects, err
}
func (c *Client) MakeRequest(method string, path string, response interface{}) *APIError {
res, err := c.Request(method, path, nil, nil)
if err != nil {
return nil, err
}
e := json.Unmarshal(res.Body, response)
if e != nil {
return &APIError{Error: &e}
}
return nil
}

Trying to test write file from goroutines in Go

Well, part of my code was working without a method approach, I'm trying to test
append text to a file and reading from goroutines, but I'm stuck here trying to
write it.
What is wrong? the file is created, but I can't append text to it, maybe something obvious, but seems I'm blind, maybe I'm failing understanding some language concepts...
package main
import (
"bufio"
"fmt"
"os"
"sync"
"time"
)
var w sync.WaitGroup
type Buffer struct {
F *os.File
}
func (buff *Buffer) Open(pathName string) (err error) {
buff.F, err = os.OpenFile(pathName, os.O_APPEND|os.O_CREATE, 0666)
if err != nil {
return
}
fmt.Println("Open() ok")
return nil
}
func (buff *Buffer) Close() (err error) {
err = buff.F.Close()
if err != nil {
return
}
fmt.Println("Close() ok")
return nil
}
func (buff *Buffer) Push(data string) (err error) {
w := bufio.NewWriter(buff.F)
_, err = fmt.Fprintf(w, "data=%s", data)
if err != nil {
return
}
w.Flush()
fmt.Println("Push() ok")
return nil
}
func checkErr(err error) {
if err != nil {
fmt.Println(err.Error())
}
}
func worker() {
var err error
buffer := new(Buffer)
err = buffer.Open("test")
checkErr(err)
err = buffer.Push("data\n")
checkErr(err)
time.Sleep(5 * time.Second)
err = buffer.Close()
checkErr(err)
w.Done()
}
func main() {
w.Add(2)
go worker()
go worker()
w.Wait()
}
Thanks
Open the file like this:
buff.F, err = os.OpenFile(pathName, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
The write flag is required to write to the file.
You missed the write error because the return from bufio Flush is ignored. Change Push to:
func (buff *Buffer) Push(data string) (err error) {
w := bufio.NewWriter(buff.F)
_, err = fmt.Fprintf(w, "data=%s", data)
if err != nil {
return
}
err = w.Flush()
if err != nil {
return err
}
fmt.Println("Push() ok")
return nil
}
To cleanly append data without intermixing with other pushes, the data must be written with a single call to the file Write method. Use a bytes.Buffer instead of a bufio.Writer to ensure a single call to the file Write method:
func (buff *Buffer) Push(data string) (err error) {
var b bytes.Buffer
_, err = fmt.Fprintf(&b, "data=%s", data)
if err != nil {
return
}
_, err := buff.F.Write(b.Bytes())
if err != nil {
return err
}
fmt.Println("Push() ok")
return nil
}

Resources