I'm trying to write a threaded Decasteljau algorithm for control polygons with any set of points in golang but can't get the goroutines to work right because their work randomly and i can't manage to get all the goroutines to work .
here's my code for the decasteljau.go file :
package main
import (
"fmt"
)
type ControlPolygon struct {
Vertices []Vertex
}
type Spline struct {
Vertices map[int]Vertex
}
type splinePoint struct {
index int
vertex Vertex
}
func (controlPolygon ControlPolygon) Decasteljau(levelOfDetail int) {
//
// LevelOfDetail is the number of points in the spline
//
spline := Spline{make(map[int]Vertex)}
splinePointsChannel := make(chan splinePoint)
for index := 1; index < levelOfDetail; index++ {
splinePoint := splinePoint{}
splinePoint.index = index
pointPosition := float64(index) / float64(levelOfDetail)
go func() {
fmt.Println("goroutine number:", index)
splinePoint.findSplinePoint(controlPolygon.Vertices, pointPosition)
splinePointsChannel <- splinePoint
}()
}
point := <-splinePointsChannel
spline.Vertices[point.index] = point.vertex
fmt.Println(spline)
}
func (point *splinePoint) findSplinePoint(vertices []Vertex, pointPosition float64) {
var interpolationPoints []Vertex
if len(vertices) == 1 {
fmt.Println("vertices : ", vertices)
point.vertex = vertices[0]
}
if len(vertices) > 1 {
for i := 0; i < len(vertices)-1; i++ {
interpolationPoint := vertices[i].GetInterpolationPoint(vertices[i+1], pointPosition)
interpolationPoints = append(interpolationPoints, interpolationPoint)
}
point.findSplinePoint(interpolationPoints, pointPosition)
fmt.Println()
} else {
fmt.Println("Done Detailing ..")
return
}
}
func main() {
v1 := Vertex{0, 0, 0}
v2 := Vertex{0, 0, 1}
v3 := Vertex{0, 1, 1}
v4 := Vertex{0, 1, 0}
vectices := []Vertex{v1, v2, v3, v4}
controlPolygon := ControlPolygon{vectices}
controlPolygon.Decasteljau(10)
}
I'm also new to go concurrency and after a lot of research i'm still wondering if i need to use buffered or unbuffered channels for my case .
I also found that goroutines are mostly used for managing networks rather than optimizing 3D so i would love to know if i'm using a good stack for writing concurrent 3D algorithms
Related
With following IDL files my intention is to measure the serialization speed of Flatbuffer . I am using golang for my analysis
namespace MyFlat;
struct Vertices {
x : double;
y :double;
}
table Polygon {
polygons : [Vertices];
}
table Layer {
polygons : [Polygon];
}
root_type Layer;
Here is the code I have written for calculation
package main
import (
"MyFlat"
"fmt"
"io/ioutil"
"log"
"strconv"
"time"
flatbuffers "github.com/google/flatbuffers/go"
)
func calculation(size int, vertices int) {
b := flatbuffers.NewBuilder(0)
var polyoffset []flatbuffers.UOffsetT
rawSize := ((16 * vertices) * size) / 1024
var vec1 flatbuffers.UOffsetT
var StartedAtMarshal time.Time
var EndedAtMarshal time.Time
StartedAtMarshal = time.Now()
for k := 0; k < size; k++ {
MyFlat.PolygonStartPolygonsVector(b, vertices)
for i := 0; i < vertices; i++ {
MyFlat.CreateVertices(b, 2.0, 2.4)
}
vec1 = b.EndVector(vertices)
MyFlat.PolygonStart(b)
MyFlat.PolygonAddPolygons(b, vec1)
polyoffset = append(polyoffset, MyFlat.PolygonEnd(b))
}
MyFlat.LayerStartPolygonsVector(b, size)
for _, offset := range polyoffset {
b.PrependUOffsetT(offset)
}
vec := b.EndVector(size)
MyFlat.LayerStart(b)
MyFlat.LayerAddPolygons(b, vec)
finalOffset := MyFlat.LayerEnd(b)
b.Finish(finalOffset)
EndedAtMarshal = time.Now()
SeElaprseTime := EndedAtMarshal.Sub(StartedAtMarshal).String()
mybyte := b.FinishedBytes()
file := "/tmp/myflat_" + strconv.Itoa(size) + ".txt"
if err := ioutil.WriteFile(file, mybyte, 0644); err != nil {
log.Fatalln("Failed to write address book:", err)
}
StartedAt := time.Now()
layer := MyFlat.GetRootAsLayer(mybyte, 0)
size = layer.PolygonsLength()
obj := &MyFlat.Polygon{}
layer.Polygons(obj, 1)
for i := 0; i < obj.PolygonsLength(); i++ {
objVertices := &MyFlat.Vertices{}
obj.Polygons(objVertices, i)
fmt.Println(objVertices.X(), objVertices.Y())
}
EndedAt := time.Now()
DeElapseTime := EndedAt.Sub(StartedAt).String()
fmt.Println(size, ",", vertices, ", ", SeElaprseTime, ",", DeElapseTime, ",", (len(mybyte) / 1024), ",", rawSize)
}
func main() {
data := []int{500000, 1000000, 1500000, 3000000, 8000000}
for _, size := range data {
//calculation(size, 5)
//calculation(size, 10)
calculation(size, 20)
}
}
Problem is I find it serialization is quite slow compared to protobuff with similar idl.
For 3M polygons serialization its taking almost 4.1167037s. Where in protobuf its taking half. Deserilization time for flatbuf is very less (in micro sec). In protobuf its quite high. But still if I add both flatbuf performance is lower.
Do you see any optimized way to serialize it. Flatbuffer is having a method createBinaryVector for byte vector but there is no direct way to serialize vector of polygon from a existing a user defined type vector.
I am adding protobuf code also
syntax = 'proto3';
package myproto;
message Polygon {
repeated double v_x = 1 ;
repeated double v_y = 2 ;
}
message CADData {
repeated Polygon polygon = 1;
string layer_name = 2;
}
Go Code with protobuf
package main
import (
"fmt"
"io/ioutil"
"log"
"math/rand"
"myproto"
"strconv"
"time"
"github.com/golang/protobuf/proto"
)
func calculation(size int, vertices int) {
var comp []*myproto.Polygon
var vx []float64
var vy []float64
for i := 0; i < vertices; i++ {
r := 0 + rand.Float64()*(10-0)
vx = append(vx, r)
vy = append(vy, r/2)
}
rawSize := ((16 * vertices) * size) / 1024
StartedAtMarshal := time.Now()
for i := 0; i < size; i++ {
comp = append(comp, &myproto.Polygon{
VX: vx,
VY: vy,
})
}
pfs := &myproto.CADData{
LayerName: "Layer",
Polygon: comp,
}
data, err := proto.Marshal(pfs)
if err != nil {
log.Fatal("marshaling error: ", err)
}
EndedAtMarshal := time.Now()
SeElaprseTime := EndedAtMarshal.Sub(StartedAtMarshal).String()
file := "/tmp/myproto_" + strconv.Itoa(size) + ".txt"
if err := ioutil.WriteFile(file, data, 0644); err != nil {
log.Fatalln("Failed to write address book:", err)
}
StartedAt := time.Now()
serialized := &myproto.CADData{}
proto.Unmarshal(data, serialized)
EndedAt := time.Now()
DeElapseTime := EndedAt.Sub(StartedAt).String()
fmt.Println(size, ",", vertices, ", ", SeElaprseTime, ",", DeElapseTime, ",", (len(data) / 1024), ",", rawSize)
}
func main() {
data := []int{500000, 1000000, 1500000, 3000000, 8000000}
for _, size := range data {
// calculation(size, 5)
//calculation(size, 10)
calculation(size, 20)
}
}
The time you give, is that for serialization, de-serialization, or both?
Your de-serialization code is likely entirely dominated by fmt.Println. Why don't you instead do sum += objVertices.X() + objVertices.Y() and print sum after timing is done? Can you pull objVertices := &MyFlat.Vertices{} outside of the loop?
You didn't post your protobuf code. Are you including in the timing the time to create the tree of objects which is being serialized (which is required for use in Protobuf but not in FlatBuffers)? Similarly, are you doing the timed (de-)serialization at least a 1000x or so, so you can include the cost of GC (Protobuf allocates a LOT of objects, FlatBuffers allocates few/none) in your comparison?
If after you do the above, it is still slower, post on the FlatBuffers github issues, the authors of the Go port may be able to help further. Make sure you post full code for both systems, and full timings.
Note generally: the design of FlatBuffers is such that it will create the biggest performance gap with Protobuf in C/C++. That said, it should still be a lot faster in Go also. There are unfortunate things about Go however that prevent it from maximizing the performance potential.
b := flatbuffers.NewBuilder(0)
I'm not sure what the "grows automatically" behavior is in Go for flatbuffers, but I'm pretty sure requiring the buffer to grow automatically is not the preferred pattern. Could you try doing your same timing comparison after initializing the buffer with flatbuffers.NewBuilder(moreBytesThanTheMessageNeeds)?
I am trying to parallelize an operation in golang and save the results in a manner that I can iterate over to sum up afterwords.
I have managed to set up the parameters so that no deadlock occurs, and I have confirmed that the operations are working and being saved correctly within the function. When I iterate over the Slice of my struct and try and sum up the results of the operation, they all remain 0. I have tried passing by reference, with pointers, and with channels (causes deadlock).
I have only found this example for help: https://golang.org/doc/effective_go.html#parallel. But this seems outdated now, as Vector as been deprecated? I also have not found any references to the way this function (in the example) was constructed (with the func (u Vector) before the name). I tried replacing this with a Slice but got compile time errors.
Any help would be very appreciated. Here is the key parts of my code:
type job struct {
a int
b int
result *big.Int
}
func choose(jobs []Job, c chan int) {
temp := new(big.Int)
for _,job := range jobs {
job.result = //perform operation on job.a and job.b
//fmt.Println(job.result)
}
c <- 1
}
func main() {
num := 100 //can be very large (why we need big.Int)
n := num
k := 0
const numCPU = 6 //runtime.NumCPU
count := new(big.Int)
// create a 2d slice of jobs, one for each core
jobs := make([][]Job, numCPU)
for (float64(k) <= math.Ceil(float64(num / 2))) {
// add one job to each core, alternating so that
// job set is similar in difficulty
for i := 0; i < numCPU; i++ {
if !(float64(k) <= math.Ceil(float64(num / 2))) {
break
}
jobs[i] = append(jobs[i], Job{n, k, new(big.Int)})
n -= 1
k += 1
}
}
c := make(chan int, numCPU)
for i := 0; i < numCPU; i++ {
go choose(jobs[i], c)
}
// drain the channel
for i := 0; i < numCPU; i++ {
<-c
}
// computations are done
for i := range jobs {
for _,job := range jobs[i] {
//fmt.Println(job.result)
count.Add(count, job.result)
}
}
fmt.Println(count)
}
Here is the code running on the go playground https://play.golang.org/p/X5IYaG36U-
As long as the []Job slice is only modified by one goroutine at a time, there's no reason you can't modify the job in place.
for i, job := range jobs {
jobs[i].result = temp.Binomial(int64(job.a), int64(job.b))
}
https://play.golang.org/p/CcEGsa1fLh
You should also use a WaitGroup, rather than rely on counting tokens in a channel yourself.
I have a need to read structure fields set from another goroutine, afaik doing so directly even when knowing for sure there will be no concurrent access(write finished before read occurred, signaled via chan struct{}) may result in stale data
Will sending a pointer to the structure(created in the 1st goroutine, modified in the 2nd, read by the 3rd) resolve the possible staleness issue, considering I can guarantee no concurrent access?
I would like to avoid copying as structure is big and contains huge Bytes.Buffer filled in the 2nd goroutine, I need to read from the 3rd
There is an option for locking, but seems like an overkill considering I know that there will be no concurrent access
There are many answers to this, and it depends to your data structure and program logic.
see: How to lock/synchronize access to a variable in Go during concurrent goroutines?
and: How to use RWMutex in Golang?
1- using Stateful Goroutines and channels
2- using sync.Mutex
3- using sync/atomic
4- using WaitGroup
5- using program logic(Semaphore)
...
1: Stateful Goroutines and channels:
I simulated very similar sample(imagine you want to read from one SSD and write to another SSD with different speed):
In this sample code one goroutine (named write) does some job prepares data and fills the big struct, and another goroutine (named read) reads data from big struct then do some job, And the manger goroutine, guarantee no concurrent access to same data.
And communication between three goroutines done with channels. And in your case you can use pointers for channel data, or global struct like this sample.
output will be like this:
mean= 36.6920166015625 stdev= 6.068973186592054
I hope this helps you to get the idea.
Working sample code:
package main
import (
"fmt"
"math"
"math/rand"
"runtime"
"sync"
"time"
)
type BigStruct struct {
big []uint16
rpos int
wpos int
full bool
empty bool
stopped bool
}
func main() {
wg.Add(1)
go write()
go read()
go manage()
runtime.Gosched()
stopCh <- <-time.After(5 * time.Second)
wg.Wait()
mean := Mean(hist)
stdev := stdDev(hist, mean)
fmt.Println("mean=", mean, "stdev=", stdev)
}
const N = 1024 * 1024 * 1024
var wg sync.WaitGroup
var stopCh chan time.Time = make(chan time.Time)
var hist []int = make([]int, 65536)
var s *BigStruct = &BigStruct{empty: true,
big: make([]uint16, N), //2GB
}
var rc chan uint16 = make(chan uint16)
var wc chan uint16 = make(chan uint16)
func next(pos int) int {
pos++
if pos >= N {
pos = 0
}
return pos
}
func manage() {
dataReady := false
var data uint16
for {
if !dataReady && !s.empty {
dataReady = true
data = s.big[s.rpos]
s.rpos++
if s.rpos >= N {
s.rpos = 0
}
s.empty = s.rpos == s.wpos
s.full = next(s.wpos) == s.rpos
}
if dataReady {
select {
case rc <- data:
dataReady = false
default:
runtime.Gosched()
}
}
if !s.full {
select {
case d := <-wc:
s.big[s.wpos] = d
s.wpos++
if s.wpos >= N {
s.wpos = 0
}
s.empty = s.rpos == s.wpos
s.full = next(s.wpos) == s.rpos
default:
runtime.Gosched()
}
}
if s.stopped {
if s.empty {
wg.Done()
return
}
}
}
}
func read() {
for {
d := <-rc
hist[d]++
}
}
func write() {
for {
wc <- uint16(rand.Intn(65536))
select {
case <-stopCh:
s.stopped = true
return
default:
runtime.Gosched()
}
}
}
func stdDev(data []int, mean float64) float64 {
sum := 0.0
for _, d := range data {
sum += math.Pow(float64(d)-mean, 2)
}
variance := sum / float64(len(data)-1)
return math.Sqrt(variance)
}
func Mean(data []int) float64 {
sum := 0.0
for _, d := range data {
sum += float64(d)
}
return sum / float64(len(data))
}
5: another way(faster) for some use cases:
here another way to use shared data structure for read job/write job/ processing job which it was separated in first post, now here doing same 3 jobs without channels and without mutex.
working sample:
package main
import (
"fmt"
"math"
"math/rand"
"time"
)
type BigStruct struct {
big []uint16
rpos int
wpos int
full bool
empty bool
stopped bool
}
func manage() {
for {
if !s.empty {
hist[s.big[s.rpos]]++ //sample read job with any time len
nextPtr(&s.rpos)
}
if !s.full && !s.stopped {
s.big[s.wpos] = uint16(rand.Intn(65536)) //sample wrire job with any time len
nextPtr(&s.wpos)
}
if s.stopped {
if s.empty {
return
}
} else {
s.stopped = time.Since(t0) >= 5*time.Second
}
}
}
func main() {
t0 = time.Now()
manage()
mean := Mean(hist)
stdev := StdDev(hist, mean)
fmt.Println("mean=", mean, "stdev=", stdev)
d0 := time.Since(t0)
fmt.Println(d0) //5.8523347s
}
var t0 time.Time
const N = 100 * 1024 * 1024
var hist []int = make([]int, 65536)
var s *BigStruct = &BigStruct{empty: true,
big: make([]uint16, N), //2GB
}
func next(pos int) int {
pos++
if pos >= N {
pos = 0
}
return pos
}
func nextPtr(pos *int) {
*pos++
if *pos >= N {
*pos = 0
}
s.empty = s.rpos == s.wpos
s.full = next(s.wpos) == s.rpos
}
func StdDev(data []int, mean float64) float64 {
sum := 0.0
for _, d := range data {
sum += math.Pow(float64(d)-mean, 2)
}
variance := sum / float64(len(data)-1)
return math.Sqrt(variance)
}
func Mean(data []int) float64 {
sum := 0.0
for _, d := range data {
sum += float64(d)
}
return sum / float64(len(data))
}
To prevent concurrent modifications to a struct while retaining the ability to read, you'd typically embed a sync.RWMutex. This is no exemption. You can simply lock your struct for writes while it is in transit and unlock it at a point in time of your convenience.
package main
import (
"fmt"
"sync"
"time"
)
// Big simulates your big struct
type Big struct {
sync.RWMutex
value string
}
// pump uses a groutine to take the slice of pointers to Big,
// locks the underlying structs and sends the pointers to
// the locked instances of Big downstream
func pump(bigs []*Big) chan *Big {
// We make the channel buffered for this example
// for illustration purposes
c := make(chan *Big, 3)
go func() {
for _, big := range bigs {
// We lock the struct before sending it to the channel
// so it can not be changed via pointer while in transit
big.Lock()
c <- big
}
close(c)
}()
return c
}
// sink reads pointers to the locked instances of Big
// reads them and unlocks them
func sink(c chan *Big) {
for big := range c {
fmt.Println(big.value)
time.Sleep(1 * time.Second)
big.Unlock()
}
}
// modify tries to achieve locks to the instances and modify them
func modify(bigs []*Big) {
for _, big := range bigs {
big.Lock()
big.value = "modified"
big.Unlock()
}
}
func main() {
bigs := []*Big{&Big{value: "Foo"}, &Big{value: "Bar"}, &Big{value: "Baz"}}
c := pump(bigs)
// For the sake of this example, we wait until all entries are
// send into the channel and hence are locked
time.Sleep(1 * time.Second)
// Now we try to modify concurrently before we even start to read
// the struct of which the pointers were sent into the channel
go modify(bigs)
sink(c)
// We use sleep here to keep waiting for modify() to finish simple.
// Usually, you'd use a sync.waitGroup
time.Sleep(1 * time.Second)
for _, big := range bigs {
fmt.Println(big.value)
}
}
Run on playground
How to realize a nested iterator that takes a depth argument. A simple iterator would be when depth = 1. it is a simple iterator which runs like a simple for loop.
func Iter () chan int {
ch := make(chan int);
go func () {
for i := 1; i < 60; i++ {
ch <- i
}
close(ch)
} ();
return ch
}
Output is 1,2,3...59
For depth = 2 Output would be "1,1" "1,2" ... "1,59" "2,1" ... "59,59"
For depth = 3 Output would be "1,1,1" ... "59,59,59"
I want to avoid a nested for loop. What is the solution here ?
I don't know if it is possible to avoid nested loops, but one solution is to use a pipeline of channels. For example:
const ITER_N = 60
// ----------------
func _goFunc1(out chan string) {
for i := 1; i < ITER_N; i++ {
out <- fmt.Sprintf("%d", i)
}
close(out)
}
func _goFuncN(in chan string, out chan string) {
for j := range in {
for i := 1; i < ITER_N; i++ {
out <- fmt.Sprintf("%s,%d", j, i)
}
}
close(out)
}
// ----------------
// create the pipeline
func IterDepth(d int) chan string {
c1 := make(chan string)
go _goFunc1(c1)
var c2 chan string
for ; d > 1; d-- {
c2 = make(chan string)
go _goFuncN(c1, c2)
c1 = c2
}
return c1
}
You can test it with:
func main() {
c := IterDepth(2)
for i := range c {
fmt.Println(i)
}
}
I usually implement iterators using closures. Multiple dimensions don't make the problem much harder. Here's one example of how to do this:
package main
import "fmt"
func iter(min, max, depth int) func() ([]int, bool) {
s := make([]int, depth)
for i := range s {
s[i] = min
}
s[0] = min - 1
return func() ([]int, bool) {
s[0]++
for i := 0; i < depth-1; i++ {
if s[i] >= max {
s[i] = min
s[i+1]++
}
}
if s[depth-1] >= max {
return nil, false
}
return s, true
}
}
func main() {
// Three dimensions, ranging between [1,4)
i := iter(1, 4, 3)
for s, ok := i(); ok; s, ok = i() {
fmt.Println(s)
}
}
Try it out on the Playground.
It'd be a simple change for example to give arguments as a single int slice instead, so that you could have per-dimension limits, if such a thing were necessary.
In Python, one can write code like this, to assign multiple values from a list:
(a, b, c, d) = [1,2,3,4]
Is there a similar set of Go library function for slices? That is, I can do this:
http://play.golang.org/p/DY1Bi5omm1
package main
func get3(s []interface{}) (
a interface{},
b interface{},
c interface{},
rest []interface{}) {
return s[0],s[1],s[2],s[4:]
}
func main() {
s := make([]interface{},5);
for i :=0 ; i < 5; i++ { s[i] = i}
a,b,c,_ := get3(s)
print(a.(int))
print(b.(int))
print(c.(int))
}
Is there a standard gophery way to do this?
And is there a way around the interface{} ugliness?
I don't think you can, not in an idiomatic/clean way at least. You CAN do multiple assignments, but you will have to pass individual values either directly or with a closure:
package main
import (
"fmt"
)
func valuesFromList(list[]int,startFrom int) func() int {
i:=startFrom
return func() int {
ret := list[i]
i++
return ret
}
}
func main () {
list := []int{0,1,2,3,4,5,6,7,8,9}
yield := valuesFromList(list,5)
//This works
a,b,c := yield(),yield(),yield()
fmt.Println(a)
fmt.Println(b)
fmt.Println(c)
//This also works
d,e,f := list[0],list[1],list[2]
fmt.Println(d)
fmt.Println(e)
fmt.Println(f)
//This won't work
//g,h,i:= list[7:9]
}
Not like that; you would need dynamic typing or parametric polymorphism, which are not available in Go. The closest I can think about is by fiddling with reflect, like this: http://play.golang.org/p/-K4jh2nZjq
// src is supposed to be []T.
// dst are supposed to be &T, except the last one, which must be a 'rest' &[]T (or nil for discarding).
// There must not be more dst vars than elements in src.
func extract(src interface{}, dst ...interface{}) {
srcV := reflect.ValueOf(src)
// Iterate over dst vars until we run out of them.
i := 0
for i = 0; i < len(dst)-1; i++ {
reflect.Indirect(reflect.ValueOf(dst[i])).Set(srcV.Index(i))
}
// Now, the rest.
restDst := dst[i]
if restDst == nil {
return
}
restV := reflect.ValueOf(restDst)
indirectRest := reflect.Indirect(restV)
l := srcV.Len() - i
indirectRest.Set(reflect.MakeSlice(restV.Type().Elem(), 0, l))
for ; i < srcV.Len(); i++ {
itemV := srcV.Index(i)
indirectRest.Set(reflect.Append(indirectRest, itemV))
}
return
}
Which then you call like:
sl := []int{1, 2, 3, 4, 5, 6} // int or any other type
var a, b, c int
var rest []int
extract(sl, &a, &b, &c, &rest)
So the ugliness doesn't get out the function.
But note that all that happens at runtime, so it's not safe nor efficient and definitely is not idiomatic Go.