Golang Cryptographic Shuffle - go

I'm trying to implement a string shuffle function in Go that uses crypto/rand instead of math/rand. The Fisher-Yates Shuffle requires random integers so I've tried to implement that functionality, without having to use crypto/rand Int which relies on math/big. Below is the best I've come up with so far but is there a better method? The fact that I can't find existing examples leads me to wonder if there's a good reason why nobody does this!
package main
import "crypto/rand"
import "fmt"
import "encoding/binary"
func randomInt(max int) int {
var n uint16
binary.Read(rand.Reader, binary.LittleEndian, &n)
return int(n) % max
}
func shuffle(s *[]string) {
slice := *s
for i := range slice {
j := randomInt(i + 1)
slice[i], slice[j] = slice[j], slice[i]
}
*s = slice
}
func main() {
slice := []string{"a", "b", "c", "d", "e", "f", "h", "i", "j", "k"}
shuffle(&slice)
fmt.Println(slice)
}

Go's math/rand library has good facilities for producing random numerical primitives from a Source.
// A Source represents a source of uniformly-distributed
// pseudo-random int64 values in the range [0, 1<<63).
type Source interface {
Int63() int64
Seed(seed int64)
}
NewSource(seed int64) returns the builtin, deterministic PRNG, but New(source Source) will allow anything that satisfies the Source interface.
Here is an example of a Source that is backed by crypto/rand.
type CryptoRandSource struct{}
func NewCryptoRandSource() CryptoRandSource {
return CryptoRandSource{}
}
func (_ CryptoRandSource) Int63() int64 {
var b [8]byte
rand.Read(b[:])
// mask off sign bit to ensure positive number
return int64(binary.LittleEndian.Uint64(b[:]) & (1<<63 - 1))
}
func (_ CryptoRandSource) Seed(_ int64) {}
You can use it like this:
r := rand.New(NewCryptoRandSource())
for i := 0; i < 10; i++ {
fmt.Println(r.Int())
}
The math/rand library has a properly implemented Intn() method which ensures a uniform distribution.
func (r *Rand) Intn(n int) int {
if n <= 0 {
panic("invalid argument to Intn")
}
if n <= 1<<31-1 {
return int(r.Int31n(int32(n)))
}
return int(r.Int63n(int64(n)))
}
func (r *Rand) Int31n(n int32) int32 {
if n <= 0 {
panic("invalid argument to Int31n")
}
if n&(n-1) == 0 { // n is power of two, can mask
return r.Int31() & (n - 1)
}
max := int32((1 << 31) - 1 - (1<<31)%uint32(n))
v := r.Int31()
for v > max {
v = r.Int31()
}
return v % n
}
func (r *Rand) Int63n(n int64) int64 {
if n <= 0 {
panic("invalid argument to Int63n")
}
if n&(n-1) == 0 { // n is power of two, can mask
return r.Int63() & (n - 1)
}
max := int64((1 << 63) - 1 - (1<<63)%uint64(n))
v := r.Int63()
for v > max {
v = r.Int63()
}
return v % n
}
Cryptographic hash functions also can be wrapped as a Source for alternate means of randomness.

The numbers from n % max are not distributed uniformly. For example,
package main
import (
"fmt"
"math"
)
func main() {
max := 7
size := math.MaxUint8
count := make([]int, size)
for i := 0; i < size; i++ {
count[i%max]++
}
fmt.Println(count[:max])
}
Output:
[37 37 37 36 36 36 36]

Based on the comments received, I think I can improve on the example in my question by adding a uniformInt function, populating a uint32 instead of a uint16 and removing the pointer to the slice.
package main
import "crypto/rand"
import "fmt"
import "encoding/binary"
func randomInt() int {
var n uint32
binary.Read(rand.Reader, binary.LittleEndian, &n)
return int(n)
}
func uniformInt(max int) (r int) {
divisor := 4294967295 / max // Max Uint32
for {
r = randomInt() / divisor
if r <= max {
break
}
}
return
}
func shuffle(slice []string) {
for i := range slice {
j := uniformInt(i + 1)
slice[i], slice[j] = slice[j], slice[i]
}
}
func main() {
slice := []string{"a", "b", "c", "d", "e", "f", "h", "i", "j", "k"}
shuffle(slice)
fmt.Println(slice)
}

Related

Heapsort implementation in Go

I'm trying to implement code from Sedgewick's Algorithms textbook. The idea is to implement heapsort with the root of the heap stored in position 1 in the array.
Given the input S O R T E X A M P L E I expect a sorted output of A E E L M O P R S T X.
I'm having a bit of trouble implementing this, even when directly trying to translate the referenced Java code. This is what I have so far, which returns the following output:
package main
import (
"bufio"
"fmt"
"os"
"reflect"
"strings"
)
type Heap struct {
PQ []interface{}
}
func (h *Heap) Sort(pq []interface{}) {
n := len(pq)
for k := n / 2; k >= 1; k-- {
Sink(pq, k, n)
}
for n > 1 {
Exchange(pq, 1, n)
n = n - 1
Sink(pq, 1, n)
}
}
func Sink(pq []interface{}, k, n int) {
fmt.Println(k, n, pq)
for 2*k <= n {
j := 2 * k
if j < n && Less(pq, j, j+1) {
j = j + 1
}
Exchange(pq, k, j)
k = j
}
}
func Exchange(pq []interface{}, j, k int) {
curr := pq[j-1]
pq[j-1] = pq[k-1]
pq[k-1] = curr
}
func Less(pq []interface{}, j, k int) bool {
x, y := pq[j-1], pq[k-1]
if reflect.TypeOf(x) != reflect.TypeOf(y) {
fmt.Println("mismatched inputs", x, y)
panic("mismatched inputs")
}
switch x.(type) {
case int:
a, b := x.(int), y.(int)
if a > b {
return false
}
case float32:
a, b := x.(int), y.(int)
if a > b {
return false
}
case float64:
a, b := x.(int), y.(int)
if a > b {
return false
}
case string:
a, b := x.(string), y.(string)
if a > b {
return false
}
default:
panic("unhandled types, please add case.")
}
return true
}
func main() {
a := readStdin()
var h *Heap = new(Heap)
h.PQ = a
h.Sort(h.PQ)
fmt.Println(h.PQ)
}
func readStdin() []interface{} {
scanner := bufio.NewScanner(os.Stdin)
var items []interface{}
for scanner.Scan() {
item := scanner.Text()
tmp := strings.SplitAfter(item, " ")
items = make([]interface{}, len(tmp)+1)
for i, item := range tmp {
items[i+1] = item
}
}
if err := scanner.Err(); err != nil {
panic(err)
}
return items
}
mismatched inputs E <nil>
panic: mismatched inputs
which panics as expected because of the comparison between nil value at index 0 and the currents slice value from 1..n. Perhaps I'm looking at this problem a bit too closely, or more than likely, I am missing a key point in the heapsort implementation altogether. Thoughts?
The standard library contains a heap package that you can use to directly implement this.
Even if you want to re-implement it yourself, the idea of using an interface to access the underlying slice is a good one -- allowing you to focus on the abstract heap operations without the mess of dealing with various types.
Here's a complete working example of heapsort, running on a slice of runes. Note that *runeSlice type implements heap.Interface by defining the three methods of sort.Interface: Len, Less, Swap, and the additional two methods from heap.Interface: Push and Pop.
package main
import (
"container/heap"
"fmt"
)
type runeSlice []rune
func (r runeSlice) Len() int { return len(r) }
func (r runeSlice) Less(i, j int) bool { return r[i] > r[j] }
func (r runeSlice) Swap(i, j int) { r[i], r[j] = r[j], r[i] }
func (r *runeSlice) Push(x interface{}) {
*r = append(*r, x.(rune))
}
func (r *runeSlice) Pop() interface{} {
x := (*r)[len(*r)-1]
*r = (*r)[:len(*r)-1]
return x
}
func main() {
a := []rune("SORTEXAMPLE")
h := runeSlice(a)
heap.Init(&h)
for i := len(a) - 1; i >= 0; i-- {
a[0], a[i] = a[i], a[0]
h = h[:i]
heap.Fix(&h, 0)
}
fmt.Println(string(a))
}

Is this a reasonable and idiomatic GoLang circular shift implementation?

Can anyone comment on whether this is a reasonable and idiomatic way of implementing circular shift of integer arrays in Go? (I deliberately chose not to use bitwise operations.)
How could it be improved?
package main
import "fmt"
func main() {
a := []int{1,2,3,4,5,6,7,8,9,10}
fmt.Println(a)
rotateR(a, 5)
fmt.Println(a)
rotateL(a, 5)
fmt.Println(a)
}
func rotateL(a []int, i int) {
for count := 1; count <= i; count++ {
tmp := a[0]
for n := 1;n < len(a);n++ {
a[n-1] = a[n]
}
a[len(a)-1] = tmp
}
}
func rotateR(a []int, i int) {
for count := 1; count <= i; count++ {
tmp := a[len(a)-1]
for n := len(a)-2;n >=0 ;n-- {
a[n+1] = a[n]
}
a[0] = tmp
}
}
Rotating the slice one position at a time, and repeating to get the total desired rotation means it will take time proportional to rotation distance × length of slice. By moving each element directly into its final position you can do this in time proportional to just the length of the slice.
The code for this is a little more tricky than you have, and you’ll need a GCD function to determine how many times to go through the slice:
func gcd(a, b int) int {
for b != 0 {
a, b = b, a % b
}
return a
}
func rotateL(a []int, i int) {
// Ensure the shift amount is less than the length of the array,
// and that it is positive.
i = i % len(a)
if i < 0 {
i += len(a)
}
for c := 0; c < gcd(i, len(a)); c++ {
t := a[c]
j := c
for {
k := j + i
// loop around if we go past the end of the slice
if k >= len(a) {
k -= len(a)
}
// end when we get to where we started
if k == c {
break
}
// move the element directly into its final position
a[j] = a[k]
j = k
}
a[j] = t
}
}
Rotating a slice of size l right by p positions is equivalent to rotating it left by l − p positions, so you can simplify your rotateR function by using rotateL:
func rotateR(a []int, i int) {
rotateL(a, len(a) - i)
}
Your code is fine for in-place modification.
Don't clearly understand what you mean by bitwise operations. Maybe this
package main
import "fmt"
func main() {
a := []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10}
fmt.Println(a)
rotateR(&a, 4)
fmt.Println(a)
rotateL(&a, 4)
fmt.Println(a)
}
func rotateL(a *[]int, i int) {
x, b := (*a)[:i], (*a)[i:]
*a = append(b, x...)
}
func rotateR(a *[]int, i int) {
x, b := (*a)[:(len(*a)-i)], (*a)[(len(*a)-i):]
*a = append(b, x...)
}
Code works https://play.golang.org/p/0VtiRFQVl7
It's called reslicing in Go vocabulary. Tradeoff is coping and looping in your snippet vs dynamic allocation in this. It's your choice, but in case of shifting 10000 elements array by one position reslicing looks much cheaper.
I like Uvelichitel solution but if you would like modular arithmetic which would be O(n) complexity
package main
func main(){
s := []string{"1", "2", "3"}
rot := 5
fmt.Println("Before RotL", s)
fmt.Println("After RotL", rotL(rot, s))
fmt.Println("Before RotR", s)
fmt.Println("After RotR", rotR(rot,s))
}
func rotL(m int, arr []string) []string{
newArr := make([]string, len(arr))
for i, k := range arr{
newPos := (((i - m) % len(arr)) + len(arr)) % len(arr)
newArr[newPos] = k
}
return newArr
}
func rotR(m int, arr []string) []string{
newArr := make([]string, len(arr))
for i, k := range arr{
newPos := (i + m) % len(arr)
newArr[newPos] = k
}
return newArr
}
If you need to enter multiple values, whatever you want (upd code Uvelichitel)
package main
import "fmt"
func main() {
var N, n int
fmt.Scan(&N)
a := make([]int, N)
for i := 0; i < N; i++ {
fmt.Scan(&a[i])
}
fmt.Scan(&n)
if n > 0 {
rotateR(&a, n%len(a))
} else {
rotateL(&a, (n*-1)%len(a))
}
for _, elem := range a {
fmt.Print(elem, " ")
}
}
func rotateL(a *[]int, i int) {
x, b := (*a)[:i], (*a)[i:]
*a = append(b, x...)
}
func rotateR(a *[]int, i int) {
x, b := (*a)[:(len(*a)-i)], (*a)[(len(*a)-i):]
*a = append(b, x...)
}

How do I reverse a slice in go?

How do I reverse an arbitrary slice ([]interface{}) in Go? I'd rather not have to write Less and Swap to use sort.Reverse. Is there a simple, builtin way to do this?
The standard library does not have a built-in function for reversing a slice. Use a for loop to reverse a slice:
for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
s[i], s[j] = s[j], s[i]
}
Use type parameters to write a generic reverse function in Go 1.18 or later:
func reverse[S ~[]E, E any](s S) {
for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
s[i], s[j] = s[j], s[i]
}
}
Use reflect.Swapper to write a function that works with arbitrary slice types in Go version 1.8 or later:
func reverse(s interface{}) {
n := reflect.ValueOf(s).Len()
swap := reflect.Swapper(s)
for i, j := 0, n-1; i < j; i, j = i+1, j-1 {
swap(i, j)
}
}
Run the code on the Go playground.
The functions in this answer reverse the slice inplace. If you do not want to modify the original slice, copy the slice before reversing the slice.
Here's another possible way to reverse generic slice (go 1.18)
// You can edit this code!
// Click here and start typing.
package main
import (
"fmt"
"sort"
)
func main() {
nums := []int64{10, 5, 15, 20, 1, 100, -1}
ReverseSlice(nums)
fmt.Println(nums)
strs := []string{"hello", "world"}
ReverseSlice(strs)
fmt.Println(strs)
runes := []rune{'h', 'e', 'l', 'l', 'o', 'w', 'o', 'r', 'l', 'd'}
ReverseSlice(runes)
for _, r := range runes {
fmt.Print(string(r), " ")
}
}
func ReverseSlice[T comparable](s []T) {
sort.SliceStable(s, func(i, j int) bool {
return i > j
})
}
Running the program above should output:
[-1 100 1 20 15 5 10]
[world hello]
d l r o w o l l e h
Program exited.
go playground
This will return a reversed slice without modifying the original slice.
Algorithm used from official wiki page: https://github.com/golang/go/wiki/SliceTricks#reversing
func reverse(s []interface{}) []interface{} {
a := make([]interface{}, len(s))
copy(a, s)
for i := len(a)/2 - 1; i >= 0; i-- {
opp := len(a) - 1 - i
a[i], a[opp] = a[opp], a[i]
}
return a
}
There are my code example, you can run it in playground
package main
import (
"fmt"
"reflect"
"errors"
)
func ReverseSlice(data interface{}) {
value := reflect.ValueOf(data)
if value.Kind() != reflect.Slice {
panic(errors.New("data must be a slice type"))
}
valueLen := value.Len()
for i := 0; i <= int((valueLen-1)/2); i++ {
reverseIndex := valueLen - 1 - i
tmp := value.Index(reverseIndex).Interface()
value.Index(reverseIndex).Set(value.Index(i))
value.Index(i).Set(reflect.ValueOf(tmp))
}
}
func main() {
names := []string{"bob", "mary", "sally", "michael"}
ReverseSlice(names)
fmt.Println(names)
}
Here is the function I'm using with generics (go 1.18+). You can use it to reverse any kind of slice or even a string (using the split/join trick). It doesn't change the original slice.
package main
import (
"fmt"
"strings"
)
func Reverse[T any](original []T) (reversed []T) {
reversed = make([]T, len(original))
copy(reversed, original)
for i := len(reversed)/2 - 1; i >= 0; i-- {
tmp := len(reversed) - 1 - i
reversed[i], reversed[tmp] = reversed[tmp], reversed[i]
}
return
}
func main() {
a := []string{"a", "b", "c"}
fmt.Println(a, Reverse(a))
b := []uint{0, 1, 2}
fmt.Println(b, Reverse(b))
c := "abc"
fmt.Println(c, strings.Join(Reverse(strings.Split(c, "")), ""))
}
Better Go Playground
This generic slice reversal function should do it for you:
func ReverseSlice[T comparable](s []T) []T {
var r []T
for i := len(s) - 1; i >= 0; i-- {
r = append(r, s[i])
}
return r
}

Sorting a Map of Structs - GOLANG

I have a map of structs that I am populating by streaming data to a Go program. The way the map is updated is similar to the example below.
Once I have this map of structs populated, what is the best (or good) way to sort this map by the values of the count field in the struct?
package main
type data struct {
count int64
}
func main() {
m := make(map[string]data)
m["x"] = data{0, 0}
if xx, ok := m["x"]; ok {
xx.count = 2
m["x"] = xx
} else {
panic("X isn't in the map")
}
}
This example can be run here: http://play.golang.org/p/OawL6QIXuO
As siritinga already pointed out, the elements of a map isn't ordered, so you cannot sort it.
What you can do is to create a slice and sort the elements using the sort package:
package main
import (
"fmt"
"sort"
)
type dataSlice []*data
type data struct {
count int64
size int64
}
// Len is part of sort.Interface.
func (d dataSlice) Len() int {
return len(d)
}
// Swap is part of sort.Interface.
func (d dataSlice) Swap(i, j int) {
d[i], d[j] = d[j], d[i]
}
// Less is part of sort.Interface. We use count as the value to sort by
func (d dataSlice) Less(i, j int) bool {
return d[i].count < d[j].count
}
func main() {
m := map[string]*data {
"x": {0, 0},
"y": {2, 9},
"z": {1, 7},
}
s := make(dataSlice, 0, len(m))
for _, d := range m {
s = append(s, d)
}
// We just add 3 to one of our structs
d := m["x"]
d.count += 3
sort.Sort(s)
for _, d := range s {
fmt.Printf("%+v\n", *d)
}
}
Output:
{count:1 size:7}
{count:2 size:9}
{count:3 size:0}
Playground
Edit
Updated the example to use pointers and to include a map so that you can both do lookups and have a slice to sort over.

bytewise compare varint encoded int64's

I'm using levigo, the leveldb bindings for Go. My keys are int64's and need to be kept sorted. By default, leveldb uses a bytewise comparator so I'm trying to use varint encoding.
func i2b(x int64) []byte {
b := make([]byte, binary.MaxVarintLen64)
n := binary.PutVarint(b, x)
return key[:n]
}
My keys are not being sorted correctly. I wrote the following as a test.
var prev int64 = 0
for i := int64(1); i < 1e5; i++ {
if bytes.Compare(i2b(i), i2b(prev)) <= 0 {
log.Fatalf("bytewise: %d > %d", b2i(prev), i)
}
prev = i
}
output: bytewise: 127 > 128
playground
I'm not sure where the problem is. Am I doing the encoding wrong? Is varint not the right encoding to use?
EDIT:
BigEndian fixed width encoding is bytewise comparable
func i2b(x int64) []byte {
b := make([]byte, 8)
binary.BigEndian.PutUint64(b, uint64(x))
return b
}
The varint encoding is not bytewise comparable* wrt to the order of the values it caries. One option how to write the ordering/collating function (cmp bellow) is for example:
package main
import (
"encoding/binary"
"log"
)
func i2b(x int64) []byte {
var b [binary.MaxVarintLen64]byte
return b[:binary.PutVarint(b[:], x)]
}
func cmp(a, b []byte) int64 {
x, n := binary.Varint(a)
if n < 0 {
log.Fatal(n)
}
y, n := binary.Varint(b)
if n < 0 {
log.Fatal(n)
}
return x - y
}
func main() {
var prev int64 = 0
for i := int64(1); i < 1e5; i++ {
if cmp(i2b(i), i2b(prev)) <= 0 {
log.Fatal("fail")
}
prev = i
}
}
Playground
(*) The reason is (also) the bit fiddling performed.

Resources