Space that is available at all Tarantool nodes (vshard) - tarantool

I use Tarantool with vshard module. When bucket_id is set, the data is distributed across the cluster. Each node has its own set of bucket_ids. How to make a space so that it is fully accessible on every node (dictionary)?

I use the following snippet to run a function on all storages (may be there's a better way).
Suppose we have a function on storages:
local function putMyData(rows)
box.atomic(function()
for _, row in ipairs(rows)
box.mySpace:put(row)
end
end)
end
box.schema.func.create("putMyData", { if_not_exists = true })
box.schema.role.grant("public", "execute", "function", "putMyData", { if_not_exists = true })
rawset(_G, "putMyData", putMyData)
Then the following helper may be used to call the function:
local function callAll(mode, fnName, args, resHandler, timeoutSec)
local replicaSets, err = vshard.router.routeall()
if err ~= nil then
error(err)
end
local count = 0
for _, _ in pairs(replicaSets) do
count = count + 1
end
local channel = fiber.channel(count)
local method
if mode == "read" then
method = "callbro"
else
method = "callrw"
end
for _, replicaSet in pairs(replicaSets) do
fiber.create(
function()
local res, fErr = replicaSet[method](replicaSet,
fnName, args, { timeout = timeoutSec or opts.timeout })
channel:put({ res = res, err = fErr })
end)
end
local results = { }
for i = 1, count do
local val = channel:get()
if val.err ~= nil then
error(val.err)
end
if resHandler == nil then
results[i] = val.res
else
resHandler(val.res, results)
end
end
return results
end

Related

Lua - FlashAir card CONFIG file gets damaged

I am using Toshiba FlashAir card to wirelessly upload photos from my drone to my web server via Lua script installed on FlashAir card. The card works fine most of the time, but sometimes the photos doesn't upload. When I connect to the card via my mac, I get this message, "CONFIG" is damaged and can't be opened.
Not sure why this is happening. The ftpUpload.lua script that does the uploading is listed below:
require("Settings")
local blackList = {}
local dateTable = {}
local parseThreads = 0
local ftpstring = "ftp://"..user..":"..passwd.."#"..server.."/"..serverDir.."/"
local lockdir = "/uploaded/" -- trailing slash required, and folder must already exist
local fileTable = {}
--Options
local sortByFileName = false -- If true sorts alphabetically by file else sorts by modified date
local jpegsOnly = true --Excludes .RAW files if set to true
local function exists(path)
if lfs.attributes(path) then
return true
else
return false
end
end
local function is_uploaded(path)
local hash = fa.hash("md5", path, "")
return exists(lockdir .. hash)
end
local function set_uploaded(path)
local hash = fa.hash("md5", path, "")
local file = io.open(lockdir .. hash, "w")
file:close()
end
local function delete(path)
-- Both of the following methods cause the next photo to be lost / not stored.
fa.remove(path)
-- fa.request("http://127.0.0.1/upload.cgi?DEL="..path)
end
--Format file date to something meanigful
local function FatDateTimeToTable(datetime_fat)
local function getbits(x, from, to)
local mask = bit32.lshift(1, to - from + 1) - 1
local shifted = bit32.rshift(x, from)
return bit32.band(shifted, mask)
end
local fatdate = bit32.rshift(datetime_fat, 16)
local day = getbits(fatdate, 0, 4)
local month = getbits(fatdate, 5, 8)
local year = getbits(fatdate, 9, 15) + 1980
local fattime = getbits(datetime_fat, 0, 15)
local sec = getbits(fattime, 0, 4) * 2
local min = getbits(fattime, 5, 10)
local hour = getbits(fattime, 11, 15)
return {
year = string.format('%02d', year),
month = string.format('%02d', month),
day = string.format('%02d', day),
hour = string.format('%02d', hour),
min = string.format('%02d', min),
sec = string.format('%02d', sec),
}
end
--Looks for value in table
local function exists_in_table(tbl, val)
for i = 1, #tbl do
if (tbl[i] == val) then
return true
end
end
return false
end
local function create_thumbs()
print("Creating Thumbs!")
for i = 1, #dateTable do
local message = "d="..dateTable[i]
local headers = {
["Content-Length"] = string.len(message),
["Content-Type"] = "application/x-www-form-urlencoded"
}
local b, c, h = fa.request{
url = "http://"..server.."/"..serverDir.."/ct.php",
method = "POST",
headers = headers,
body = message
}
end
print("COMPLETE!")
end
local function upload_file(folder, file, subfolder)
local path = folder .. "/" .. file
-- Open the log file
local outfile = io.open(logfile, "a")
outfile:write(file .. " ... ")
local url = ftpstring..subfolder.."/"..file
local response = fa.ftp("put", url, path)
--print("Uploading", url)
--Check to see if it worked, and log the result!
if response ~= nil then
print("Success!")
outfile:write(" Success!\n")
set_uploaded(path)
if delete_after_upload == true then
print("Deleting " .. file)
outfile:write("Deleting " .. file .. "\n")
sleep(1000)
delete(path)
sleep(1000)
end
else
print(" Fail ")
outfile:write(" Fail\n")
end
--Close our log file
outfile:close()
end
local function sort_upload_table()
if (sortByFileName) then
print("Sorting filenames alphabetically")
table.sort(fileTable, function(a,b)
return a.file>b.file
end)
else
print("Sorting filenames by modded date")
table.sort(fileTable, function(a,b)
return a.sortDate>b.sortDate
end)
end
end
local function run_upload()
print("Starting upload")
for i = 1, #fileTable do
local ft = fileTable[i]
print("Uploading:", ft.folder, ft.file, ft.dateString)
upload_file(ft.folder, ft.file, ft.dateString)
end
create_thumbs()
end
local function walk_directory(folder)
parseThreads = parseThreads+1
for file in lfs.dir(folder) do
local path = folder .. "/" .. file
local skip = string.sub( file, 1, 2 ) == "._"
local attr = lfs.attributes(path)
local dt={}
if (not skip) then
print( "Found "..attr.mode..": " .. path )
if attr.mode == "file" then
local dateString = ""
if (attr.modification~=nil) then
dt = FatDateTimeToTable(attr.modification)
dateString = dt.day.."-"..dt.month.."-"..dt.year
end
if not is_uploaded(path) then
if (not exists_in_table(blackList, dateString)) then
local s,f = true, true
if (jpegsOnly) then
s,f = string.find(string.lower(file), ".jpg")
end
if (s and f) then
fileTable[#fileTable+1] =
{
folder = folder,
file = file,
dateString = dateString,
sortDate=dt.year..dt.month..dt.day..dt.hour..dt.min..dt.sec
}
--upload_file(folder, file, dateString)
end
else
print("Skipping ".. dateString.." - Blacklisted")
end
else
print(path .. " previously uploaded, skipping")
end
elseif attr.mode == "directory" then
print("Entering " .. path)
walk_directory(path)
end
end
end
parseThreads = parseThreads-1
if (parseThreads == 0) then
--create_thumbs()
sort_upload_table()
run_upload()
end
end
local function create_folders(folder)
if (#dateTable==0) then
print("ERROR: DID NOT FIND ANY DATES!")
return
end
for i = 1, #dateTable do
local message = "d="..dateTable[i]
local headers = {
["Content-Length"] = string.len(message),
["Content-Type"] = "application/x-www-form-urlencoded"
}
local b, c, h = fa.request{
url = "http://"..server.."/"..serverDir.."/cd.php",
method = "POST",
headers = headers,
body = message
}
if (b~=nil) then
b = string.gsub(b, "\n", "")
b = string.gsub(b, "\r", "")
end
if (b and b == "success") then
print("SUCCESS FROM SERVER FOR FOLDER:"..dateTable[i].."<<")
else
print("FAILED RESPONSE FROM SERVER FOR FOLDER:"..dateTable[i].."<<")
print("ADDING TO BLACKLIST")
blackList[#blackList+1] = dateTable[i]
end
end
print("OK FTP Starting...")
walk_directory(folder)
end
local function get_folders(folder)
parseThreads = parseThreads+1
local tableCount = 1
--Get the date range from the file
for file in lfs.dir(folder) do
local path = folder .. "/" .. file
local skip = string.sub( file, 1, 2 ) == "._"
local attr = lfs.attributes(path)
if (not skip) then
if (attr.mode == "file") then
print( "Datesearch Found "..attr.mode..": " .. path )
local dateString = ""
if (attr.modification~=nil) then
local dt = FatDateTimeToTable(attr.modification)
dateString = dt.day.."-"..dt.month.."-"..dt.year
end
if (not exists_in_table(dateTable, dateString)) then
dateTable[#dateTable+1] = dateString
end
elseif attr.mode == "directory" then
print("Datesearch Entering " .. path)
get_folders(path)
end
end
end
parseThreads = parseThreads-1
if (parseThreads == 0) then
create_folders(folder)
end
end
-- wait for wifi to connect
while string.sub(fa.ReadStatusReg(),13,13) ~= "a" do
print("Wifi not connected. Waiting...")
sleep(1000)
end
sleep(30*1000)
get_folders(folder)

spark streaming mapwithstate's confuse

as the code show:
val mappedData = new ArrayBuffer[E]
val wrappedState = new StateImpl[S]()
// Call the mapping function on each record in the data iterator, and accordingly
// update the states touched, and collect the data returned by the mapping function
dataIterator.foreach { case (key, value) =>
wrappedState.wrap(newStateMap.get(key))
val returned = mappingFunction(batchTime, key, Some(value), wrappedState)
if (wrappedState.isRemoved) {
newStateMap.remove(key)
} else if (wrappedState.isUpdated
|| (wrappedState.exists && timeoutThresholdTime.isDefined)) {
newStateMap.put(key, wrappedState.get(), batchTime.milliseconds)
}
mappedData ++= returned
}
// Get the timed out state records, call the mapping function on each and collect the
// data returned
if (removeTimedoutData && timeoutThresholdTime.isDefined) {
newStateMap.getByTime(timeoutThresholdTime.get).foreach { case (key, state, _) =>
wrappedState.wrapTimingOutState(state)
val returned = mappingFunction(batchTime, key, None, wrappedState)
mappedData ++= returned
newStateMap.remove(key)
}
}
Spark streaming mapWithState timeout delayed?
deltaMap ll mark for deleting when the key's removeTimedoutData=true
override def remove(key: K): Unit = {
val stateInfo = deltaMap(key)
if (stateInfo != null) {
stateInfo.markDeleted()
} else {
val newInfo = new StateInfo[S](deleted = true)
deltaMap.update(key, newInfo)
}
}
the openhashmap ll remove the key when DELTA_CHAIN_LENGTH_THRESHOLD >= 20
my question is :
1: a key which in this current batch timeout ll be executed because "wrappedState.exists && timeoutThresholdTime.isDefined" and "removeTimedoutData && timeoutThresholdTime.isDefined " all true when checkpoint invokes
override def checkpoint(): Unit = {
super.checkpoint()
doFullScan = true
}
but what does it means by executing "mappedData ++= returned" two times for a timeout key.
val returned = mappingFunction(batchTime, key, Some(value), wrappedState)
mappedData ++= returned
and
val returned = mappingFunction(batchTime, key, None, wrappedState)
mappedData ++= returned
2: when a key is marked for delete but not remove from openhashmap ,then next batch data which contain this key comes,"wrappedState.exists && timeoutThresholdTime.isDefined" and "removeTimedoutData && timeoutThresholdTime.isDefined" still all true and they ll be executed another times?
i reviewed the code of mapwithstate again and got answer
my question is :
1: a key which in this current batch timeout ll be executed because "wrappedState.exists && timeoutThresholdTime.isDefined" and "removeTimedoutData && timeoutThresholdTime.isDefined " all true when checkpoint invokes
a key this current batch timeout then the key have to wait for "doFullScan = true" which means "batchtime*DEFAULT_CHECKPOINT_DURATION_MULTIPLIER(defalut:10)",now the key already timeout for sometimes and "doFullScan = true",the key ll be update it's state first:
if (wrappedState.isUpdated|| (wrappedState.exists && timeoutThresholdTime.isDefined)) {
newStateMap.put(key, wrappedState.get(), batchTime.milliseconds)
}
then the key meet the conditions :
if (removeTimedoutData && timeoutThresholdTime.isDefined) {
newStateMap.getByTime(timeoutThresholdTime.get).foreach { case (key, state, _) =>
wrappedState.wrapTimingOutState(state)
val returned = mappingFunction(batchTime, key, None, wrappedState)
mappedData ++= returned
newStateMap.remove(key)
}
in this code,first of all Get all the keys and states whose updated time is older than the give threshold time and then change the key's flag "defined = true timingOut = true removed = false updated = false"
the code "mappingFunction(batchTime, key, None, wrappedState)" ll be executed and the value of this key ll be None and in mapwihtstateFunction u can do something by "if(state.isTimeout){dosomething}"
questions 2
a key was marked as delete in stateMap then it ll be remove form openhashmap when "DELTA_CHAIN_LENGTH_THRESHOLD >= 20" but even if the key not be removed from openhashmap and u can not get it from statMap because statemap do not return a key with "delete flag"

Scala/functional/without libs - check if string permutation of other

How could you check to see if one string is a permutation of another using scala/functional programming with out complex pre-built functions like sorted()?
I'm a Python dev and what I think trips me up the most is that you can't just iterate through a dictionary of character counts comparing to another dictionary of character counts, then just exit when there isn't a match, you can't just call break.
Assume this is the starting point, based on your description:
val a = "aaacddba"
val b = "aabaacdd"
def counts(s: String) = s.groupBy(identity).mapValues(_.size)
val aCounts = counts(a)
val bCounts = counts(b)
This is the simplest way:
aCounts == bCounts // true
This is precisely what you described:
def isPerm(aCounts: Map[Char,Int], bCounts: Map[Char,Int]): Boolean = {
if (aCounts.size != bCounts.size)
return false
for ((k,v) <- aCounts) {
if (bCounts.getOrElse(k, 0) != v)
return false
}
return true
}
This is your method, but more scala-ish. (It also breaks as soon as a mismatch is found, because of how foreach is implemented):
(aCounts.size == bCounts.size) &&
aCounts.forall { case (k,v) => bCounts.getOrElse(k, 0) == v }
(Also, Scala does have break.)
Also, also: you should read the answer to this question.
Another option using recursive function, which will also 'break' immediately once mismatch is detected:
import scala.annotation.tailrec
#tailrec
def isPerm1(a: String, b: String): Boolean = {
if (a.length == b.length) {
a.headOption match {
case Some(c) =>
val i = b.indexOf(c)
if (i >= 0) {
isPerm1(a.tail, b.substring(0, i) + b.substring(i + 1))
} else {
false
}
case None => true
}
} else {
false
}
}
Out of my own curiosity I also create two more versions which use char counts map for matching:
def isPerm2(a: String, b: String): Boolean = {
val cntsA = a.groupBy(identity).mapValues(_.size)
val cntsB = b.groupBy(identity).mapValues(_.size)
cntsA == cntsB
}
and
def isPerm3(a: String, b: String): Boolean = {
val cntsA = a.groupBy(identity).mapValues(_.size)
val cntsB = b.groupBy(identity).mapValues(_.size)
(cntsA == cntsB) && cntsA.forall { case (k, v) => cntsB.getOrElse(k, 0) == v }
}
and roughly compare their performance by:
def time[R](block: => R): R = {
val t0 = System.nanoTime()
val result = block // call-by-name
val t1 = System.nanoTime()
println("Elapsed time: " + (t1 - t0) + "ns")
result
}
// Match
time((1 to 10000).foreach(_ => isPerm1("apple"*100,"elppa"*100)))
time((1 to 10000).foreach(_ => isPerm2("apple"*100,"elppa"*100)))
time((1 to 10000).foreach(_ => isPerm3("apple"*100,"elppa"*100)))
// Mismatch
time((1 to 10000).foreach(_ => isPerm1("xpple"*100,"elppa"*100)))
time((1 to 10000).foreach(_ => isPerm2("xpple"*100,"elppa"*100)))
time((1 to 10000).foreach(_ => isPerm3("xpple"*100,"elppa"*100)))
and the result is:
Match cases
isPerm1 = 2337999406ns
isPerm2 = 383375133ns
isPerm3 = 382514833ns
Mismatch cases
isPerm1 = 29573489ns
isPerm2 = 381622225ns
isPerm3 = 417863227ns
As can be expected, the char counts map speeds up positive cases but can slow down negative cases (overhead on building the char counts map).

Efficiently storing and iterating over Lua multi-level queue

This is the add format:
AddToQueue( String identifier, function callfunc, int priority )
priority is guaranteed to be 0 to 4, with 0 being the highest priority. My current setup is this:
local tQueue = {
[0] = {},
[1] = {},
[2] = {},
[3] = {},
[4] = {}
}
function AddToQueue( sName, funcCallback, iPriority )
queue[iPriority or 0][sName] = funcCallback
end
function CallQueue()
for i = 0, 4 do
for _, func in pairs( queue[i] ) do
local retval = func()
if ( retval ~= nil ) then
return retval
end
end
end
end
This works, but I want to know if there's a better way to store and iterate the functions to prevent doing 5 pair loops every call. Thanks!
If you iterate your queue frequently, and new callbacks additions is infrequent, then you can just store everything in single table, sorting it each time you've added new callback.

Lua - how to sort a table by the value chain

I'm looking for a method of sorting a Lua table by its values chain. Say, the table:
local vals = {
{ id = "checkpoint4" },
{ id = "checkpoint1", nextid = "checkpoint2" },
{ id = "checkpoint3", nextid = "checkpoint4" },
{ id = "checkpoint2", nextid = "checkpoint3" },
}
Should transform into this after sorting:
local vals = {
{ id = "checkpoint1", nextid = "checkpoint2" },
{ id = "checkpoint2", nextid = "checkpoint3" },
{ id = "checkpoint3", nextid = "checkpoint4" },
{ id = "checkpoint4" },
}
It's not essentially with the exact same names, they might vary. I wanted to make the comparison of numbers after "checkpoint", but it turned out that I have to work with dynamic things like this (already sorted the way I want it to be):
local vals = {
{ id = "checkpoint1", nextid = "cp" },
{ id = "cp", nextid = "chp" },
{ id = "chp", nextid = "mynextcheckpoint" },
{ id = "mynextcheckpoint"},
}
Thanks.
What you are describing is called topological sorting. However, since this is a restricted case, we do not have to implement a complete topological sorting algorithm:
function sort_list(tbl)
local preceding = {}
local ending
local sorted = {}
for i, e in ipairs(tbl) do
if e.nextid == nil then
ending = e
else
preceding[e.nextid] = i
end
end
if ending == nil then
return nil, "no ending"
end
local j = #tbl
while ending ~= nil do
sorted[j] = ending
ending = tbl[preceding[ending.id]]
j = j - 1
end
if sorted[1] == nil then
return nil, "incomplete list"
end
return sorted
end
Usage:
local sorted = sort_list(vals)
local id2val, tailsizes = {}, {}
for _, val in ipairs(vals) do id2val[val.id] = val end
local function tailsize(val) -- memoized calculation of tails sizes
if not tailsizes[val] then
tailsizes[val] = 0 -- cycle-proof
if val.nextid and id2val[val.nextid] then -- dangling nextid proof
tailsizes[val] = tailsize(id2val[val.nextid]) + 1
end
end
return tailsizes[val]
end
-- sorting according to tails sizes
table.sort(vals, function(a,b) return tailsize(a) > tailsize(b) end)

Resources