quantstrat::apply.paramset error: attempt to select less than one element - quantstrat

I modified quantstrat demo bee and tried to run optimization on indicator parameters. However, I have tried to simplify optimization as much as I can, but I still got the same error as following:
error calling combine function:
<simpleError in fun(result.1, result.2, result.3, result.4, result.5, result.6, result.7, result.8, result.9, result.10): attempt to select less than one element>
Error in .subset2(x, i, exact = exact) : subscript out of bounds
my sessionInfo output:
R version 3.2.4 (2016-03-10)
Platform: x86_64-apple-darwin13.4.0 (64-bit)
Running under: OS X 10.11.4 (El Capitan)
locale:
[1] en_US.UTF-8/en_US.UTF-8/en_US.UTF-8/C/en_US.UTF-8/en_US.UTF-8
attached base packages:
[1] parallel stats graphics grDevices utils
[6] datasets methods base
other attached packages:
[1] doMC_1.3.4
[2] iterators_1.0.8
[3] lubridate_1.5.0
[4] dplyr_0.4.3
[5] quantstrat_0.9.1739
[6] foreach_1.4.3
[7] blotter_0.9.1741
[8] PerformanceAnalytics_1.4.4000
[9] FinancialInstrument_1.2.0
[10] quantmod_0.4-5
[11] TTR_0.23-1
[12] xts_0.9-7
[13] zoo_1.7-12
loaded via a namespace (and not attached):
[1] Rcpp_0.12.3 magrittr_1.5
[3] lattice_0.20-33 R6_2.1.2
[5] stringr_1.0.0 tools_3.2.4
[7] grid_3.2.4 DBI_0.3.1
[9] htmltools_0.3 yaml_2.1.13
[11] lazyeval_0.1.10 assertthat_0.1
[13] digest_0.6.9 codetools_0.2-14
[15] rsconnect_0.4.1.4 rmarkdown_0.9.5
[17] stringi_1.0-1 compiler_3.2.4
Here is my code and data is to be download from here. Could anyone help me to figure out where goes wrong? thanks
####################################################################
#### Load packages
#####################################################################
library(dplyr)
suppressMessages(require(quantstrat))
####################################################################
#### remove objects
########################################################################
rm(list = ls(all.names = T))
if(!exists(".instrument")) .instrument <<- new.env()
if(!exists(".blotter")) .blotter <<- new.env()
if(!exists(".strategy")) .strategy <- new.env()
########################################################################
#### DEFINE VARIABLES or parameters
########################################################################
initDate = "2000-01-01"
symbol.st = 'CYB_DAY'
portf.st = 'bug'
acct.st = 'colony'
strat.st = 'bee'
initEq = 100000
nFast = 10
nSlow = 30
nSd = 1
########################################################################
#### GET DATA
###############################################################
# set your working directory where the data is stored
setwd("/Users/Natsume/Documents/data_exploration_r/data")
currency('USD') # initiate currency
stock(symbol.st ,currency='USD', multiplier=1) # initiate stock
getSymbols(Symbols = symbol.st, src = "csv")
initPortf(
portf.st,
symbol.st,
initDate=initDate) # initiate portfolio
initAcct(
acct.st,
portf.st,
initEq=initEq,
initDate=initDate) # initiate account
initOrders(
portf.st,
initDate=initDate ) # initiate order_book
bee = strategy(strat.st) # create strategy object
addPosLimit(
portfolio=portf.st,
symbol=symbol.st,
timestamp=initDate,
maxpos=300, longlevels = 3) # only trade in one direction once
bee <- add.indicator(
strategy = strat.st,
name = 'BBands', # TA name
arguments = list(HLC=quote(HLC(mktdata)),
n=nSlow,
sd=nSd),
label = 'BBand')
#### SMA column
bee <- add.indicator(
strategy = strat.st,
name = 'SMA', # TA name
arguments = list(x=quote(Cl(mktdata)),
n=nFast),
label = 'MA' )
bee <- add.signal(
strategy = strat.st,
name = 'sigCrossover',
arguments = list(columns=c('MA','dn'),
relationship='lt'),
label = 'MA.lt.dn')
#### SMA cross over upperBand
bee <- add.signal(
strategy = strat.st,
name = 'sigCrossover',
arguments = list(columns=c('MA','up'),
relationship='gt'),
label = 'MA.gt.up')
bee <- add.rule(
strategy = strat.st,
name = 'ruleSignal',
arguments = list(sigcol = 'MA.gt.up',
sigval = TRUE,
replace = F,
orderqty = 100,
ordertype = 'market',
# one of "market","limit","stoplimit", "stoptrailing", or "iceberg"
orderside = 'long',
osFUN = 'osMaxPos'),
type = 'enter',
label = 'EnterLONG')
#### exitLong when SMA cross under LowerBand
bee <- add.rule(
strategy = strat.st,
name = 'ruleSignal',
arguments = list(sigcol = 'MA.lt.dn',
sigval = TRUE,
replace = F,
orderqty = 'all',
ordertype = 'market',
orderside = 'long'),
type = 'exit',
label = 'ExitLONG')
#### enterShort when SMA cross under LowerBand
bee <- add.rule(
strategy = strat.st,
name = 'ruleSignal',
arguments = list(sigcol = 'MA.lt.dn',
sigval = TRUE,
replace = F,
orderqty = -100,
ordertype = 'market',
orderside = 'short',
osFUN = 'osMaxPos'),
type = 'enter',
label = 'EnterSHORT')
#### exitShort when SMA cross over upperBand
bee <- add.rule(
strategy = strat.st,
name = 'ruleSignal',
arguments = list(sigcol = 'MA.gt.up',
sigval = TRUE,
replace = F,
orderqty = 'all',
ordertype = 'market',
orderside = 'short'),
type = 'exit',
label = 'ExitSHORT')
applyStrategy(
strat.st,
portf.st,
prefer='Open', # why prefer='Open'
verbose=T)
updatePortf(
portf.st) #,
updateAcct(
acct.st) # ,
updateEndEq(
Account = acct.st)#,
### User Set up pf parameter ranges to test
.nFastList = 5:13
.nSlowList = 10:40
.nSdList = 1:3
# number of random samples of the parameter distribution to use for random run
.nsamples = 10
add.distribution(strat.st,
paramset.label = 'SMA_BBparams',
component.type = 'indicator',
component.label = 'BBand', #this is the label given to the indicator in the strat
variable = list(n = .nSlowList),
label = 'BBandMA'
)
add.distribution(strat.st,
paramset.label = 'SMA_BBparams',
component.type = 'indicator',
component.label = 'BBand', #this is the label given to the indicator in the strat
variable = list(sd = .nSdList),
label = 'BBandSD'
)
add.distribution(strat.st,
paramset.label = 'SMA_BBparams',
component.type = 'indicator',
component.label = 'MA', #this is the label given to the indicator in the strat
variable = list(n = .nFastList),
label = 'MAn'
)
add.distribution.constraint(strat.st,
paramset.label = 'SMA_BBparams',
distribution.label.1 = 'BBandMA',
distribution.label.2 = 'MAn',
operator = '>',
label = 'BBandMA>MAn'
)
### parallel computing to speed up
if( Sys.info()['sysname'] == "Windows" )
{
library(doParallel)
# registerDoParallel(cores=detectCores())
registerDoSEQ()
} else {
library(doMC)
registerDoMC(cores=detectCores())
}
results <- apply.paramset(strat.st,
paramset.label='SMA_BBparams',
portfolio.st=portf.st,
account.st=acct.st,
nsamples= .nsamples, # take all options
#.nsamples, only take 10 samples
verbose=TRUE)
results$tradeStats %>% View()

Related

Error with the message "Cannot cast (inet::physicallayer::Ieee80211DimensionalTransmission*) to type 'const inet::physicallayer::IScalarSignal *'"

I use Omnet++ and Inet 4.4, I want to simulate a scenario to investigate the effect of interference, which is as follows:
In a network consisting of two pairs of nodes including a node called source and a node called destination, in which node source sends packets to node destination and is tuned in channel 2 of Ieee802.11b/g, let's examine the second pair consisting of node Node1 and node Node2, which node Node1 sends packets to node Node 4 and is tuned in channel 4.
But recently a problem appear very often and the simulations are stopped, the error is this:
check_and_cast(): Cannot cast (inet::physicallayer::Ieee80211DimensionalTransmission*) to type 'const inet::physicallayer::IScalarSignal *' -- in module (inet::physicallayer::Ieee80211Radio) AnalogModelShowcaseDistanceNetworkRM.source.wlan[1].radio (id=200), at t=0.001s, event #24
My omnetpp.ini is as:
[Config Distance]
network = AnalogModelShowcaseDistanceNetworkRM
sim-time-limit = 5s
# Maryam **.radio.packetErrorRate.result-recording-modes = +vector
# Maryam **.radio.bitErrorRate.result-recording-modes = +vector
# application parameters
*.source.numApps = 1
*.source.app[0].typename = "UdpBasicApp"
*.source.app[*].destAddresses = "destination"
*.source.app[*].destPort = 1000
*.source.app[*].messageLength = 1000byte
*.source.app[*].sendInterval = 1ms
*.destination.numApps = 1
*.destination.app[0].typename = "UdpSink"
*.destination.app[*].localPort = 1000
*.Node1.numApps = 1
*.Node1.app[0].typename = "UdpBasicApp"
*.Node1.app[*].destAddresses = "Node2"
*.Node1.app[*].destPort = 1001
*.Node1.app[*].messageLength = 1000byte
*.Node1.app[*].sendInterval = 1ms
*.Node2.numApps = 1
*.Node2.app[0].typename = "UdpSink"
*.Node2.app[*].localPort = 1001
*.source.numWlanInterfaces = 2
*.destination.numWlanInterfaces = 2
*.Node1.numWlanInterfaces = 2
*.Node2.numWlanInterfaces = 2
*.source.wlan[*].radio.typename = "Ieee80211DimensionalRadio"
*.destination.wlan[*].radio.typename = "Ieee80211DimensionalRadio"
*.Node*.wlan[*].radio.typename = "Ieee80211DimensionalRadio"
*.source.wlan[*].radio.centerFrequency = 2.412GHz
*.source.wlan[*].radio.bandwidth = 2MHz
*.source.wlan[*].radio.transmitter.power = 2mW
*.source.wlan[*].radio.transmitter.bitrate = 2Mbps
*.source.wlan[*].radio.transmitter.preambleDuration = 0s
*.source.wlan[*].radio.transmitter.headerLength = 96b
*.source.wlan[*].radio.transmitter.modulation = "BPSK"
*.source.wlan[*].radio.receiver.sensitivity = -85dBm
*.source.wlan[*].radio.receiver.energyDetection = -85dBm
*.source.wlan[*].radio.receiver.snirThreshold = 4dB
*.destination.wlan[*].radio.centerFrequency = 2.412GHz
*.destination.wlan[*].radio.bandwidth = 2MHz
*.destination.wlan[*].radio.transmitter.power = 2mW
*.destination.wlan[*].radio.transmitter.bitrate = 2Mbps
*.destination.wlan[*].radio.transmitter.preambleDuration = 0s
*.destination.wlan[*].radio.transmitter.headerLength = 96b
*.destination.wlan[*].radio.transmitter.modulation = "BPSK"
*.destination.wlan[*].radio.receiver.sensitivity = -85dBm
*.destination.wlan[*].radio.receiver.energyDetection = -85dBm
*.destination.wlan[*].radio.receiver.snirThreshold = 4dB
*.Node*.wlan[*].radio.centerFrequency = 2.412GHz
*.Node*.wlan[*].radio.bandwidth = 2MHz
*.Node*.wlan[*].radio.transmitter.power = 2mW
*.Node*.wlan[*].radio.transmitter.bitrate = 2Mbps
*.Node*.wlan[*].radio.transmitter.preambleDuration = 0s
*.Node*.wlan[*].radio.transmitter.headerLength = 96b
*.Node*.wlan[*].radio.transmitter.modulation = "BPSK"
*.Node*.wlan[*].radio.receiver.sensitivity = -85dBm
*.Node*.wlan[*].radio.receiver.energyDetection = -85dBm
*.Node*.wlan[*].radio.receiver.snirThreshold = 4dB
*.source.wlan[0].radio.channelNumber = 2
*.destination.wlan[0].radio.channelNumber = 2
*.Node1.wlan[0].radio.channelNumber = 4
*.Node2.wlan[0].radio.channelNumber = 4
# mobility parameters
*.destination.mobility.typename = "LinearMobility"
*.destination.mobility.initialMovementHeading = 0deg
*.destination.mobility.speed = 200mps
*.destination.mobility.constraintAreaMinX = 500m
*.destination.mobility.constraintAreaMaxX = 1200m
# wlan
*.source.**.transmitter.power = 12mW
*.source.**.displayCommunicationRange = true
**.backgroundNoise.power = -105dBm
**.wlan*.mac.*.rateSelection.dataFrameBitrate = 54Mbps
**.wlan*.mac.dcf.channelAccess.pendingQueue.packetCapacity = 14
# visualizer parameters
*.visualizer.*.numStatisticVisualizers = 2
*.visualizer.*.statisticVisualizer[0].signalName = "packetSentToUpper"
*.visualizer.*.statisticVisualizer[0].statisticExpression = "packetErrorRate"
*.visualizer.*.statisticVisualizer[0].sourceFilter = "*.destination.wlan[*].radio"
*.visualizer.*.statisticVisualizer[0].format = "packetErrorRate(Maryam): %v"
*.visualizer.*.statisticVisualizer[1].signalName = "packetSentToUpper"
*.visualizer.*.statisticVisualizer[1].statisticExpression = "minimumSnir"
*.visualizer.*.statisticVisualizer[1].sourceFilter = "*.destination.wlan[*].radio"
*.visualizer.*.statisticVisualizer[1].format = "SNIR(Maryam): %v"
*.visualizer.*.statisticVisualizer[1].placementHint = "topLeft"
*.visualizer.*.dataLinkVisualizer[0].displayLinks = true
*.visualizer.*.packetDropVisualizer[0].displayPacketDrops = true
*.visualizer.*.packetDropVisualizer[0].nodeFilter = "destination"
*.visualizer.*.packetDropVisualizer[0].labelFormat = "(Maryam) %r"
*.visualizer.*.infoVisualizer[0].displayInfos = true
*.visualizer.*.infoVisualizer[0].modules = "*.destination.app[0]"
How do I solve this?
You need to set the radioMedium type to dimensional as well
radioMedium: Ieee80211DimensionalRadioMedium {
parameters:
#display("p=62.247997,287.14398");
}
In the NED file.

How to correctly use Distributed Data Parallel when customizing 'parameters' in the model ?

I have customized a parameter in my model:
self.params = list(self.backbone.parameters())
for head in self.headlist:
self.params += list(head.parameters())
When I wrap my model with DDP, an error occurs when defining the optimizer
optimizer = optim.SGD(model.params, lr=FLAGS.lr, momentum=FLAGS.momentum, weight_decay=FLAGS.weight_decay)
AttributeError 'DistributedDataParallel' object has no attribute 'params '
I think the error is probably caused by my customized "self.params"
Is the following code correct:
model = torch.nn.parallel.DistributedDataParallel(model,device_ids=local_rank)
model_without_ddp = model.module
**
optimizer = optim.SGD(model_without_ddp.params, lr=FLAGS.lr, momentum=FLAGS.momentum, weight_decay=FLAGS.weight_decay)
Or is there any simpler code?
###################################
The detailed definition of the network is as follows:
class multiheadModel():
def __init__(self, num_heads, device, model_name):
self.device = device
self.num_heads = num_heads # global+K
if model_name == 'fcn8s':
self.backbone = VGG16_FCN8s(num_classes=19, backbone=1, head=0).to(device)
self.headlist = [VGG16_FCN8s(num_classes=19, backbone=0, head=1).to(device) for i in range(num_heads)]
self.model = VGG16_FCN8s(num_classes=19).to(device)
for name, param in self.backbone.named_parameters():
if ('conv3' in name) or ('conv4' in name):
param.requires_grad = True
else:
param.requires_grad = False
elif model_name == 'deeplab':
self.backbone = Res_Deeplab(num_classes=19, backbone=1, head=0).to(device)
self.headlist = [Res_Deeplab(num_classes=19, backbone=0, head=1).to(device) for i in range(num_heads)]
self.model = Res_Deeplab(num_classes=19).to(device)
for name, param in self.backbone.named_parameters():
if 'layer3' in name:
param.requires_grad = True
else:
param.required_grad = False
else:
print('ERROR : wrong model name')
sys.exit()
self.params = list(self.backbone.parameters())
for head in self.headlist:
self.params += list(head.parameters())
self.loss_fn = None
#self.k2head = {0:2,1:1,2:0,3:0,4:0,5:4,6:4,7:5}
#self.k2head = {0:2,1:1,2:0,3:0,4:0,5:3,6:3,7:4}
self.k2head = {0:2,1:1,2:0,3:0,4:3,5:3,6:4}
# set train and eval mode
def train(self):
self.backbone.train()
for head in self.headlist:
head.train()
def eval(self):
self.backbone.eval()
for head in self.headlist:
head.eval()
def computePredLoss(self, rgb, lbl, k):
x = self.backbone(rgb)
head_id = list(range(self.num_heads))
head_id.remove(self.k2head[k])
input_size = rgb.size()[2:]
loss = 0
for i in head_id:
pred = self.headlist[i](x)
pred = F.interpolate(pred, size=input_size, mode='bilinear', align_corners=True)
loss += self.loss_fn(pred, lbl)
return pred, loss
def forward(self, input):
output = {}
if "label" in input:
self.train()
pred,loss = self.computePredLoss(input['rgb'], input['label'], input['k'])
output['pred'], output['loss']=pred, loss
else:
self.eval()
x = self.backbone(input['rgb'])
k = -1
if "k" in input:
k = self.k2head[input['k']]
pred = self.headlist[k](x)
input_size = input['rgb'].size()[2:]
pred = F.interpolate(pred, size=input_size, mode='bilinear', align_corners=True)
output['pred'] = pred
return output
def validate(self, loader, k=-2):
self.eval()
if k!=-2:
val_metrics = StreamSegMetrics(19)
val_metrics.reset()
with torch.no_grad():
for i, (batch, rgb_batch) in enumerate(loader):
rgb_batch = rgb_batch.to(device=self.device, dtype=torch.float)
batch = batch.to(device=self.device, dtype=torch.int64)
input_size = rgb_batch.size()[2:]
x = self.backbone(rgb_batch)
pred = self.headlist[k](x)
pred = F.interpolate(pred, size=input_size, mode='bilinear', align_corners=True)
preds = pred.detach().max(dim=1)[1].cpu().numpy()
targets = batch.cpu().numpy()
val_metrics.update(targets, preds)
score = val_metrics.get_results()
else:
val_metrics = [StreamSegMetrics(19) for i in range(self.num_heads)]
for metric in val_metrics:
metric.reset()
with torch.no_grad():
for i, (batch, rgb_batch) in enumerate(loader):
rgb_batch = rgb_batch.to(device=self.device, dtype=torch.float)
batch = batch.to(device=self.device, dtype=torch.int64)
input_size = rgb_batch.size()[2:]
x = self.backbone(rgb_batch)
for k in range(self.num_heads):
pred = self.headlist[k](x)
pred = F.interpolate(pred, size=input_size, mode='bilinear', align_corners=True)
preds = pred.detach().max(dim=1)[1].cpu().numpy()
targets = batch.cpu().numpy()
val_metrics[k].update(targets, preds)
score = [val_metrics[k].get_results() for k in range(self.num_heads)]
return score
def getHeadPaths(self, model_path, iteration=-1):
head_paths = []
if '_iter' in model_path:
base_path = model_path.split('_iter')[0]
else:
base_path = model_path.split('.pth')[0]
if iteration==-1:
for i in range(self.num_heads-1):
head_paths.append(base_path+'_except_g'+chr(97+i)+'.pth')
head_paths.append(model_path)
else:
for i in range(self.num_heads-1):
head_paths.append(base_path+'_except_g'+chr(97+i)+'_iter'+str(iteration)+'.pth')
head_paths.append(base_path+'_iter'+str(iteration)+'.pth')
return head_paths
def save(self, model_path, iteration=-1):
self.model.load_state_dict(self.backbone.state_dict(), strict=False)
head_paths = self.getHeadPaths(model_path, iteration)
for i in range(self.num_heads):
self.model.load_state_dict(self.headlist[i].state_dict(), strict=False)
torch.save(self.model.state_dict(), head_paths[i])
def load(self, model_path):
iteration = -1
if '_iter' in model_path:
iteration = int(model_path.split('_iter')[1].split('.pth')[0])
self.model.load_state_dict(torch.load(model_path))
self.backbone.load_state_dict(self.model.state_dict(), strict=False)
head_paths = self.getHeadPaths(model_path, iteration)
existance = 1
for path in head_paths:
if os.path.isfile(path)==False:
existance = 0
if existance==1:
print('loading from multiheads')
for i in range(self.num_heads):
self.model.load_state_dict(torch.load(head_paths[i]))
self.headlist[i].load_state_dict(self.model.state_dict(), strict=False)
else:
print('loading from singlehead')
for i in range(self.num_heads):
self.model.load_state_dict(torch.load(head_paths[-1]))
self.headlist[i].load_state_dict(self.model.state_dict(), strict=False)
def __call__(self, input):
return self.forward(input)

tensorflow: After adding a rnn the whole work doesn't work

I have downloaded a code of FCN for image segmentation and it ran well. Now I want to add a rnn layer attempting to refine the result according to the work "ReSeg: A Recurrent Neural Network-Based Model for Semantic Segmentation". My code shows as follows:
This part is for the inference:
def inference(image, keep_prob):
"""
Semantic segmentation network definition
:param image: input image. Should have values in range 0-255
:param keep_prob:
:return:
"""
print("setting up vgg initialized conv layers ...")
#model_data = utils.get_model_data(FLAGS.model_dir, MODEL_URL)
model_data = scipy.io.loadmat("H:/Deep Learning/FCN.tensorflow-master/imagenet-vgg-verydeep-19.mat")
mean = model_data['normalization'][0][0][0]
mean_pixel = np.mean(mean, axis=(0, 1))
weights = np.squeeze(model_data['layers'])
processed_image = utils.process_image(image, mean_pixel)
with tf.variable_scope("inference"):
image_net = vgg_net(weights, processed_image)
conv_final_layer = image_net["conv5_3"]
pool5 = utils.max_pool_2x2(conv_final_layer)
W6 = utils.weight_variable([7, 7, 512, 4096], name="W6")
b6 = utils.bias_variable([4096], name="b6")
conv6 = utils.conv2d_basic(pool5, W6, b6)
relu6 = tf.nn.relu(conv6, name="relu6")
if FLAGS.debug:
utils.add_activation_summary(relu6)
relu_dropout6 = tf.nn.dropout(relu6, keep_prob=keep_prob)
W7 = utils.weight_variable([1, 1, 4096, 4096], name="W7")
b7 = utils.bias_variable([4096], name="b7")
conv7 = utils.conv2d_basic(relu_dropout6, W7, b7)
relu7 = tf.nn.relu(conv7, name="relu7")
if FLAGS.debug:
utils.add_activation_summary(relu7)
relu_dropout7 = tf.nn.dropout(relu7, keep_prob=keep_prob)
W8 = utils.weight_variable([1, 1, 4096, NUM_OF_CLASSESS], name="W8")
b8 = utils.bias_variable([NUM_OF_CLASSESS], name="b8")
conv8 = utils.conv2d_basic(relu_dropout7, W8, b8)
# annotation_pred1 = tf.argmax(conv8, dimension=3, name="prediction1")
# now to upscale to actual image size
deconv_shape1 = image_net["pool4"].get_shape()
W_t1 = utils.weight_variable([4, 4, deconv_shape1[3].value, NUM_OF_CLASSESS], name="W_t1")
b_t1 = utils.bias_variable([deconv_shape1[3].value], name="b_t1")
conv_t1 = utils.conv2d_transpose_strided(conv8, W_t1, b_t1, output_shape=tf.shape(image_net["pool4"]))
#fuse_1 = tf.add(conv_t1, image_net["pool4"], name="fuse_1")
deconv_shape2 = image_net["pool3"].get_shape()
W_t2 = utils.weight_variable([4, 4, deconv_shape2[3].value, deconv_shape1[3].value], name="W_t2")
b_t2 = utils.bias_variable([deconv_shape2[3].value], name="b_t2")
conv_t2 = utils.conv2d_transpose_strided(conv_t1, W_t2, b_t2, output_shape=tf.shape(image_net["pool3"]))
#fuse_2 = tf.add(conv_t2, image_net["pool3"], name="fuse_2")
shape = tf.shape(image)
deconv_shape3 = tf.stack([shape[0], shape[1], shape[2], NUM_OF_CLASSESS])
W_t3 = utils.weight_variable([16, 16, NUM_OF_CLASSESS, deconv_shape2[3].value], name="W_t3")
b_t3 = utils.bias_variable([NUM_OF_CLASSESS], name="b_t3")
conv_t3 = utils.conv2d_transpose_strided(conv_t2, W_t3, b_t3, output_shape=deconv_shape3, stride=8)
/////////////////////////////////////////////////////this is from where i added the rnn
shape_5 = tf.shape(image)
W_a = 224
H_a = 224
p_size_a = NUM_OF_CLASSESS
# x = tf.reshape(conv_t1, [shape_5[0],H_a,W_a, p_size_a])
x = tf.transpose(conv_t3, perm=[0,2,1,3])
x = tf.reshape(x,[-1,H_a,p_size_a])
mat = tf.unstack(x, H_a, 1)
lstm_fw_cell = rnn.BasicLSTMCell(N_HIDDEN, forget_bias=1.0)
lstm_bw_cell = rnn.BasicLSTMCell(N_HIDDEN, forget_bias=1.0)
#with tf.variable_scope('rnn1_1'):
try:
outputs, _, _ = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, mat,
dtype=tf.float32,scope='rnn1_1')
except Exception: # Old TensorFlow version only returns outputs not states
outputs = rnn.static_bidirectional_rnn(lstm_fw_cell, lstm_bw_cell, mat,
dtype=tf.float32)
outputs1 = tf.reshape(outputs,[H_a, shape_5[0], W_a, 2 * N_HIDDEN])
outputs1 = tf.transpose(outputs1,(1,0,2,3))
x_1 = tf.reshape(outputs1,[-1,W_a,2 * N_HIDDEN])
mat_1 = tf.unstack(x_1, W_a, 1)
lstm_lw_cell = rnn.BasicLSTMCell(N_HIDDEN, forget_bias=1.0)
lstm_rw_cell = rnn.BasicLSTMCell(N_HIDDEN, forget_bias=1.0)
#with tf.variable_scope('rnn1_2'):
try:
outputs2, _, _ = rnn.static_bidirectional_rnn(lstm_lw_cell, lstm_rw_cell, mat_1,
dtype=tf.float32,scope = 'rnn1_2')
except Exception: # Old TensorFlow version only returns outputs not states
outputs2 = rnn.static_bidirectional_rnn(lstm_lw_cell, lstm_rw_cell, mat_1,
dtype=tf.float32)
outputs2 = tf.reshape(outputs,[W_a, shape_5[0], H_a, 2 * N_HIDDEN])
outputs2 = tf.transpose(outputs2,(1,2,0,3))
///////////////////////////////////////////////////till here
annotation_pred = tf.argmax(outputs2, dimension=3, name="prediction")
return tf.expand_dims(annotation_pred, dim=3), outputs2
and this part is for the training:
def train(loss_val, var_list):
optimizer = tf.train.AdamOptimizer(FLAGS.learning_rate)
grads = optimizer.compute_gradients(loss_val, var_list=var_list)
if FLAGS.debug:
# print(len(var_list))
for grad, var in grads:
utils.add_gradient_summary(grad, var)
return optimizer.apply_gradients(grads)
def main(argv=None):
keep_probability = tf.placeholder(tf.float32, name="keep_probabilty")
image = tf.placeholder(tf.float32, shape=[None, IMAGE_SIZE, IMAGE_SIZE, 3], name="input_image")
annotation = tf.placeholder(tf.int32, shape=[None, IMAGE_SIZE, IMAGE_SIZE, 1], name="annotation")
pred_annotation, logits = inference(image, keep_probability)
tf.summary.image("input_image", image, max_outputs=2)
tf.summary.image("ground_truth", tf.cast(annotation, tf.uint8), max_outputs=2)
tf.summary.image("pred_annotation", tf.cast(pred_annotation, tf.uint8), max_outputs=2)
loss = tf.reduce_mean((tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits,
labels=tf.squeeze(annotation, squeeze_dims=[3]),
name="entropy")))
tf.summary.scalar("entropy", loss)
trainable_var = tf.trainable_variables()
if FLAGS.debug:
for var in trainable_var:
utils.add_to_regularization_and_summary(var)
train_op = train(loss, trainable_var)
print("Setting up summary op...")
summary_op = tf.summary.merge_all()
print("Setting up image reader...")
train_records, valid_records = scene_parsing.read_dataset(FLAGS.data_dir)
print(len(train_records))
print(len(valid_records))
print("Setting up dataset reader")
image_options = {'resize': True, 'resize_size': IMAGE_SIZE}
if FLAGS.mode == 'train':
train_dataset_reader = dataset.BatchDatset(train_records, image_options)
validation_dataset_reader = dataset.BatchDatset(valid_records, image_options)
sess = tf.Session()
print("Setting up Saver...")
saver = tf.train.Saver()
summary_writer = tf.summary.FileWriter(FLAGS.logs_dir, sess.graph)
sess.run(tf.global_variables_initializer())
ckpt = tf.train.get_checkpoint_state(FLAGS.logs_dir)
if ckpt and ckpt.model_checkpoint_path:
saver.restore(sess, ckpt.model_checkpoint_path)
print("Model restored...")
if FLAGS.mode == "train":
for itr in xrange(MAX_ITERATION):
train_images, train_annotations = train_dataset_reader.next_batch(FLAGS.batch_size)
feed_dict = {image: train_images, annotation: train_annotations, keep_probability: 0.85}
sess.run(train_op, feed_dict=feed_dict)
if itr % 10 == 0:
train_loss, summary_str = sess.run([loss, summary_op], feed_dict=feed_dict)
print("Step: %d, Train_loss:%g" % (itr, train_loss))
summary_writer.add_summary(summary_str, itr)
if itr % 500 == 0:
valid_images, valid_annotations = validation_dataset_reader.next_batch(FLAGS.batch_size)
valid_loss = sess.run(loss, feed_dict={image: valid_images, annotation: valid_annotations,
keep_probability: 1.0})
print("%s ---> Validation_loss: %g" % (datetime.datetime.now(), valid_loss))
saver.save(sess, FLAGS.logs_dir + "model.ckpt", itr)
elif FLAGS.mode == "visualize":
valid_images, valid_annotations = validation_dataset_reader.get_random_batch(FLAGS.batch_size)
pred = sess.run(pred_annotation, feed_dict={image: valid_images, annotation: valid_annotations,
keep_probability: 1.0})
valid_annotations = np.squeeze(valid_annotations, axis=3)
pred = np.squeeze(pred, axis=3)
for itr in range(FLAGS.batch_size):
utils.save_image(valid_images[itr].astype(np.uint8), FLAGS.logs_dir, name="inp_" + str(5+itr))
utils.save_image(valid_annotations[itr].astype(np.uint8), FLAGS.logs_dir, name="gt_" + str(5+itr))
utils.save_image(pred[itr].astype(np.uint8), FLAGS.logs_dir, name="pred_" + str(5+itr))
print("Saved image: %d" % itr)
The error was described as:
Not found: Key inference/rnn1_2/fw/basic_lstm_cell/weights not found in checkpoint
So i think there must be something wrong with the variables.
I'll be very appreciate if someone could tell me how to fix it!
looking forward to your help!

Loading test data using batch Tensorflow

The following code is my pipeline for reading images and labels from files:
import tensorflow as tf
import numpy as np
import tflearn.data_utils
from tensorflow.python.framework import ops
from tensorflow.python.framework import dtypes
import sys
#process labels in the input file
def process_label(label):
info=np.zeros(6)
...
return info
def read_label_file(file):
f = open(file, "r")
filepaths = []
labels = []
lines = []
for line in f:
tokens = line.split(",")
filepaths.append([tokens[0],tokens[1],tokens[2]])
labels.append(process_label(tokens[3:]))
lines.append(line)
return filepaths, np.vstack(labels), lines
def get_data_batches(params):
# reading labels and file path
train_filepaths, train_labels, train_line = read_label_file(params.train_info)
test_filepaths, test_labels, test_line = read_label_file(params.test_info)
# convert string into tensors
train_images = ops.convert_to_tensor(train_filepaths)
train_labels = ops.convert_to_tensor(train_labels)
train_line = ops.convert_to_tensor(train_line)
test_images = ops.convert_to_tensor(test_filepaths)
test_labels = ops.convert_to_tensor(test_labels)
test_line = ops.convert_to_tensor(test_line)
# create input queues
train_input_queue = tf.train.slice_input_producer([train_images, train_labels, train_line], shuffle=params.shuffle)
test_input_queue = tf.train.slice_input_producer([test_images, test_labels, test_line],shuffle=False)
# process path and string tensor into an image and a label
train_image=None
for i in range(train_input_queue[0].get_shape()[0]):
file_content = tf.read_file(params.path_prefix+train_input_queue[0][i])
train_imageT = (tf.to_float(tf.image.decode_jpeg(file_content, channels=params.num_channels)))*(1.0/255)
train_imageT = tf.image.resize_images(train_imageT,[params.load_size[0],params.load_size[1]])
train_imageT = tf.random_crop(train_imageT,size=[params.crop_size[0],params.crop_size[1],params.num_channels])
train_imageT = tf.image.random_flip_up_down(train_imageT)
train_imageT = tf.image.per_image_standardization(train_imageT)
if(i==0):
train_image = train_imageT
else:
train_image = tf.concat([train_image, train_imageT], 2)
train_label = train_input_queue[1]
train_lineInfo = train_input_queue[2]
test_image=None
for i in range(test_input_queue[0].get_shape()[0]):
file_content = tf.read_file(params.path_prefix+test_input_queue[0][i])
test_imageT = tf.to_float(tf.image.decode_jpeg(file_content, channels=params.num_channels))*(1.0/255)
test_imageT = tf.image.resize_images(test_imageT,[params.load_size[0],params.load_size[1]])
test_imageT = tf.image.central_crop(test_imageT, (params.crop_size[0]+0.0)/params.load_size[0])
test_imageT = tf.image.per_image_standardization(test_imageT)
if(i==0):
test_image = test_imageT
else:
test_image = tf.concat([test_image, test_imageT],2)
test_label = test_input_queue[1]
test_lineInfo = test_input_queue[2]
# define tensor shape
train_image.set_shape([params.crop_size[0], params.crop_size[1], params.num_channels*3])
train_label.set_shape([66])
test_image.set_shape( [params.crop_size[0], params.crop_size[1], params.num_channels*3])
test_label.set_shape([66])
# collect batches of images before processing
train_image_batch, train_label_batch, train_lineno = tf.train.batch([train_image, train_label, train_lineInfo],batch_size=params.batch_size,num_threads=params.num_threads,allow_smaller_final_batch=True)
test_image_batch, test_label_batch, test_lineno = tf.train.batch([test_image, test_label, test_lineInfo],batch_size=params.test_size,num_threads=params.num_threads,allow_smaller_final_batch=True)
if(params.loadSlice=='all'):
return train_image_batch, train_label_batch, train_lineno, test_image_batch, test_label_batch, test_lineno
elif params.loadSlice=='train':
return train_image_batch, train_label_batch
elif params.loadSlice=='test':
return test_image_batch, test_label_batch
elif params.loadSlice=='train_info':
return train_image_batch, train_label_batch, train_lineno
elif params.loadSlice=='test_info':
return test_image_batch, test_label_batch, test_lineno
else:
return train_image_batch, train_label_batch, test_image_batch, test_label_batch
I want to use the same pipeline for loading the test data. The size of my test data is huge and I cannot load all of them at once.
I have 20453 test examples which is not an integer multiply of the batch size (here 512).
How can I read all of my test examples via this pipeline one and only one time and then measure the performance on them?
Currently, I am using this code for batching my test data and it does not work. It always read a full batch from the queue even when I set allow_smaller_final_batch to True
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
saver.restore(sess,"checkpoints2/snapshot-16")
coord = tf.train.Coordinator()
threads = tf.train.start_queue_runners(coord=coord)
more = True
num_examples=0
while(more):
img_test, lbl_test, lbl_line=sess.run([test_image_batch,test_label_batch,test_lineno])
print(lbl_test.shape)
size=lbl_test.shape[0]
num_examples += size
if size<args.batch_size:
more = False
sess.close()
This is the code of my model:
from tflearn.layers.core import input_data, dropout, fully_connected
from tflearn.layers.conv import conv_2d, max_pool_2d
from tflearn.layers.normalization import local_response_normalization
from tflearn.layers.normalization import batch_normalization
from tflearn.layers.estimator import regression
from tflearn.activations import relu
def get_alexnet(x,num_output):
network = conv_2d(x, 64, 11, strides=4)
network = batch_normalization(network,epsilon=0.001)
network = relu (network)
network = max_pool_2d(network, 3, strides=2)
network = conv_2d(network, 192, 5)
network = batch_normalization(network,epsilon=0.001)
network = relu(network)
network = max_pool_2d(network, 3, strides=2)
network = conv_2d(network, 384, 3)
network = batch_normalization(network,epsilon=0.0001)
network = relu(network)
network = conv_2d(network, 256, 3)
network = batch_normalization(network,epsilon=0.001)
network = relu(network)
network = conv_2d(network, 256, 3)
network = batch_normalization(network,epsilon=0.001)
network = relu(network)
network = max_pool_2d(network, 3, strides=2)
network = fully_connected(network, 4096)
network = batch_normalization(network,epsilon=0.001)
network = relu(network)
network = dropout(network, 0.5)
network = fully_connected(network, 4096)
network = batch_normalization(network,epsilon=0.001)
network = relu(network)
network = dropout(network, 0.5)
network1 = fully_connected(network, num_output)
network2 = fully_connected(network, 12)
network3 = fully_connected(network,6)
return network1,network2,network3
This simply could be achieved by setting num_epochs=1 and allow_smaller_final_batch= True!
One solution is set batch_size=size of test set

Reading memory and Access is Denied

I need to access all memory of a running process in my local Windows 7-64bit. I am new to winapi.
Here is my problem; Whenever I try to Open a process and reads its memory, I get Access is Denied error.
I searched and found something. It is said that If I run the main process as Administrator and use PROCESS_ALL_ACCESS on OpenProcess, I would have enough right to do it as it is said. OK, I did it. but nothing is changed. On reading memory, I still get Access is Denied.
So, I kept searching and found another thing which is enabling SeDebugPrivilege. I have also done that but nothing is changed. I still get the error.
I've read the quest and his answer here;
Windows Vista/Win7 Privilege Problem: SeDebugPrivilege & OpenProcess .
But as I said, I am really new to winapi. I could not solve my problem yet. Is there anything which which I need to configure in my local operating system?
Here is my Python code with pywin32;
from _ctypes import byref, sizeof, Structure
from ctypes import windll, WinError, c_buffer, c_void_p, create_string_buffer
from ctypes.wintypes import *
import win32security
import win32api
import gc
import ntsecuritycon
from struct import Struct
from win32con import PROCESS_ALL_ACCESS
from struct import calcsize
MEMORY_STATES = {0x1000: "MEM_COMMIT", 0x10000: "MEM_FREE", 0x2000: "MEM_RESERVE"}
MEMORY_PROTECTIONS = {0x10: "PAGE_EXECUTE", 0x20: "PAGE_EXECUTE_READ", 0x40: "PAGEEXECUTE_READWRITE",
0x80: "PAGE_EXECUTE_WRITECOPY", 0x01: "PAGE_NOACCESS", 0x04: "PAGE_READWRITE",
0x08: "PAGE_WRITECOPY"}
MEMORY_TYPES = {0x1000000: "MEM_IMAGE", 0x40000: "MEM_MAPPED", 0x20000: "MEM_PRIVATE"}
class MEMORY_BASIC_INFORMATION(Structure):
_fields_ = [
("BaseAddress", c_void_p),
("AllocationBase", c_void_p),
("AllocationProtect", DWORD),
("RegionSize", UINT),
("State", DWORD),
("Protect", DWORD),
("Type", DWORD)
]
class SYSTEM_INFO(Structure):
_fields_ = [("wProcessorArchitecture", WORD),
("wReserved", WORD),
("dwPageSize", DWORD),
("lpMinimumApplicationAddress", DWORD),
("lpMaximumApplicationAddress", DWORD),
("dwActiveProcessorMask", DWORD),
("dwNumberOfProcessors", DWORD),
("dwProcessorType", DWORD),
("dwAllocationGranularity", DWORD),
("wProcessorLevel", WORD),
("wProcessorRevision", WORD)]
class PyMEMORY_BASIC_INFORMATION:
def __init__(self, MBI):
self.MBI = MBI
self.set_attributes()
def set_attributes(self):
self.BaseAddress = self.MBI.BaseAddress
self.AllocationBase = self.MBI.AllocationBase
self.AllocationProtect = MEMORY_PROTECTIONS.get(self.MBI.AllocationProtect, self.MBI.AllocationProtect)
self.RegionSize = self.MBI.RegionSize
self.State = MEMORY_STATES.get(self.MBI.State, self.MBI.State)
# self.Protect = self.MBI.Protect # uncomment this and comment next line if you want to do a bitwise check on Protect.
self.Protect = MEMORY_PROTECTIONS.get(self.MBI.Protect, self.MBI.Protect)
self.Type = MEMORY_TYPES.get(self.MBI.Type, self.MBI.Type)
ASSUME_ALIGNMENT = True
class TARGET:
"""Given a ctype (initialized or not) this coordinates all the information needed to read, write and compare."""
def __init__(self, ctype):
self.alignment = 1
self.ctype = ctype
# size of target data
self.size = sizeof(ctype)
self.type = ctype._type_
# get the format type needed for struct.unpack/pack.
while hasattr(self.type, "_type_"):
self.type = self.type._type_
# string_buffers and char arrays have _type_ 'c'
# but that makes it slightly slower to unpack
# so swap is for 's'.
if self.type == "c":
self.type = "s"
# calculate byte alignment. this speeds up scanning substantially
# because we can read and compare every alignment bytes
# instead of every single byte.
# although if we are scanning for a string the alignment is defaulted to 1 \
# (im not sure if this is correct).
elif ASSUME_ALIGNMENT:
# calc alignment
divider = 1
for i in xrange(4):
divider *= 2
if not self.size % divider:
self.alignment = divider
# size of target ctype.
self.type_size = calcsize(self.type)
# length of target / array length.
self.length = self.size / self.type_size
self.value = getattr(ctype, "raw", ctype.value)
# the format string used for struct.pack/unpack.
self.format = str(self.length) + self.type
# efficient packer / unpacker for our own format.
self.packer = Struct(self.format)
def get_packed(self):
"""Gets the byte representation of the ctype value for use with WriteProcessMemory."""
return self.packer.pack(self.value)
def __str__(self):
return str(self.ctype)[:10] + "..." + " <" + str(self.value)[:10] + "..." + ">"
class Memory(object):
def __init__(self, process_handle, target):
self._process_handle = process_handle
self._target = target
self.found = []
self.__scann_process()
def __scann_process(self):
"""scans a processes pages for the target value."""
si = SYSTEM_INFO()
psi = byref(si)
windll.kernel32.GetSystemInfo(psi)
base_address = si.lpMinimumApplicationAddress
max_address = si.lpMaximumApplicationAddress
page_address = base_address
while page_address < max_address:
page_address = self.__scan_page(page_address)
if len(self.found) >= 60000000:
print("[Warning] Scan ended early because too many addresses were found to hold the target data.")
break
gc.collect()
return self.found
def __scan_page(self, page_address):
"""Scans the entire page for TARGET instance and returns the next page address and found addresses."""
information = self.VirtualQueryEx(page_address)
base_address = information.BaseAddress
region_size = information.RegionSize
next_region = base_address + region_size
size = self._target.size
target_value = self._target.value
step = self._target.alignment
unpacker = self._target.packer.unpack
if information.Type != "MEM_PRIVATE" or \
region_size < size or \
information.State != "MEM_COMMIT" or \
information.Protect not in ["PAGE_EXECUTE_READ", "PAGEEXECUTE_READWRITE", "PAGE_READWRITE"]:
return next_region
page_bytes = self.ReadMemory(base_address, region_size)
for i in xrange(0, (region_size - size), step):
partial = page_bytes[i:i + size]
if unpacker(partial)[0] == target_value:
self.found.append(base_address + i)
del page_bytes # free the buffer
return next_region
def ReadMemory(self, address, size):
cbuffer = c_buffer(size)
success = windll.kernel32.ReadProcessMemory(
self._process_handle,
address,
cbuffer,
size,
0)
assert success, "ReadMemory Failed with success == %s and address == %s and size == %s.\n%s" % (
success, address, size, WinError(win32api.GetLastError()))
return cbuffer.raw
def VirtualQueryEx(self, address):
MBI = MEMORY_BASIC_INFORMATION()
MBI_pointer = byref(MBI)
size = sizeof(MBI)
success = windll.kernel32.VirtualQueryEx(
self._process_handle,
address,
MBI_pointer,
size)
assert success, "VirtualQueryEx Failed with success == %s.\n%s" % (
success, WinError(win32api.GetLastError())[1])
assert success == size, "VirtualQueryEx Failed because not all data was written."
return PyMEMORY_BASIC_INFORMATION(MBI)
def AdjustPrivilege(priv):
flags = win32security.TOKEN_ADJUST_PRIVILEGES | win32security.TOKEN_QUERY
p = win32api.GetCurrentProcess()
htoken = win32security.OpenProcessToken(p, flags)
id = win32security.LookupPrivilegeValue(None, priv)
newPrivileges = [(id, win32security.SE_PRIVILEGE_ENABLED)]
win32security.AdjustTokenPrivileges(htoken, 0, newPrivileges)
win32api.CloseHandle(htoken)
def OpenProcess(pid=win32api.GetCurrentProcessId()):
# ntsecuritycon.SE_DEBUG_NAME = "SeDebugPrivilege"
AdjustPrivilege(ntsecuritycon.SE_DEBUG_NAME)
phandle = windll.kernel32.OpenProcess( \
PROCESS_ALL_ACCESS,
0,
pid)
assert phandle, "Failed to open process!\n%s" % WinError(win32api.GetLastError())[1]
return phandle
PID = 22852
process_handle = OpenProcess(PID)
Memory(process_handle, TARGET(create_string_buffer("1456")))
Here is the error I always get;
AssertionError: ReadMemory Failed with success == 0 and address == 131072 and size == 4096.
[Error 5] Access is denied.
I do not know what information else about my code and my personal Windows 7 operating system, I should provide to you. If you need to know more, please ask it from me, I will provide it to solve that problem.
I guess, this is about a lack of configuration in my operating system , not about pywin32. I'll be waiting for your solutions.

Resources