QPixmap width and height return 0 - image

problemPicture
ENV
Win10
python3.7
PyQt5
Desc
With the following code, where image_path is the absolute path to problemPicture.(this is not a question about cannot find the pic)
pixmapHight and pixmapWidth will get 0, but the picture can be well openned and viewed with default APP
And for most pictures, this code works well, but fail on this special one.
so can anyone explain it and give me some advice?
pixmap = QPixmap(image_path)
pixmapHight = pixmap.height()
pixmapWidth = pixmap.width()
PS this question seems the same, but the answer is not accepted, and neither to my question.

I find it is caused by wrong suffix.
The pic saved locally is suffixed by .png, but the real format is jpg.
When specifying with right format, eg: QPixmap(image_path, format = 'jpg'), I can get correct height and width.
here is my sample code to check real format:
import os
import sys
import imghdr
from PyQt5.QtGui import *
from PyQt5.QtCore import *
from PyQt5.QtWidgets import *
class DemoApp(QMainWindow):
def is_type_wrong(self, path):
current_type = path[path.rfind('.')+1:]
real_type = 'xxx'
if path.lower().endswith('.gif') or path.lower().endswith('.jpg') or path.lower().endswith('.png'):
header = []
with open(path, 'rb') as f:
while(len(header) < 5):
header.append(f.read(1))
print(header)
if (header[0] == b'\x47' and header[1] and b'\x49' and header[2] == b'\x46' and header[3] == b'\x38'):
real_type = 'gif'
if (header[0] == b'\xff' and header[1] == b'\xd8'):
real_type = 'jpg'
if (header[0] == b'\x89' and header[1] == b'\x50' and header[2] == b'\x4e' and header[3] == b'\x47' and header[4] == b'\x0D'):
real_type = 'png'
return current_type != real_type, real_type
def __init__(self, parent=None):
super(DemoApp, self).__init__(parent)
image_path_list = ['D:\FailPic.png', 'D:\OkPic.png', 'D:\FromWeb.jpg']
for image_path in image_path_list:
if os.path.exists(image_path):
fmt = imghdr.what(image_path)
if fmt == None:
# I get one picture still can be opened with default app while imghdr.what return None
isWrongType, real_type = self.is_type_wrong(image_path)
if isWrongType:
fmt = real_type
else:
print("!!! corrupted picture: %s" %image_path)
pixmap = QPixmap(image_path, format = fmt)
pixmapHight = pixmap.height()
pixmapWidth = pixmap.width()
print("[%s]isNull:%d pixmapHight: %f, pixmapWidth: %f, fmt: %s" %(image_path, pixmap.isNull(), pixmapHight, pixmapWidth, fmt))
else:
print("pic % not found" %(image_path))
if __name__ == "__main__":
app = QApplication(sys.argv)
main_window = DemoApp()
main_window.show()
sys.exit(app.exec_())

Related

Fine tuning Bert for NER attempt on Mac OS

I'm using a MacBook Air/OS Monterey 12.5 (There are updates available; Ventura 13.1
Python version 3.10.8 and also tried using 3.11
Pylance has pointed that all the imports I was trying to execute were not being resolved so I changed the VS Code interpreter to Python 3.10.
Anyways, here's the code:
import pandas as pd
import torch
import numpy as np
from tqdm import tqdm
from transformers import BertTokenizerFast
from transformers import BertForTokenClassification
from torch.utils.data import Dataset, DataLoader
df = pd.read_csv('ner.csv')
labels = [i.split() for i in df['labels'].values.tolist()]
unique_labels = set()
for lb in labels:
[unique_labels.add(i) for i in lb if i not in unique_labels]
# print(unique_labels)
labels_to_ids = {k: v for v, k in enumerate(sorted(unique_labels))}
ids_to_labels = {v: k for v, k in enumerate(sorted(unique_labels))}
# print(labels_to_ids)
text = df['text'].values.tolist()
example = text[36]
#print(example)
tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')
text_tokenized = tokenizer(example, padding='max_length', max_length=512, truncation=True, return_tensors='pt')
'''
print(text_tokenized)
print(tokenizer.decode(text_tokenized.input_ids[0]))
'''
def align_label_example(tokenized_input, labels):
word_ids = tokenized_input.word_ids()
previous_word_idx = None
label_ids = []
for word_idx in word_ids:
if word_idx is None:
label_ids.append(-100)
elif word_idx != previous_word_idx:
try:
label_ids.append(labels_to_ids[labels[word_idx]])
except:
label_ids.append(-100)
else:
label_ids.append(labels_to_ids[labels[word_idx]] if label_all_tokens else -100)
previous_word_idx = word_idx
return label_ids;
label = labels[36]
label_all_tokens = False
new_label = align_label_example(text_tokenized, label)
'''
print(new_label)
print(tokenizer.convert_ids_to_tokens(text_tokenized['input_ids'][0]))
'''
def align_label(texts, labels):
tokenized_inputs = tokenizer(texts, padding='max_length', max_length=512, truncation=True)
word_ids = tokenized_inputs.word_ids()
previous_word_idx = None
label_ids = []
for word_idx in word_ids:
if word_idx is None:
label_ids.append(-100)
elif word_idx != previous_word_idx:
try:
label_ids.append(labels_to_ids[labels[word_idx]])
except:
label_ids.append(-100)
else:
try:
label_ids.append(labels_to_ids[labels[word_idx]] if label_all_tokens else -100)
except:
label_ids.append(-100)
previous_word_idx = word_idx
return label_ids
class DataSequence(torch.utils.data.Dataset):
def __init__(self, df):
lb = [i.split() for i in df['labels'].values.tolist()]
txt = df['text'].values.tolist()
self.texts = [tokenizer(str(i),
padding='max_length', max_length=512, truncation=True, return_tensors='pt') for i in txt]
self.labels = [align_label(i,j) for i,j in zip(txt, lb)]
def __len__(self):
return len(self.labels)
def get_batch_labels(self, idx):
return torch.LongTensor(self.labels[idx])
def __getitem__(self, idx):
batch_data = self.get_batch_data(idx)
batch_labels = self.get_batch_labels(idx)
return batch_data, batch_labels
df = df[0:1000]
df_train, df_val, df_test = np.split(df.sample(frac=1, random_state=42),
[int(.8 * len(df)), int(.9 * len(df))])
class BertModel(torch.nn.Module):
def __init__(self):
super(BertModel, self).__init__()
self.bert = BertForTokenClassification.from_pretrained('bert-base-cased', num_labels=len(unique_labels))
def forward(self, input_id, mask, label):
output = self.bert(input_ids=input_id, attention_mask=mask, labels=label, return_dict=False)
return output
def train_loop(model, df_train, df_val):
train_dataset = DataSequence(df_train)
val_dataset = DataSequence(df_val)
train_dataloader = DataLoader(train_dataset, num_workers=4, batch_size=BATCH_SIZE, shuffle=True)
val_dataloader = DataLoader(val_dataset, num_workers=4, batch_size=BATCH_SIZE)
use_cuda = torch.cuda.is_available()
device = torch.device('cuda' if use_cuda else 'cpu')
optimizer = torch.optim.SGD(model.parameters(), lr=LEARNING_RATE)
if use_cuda:
model = model.cuda()
best_acc = 0
best_loss = 1000
for epoch_num in range(EPOCHS):
total_acc_train = 0
total_loss_train = 0
model.train()
for train_data, train_label in tqdm(train_dataloader):
train_label = train_label.to(device)
mask = train_data['attention_mask'].squeeze(1).to(device)
input_id = train_data['input_ids'].squeeze(1).to(device)
optimizer.zero_grad()
loss, logits = model(input_id, mask, train_label)
for i in range(logits.shape[0]):
logits_clean = logits[i][train_label[i] != -100]
label_clean = train_label[i][train_label[i] != -100]
predictions = logits_clean.argmax(dim=1)
acc = (predictions == label_clean).float().mean()
total_acc_train += acc
total_loss_train += loss.item()
loss.backward()
optimizer.step()
model.eval()
total_acc_val = 0
total_loss_val = 0
for val_data, val_label in val_dataloader:
val_label = val_label.to(device)
mask = val_data['attention_mask'].squeeze(1).to(device)
input_id = val_data['input_ids'].squeeze(1).to(device)
loss, logits = model(input_id, mask, val_label)
for i in range(logits.shape[0]):
logits_clean = logits[i][val_label[i] != -100]
label_clean = val_label[i][val_label[i] != -100]
predictions = logits_clean.argmax(dim=1)
acc = (predictions == label_clean).float().mean()
total_acc_val += acc
total_loss_val += loss.item()
val_accuracy = total_acc_val / len(df_val)
val_loss = total_loss_val / len(df_val)
print(
f'Epochs: {epoch_num + 1} | Loss: {total_loss_train / len(df_train): .3f} | Accuracy: {total_acc_train / len(df_train): .3f} | Val_Loss: {total_loss_val / len(df_val): .3f} | Accuracy: {total_acc_val / len(df_val): .3f}')
LEARNING_RATE = 5e-3
EPOCHS = 5
BATCH_SIZE = 2
model = BertModel()
train_loop(model, df_train, df_val)
And the debugger says:
Exception has occurred: RuntimeError (note: full exception trace is shown but execution is paused at: <module>)
An attempt has been made to start a new process before the
current process has finished its bootstrapping phase.
This probably means that you are not using fork to start your
child processes and you have forgotten to use the proper idiom
in the main module:
if __name__ == '__main__':
freeze_support()
...
The "freeze_support()" line can be omitted if the program
is not going to be frozen to produce an executable.
File "/Users/filipedonatti/Projects/pyCodes/second_try.py", line 141, in train_loop
for train_data, train_label in tqdm(train_dataloader):
File "/Users/filipedonatti/Projects/pyCodes/second_try.py", line 197, in <module>
train_loop(model, df_train, df_val)
File "<string>", line 1, in <module> (Current frame)
By the way,
Despite using Mac, I have downloaded Anaconda-Navigator, however I've been trying and executing this code on VS Code. I've downloaded numpy, torch, datasets and other libraries through Brew with the pip3 command.
I'm at a loss, I can run the code on a google collab notebook or Jupiter notebook, and I know training models and such in my humble Mac would not be advised, but I am just exercising this so I can train and use the model in a much more powerful machine.
Please help me with this issue, I've been trying to find a solution for days.
Peace and happy holidays.
I've tried solving the issue by writing:
if __name__ == '__main__':
freeze_support()
I've tried using this:
import parallelTestModule
extractor = parallelTestModule.ParallelExtractor()
extractor.runInParallel(numProcesses=2, numThreads=4)
So...
It turns out the correct way to solve this is to implement a function to train the loop as such:
def run():
model = BertModel()
torch.multiprocessing.freeze_support()
print('loop')
train_loop(model, df_train, df_val)
if __name__ == '__main__':
run()
Redefining that train_loop line in the end. Issue solved. For more see this link: https://github.com/pytorch/pytorch/issues/5858

What's wrong with Windows Task Sheduler + pytesseract + multiprocessing?

Have the python code with pytesseract & multiprocessing. When I start the code manually from PyCharm it works fine with any number of threads. When I start the code with Win Task Sheduler with 'threads=1' it works fine.
However if I start the code with Win Task Sheduler with 'threads=2' or more than 2, it finishes without processing the images and without any errors.
I've got log messages like this. Script starts but does nothing and there is no any errors in Win logs
2020-05-24 13:09:31,834;START
2020-05-24 13:09:31,834;threads: 2
2020-05-24 13:10:31,832;START
2020-05-24 13:10:31,832;threads: 2
2020-05-24 13:11:31,851;START
2020-05-24 13:11:31,851;threads: 2
Code
from PIL import Image
import pytesseract
from pytesseract import Output
import datetime
from glob import glob
import os
import multiprocessing as multiprocessing
import cv2
import logging
def loggerinit(name, filename, overwrite):
logger = logging.getLogger(name)
logger.setLevel(logging.INFO)
# create the logging file handler
fh = logging.FileHandler(filename, encoding = 'UTF-8')
formatter = logging.Formatter('%(asctime)s;%(message)s')
fh.setFormatter(formatter)
# add handler to logger object
logger.addHandler(fh)
return logger
def getfiles(dirname, mask):
return glob(os.path.join(dirname, mask))
def tess_file(fname):
img = cv2.imread(fname)
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)
im_for_T = Image.fromarray(img)
pytesseract.pytesseract.tesseract_cmd = 'C://Tesseract-OCR//tesseract.exe'
TESSDATA_PREFIX = 'C:/Program Files/Tesseract-OCR/tessdata'
try:
os.environ['OMP_THREAD_LIMIT'] = '1'
tess_data = pytesseract.image_to_osd(im_for_T, output_type=Output.DICT)
return fname, tess_data
except:
return fname, None
if __name__ == '__main__':
logger = loggerinit('tess', 'tess.log', False)
files = getfiles('Croped', '*.jpg')
t1 = datetime.datetime.now()
logger.info('START')
threads = 2
logger.info('threads: ' + str(threads))
p = multiprocessing.Pool(threads)
results = p.map(tess_file,files)
e = []
for r in results:
if type(r) == type(None):
e.append('OCR error: ' + r)
else:
print(r[0],". rotate: ",r[1]['rotate'])
p.close()
p.join()
t2 = datetime.datetime.now()
delta = (t2 - t1).total_seconds()
print('Total time: ', delta)
print('Files: ', len(files))
logger.info('Files: ' + str(len(files)))
logger.info('Stop.' + 'Total time: ' + str(delta))
# Print error if exist
for ee in e:
print(ee)
Whats wrong? How can I fix this issue?

I am trying to bundle my Python(3.5.3) Tkinter app using cx_Freeze(5.1.1). When I hide my command prompt the app doesn't work.

As suggested here I use it to hide my command prompt in my setup.py file. It does hide my command prompt but the app does not work. Basically I am trying to make a Windows native Microsoft MSI for my GUI that I have built for youtube-dl command line tool that is used to consume media from some of the most popular video hosting sites. Any help is much appreciated. Here is my app.py:-
from tkinter import *
from tkinter import ttk
from tkinter import messagebox
from tkinter import filedialog
from tkinter.ttk import Progressbar
import youtube_dl
import threading
import os
download_folder = os.path.expanduser("~")+"/Downloads/"
download_folder_chosen = ""
window = Tk()
window.title("IOB Youtube Downloader")
window.geometry('510x100')
def my_hook(d):
if d:
if d['status'] == 'downloading':
percent_done = d['_percent_str']
percent_done = percent_done.replace(" ", "")
percent_done = percent_done.replace("%", "")
bar['value'] = percent_done
bar.grid(column=1, row=2, pady=15)
bar_lbl.configure(text=percent_done + "%")
bar_lbl.grid(column=1, row=3)
txt['state'] = DISABLED
btn['state'] = DISABLED
if d['status'] == 'finished':
bar.grid_forget()
txt['state'] = NORMAL
btn['state'] = NORMAL
bar_lbl.configure(text="Download Completed !!!")
bar_lbl.grid(column=1, row=2)
messagebox.showinfo('IOB Youtube Downloader', 'Download Complete')
if d['status'] == 'error':
print("\n"*10)
print(d)
messagebox.showerror('IOB Youtube Downloader', 'Download Error')
else:
bar_lbl.configure(text="Download Error. Please try again !!!")
bar_lbl.grid(column=1, row=2)
def start_thread():
t1 = threading.Thread(target=clicked, args=())
t1.start()
def clicked():
res = txt.get()
if download_folder_chosen != "":
location = download_folder_chosen + "/"
else:
location = download_folder
ydl_opts = {
'progress_hooks': [my_hook],
'format': 'best',
'outtmpl': location + u'%(title)s-%(id)s.%(ext)s',
}
try:
with youtube_dl.YoutubeDL(ydl_opts) as ydl:
ydl.download([res])
except:
messagebox.showerror('IOB Youtube Downloader', 'Download Error')
def choose_directory():
global download_folder_chosen
current_directory = filedialog.askdirectory()
download_folder_chosen = current_directory
messagebox.showinfo('IOB Youtube Downloader', 'Download Location:- ' + download_folder_chosen)
style = ttk.Style()
style.theme_use('default')
style.configure("blue.Horizontal.TProgressbar", background='blue')
bar = Progressbar(window, length=200, style='black.Horizontal.TProgressbar')
bar_lbl = Label(window, text="")
lbl = Label(window, text="Paste URL")
lbl.grid(column=0, row=0)
txt = Entry(window,width=60)
txt.grid(column=1, row=0)
btn = Button(window, text="Download", command=start_thread)
btn.grid(column=2, row=0)
btn2 = Button(window, text="...", command=choose_directory)
btn2.grid(column=3, row=0)
window.iconbitmap('favicon.ico')
window.mainloop()
And here is my setup.py file that I use to build the bundle exe using cx_Freeze.
from cx_Freeze import setup, Executable
import sys
import os
base = None
if sys.platform == 'win32':
base = "Win32GUI"
os.environ["TCL_LIBRARY"] = r"C:\Python35\tcl\tcl8.6"
os.environ["TK_LIBRARY"] = r"C:\Python35\tcl\tk8.6"
setup(
name = "IOB Youtube Downloader",
options = {"build_exe": {"packages":["tkinter",], "include_files":[r"C:\Python35\DLLs\tk86t.dll", r"C:\Python35\DLLs\tcl86t.dll", r"E:\Youtube_Downloader\Src\favicon.ico"]}},
version = "1.0",
author = "IO-Bridges",
description = "Download videos from all popular video streaming sites.",
executables = [Executable
(
r"downloader.py",
# base=base, <---- Here setting the base
shortcutName="IOB Youtube Downloader",
shortcutDir="DesktopFolder",
icon="favicon.ico"
)]
)

Python compiled code crashes

I made this game in python 2.7 and it worked in .py format but when I compiled it with py2exe it suddenly broke. It gave me this error:
"Microsoft Visuak C++ Runtime Library
Runtime Error!
Program C:\Python27\Helmetdodger\Stuffz\main.exe
This application has requested the Runtime to terminate it in an unusual way.
Please contact the program's support team."
The code:
import pygame
import random, sys, os
from pygame.locals import *
WINDOWWIDTH = 1200
WINDOWHEIGHT = 900
TEXTCOLOR = (255, 255, 255)
BACKGROUNDCOLOR = (0, 0, 0)
FPS = 40
BADDIEMINSIZE = 10
BADDIEMAXSIZE = 40
BADDIEMINSPEED = 1
BADDIEMAXSPEED = 8
ADDNEWBADDIERATE = 6
PLAYERMOVERATE = 5
def terminate():
pygame.quit()
os.exit(1)
def waitForPlayerToPressKey():
while True:
for event in pygame.event.get():
if event.type == QUIT:
terminate()
if event.type == KEYDOWN:
if event.key == K_ESCAPE: # pressing escape quits
terminate()
return
def playerHasHitBaddie(playerRect, baddies):
for b in baddies:
if playerRect.colliderect(b['rect']):
return True
return False
def drawText(text, font, surface, x, y):
textobj = font.render(text, 1, TEXTCOLOR)
textrect = textobj.get_rect()
textrect.topleft = (x, y)
surface.blit(textobj, textrect)
# set up pygame, the window, and the mouse cursor
pygame.init()
mainClock = pygame.time.Clock()
windowSurface = pygame.display.set_mode((WINDOWWIDTH, WINDOWHEIGHT))
pygame.display.set_caption('Helmetdodger')
pygame.mouse.set_visible(False)
# set up fonts
font = pygame.font.SysFont(None, 48)
# set up sounds
gameOverSound = pygame.mixer.Sound('gameover.wav')
pygame.mixer.music.load('background.mid')
# set up images
playerImage1 = pygame.image.load('player1.png')
powerImage = pygame.image.load('power.png')
playerRect = playerImage1.get_rect()
baddieImage1 = pygame.image.load('baddie.png')
baddieImage2 = pygame.image.load('baddie1.png')
baddieImage3 = pygame.image.load('baddie2.png')
baddieImage4 = pygame.image.load('baddie3.png')
baddieImage5 = pygame.image.load('baddie4.png')
baddieImage6 = pygame.image.load('baddie5.png')
baddieImage7 = pygame.image.load('baddie6.png')
baddieImage8 = pygame.image.load('baddie7.png')
baddieImage9 = pygame.image.load('baddie8.png')
baddieImage10 = pygame.image.load('baddie9.png')
baddieImage11 = pygame.image.load('baddie10.png')
baddieImage12 = pygame.image.load('baddie11.png')
baddieImage13 = pygame.image.load('baddie12.png')
baddieImage14 = pygame.image.load('baddie13.png')
baddieImage15 = pygame.image.load('baddie14.png')
baddieImage16 = pygame.image.load('baddie15.png')
baddieImage17 = pygame.image.load('baddie16.png')
baddieImage18 = pygame.image.load('baddie17.png')
baddieImage19 = pygame.image.load('baddie18.png')
baddieImage20 = pygame.image.load('baddie19.png')
baddieImage21 = pygame.image.load('baddie20.png')
baddieImage22 = pygame.image.load('baddie21.png')
baddieImage23 = pygame.image.load('baddie22.png')
baddieImage24 = pygame.image.load('baddie23.png')
baddieImage25 = pygame.image.load('baddie24.png')
baddieImage26 = pygame.image.load('baddie25.png')
baddieImage27 = pygame.image.load('baddie26.png')
baddieImage28 = pygame.image.load('baddie27.png')
baddieImage29 = pygame.image.load('baddie28.png')
baddieImage30 = pygame.image.load('baddie29.png')
baddieImages = [baddieImage1, baddieImage2, baddieImage3, baddieImage4, baddieImage5, baddieImage6, baddieImage7, baddieImage8, baddieImage9, baddieImage10, baddieImage11, baddieImage12, baddieImage13, baddieImage14, baddieImage15, baddieImage16, baddieImage17, baddieImage18, baddieImage19, baddieImage20, baddieImage21, baddieImage22, baddieImage23, baddieImage24, baddieImage25, baddieImage26, baddieImage27, baddieImage28, baddieImage29, baddieImage20]
# show the "Start" screen
drawText('Helmetdodger', font, windowSurface, (WINDOWWIDTH / 3), (WINDOWHEIGHT / 3))
drawText('Press a key to start.', font, windowSurface, (WINDOWWIDTH / 3) - 30, (WINDOWHEIGHT / 3) + 50)
pygame.display.update()
waitForPlayerToPressKey()
#Get highscore
topScore = 0
try:
file = open('hs.txt', "r")
topScore = file.read()
topScore = int(topScore)
file.close()
except:
topScore = 0
while True:
# set up the start of the game
baddies = []
score = 0
playerRect.topleft = (WINDOWWIDTH / 2, WINDOWHEIGHT - 50)
moveLeft = moveRight = moveUp = moveDown = False
reverseCheat = slowCheat = False
baddieAddCounter = 0
pygame.mixer.music.play(-1, 0.0)
powerCount = 0
while True: # the game loop runs while the game part is playing
score += 1 # increase score
for event in pygame.event.get():
if event.type == QUIT:
terminate()
if event.type == KEYDOWN:
if event.key == ord('z'):
reverseCheat = True
if event.key == ord('x'):
slowCheat = True
if event.key == K_LEFT or event.key == ord('a'):
moveRight = False
moveLeft = True
if event.key == K_RIGHT or event.key == ord('d'):
moveLeft = False
moveRight = True
if event.key == K_UP or event.key == ord('w'):
moveDown = False
moveUp = True
if event.key == K_DOWN or event.key == ord('s'):
moveUp = False
moveDown = True
if event.type == KEYUP:
if event.key == ord('z'):
reverseCheat = False
score = 0
if event.key == ord('x'):
slowCheat = False
score = 0
if event.key == K_ESCAPE:
terminate()
if event.key == K_LEFT or event.key == ord('a'):
moveLeft = False
if event.key == K_RIGHT or event.key == ord('d'):
moveRight = False
if event.key == K_UP or event.key == ord('w'):
moveUp = False
if event.key == K_DOWN or event.key == ord('s'):
moveDown = False
if event.type == MOUSEMOTION:
# If the mouse moves, move the player where the cursor is.
playerRect.move_ip(event.pos[0] - playerRect.centerx, event.pos[1] - playerRect.centery)
# Add new baddies at the top of the screen, if needed.
if not reverseCheat and not slowCheat:
baddieAddCounter += 1
if baddieAddCounter == ADDNEWBADDIERATE:
baddieCount = random.randrange(len(baddieImages))
baddieAddCounter = 0
baddieSize = random.randint(BADDIEMINSIZE, BADDIEMAXSIZE)
newBaddie = {'rect': pygame.Rect(random.randint(0, WINDOWWIDTH-baddieSize), 0 - baddieSize, baddieSize, baddieSize),
'speed': random.randint(BADDIEMINSPEED, BADDIEMAXSPEED),
'surface':pygame.transform.scale(baddieImages[baddieCount], (baddieSize, baddieSize)),
}
baddies.append(newBaddie)
# Move the player around.
if moveLeft and playerRect.left > 0:
playerRect.move_ip(-1 * PLAYERMOVERATE, 0)
if moveRight and playerRect.right < WINDOWWIDTH:
playerRect.move_ip(PLAYERMOVERATE, 0)
if moveUp and playerRect.top > 0:
playerRect.move_ip(0, -1 * PLAYERMOVERATE)
if moveDown and playerRect.bottom < WINDOWHEIGHT:
playerRect.move_ip(0, PLAYERMOVERATE)
# Move the mouse cursor to match the player.
pygame.mouse.set_pos(playerRect.centerx, playerRect.centery)
# Move the baddies down.
for b in baddies:
if not reverseCheat and not slowCheat:
b['rect'].move_ip(0, b['speed'])
elif reverseCheat:
b['rect'].move_ip(0, -5)
elif slowCheat:
b['rect'].move_ip(0, 1)
# Delete baddies that have fallen past the bottom.
for b in baddies[:]:
if b['rect'].top > WINDOWHEIGHT:
baddies.remove(b)
# Draw the game world on the window.
windowSurface.fill(BACKGROUNDCOLOR)
# Draw the score and top score.
drawText('Score: %s' % (score), font, windowSurface, 10, 0)
drawText('Top Score: %s' % (topScore), font, windowSurface, 10, 40)
# Draw the player's rectangle
windowSurface.blit(playerImage1, playerRect)
# Draw each baddie
for b in baddies:
windowSurface.blit(b['surface'], b['rect'])
pygame.display.update()
# Check if any of the baddies have hit the player.
if playerHasHitBaddie(playerRect, baddies):
if score > topScore:
topScore = score # set new top score
file = open("hs.txt", "w")
score = str(score)
file.write(score)
file.close()
break
mainClock.tick(FPS)
# Stop the game and show the "Game Over" screen.
pygame.mixer.music.stop()
gameOverSound.play()
drawText('GAME OVER', font, windowSurface, (WINDOWWIDTH / 3), (WINDOWHEIGHT / 3))
drawText('Press a key to play again.', font, windowSurface, (WINDOWWIDTH / 3) - 80, (WINDOWHEIGHT / 3) + 50)
pygame.display.update()
waitForPlayerToPressKey()
gameOverSound.stop()
My buildscript was incorrect, I found a better buildscript online to use instead
try:
from distutils.core import setup
import py2exe, pygame
from modulefinder import Module
import glob, fnmatch
import sys, os, shutil
import operator
except ImportError, message:
raise SystemExit, "Unable to load module. %s" % message
#hack which fixes the pygame mixer and pygame font
origIsSystemDLL = py2exe.build_exe.isSystemDLL # save the orginal before we edit it
def isSystemDLL(pathname):
# checks if the freetype and ogg dll files are being included
if os.path.basename(pathname).lower() in ("libfreetype-6.dll", "libogg-0.dll","sdl_ttf.dll"): # "sdl_ttf.dll" added by arit.
return 0
return origIsSystemDLL(pathname) # return the orginal function
py2exe.build_exe.isSystemDLL = isSystemDLL # override the default function with this one
class pygame2exe(py2exe.build_exe.py2exe): #This hack make sure that pygame default font is copied: no need to modify code for specifying default font
def copy_extensions(self, extensions):
#Get pygame default font
pygamedir = os.path.split(pygame.base.__file__)[0]
pygame_default_font = os.path.join(pygamedir, pygame.font.get_default_font())
#Add font to list of extension to be copied
extensions.append(Module("pygame.font", pygame_default_font))
py2exe.build_exe.py2exe.copy_extensions(self, extensions)
class BuildExe:
def __init__(self):
#Name of starting + .py
self.script = "raw.py"
#Name of program
self.project_name = "main"
#Project url
self.project_url = "about:none"
#Version of program
self.project_version = "0.0"
#License of the program
self.license = "MyApps License"
#Auhor of program
self.author_name = "Me"
self.author_email = "example#example.com"
self.copyright = "Copyright (c) 2009 Me."
#Description
self.project_description = "MyApps Description"
#Icon file (None will use pygame default icon)
self.icon_file = None
#Extra files/dirs copied to game
self.extra_datas = []
#Extra/excludes python modules
self.extra_modules = []
self.exclude_modules = []
#DLL Excludes
self.exclude_dll = ['']
#python scripts (strings) to be included, seperated by a comma
self.extra_scripts = []
#Zip file name (None will bundle files in exe instead of zip file)
self.zipfile_name = None
#Dist directory
self.dist_dir ='C:\Python27\dist'
## Code from DistUtils tutorial at http://wiki.python.org/moin/Distutils/Tutorial
## Originally borrowed from wxPython's setup and config files
def opj(self, *args):
path = os.path.join(*args)
return os.path.normpath(path)
def find_data_files(self, srcdir, *wildcards, **kw):
# get a list of all files under the srcdir matching wildcards,
# returned in a format to be used for install_data
def walk_helper(arg, dirname, files):
if '.svn' in dirname:
return
names = []
lst, wildcards = arg
for wc in wildcards:
wc_name = self.opj(dirname, wc)
for f in files:
filename = self.opj(dirname, f)
if fnmatch.fnmatch(filename, wc_name) and not os.path.isdir(filename):
names.append(filename)
if names:
lst.append( (dirname, names ) )
file_list = []
recursive = kw.get('recursive', True)
if recursive:
os.path.walk(srcdir, walk_helper, (file_list, wildcards))
else:
walk_helper((file_list, wildcards),
srcdir,
[os.path.basename(f) for f in glob.glob(self.opj(srcdir, '*'))])
return file_list
def run(self):
if os.path.isdir(self.dist_dir): #Erase previous destination dir
shutil.rmtree(self.dist_dir)
#Use the default pygame icon, if none given
if self.icon_file == None:
path = os.path.split(pygame.__file__)[0]
self.icon_file = os.path.join(path, 'pygame.ico')
#List all data files to add
extra_datas = []
for data in self.extra_datas:
if os.path.isdir(data):
extra_datas.extend(self.find_data_files(data, '*'))
else:
extra_datas.append(('.', [data]))
setup(
cmdclass = {'py2exe': pygame2exe},
version = self.project_version,
description = self.project_description,
name = self.project_name,
url = self.project_url,
author = self.author_name,
author_email = self.author_email,
license = self.license,
# targets to build
windows = [{
'script': self.script,
'icon_resources': [(0, self.icon_file)],
'copyright': self.copyright
}],
options = {'py2exe': {'optimize': 2, 'bundle_files': 2, 'compressed': True, \
'excludes': self.exclude_modules, 'packages': self.extra_modules, \
'dll_excludes': self.exclude_dll,
'includes': self.extra_scripts} },
zipfile = self.zipfile_name,
data_files = extra_datas,
dist_dir = self.dist_dir
)
if os.path.isdir('build'): #Clean up build dir
shutil.rmtree('build')
if __name__ == '__main__':
if operator.lt(len(sys.argv), 2):
sys.argv.append('py2exe')
BuildExe().run() #Run generation
raw_input("Press any key to continue") #Pause to let user see that things ends

Graphite / Carbon / Ceres node overlap

I'm working with Graphite monitoring using Carbon and Ceres as the storage method. I have some problems with correcting bad data. It seems that (due to various problems) I've ended up with overlapping files. That is, since Carbon / Ceres stores the data as timestamp#interval.slice, I can have two or more files with overlapping time ranges.
There are two kinds of overlaps:
File A: +------------+ orig file
File B: +-----+ subset
File C: +---------+ overlap
This is causing problems because the existing tools available (ceres-maintenance defrag and rollup) don't cope with these overlaps. Instead, they skip the directory and move on. This is a problem, obviously.
I've created a script that fixes this problem, as follows:
For subsets, just delete the subset file.
For overlaps, using the file system 'truncate' on the orig file at the point where the next file starts. While it is possible to cut off the start of the overlap file and rename it properly, I would suggest that this is fraught with danger.
I've found that it's possible to do this in two ways:
Walk the dirs and iterate over the files, fixing as you go, and find the file subsets, remove them;
Walk the dirs and fix all the problems in a dir before moving on. This is BY FAR the faster approach, since the dir walk is hugely time consuming.
Code:
#!/usr/bin/env python2.6
################################################################################
import io
import os
import time
import sys
import string
import logging
import unittest
import datetime
import random
import zmq
import json
import socket
import traceback
import signal
import select
import simplejson
import cPickle as pickle
import re
import shutil
import collections
from pymongo import Connection
from optparse import OptionParser
from pprint import pprint, pformat
################################################################################
class SliceFile(object):
def __init__(self, fname):
self.name = fname
basename = fname.split('/')[-1]
fnArray = basename.split('#')
self.timeStart = int(fnArray[0])
self.freq = int(fnArray[1].split('.')[0])
self.size = None
self.numPoints = None
self.timeEnd = None
self.deleted = False
def __repr__(self):
out = "Name: %s, tstart=%s tEnd=%s, freq=%s, size=%s, npoints=%s." % (
self.name, self.timeStart, self.timeEnd, self.freq, self.size, self.numPoints)
return out
def setVars(self):
self.size = os.path.getsize(self.name)
self.numPoints = int(self.size / 8)
self.timeEnd = self.timeStart + (self.numPoints * self.freq)
################################################################################
class CeresOverlapFixup(object):
def __del__(self):
import datetime
self.writeLog("Ending at %s" % (str(datetime.datetime.today())))
self.LOGFILE.flush()
self.LOGFILE.close()
def __init__(self):
self.verbose = False
self.debug = False
self.LOGFILE = open("ceresOverlapFixup.log", "a")
self.badFilesList = set()
self.truncated = 0
self.subsets = 0
self.dirsExamined = 0
self.lastStatusTime = 0
def getOptionParser(self):
return OptionParser()
def getOptions(self):
parser = self.getOptionParser()
parser.add_option("-d", "--debug", action="store_true", dest="debug", default=False, help="debug mode for this program, writes debug messages to logfile." )
parser.add_option("-v", "--verbose", action="store_true", dest="verbose", default=False, help="verbose mode for this program, prints a lot to stdout." )
parser.add_option("-b", "--basedir", action="store", type="string", dest="basedir", default=None, help="base directory location to start converting." )
(options, args) = parser.parse_args()
self.debug = options.debug
self.verbose = options.verbose
self.basedir = options.basedir
assert self.basedir, "must provide base directory."
# Examples:
# ./updateOperations/1346805360#60.slice
# ./updateOperations/1349556660#60.slice
# ./updateOperations/1346798040#60.slice
def getFileData(self, inFilename):
ret = SliceFile(inFilename)
ret.setVars()
return ret
def removeFile(self, inFilename):
os.remove(inFilename)
#self.writeLog("removing file: %s" % (inFilename))
self.subsets += 1
def truncateFile(self, fname, newSize):
if self.verbose:
self.writeLog("Truncating file, name=%s, newsize=%s" % (pformat(fname), pformat(newSize)))
IFD = None
try:
IFD = os.open(fname, os.O_RDWR|os.O_CREAT)
os.ftruncate(IFD, newSize)
os.close(IFD)
self.truncated += 1
except:
self.writeLog("Exception during truncate: %s" % (traceback.format_exc()))
try:
os.close(IFD)
except:
pass
return
def printStatus(self):
now = self.getNowTime()
if ((now - self.lastStatusTime) > 10):
self.writeLog("Status: time=%d, Walked %s dirs, subsetFilesRemoved=%s, truncated %s files." % (now, self.dirsExamined, self.subsets, self.truncated))
self.lastStatusTime = now
def fixupThisDir(self, inPath, inFiles):
# self.writeLog("Fixing files in dir: %s" % (inPath))
if not '.ceres-node' in inFiles:
# self.writeLog("--> Not a slice directory, skipping.")
return
self.dirsExamined += 1
sortedFiles = sorted(inFiles)
sortedFiles = [x for x in sortedFiles if ((x != '.ceres-node') and (x.count('#') > 0)) ]
lastFile = None
fileObjList = []
for thisFile in sortedFiles:
wholeFilename = os.path.join(inPath, thisFile)
try:
curFile = self.getFileData(wholeFilename)
fileObjList.append(curFile)
except:
self.badFilesList.add(wholeFilename)
self.writeLog("ERROR: file %s, %s" % (wholeFilename, traceback.format_exc()))
# name is timeStart, really.
fileObjList = sorted(fileObjList, key=lambda thisObj: thisObj.name)
while fileObjList:
self.printStatus()
changes = False
firstFile = fileObjList[0]
removedFiles = []
for curFile in fileObjList[1:]:
if (curFile.timeEnd <= firstFile.timeEnd):
# have subset file. elim.
self.removeFile(curFile.name)
removedFiles.append(curFile.name)
self.subsets += 1
changes = True
if self.verbose:
self.writeLog("Subset file situation. First=%s, overlap=%s" % (firstFile, curFile))
fileObjList = [x for x in fileObjList if x.name not in removedFiles]
if (len(fileObjList) < 2):
break
secondFile = fileObjList[1]
# LT is right. FirstFile's timeEnd is always the first open time after first is done.
# so, first starts#100, len=2, end=102, positions used=100,101. second start#102 == OK.
if (secondFile.timeStart < firstFile.timeEnd):
# truncate first file.
# file_A (last): +---------+
# file_B (curr): +----------+
# solve by truncating previous file at startpoint of current file.
newLenFile_A_seconds = int(secondFile.timeStart - firstFile.timeStart)
newFile_A_datapoints = int(newLenFile_A_seconds / firstFile.freq)
newFile_A_bytes = int(newFile_A_datapoints) * 8
if (not newFile_A_bytes):
fileObjList = fileObjList[1:]
continue
assert newFile_A_bytes, "Must have size. newLenFile_A_seconds=%s, newFile_A_datapoints=%s, newFile_A_bytes=%s." % (newLenFile_A_seconds, newFile_A_datapoints, newFile_A_bytes)
self.truncateFile(firstFile.name, newFile_A_bytes)
if self.verbose:
self.writeLog("Truncate situation. First=%s, overlap=%s" % (firstFile, secondFile))
self.truncated += 1
fileObjList = fileObjList[1:]
changes = True
if not changes:
fileObjList = fileObjList[1:]
def getNowTime(self):
return time.time()
def walkDirStructure(self):
startTime = self.getNowTime()
self.lastStatusTime = startTime
updateStatsDict = {}
self.okayFiles = 0
emptyFiles = 0
for (thisPath, theseDirs, theseFiles) in os.walk(self.basedir):
self.printStatus()
self.fixupThisDir(thisPath, theseFiles)
self.dirsExamined += 1
endTime = time.time()
# time.sleep(11)
self.printStatus()
self.writeLog( "now = %s, started at %s, elapsed time = %s seconds." % (startTime, endTime, endTime - startTime))
self.writeLog( "Done.")
def writeLog(self, instring):
print instring
print >> self.LOGFILE, instring
self.LOGFILE.flush()
def main(self):
self.getOptions()
self.walkDirStructure()

Resources