Celery not starting in OS X - dbm.error: db type is dbm.gnu, but the module is not available - macos

I'm trying to run celery worker in OS X (Mavericks). I activated virtual environment (python 3.4) and tried to start Celery with this argument:
celery worker --app=scheduling -linfo
Where scheduling is my celery app.
But I ended up with this error: dbm.error: db type is dbm.gnu, but the module is not available
Complete stacktrace:
Traceback (most recent call last):
File "/Users/other/PhoenixEnv/lib/python3.4/site-packages/kombu/utils/__init__.py", line 320, in __get__
return obj.__dict__[self.__name__]
KeyError: 'db'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "/Users/other/PhoenixEnv/bin/celery", line 9, in <module>
load_entry_point('celery==3.1.9', 'console_scripts', 'celery')()
File "/Users/other/PhoenixEnv/lib/python3.4/site-packages/celery/__main__.py", line 30, in main
main()
File "/Users/other/PhoenixEnv/lib/python3.4/site-packages/celery/bin/celery.py", line 80, in main
cmd.execute_from_commandline(argv)
File "/Users/other/PhoenixEnv/lib/python3.4/site-packages/celery/bin/celery.py", line 768, in execute_from_commandline
super(CeleryCommand, self).execute_from_commandline(argv)))
File "/Users/other/PhoenixEnv/lib/python3.4/site-packages/celery/bin/base.py", line 308, in execute_from_commandline
return self.handle_argv(self.prog_name, argv[1:])
File "/Users/other/PhoenixEnv/lib/python3.4/site-packages/celery/bin/celery.py", line 760, in handle_argv
return self.execute(command, argv)
File "/Users/other/PhoenixEnv/lib/python3.4/site-packages/celery/bin/celery.py", line 692, in execute
).run_from_argv(self.prog_name, argv[1:], command=argv[0])
File "/Users/other/PhoenixEnv/lib/python3.4/site-packages/celery/bin/worker.py", line 175, in run_from_argv
return self(*args, **options)
File "/Users/other/PhoenixEnv/lib/python3.4/site-packages/celery/bin/base.py", line 271, in __call__
ret = self.run(*args, **kwargs)
File "/Users/other/PhoenixEnv/lib/python3.4/site-packages/celery/bin/worker.py", line 209, in run
).start()
File "/Users/other/PhoenixEnv/lib/python3.4/site-packages/celery/worker/__init__.py", line 100, in __init__
self.setup_instance(**self.prepare_args(**kwargs))
File "/Users/other/PhoenixEnv/lib/python3.4/site-packages/celery/worker/__init__.py", line 141, in setup_instance
self.blueprint.apply(self, **kwargs)
File "/Users/other/PhoenixEnv/lib/python3.4/site-packages/celery/bootsteps.py", line 221, in apply
step.include(parent)
File "/Users/other/PhoenixEnv/lib/python3.4/site-packages/celery/bootsteps.py", line 347, in include
return self._should_include(parent)[0]
File "/Users/other/PhoenixEnv/lib/python3.4/site-packages/celery/bootsteps.py", line 343, in _should_include
return True, self.create(parent)
File "/Users/other/PhoenixEnv/lib/python3.4/site-packages/celery/worker/components.py", line 220, in create
w._persistence = w.state.Persistent(w.state, w.state_db, w.app.clock)
File "/Users/other/PhoenixEnv/lib/python3.4/site-packages/celery/worker/state.py", line 161, in __init__
self.merge()
File "/Users/other/PhoenixEnv/lib/python3.4/site-packages/celery/worker/state.py", line 169, in merge
self._merge_with(self.db)
File "/Users/other/PhoenixEnv/lib/python3.4/site-packages/kombu/utils/__init__.py", line 322, in __get__
value = obj.__dict__[self.__name__] = self.__get(obj)
File "/Users/other/PhoenixEnv/lib/python3.4/site-packages/celery/worker/state.py", line 238, in db
return self.open()
File "/Users/other/PhoenixEnv/lib/python3.4/site-packages/celery/worker/state.py", line 165, in open
self.filename, protocol=self.protocol, writeback=True,
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/shelve.py", line 239, in open
return DbfilenameShelf(filename, flag, protocol, writeback)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/shelve.py", line 223, in __init__
Shelf.__init__(self, dbm.open(filename, flag), protocol, writeback)
File "/Library/Frameworks/Python.framework/Versions/3.4/lib/python3.4/dbm/__init__.py", line 91, in open
"available".format(result))
dbm.error: db type is dbm.gnu, but the module is not available
Please help.

I switched to python3.5 and got the same error. On Ubuntu I could fix it with
aptitude install python3.5-gdbm

I got the same error. On Macbook I could fix it with
brew install gdb

Related

how to read from h5py in multiprocessing without errors

I have code like:
def get_df(path, key):
with h5py.File(path) as hdf:
df = pd.DataFrame(np.array(hdf[key]))
return df
def f(key):
df = get_df(path, key)
...transform df...
return df
with multiprocessing.Pool(n_cpus) as pool:
rvals = pool.map(f, keys)
And I'm getting this error many times:
terminate called after throwing an instance of 'std::system_error'
what(): Resource temporarily unavailable
and one copy of this error:
Exception in thread Thread-1:
Traceback (most recent call last):
File "/home/lenail/.conda/envs/py38/lib/python3.8/threading.py", line 932, in _bootstrap_inner
self.run()
File "/home/lenail/.conda/envs/py38/lib/python3.8/threading.py", line 870, in run
self._target(*self._args, **self._kwargs)
File "/home/lenail/.conda/envs/py38/lib/python3.8/multiprocessing/pool.py", line 513, in _handle_workers
cls._maintain_pool(ctx, Process, processes, pool, inqueue,
File "/home/lenail/.conda/envs/py38/lib/python3.8/multiprocessing/pool.py", line 337, in _maintain_pool
Pool._repopulate_pool_static(ctx, Process, processes, pool,
File "/home/lenail/.conda/envs/py38/lib/python3.8/multiprocessing/pool.py", line 326, in _repopulate_pool_static
w.start()
File "/home/lenail/.conda/envs/py38/lib/python3.8/multiprocessing/process.py", line 121, in start
self._popen = self._Popen(self)
File "/home/lenail/.conda/envs/py38/lib/python3.8/multiprocessing/context.py", line 277, in _Popen
return Popen(process_obj)
File "/home/lenail/.conda/envs/py38/lib/python3.8/multiprocessing/popen_fork.py", line 19, in __init__
self._launch(process_obj)
File "/home/lenail/.conda/envs/py38/lib/python3.8/multiprocessing/popen_fork.py", line 70, in _launch
self.pid = os.fork()
BlockingIOError: [Errno 11] Resource temporarily unavailable
any ideas why this might be or how to fix it? I had read that h5py supports multiprocessing.
Python 3.8.13 | packaged by conda-forge
>>> h5py.__version__
'3.6.0'

ERROR: Could not install packages due to an OSError: [Errno 22]

Hi im trying to install voevent-parse via pip and run into the following error:
ERROR: Could not install packages due to an OSError. Traceback (most recent call last): File "c:\users\timo\appdata\local\programs\python\python39\lib\site-packages\pip\_vendor\resolvelib\resolvers.py", line 171, in _merge_into_criterion
crit = self.state.criteria[name] KeyError: 'voevent-parse'
During handling of the above exception, another exception occurred:
Traceback (most recent call last):
File "c:\users\timo\appdata\local\programs\python\python39\lib\site-packages\pip\_internal\commands\install.py", line 316, in run
requirement_set = resolver.resolve(
File "c:\users\timo\appdata\local\programs\python\python39\lib\site-packages\pip\_internal\resolution\resolvelib\resolver.py", line 121, in resolve
self._result = resolver.resolve(
File "c:\users\timo\appdata\local\programs\python\python39\lib\site-packages\pip\_vendor\resolvelib\resolvers.py", line 453, in resolve
state = resolution.resolve(requirements, max_rounds=max_rounds)
File "c:\users\timo\appdata\local\programs\python\python39\lib\site-packages\pip\_vendor\resolvelib\resolvers.py", line 318, in resolve
name, crit = self._merge_into_criterion(r, parent=None)
File "c:\users\timo\appdata\local\programs\python\python39\lib\site-packages\pip\_vendor\resolvelib\resolvers.py", line 173, in _merge_into_criterion
crit = Criterion.from_requirement(self._p, requirement, parent)
File "c:\users\timo\appdata\local\programs\python\python39\lib\site-packages\pip\_vendor\resolvelib\resolvers.py", line 82, in from_requirement
if not cands:
File "c:\users\timo\appdata\local\programs\python\python39\lib\site-packages\pip\_vendor\resolvelib\structs.py", line 124, in __bool__
return bool(self._sequence)
File "c:\users\timo\appdata\local\programs\python\python39\lib\site-packages\pip\_internal\resolution\resolvelib\found_candidates.py", line 99, in __bool__
return any(self)
File "c:\users\timo\appdata\local\programs\python\python39\lib\site-packages\pip\_internal\resolution\resolvelib\factory.py", line 237, in iter_index_candidates
candidate = self._make_candidate_from_link(
File "c:\users\timo\appdata\local\programs\python\python39\lib\site-packages\pip\_internal\resolution\resolvelib\factory.py", line 165, in _make_candidate_from_link
self._link_candidate_cache[link] = LinkCandidate(
File "c:\users\timo\appdata\local\programs\python\python39\lib\site-packages\pip\_internal\resolution\resolvelib\candidates.py", line 300, in __init__
super().__init__(
File "c:\users\timo\appdata\local\programs\python\python39\lib\site-packages\pip\_internal\resolution\resolvelib\candidates.py", line 144, in __init__
self.dist = self._prepare()
File "c:\users\timo\appdata\local\programs\python\python39\lib\site-packages\pip\_internal\resolution\resolvelib\candidates.py", line 226, in _prepare
dist = self._prepare_distribution()
File "c:\users\timo\appdata\local\programs\python\python39\lib\site-packages\pip\_internal\resolution\resolvelib\candidates.py", line 311, in _prepare_distribution
return self._factory.preparer.prepare_linked_requirement(
File "c:\users\timo\appdata\local\programs\python\python39\lib\site-packages\pip\_internal\operations\prepare.py", line 457, in prepare_linked_requirement
return self._prepare_linked_requirement(req, parallel_builds)
File "c:\users\timo\appdata\local\programs\python\python39\lib\site-packages\pip\_internal\operations\prepare.py", line 480, in _prepare_linked_requirement
local_file = unpack_url(
File "c:\users\timo\appdata\local\programs\python\python39\lib\site-packages\pip\_internal\operations\prepare.py", line 240, in unpack_url
unpack_file(file.path, location, file.content_type)
File "c:\users\timo\appdata\local\programs\python\python39\lib\site-packages\pip\_internal\utils\unpacking.py", line 266, in unpack_file
untar_file(filename, location)
File "c:\users\timo\appdata\local\programs\python\python39\lib\site-packages\pip\_internal\utils\unpacking.py", line 230, in untar_file
with open(path, 'wb') as destfp:
OSError: [Errno 22] Invalid argument: 'C:\\Users\\Timo\\AppData\\Local\\Temp\\pip-install-wpb7xknq\\voevent-parse_090e9cbdcae64f31a771cb0698293f1a\\src/voeventparse/fixtures/MOA_Lensing_Event_2015-07-10T14:50:54.00.xml'
Removed build tracker: 'C:\\Users\\Timo\\AppData\\Local\\Temp\\pip-req-tracker-cmjrk9m4'
Im using the latest Windows 10 version and latest pip version
Please consider running your command line in administrator

while launching spyder , spyder kernel is showing this error. and anaconda is also not installing other environments

Traceback (most recent call last):
File "C:\ProgramData\Anaconda3\lib\site‑packages\spyder\plugins\ipythonconsole\plugin.py", line 1223, in create_kernel_manager_and_kernel_client
kernel_manager.start_kernel(stderr=stderr_handle)
File "C:\Users\Mahima Kaushik\AppData\Roaming\Python\Python38\site‑packages\jupyter_client\manager.py", line 309, in start_kernel
kernel_cmd, kw = self.pre_start_kernel(**kw)
File "C:\Users\Mahima Kaushik\AppData\Roaming\Python\Python38\site‑packages\jupyter_client\manager.py", line 256, in pre_start_kernel
self.write_connection_file()
File "C:\Users\Mahima Kaushik\AppData\Roaming\Python\Python38\site‑packages\jupyter_client\connect.py", line 468, in write_connection_file
self.connection_file, cfg = write_connection_file(self.connection_file,
File "C:\Users\Mahima Kaushik\AppData\Roaming\Python\Python38\site‑packages\jupyter_client\connect.py", line 138, in write_connection_file
with secure_write(fname) as f:
File "C:\ProgramData\Anaconda3\lib\contextlib.py", line 113, in enter
return next(self.gen)
File "C:\Users\Mahima Kaushik\AppData\Roaming\Python\Python38\site‑packages\jupyter_core\paths.py", line 435, in secure_write
win32_restrict_file_to_user(fname)
File "C:\Users\Mahima Kaushik\AppData\Roaming\Python\Python38\site‑packages\jupyter_core\paths.py", line 361, in win32_restrict_file_to_user
import win32api
ImportError: DLL load failed while importing win32api: %1 is not a valid Win32 application.
https://github.com/spyder-ide/spyder/issues/12040#issuecomment-705033049
https://github.com/spyder-ide/spyder/issues/12040#issuecomment-705057347
I removed the following directory to fix my problem:
C:\Users\Mahima Kaushik\AppData\Roaming\Python

Setting up Python Virtual Env Initiating Jupyter Notebook gaierror: [Errno -2] Name or service not known

When I try to Initiate Jupyter notebook in the virtual env, I get the following error - .gaierror: [Errno -2] Name or service not known
Virtual Env -wget https://pypi.python.org/packages/source/v/virtualenv/virtualenv-12.0.7.tar.gz
Traceback (most recent call last):
File "/home/vraja/pyspark/bin/jupyter-notebook", line 11, in <module>
sys.exit(main())
File "/home/vraja/pyspark/lib/python2.7/site-
packages/jupyter_core/application.py", line 266, in launch_instance
return super(JupyterApp, cls).launch_instance(argv=argv, **kwargs)
File "/home/vraja/pyspark/lib/python2.7/site-
packages/traitlets/config/application.py", line 657, in launch_instance
app.initialize(argv)
File "<decorator-gen-7>", line 2, in initialize
File "/home/vraja/pyspark/lib/python2.7/site-
packages/traitlets/config/application.py", line 87, in catch_config_error
return method(app, *args, **kwargs)
File "/home/vraja/pyspark/lib/python2.7/site-
packages/notebook/notebookapp.py", line 1629, in initialize
self.init_webapp()
File "/home/vraja/pyspark/lib/python2.7/site-
packages/notebook/notebookapp.py", line 1379, in init_webapp
self.jinja_environment_options,
File "/home/vraja/pyspark/lib/python2.7/site-
packages/notebook/notebookapp.py", line 158, in __init__
default_url, settings_overrides, jinja_env_options)
File "/home/vraja/pyspark/lib/python2.7/site-
packages/notebook/notebookapp.py", line 251, in init_settings
allow_remote_access=jupyter_app.allow_remote_access,
File "/home/vraja/pyspark/lib/python2.7/site-
packages/traitlets/traitlets.py", line 556, in __get__
return self.get(obj, cls)
File "/home/vraja/pyspark/lib/python2.7/site-
packages/traitlets/traitlets.py", line 535, in get
value = self._validate(obj, dynamic_default())
File "/home/vraja/pyspark/lib/python2.7/site-
packages/notebook/notebookapp.py", line 872, in _default_allow_remote
for info in socket.getaddrinfo(self.ip, self.port, 0,
socket.SOCK_STREAM):
socket.gaierror: [Errno -2] Name or service not known
(pyspark)-sh-4.1$
did you set profile variable ?..
Please add below env variable in profile file.
export PYSPARK_DRIVER_PYTHON=jupyter
export PYSPARK_DRIVER_PYTHON_OPTS='notebook'
Changed NotebookApp.ip='*' to Changed NotebookApp.ip='0.0.0.0' according to https://github.com/jupyterlab/jupyterlab/issues/5410 (Pazitos10 comment)

Google Apis Client Generator Python Not Running

I'm trying to execute Google APIs Client Generator to build new ADMIN-SDK for Java Language however I'm stuck in the generator, I've tried step by step using python2.6 and python2.7 both had similar outputs.
$ python $(/bin/pwd)/googleapis/codegen/generate_library.py --api_name=plus --api_version=v1 --language=java --output_dir=/tmp/gen --language=java
Traceback (most recent call last): File
"/Users/alejacquet/Development/git/google-apis-client-generator/src/googleapis/codegen/generate_library.py",
line 245, in
app.run() File "/Library/Python/2.7/site-packages/google_apputils-0.3.0-py2.7.egg/google/apputils/app.py",
line 216, in run
return _actual_start() File "/Library/Python/2.7/site-packages/google_apputils-0.3.0-py2.7.egg/google/apputils/app.py",
line 244, in _actual_start
really_start() File "/Library/Python/2.7/site-packages/google_apputils-0.3.0-py2.7.egg/google/apputils/app.py",
line 203, in really_start
sys.exit(main(argv)) File "/Users/alejacquet/Development/git/google-apis-client-generator/src/googleapis/codegen/generate_library.py",
line 152, in main
language_variant=FLAGS.language_variant) File "/Users/alejacquet/Development/git/google-apis-client-generator/src/googleapis/codegen/generate_library.py",
line 179, in Generate
language_variations = Targets().VariationsForLanguage(language) File
"/Users/alejacquet/Development/git/google-apis-client-generator/src/googleapis/codegen/targets.py",
line 68, in init
files.GetFileContents(self.targets_path)) File "/Users/alejacquet/Development/git/google-apis-client-generator/src/googleapis/codegen/json_with_comments.py",
line 54, in Loads
return json.loads(stripped, **kw) File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/init.py",
line 326, in loads
File "/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/decoder.py",
line 366, in decode File
"/System/Library/Frameworks/Python.framework/Versions/2.7/lib/python2.7/json/decoder.py",
line 382, in raw_decode ValueError: Expecting property name: line 9
column 9 (char 198)
Any ideas?
You may have to specify --language_variant=XXX, where XXX is one of the directory names under .../templates/java

Resources