Django Celery issues within Docker - django-celery

Question

Having a major issue I cannot track down and I'm spinning my wheels. Any help would be greatly appreciated. I have django, celerybeat and celeryworker containers that all use the same base image of django.
local.yml
version: "3"
volumes:
local_postgres_data: {}
local_postgres_data_backups: {}
services:
django: &django
build:
context: .
dockerfile: ./compose/local/django/Dockerfile
image: local_django
container_name: django
depends_on:
- postgres
- mailhog
- redis
volumes:
- .:/app
env_file:
- ./.envs/.local/.django
- ./.envs/.local/.postgres
ports:
- "8000:8000"
command: /start
restart: always
postgres:
build:
context: .
dockerfile: ./compose/production/postgres/Dockerfile
image: production_postgres
container_name: postgres
volumes:
- local_postgres_data:/var/lib/postgresql/data
- local_postgres_data_backups:/backups
env_file:
- ./.envs/.local/.postgres
ports:
- "5432:5432"
mailhog:
image: mailhog/mailhog:v1.0.0
container_name: mailhog
ports:
- "8025:8025"
logging:
driver: "none" # disable saving logs
redis:
image: redis:6
container_name: redis
celeryworker:
<<: *django
image: local_celeryworker
container_name: celeryworker
depends_on:
- redis
- postgres
- mailhog
ports: []
command: /start-celeryworker
restart: always
celerybeat:
<<: *django
image: local_celerybeat
container_name: celerybeat
depends_on:
- redis
- postgres
- mailhog
ports: []
command: /start-celerybeat
restart: always
node:
build:
context: .
dockerfile: ./compose/local/node/Dockerfile
image: local_node
container_name: node
depends_on:
- django
volumes:
- .:/app
# http://jdlm.info/articles/2016/03/06/lessons-building-node-app-docker.html
- /app/node_modules
command: npm run dev
ports:
- "3000:3000"
# Expose browsersync UI: https://www.browsersync.io/docs/options/#option-ui
- "3001:3001"
compose/local/django/Dockerfile
ARG PYTHON_VERSION=3.9-slim-bullseye
# define an alias for the specfic python version used in this file.
FROM python:${PYTHON_VERSION} as python
# Python build stage
FROM python as python-build-stage
ARG BUILD_ENVIRONMENT=local
# Install apt packages
RUN apt-get update && apt-get install --no-install-recommends -y \
# dependencies for building Python packages
build-essential \
# psycopg2 dependencies
libpq-dev
# Requirements are installed here to ensure they will be cached.
COPY ./requirements .
# Create Python Dependency and Sub-Dependency Wheels.
RUN pip wheel --wheel-dir /usr/src/app/wheels \
-r ${BUILD_ENVIRONMENT}.txt
# Python 'run' stage
FROM python as python-run-stage
ARG BUILD_ENVIRONMENT=local
ARG APP_HOME=/app
ENV PYTHONUNBUFFERED 1
ENV PYTHONDONTWRITEBYTECODE 1
ENV BUILD_ENV ${BUILD_ENVIRONMENT}
WORKDIR ${APP_HOME}
# Install required system dependencies
RUN apt-get update && apt-get install --no-install-recommends -y \
# psycopg2 dependencies
libpq-dev \
# Translations dependencies
gettext \
# cleaning up unused files
&& apt-get purge -y --auto-remove -o APT::AutoRemove::RecommendsImportant=false \
&& rm -rf /var/lib/apt/lists/*
# All absolute dir copies ignore workdir instruction. All relative dir copies are wrt to the workdir instruction
# copy python dependency wheels from python-build-stage
COPY --from=python-build-stage /usr/src/app/wheels /wheels/
# use wheels to install python dependencies
RUN pip install --no-cache-dir --no-index --find-links=/wheels/ /wheels/* \
&& rm -rf /wheels/
COPY ./compose/production/django/entrypoint /entrypoint
RUN sed -i 's/\r$//g' /entrypoint
RUN chmod +x /entrypoint
COPY ./compose/local/django/start /start
RUN sed -i 's/\r$//g' /start
RUN chmod +x /start
COPY ./compose/local/django/celery/worker/start /start-celeryworker
RUN sed -i 's/\r$//g' /start-celeryworker
RUN chmod +x /start-celeryworker
COPY ./compose/local/django/celery/beat/start /start-celerybeat
RUN sed -i 's/\r$//g' /start-celerybeat
RUN chmod +x /start-celerybeat
COPY ./compose/local/django/celery/flower/start /start-flower
RUN sed -i 's/\r$//g' /start-flower
RUN chmod +x /start-flower
# copy application code to WORKDIR
COPY . ${APP_HOME}
ENTRYPOINT ["/entrypoint"]
config/celery_app.py
import os
from celery import Celery
# set the default Django settings module for the 'celery' program.
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.local")
app = Celery("myapp")
# Using a string here means the worker doesn't have to serialize
# the configuration object to child processes.
# - namespace='CELERY' means all celery-related configuration keys
# should have a `CELERY_` prefix.
app.config_from_object("django.conf:settings", namespace="CELERY")
# Load task modules from all registered Django app configs.
app.autodiscover_tasks()
config/__init__.py
# This will make sure the app is always imported when
# Django starts so that shared_task will use this app.
from .celery_app import app as celery_app
__all__ = ("celery_app",)
compose/local/django/celery/beat/start
#!/bin/bash
set -o errexit
set -o nounset
rm -f './celerybeat.pid'
celery -A config.celery_app beat -l INFO
compose/local/django/celery/worker/start
#!/bin/bash
set -o errexit
set -o nounset
watchgod celery.__main__.main --args -A config.celery_app worker -l INFO
When running docker-compose up without celeryworker or celerybeat the modules are found fine in the django container. But as soon as I add the celeryworker or celeryworker I get the following about not being able to find modules.
celery beat v5.2.7 (dawn-chorus) is starting.
2022-08-09 14:07:41,879 [celery.utils.dispatch.signal:280] ERROR - Signal handler <bound method DjangoFixup.on_import_modules of <celery.fixups.django.DjangoFixup object at 0x7f6cff68e670>> raised: ModuleNotFoundError("No module named 'customadmin'")
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/celery/utils/dispatch/signal.py", line 276, in send
response = receiver(signal=self, sender=sender, **named)
File "/usr/local/lib/python3.9/site-packages/celery/fixups/django.py", line 82, in on_import_modules
self.worker_fixup.validate_models()
File "/usr/local/lib/python3.9/site-packages/celery/fixups/django.py", line 120, in validate_models
self.django_setup()
File "/usr/local/lib/python3.9/site-packages/celery/fixups/django.py", line 116, in django_setup
django.setup()
File "/usr/local/lib/python3.9/site-packages/django/__init__.py", line 24, in setup
apps.populate(settings.INSTALLED_APPS)
File "/usr/local/lib/python3.9/site-packages/django/apps/registry.py", line 91, in populate
app_config = AppConfig.create(entry)
File "/usr/local/lib/python3.9/site-packages/django/apps/config.py", line 224, in create
import_module(entry)
File "/usr/local/lib/python3.9/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1030, in _gcd_import
File "<frozen importlib._bootstrap>", line 1007, in _find_and_load
File "<frozen importlib._bootstrap>", line 984, in _find_and_load_unlocked
ModuleNotFoundError: No module named 'customadmin'
2022-08-09 14:07:41,880 [celery.utils.dispatch.signal:280] ERROR - Signal handler <promise#0x7f6cff6491f0 --> <bound method Celery._autodiscover_tasks of <Celery bahamar at 0x7f6cff68eee0>>> raised: AppRegistryNotReady("Apps aren't loaded yet.")
Traceback (most recent call last):
File "/usr/local/lib/python3.9/site-packages/celery/utils/dispatch/signal.py", line 276, in send
response = receiver(signal=self, sender=sender, **named)
File "/usr/local/lib/python3.9/site-packages/vine/promises.py", line 160, in __call__
return self.throw()
File "/usr/local/lib/python3.9/site-packages/vine/promises.py", line 157, in __call__
retval = fun(*final_args, **final_kwargs)
File "/usr/local/lib/python3.9/site-packages/celery/app/base.py", line 689, in _autodiscover_tasks
return self._autodiscover_tasks_from_fixups(related_name)
File "/usr/local/lib/python3.9/site-packages/celery/app/base.py", line 698, in _autodiscover_tasks_from_fixups
return self._autodiscover_tasks_from_names([
File "/usr/local/lib/python3.9/site-packages/celery/app/base.py", line 701, in <listcomp>
for pkg in fixup.autodiscover_tasks()
File "/usr/local/lib/python3.9/site-packages/celery/fixups/django.py", line 92, in autodiscover_tasks
return [config.name for config in apps.get_app_configs()]
File "/usr/local/lib/python3.9/site-packages/django/apps/registry.py", line 145, in get_app_configs
self.check_apps_ready()
File "/usr/local/lib/python3.9/site-packages/django/apps/registry.py", line 136, in check_apps_ready
raise AppRegistryNotReady("Apps aren't loaded yet.")
django.core.exceptions.AppRegistryNotReady: Apps aren't loaded yet.
Traceback (most recent call last):
File "/usr/local/bin/celery", line 8, in <module>
sys.exit(main())
File "/usr/local/lib/python3.9/site-packages/celery/__main__.py", line 15, in main
sys.exit(_main())
File "/usr/local/lib/python3.9/site-packages/celery/bin/celery.py", line 217, in main
return celery(auto_envvar_prefix="CELERY")
File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1130, in __call__
return self.main(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1055, in main
rv = self.invoke(ctx)
File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1657, in invoke
return _process_result(sub_ctx.command.invoke(sub_ctx))
File "/usr/local/lib/python3.9/site-packages/click/core.py", line 1404, in invoke
return ctx.invoke(self.callback, **ctx.params)
File "/usr/local/lib/python3.9/site-packages/click/core.py", line 760, in invoke
return __callback(*args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/click/decorators.py", line 26, in new_func
return f(get_current_context(), *args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/celery/bin/base.py", line 134, in caller
return f(ctx, *args, **kwargs)
File "/usr/local/lib/python3.9/site-packages/celery/bin/beat.py", line 72, in beat
return beat().run()
File "/usr/local/lib/python3.9/site-packages/celery/apps/beat.py", line 75, in run
self.init_loader()
File "/usr/local/lib/python3.9/site-packages/celery/apps/beat.py", line 124, in init_loader
self.app.loader.init_worker()
File "/usr/local/lib/python3.9/site-packages/celery/loaders/base.py", line 111, in init_worker
self.import_default_modules()
File "/usr/local/lib/python3.9/site-packages/celery/loaders/base.py", line 105, in import_default_modules
raise response
File "/usr/local/lib/python3.9/site-packages/celery/utils/dispatch/signal.py", line 276, in send
response = receiver(signal=self, sender=sender, **named)
File "/usr/local/lib/python3.9/site-packages/celery/fixups/django.py", line 82, in on_import_modules
self.worker_fixup.validate_models()
File "/usr/local/lib/python3.9/site-packages/celery/fixups/django.py", line 120, in validate_models
self.django_setup()
File "/usr/local/lib/python3.9/site-packages/celery/fixups/django.py", line 116, in django_setup
django.setup()
File "/usr/local/lib/python3.9/site-packages/django/__init__.py", line 24, in setup
apps.populate(settings.INSTALLED_APPS)
File "/usr/local/lib/python3.9/site-packages/django/apps/registry.py", line 91, in populate
app_config = AppConfig.create(entry)
File "/usr/local/lib/python3.9/site-packages/django/apps/config.py", line 224, in create
import_module(entry)
File "/usr/local/lib/python3.9/importlib/__init__.py", line 127, in import_module
return _bootstrap._gcd_import(name[level:], package, level)
File "<frozen importlib._bootstrap>", line 1030, in _gcd_import
File "<frozen importlib._bootstrap>", line 1007, in _find_and_load
File "<frozen importlib._bootstrap>", line 984, in _find_and_load_unlocked
ModuleNotFoundError: No module named 'customadmin'
customadmin just so happens to be the first app of many apps in my LOCAL_APPS.
Django 3.2.14
celery 5.2.7
django-celery-beat 2.3.0
django-celery-results 2.2.0