diff --git a/.env.example b/.env.example index a7a37c81e..dbb5e0bf7 100644 --- a/.env.example +++ b/.env.example @@ -22,6 +22,7 @@ POSTGRES_USER=fedireads POSTGRES_DB=fedireads POSTGRES_HOST=db -RABBITMQ_DEFAULT_USER=rabbit -RABBITMQ_DEFAULT_PASS=changeme -CELERY_BROKER=amqp://rabbit:changeme@rabbitmq:5672 +CELERY_BROKER=redis://redis:6379/0 +CELERY_RESULT_BACKEND=redis://redis:6379/0 + +FLOWER_PORT=5555 diff --git a/Dockerfile b/Dockerfile index c845035f9..f6dbb7402 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,3 +5,5 @@ WORKDIR /app COPY requirements.txt /app/ RUN pip install -r requirements.txt COPY ./fedireads /app +COPY ./fr_celery /app +EXPOSE 5555 diff --git a/docker-compose.yml b/docker-compose.yml index a119f25d0..a8adde912 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -20,25 +20,32 @@ services: - celery_worker networks: - main - rabbitmq: + redis: + image: redis env_file: .env - image: rabbitmq:latest + ports: + - "6379:6379" networks: - main - ports: - - "5672:5672" restart: on-failure celery_worker: env_file: .env build: . networks: - main - command: celery -A fedireads worker -l info + command: celery -A fr_celery worker -l info volumes: - .:/app depends_on: - db - - rabbitmq + - redis + restart: on-failure + flower: + image: mher/flower + command: ["flower", "--broker=redis://redis:6379/0", "--port=5555"] + env_file: .env + ports: + - "5555:5555" restart: on-failure volumes: pgdata: diff --git a/fedireads/__init__.py b/fedireads/__init__.py index 18666a189..e69de29bb 100644 --- a/fedireads/__init__.py +++ b/fedireads/__init__.py @@ -1,9 +0,0 @@ -''' we need this file to initialize celery ''' -from __future__ import absolute_import, unicode_literals - -# This will make sure the app is always imported when -# Django starts so that shared_task will use this app. -from .celery import app as celery_app - -__all__ = ('celery_app',) - diff --git a/fedireads/broadcast.py b/fedireads/broadcast.py index 3a62afeb0..0b83ab899 100644 --- a/fedireads/broadcast.py +++ b/fedireads/broadcast.py @@ -9,6 +9,7 @@ import requests from urllib.parse import urlparse from fedireads import models +from fedireads.tasks import app def get_recipients(user, post_privacy, direct_recipients=None, limit=False): @@ -39,6 +40,9 @@ def get_recipients(user, post_privacy, direct_recipients=None, limit=False): fedireads_user = limit == 'fedireads' followers = user.followers.filter(fedireads_user=fedireads_user).all() + # we don't need to broadcast to ourself + followers = followers.filter(local=False) + # TODO I don't think this is actually accomplishing pubic/followers only? if post_privacy == 'public': # post to public shared inboxes @@ -58,6 +62,13 @@ def get_recipients(user, post_privacy, direct_recipients=None, limit=False): def broadcast(sender, activity, recipients): ''' send out an event ''' + broadcast_task.delay(sender.id, activity, recipients) + + +@app.task +def broadcast_task(sender_id, activity, recipients): + ''' the celery task for broadcast ''' + sender = models.User.objects.get(id=sender_id) errors = [] for recipient in recipients: try: diff --git a/fedireads/incoming.py b/fedireads/incoming.py index a766cb688..773e4fbe8 100644 --- a/fedireads/incoming.py +++ b/fedireads/incoming.py @@ -13,6 +13,21 @@ import requests from fedireads import models, outgoing from fedireads import status as status_builder from fedireads.remote_user import get_or_create_remote_user +from fedireads.tasks import app + + +@csrf_exempt +def inbox(request, username): + ''' incoming activitypub events ''' + # TODO: should do some kind of checking if the user accepts + # this action from the sender probably? idk + # but this will just throw a 404 if the user doesn't exist + try: + models.User.objects.get(localname=username) + except models.User.DoesNotExist: + return HttpResponseNotFound() + + return shared_inbox(request) @csrf_exempt @@ -40,7 +55,7 @@ def shared_inbox(request): 'Like': handle_favorite, 'Announce': handle_boost, 'Add': { - 'Tag': handle_add, + 'Tag': handle_tag, }, 'Undo': { 'Follow': handle_unfollow, @@ -57,10 +72,11 @@ def shared_inbox(request): if isinstance(handler, dict): handler = handler.get(activity['object']['type'], None) - if handler: - return handler(activity) + if not handler: + return HttpResponseNotFound() - return HttpResponseNotFound() + handler.delay(activity) + return HttpResponse() def verify_signature(request): @@ -109,20 +125,7 @@ def verify_signature(request): return True -@csrf_exempt -def inbox(request, username): - ''' incoming activitypub events ''' - # TODO: should do some kind of checking if the user accepts - # this action from the sender probably? idk - # but this will just throw a 404 if the user doesn't exist - try: - models.User.objects.get(localname=username) - except models.User.DoesNotExist: - return HttpResponseNotFound() - - return shared_inbox(request) - - +@app.task def handle_follow(activity): ''' someone wants to follow a local user ''' # figure out who they want to follow @@ -141,7 +144,7 @@ def handle_follow(activity): # Duplicate follow request. Not sure what the correct behaviour is, but # just dropping it works for now. We should perhaps generate the # Accept, but then do we need to match the activity id? - return HttpResponse() + return if not to_follow.manually_approves_followers: status_builder.create_notification( @@ -156,9 +159,9 @@ def handle_follow(activity): 'FOLLOW_REQUEST', related_user=user ) - return HttpResponse() +@app.task def handle_unfollow(activity): ''' unfollow a local user ''' obj = activity['object'] @@ -172,9 +175,9 @@ def handle_unfollow(activity): return HttpResponseNotFound() to_unfollow.followers.remove(requester) - return HttpResponse() +@app.task def handle_follow_accept(activity): ''' hurray, someone remote accepted a follow request ''' # figure out who they want to follow @@ -191,9 +194,9 @@ def handle_follow_accept(activity): except models.UserFollowRequest.DoesNotExist: pass accepter.followers.add(requester) - return HttpResponse() +@app.task def handle_follow_reject(activity): ''' someone is rejecting a follow request ''' requester = models.User.objects.get(actor=activity['object']['actor']) @@ -208,8 +211,8 @@ def handle_follow_reject(activity): except models.UserFollowRequest.DoesNotExist: pass - return HttpResponse() +@app.task def handle_create(activity): ''' someone did something, good on them ''' user = get_or_create_remote_user(activity['actor']) @@ -219,7 +222,7 @@ def handle_create(activity): if user.local: # we really oughtn't even be sending in this case - return HttpResponse() + return if activity['object'].get('fedireadsType') in ['Review', 'Comment'] and \ 'inReplyToBook' in activity['object']: @@ -251,9 +254,9 @@ def handle_create(activity): except ValueError: return HttpResponseBadRequest() - return HttpResponse() +@app.task def handle_favorite(activity): ''' approval of your good good post ''' try: @@ -261,7 +264,7 @@ def handle_favorite(activity): status = models.Status.objects.get(id=status_id) liker = get_or_create_remote_user(activity['actor']) except (models.Status.DoesNotExist, models.User.DoesNotExist): - return HttpResponseNotFound() + return if not liker.local: status_builder.create_favorite_from_activity(liker, activity) @@ -272,21 +275,20 @@ def handle_favorite(activity): related_user=liker, related_status=status, ) - return HttpResponse() +@app.task def handle_unfavorite(activity): ''' approval of your good good post ''' - try: - favorite_id = activity['object']['id'] - fav = status_builder.get_favorite(favorite_id) - except models.Favorite.DoesNotExist: + favorite_id = activity['object']['id'] + fav = status_builder.get_favorite(favorite_id) + if not fav: return HttpResponseNotFound() fav.delete() - return HttpResponse() +@app.task def handle_boost(activity): ''' someone gave us a boost! ''' try: @@ -306,16 +308,12 @@ def handle_boost(activity): related_status=status, ) - return HttpResponse() -def handle_add(activity): +@app.task +def handle_tag(activity): ''' someone is tagging or shelving a book ''' - if activity['object']['type'] == 'Tag': - user = get_or_create_remote_user(activity['actor']) - if not user.local: - book = activity['target']['id'].split('/')[-1] - status_builder.create_tag(user, book, activity['object']['name']) - return HttpResponse() - return HttpResponse() - return HttpResponseNotFound() + user = get_or_create_remote_user(activity['actor']) + if not user.local: + book = activity['target']['id'].split('/')[-1] + status_builder.create_tag(user, book, activity['object']['name']) diff --git a/fedireads/outgoing.py b/fedireads/outgoing.py index e06870484..73e373db8 100644 --- a/fedireads/outgoing.py +++ b/fedireads/outgoing.py @@ -81,9 +81,7 @@ def handle_account_search(query): def handle_follow(user, to_follow): ''' someone local wants to follow someone ''' activity = activitypub.get_follow_request(user, to_follow) - errors = broadcast(user, activity, [to_follow.inbox]) - for error in errors: - raise(error['error']) + broadcast(user, activity, [to_follow.inbox]) def handle_unfollow(user, to_unfollow): @@ -93,10 +91,8 @@ def handle_unfollow(user, to_unfollow): user_object=to_unfollow ) activity = activitypub.get_unfollow(relationship) - errors = broadcast(user, activity, [to_unfollow.inbox]) + broadcast(user, activity, [to_unfollow.inbox]) to_unfollow.followers.remove(user) - for error in errors: - raise(error['error']) def handle_accept(user, to_follow, follow_request): @@ -187,7 +183,8 @@ def handle_import_books(user, items): create_activity = activitypub.get_create( user, activitypub.get_status(status)) - broadcast(user, create_activity, get_recipients(user, 'public')) + recipients = get_recipients(user, 'public') + broadcast(user, create_activity, recipients) def handle_review(user, book, name, content, rating): diff --git a/fedireads/settings.py b/fedireads/settings.py index 3c951496d..ff93ad149 100644 --- a/fedireads/settings.py +++ b/fedireads/settings.py @@ -5,6 +5,13 @@ from environs import Env env = Env() +# celery +CELERY_BROKER = env('CELERY_BROKER') +CELERY_RESULT_BACKEND = env('CELERY_RESULT_BACKEND') +CELERY_ACCEPT_CONTENT = ['application/json'] +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_SERIALIZER = 'json' + # Build paths inside the project like this: os.path.join(BASE_DIR, ...) BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -21,12 +28,6 @@ DOMAIN = env('DOMAIN') ALLOWED_HOSTS = env.list('ALLOWED_HOSTS', ['*']) OL_URL = env('OL_URL') -# celery/rebbitmq -CELERY_BROKER_URL = env('CELERY_BROKER') -CELERY_ACCEPT_CONTENT = ['json'] -CELERY_TASK_SERIALIZER = 'json' -CELERY_RESULT_BACKEND = 'amqp' - # Application definition INSTALLED_APPS = [ diff --git a/fedireads/tasks.py b/fedireads/tasks.py new file mode 100644 index 000000000..9492fa69e --- /dev/null +++ b/fedireads/tasks.py @@ -0,0 +1,14 @@ +''' background tasks ''' +from celery import Celery +import os + +from fedireads import settings + +# set the default Django settings module for the 'celery' program. +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fr_celery.settings') +app = Celery( + 'tasks', + broker=settings.CELERY_BROKER, +) + + diff --git a/fr_celery/__init__.py b/fr_celery/__init__.py new file mode 100644 index 000000000..3e6ab9e52 --- /dev/null +++ b/fr_celery/__init__.py @@ -0,0 +1,10 @@ +''' we need this file to initialize celery ''' +from __future__ import absolute_import, unicode_literals + +# This will make sure the app is always imported when +# Django starts so that shared_task will use this app. +from .celery import app as celery_app + +__all__ = ('celery_app',) + + diff --git a/fr_celery/asgi.py b/fr_celery/asgi.py new file mode 100644 index 000000000..f66a43b91 --- /dev/null +++ b/fr_celery/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for fr_celery project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fr_celery.settings') + +application = get_asgi_application() diff --git a/fedireads/celery.py b/fr_celery/celery.py similarity index 69% rename from fedireads/celery.py rename to fr_celery/celery.py index 7c26dc069..45c130d9c 100644 --- a/fedireads/celery.py +++ b/fr_celery/celery.py @@ -1,13 +1,14 @@ from __future__ import absolute_import, unicode_literals +from . import settings import os from celery import Celery # set the default Django settings module for the 'celery' program. -os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fedireads.settings') +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fr_celery.settings') -app = Celery('fedireads') +app = Celery('fr_celery') # Using a string here means the worker doesn't have to serialize # the configuration object to child processes. @@ -17,8 +18,6 @@ app.config_from_object('django.conf:settings', namespace='CELERY') # Load task modules from all registered Django app configs. app.autodiscover_tasks() +app.autodiscover_tasks(['fedireads'], related_name='incoming') +app.autodiscover_tasks(['fedireads'], related_name='broadcast') - -@app.task(bind=True) -def debug_task(self): - print('Request: {0!r}'.format(self.request)) diff --git a/fr_celery/settings.py b/fr_celery/settings.py new file mode 100644 index 000000000..48aa1643b --- /dev/null +++ b/fr_celery/settings.py @@ -0,0 +1,146 @@ +""" +Django settings for fr_celery project. + +Generated by 'django-admin startproject' using Django 3.0.3. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.0/ref/settings/ +""" + +import os +from environs import Env + +env = Env() + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +# celery/rebbitmq +CELERY_BROKER_URL = env('CELERY_BROKER') +CELERY_ACCEPT_CONTENT = ['json'] +CELERY_TASK_SERIALIZER = 'json' +CELERY_RESULT_BACKEND = 'redis' + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +SECRET_KEY = '0a^0gpwjc1ap+lb$dinin=efc@e&_0%102$o3(>9e7lndiaw' + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'fr_celery', + 'fedireads', + 'celery', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'fr_celery.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'fr_celery.wsgi.application' + + +# Database +# https://docs.djangoproject.com/en/3.0/ref/settings/#databases + +FEDIREADS_DATABASE_BACKEND = env('FEDIREADS_DATABASE_BACKEND', 'postgres') + +FEDIREADS_DBS = { + 'postgres': { + 'ENGINE': 'django.db.backends.postgresql_psycopg2', + 'NAME': env('POSTGRES_DB', 'fedireads'), + 'USER': env('POSTGRES_USER', 'fedireads'), + 'PASSWORD': env('POSTGRES_PASSWORD', 'fedireads'), + 'HOST': env('POSTGRES_HOST', ''), + 'PORT': 5432 + }, + 'sqlite': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.path.join(BASE_DIR, 'fedireads.db') + } +} + +DATABASES = { + 'default': FEDIREADS_DBS[FEDIREADS_DATABASE_BACKEND] +} + + +# Password validation +# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/3.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.0/howto/static-files/ + +STATIC_URL = '/static/' diff --git a/fr_celery/urls.py b/fr_celery/urls.py new file mode 100644 index 000000000..c8cc543b4 --- /dev/null +++ b/fr_celery/urls.py @@ -0,0 +1,21 @@ +"""fr_celery URL Configuration + +The `urlpatterns` list routes URLs to views. For more information please see: + https://docs.djangoproject.com/en/3.0/topics/http/urls/ +Examples: +Function views + 1. Add an import: from my_app import views + 2. Add a URL to urlpatterns: path('', views.home, name='home') +Class-based views + 1. Add an import: from other_app.views import Home + 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') +Including another URLconf + 1. Import the include() function: from django.urls import include, path + 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) +""" +from django.contrib import admin +from django.urls import path + +urlpatterns = [ + path('admin/', admin.site.urls), +] diff --git a/fr_celery/wsgi.py b/fr_celery/wsgi.py new file mode 100644 index 000000000..381265564 --- /dev/null +++ b/fr_celery/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for fr_celery project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'fr_celery.settings') + +application = get_wsgi_application() diff --git a/requirements.txt b/requirements.txt index b8836226b..13baa2257 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,8 +2,10 @@ celery==4.4.2 Django==3.0.3 django-model-utils==4.0.0 environs==7.2.0 +flower==0.9.4 Pillow==7.0.0 psycopg2==2.8.4 pycryptodome==3.9.4 python-dateutil==2.8.1 +redis==3.4.1 requests==2.22.0