diff --git a/.env.example b/.env.example index 3b541eb7c..fb0f7308d 100644 --- a/.env.example +++ b/.env.example @@ -8,7 +8,7 @@ USE_HTTPS=true DOMAIN=your.domain.here EMAIL=your@email.here -# Instance defualt language (see options at bookwyrm/settings.py "LANGUAGES" +# Instance default language (see options at bookwyrm/settings.py "LANGUAGES" LANGUAGE_CODE="en-us" # Used for deciding which editions to prefer DEFAULT_LANGUAGE="English" diff --git a/bookwyrm/activitypub/base_activity.py b/bookwyrm/activitypub/base_activity.py index ef7ecd0b2..e56d3b7ba 100644 --- a/bookwyrm/activitypub/base_activity.py +++ b/bookwyrm/activitypub/base_activity.py @@ -127,7 +127,7 @@ class ActivityObject: if ( allow_create and hasattr(model, "ignore_activity") - and model.ignore_activity(self) + and model.ignore_activity(self, allow_external_connections) ): return None diff --git a/bookwyrm/activitystreams.py b/bookwyrm/activitystreams.py index 74471883e..92076e6b8 100644 --- a/bookwyrm/activitystreams.py +++ b/bookwyrm/activitystreams.py @@ -38,11 +38,14 @@ class ActivityStream(RedisStore): def add_status(self, status, increment_unread=False): """add a status to users' feeds""" + audience = self.get_audience(status) # the pipeline contains all the add-to-stream activities - pipeline = self.add_object_to_related_stores(status, execute=False) + pipeline = self.add_object_to_stores( + status, self.get_stores_for_users(audience), execute=False + ) if increment_unread: - for user_id in self.get_audience(status): + for user_id in audience: # add to the unread status count pipeline.incr(self.unread_id(user_id)) # add to the unread status count for status type @@ -102,9 +105,16 @@ class ActivityStream(RedisStore): """go from zero to a timeline""" self.populate_store(self.stream_id(user.id)) + @tracer.start_as_current_span("ActivityStream._get_audience") def _get_audience(self, status): # pylint: disable=no-self-use - """given a status, what users should see it""" - # direct messages don't appeard in feeds, direct comments/reviews/etc do + """given a status, what users should see it, excluding the author""" + trace.get_current_span().set_attribute("status_type", status.status_type) + trace.get_current_span().set_attribute("status_privacy", status.privacy) + trace.get_current_span().set_attribute( + "status_reply_parent_privacy", + status.reply_parent.privacy if status.reply_parent else None, + ) + # direct messages don't appear in feeds, direct comments/reviews/etc do if status.privacy == "direct" and status.status_type == "Note": return [] @@ -119,15 +129,13 @@ class ActivityStream(RedisStore): # only visible to the poster and mentioned users if status.privacy == "direct": audience = audience.filter( - Q(id=status.user.id) # if the user is the post's author - | Q(id__in=status.mention_users.all()) # if the user is mentioned + Q(id__in=status.mention_users.all()) # if the user is mentioned ) # don't show replies to statuses the user can't see elif status.reply_parent and status.reply_parent.privacy == "followers": audience = audience.filter( - Q(id=status.user.id) # if the user is the post's author - | Q(id=status.reply_parent.user.id) # if the user is the OG author + Q(id=status.reply_parent.user.id) # if the user is the OG author | ( Q(following=status.user) & Q(following=status.reply_parent.user) ) # if the user is following both authors @@ -136,8 +144,7 @@ class ActivityStream(RedisStore): # only visible to the poster's followers and tagged users elif status.privacy == "followers": audience = audience.filter( - Q(id=status.user.id) # if the user is the post's author - | Q(following=status.user) # if the user is following the author + Q(following=status.user) # if the user is following the author ) return audience.distinct() @@ -145,10 +152,15 @@ class ActivityStream(RedisStore): def get_audience(self, status): """given a status, what users should see it""" trace.get_current_span().set_attribute("stream_id", self.key) - return [user.id for user in self._get_audience(status)] + audience = self._get_audience(status) + status_author = models.User.objects.filter( + is_active=True, local=True, id=status.user.id + ) + return list({user.id for user in list(audience) + list(status_author)}) - def get_stores_for_object(self, obj): - return [self.stream_id(user_id) for user_id in self.get_audience(obj)] + def get_stores_for_users(self, user_ids): + """convert a list of user ids into redis store ids""" + return [self.stream_id(user_id) for user_id in user_ids] def get_statuses_for_user(self, user): # pylint: disable=no-self-use """given a user, what statuses should they see on this stream""" @@ -173,11 +185,13 @@ class HomeStream(ActivityStream): audience = super()._get_audience(status) if not audience: return [] - # if the user is the post's author - ids_self = [user.id for user in audience.filter(Q(id=status.user.id))] # if the user is following the author - ids_following = [user.id for user in audience.filter(Q(following=status.user))] - return ids_self + ids_following + audience = audience.filter(following=status.user) + # if the user is the post's author + status_author = models.User.objects.filter( + is_active=True, local=True, id=status.user.id + ) + return list({user.id for user in list(audience) + list(status_author)}) def get_statuses_for_user(self, user): return models.Status.privacy_filter( @@ -197,11 +211,11 @@ class LocalStream(ActivityStream): key = "local" - def _get_audience(self, status): + def get_audience(self, status): # this stream wants no part in non-public statuses if status.privacy != "public" or not status.user.local: return [] - return super()._get_audience(status) + return super().get_audience(status) def get_statuses_for_user(self, user): # all public statuses by a local user @@ -218,13 +232,6 @@ class BooksStream(ActivityStream): def _get_audience(self, status): """anyone with the mentioned book on their shelves""" - # only show public statuses on the books feed, - # and only statuses that mention books - if status.privacy != "public" or not ( - status.mention_books.exists() or hasattr(status, "book") - ): - return [] - work = ( status.book.parent_work if hasattr(status, "book") @@ -236,6 +243,16 @@ class BooksStream(ActivityStream): return [] return audience.filter(shelfbook__book__parent_work=work).distinct() + def get_audience(self, status): + # only show public statuses on the books feed, + # and only statuses that mention books + if status.privacy != "public" or not ( + status.mention_books.exists() or hasattr(status, "book") + ): + return [] + + return super().get_audience(status) + def get_statuses_for_user(self, user): """any public status that mentions the user's books""" books = user.shelfbook_set.values_list( @@ -514,7 +531,9 @@ def remove_status_task(status_ids): for stream in streams.values(): for status in statuses: - stream.remove_object_from_related_stores(status) + stream.remove_object_from_stores( + status, stream.get_stores_for_users(stream.get_audience(status)) + ) @app.task(queue=HIGH, ignore_result=True) @@ -563,10 +582,10 @@ def handle_boost_task(boost_id): for stream in streams.values(): # people who should see the boost (not people who see the original status) - audience = stream.get_stores_for_object(instance) - stream.remove_object_from_related_stores(boosted, stores=audience) + audience = stream.get_stores_for_users(stream.get_audience(instance)) + stream.remove_object_from_stores(boosted, audience) for status in old_versions: - stream.remove_object_from_related_stores(status, stores=audience) + stream.remove_object_from_stores(status, audience) def get_status_type(status): diff --git a/bookwyrm/apps.py b/bookwyrm/apps.py index 7f684722d..b0c3e3fa4 100644 --- a/bookwyrm/apps.py +++ b/bookwyrm/apps.py @@ -40,6 +40,7 @@ class BookwyrmConfig(AppConfig): from bookwyrm.telemetry import open_telemetry open_telemetry.instrumentDjango() + open_telemetry.instrumentPostgres() if settings.ENABLE_PREVIEW_IMAGES and settings.FONTS: # Download any fonts that we don't have yet diff --git a/bookwyrm/connectors/abstract_connector.py b/bookwyrm/connectors/abstract_connector.py index 0e04ffaf2..bbe40f928 100644 --- a/bookwyrm/connectors/abstract_connector.py +++ b/bookwyrm/connectors/abstract_connector.py @@ -52,7 +52,7 @@ class AbstractMinimalConnector(ABC): return f"{self.search_url}{quote_plus(query)}" def process_search_response(self, query, data, min_confidence): - """Format the search results based on the formt of the query""" + """Format the search results based on the format of the query""" if maybe_isbn(query): return list(self.parse_isbn_search_data(data))[:10] return list(self.parse_search_data(data, min_confidence))[:10] @@ -321,7 +321,7 @@ def infer_physical_format(format_text): def unique_physical_format(format_text): - """only store the format if it isn't diretly in the format mappings""" + """only store the format if it isn't directly in the format mappings""" format_text = format_text.lower() if format_text in format_mappings: # try a direct match, so saving this would be redundant diff --git a/bookwyrm/connectors/connector_manager.py b/bookwyrm/connectors/connector_manager.py index 4330d4ac2..d7e2aad4b 100644 --- a/bookwyrm/connectors/connector_manager.py +++ b/bookwyrm/connectors/connector_manager.py @@ -73,7 +73,7 @@ async def async_connector_search(query, items, min_confidence): def search(query, min_confidence=0.1, return_first=False): - """find books based on arbitary keywords""" + """find books based on arbitrary keywords""" if not query: return [] results = [] diff --git a/bookwyrm/connectors/inventaire.py b/bookwyrm/connectors/inventaire.py index a330b2c4a..f3e24c0ec 100644 --- a/bookwyrm/connectors/inventaire.py +++ b/bookwyrm/connectors/inventaire.py @@ -97,7 +97,7 @@ class Connector(AbstractConnector): ) def parse_isbn_search_data(self, data): - """got some daaaata""" + """got some data""" results = data.get("entities") if not results: return diff --git a/bookwyrm/forms/admin.py b/bookwyrm/forms/admin.py index 1ad158119..72f50ccb8 100644 --- a/bookwyrm/forms/admin.py +++ b/bookwyrm/forms/admin.py @@ -15,7 +15,7 @@ from .custom_form import CustomForm, StyledForm # pylint: disable=missing-class-docstring class ExpiryWidget(widgets.Select): def value_from_datadict(self, data, files, name): - """human-readable exiration time buckets""" + """human-readable expiration time buckets""" selected_string = super().value_from_datadict(data, files, name) if selected_string == "day": diff --git a/bookwyrm/lists_stream.py b/bookwyrm/lists_stream.py index 7426488ce..2a92103e5 100644 --- a/bookwyrm/lists_stream.py +++ b/bookwyrm/lists_stream.py @@ -24,8 +24,7 @@ class ListsStream(RedisStore): def add_list(self, book_list): """add a list to users' feeds""" - # the pipeline contains all the add-to-stream activities - self.add_object_to_related_stores(book_list) + self.add_object_to_stores(book_list, self.get_stores_for_object(book_list)) def add_user_lists(self, viewer, user): """add a user's lists to another user's feed""" @@ -86,18 +85,19 @@ class ListsStream(RedisStore): if group: audience = audience.filter( Q(id=book_list.user.id) # if the user is the list's owner - | Q(following=book_list.user) # if the user is following the pwmer + | Q(following=book_list.user) # if the user is following the owner # if a user is in the group | Q(memberships__group__id=book_list.group.id) ) else: audience = audience.filter( Q(id=book_list.user.id) # if the user is the list's owner - | Q(following=book_list.user) # if the user is following the pwmer + | Q(following=book_list.user) # if the user is following the owner ) return audience.distinct() def get_stores_for_object(self, obj): + """the stores that an object belongs in""" return [self.stream_id(u) for u in self.get_audience(obj)] def get_lists_for_user(self, user): # pylint: disable=no-self-use @@ -233,7 +233,7 @@ def remove_list_task(list_id, re_add=False): # delete for every store stores = [ListsStream().stream_id(idx) for idx in stores] - ListsStream().remove_object_from_related_stores(list_id, stores=stores) + ListsStream().remove_object_from_stores(list_id, stores) if re_add: add_list_task.delay(list_id) diff --git a/bookwyrm/management/commands/deduplicate_book_data.py b/bookwyrm/management/commands/deduplicate_book_data.py index ed01a7843..5ca8496b0 100644 --- a/bookwyrm/management/commands/deduplicate_book_data.py +++ b/bookwyrm/management/commands/deduplicate_book_data.py @@ -68,12 +68,12 @@ def dedupe_model(model): class Command(BaseCommand): - """dedplucate allllll the book data models""" + """deduplicate allllll the book data models""" help = "merges duplicate book data" # pylint: disable=no-self-use,unused-argument def handle(self, *args, **options): - """run deudplications""" + """run deduplications""" dedupe_model(models.Edition) dedupe_model(models.Work) dedupe_model(models.Author) diff --git a/bookwyrm/management/commands/remove_editions.py b/bookwyrm/management/commands/remove_editions.py index 9eb9b7da8..5cb430a93 100644 --- a/bookwyrm/management/commands/remove_editions.py +++ b/bookwyrm/management/commands/remove_editions.py @@ -33,10 +33,10 @@ def remove_editions(): class Command(BaseCommand): - """dedplucate allllll the book data models""" + """deduplicate allllll the book data models""" help = "merges duplicate book data" # pylint: disable=no-self-use,unused-argument def handle(self, *args, **options): - """run deudplications""" + """run deduplications""" remove_editions() diff --git a/bookwyrm/management/commands/revoke_preview_image_tasks.py b/bookwyrm/management/commands/revoke_preview_image_tasks.py index 6d6e59e8f..7b0947b12 100644 --- a/bookwyrm/management/commands/revoke_preview_image_tasks.py +++ b/bookwyrm/management/commands/revoke_preview_image_tasks.py @@ -9,7 +9,7 @@ class Command(BaseCommand): # pylint: disable=unused-argument def handle(self, *args, **options): - """reveoke nonessential low priority tasks""" + """revoke nonessential low priority tasks""" types = [ "bookwyrm.preview_images.generate_edition_preview_image_task", "bookwyrm.preview_images.generate_user_preview_image_task", diff --git a/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py b/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py index c06fa40a0..f25bafe15 100644 --- a/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py +++ b/bookwyrm/migrations/0006_auto_20200221_1702_squashed_0064_merge_20201101_1913.py @@ -1467,7 +1467,7 @@ class Migration(migrations.Migration): ( "expiry", models.DateTimeField( - default=bookwyrm.models.site.get_passowrd_reset_expiry + default=bookwyrm.models.site.get_password_reset_expiry ), ), ( diff --git a/bookwyrm/migrations/0101_auto_20210929_1847.py b/bookwyrm/migrations/0101_auto_20210929_1847.py index 3fca28eac..967b59819 100644 --- a/bookwyrm/migrations/0101_auto_20210929_1847.py +++ b/bookwyrm/migrations/0101_auto_20210929_1847.py @@ -6,7 +6,7 @@ from bookwyrm.connectors.abstract_connector import infer_physical_format def infer_format(app_registry, schema_editor): - """set the new phsyical format field based on existing format data""" + """set the new physical format field based on existing format data""" db_alias = schema_editor.connection.alias editions = ( diff --git a/bookwyrm/migrations/0102_remove_connector_local.py b/bookwyrm/migrations/0102_remove_connector_local.py index 857f0f589..9bfd8b1d0 100644 --- a/bookwyrm/migrations/0102_remove_connector_local.py +++ b/bookwyrm/migrations/0102_remove_connector_local.py @@ -5,7 +5,7 @@ from bookwyrm.settings import DOMAIN def remove_self_connector(app_registry, schema_editor): - """set the new phsyical format field based on existing format data""" + """set the new physical format field based on existing format data""" db_alias = schema_editor.connection.alias app_registry.get_model("bookwyrm", "Connector").objects.using(db_alias).filter( connector_file="self_connector" diff --git a/bookwyrm/models/activitypub_mixin.py b/bookwyrm/models/activitypub_mixin.py index ee8b0d26a..479fa7203 100644 --- a/bookwyrm/models/activitypub_mixin.py +++ b/bookwyrm/models/activitypub_mixin.py @@ -25,7 +25,7 @@ from bookwyrm.tasks import app, MEDIUM, BROADCAST from bookwyrm.models.fields import ImageField, ManyToManyField logger = logging.getLogger(__name__) -# I tried to separate these classes into mutliple files but I kept getting +# I tried to separate these classes into multiple files but I kept getting # circular import errors so I gave up. I'm sure it could be done though! PropertyField = namedtuple("PropertyField", ("set_activity_from_field")) @@ -91,7 +91,7 @@ class ActivitypubMixin: @classmethod def find_existing(cls, data): - """compare data to fields that can be used for deduplation. + """compare data to fields that can be used for deduplication. This always includes remote_id, but can also be unique identifiers like an isbn for an edition""" filters = [] @@ -234,8 +234,8 @@ class ObjectMixin(ActivitypubMixin): activity = self.to_create_activity(user) self.broadcast(activity, user, software=software, queue=priority) except AttributeError: - # janky as heck, this catches the mutliple inheritence chain - # for boosts and ignores this auxilliary broadcast + # janky as heck, this catches the multiple inheritance chain + # for boosts and ignores this auxiliary broadcast return return @@ -311,7 +311,7 @@ class OrderedCollectionPageMixin(ObjectMixin): @property def collection_remote_id(self): - """this can be overriden if there's a special remote id, ie outbox""" + """this can be overridden if there's a special remote id, ie outbox""" return self.remote_id def to_ordered_collection( @@ -339,7 +339,7 @@ class OrderedCollectionPageMixin(ObjectMixin): activity["id"] = remote_id paginated = Paginator(queryset, PAGE_LENGTH) - # add computed fields specific to orderd collections + # add computed fields specific to ordered collections activity["totalItems"] = paginated.count activity["first"] = f"{remote_id}?page=1" activity["last"] = f"{remote_id}?page={paginated.num_pages}" @@ -405,7 +405,7 @@ class CollectionItemMixin(ActivitypubMixin): # first off, we want to save normally no matter what super().save(*args, **kwargs) - # list items can be updateda, normally you would only broadcast on created + # list items can be updated, normally you would only broadcast on created if not broadcast or not self.user.local: return @@ -581,7 +581,7 @@ async def sign_and_send( def to_ordered_collection_page( queryset, remote_id, id_only=False, page=1, pure=False, **kwargs ): - """serialize and pagiante a queryset""" + """serialize and paginate a queryset""" paginated = Paginator(queryset, PAGE_LENGTH) activity_page = paginated.get_page(page) diff --git a/bookwyrm/models/annual_goal.py b/bookwyrm/models/annual_goal.py index 0eefacb32..d36b822df 100644 --- a/bookwyrm/models/annual_goal.py +++ b/bookwyrm/models/annual_goal.py @@ -24,7 +24,7 @@ class AnnualGoal(BookWyrmModel): ) class Meta: - """unqiueness constraint""" + """uniqueness constraint""" unique_together = ("user", "year") diff --git a/bookwyrm/models/book.py b/bookwyrm/models/book.py index a5be51a29..4e7ffcad3 100644 --- a/bookwyrm/models/book.py +++ b/bookwyrm/models/book.py @@ -321,7 +321,7 @@ class Edition(Book): def get_rank(self): """calculate how complete the data is on this edition""" rank = 0 - # big ups for havinga cover + # big ups for having a cover rank += int(bool(self.cover)) * 3 # is it in the instance's preferred language? rank += int(bool(DEFAULT_LANGUAGE in self.languages)) diff --git a/bookwyrm/models/favorite.py b/bookwyrm/models/favorite.py index 4c3675219..98fbce550 100644 --- a/bookwyrm/models/favorite.py +++ b/bookwyrm/models/favorite.py @@ -20,8 +20,9 @@ class Favorite(ActivityMixin, BookWyrmModel): activity_serializer = activitypub.Like + # pylint: disable=unused-argument @classmethod - def ignore_activity(cls, activity): + def ignore_activity(cls, activity, allow_external_connections=True): """don't bother with incoming favs of unknown statuses""" return not Status.objects.filter(remote_id=activity.object).exists() diff --git a/bookwyrm/models/fields.py b/bookwyrm/models/fields.py index 62dc8f0d9..3fe035f58 100644 --- a/bookwyrm/models/fields.py +++ b/bookwyrm/models/fields.py @@ -71,11 +71,11 @@ class ActivitypubFieldMixin: def set_field_from_activity( self, instance, data, overwrite=True, allow_external_connections=True ): - """helper function for assinging a value to the field. Returns if changed""" + """helper function for assigning a value to the field. Returns if changed""" try: value = getattr(data, self.get_activitypub_field()) except AttributeError: - # masssively hack-y workaround for boosts + # massively hack-y workaround for boosts if self.get_activitypub_field() != "attributedTo": raise value = getattr(data, "actor") @@ -221,7 +221,7 @@ PrivacyLevels = [ class PrivacyField(ActivitypubFieldMixin, models.CharField): - """this maps to two differente activitypub fields""" + """this maps to two different activitypub fields""" public = "https://www.w3.org/ns/activitystreams#Public" @@ -436,7 +436,7 @@ class ImageField(ActivitypubFieldMixin, models.ImageField): def set_field_from_activity( self, instance, data, save=True, overwrite=True, allow_external_connections=True ): - """helper function for assinging a value to the field""" + """helper function for assigning a value to the field""" value = getattr(data, self.get_activitypub_field()) formatted = self.field_from_activity( value, allow_external_connections=allow_external_connections diff --git a/bookwyrm/models/link.py b/bookwyrm/models/link.py index 56b096bc2..d334a9d29 100644 --- a/bookwyrm/models/link.py +++ b/bookwyrm/models/link.py @@ -31,7 +31,7 @@ class Link(ActivitypubMixin, BookWyrmModel): @property def name(self): - """link name via the assocaited domain""" + """link name via the associated domain""" return self.domain.name def save(self, *args, **kwargs): diff --git a/bookwyrm/models/notification.py b/bookwyrm/models/notification.py index 29f7b0c2d..522038f9a 100644 --- a/bookwyrm/models/notification.py +++ b/bookwyrm/models/notification.py @@ -284,7 +284,7 @@ def notify_user_on_list_item_add(sender, instance, created, *args, **kwargs): return list_owner = instance.book_list.user - # create a notification if somoene ELSE added to a local user's list + # create a notification if someone ELSE added to a local user's list if list_owner.local and list_owner != instance.user: # keep the related_user singular, group the items Notification.notify_list_item(list_owner, instance) diff --git a/bookwyrm/models/readthrough.py b/bookwyrm/models/readthrough.py index 239ec56be..4911c715b 100644 --- a/bookwyrm/models/readthrough.py +++ b/bookwyrm/models/readthrough.py @@ -8,7 +8,7 @@ from .base_model import BookWyrmModel class ProgressMode(models.TextChoices): - """types of prgress available""" + """types of progress available""" PAGE = "PG", "page" PERCENT = "PCT", "percent" diff --git a/bookwyrm/models/relationship.py b/bookwyrm/models/relationship.py index 422967855..4754bea36 100644 --- a/bookwyrm/models/relationship.py +++ b/bookwyrm/models/relationship.py @@ -34,7 +34,7 @@ class UserRelationship(BookWyrmModel): @property def recipients(self): - """the remote user needs to recieve direct broadcasts""" + """the remote user needs to receive direct broadcasts""" return [u for u in [self.user_subject, self.user_object] if not u.local] def save(self, *args, **kwargs): diff --git a/bookwyrm/models/shelf.py b/bookwyrm/models/shelf.py index 8e754bc47..c52cb6ab8 100644 --- a/bookwyrm/models/shelf.py +++ b/bookwyrm/models/shelf.py @@ -80,7 +80,7 @@ class Shelf(OrderedCollectionMixin, BookWyrmModel): raise PermissionDenied() class Meta: - """user/shelf unqiueness""" + """user/shelf uniqueness""" unique_together = ("user", "identifier") diff --git a/bookwyrm/models/site.py b/bookwyrm/models/site.py index 35f007be2..a27c4b70d 100644 --- a/bookwyrm/models/site.py +++ b/bookwyrm/models/site.py @@ -209,7 +209,7 @@ class InviteRequest(BookWyrmModel): super().save(*args, **kwargs) -def get_passowrd_reset_expiry(): +def get_password_reset_expiry(): """give people a limited time to use the link""" now = timezone.now() return now + datetime.timedelta(days=1) @@ -219,7 +219,7 @@ class PasswordReset(models.Model): """gives someone access to create an account on the instance""" code = models.CharField(max_length=32, default=new_access_code) - expiry = models.DateTimeField(default=get_passowrd_reset_expiry) + expiry = models.DateTimeField(default=get_password_reset_expiry) user = models.OneToOneField(User, on_delete=models.CASCADE) def valid(self): diff --git a/bookwyrm/models/status.py b/bookwyrm/models/status.py index 2f1999bf2..e51f2ba07 100644 --- a/bookwyrm/models/status.py +++ b/bookwyrm/models/status.py @@ -116,10 +116,16 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel): return list(set(mentions)) @classmethod - def ignore_activity(cls, activity): # pylint: disable=too-many-return-statements + def ignore_activity( + cls, activity, allow_external_connections=True + ): # pylint: disable=too-many-return-statements """keep notes if they are replies to existing statuses""" if activity.type == "Announce": - boosted = activitypub.resolve_remote_id(activity.object, get_activity=True) + boosted = activitypub.resolve_remote_id( + activity.object, + get_activity=True, + allow_external_connections=allow_external_connections, + ) if not boosted: # if we can't load the status, definitely ignore it return True diff --git a/bookwyrm/redis_store.py b/bookwyrm/redis_store.py index f25829f5c..e188487aa 100644 --- a/bookwyrm/redis_store.py +++ b/bookwyrm/redis_store.py @@ -16,12 +16,12 @@ class RedisStore(ABC): """the object and rank""" return {obj.id: self.get_rank(obj)} - def add_object_to_related_stores(self, obj, execute=True): - """add an object to all suitable stores""" + def add_object_to_stores(self, obj, stores, execute=True): + """add an object to a given set of stores""" value = self.get_value(obj) # we want to do this as a bulk operation, hence "pipeline" pipeline = r.pipeline() - for store in self.get_stores_for_object(obj): + for store in stores: # add the status to the feed pipeline.zadd(store, value) # trim the store @@ -32,14 +32,14 @@ class RedisStore(ABC): # and go! return pipeline.execute() - def remove_object_from_related_stores(self, obj, stores=None): + # pylint: disable=no-self-use + def remove_object_from_stores(self, obj, stores): """remove an object from all stores""" - # if the stoers are provided, the object can just be an id + # if the stores are provided, the object can just be an id if stores and isinstance(obj, int): obj_id = obj else: obj_id = obj.id - stores = self.get_stores_for_object(obj) if stores is None else stores pipeline = r.pipeline() for store in stores: pipeline.zrem(store, -1, obj_id) @@ -82,10 +82,6 @@ class RedisStore(ABC): def get_objects_for_store(self, store): """a queryset of what should go in a store, used for populating it""" - @abstractmethod - def get_stores_for_object(self, obj): - """the stores that an object belongs in""" - @abstractmethod def get_rank(self, obj): """how to rank an object""" diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 419b76195..8dcf90fcb 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -226,7 +226,7 @@ STREAMS = [ # total time in seconds that the instance will spend searching connectors SEARCH_TIMEOUT = env.int("SEARCH_TIMEOUT", 8) # timeout for a query to an individual connector -QUERY_TIMEOUT = env.int("QUERY_TIMEOUT", 5) +QUERY_TIMEOUT = env.int("INTERACTIVE_QUERY_TIMEOUT", env.int("QUERY_TIMEOUT", 5)) # Redis cache backend if env.bool("USE_DUMMY_CACHE", False): diff --git a/bookwyrm/static/css/bookwyrm/components/_book_cover.scss b/bookwyrm/static/css/bookwyrm/components/_book_cover.scss index d1125197e..48b564a0b 100644 --- a/bookwyrm/static/css/bookwyrm/components/_book_cover.scss +++ b/bookwyrm/static/css/bookwyrm/components/_book_cover.scss @@ -5,7 +5,7 @@ * - .book-cover is positioned and sized based on its container. * * To have the cover within specific dimensions, specify a width or height for - * standard bulma’s named breapoints: + * standard bulma’s named breakpoints: * * `is-(w|h)-(auto|xs|s|m|l|xl|xxl)[-(mobile|tablet|desktop)]` * @@ -43,7 +43,7 @@ max-height: 100%; /* Useful when stretching under-sized images. */ - image-rendering: optimizequality; + image-rendering: optimizeQuality; image-rendering: smooth; } diff --git a/bookwyrm/static/js/bookwyrm.js b/bookwyrm/static/js/bookwyrm.js index 6a6c0217f..ceed12eba 100644 --- a/bookwyrm/static/js/bookwyrm.js +++ b/bookwyrm/static/js/bookwyrm.js @@ -5,7 +5,7 @@ let BookWyrm = new (class { constructor() { this.MAX_FILE_SIZE_BYTES = 10 * 1000000; this.initOnDOMLoaded(); - this.initReccuringTasks(); + this.initRecurringTasks(); this.initEventListeners(); } @@ -77,7 +77,7 @@ let BookWyrm = new (class { /** * Execute recurring tasks. */ - initReccuringTasks() { + initRecurringTasks() { // Polling document.querySelectorAll("[data-poll]").forEach((liveArea) => this.polling(liveArea)); } diff --git a/bookwyrm/static/js/forms.js b/bookwyrm/static/js/forms.js index a48675b35..08066f137 100644 --- a/bookwyrm/static/js/forms.js +++ b/bookwyrm/static/js/forms.js @@ -2,7 +2,7 @@ "use strict"; /** - * Remoev input field + * Remove input field * * @param {event} the button click event */ diff --git a/bookwyrm/suggested_users.py b/bookwyrm/suggested_users.py index ea6b1c55d..0ae2a1b15 100644 --- a/bookwyrm/suggested_users.py +++ b/bookwyrm/suggested_users.py @@ -4,13 +4,16 @@ import logging from django.dispatch import receiver from django.db import transaction from django.db.models import signals, Count, Q, Case, When, IntegerField +from opentelemetry import trace from bookwyrm import models from bookwyrm.redis_store import RedisStore, r from bookwyrm.tasks import app, LOW, MEDIUM +from bookwyrm.telemetry import open_telemetry logger = logging.getLogger(__name__) +tracer = open_telemetry.tracer() class SuggestedUsers(RedisStore): @@ -49,30 +52,34 @@ class SuggestedUsers(RedisStore): ) def get_stores_for_object(self, obj): + """the stores that an object belongs in""" return [self.store_id(u) for u in self.get_users_for_object(obj)] def get_users_for_object(self, obj): # pylint: disable=no-self-use """given a user, who might want to follow them""" - return models.User.objects.filter(local=True,).exclude( + return models.User.objects.filter(local=True, is_active=True).exclude( Q(id=obj.id) | Q(followers=obj) | Q(id__in=obj.blocks.all()) | Q(blocks=obj) ) + @tracer.start_as_current_span("SuggestedUsers.rerank_obj") def rerank_obj(self, obj, update_only=True): """update all the instances of this user with new ranks""" + trace.get_current_span().set_attribute("update_only", update_only) pipeline = r.pipeline() for store_user in self.get_users_for_object(obj): - annotated_user = get_annotated_users( - store_user, - id=obj.id, - ).first() - if not annotated_user: - continue + with tracer.start_as_current_span("SuggestedUsers.rerank_obj/user") as _: + annotated_user = get_annotated_users( + store_user, + id=obj.id, + ).first() + if not annotated_user: + continue - pipeline.zadd( - self.store_id(store_user), - self.get_value(annotated_user), - xx=update_only, - ) + pipeline.zadd( + self.store_id(store_user), + self.get_value(annotated_user), + xx=update_only, + ) pipeline.execute() def rerank_user_suggestions(self, user): @@ -254,7 +261,9 @@ def rerank_user_task(user_id, update_only=False): def remove_user_task(user_id): """do the hard work in celery""" user = models.User.objects.get(id=user_id) - suggested_users.remove_object_from_related_stores(user) + suggested_users.remove_object_from_stores( + user, suggested_users.get_stores_for_object(user) + ) @app.task(queue=MEDIUM, ignore_result=True) @@ -268,7 +277,9 @@ def remove_suggestion_task(user_id, suggested_user_id): def bulk_remove_instance_task(instance_id): """remove a bunch of users from recs""" for user in models.User.objects.filter(federated_server__id=instance_id): - suggested_users.remove_object_from_related_stores(user) + suggested_users.remove_object_from_stores( + user, suggested_users.get_stores_for_object(user) + ) @app.task(queue=LOW, ignore_result=True) diff --git a/bookwyrm/telemetry/open_telemetry.py b/bookwyrm/telemetry/open_telemetry.py index 2798582d0..00b24d4b0 100644 --- a/bookwyrm/telemetry/open_telemetry.py +++ b/bookwyrm/telemetry/open_telemetry.py @@ -22,6 +22,12 @@ def instrumentDjango(): DjangoInstrumentor().instrument() +def instrumentPostgres(): + from opentelemetry.instrumentation.psycopg2 import Psycopg2Instrumentor + + Psycopg2Instrumentor().instrument() + + def instrumentCelery(): from opentelemetry.instrumentation.celery import CeleryInstrumentor from celery.signals import worker_process_init diff --git a/bookwyrm/templates/book/book.html b/bookwyrm/templates/book/book.html index e9eff99ab..1024255e6 100644 --- a/bookwyrm/templates/book/book.html +++ b/bookwyrm/templates/book/book.html @@ -46,7 +46,13 @@ - ({{ book.series }}{% if book.series_number %} #{{ book.series_number }}{% endif %}) + {% if book.authors.exists %} + + {% endif %} + {{ book.series }}{% if book.series_number %} #{{ book.series_number }}{% endif %} + {% if book.authors.exists %} + + {% endif %} {% endif %}

{% endif %} diff --git a/bookwyrm/templates/lists/created_text.html b/bookwyrm/templates/lists/created_text.html index f5405b64a..b9e188686 100644 --- a/bookwyrm/templates/lists/created_text.html +++ b/bookwyrm/templates/lists/created_text.html @@ -3,7 +3,7 @@ {% if list.curation == 'group' %} {% blocktrans with username=list.user.display_name userpath=list.user.local_path groupname=list.group.name grouppath=list.group.local_path %}Created by {{ username }} and managed by {{ groupname }}{% endblocktrans %} -{% elif list.curation != 'open' %} +{% elif list.curation == 'curated' %} {% blocktrans with username=list.user.display_name path=list.user.local_path %}Created and curated by {{ username }}{% endblocktrans %} {% else %} {% blocktrans with username=list.user.display_name path=list.user.local_path %}Created by {{ username }}{% endblocktrans %} diff --git a/bookwyrm/templates/preferences/edit_user.html b/bookwyrm/templates/preferences/edit_user.html index 493b18d2f..f2b14babf 100644 --- a/bookwyrm/templates/preferences/edit_user.html +++ b/bookwyrm/templates/preferences/edit_user.html @@ -133,7 +133,7 @@ {% url 'user-shelves' request.user.localname as path %}

- {% blocktrans %}Looking for shelf privacy? You can set a sepearate visibility level for each of your shelves. Go to Your Books, pick a shelf from the tab bar, and click "Edit shelf."{% endblocktrans %} + {% blocktrans %}Looking for shelf privacy? You can set a separate visibility level for each of your shelves. Go to Your Books, pick a shelf from the tab bar, and click "Edit shelf."{% endblocktrans %}

diff --git a/bookwyrm/templates/settings/celery.html b/bookwyrm/templates/settings/celery.html index 65315da01..5f79dfd9d 100644 --- a/bookwyrm/templates/settings/celery.html +++ b/bookwyrm/templates/settings/celery.html @@ -116,6 +116,35 @@ {% endif %} +
+

{% trans "Clear Queues" %}

+ +
+ + {% trans "Clearing queues can cause serious problems including data loss! Only play with this if you really know what you're doing. You must shut down the Celery worker before you do this." %} +
+ +
+ {% csrf_token %} + +
+
+

{{ form.queues.label_tag }}

+ {{ form.queues }} +
+ +
+

{{ form.tasks.label_tag }}

+ {{ form.tasks }} +
+
+ +
+ +
+
+
+ {% if errors %}

{% trans "Errors" %}

diff --git a/bookwyrm/templates/settings/dashboard/dashboard.html b/bookwyrm/templates/settings/dashboard/dashboard.html index 99c0e9621..4c109c7e1 100644 --- a/bookwyrm/templates/settings/dashboard/dashboard.html +++ b/bookwyrm/templates/settings/dashboard/dashboard.html @@ -16,7 +16,7 @@

{{ users|intcomma }}

-
+

{% trans "Active this month" %}

{{ active_users|intcomma }}

diff --git a/bookwyrm/templatetags/book_display_tags.py b/bookwyrm/templatetags/book_display_tags.py index 56eb096ec..0a0f228d8 100644 --- a/bookwyrm/templatetags/book_display_tags.py +++ b/bookwyrm/templatetags/book_display_tags.py @@ -18,7 +18,7 @@ def get_book_description(book): if book.description: return book.description if book.parent_work: - # this shoud always be true + # this should always be true return book.parent_work.description return None diff --git a/bookwyrm/templatetags/shelf_tags.py b/bookwyrm/templatetags/shelf_tags.py index 1fb799883..5e6850363 100644 --- a/bookwyrm/templatetags/shelf_tags.py +++ b/bookwyrm/templatetags/shelf_tags.py @@ -37,7 +37,7 @@ def get_next_shelf(current_shelf): @register.filter(name="translate_shelf_name") def get_translated_shelf_name(shelf): - """produced translated shelf nidentifierame""" + """produce translated shelf identifiername""" if not shelf: return "" # support obj or dict diff --git a/bookwyrm/templatetags/utilities.py b/bookwyrm/templatetags/utilities.py index 834d39a14..4aaf6b8a7 100644 --- a/bookwyrm/templatetags/utilities.py +++ b/bookwyrm/templatetags/utilities.py @@ -19,7 +19,7 @@ def get_uuid(identifier): @register.simple_tag(takes_context=False) def join(*args): - """concatenate an arbitary set of values""" + """concatenate an arbitrary set of values""" return "_".join(str(a) for a in args) diff --git a/bookwyrm/tests/activitypub/test_author.py b/bookwyrm/tests/activitypub/test_author.py index 61d525fc0..51beac49a 100644 --- a/bookwyrm/tests/activitypub/test_author.py +++ b/bookwyrm/tests/activitypub/test_author.py @@ -19,7 +19,7 @@ class Author(TestCase): ) def test_serialize_model(self): - """check presense of author fields""" + """check presence of author fields""" activity = self.author.to_activity() self.assertEqual(activity["id"], self.author.remote_id) self.assertIsInstance(activity["aliases"], list) diff --git a/bookwyrm/tests/activitypub/test_base_activity.py b/bookwyrm/tests/activitypub/test_base_activity.py index df243d0db..c9022d35c 100644 --- a/bookwyrm/tests/activitypub/test_base_activity.py +++ b/bookwyrm/tests/activitypub/test_base_activity.py @@ -59,7 +59,7 @@ class BaseActivity(TestCase): self.assertIsInstance(representative, models.User) def test_init(self, *_): - """simple successfuly init""" + """simple successfully init""" instance = ActivityObject(id="a", type="b") self.assertTrue(hasattr(instance, "id")) self.assertTrue(hasattr(instance, "type")) diff --git a/bookwyrm/tests/activitypub/test_quotation.py b/bookwyrm/tests/activitypub/test_quotation.py index c90348bc3..678ee7aa3 100644 --- a/bookwyrm/tests/activitypub/test_quotation.py +++ b/bookwyrm/tests/activitypub/test_quotation.py @@ -1,4 +1,4 @@ -""" quotation activty object serializer class """ +""" quotation activity object serializer class """ import json import pathlib from unittest.mock import patch @@ -30,7 +30,7 @@ class Quotation(TestCase): self.status_data = json.loads(datafile.read_bytes()) def test_quotation_activity(self): - """create a Quoteation ap object from json""" + """create a Quotation ap object from json""" quotation = activitypub.Quotation(**self.status_data) self.assertEqual(quotation.type, "Quotation") diff --git a/bookwyrm/tests/activitystreams/test_tasks.py b/bookwyrm/tests/activitystreams/test_tasks.py index 2e89f6ccc..82b8c2e5a 100644 --- a/bookwyrm/tests/activitystreams/test_tasks.py +++ b/bookwyrm/tests/activitystreams/test_tasks.py @@ -50,7 +50,7 @@ class Activitystreams(TestCase): self.assertEqual(args[1], self.book) def test_remove_book_statuses_task(self): - """remove stauses related to a book""" + """remove statuses related to a book""" with patch("bookwyrm.activitystreams.BooksStream.remove_book_statuses") as mock: activitystreams.remove_book_statuses_task(self.local_user.id, self.book.id) self.assertTrue(mock.called) @@ -75,7 +75,7 @@ class Activitystreams(TestCase): def test_remove_status_task(self): """remove a status from all streams""" with patch( - "bookwyrm.activitystreams.ActivityStream.remove_object_from_related_stores" + "bookwyrm.activitystreams.ActivityStream.remove_object_from_stores" ) as mock: activitystreams.remove_status_task(self.status.id) self.assertEqual(mock.call_count, 3) @@ -132,8 +132,8 @@ class Activitystreams(TestCase): self.assertEqual(args[0], self.local_user) self.assertEqual(args[1], self.another_user) - @patch("bookwyrm.activitystreams.LocalStream.remove_object_from_related_stores") - @patch("bookwyrm.activitystreams.BooksStream.remove_object_from_related_stores") + @patch("bookwyrm.activitystreams.LocalStream.remove_object_from_stores") + @patch("bookwyrm.activitystreams.BooksStream.remove_object_from_stores") @patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") def test_boost_to_another_timeline(self, *_): """boost from a non-follower doesn't remove original status from feed""" @@ -144,7 +144,7 @@ class Activitystreams(TestCase): user=self.another_user, ) with patch( - "bookwyrm.activitystreams.HomeStream.remove_object_from_related_stores" + "bookwyrm.activitystreams.HomeStream.remove_object_from_stores" ) as mock: activitystreams.handle_boost_task(boost.id) @@ -152,10 +152,10 @@ class Activitystreams(TestCase): self.assertEqual(mock.call_count, 1) call_args = mock.call_args self.assertEqual(call_args[0][0], status) - self.assertEqual(call_args[1]["stores"], [f"{self.another_user.id}-home"]) + self.assertEqual(call_args[0][1], [f"{self.another_user.id}-home"]) - @patch("bookwyrm.activitystreams.LocalStream.remove_object_from_related_stores") - @patch("bookwyrm.activitystreams.BooksStream.remove_object_from_related_stores") + @patch("bookwyrm.activitystreams.LocalStream.remove_object_from_stores") + @patch("bookwyrm.activitystreams.BooksStream.remove_object_from_stores") @patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") def test_boost_to_another_timeline_remote(self, *_): """boost from a remote non-follower doesn't remove original status from feed""" @@ -166,7 +166,7 @@ class Activitystreams(TestCase): user=self.remote_user, ) with patch( - "bookwyrm.activitystreams.HomeStream.remove_object_from_related_stores" + "bookwyrm.activitystreams.HomeStream.remove_object_from_stores" ) as mock: activitystreams.handle_boost_task(boost.id) @@ -174,10 +174,10 @@ class Activitystreams(TestCase): self.assertEqual(mock.call_count, 1) call_args = mock.call_args self.assertEqual(call_args[0][0], status) - self.assertEqual(call_args[1]["stores"], []) + self.assertEqual(call_args[0][1], []) - @patch("bookwyrm.activitystreams.LocalStream.remove_object_from_related_stores") - @patch("bookwyrm.activitystreams.BooksStream.remove_object_from_related_stores") + @patch("bookwyrm.activitystreams.LocalStream.remove_object_from_stores") + @patch("bookwyrm.activitystreams.BooksStream.remove_object_from_stores") @patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") def test_boost_to_following_timeline(self, *_): """add a boost and deduplicate the boosted status on the timeline""" @@ -189,17 +189,17 @@ class Activitystreams(TestCase): user=self.another_user, ) with patch( - "bookwyrm.activitystreams.HomeStream.remove_object_from_related_stores" + "bookwyrm.activitystreams.HomeStream.remove_object_from_stores" ) as mock: activitystreams.handle_boost_task(boost.id) self.assertTrue(mock.called) call_args = mock.call_args self.assertEqual(call_args[0][0], status) - self.assertTrue(f"{self.another_user.id}-home" in call_args[1]["stores"]) - self.assertTrue(f"{self.local_user.id}-home" in call_args[1]["stores"]) + self.assertTrue(f"{self.another_user.id}-home" in call_args[0][1]) + self.assertTrue(f"{self.local_user.id}-home" in call_args[0][1]) - @patch("bookwyrm.activitystreams.LocalStream.remove_object_from_related_stores") - @patch("bookwyrm.activitystreams.BooksStream.remove_object_from_related_stores") + @patch("bookwyrm.activitystreams.LocalStream.remove_object_from_stores") + @patch("bookwyrm.activitystreams.BooksStream.remove_object_from_stores") @patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") def test_boost_to_same_timeline(self, *_): """add a boost and deduplicate the boosted status on the timeline""" @@ -210,10 +210,10 @@ class Activitystreams(TestCase): user=self.local_user, ) with patch( - "bookwyrm.activitystreams.HomeStream.remove_object_from_related_stores" + "bookwyrm.activitystreams.HomeStream.remove_object_from_stores" ) as mock: activitystreams.handle_boost_task(boost.id) self.assertTrue(mock.called) call_args = mock.call_args self.assertEqual(call_args[0][0], status) - self.assertEqual(call_args[1]["stores"], [f"{self.local_user.id}-home"]) + self.assertEqual(call_args[0][1], [f"{self.local_user.id}-home"]) diff --git a/bookwyrm/tests/connectors/test_openlibrary_connector.py b/bookwyrm/tests/connectors/test_openlibrary_connector.py index 05ba39ab9..01b9b9f6a 100644 --- a/bookwyrm/tests/connectors/test_openlibrary_connector.py +++ b/bookwyrm/tests/connectors/test_openlibrary_connector.py @@ -46,7 +46,7 @@ class Openlibrary(TestCase): data = {"key": "/work/OL1234W"} result = self.connector.get_remote_id_from_data(data) self.assertEqual(result, "https://openlibrary.org/work/OL1234W") - # error handlding + # error handling with self.assertRaises(ConnectorException): self.connector.get_remote_id_from_data({}) diff --git a/bookwyrm/tests/lists_stream/test_tasks.py b/bookwyrm/tests/lists_stream/test_tasks.py index 55c5d98c8..2e01cecad 100644 --- a/bookwyrm/tests/lists_stream/test_tasks.py +++ b/bookwyrm/tests/lists_stream/test_tasks.py @@ -59,7 +59,7 @@ class Activitystreams(TestCase): def test_remove_list_task(self, *_): """remove a list from all streams""" with patch( - "bookwyrm.lists_stream.ListsStream.remove_object_from_related_stores" + "bookwyrm.lists_stream.ListsStream.remove_object_from_stores" ) as mock: lists_stream.remove_list_task(self.list.id) self.assertEqual(mock.call_count, 1) diff --git a/bookwyrm/tests/models/test_activitypub_mixin.py b/bookwyrm/tests/models/test_activitypub_mixin.py index fdd1883a8..a465c2c12 100644 --- a/bookwyrm/tests/models/test_activitypub_mixin.py +++ b/bookwyrm/tests/models/test_activitypub_mixin.py @@ -245,7 +245,7 @@ class ActivitypubMixins(TestCase): # ObjectMixin def test_object_save_create(self, *_): - """should save uneventufully when broadcast is disabled""" + """should save uneventfully when broadcast is disabled""" class Success(Exception): """this means we got to the right method""" @@ -276,7 +276,7 @@ class ActivitypubMixins(TestCase): ObjectModel(user=None).save() def test_object_save_update(self, *_): - """should save uneventufully when broadcast is disabled""" + """should save uneventfully when broadcast is disabled""" class Success(Exception): """this means we got to the right method""" diff --git a/bookwyrm/tests/models/test_base_model.py b/bookwyrm/tests/models/test_base_model.py index 8a8be2148..b94592571 100644 --- a/bookwyrm/tests/models/test_base_model.py +++ b/bookwyrm/tests/models/test_base_model.py @@ -51,7 +51,7 @@ class BaseModel(TestCase): def test_set_remote_id(self): """this function sets remote ids after creation""" - # using Work because it BookWrymModel is abstract and this requires save + # using Work because it BookWyrmModel is abstract and this requires save # Work is a relatively not-fancy model. instance = models.Work.objects.create(title="work title") instance.remote_id = None diff --git a/bookwyrm/tests/models/test_fields.py b/bookwyrm/tests/models/test_fields.py index fc8d7387d..553a533d5 100644 --- a/bookwyrm/tests/models/test_fields.py +++ b/bookwyrm/tests/models/test_fields.py @@ -29,7 +29,7 @@ from bookwyrm.settings import DOMAIN @patch("bookwyrm.activitystreams.populate_stream_task.delay") @patch("bookwyrm.lists_stream.populate_lists_task.delay") class ModelFields(TestCase): - """overwrites standard model feilds to work with activitypub""" + """overwrites standard model fields to work with activitypub""" def test_validate_remote_id(self, *_): """should look like a url""" @@ -125,7 +125,7 @@ class ModelFields(TestCase): instance.run_validators("@example.com") instance.run_validators("mouse@examplecom") instance.run_validators("one two@fish.aaaa") - instance.run_validators("a*&@exampke.com") + instance.run_validators("a*&@example.com") instance.run_validators("trailingwhite@example.com ") self.assertIsNone(instance.run_validators("mouse@example.com")) self.assertIsNone(instance.run_validators("mo-2use@ex3ample.com")) @@ -292,7 +292,7 @@ class ModelFields(TestCase): self.assertEqual(value.name, "MOUSE?? MOUSE!!") def test_foreign_key_from_activity_dict(self, *_): - """test recieving activity json""" + """test receiving activity json""" instance = fields.ForeignKey(User, on_delete=models.CASCADE) datafile = pathlib.Path(__file__).parent.joinpath("../data/ap_user.json") userdata = json.loads(datafile.read_bytes()) diff --git a/bookwyrm/tests/models/test_status_model.py b/bookwyrm/tests/models/test_status_model.py index 177bedb24..1bbca1896 100644 --- a/bookwyrm/tests/models/test_status_model.py +++ b/bookwyrm/tests/models/test_status_model.py @@ -397,7 +397,7 @@ class Status(TestCase): # pylint: disable=unused-argument def test_create_broadcast(self, one, two, broadcast_mock, *_): - """should send out two verions of a status on create""" + """should send out two versions of a status on create""" models.Comment.objects.create( content="hi", user=self.local_user, book=self.book ) diff --git a/bookwyrm/tests/templatetags/test_markdown.py b/bookwyrm/tests/templatetags/test_markdown.py index ba283a4f2..5b5959ad3 100644 --- a/bookwyrm/tests/templatetags/test_markdown.py +++ b/bookwyrm/tests/templatetags/test_markdown.py @@ -7,7 +7,7 @@ class MarkdownTags(TestCase): """lotta different things here""" def test_get_markdown(self): - """mardown format data""" + """markdown format data""" result = markdown.get_markdown("_hi_") self.assertEqual(result, "

hi

") diff --git a/bookwyrm/tests/templatetags/test_rating_tags.py b/bookwyrm/tests/templatetags/test_rating_tags.py index a06ee9402..5abfa471a 100644 --- a/bookwyrm/tests/templatetags/test_rating_tags.py +++ b/bookwyrm/tests/templatetags/test_rating_tags.py @@ -41,7 +41,7 @@ class RatingTags(TestCase): @patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async") def test_get_rating(self, *_): """privacy filtered rating. Commented versions are how it ought to work with - subjective ratings, which are currenly not used for performance reasons.""" + subjective ratings, which are currently not used for performance reasons.""" # follows-only: not included models.ReviewRating.objects.create( user=self.remote_user, diff --git a/bookwyrm/tests/test_postgres.py b/bookwyrm/tests/test_postgres.py index 94a8090f4..8fc3c9d59 100644 --- a/bookwyrm/tests/test_postgres.py +++ b/bookwyrm/tests/test_postgres.py @@ -30,7 +30,7 @@ class PostgresTriggers(TestCase): title="The Long Goodbye", subtitle="wow cool", series="series name", - languages=["irrelevent"], + languages=["irrelevant"], ) book.authors.add(author) book.refresh_from_db() @@ -40,7 +40,7 @@ class PostgresTriggers(TestCase): "'cool':5B 'goodby':3A 'long':2A 'name':9 'rays':7C 'seri':8 'the':6C 'wow':4B", ) - def test_seach_vector_on_author_update(self, _): + def test_search_vector_on_author_update(self, _): """update search when an author name changes""" author = models.Author.objects.create(name="The Rays") book = models.Edition.objects.create( @@ -53,7 +53,7 @@ class PostgresTriggers(TestCase): self.assertEqual(book.search_vector, "'goodby':3A 'jeremy':4C 'long':2A") - def test_seach_vector_on_author_delete(self, _): + def test_search_vector_on_author_delete(self, _): """update search when an author name changes""" author = models.Author.objects.create(name="Jeremy") book = models.Edition.objects.create( diff --git a/bookwyrm/tests/test_signing.py b/bookwyrm/tests/test_signing.py index 37d841b33..d61c32df5 100644 --- a/bookwyrm/tests/test_signing.py +++ b/bookwyrm/tests/test_signing.py @@ -107,7 +107,7 @@ class Signature(TestCase): @responses.activate def test_remote_signer(self): - """signtures for remote users""" + """signatures for remote users""" datafile = pathlib.Path(__file__).parent.joinpath("data/ap_user.json") data = json.loads(datafile.read_bytes()) data["id"] = self.fake_remote.remote_id diff --git a/bookwyrm/tests/views/inbox/test_inbox_delete.py b/bookwyrm/tests/views/inbox/test_inbox_delete.py index b4863aad5..0fb108e22 100644 --- a/bookwyrm/tests/views/inbox/test_inbox_delete.py +++ b/bookwyrm/tests/views/inbox/test_inbox_delete.py @@ -58,7 +58,7 @@ class InboxActivities(TestCase): with patch("bookwyrm.activitystreams.remove_status_task.delay") as redis_mock: views.inbox.activity_task(activity) self.assertTrue(redis_mock.called) - # deletion doens't remove the status, it turns it into a tombstone + # deletion doesn't remove the status, it turns it into a tombstone status = models.Status.objects.get() self.assertTrue(status.deleted) self.assertIsInstance(status.deleted_date, datetime) @@ -87,7 +87,7 @@ class InboxActivities(TestCase): with patch("bookwyrm.activitystreams.remove_status_task.delay") as redis_mock: views.inbox.activity_task(activity) self.assertTrue(redis_mock.called) - # deletion doens't remove the status, it turns it into a tombstone + # deletion doesn't remove the status, it turns it into a tombstone status = models.Status.objects.get() self.assertTrue(status.deleted) self.assertIsInstance(status.deleted_date, datetime) diff --git a/bookwyrm/tests/views/landing/test_login.py b/bookwyrm/tests/views/landing/test_login.py index d76e9a55f..eab082609 100644 --- a/bookwyrm/tests/views/landing/test_login.py +++ b/bookwyrm/tests/views/landing/test_login.py @@ -114,7 +114,7 @@ class LoginViews(TestCase): view = views.Login.as_view() form = forms.LoginForm() form.data["localname"] = "mouse" - form.data["password"] = "passsword1" + form.data["password"] = "password1" request = self.factory.post("", form.data) request.user = self.anonymous_user diff --git a/bookwyrm/tests/views/landing/test_password.py b/bookwyrm/tests/views/landing/test_password.py index c7c7e05d5..c1adf61e9 100644 --- a/bookwyrm/tests/views/landing/test_password.py +++ b/bookwyrm/tests/views/landing/test_password.py @@ -72,7 +72,7 @@ class PasswordViews(TestCase): validate_html(result.render()) self.assertEqual(result.status_code, 200) - def test_password_reset_nonexistant_code(self): + def test_password_reset_nonexistent_code(self): """there are so many views, this just makes sure it LOADS""" view = views.PasswordReset.as_view() request = self.factory.get("") diff --git a/bookwyrm/tests/views/test_status.py b/bookwyrm/tests/views/test_status.py index 7c64fdb0c..5874d9f2f 100644 --- a/bookwyrm/tests/views/test_status.py +++ b/bookwyrm/tests/views/test_status.py @@ -234,7 +234,7 @@ class StatusViews(TestCase): ) def test_create_status_reply_with_mentions(self, *_): - """reply to a post with an @mention'ed user""" + """reply to a post with an @mention'd user""" view = views.CreateStatus.as_view() user = models.User.objects.create_user( "rat", "rat@rat.com", "password", local=True, localname="rat" @@ -356,12 +356,12 @@ class StatusViews(TestCase): self.assertEqual(len(hashtags), 2) self.assertEqual(list(status.mention_hashtags.all()), list(hashtags)) - hashtag_exising = models.Hashtag.objects.filter(name="#existing").first() + hashtag_existing = models.Hashtag.objects.filter(name="#existing").first() hashtag_new = models.Hashtag.objects.filter(name="#NewTag").first() self.assertEqual( status.content, "

this is an " - + f'' + + f'' + "#EXISTING hashtag but all uppercase, this one is " + f'' + "#NewTag.

", diff --git a/bookwyrm/tests/views/test_wellknown.py b/bookwyrm/tests/views/test_wellknown.py index 465f39b40..80f5a56ae 100644 --- a/bookwyrm/tests/views/test_wellknown.py +++ b/bookwyrm/tests/views/test_wellknown.py @@ -53,7 +53,7 @@ class WellknownViews(TestCase): data = json.loads(result.getvalue()) self.assertEqual(data["subject"], "acct:mouse@local.com") - def test_webfinger_case_sensitivty(self): + def test_webfinger_case_sensitivity(self): """ensure that webfinger queries are not case sensitive""" request = self.factory.get("", {"resource": "acct:MoUsE@local.com"}) request.user = self.anonymous_user diff --git a/bookwyrm/views/admin/celery_status.py b/bookwyrm/views/admin/celery_status.py index e0b1fe18c..6263d8654 100644 --- a/bookwyrm/views/admin/celery_status.py +++ b/bookwyrm/views/admin/celery_status.py @@ -1,10 +1,13 @@ """ celery status """ +import json + from django.contrib.auth.decorators import login_required, permission_required from django.http import HttpResponse from django.template.response import TemplateResponse from django.utils.decorators import method_decorator from django.views import View from django.views.decorators.http import require_GET +from django import forms import redis from celerywyrm import settings @@ -46,21 +49,68 @@ class CeleryStatus(View): queues = None errors.append(err) + form = ClearCeleryForm() + data = { "stats": stats, "active_tasks": active_tasks, "queues": queues, + "form": form, "errors": errors, } return TemplateResponse(request, "settings/celery.html", data) + def post(self, request): + """Submit form to clear queues""" + form = ClearCeleryForm(request.POST) + if form.is_valid(): + if len(celery.control.ping()) != 0: + return HttpResponse( + "Refusing to delete tasks while Celery worker is active" + ) + pipeline = r.pipeline() + for queue in form.cleaned_data["queues"]: + for task in r.lrange(queue, 0, -1): + task_json = json.loads(task) + if task_json["headers"]["task"] in form.cleaned_data["tasks"]: + pipeline.lrem(queue, 0, task) + results = pipeline.execute() + + return HttpResponse(f"Deleted {sum(results)} tasks") + + +class ClearCeleryForm(forms.Form): + """Form to clear queues""" + + queues = forms.MultipleChoiceField( + label="Queues", + choices=[ + (LOW, "Low prioirty"), + (MEDIUM, "Medium priority"), + (HIGH, "High priority"), + (IMPORTS, "Imports"), + (BROADCAST, "Broadcasts"), + ], + widget=forms.CheckboxSelectMultiple, + ) + tasks = forms.MultipleChoiceField( + label="Tasks", choices=[], widget=forms.CheckboxSelectMultiple + ) + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + celery.loader.import_default_modules() + self.fields["tasks"].choices = sorted( + [(k, k) for k in celery.tasks.keys() if not k.startswith("celery.")] + ) + @require_GET # pylint: disable=unused-argument def celery_ping(request): """Just tells you if Celery is on or not""" try: - ping = celery.control.inspect().ping() + ping = celery.control.inspect().ping(timeout=5) if ping: return HttpResponse() # pylint: disable=broad-except diff --git a/bookwyrm/views/admin/dashboard.py b/bookwyrm/views/admin/dashboard.py index b49c5a238..9d256fc6c 100644 --- a/bookwyrm/views/admin/dashboard.py +++ b/bookwyrm/views/admin/dashboard.py @@ -76,7 +76,7 @@ class Dashboard(View): def get_charts_and_stats(request): - """Defines the dashbaord charts""" + """Defines the dashboard charts""" interval = int(request.GET.get("days", 1)) now = timezone.now() start = request.GET.get("start") diff --git a/bookwyrm/views/books/edit_book.py b/bookwyrm/views/books/edit_book.py index 167bd4b46..97b012db8 100644 --- a/bookwyrm/views/books/edit_book.py +++ b/bookwyrm/views/books/edit_book.py @@ -154,7 +154,7 @@ def add_authors(request, data): data["author_matches"] = [] data["isni_matches"] = [] - # creting a book or adding an author to a book needs another step + # creating a book or adding an author to a book needs another step data["confirm_mode"] = True # this isn't preserved because it isn't part of the form obj data["remove_authors"] = request.POST.getlist("remove_authors") diff --git a/bookwyrm/views/inbox.py b/bookwyrm/views/inbox.py index 1c6c64228..5264eb395 100644 --- a/bookwyrm/views/inbox.py +++ b/bookwyrm/views/inbox.py @@ -103,7 +103,7 @@ def raise_is_blocked_activity(activity_json): def sometimes_async_activity_task(activity_json, queue=MEDIUM): """Sometimes we can effectively respond to a request without queuing a new task, - and whever that is possible, we should do it.""" + and whenever that is possible, we should do it.""" activity = activitypub.parse(activity_json) # try resolving this activity without making any http requests diff --git a/bookwyrm/views/list/curate.py b/bookwyrm/views/list/curate.py index 7155ffc43..cf41636ba 100644 --- a/bookwyrm/views/list/curate.py +++ b/bookwyrm/views/list/curate.py @@ -14,7 +14,7 @@ from bookwyrm.views.list.list import normalize_book_list_ordering # pylint: disable=no-self-use @method_decorator(login_required, name="dispatch") class Curate(View): - """approve or discard list suggestsions""" + """approve or discard list suggestions""" def get(self, request, list_id): """display a pending list""" diff --git a/bookwyrm/views/list/embed.py b/bookwyrm/views/list/embed.py index 9d0078b65..a62c9c1ba 100644 --- a/bookwyrm/views/list/embed.py +++ b/bookwyrm/views/list/embed.py @@ -14,7 +14,7 @@ from bookwyrm.settings import PAGE_LENGTH # pylint: disable=no-self-use class EmbedList(View): - """embeded book list page""" + """embedded book list page""" def get(self, request, list_id, list_key): """display a book list""" diff --git a/bookwyrm/views/list/list.py b/bookwyrm/views/list/list.py index 24d44d183..30d6f970a 100644 --- a/bookwyrm/views/list/list.py +++ b/bookwyrm/views/list/list.py @@ -8,7 +8,7 @@ from django.db import transaction from django.db.models import Avg, DecimalField, Q, Max from django.db.models.functions import Coalesce from django.http import HttpResponseBadRequest, HttpResponse -from django.shortcuts import get_object_or_404 +from django.shortcuts import get_object_or_404, redirect from django.template.response import TemplateResponse from django.urls import reverse from django.utils.decorators import method_decorator @@ -183,7 +183,7 @@ def delete_list(request, list_id): book_list.raise_not_deletable(request.user) book_list.delete() - return redirect_to_referer(request, "lists") + return redirect("/list") @require_POST diff --git a/bookwyrm/views/reading.py b/bookwyrm/views/reading.py index 958917eaa..65870b8dc 100644 --- a/bookwyrm/views/reading.py +++ b/bookwyrm/views/reading.py @@ -186,7 +186,7 @@ def update_readthrough_on_shelve( active_readthrough = models.ReadThrough.objects.create( user=user, book=annotated_book ) - # santiize and set dates + # sanitize and set dates active_readthrough.start_date = load_date_in_user_tz_as_utc(start_date, user) # if the stop or finish date is set, the readthrough will be set as inactive active_readthrough.finish_date = load_date_in_user_tz_as_utc(finish_date, user) diff --git a/bookwyrm/views/status.py b/bookwyrm/views/status.py index a8c874c13..e3a7481f8 100644 --- a/bookwyrm/views/status.py +++ b/bookwyrm/views/status.py @@ -232,7 +232,7 @@ def find_mentions(user, content): if not content: return {} # The regex has nested match groups, so the 0th entry has the full (outer) match - # And beacuse the strict username starts with @, the username is 1st char onward + # And because the strict username starts with @, the username is 1st char onward usernames = [m[0][1:] for m in re.findall(regex.STRICT_USERNAME, content)] known_users = ( diff --git a/celerywyrm/apps.py b/celerywyrm/apps.py index bb2d27edd..bf443afdb 100644 --- a/celerywyrm/apps.py +++ b/celerywyrm/apps.py @@ -11,3 +11,4 @@ class CelerywyrmConfig(AppConfig): from bookwyrm.telemetry import open_telemetry open_telemetry.instrumentCelery() + open_telemetry.instrumentPostgres() diff --git a/celerywyrm/settings.py b/celerywyrm/settings.py index c1e533ac3..aa08a2417 100644 --- a/celerywyrm/settings.py +++ b/celerywyrm/settings.py @@ -3,6 +3,8 @@ # pylint: disable=unused-wildcard-import from bookwyrm.settings import * +QUERY_TIMEOUT = env.int("CELERY_QUERY_TIMEOUT", env.int("QUERY_TIMEOUT", 30)) + # pylint: disable=line-too-long REDIS_BROKER_PASSWORD = requests.utils.quote(env("REDIS_BROKER_PASSWORD", "")) REDIS_BROKER_HOST = env("REDIS_BROKER_HOST", "redis_broker") diff --git a/locale/en_US/LC_MESSAGES/django.po b/locale/en_US/LC_MESSAGES/django.po index e55690723..65000c1d4 100644 --- a/locale/en_US/LC_MESSAGES/django.po +++ b/locale/en_US/LC_MESSAGES/django.po @@ -1,4 +1,4 @@ -# Stub English-language trnaslation file +# Stub English-language translation file # Copyright (C) 2021 Mouse Reeve # This file is distributed under the same license as the BookWyrm package. # Mouse Reeve , 2021 @@ -4045,7 +4045,7 @@ msgstr "" #: bookwyrm/templates/preferences/edit_user.html:136 #, python-format -msgid "Looking for shelf privacy? You can set a sepearate visibility level for each of your shelves. Go to Your Books, pick a shelf from the tab bar, and click \"Edit shelf.\"" +msgid "Looking for shelf privacy? You can set a separate visibility level for each of your shelves. Go to Your Books, pick a shelf from the tab bar, and click \"Edit shelf.\"" msgstr "" #: bookwyrm/templates/preferences/export.html:4 diff --git a/requirements.txt b/requirements.txt index f8f1ab937..3da1d5082 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,6 +29,7 @@ opentelemetry-api==1.16.0 opentelemetry-exporter-otlp-proto-grpc==1.16.0 opentelemetry-instrumentation-celery==0.37b0 opentelemetry-instrumentation-django==0.37b0 +opentelemetry-instrumentation-psycopg2==0.37b0 opentelemetry-sdk==1.16.0 protobuf==3.20.* pyotp==2.8.0