zedfrigg/bookwyrm
1
0
Fork 0
Code Activity

Merge branch 'main' into main

Browse source
This commit is contained in:
Jascha Ezra Urbach 2023-03-21 05:49:52 +01:00 • committed by GitHub
commit bbed08e182
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
114 changed files with 5114 additions and 2390 deletions
Download patch file Download diff file Expand all files Collapse all files

2
bookwyrm/activitypub/__init__.py
View file

@ -3,7 +3,7 @@ import inspect
import sys
from .base_activity import ActivityEncoder, Signature, naive_parse
from .base_activity import Link, Mention
from .base_activity import Link, Mention, Hashtag
from .base_activity import ActivitySerializerError, resolve_remote_id
from .image import Document, Image
from .note import Note, GeneratedNote, Article, Comment, Quotation

73
bookwyrm/activitypub/base_activity.py
View file

@ -100,9 +100,27 @@ class ActivityObject:
# pylint: disable=too-many-locals,too-many-branches,too-many-arguments
def to_model(
self, model=None, instance=None, allow_create=True, save=True, overwrite=True
self,
model=None,
instance=None,
allow_create=True,
save=True,
overwrite=True,
allow_external_connections=True,
):
"""convert from an activity to a model instance"""
"""convert from an activity to a model instance. Args:
model: the django model that this object is being converted to
(will guess if not known)
instance: an existing database entry that is going to be updated by
this activity
allow_create: whether a new object should be created if there is no
existing object is provided or found matching the remote_id
save: store in the database if true, return an unsaved model obj if false
overwrite: replace fields in the database with this activity if true,
only update blank fields if false
allow_external_connections: look up missing data if true,
throw an exception if false and an external connection is needed
"""
model = model or get_model_from_type(self.type)
# only reject statuses if we're potentially creating them
@ -127,7 +145,10 @@ class ActivityObject:
for field in instance.simple_fields:
try:
changed = field.set_field_from_activity(
instance, self, overwrite=overwrite
instance,
self,
overwrite=overwrite,
allow_external_connections=allow_external_connections,
)
if changed:
update_fields.append(field.name)
@ -138,7 +159,11 @@ class ActivityObject:
# too early and jank up users
for field in instance.image_fields:
changed = field.set_field_from_activity(
instance, self, save=save, overwrite=overwrite
instance,
self,
save=save,
overwrite=overwrite,
allow_external_connections=allow_external_connections,
)
if changed:
update_fields.append(field.name)
@ -161,8 +186,12 @@ class ActivityObject:
# add many to many fields, which have to be set post-save
for field in instance.many_to_many_fields:
# mention books/users, for example
field.set_field_from_activity(instance, self)
# mention books/users/hashtags, for example
field.set_field_from_activity(
instance,
self,
allow_external_connections=allow_external_connections,
)
# reversed relationships in the models
for (
@ -212,7 +241,7 @@ class ActivityObject:
return data
@app.task(queue=MEDIUM)
@app.task(queue=MEDIUM, ignore_result=True)
@transaction.atomic
def set_related_field(
model_name, origin_model_name, related_field_name, related_remote_id, data
@ -266,10 +295,22 @@ def get_model_from_type(activity_type):
return model[0]
# pylint: disable=too-many-arguments
def resolve_remote_id(
remote_id, model=None, refresh=False, save=True, get_activity=False
remote_id,
model=None,
refresh=False,
save=True,
get_activity=False,
allow_external_connections=True,
):
"""take a remote_id and return an instance, creating if necessary"""
"""take a remote_id and return an instance, creating if necessary. Args:
remote_id: the unique url for looking up the object in the db or by http
model: a string or object representing the model that corresponds to the object
save: whether to return an unsaved database entry or a saved one
get_activity: whether to return the activitypub object or the model object
allow_external_connections: whether to make http connections
"""
if model: # a bonus check we can do if we already know the model
if isinstance(model, str):
model = apps.get_model(f"bookwyrm.{model}", require_ready=True)
@ -277,6 +318,13 @@ def resolve_remote_id(
if result and not refresh:
return result if not get_activity else result.to_activity_dataclass()
# The above block will return the object if it already exists in the database.
# If it doesn't, an external connection would be needed, so check if that's cool
if not allow_external_connections:
raise ActivitySerializerError(
"Unable to serialize object without making external HTTP requests"
)
# load the data and create the object
try:
data = get_data(remote_id)
@ -378,3 +426,10 @@ class Mention(Link):
"""a subtype of Link for mentioning an actor"""
type: str = "Mention"
@dataclass(init=False)
class Hashtag(Link):
"""a subtype of Link for mentioning a hashtag"""
type: str = "Hashtag"

48
bookwyrm/activitypub/note.py
View file

@ -1,9 +1,12 @@
""" note serializer and children thereof """
from dataclasses import dataclass, field
from typing import Dict, List
from django.apps import apps
import re
from .base_activity import ActivityObject, Link
from django.apps import apps
from django.db import IntegrityError, transaction
from .base_activity import ActivityObject, ActivitySerializerError, Link
from .image import Document
@ -38,6 +41,47 @@ class Note(ActivityObject):
updated: str = None
type: str = "Note"
# pylint: disable=too-many-arguments
def to_model(
self,
model=None,
instance=None,
allow_create=True,
save=True,
overwrite=True,
allow_external_connections=True,
):
instance = super().to_model(
model, instance, allow_create, save, overwrite, allow_external_connections
)
if instance is None:
return instance
# Replace links to hashtags in content with local URLs
changed_content = False
for hashtag in instance.mention_hashtags.all():
updated_content = re.sub(
rf'(<a href=")[^"]*(" data-mention="hashtag">{hashtag.name}</a>)',
rf"\1{hashtag.remote_id}\2",
instance.content,
flags=re.IGNORECASE,
)
if instance.content != updated_content:
instance.content = updated_content
changed_content = True
if not save or not changed_content:
return instance
with transaction.atomic():
try:
instance.save(broadcast=False, update_fields=["content"])
except IntegrityError as e:
raise ActivitySerializerError(e)
return instance
@dataclass(init=False)
class Article(Note):

65
bookwyrm/activitypub/verbs.py
View file

@ -14,12 +14,12 @@ class Verb(ActivityObject):
actor: str
object: ActivityObject
def action(self):
def action(self, allow_external_connections=True):
"""usually we just want to update and save"""
# self.object may return None if the object is invalid in an expected way
# ie, Question type
if self.object:
self.object.to_model()
self.object.to_model(allow_external_connections=allow_external_connections)
# pylint: disable=invalid-name
@ -42,7 +42,7 @@ class Delete(Verb):
cc: List[str] = field(default_factory=lambda: [])
type: str = "Delete"
def action(self):
def action(self, allow_external_connections=True):
"""find and delete the activity object"""
if not self.object:
return
@ -52,7 +52,11 @@ class Delete(Verb):
model = apps.get_model("bookwyrm.User")
obj = model.find_existing_by_remote_id(self.object)
else:
obj = self.object.to_model(save=False, allow_create=False)
obj = self.object.to_model(
save=False,
allow_create=False,
allow_external_connections=allow_external_connections,
)
if obj:
obj.delete()
@ -67,11 +71,13 @@ class Update(Verb):
to: List[str]
type: str = "Update"
def action(self):
def action(self, allow_external_connections=True):
"""update a model instance from the dataclass"""
if not self.object:
return
self.object.to_model(allow_create=False)
self.object.to_model(
allow_create=False, allow_external_connections=allow_external_connections
)
@dataclass(init=False)
@ -80,7 +86,7 @@ class Undo(Verb):
type: str = "Undo"
def action(self):
def action(self, allow_external_connections=True):
"""find and remove the activity object"""
if isinstance(self.object, str):
# it may be that something should be done with these, but idk what
@ -92,13 +98,28 @@ class Undo(Verb):
model = None
if self.object.type == "Follow":
model = apps.get_model("bookwyrm.UserFollows")
obj = self.object.to_model(model=model, save=False, allow_create=False)
obj = self.object.to_model(
model=model,
save=False,
allow_create=False,
allow_external_connections=allow_external_connections,
)
if not obj:
# this could be a follow request not a follow proper
model = apps.get_model("bookwyrm.UserFollowRequest")
obj = self.object.to_model(model=model, save=False, allow_create=False)
obj = self.object.to_model(
model=model,
save=False,
allow_create=False,
allow_external_connections=allow_external_connections,
)
else:
obj = self.object.to_model(model=model, save=False, allow_create=False)
obj = self.object.to_model(
model=model,
save=False,
allow_create=False,
allow_external_connections=allow_external_connections,
)
if not obj:
# if we don't have the object, we can't undo it. happens a lot with boosts
return
@ -112,9 +133,9 @@ class Follow(Verb):
object: str
type: str = "Follow"
def action(self):
def action(self, allow_external_connections=True):
"""relationship save"""
self.to_model()
self.to_model(allow_external_connections=allow_external_connections)
@dataclass(init=False)
@ -124,9 +145,9 @@ class Block(Verb):
object: str
type: str = "Block"
def action(self):
def action(self, allow_external_connections=True):
"""relationship save"""
self.to_model()
self.to_model(allow_external_connections=allow_external_connections)
@dataclass(init=False)
@ -136,7 +157,7 @@ class Accept(Verb):
object: Follow
type: str = "Accept"
def action(self):
def action(self, allow_external_connections=True):
"""accept a request"""
obj = self.object.to_model(save=False, allow_create=True)
obj.accept()
@ -149,7 +170,7 @@ class Reject(Verb):
object: Follow
type: str = "Reject"
def action(self):
def action(self, allow_external_connections=True):
"""reject a follow request"""
obj = self.object.to_model(save=False, allow_create=False)
obj.reject()
@ -163,7 +184,7 @@ class Add(Verb):
object: CollectionItem
type: str = "Add"
def action(self):
def action(self, allow_external_connections=True):
"""figure out the target to assign the item to a collection"""
target = resolve_remote_id(self.target)
item = self.object.to_model(save=False)
@ -177,7 +198,7 @@ class Remove(Add):
type: str = "Remove"
def action(self):
def action(self, allow_external_connections=True):
"""find and remove the activity object"""
obj = self.object.to_model(save=False, allow_create=False)
if obj:
@ -191,9 +212,9 @@ class Like(Verb):
object: str
type: str = "Like"
def action(self):
def action(self, allow_external_connections=True):
"""like"""
self.to_model()
self.to_model(allow_external_connections=allow_external_connections)
# pylint: disable=invalid-name
@ -207,6 +228,6 @@ class Announce(Verb):
object: str
type: str = "Announce"
def action(self):
def action(self, allow_external_connections=True):
"""boost"""
self.to_model()
self.to_model(allow_external_connections=allow_external_connections)

129
bookwyrm/activitystreams.py
View file

@ -13,18 +13,18 @@ from bookwyrm.tasks import app, LOW, MEDIUM, HIGH
class ActivityStream(RedisStore):
"""a category of activity stream (like home, local, books)"""
def stream_id(self, user):
def stream_id(self, user_id):
"""the redis key for this user's instance of this stream"""
return f"{user.id}-{self.key}"
return f"{user_id}-{self.key}"
def unread_id(self, user):
def unread_id(self, user_id):
"""the redis key for this user's unread count for this stream"""
stream_id = self.stream_id(user)
stream_id = self.stream_id(user_id)
return f"{stream_id}-unread"
def unread_by_status_type_id(self, user):
def unread_by_status_type_id(self, user_id):
"""the redis key for this user's unread count for this stream"""
stream_id = self.stream_id(user)
stream_id = self.stream_id(user_id)
return f"{stream_id}-unread-by-type"
def get_rank(self, obj): # pylint: disable=no-self-use
@ -37,12 +37,12 @@ class ActivityStream(RedisStore):
pipeline = self.add_object_to_related_stores(status, execute=False)
if increment_unread:
for user in self.get_audience(status):
for user_id in self.get_audience(status):
# add to the unread status count
pipeline.incr(self.unread_id(user))
pipeline.incr(self.unread_id(user_id))
# add to the unread status count for status type
pipeline.hincrby(
self.unread_by_status_type_id(user), get_status_type(status), 1
self.unread_by_status_type_id(user_id), get_status_type(status), 1
)
# and go!
@ -52,21 +52,21 @@ class ActivityStream(RedisStore):
"""add a user's statuses to another user's feed"""
# only add the statuses that the viewer should be able to see (ie, not dms)
statuses = models.Status.privacy_filter(viewer).filter(user=user)
self.bulk_add_objects_to_store(statuses, self.stream_id(viewer))
self.bulk_add_objects_to_store(statuses, self.stream_id(viewer.id))
def remove_user_statuses(self, viewer, user):
"""remove a user's status from another user's feed"""
# remove all so that followers only statuses are removed
statuses = user.status_set.all()
self.bulk_remove_objects_from_store(statuses, self.stream_id(viewer))
self.bulk_remove_objects_from_store(statuses, self.stream_id(viewer.id))
def get_activity_stream(self, user):
"""load the statuses to be displayed"""
# clear unreads for this feed
r.set(self.unread_id(user), 0)
r.delete(self.unread_by_status_type_id(user))
r.set(self.unread_id(user.id), 0)
r.delete(self.unread_by_status_type_id(user.id))
statuses = self.get_store(self.stream_id(user))
statuses = self.get_store(self.stream_id(user.id))
return (
models.Status.objects.select_subclasses()
.filter(id__in=statuses)
@ -83,11 +83,11 @@ class ActivityStream(RedisStore):
def get_unread_count(self, user):
"""get the unread status count for this user's feed"""
return int(r.get(self.unread_id(user)) or 0)
return int(r.get(self.unread_id(user.id)) or 0)
def get_unread_count_by_status_type(self, user):
"""get the unread status count for this user's feed's status types"""
status_types = r.hgetall(self.unread_by_status_type_id(user))
status_types = r.hgetall(self.unread_by_status_type_id(user.id))
return {
str(key.decode("utf-8")): int(value) or 0
for key, value in status_types.items()
@ -95,9 +95,9 @@ class ActivityStream(RedisStore):
def populate_streams(self, user):
"""go from zero to a timeline"""
self.populate_store(self.stream_id(user))
self.populate_store(self.stream_id(user.id))
def get_audience(self, status): # pylint: disable=no-self-use
def _get_audience(self, status): # pylint: disable=no-self-use
"""given a status, what users should see it"""
# direct messages don't appeard in feeds, direct comments/reviews/etc do
if status.privacy == "direct" and status.status_type == "Note":
@ -136,8 +136,12 @@ class ActivityStream(RedisStore):
)
return audience.distinct()
def get_audience(self, status): # pylint: disable=no-self-use
"""given a status, what users should see it"""
return [user.id for user in self._get_audience(status)]
def get_stores_for_object(self, obj):
return [self.stream_id(u) for u in self.get_audience(obj)]
return [self.stream_id(user_id) for user_id in self.get_audience(obj)]
def get_statuses_for_user(self, user): # pylint: disable=no-self-use
"""given a user, what statuses should they see on this stream"""
@ -157,13 +161,14 @@ class HomeStream(ActivityStream):
key = "home"
def get_audience(self, status):
audience = super().get_audience(status)
audience = super()._get_audience(status)
if not audience:
return []
return audience.filter(
Q(id=status.user.id) # if the user is the post's author
| Q(following=status.user) # if the user is following the author
).distinct()
# if the user is the post's author
ids_self = [user.id for user in audience.filter(Q(id=status.user.id))]
# if the user is following the author
ids_following = [user.id for user in audience.filter(Q(following=status.user))]
return ids_self + ids_following
def get_statuses_for_user(self, user):
return models.Status.privacy_filter(
@ -183,11 +188,11 @@ class LocalStream(ActivityStream):
key = "local"
def get_audience(self, status):
def _get_audience(self, status):
# this stream wants no part in non-public statuses
if status.privacy != "public" or not status.user.local:
return []
return super().get_audience(status)
return super()._get_audience(status)
def get_statuses_for_user(self, user):
# all public statuses by a local user
@ -202,7 +207,7 @@ class BooksStream(ActivityStream):
key = "books"
def get_audience(self, status):
def _get_audience(self, status):
"""anyone with the mentioned book on their shelves"""
# only show public statuses on the books feed,
# and only statuses that mention books
@ -217,7 +222,7 @@ class BooksStream(ActivityStream):
else status.mention_books.first().parent_work
)
audience = super().get_audience(status)
audience = super()._get_audience(status)
if not audience:
return []
return audience.filter(shelfbook__book__parent_work=work).distinct()
@ -244,38 +249,38 @@ class BooksStream(ActivityStream):
def add_book_statuses(self, user, book):
"""add statuses about a book to a user's feed"""
work = book.parent_work
statuses = (
models.Status.privacy_filter(
user,
privacy_levels=["public"],
)
.filter(
Q(comment__book__parent_work=work)
| Q(quotation__book__parent_work=work)
| Q(review__book__parent_work=work)
| Q(mention_books__parent_work=work)
)
.distinct()
statuses = models.Status.privacy_filter(
user,
privacy_levels=["public"],
)
self.bulk_add_objects_to_store(statuses, self.stream_id(user))
book_comments = statuses.filter(Q(comment__book__parent_work=work))
book_quotations = statuses.filter(Q(quotation__book__parent_work=work))
book_reviews = statuses.filter(Q(review__book__parent_work=work))
book_mentions = statuses.filter(Q(mention_books__parent_work=work))
self.bulk_add_objects_to_store(book_comments, self.stream_id(user.id))
self.bulk_add_objects_to_store(book_quotations, self.stream_id(user.id))
self.bulk_add_objects_to_store(book_reviews, self.stream_id(user.id))
self.bulk_add_objects_to_store(book_mentions, self.stream_id(user.id))
def remove_book_statuses(self, user, book):
"""add statuses about a book to a user's feed"""
work = book.parent_work
statuses = (
models.Status.privacy_filter(
user,
privacy_levels=["public"],
)
.filter(
Q(comment__book__parent_work=work)
| Q(quotation__book__parent_work=work)
| Q(review__book__parent_work=work)
| Q(mention_books__parent_work=work)
)
.distinct()
statuses = models.Status.privacy_filter(
user,
privacy_levels=["public"],
)
self.bulk_remove_objects_from_store(statuses, self.stream_id(user))
book_comments = statuses.filter(Q(comment__book__parent_work=work))
book_quotations = statuses.filter(Q(quotation__book__parent_work=work))
book_reviews = statuses.filter(Q(review__book__parent_work=work))
book_mentions = statuses.filter(Q(mention_books__parent_work=work))
self.bulk_remove_objects_from_store(book_comments, self.stream_id(user.id))
self.bulk_remove_objects_from_store(book_quotations, self.stream_id(user.id))
self.bulk_remove_objects_from_store(book_reviews, self.stream_id(user.id))
self.bulk_remove_objects_from_store(book_mentions, self.stream_id(user.id))
# determine which streams are enabled in settings.py
@ -466,7 +471,7 @@ def remove_statuses_on_unshelve(sender, instance, *args, **kwargs):
# ---- TASKS
@app.task(queue=LOW)
@app.task(queue=LOW, ignore_result=True)
def add_book_statuses_task(user_id, book_id):
"""add statuses related to a book on shelve"""
user = models.User.objects.get(id=user_id)
@ -474,7 +479,7 @@ def add_book_statuses_task(user_id, book_id):
BooksStream().add_book_statuses(user, book)
@app.task(queue=LOW)
@app.task(queue=LOW, ignore_result=True)
def remove_book_statuses_task(user_id, book_id):
"""remove statuses about a book from a user's books feed"""
user = models.User.objects.get(id=user_id)
@ -482,7 +487,7 @@ def remove_book_statuses_task(user_id, book_id):
BooksStream().remove_book_statuses(user, book)
@app.task(queue=MEDIUM)
@app.task(queue=MEDIUM, ignore_result=True)
def populate_stream_task(stream, user_id):
"""background task for populating an empty activitystream"""
user = models.User.objects.get(id=user_id)
@ -490,7 +495,7 @@ def populate_stream_task(stream, user_id):
stream.populate_streams(user)
@app.task(queue=MEDIUM)
@app.task(queue=MEDIUM, ignore_result=True)
def remove_status_task(status_ids):
"""remove a status from any stream it might be in"""
# this can take an id or a list of ids
@ -503,7 +508,7 @@ def remove_status_task(status_ids):
stream.remove_object_from_related_stores(status)
@app.task(queue=HIGH)
@app.task(queue=HIGH, ignore_result=True)
def add_status_task(status_id, increment_unread=False):
"""add a status to any stream it should be in"""
status = models.Status.objects.select_subclasses().get(id=status_id)
@ -515,7 +520,7 @@ def add_status_task(status_id, increment_unread=False):
stream.add_status(status, increment_unread=increment_unread)
@app.task(queue=MEDIUM)
@app.task(queue=MEDIUM, ignore_result=True)
def remove_user_statuses_task(viewer_id, user_id, stream_list=None):
"""remove all statuses by a user from a viewer's stream"""
stream_list = [streams[s] for s in stream_list] if stream_list else streams.values()
@ -525,7 +530,7 @@ def remove_user_statuses_task(viewer_id, user_id, stream_list=None):
stream.remove_user_statuses(viewer, user)
@app.task(queue=MEDIUM)
@app.task(queue=MEDIUM, ignore_result=True)
def add_user_statuses_task(viewer_id, user_id, stream_list=None):
"""add all statuses by a user to a viewer's stream"""
stream_list = [streams[s] for s in stream_list] if stream_list else streams.values()
@ -535,7 +540,7 @@ def add_user_statuses_task(viewer_id, user_id, stream_list=None):
stream.add_user_statuses(viewer, user)
@app.task(queue=MEDIUM)
@app.task(queue=MEDIUM, ignore_result=True)
def handle_boost_task(boost_id):
"""remove the original post and other, earlier boosts"""
instance = models.Status.objects.get(id=boost_id)

2
bookwyrm/apps.py
View file

@ -35,7 +35,7 @@ class BookwyrmConfig(AppConfig):
# pylint: disable=no-self-use
def ready(self):
"""set up OTLP and preview image files, if desired"""
if settings.OTEL_EXPORTER_OTLP_ENDPOINT:
if settings.OTEL_EXPORTER_OTLP_ENDPOINT or settings.OTEL_EXPORTER_CONSOLE:
# pylint: disable=import-outside-toplevel
from bookwyrm.telemetry import open_telemetry

4
bookwyrm/connectors/connector_manager.py
View file

@ -143,7 +143,7 @@ def get_or_create_connector(remote_id):
return load_connector(connector_info)
@app.task(queue=LOW)
@app.task(queue=LOW, ignore_result=True)
def load_more_data(connector_id, book_id):
"""background the work of getting all 10,000 editions of LoTR"""
connector_info = models.Connector.objects.get(id=connector_id)
@ -152,7 +152,7 @@ def load_more_data(connector_id, book_id):
connector.expand_book_data(book)
@app.task(queue=LOW)
@app.task(queue=LOW, ignore_result=True)
def create_edition_task(connector_id, work_id, data):
"""separate task for each of the 10,000 editions of LoTR"""
connector_info = models.Connector.objects.get(id=connector_id)

2
bookwyrm/emailing.py
View file

@ -75,7 +75,7 @@ def format_email(email_name, data):
return (subject, html_content, text_content)
@app.task(queue=HIGH)
@app.task(queue=HIGH, ignore_result=True)
def send_email(recipient, subject, html_content, text_content):
"""use a task to send the email"""
email = EmailMultiAlternatives(

10
bookwyrm/lists_stream.py
View file

@ -217,14 +217,14 @@ def add_list_on_account_create_command(user_id):
# ---- TASKS
@app.task(queue=MEDIUM)
@app.task(queue=MEDIUM, ignore_result=True)
def populate_lists_task(user_id):
"""background task for populating an empty list stream"""
user = models.User.objects.get(id=user_id)
ListsStream().populate_lists(user)
@app.task(queue=MEDIUM)
@app.task(queue=MEDIUM, ignore_result=True)
def remove_list_task(list_id, re_add=False):
"""remove a list from any stream it might be in"""
stores = models.User.objects.filter(local=True, is_active=True).values_list(
@ -239,14 +239,14 @@ def remove_list_task(list_id, re_add=False):
add_list_task.delay(list_id)
@app.task(queue=HIGH)
@app.task(queue=HIGH, ignore_result=True)
def add_list_task(list_id):
"""add a list to any stream it should be in"""
book_list = models.List.objects.get(id=list_id)
ListsStream().add_list(book_list)
@app.task(queue=MEDIUM)
@app.task(queue=MEDIUM, ignore_result=True)
def remove_user_lists_task(viewer_id, user_id, exclude_privacy=None):
"""remove all lists by a user from a viewer's stream"""
viewer = models.User.objects.get(id=viewer_id)
@ -254,7 +254,7 @@ def remove_user_lists_task(viewer_id, user_id, exclude_privacy=None):
ListsStream().remove_user_lists(viewer, user, exclude_privacy=exclude_privacy)
@app.task(queue=MEDIUM)
@app.task(queue=MEDIUM, ignore_result=True)
def add_user_lists_task(viewer_id, user_id):
"""add all lists by a user to a viewer's stream"""
viewer = models.User.objects.get(id=viewer_id)

46
bookwyrm/migrations/0174_auto_20230222_1742.py Normal file
View file

@ -0,0 +1,46 @@
# Generated by Django 3.2.18 on 2023-02-22 17:42
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0174_auto_20230130_1240"),
]
operations = [
migrations.AddField(
model_name="notification",
name="related_link_domains",
field=models.ManyToManyField(to="bookwyrm.LinkDomain"),
),
migrations.AlterField(
model_name="notification",
name="notification_type",
field=models.CharField(
choices=[
("FAVORITE", "Favorite"),
("REPLY", "Reply"),
("MENTION", "Mention"),
("TAG", "Tag"),
("FOLLOW", "Follow"),
("FOLLOW_REQUEST", "Follow Request"),
("BOOST", "Boost"),
("IMPORT", "Import"),
("ADD", "Add"),
("REPORT", "Report"),
("LINK_DOMAIN", "Link Domain"),
("INVITE", "Invite"),
("ACCEPT", "Accept"),
("JOIN", "Join"),
("LEAVE", "Leave"),
("REMOVE", "Remove"),
("GROUP_PRIVACY", "Group Privacy"),
("GROUP_NAME", "Group Name"),
("GROUP_DESCRIPTION", "Group Description"),
],
max_length=255,
),
),
]

53
bookwyrm/migrations/0176_hashtag_support.py Normal file
View file

@ -0,0 +1,53 @@
# Generated by Django 3.2.16 on 2022-12-17 19:28
import bookwyrm.models.fields
import django.contrib.postgres.fields.citext
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0174_auto_20230130_1240"),
]
operations = [
migrations.CreateModel(
name="Hashtag",
fields=[
(
"id",
models.AutoField(
auto_created=True,
primary_key=True,
serialize=False,
verbose_name="ID",
),
),
("created_date", models.DateTimeField(auto_now_add=True)),
("updated_date", models.DateTimeField(auto_now=True)),
(
"remote_id",
bookwyrm.models.fields.RemoteIdField(
max_length=255,
null=True,
validators=[bookwyrm.models.fields.validate_remote_id],
),
),
(
"name",
django.contrib.postgres.fields.citext.CICharField(max_length=256),
),
],
options={
"abstract": False,
},
),
migrations.AddField(
model_name="status",
name="mention_hashtags",
field=bookwyrm.models.fields.TagField(
related_name="mention_hashtag", to="bookwyrm.Hashtag"
),
),
]

13
bookwyrm/migrations/0177_merge_0174_auto_20230222_1742_0176_hashtag_support.py Normal file
View file

@ -0,0 +1,13 @@
# Generated by Django 3.2.18 on 2023-03-12 23:41
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
("bookwyrm", "0174_auto_20230222_1742"),
("bookwyrm", "0176_hashtag_support"),
]
operations = []

2
bookwyrm/models/__init__.py
View file

@ -34,6 +34,8 @@ from .antispam import EmailBlocklist, IPBlocklist, AutoMod, automod_task
from .notification import Notification
from .hashtag import Hashtag
cls_members = inspect.getmembers(sys.modules[__name__], inspect.isclass)
activity_models = {
c[1].activity_serializer.__name__: c[1]

2
bookwyrm/models/activitypub_mixin.py
View file

@ -506,7 +506,7 @@ def unfurl_related_field(related_field, sort_field=None):
return related_field.remote_id
@app.task(queue=BROADCAST)
@app.task(queue=BROADCAST, ignore_result=True)
def broadcast_task(sender_id: int, activity: str, recipients: List[str]):
"""the celery task for broadcast"""
user_model = apps.get_model("bookwyrm.User", require_ready=True)

2
bookwyrm/models/antispam.py
View file

@ -65,7 +65,7 @@ class AutoMod(AdminModel):
created_by = models.ForeignKey("User", on_delete=models.PROTECT)
@app.task(queue=LOW)
@app.task(queue=LOW, ignore_result=True)
def automod_task():
"""Create reports"""
if not AutoMod.objects.exists():

87
bookwyrm/models/fields.py
View file

@ -7,6 +7,7 @@ from urllib.parse import urljoin
import dateutil.parser
from dateutil.parser import ParserError
from django.contrib.postgres.fields import ArrayField as DjangoArrayField
from django.contrib.postgres.fields import CICharField as DjangoCICharField
from django.core.exceptions import ValidationError
from django.db import models
from django.forms import ClearableFileInput, ImageField as DjangoImageField
@ -67,7 +68,9 @@ class ActivitypubFieldMixin:
self.activitypub_field = activitypub_field
super().__init__(*args, **kwargs)
def set_field_from_activity(self, instance, data, overwrite=True):
def set_field_from_activity(
self, instance, data, overwrite=True, allow_external_connections=True
):
"""helper function for assinging a value to the field. Returns if changed"""
try:
value = getattr(data, self.get_activitypub_field())
@ -76,7 +79,9 @@ class ActivitypubFieldMixin:
if self.get_activitypub_field() != "attributedTo":
raise
value = getattr(data, "actor")
formatted = self.field_from_activity(value)
formatted = self.field_from_activity(
value, allow_external_connections=allow_external_connections
)
if formatted is None or formatted is MISSING or formatted == {}:
return False
@ -116,7 +121,8 @@ class ActivitypubFieldMixin:
return {self.activitypub_wrapper: value}
return value
def field_from_activity(self, value):
# pylint: disable=unused-argument
def field_from_activity(self, value, allow_external_connections=True):
"""formatter to convert activitypub into a model value"""
if value and hasattr(self, "activitypub_wrapper"):
value = value.get(self.activitypub_wrapper)
@ -138,7 +144,7 @@ class ActivitypubRelatedFieldMixin(ActivitypubFieldMixin):
self.load_remote = load_remote
super().__init__(*args, **kwargs)
def field_from_activity(self, value):
def field_from_activity(self, value, allow_external_connections=True):
if not value:
return None
@ -159,7 +165,11 @@ class ActivitypubRelatedFieldMixin(ActivitypubFieldMixin):
if not self.load_remote:
# only look in the local database
return related_model.find_existing_by_remote_id(value)
return activitypub.resolve_remote_id(value, model=related_model)
return activitypub.resolve_remote_id(
value,
model=related_model,
allow_external_connections=allow_external_connections,
)
class RemoteIdField(ActivitypubFieldMixin, models.CharField):
@ -219,7 +229,9 @@ class PrivacyField(ActivitypubFieldMixin, models.CharField):
super().__init__(*args, max_length=255, choices=PrivacyLevels, default="public")
# pylint: disable=invalid-name
def set_field_from_activity(self, instance, data, overwrite=True):
def set_field_from_activity(
self, instance, data, overwrite=True, allow_external_connections=True
):
if not overwrite:
return False
@ -234,7 +246,11 @@ class PrivacyField(ActivitypubFieldMixin, models.CharField):
break
if not user_field:
raise ValidationError("No user field found for privacy", data)
user = activitypub.resolve_remote_id(getattr(data, user_field), model="User")
user = activitypub.resolve_remote_id(
getattr(data, user_field),
model="User",
allow_external_connections=allow_external_connections,
)
if to == [self.public]:
setattr(instance, self.name, "public")
@ -295,13 +311,17 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
self.link_only = link_only
super().__init__(*args, **kwargs)
def set_field_from_activity(self, instance, data, overwrite=True):
def set_field_from_activity(
self, instance, data, overwrite=True, allow_external_connections=True
):
"""helper function for assigning a value to the field"""
if not overwrite and getattr(instance, self.name).exists():
return False
value = getattr(data, self.get_activitypub_field())
formatted = self.field_from_activity(value)
formatted = self.field_from_activity(
value, allow_external_connections=allow_external_connections
)
if formatted is None or formatted is MISSING:
return False
getattr(instance, self.name).set(formatted)
@ -313,7 +333,7 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
return f"{value.instance.remote_id}/{self.name}"
return [i.remote_id for i in value.all()]
def field_from_activity(self, value):
def field_from_activity(self, value, allow_external_connections=True):
if value is None or value is MISSING:
return None
if not isinstance(value, list):
@ -326,7 +346,11 @@ class ManyToManyField(ActivitypubFieldMixin, models.ManyToManyField):
except ValidationError:
continue
items.append(
activitypub.resolve_remote_id(remote_id, model=self.related_model)
activitypub.resolve_remote_id(
remote_id,
model=self.related_model,
allow_external_connections=allow_external_connections,
)
)
return items
@ -353,7 +377,7 @@ class TagField(ManyToManyField):
)
return tags
def field_from_activity(self, value):
def field_from_activity(self, value, allow_external_connections=True):
if not isinstance(value, list):
return None
items = []
@ -365,9 +389,22 @@ class TagField(ManyToManyField):
if tag_type != self.related_model.activity_serializer.type:
# tags can contain multiple types
continue
items.append(
activitypub.resolve_remote_id(link.href, model=self.related_model)
)
if tag_type == "Hashtag":
# we already have all data to create hashtags,
# no need to fetch from remote
item = self.related_model.activity_serializer(**link_json)
hashtag = item.to_model(model=self.related_model, save=True)
items.append(hashtag)
else:
# for other tag types we fetch them remotely
items.append(
activitypub.resolve_remote_id(
link.href,
model=self.related_model,
allow_external_connections=allow_external_connections,
)
)
return items
@ -390,11 +427,15 @@ class ImageField(ActivitypubFieldMixin, models.ImageField):
self.alt_field = alt_field
super().__init__(*args, **kwargs)
# pylint: disable=arguments-differ,arguments-renamed
def set_field_from_activity(self, instance, data, save=True, overwrite=True):
# pylint: disable=arguments-differ,arguments-renamed,too-many-arguments
def set_field_from_activity(
self, instance, data, save=True, overwrite=True, allow_external_connections=True
):
"""helper function for assinging a value to the field"""
value = getattr(data, self.get_activitypub_field())
formatted = self.field_from_activity(value)
formatted = self.field_from_activity(
value, allow_external_connections=allow_external_connections
)
if formatted is None or formatted is MISSING:
return False
@ -426,7 +467,7 @@ class ImageField(ActivitypubFieldMixin, models.ImageField):
return activitypub.Document(url=url, name=alt)
def field_from_activity(self, value):
def field_from_activity(self, value, allow_external_connections=True):
image_slug = value
# when it's an inline image (User avatar/icon, Book cover), it's a json
# blob, but when it's an attached image, it's just a url
@ -481,7 +522,7 @@ class DateTimeField(ActivitypubFieldMixin, models.DateTimeField):
return None
return value.isoformat()
def field_from_activity(self, value):
def field_from_activity(self, value, allow_external_connections=True):
try:
date_value = dateutil.parser.parse(value)
try:
@ -495,7 +536,7 @@ class DateTimeField(ActivitypubFieldMixin, models.DateTimeField):
class HtmlField(ActivitypubFieldMixin, models.TextField):
"""a text field for storing html"""
def field_from_activity(self, value):
def field_from_activity(self, value, allow_external_connections=True):
if not value or value == MISSING:
return None
return clean(value)
@ -515,6 +556,10 @@ class CharField(ActivitypubFieldMixin, models.CharField):
"""activitypub-aware char field"""
class CICharField(ActivitypubFieldMixin, DjangoCICharField):
"""activitypub-aware cichar field"""
class URLField(ActivitypubFieldMixin, models.URLField):
"""activitypub-aware url field"""

23
bookwyrm/models/hashtag.py Normal file
View file

@ -0,0 +1,23 @@
""" model for tags """
from bookwyrm import activitypub
from .activitypub_mixin import ActivitypubMixin
from .base_model import BookWyrmModel
from .fields import CICharField
class Hashtag(ActivitypubMixin, BookWyrmModel):
"a hashtag which can be used in statuses"
name = CICharField(
max_length=256,
blank=False,
null=False,
activitypub_field="name",
deduplication_field=True,
)
name_field = "name"
activity_serializer = activitypub.Hashtag
def __repr__(self):
return f"<{self.__class__} id={self.id} name={self.name}>"

4
bookwyrm/models/import_job.py
View file

@ -327,7 +327,7 @@ class ImportItem(models.Model):
)
@app.task(queue=IMPORTS)
@app.task(queue=IMPORTS, ignore_result=True)
def start_import_task(job_id):
"""trigger the child tasks for each row"""
job = ImportJob.objects.get(id=job_id)
@ -346,7 +346,7 @@ def start_import_task(job_id):
job.save()
@app.task(queue=IMPORTS)
@app.task(queue=IMPORTS, ignore_result=True)
def import_item_task(item_id):
"""resolve a row into a book"""
item = ImportItem.objects.get(id=item_id)

28
bookwyrm/models/notification.py
View file

@ -2,8 +2,8 @@
from django.db import models, transaction
from django.dispatch import receiver
from .base_model import BookWyrmModel
from . import Boost, Favorite, GroupMemberInvitation, ImportJob, ListItem, Report
from . import Status, User, UserFollowRequest
from . import Boost, Favorite, GroupMemberInvitation, ImportJob, LinkDomain
from . import ListItem, Report, Status, User, UserFollowRequest
class Notification(BookWyrmModel):
@ -28,6 +28,7 @@ class Notification(BookWyrmModel):
# Admin
REPORT = "REPORT"
LINK_DOMAIN = "LINK_DOMAIN"
# Groups
INVITE = "INVITE"
@ -43,7 +44,7 @@ class Notification(BookWyrmModel):
NotificationType = models.TextChoices(
# there has got be a better way to do this
"NotificationType",
f"{FAVORITE} {REPLY} {MENTION} {TAG} {FOLLOW} {FOLLOW_REQUEST} {BOOST} {IMPORT} {ADD} {REPORT} {INVITE} {ACCEPT} {JOIN} {LEAVE} {REMOVE} {GROUP_PRIVACY} {GROUP_NAME} {GROUP_DESCRIPTION}",
f"{FAVORITE} {REPLY} {MENTION} {TAG} {FOLLOW} {FOLLOW_REQUEST} {BOOST} {IMPORT} {ADD} {REPORT} {LINK_DOMAIN} {INVITE} {ACCEPT} {JOIN} {LEAVE} {REMOVE} {GROUP_PRIVACY} {GROUP_NAME} {GROUP_DESCRIPTION}",
)
user = models.ForeignKey("User", on_delete=models.CASCADE)
@ -64,6 +65,7 @@ class Notification(BookWyrmModel):
"ListItem", symmetrical=False, related_name="notifications"
)
related_reports = models.ManyToManyField("Report", symmetrical=False)
related_link_domains = models.ManyToManyField("LinkDomain", symmetrical=False)
@classmethod
@transaction.atomic
@ -241,6 +243,26 @@ def notify_admins_on_report(sender, instance, created, *args, **kwargs):
notification.related_reports.add(instance)
@receiver(models.signals.post_save, sender=LinkDomain)
@transaction.atomic
# pylint: disable=unused-argument
def notify_admins_on_link_domain(sender, instance, created, *args, **kwargs):
"""a new link domain needs to be verified"""
if not created:
# otherwise you'll get a notification when you approve a domain
return
# moderators and superusers should be notified
admins = User.admins()
for admin in admins:
notification, _ = Notification.objects.get_or_create(
user=admin,
notification_type=Notification.LINK_DOMAIN,
read=False,
)
notification.related_link_domains.add(instance)
@receiver(models.signals.post_save, sender=GroupMemberInvitation)
# pylint: disable=unused-argument
def notify_user_on_group_invite(sender, instance, *args, **kwargs):

6
bookwyrm/models/relationship.py
View file

@ -171,7 +171,11 @@ class UserFollowRequest(ActivitypubMixin, UserRelationship):
return
with transaction.atomic():
UserFollows.from_request(self)
try:
UserFollows.from_request(self)
except IntegrityError:
# this just means we already saved this relationship
pass
if self.id:
self.delete()

1
bookwyrm/models/status.py
View file

@ -34,6 +34,7 @@ class Status(OrderedCollectionPageMixin, BookWyrmModel):
raw_content = models.TextField(blank=True, null=True)
mention_users = fields.TagField("User", related_name="mention_user")
mention_books = fields.TagField("Edition", related_name="mention_book")
mention_hashtags = fields.TagField("Hashtag", related_name="mention_hashtag")
local = models.BooleanField(default=True)
content_warning = fields.CharField(
max_length=500, blank=True, null=True, activitypub_field="summary"

4
bookwyrm/models/user.py
View file

@ -469,7 +469,7 @@ class KeyPair(ActivitypubMixin, BookWyrmModel):
return super().save(*args, **kwargs)
@app.task(queue=LOW)
@app.task(queue=LOW, ignore_result=True)
def set_remote_server(user_id):
"""figure out the user's remote server in the background"""
user = User.objects.get(id=user_id)
@ -513,7 +513,7 @@ def get_or_create_remote_server(domain, refresh=False):
return server
@app.task(queue=LOW)
@app.task(queue=LOW, ignore_result=True)
def get_remote_reviews(outbox):
"""ingest reviews by a new remote bookwyrm user"""
outbox_page = outbox + "?page=true&type=Review"

8
bookwyrm/preview_images.py
View file

@ -420,7 +420,7 @@ def save_and_cleanup(image, instance=None):
# pylint: disable=invalid-name
@app.task(queue=LOW)
@app.task(queue=LOW, ignore_result=True)
def generate_site_preview_image_task():
"""generate preview_image for the website"""
if not settings.ENABLE_PREVIEW_IMAGES:
@ -445,7 +445,7 @@ def generate_site_preview_image_task():
# pylint: disable=invalid-name
@app.task(queue=LOW)
@app.task(queue=LOW, ignore_result=True)
def generate_edition_preview_image_task(book_id):
"""generate preview_image for a book"""
if not settings.ENABLE_PREVIEW_IMAGES:
@ -470,7 +470,7 @@ def generate_edition_preview_image_task(book_id):
save_and_cleanup(image, instance=book)
@app.task(queue=LOW)
@app.task(queue=LOW, ignore_result=True)
def generate_user_preview_image_task(user_id):
"""generate preview_image for a user"""
if not settings.ENABLE_PREVIEW_IMAGES:
@ -496,7 +496,7 @@ def generate_user_preview_image_task(user_id):
save_and_cleanup(image, instance=user)
@app.task(queue=LOW)
@app.task(queue=LOW, ignore_result=True)
def remove_user_preview_image_task(user_id):
"""remove preview_image for a user"""
if not settings.ENABLE_PREVIEW_IMAGES:

54
bookwyrm/settings.py
View file

@ -4,6 +4,7 @@ from environs import Env
import requests
from django.utils.translation import gettext_lazy as _
from django.core.exceptions import ImproperlyConfigured
# pylint: disable=line-too-long
@ -11,22 +12,22 @@ from django.utils.translation import gettext_lazy as _
env = Env()
env.read_env()
DOMAIN = env("DOMAIN")
VERSION = "0.5.5"
VERSION = "0.6.0"
RELEASE_API = env(
"RELEASE_API",
"https://api.github.com/repos/bookwyrm-social/bookwyrm/releases/latest",
)
PAGE_LENGTH = env("PAGE_LENGTH", 15)
PAGE_LENGTH = env.int("PAGE_LENGTH", 15)
DEFAULT_LANGUAGE = env("DEFAULT_LANGUAGE", "English")
JS_CACHE = "cd848b9a"
JS_CACHE = "a7d4e720"
# email
EMAIL_BACKEND = env("EMAIL_BACKEND", "django.core.mail.backends.smtp.EmailBackend")
EMAIL_HOST = env("EMAIL_HOST")
EMAIL_PORT = env("EMAIL_PORT", 587)
EMAIL_PORT = env.int("EMAIL_PORT", 587)
EMAIL_HOST_USER = env("EMAIL_HOST_USER")
EMAIL_HOST_PASSWORD = env("EMAIL_HOST_PASSWORD")
EMAIL_USE_TLS = env.bool("EMAIL_USE_TLS", True)
@ -68,13 +69,15 @@ FONT_DIR = os.path.join(STATIC_ROOT, "fonts")
# Quick-start development settings - unsuitable for production
# See https://docs.djangoproject.com/en/3.2/howto/deployment/checklist/
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = env("SECRET_KEY")
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = env.bool("DEBUG", True)
USE_HTTPS = env.bool("USE_HTTPS", not DEBUG)
# SECURITY WARNING: keep the secret key used in production secret!
SECRET_KEY = env("SECRET_KEY")
if not DEBUG and SECRET_KEY == "7(2w1sedok=aznpq)ta1mc4i%4h=xx@hxwx*o57ctsuml0x%fr":
raise ImproperlyConfigured("You must change the SECRET_KEY env variable")
ALLOWED_HOSTS = env.list("ALLOWED_HOSTS", ["*"])
# Application definition
@ -205,14 +208,14 @@ WSGI_APPLICATION = "bookwyrm.wsgi.application"
# redis/activity streams settings
REDIS_ACTIVITY_HOST = env("REDIS_ACTIVITY_HOST", "localhost")
REDIS_ACTIVITY_PORT = env("REDIS_ACTIVITY_PORT", 6379)
REDIS_ACTIVITY_PORT = env.int("REDIS_ACTIVITY_PORT", 6379)
REDIS_ACTIVITY_PASSWORD = requests.utils.quote(env("REDIS_ACTIVITY_PASSWORD", ""))
REDIS_ACTIVITY_DB_INDEX = env("REDIS_ACTIVITY_DB_INDEX", 0)
REDIS_ACTIVITY_DB_INDEX = env.int("REDIS_ACTIVITY_DB_INDEX", 0)
REDIS_ACTIVITY_URL = env(
"REDIS_ACTIVITY_URL",
f"redis://:{REDIS_ACTIVITY_PASSWORD}@{REDIS_ACTIVITY_HOST}:{REDIS_ACTIVITY_PORT}/{REDIS_ACTIVITY_DB_INDEX}",
)
MAX_STREAM_LENGTH = int(env("MAX_STREAM_LENGTH", 200))
MAX_STREAM_LENGTH = env.int("MAX_STREAM_LENGTH", 200)
STREAMS = [
{"key": "home", "name": _("Home Timeline"), "shortname": _("Home")},
@ -221,12 +224,12 @@ STREAMS = [
# Search configuration
# total time in seconds that the instance will spend searching connectors
SEARCH_TIMEOUT = int(env("SEARCH_TIMEOUT", 8))
SEARCH_TIMEOUT = env.int("SEARCH_TIMEOUT", 8)
# timeout for a query to an individual connector
QUERY_TIMEOUT = int(env("QUERY_TIMEOUT", 5))
QUERY_TIMEOUT = env.int("QUERY_TIMEOUT", 5)
# Redis cache backend
if env("USE_DUMMY_CACHE", False):
if env.bool("USE_DUMMY_CACHE", False):
CACHES = {
"default": {
"BACKEND": "django.core.cache.backends.dummy.DummyCache",
@ -256,7 +259,7 @@ DATABASES = {
"USER": env("POSTGRES_USER", "bookwyrm"),
"PASSWORD": env("POSTGRES_PASSWORD", "bookwyrm"),
"HOST": env("POSTGRES_HOST", ""),
"PORT": env("PGPORT", 5432),
"PORT": env.int("PGPORT", 5432),
},
}
@ -341,6 +344,7 @@ if USE_HTTPS:
CSRF_COOKIE_SECURE = True
USE_S3 = env.bool("USE_S3", False)
USE_AZURE = env.bool("USE_AZURE", False)
if USE_S3:
# AWS settings
@ -364,6 +368,27 @@ if USE_S3:
DEFAULT_FILE_STORAGE = "bookwyrm.storage_backends.ImagesStorage"
CSP_DEFAULT_SRC = ["'self'", AWS_S3_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
CSP_SCRIPT_SRC = ["'self'", AWS_S3_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
elif USE_AZURE:
AZURE_ACCOUNT_NAME = env("AZURE_ACCOUNT_NAME")
AZURE_ACCOUNT_KEY = env("AZURE_ACCOUNT_KEY")
AZURE_CONTAINER = env("AZURE_CONTAINER")
AZURE_CUSTOM_DOMAIN = env("AZURE_CUSTOM_DOMAIN")
# Azure Static settings
STATIC_LOCATION = "static"
STATIC_URL = (
f"{PROTOCOL}://{AZURE_CUSTOM_DOMAIN}/{AZURE_CONTAINER}/{STATIC_LOCATION}/"
)
STATICFILES_STORAGE = "bookwyrm.storage_backends.AzureStaticStorage"
# Azure Media settings
MEDIA_LOCATION = "images"
MEDIA_URL = (
f"{PROTOCOL}://{AZURE_CUSTOM_DOMAIN}/{AZURE_CONTAINER}/{MEDIA_LOCATION}/"
)
MEDIA_FULL_URL = MEDIA_URL
STATIC_FULL_URL = STATIC_URL
DEFAULT_FILE_STORAGE = "bookwyrm.storage_backends.AzureImagesStorage"
CSP_DEFAULT_SRC = ["'self'", AZURE_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
CSP_SCRIPT_SRC = ["'self'", AZURE_CUSTOM_DOMAIN] + CSP_ADDITIONAL_HOSTS
else:
STATIC_URL = "/static/"
MEDIA_URL = "/images/"
@ -377,6 +402,7 @@ CSP_INCLUDE_NONCE_IN = ["script-src"]
OTEL_EXPORTER_OTLP_ENDPOINT = env("OTEL_EXPORTER_OTLP_ENDPOINT", None)
OTEL_EXPORTER_OTLP_HEADERS = env("OTEL_EXPORTER_OTLP_HEADERS", None)
OTEL_SERVICE_NAME = env("OTEL_SERVICE_NAME", None)
OTEL_EXPORTER_CONSOLE = env.bool("OTEL_EXPORTER_CONSOLE", False)
TWO_FACTOR_LOGIN_MAX_SECONDS = env.int("TWO_FACTOR_LOGIN_MAX_SECONDS", 60)
TWO_FACTOR_LOGIN_VALIDITY_WINDOW = env.int("TWO_FACTOR_LOGIN_VALIDITY_WINDOW", 2)

14
bookwyrm/static/js/bookwyrm.js
View file

@ -95,7 +95,6 @@ let BookWyrm = new (class {
/**
* Update a counter with recurring requests to the API
* The delay is slightly randomized and increased on each cycle.
*
* @param {Object} counter - DOM node
* @param {int} delay - frequency for polling in ms
@ -104,16 +103,19 @@ let BookWyrm = new (class {
polling(counter, delay) {
const bookwyrm = this;
delay = delay || 10000;
delay += Math.random() * 1000;
delay = delay || 5 * 60 * 1000 + (Math.random() - 0.5) * 30 * 1000;
setTimeout(
function () {
fetch("/api/updates/" + counter.dataset.poll)
.then((response) => response.json())
.then((data) => bookwyrm.updateCountElement(counter, data));
bookwyrm.polling(counter, delay * 1.25);
.then((data) => {
bookwyrm.updateCountElement(counter, data);
bookwyrm.polling(counter);
})
.catch(() => {
bookwyrm.polling(counter, delay * 1.1);
});
},
delay,
counter

14
bookwyrm/storage_backends.py
View file

@ -2,6 +2,7 @@
import os
from tempfile import SpooledTemporaryFile
from storages.backends.s3boto3 import S3Boto3Storage
from storages.backends.azure_storage import AzureStorage
class StaticStorage(S3Boto3Storage): # pylint: disable=abstract-method
@ -47,3 +48,16 @@ class ImagesStorage(S3Boto3Storage): # pylint: disable=abstract-method
# Upload the object which will auto close the
# content_autoclose instance
return super()._save(name, content_autoclose)
class AzureStaticStorage(AzureStorage): # pylint: disable=abstract-method
"""Storage class for Static contents"""
location = "static"
class AzureImagesStorage(AzureStorage): # pylint: disable=abstract-method
"""Storage class for Image files"""
location = "images"
overwrite_files = False

12
bookwyrm/suggested_users.py
View file

@ -237,41 +237,41 @@ def domain_level_update(sender, instance, created, update_fields=None, **kwargs)
# ------------------- TASKS
@app.task(queue=LOW)
@app.task(queue=LOW, ignore_result=True)
def rerank_suggestions_task(user_id):
"""do the hard work in celery"""
suggested_users.rerank_user_suggestions(user_id)
@app.task(queue=LOW)
@app.task(queue=LOW, ignore_result=True)
def rerank_user_task(user_id, update_only=False):
"""do the hard work in celery"""
user = models.User.objects.get(id=user_id)
suggested_users.rerank_obj(user, update_only=update_only)
@app.task(queue=LOW)
@app.task(queue=LOW, ignore_result=True)
def remove_user_task(user_id):
"""do the hard work in celery"""
user = models.User.objects.get(id=user_id)
suggested_users.remove_object_from_related_stores(user)
@app.task(queue=MEDIUM)
@app.task(queue=MEDIUM, ignore_result=True)
def remove_suggestion_task(user_id, suggested_user_id):
"""remove a specific user from a specific user's suggestions"""
suggested_user = models.User.objects.get(id=suggested_user_id)
suggested_users.remove_suggestion(user_id, suggested_user)
@app.task(queue=LOW)
@app.task(queue=LOW, ignore_result=True)
def bulk_remove_instance_task(instance_id):
"""remove a bunch of users from recs"""
for user in models.User.objects.filter(federated_server__id=instance_id):
suggested_users.remove_object_from_related_stores(user)
@app.task(queue=LOW)
@app.task(queue=LOW, ignore_result=True)
def bulk_add_instance_task(instance_id):
"""remove a bunch of users from recs"""
for user in models.User.objects.filter(federated_server__id=instance_id):

13
bookwyrm/telemetry/open_telemetry.py
View file

@ -1,10 +1,19 @@
from opentelemetry import trace
from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import OTLPSpanExporter
from opentelemetry.sdk.trace import TracerProvider
from opentelemetry.sdk.trace.export import BatchSpanProcessor
from opentelemetry.sdk.trace.export import BatchSpanProcessor, ConsoleSpanExporter
from bookwyrm import settings
trace.set_tracer_provider(TracerProvider())
trace.get_tracer_provider().add_span_processor(BatchSpanProcessor(OTLPSpanExporter()))
if settings.OTEL_EXPORTER_CONSOLE:
trace.get_tracer_provider().add_span_processor(
BatchSpanProcessor(ConsoleSpanExporter())
)
else:
trace.get_tracer_provider().add_span_processor(
BatchSpanProcessor(OTLPSpanExporter())
)
def instrumentDjango():

2
bookwyrm/templates/book/book.html
View file

@ -82,6 +82,8 @@
src="{% static "images/no_cover.jpg" %}"
alt=""
aria-hidden="true"
loading="lazy"
decoding="async"
>
<span class="cover-caption">
<span>{{ book.alt_text }}</span>

2
bookwyrm/templates/book/cover_show_modal.html
View file

@ -5,7 +5,7 @@
<div class="modal-background" data-modal-close></div><!-- modal background -->
<div class="modal-card is-align-items-center" role="dialog" aria-modal="true" tabindex="-1" aria-label="{% trans 'Book cover preview' %}">
<div class="cover-container">
<img class="book-cover" src="{% get_media_prefix %}{{ book.cover }}" itemprop="thumbnailUrl" alt="">
<img class="book-cover" src="{% get_media_prefix %}{{ book.cover }}" itemprop="thumbnailUrl" alt="" loading="lazy" decoding="async">
</div>
</div>
<button type="button" data-modal-close class="modal-close is-large" aria-label="{% trans 'Close' %}"></button>

8
bookwyrm/templates/book/edit/edit_book.html
View file

@ -37,6 +37,14 @@
{% endif %}
</header>
{% if form.errors %}
<div class="block">
<p class="notification is-danger is-light">
{% trans "Failed to save book, see errors below for more information." %}
</p>
</div>
{% endif %}
<form
class="block"
{% if book.id %}

2
bookwyrm/templates/email/html_layout.html
View file

@ -2,7 +2,7 @@
<div style="font-family: BlinkMacSystemFont,-apple-system,'Segoe UI',Roboto,Oxygen,Ubuntu,Cantarell,'Fira Sans','Droid Sans','Helvetica Neue',Helvetica,Arial,sans-serif; border-radius: 6px; background-color: #efefef; max-width: 632px;">
<div style="padding: 1rem; overflow: auto;">
<div style="float: left; margin-right: 1rem;">
<a style="color: #3273dc;" href="https://{{ domain }}" style="text-decoration: none;"><img src="{{ logo }}" alt="logo"></a>
<a style="color: #3273dc;" href="https://{{ domain }}" style="text-decoration: none;"><img src="{{ logo }}" alt="logo" loading="lazy" decoding="async"></a>
</div>
<div>
<a style="color: black; text-decoration: none" href="https://{{ domain }}" style="text-decoration: none;"><strong>{{ site_name }}</strong><br>

2
bookwyrm/templates/embed-layout.html
View file

@ -17,7 +17,7 @@
<header class="section py-3">
<a href="/" class="is-flex is-align-items-center">
<img class="image logo is-flex-shrink-0" style="height: 32px" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" alt="{% blocktrans with site_name=site.name %}{{ site_name }} home page{% endblocktrans %}">
<img class="image logo is-flex-shrink-0" style="height: 32px" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" alt="{% blocktrans with site_name=site.name %}{{ site_name }} home page{% endblocktrans %}" loading="lazy" decoding="async">
<span class="title is-5 ml-2">{{ site.name }}</span>
</a>
</header>

2
bookwyrm/templates/feed/layout.html
View file

@ -23,7 +23,7 @@
{% block panel %}{% endblock %}
{% if activities %}
{% include 'snippets/pagination.html' with page=activities path=path anchor="#feed" %}
{% include 'snippets/pagination.html' with page=activities path=path anchor="#feed" mode="chronological" %}
{% endif %}
</div>
</div>

2
bookwyrm/templates/get_started/layout.html
View file

@ -15,6 +15,8 @@
src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}"
aria-hidden="true"
alt="{{ site.name }}"
loading="lazy"
decoding="async"
>
<h1 class="modal-card-title" id="get_started_header">
{% blocktrans %}Welcome to {{ site_name }}!{% endblocktrans %}

32
bookwyrm/templates/hashtag.html Normal file
View file

@ -0,0 +1,32 @@
{% extends "layout.html" %}
{% load i18n %}
{% block title %}{{ hashtag }}{% endblock %}
{% block content %}
<div class="container is-max-desktop">
<section class="block">
<header class="block content has-text-centered">
<h1 class="title">{{ hashtag }}</h1>
<p class="subtitle">
{% blocktrans trimmed with site_name=site.name %}
See tagged statuses in the local {{ site_name }} community
{% endblocktrans %}
</p>
</header>
{% for activity in activities %}
<div class="block">
{% include 'snippets/status/status.html' with status=activity %}
</div>
{% endfor %}
{% if not activities %}
<div class="block">
<p>{% trans "No activities for this hashtag yet!" %}</p>
</div>
{% endif %}
{% include 'snippets/pagination.html' with page=activities path=path %}
</section>
</div>
{% endblock %}

2
bookwyrm/templates/layout.html
View file

@ -28,7 +28,7 @@
{% with notification_count=request.user.unread_notification_count has_unread_mentions=request.user.has_unread_mentions %}
<div class="navbar-brand">
<a class="navbar-item" href="/">
<img class="image logo" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" alt="{% blocktrans with site_name=site.name %}{{ site_name }} home page{% endblocktrans %}">
<img class="image logo" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" alt="{% blocktrans with site_name=site.name %}{{ site_name }} home page{% endblocktrans %}" loading="lazy" decoding="async">
</a>
<form class="navbar-item column is-align-items-start pt-5" action="{% url 'search' %}">
<div class="field has-addons">

2
bookwyrm/templates/notifications/item.html
View file

@ -17,6 +17,8 @@
{% include 'notifications/items/add.html' %}
{% elif notification.notification_type == 'REPORT' %}
{% include 'notifications/items/report.html' %}
{% elif notification.notification_type == 'LINK_DOMAIN' %}
{% include 'notifications/items/link_domain.html' %}
{% elif notification.notification_type == 'INVITE' %}
{% include 'notifications/items/invite.html' %}
{% elif notification.notification_type == 'ACCEPT' %}

20
bookwyrm/templates/notifications/items/link_domain.html Normal file
View file

@ -0,0 +1,20 @@
{% extends 'notifications/items/layout.html' %}
{% load humanize %}
{% load i18n %}
{% block primary_link %}{% spaceless %}
{% url 'settings-link-domain' %}
{% endspaceless %}{% endblock %}
{% block icon %}
<span class="icon icon-warning"></span>
{% endblock %}
{% block description %}
{% url 'settings-link-domain' as path %}
{% blocktrans trimmed count counter=notification.related_link_domains.count with display_count=notification.related_link_domains.count|intcomma %}
A new <a href="{{ path }}">link domain</a> needs review
{% plural %}
{{ display_count }} new <a href="{{ path }}">link domains</a> need moderation
{% endblocktrans %}
{% endblock %}

2
bookwyrm/templates/ostatus/template.html
View file

@ -21,7 +21,7 @@
<nav class="navbar" aria-label="main navigation">
<div class="container">
<div class="navbar-brand">
<img class="image logo navbar-item" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static 'images/logo-small.png' %}{% endif %}" alt="Home page">
<img class="image logo navbar-item" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static 'images/logo-small.png' %}{% endif %}" alt="Home page" loading="lazy" decoding="async">
<h2 class="navbar-item subtitle">{% block heading %}{% endblock %}</h2>
</div>
</div>

2
bookwyrm/templates/setup/layout.html
View file

@ -13,6 +13,8 @@
class="image logo"
src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}"
alt="{% blocktrans with site_name=site.name %}{{ site_name }} home page{% endblocktrans %}"
loading="lazy"
decoding="async"
>
</span>
<div class="navbar-item is-align-items-start pt-5 is-flex-grow-1">

2
bookwyrm/templates/snippets/about.html
View file

@ -3,7 +3,7 @@
<div class="columns">
<div class="column is-narrow is-hidden-mobile">
<figure class="block is-w-{% if size %}{{ size }}{% else %}xl{% endif %}">
<img src="{% if site.logo %}{% get_media_prefix %}{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}" alt="BookWyrm logo">
<img src="{% if site.logo %}{% get_media_prefix %}{{ site.logo }}{% else %}{% static "images/logo.png" %}{% endif %}" alt="BookWyrm logo" loading="lazy" decoding="async">
</figure>
</div>
<div class="content">

2
bookwyrm/templates/snippets/avatar.html
View file

@ -5,4 +5,6 @@
src="{% if user.avatar %}{% get_media_prefix %}{{ user.avatar }}{% else %}{% static "images/default_avi.jpg" %}{% endif %}"
{% if ariaHide %}aria-hidden="true"{% endif %}
alt="{{ user.alt_text }}"
loading="lazy"
decoding="async"
>

2
bookwyrm/templates/snippets/book_cover.html
View file

@ -13,6 +13,8 @@
src="{{ book.cover }}"
itemprop="thumbnailUrl"
alt="{{ book.alt_text|default:'' }}"
loading="lazy"
decoding="async"
>
{% else %}

12
bookwyrm/templates/snippets/pagination.html
View file

@ -9,7 +9,11 @@
{% endif %}>
<span class="icon icon-arrow-left" aria-hidden="true"></span>
{% trans "Older" %}
{% if mode == "chronological" %}
{% trans "Newer" %}
{% else %}
{% trans "Previous" %}
{% endif %}
</a>
<a
@ -20,7 +24,11 @@
aria-hidden="true"
{% endif %}>
{% trans "Newer" %}
{% if mode == "chronological" %}
{% trans "Older" %}
{% else %}
{% trans "Next" %}
{% endif %}
<span class="icon icon-arrow-right" aria-hidden="true"></span>
</a>

2
bookwyrm/templates/snippets/status/content_status.html
View file

@ -133,6 +133,8 @@
alt="{{ attachment.caption }}"
title="{{ attachment.caption }}"
{% endif %}
loading="lazy"
decoding="async"
>
</a>
</figure>

2
bookwyrm/templates/two_factor_auth/two_factor_login.html
View file

@ -15,7 +15,7 @@
<div class="container">
<div class="navbar-brand">
<a class="navbar-item" href="/">
<img class="image logo" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" alt="{% blocktrans with site_name=site.name %}{{ site_name }} home page{% endblocktrans %}">
<img class="image logo" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" alt="{% blocktrans with site_name=site.name %}{{ site_name }} home page{% endblocktrans %}" loading="lazy" decoding="async">
</a>
</div>
</div>

2
bookwyrm/templates/two_factor_auth/two_factor_prompt.html
View file

@ -15,7 +15,7 @@
<div class="container">
<div class="navbar-brand">
<a class="navbar-item" href="/">
<img class="image logo" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" alt="{% blocktrans with site_name=site.name %}{{ site_name }} home page{% endblocktrans %}">
<img class="image logo" src="{% if site.logo_small %}{% get_media_prefix %}{{ site.logo_small }}{% else %}{% static "images/logo-small.png" %}{% endif %}" alt="{% blocktrans with site_name=site.name %}{{ site_name }} home page{% endblocktrans %}" loading="lazy" decoding="async">
</a>
</div>
</div>

2
bookwyrm/templates/user/reviews_comments.html
View file

@ -25,6 +25,6 @@
</div>
{% endif %}
{% include 'snippets/pagination.html' with page=activities path=path %}
{% include 'snippets/pagination.html' with page=activities path=path mode="chronological" %}
</div>
{% endblock %}

4
bookwyrm/templates/user/user.html
View file

@ -87,7 +87,7 @@
{% trans "Back" %}
</span>
</summary>
<div class="dropdown-menu">
<ul
class="dropdown-content"
@ -130,7 +130,7 @@
</div>
{% endif %}
{% include 'snippets/pagination.html' with page=activities path=user.local_path anchor="#feed" %}
{% include 'snippets/pagination.html' with page=activities path=user.local_path anchor="#feed" mode="chronological" %}
</div>
{% endblock %}

9
bookwyrm/tests/activitypub/test_base_activity.py
View file

@ -183,12 +183,21 @@ class BaseActivity(TestCase):
"name": "gerald j. books",
"href": "http://book.com/book",
},
{
"type": "Hashtag",
"name": "#BookClub",
"href": "http://example.com/tags/BookClub",
},
],
)
update_data.to_model(model=models.Status, instance=status)
self.assertEqual(status.mention_users.first(), self.user)
self.assertEqual(status.mention_books.first(), book)
hashtag = models.Hashtag.objects.filter(name="#BookClub").first()
self.assertIsNotNone(hashtag)
self.assertEqual(status.mention_hashtags.first(), hashtag)
@responses.activate
def test_to_model_one_to_many(self, *_):
"""these are reversed relationships, where the secondary object

64
bookwyrm/tests/activitypub/test_note.py Normal file
View file

@ -0,0 +1,64 @@
""" tests functionality specifically for the Note ActivityPub dataclass"""
from unittest.mock import patch
from django.test import TestCase
from bookwyrm import activitypub
from bookwyrm import models
class Note(TestCase):
"""the model-linked ActivityPub dataclass for Note-based types"""
# pylint: disable=invalid-name
def setUp(self):
"""create a shared user"""
with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
"bookwyrm.activitystreams.populate_stream_task.delay"
), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
self.user = models.User.objects.create_user(
"mouse", "mouse@mouse.mouse", "mouseword", local=True, localname="mouse"
)
self.user.remote_id = "https://test-instance.org/user/critic"
self.user.save(broadcast=False, update_fields=["remote_id"])
self.book = models.Edition.objects.create(
title="Test Edition", remote_id="http://book.com/book"
)
def test_to_model_hashtag_postprocess_content(self):
"""test that hashtag links are post-processed and link to local URLs"""
update_data = activitypub.Comment(
id="https://test-instance.org/user/critic/comment/42",
attributedTo=self.user.remote_id,
inReplyToBook=self.book.remote_id,
content="<p>This is interesting "
+ '<a href="https://test-instance.org/hashtag/2" data-mention="hashtag">'
+ "#bookclub</a></p>",
published="2023-02-17T23:12:59.398030+00:00",
to=[],
cc=[],
tag=[
{
"type": "Edition",
"name": "gerald j. books",
"href": "http://book.com/book",
},
{
"type": "Hashtag",
"name": "#BookClub",
"href": "https://test-instance.org/hashtag/2",
},
],
)
instance = update_data.to_model(model=models.Status)
self.assertIsNotNone(instance)
hashtag = models.Hashtag.objects.filter(name="#BookClub").first()
self.assertIsNotNone(hashtag)
self.assertEqual(
instance.content,
"<p>This is interesting "
+ f'<a href="{hashtag.remote_id}" data-mention="hashtag">'
+ "#bookclub</a></p>",
)

38
bookwyrm/tests/activitystreams/test_abstractstream.py
View file

@ -53,18 +53,18 @@ class Activitystreams(TestCase):
def test_activitystream_class_ids(self, *_):
"""the abstract base class for stream objects"""
self.assertEqual(
self.test_stream.stream_id(self.local_user),
self.test_stream.stream_id(self.local_user.id),
f"{self.local_user.id}-test",
)
self.assertEqual(
self.test_stream.unread_id(self.local_user),
self.test_stream.unread_id(self.local_user.id),
f"{self.local_user.id}-test-unread",
)
def test_unread_by_status_type_id(self, *_):
"""stream for status type"""
self.assertEqual(
self.test_stream.unread_by_status_type_id(self.local_user),
self.test_stream.unread_by_status_type_id(self.local_user.id),
f"{self.local_user.id}-test-unread-by-type",
)
@ -118,9 +118,9 @@ class Activitystreams(TestCase):
)
users = self.test_stream.get_audience(status)
# remote users don't have feeds
self.assertFalse(self.remote_user in users)
self.assertTrue(self.local_user in users)
self.assertTrue(self.another_user in users)
self.assertFalse(self.remote_user.id in users)
self.assertTrue(self.local_user.id in users)
self.assertTrue(self.another_user.id in users)
def test_abstractstream_get_audience_direct(self, *_):
"""get a list of users that should see a status"""
@ -141,9 +141,9 @@ class Activitystreams(TestCase):
)
status.mention_users.add(self.local_user)
users = self.test_stream.get_audience(status)
self.assertTrue(self.local_user in users)
self.assertFalse(self.another_user in users)
self.assertFalse(self.remote_user in users)
self.assertTrue(self.local_user.id in users)
self.assertFalse(self.another_user.id in users)
self.assertFalse(self.remote_user.id in users)
def test_abstractstream_get_audience_followers_remote_user(self, *_):
"""get a list of users that should see a status"""
@ -153,7 +153,7 @@ class Activitystreams(TestCase):
privacy="followers",
)
users = self.test_stream.get_audience(status)
self.assertFalse(users.exists())
self.assertEqual(users, [])
def test_abstractstream_get_audience_followers_self(self, *_):
"""get a list of users that should see a status"""
@ -164,9 +164,9 @@ class Activitystreams(TestCase):
book=self.book,
)
users = self.test_stream.get_audience(status)
self.assertTrue(self.local_user in users)
self.assertFalse(self.another_user in users)
self.assertFalse(self.remote_user in users)
self.assertTrue(self.local_user.id in users)
self.assertFalse(self.another_user.id in users)
self.assertFalse(self.remote_user.id in users)
def test_abstractstream_get_audience_followers_with_mention(self, *_):
"""get a list of users that should see a status"""
@ -179,9 +179,9 @@ class Activitystreams(TestCase):
status.mention_users.add(self.local_user)
users = self.test_stream.get_audience(status)
self.assertTrue(self.local_user in users)
self.assertFalse(self.another_user in users)
self.assertFalse(self.remote_user in users)
self.assertTrue(self.local_user.id in users)
self.assertFalse(self.another_user.id in users)
self.assertFalse(self.remote_user.id in users)
def test_abstractstream_get_audience_followers_with_relationship(self, *_):
"""get a list of users that should see a status"""
@ -193,6 +193,6 @@ class Activitystreams(TestCase):
book=self.book,
)
users = self.test_stream.get_audience(status)
self.assertFalse(self.local_user in users)
self.assertFalse(self.another_user in users)
self.assertFalse(self.remote_user in users)
self.assertFalse(self.local_user.id in users)
self.assertFalse(self.another_user.id in users)
self.assertFalse(self.remote_user.id in users)

29
bookwyrm/tests/activitystreams/test_booksstream.py
View file

@ -1,4 +1,6 @@
""" testing activitystreams """
import itertools
from unittest.mock import patch
from django.test import TestCase
from bookwyrm import activitystreams, models
@ -69,12 +71,29 @@ class Activitystreams(TestCase):
shelf=self.local_user.shelf_set.first(),
book=self.book,
)
class RedisMockCounter:
"""keep track of calls to mock redis store"""
calls = []
def bulk_add_objects_to_store(self, objs, store):
"""keep track of bulk_add_objects_to_store calls"""
self.calls.append((objs, store))
redis_mock_counter = RedisMockCounter()
with patch(
"bookwyrm.activitystreams.BooksStream.bulk_add_objects_to_store"
) as redis_mock:
redis_mock.side_effect = redis_mock_counter.bulk_add_objects_to_store
activitystreams.BooksStream().add_book_statuses(self.local_user, self.book)
args = redis_mock.call_args[0]
queryset = args[0]
self.assertEqual(queryset.count(), 1)
self.assertTrue(status in queryset)
self.assertEqual(args[1], f"{self.local_user.id}-books")
self.assertEqual(sum(map(lambda x: x[0].count(), redis_mock_counter.calls)), 1)
self.assertTrue(
status
in itertools.chain.from_iterable(
map(lambda x: x[0], redis_mock_counter.calls)
)
)
for call in redis_mock_counter.calls:
self.assertEqual(call[1], f"{self.local_user.id}-books")

10
bookwyrm/tests/activitystreams/test_homestream.py
View file

@ -44,7 +44,7 @@ class Activitystreams(TestCase):
user=self.remote_user, content="hi", privacy="public"
)
users = activitystreams.HomeStream().get_audience(status)
self.assertFalse(users.exists())
self.assertEqual(users, [])
def test_homestream_get_audience_with_mentions(self, *_):
"""get a list of users that should see a status"""
@ -53,8 +53,8 @@ class Activitystreams(TestCase):
)
status.mention_users.add(self.local_user)
users = activitystreams.HomeStream().get_audience(status)
self.assertFalse(self.local_user in users)
self.assertFalse(self.another_user in users)
self.assertFalse(self.local_user.id in users)
self.assertFalse(self.another_user.id in users)
def test_homestream_get_audience_with_relationship(self, *_):
"""get a list of users that should see a status"""
@ -63,5 +63,5 @@ class Activitystreams(TestCase):
user=self.remote_user, content="hi", privacy="public"
)
users = activitystreams.HomeStream().get_audience(status)
self.assertTrue(self.local_user in users)
self.assertFalse(self.another_user in users)
self.assertTrue(self.local_user.id in users)
self.assertFalse(self.another_user.id in users)

10
bookwyrm/tests/activitystreams/test_localstream.py
View file

@ -54,8 +54,8 @@ class Activitystreams(TestCase):
user=self.local_user, content="hi", privacy="public"
)
users = activitystreams.LocalStream().get_audience(status)
self.assertTrue(self.local_user in users)
self.assertTrue(self.another_user in users)
self.assertTrue(self.local_user.id in users)
self.assertTrue(self.another_user.id in users)
def test_localstream_get_audience_unlisted(self, *_):
"""get a list of users that should see a status"""
@ -88,7 +88,7 @@ class Activitystreams(TestCase):
)
# yes book, yes audience
audience = activitystreams.BooksStream().get_audience(status)
self.assertTrue(self.local_user in audience)
self.assertTrue(self.local_user.id in audience)
def test_localstream_get_audience_books_book_field(self, *_):
"""get a list of users that should see a status"""
@ -102,7 +102,7 @@ class Activitystreams(TestCase):
)
# yes book, yes audience
audience = activitystreams.BooksStream().get_audience(status)
self.assertTrue(self.local_user in audience)
self.assertTrue(self.local_user.id in audience)
def test_localstream_get_audience_books_alternate_edition(self, *_):
"""get a list of users that should see a status"""
@ -119,7 +119,7 @@ class Activitystreams(TestCase):
)
# yes book, yes audience
audience = activitystreams.BooksStream().get_audience(status)
self.assertTrue(self.local_user in audience)
self.assertTrue(self.local_user.id in audience)
def test_localstream_get_audience_books_non_public(self, *_):
"""get a list of users that should see a status"""

2
bookwyrm/tests/models/test_status_model.py
View file

@ -265,7 +265,7 @@ class Status(TestCase):
self.assertEqual(activity["attachment"][0]["type"], "Document")
self.assertTrue(
re.match(
r"https:\/\/your.domain.here\/images\/covers\/test_[A-z0-9]+.jpg",
r"https:\/\/your.domain.here\/images\/covers\/test(_[A-z0-9]+)?.jpg",
activity["attachment"][0]["url"],
)
)

27
bookwyrm/tests/test_signing.py
View file

@ -95,7 +95,8 @@ class Signature(TestCase):
def test_correct_signature(self):
"""this one should just work"""
response = self.send_test_request(sender=self.mouse)
with patch("bookwyrm.models.relationship.UserFollowRequest.accept"):
response = self.send_test_request(sender=self.mouse)
self.assertEqual(response.status_code, 200)
def test_wrong_signature(self):
@ -124,8 +125,12 @@ class Signature(TestCase):
)
with patch("bookwyrm.models.user.get_remote_reviews.delay"):
response = self.send_test_request(sender=self.fake_remote)
with patch(
"bookwyrm.models.relationship.UserFollowRequest.accept"
) as accept_mock:
response = self.send_test_request(sender=self.fake_remote)
self.assertEqual(response.status_code, 200)
self.assertTrue(accept_mock.called)
@responses.activate
def test_key_needs_refresh(self):
@ -148,16 +153,28 @@ class Signature(TestCase):
with patch("bookwyrm.models.user.get_remote_reviews.delay"):
# Key correct:
response = self.send_test_request(sender=self.fake_remote)
with patch(
"bookwyrm.models.relationship.UserFollowRequest.accept"
) as accept_mock:
response = self.send_test_request(sender=self.fake_remote)
self.assertEqual(response.status_code, 200)
self.assertTrue(accept_mock.called)
# Old key is cached, so still works:
response = self.send_test_request(sender=self.fake_remote)
with patch(
"bookwyrm.models.relationship.UserFollowRequest.accept"
) as accept_mock:
response = self.send_test_request(sender=self.fake_remote)
self.assertEqual(response.status_code, 200)
self.assertTrue(accept_mock.called)
# Try with new key:
response = self.send_test_request(sender=new_sender)
with patch(
"bookwyrm.models.relationship.UserFollowRequest.accept"
) as accept_mock:
response = self.send_test_request(sender=new_sender)
self.assertEqual(response.status_code, 200)
self.assertTrue(accept_mock.called)
# Now the old key will fail:
response = self.send_test_request(sender=self.fake_remote)

13
bookwyrm/tests/views/preferences/test_export.py
View file

@ -1,7 +1,7 @@
""" test for app action functionality """
from unittest.mock import patch
from django.http import StreamingHttpResponse
from django.http import HttpResponse
from django.test import TestCase
from django.test.client import RequestFactory
@ -57,13 +57,12 @@ class ExportViews(TestCase):
request = self.factory.post("")
request.user = self.local_user
export = views.Export.as_view()(request)
self.assertIsInstance(export, StreamingHttpResponse)
self.assertIsInstance(export, HttpResponse)
self.assertEqual(export.status_code, 200)
result = list(export.streaming_content)
# pylint: disable=line-too-long
self.assertEqual(
result[0],
b"title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,rating,review_name,review_cw,review_content\r\n",
export.content,
b"title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,rating,review_name,review_cw,review_content\r\nTest Book,,"
+ self.book.remote_id.encode("utf-8")
+ b",,,,,beep,,,,,,123456789X,9781234567890,,,,,\r\n",
)
expected = f"Test Book,,{self.book.remote_id},,,,,beep,,,,,,123456789X,9781234567890,,,,,\r\n"
self.assertEqual(result[1].decode("utf-8"), expected)

197
bookwyrm/tests/views/test_hashtag.py Normal file
View file

@ -0,0 +1,197 @@
""" tests for hashtag view """
from unittest.mock import patch
from django.contrib.auth.models import AnonymousUser
from django.http import Http404
from django.template.response import TemplateResponse
from django.test import TestCase
from django.test.client import RequestFactory
from bookwyrm import models, views
from bookwyrm.tests.validate_html import validate_html
class HashtagView(TestCase):
"""hashtag view"""
def setUp(self):
self.factory = RequestFactory()
with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch(
"bookwyrm.activitystreams.populate_stream_task.delay"
), patch("bookwyrm.lists_stream.populate_lists_task.delay"):
self.local_user = models.User.objects.create_user(
"mouse@local.com",
"mouse@mouse.com",
"mouseword",
local=True,
localname="mouse",
remote_id="https://example.com/users/mouse",
)
self.follower_user = models.User.objects.create_user(
"follower@local.com",
"follower@email.com",
"followerword",
local=True,
localname="follower",
remote_id="https://example.com/users/follower",
)
self.local_user.followers.add(self.follower_user)
self.other_user = models.User.objects.create_user(
"other@local.com",
"other@email.com",
"otherword",
local=True,
localname="other",
remote_id="https://example.com/users/other",
)
self.work = models.Work.objects.create(title="Test Work")
self.book = models.Edition.objects.create(
title="Example Edition",
remote_id="https://example.com/book/1",
parent_work=self.work,
)
self.hashtag_bookclub = models.Hashtag.objects.create(name="#BookClub")
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.activitystreams.add_status_task.delay"):
self.statuses_bookclub = [
models.Comment.objects.create(
book=self.book, user=self.local_user, content="#BookClub"
),
]
for status in self.statuses_bookclub:
status.mention_hashtags.add(self.hashtag_bookclub)
self.anonymous_user = AnonymousUser
self.anonymous_user.is_authenticated = False
models.SiteSettings.objects.create()
def test_hashtag_page(self):
"""just make sure it loads"""
view = views.Hashtag.as_view()
request = self.factory.get("")
request.user = self.local_user
result = view(request, self.hashtag_bookclub.id)
self.assertIsInstance(result, TemplateResponse)
validate_html(result.render())
self.assertEqual(result.status_code, 200)
self.assertEqual(len(result.context_data["activities"]), 1)
def test_privacy_direct(self):
"""ensure statuses with privacy set to direct are always filtered out"""
view = views.Hashtag.as_view()
request = self.factory.get("")
hashtag = models.Hashtag.objects.create(name="#test")
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.activitystreams.add_status_task.delay"):
status = models.Comment.objects.create(
user=self.local_user, book=self.book, content="#test", privacy="direct"
)
status.mention_hashtags.add(hashtag)
for user in [
self.local_user,
self.follower_user,
self.other_user,
self.anonymous_user,
]:
request.user = user
result = view(request, hashtag.id)
self.assertNotIn(status, result.context_data["activities"])
def test_privacy_unlisted(self):
"""ensure statuses with privacy set to unlisted are always filtered out"""
view = views.Hashtag.as_view()
request = self.factory.get("")
hashtag = models.Hashtag.objects.create(name="#test")
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.activitystreams.add_status_task.delay"):
status = models.Comment.objects.create(
user=self.local_user,
book=self.book,
content="#test",
privacy="unlisted",
)
status.mention_hashtags.add(hashtag)
for user in [
self.local_user,
self.follower_user,
self.other_user,
self.anonymous_user,
]:
request.user = user
result = view(request, hashtag.id)
self.assertNotIn(status, result.context_data["activities"])
def test_privacy_following(self):
"""ensure only creator and followers can see statuses with privacy
set to followers"""
view = views.Hashtag.as_view()
request = self.factory.get("")
hashtag = models.Hashtag.objects.create(name="#test")
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.activitystreams.add_status_task.delay"):
status = models.Comment.objects.create(
user=self.local_user,
book=self.book,
content="#test",
privacy="followers",
)
status.mention_hashtags.add(hashtag)
for user in [self.local_user, self.follower_user]:
request.user = user
result = view(request, hashtag.id)
self.assertIn(status, result.context_data["activities"])
for user in [self.other_user, self.anonymous_user]:
request.user = user
result = view(request, hashtag.id)
self.assertNotIn(status, result.context_data["activities"])
def test_not_found(self):
"""make sure 404 is rendered"""
view = views.Hashtag.as_view()
request = self.factory.get("")
request.user = self.local_user
with self.assertRaises(Http404):
view(request, 42)
def test_empty(self):
"""hashtag without any statuses should still render"""
view = views.Hashtag.as_view()
request = self.factory.get("")
request.user = self.local_user
hashtag_empty = models.Hashtag.objects.create(name="#empty")
result = view(request, hashtag_empty.id)
self.assertIsInstance(result, TemplateResponse)
validate_html(result.render())
self.assertEqual(result.status_code, 200)
self.assertEqual(len(result.context_data["activities"]), 0)
def test_logged_out(self):
"""make sure it loads all activities"""
view = views.Hashtag.as_view()
request = self.factory.get("")
request.user = self.anonymous_user
result = view(request, self.hashtag_bookclub.id)
self.assertIsInstance(result, TemplateResponse)
validate_html(result.render())
self.assertEqual(result.status_code, 200)
self.assertEqual(len(result.context_data["activities"]), 1)

68
bookwyrm/tests/views/test_status.py
View file

@ -6,7 +6,7 @@ from django.test import TestCase, TransactionTestCase
from django.test.client import RequestFactory
from bookwyrm import forms, models, views
from bookwyrm.views.status import find_mentions
from bookwyrm.views.status import find_mentions, find_or_create_hashtags
from bookwyrm.settings import DOMAIN
from bookwyrm.tests.validate_html import validate_html
@ -95,6 +95,7 @@ class StatusViews(TestCase):
local=True,
localname="nutria",
)
self.existing_hashtag = models.Hashtag.objects.create(name="#existing")
with patch("bookwyrm.models.user.set_remote_server"):
self.remote_user = models.User.objects.create_user(
"rat",
@ -333,6 +334,71 @@ class StatusViews(TestCase):
result = find_mentions(self.local_user, "@beep@beep.com")
self.assertEqual(result, {})
def test_create_status_hashtags(self, *_):
"""#mention a hashtag in a post"""
view = views.CreateStatus.as_view()
form = forms.CommentForm(
{
"content": "this is an #EXISTING hashtag but all uppercase, "
+ "this one is #NewTag.",
"user": self.local_user.id,
"book": self.book.id,
"privacy": "public",
}
)
request = self.factory.post("", form.data)
request.user = self.local_user
view(request, "comment")
status = models.Status.objects.get()
hashtags = models.Hashtag.objects.all()
self.assertEqual(len(hashtags), 2)
self.assertEqual(list(status.mention_hashtags.all()), list(hashtags))
hashtag_exising = models.Hashtag.objects.filter(name="#existing").first()
hashtag_new = models.Hashtag.objects.filter(name="#NewTag").first()
self.assertEqual(
status.content,
"<p>this is an "
+ f'<a href="{hashtag_exising.remote_id}" data-mention="hashtag">'
+ "#EXISTING</a> hashtag but all uppercase, this one is "
+ f'<a href="{hashtag_new.remote_id}" data-mention="hashtag">'
+ "#NewTag</a>.</p>",
)
def test_find_or_create_hashtags(self, *_):
"""detect and look up #hashtags"""
result = find_or_create_hashtags("no hashtag to be found here")
self.assertEqual(result, {})
result = find_or_create_hashtags("#existing")
self.assertEqual(result["#existing"], self.existing_hashtag)
result = find_or_create_hashtags("leading text #existing")
self.assertEqual(result["#existing"], self.existing_hashtag)
result = find_or_create_hashtags("leading #existing trailing")
self.assertEqual(result["#existing"], self.existing_hashtag)
self.assertIsNone(models.Hashtag.objects.filter(name="new").first())
result = find_or_create_hashtags("leading #new trailing")
new_hashtag = models.Hashtag.objects.filter(name="#new").first()
self.assertIsNotNone(new_hashtag)
self.assertEqual(result["#new"], new_hashtag)
result = find_or_create_hashtags("leading #existing #new trailing")
self.assertEqual(result["#existing"], self.existing_hashtag)
self.assertEqual(result["#new"], new_hashtag)
result = find_or_create_hashtags("#Braunbär")
hashtag = models.Hashtag.objects.filter(name="#Braunbär").first()
self.assertEqual(result["#Braunbär"], hashtag)
result = find_or_create_hashtags("#ひぐま")
hashtag = models.Hashtag.objects.filter(name="#ひぐま").first()
self.assertEqual(result["#ひぐま"], hashtag)
def test_format_links_simple_url(self, *_):
"""find and format urls into a tags"""
url = "http://www.fish.com/"

12
bookwyrm/urls.py
View file

@ -356,6 +356,15 @@ urlpatterns = [
name="notifications",
),
re_path(r"^directory/?", views.Directory.as_view(), name="directory"),
# hashtag
re_path(
r"^hashtag/(?P<hashtag_id>\d+)/?$", views.Hashtag.as_view(), name="hashtag"
),
re_path(
rf"^hashtag/(?P<hashtag_id>\d+){regex.SLUG}/?$",
views.Hashtag.as_view(),
name="hashtag",
),
# Get started
re_path(
r"^get-started/profile/?$",
@ -764,3 +773,6 @@ urlpatterns = [
),
path("guided-tour/<tour>", views.toggle_guided_tour),
] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT)
# pylint: disable=invalid-name
handler500 = "bookwyrm.views.server_error"

1
bookwyrm/utils/regex.py
View file

@ -7,5 +7,6 @@ USERNAME = rf"{LOCALNAME}(@{DOMAIN})?"
STRICT_USERNAME = rf"(\B{STRICT_LOCALNAME}(@{DOMAIN})?\b)"
FULL_USERNAME = rf"{LOCALNAME}@{DOMAIN}\b"
SLUG = r"/s/(?P<slug>[-_a-z0-9]*)"
HASHTAG = r"(#[^!@#$%^&*(),.?\":{}|<>\s]+)"
# should match (BookWyrm/1.0.0; or (BookWyrm/99.1.2;
BOOKWYRM_USER_AGENT = r"\(BookWyrm/[0-9]+\.[0-9]+\.[0-9]+;"

2
bookwyrm/utils/sanitizer.py
View file

@ -21,6 +21,6 @@ def clean(input_text):
"ol",
"li",
],
attributes=["href", "rel", "src", "alt"],
attributes=["href", "rel", "src", "alt", "data-mention"],
strip=True,
)

2
bookwyrm/views/__init__.py
View file

@ -130,6 +130,7 @@ from .group import (
accept_membership,
reject_membership,
)
from .hashtag import Hashtag
from .inbox import Inbox
from .interaction import Favorite, Unfavorite, Boost, Unboost
from .isbn import Isbn
@ -164,3 +165,4 @@ from .annual_summary import (
summary_add_key,
summary_revoke_key,
)
from .server_error import server_error

4
bookwyrm/views/admin/dashboard.py
View file

@ -16,7 +16,6 @@ from csp.decorators import csp_update
from bookwyrm import models, settings
from bookwyrm.connectors.abstract_connector import get_data
from bookwyrm.connectors.connector_manager import ConnectorException
from bookwyrm.utils import regex
@ -61,6 +60,7 @@ class Dashboard(View):
)
# check version
try:
release = get_data(settings.RELEASE_API, timeout=3)
available_version = release.get("tag_name", None)
@ -69,7 +69,7 @@ class Dashboard(View):
):
data["current_version"] = settings.VERSION
data["available_version"] = available_version
except ConnectorException:
except: # pylint: disable= bare-except
pass
return TemplateResponse(request, "settings/dashboard/dashboard.html", data)

54
bookwyrm/views/hashtag.py Normal file
View file

@ -0,0 +1,54 @@
""" listing statuses for a given hashtag """
from django.core.paginator import Paginator
from django.db.models import Q
from django.views import View
from django.shortcuts import get_object_or_404
from django.template.response import TemplateResponse
from bookwyrm import models
from bookwyrm.settings import PAGE_LENGTH
from bookwyrm.views.helpers import maybe_redirect_local_path
# pylint: disable= no-self-use
class Hashtag(View):
"""listing statuses for a given hashtag"""
# pylint: disable=unused-argument
def get(self, request, hashtag_id, slug=None):
"""show hashtag with related statuses"""
hashtag = get_object_or_404(models.Hashtag, id=hashtag_id)
if redirect_local_path := maybe_redirect_local_path(request, hashtag):
return redirect_local_path
activities = (
models.Status.privacy_filter(
request.user,
)
.filter(
Q(mention_hashtags=hashtag),
)
.exclude(
privacy__in=["direct", "unlisted"],
)
.select_related(
"user",
"reply_parent",
"review__book",
"comment__book",
"quotation__book",
)
.prefetch_related(
"mention_books",
"mention_users",
"attachments",
)
)
paginated = Paginator(activities, PAGE_LENGTH)
data = {
"hashtag": hashtag.name,
"activities": paginated.get_page(request.GET.get("page", 1)),
}
return TemplateResponse(request, "hashtag.html", data)

17
bookwyrm/views/inbox.py
View file

@ -64,7 +64,7 @@ class Inbox(View):
high = ["Follow", "Accept", "Reject", "Block", "Unblock", "Undo"]
priority = HIGH if activity_json["type"] in high else MEDIUM
activity_task.apply_async(args=(activity_json,), queue=priority)
sometimes_async_activity_task(activity_json, queue=priority)
return HttpResponse()
@ -102,7 +102,20 @@ def raise_is_blocked_activity(activity_json):
raise PermissionDenied()
@app.task(queue=MEDIUM)
def sometimes_async_activity_task(activity_json, queue=MEDIUM):
"""Sometimes we can effectively respond to a request without queuing a new task,
and whever that is possible, we should do it."""
activity = activitypub.parse(activity_json)
# try resolving this activity without making any http requests
try:
activity.action(allow_external_connections=False)
except activitypub.ActivitySerializerError:
# if that doesn't work, run it asynchronously
activity_task.apply_async(args=(activity_json,), queue=queue)
@app.task(queue=MEDIUM, ignore_result=True)
def activity_task(activity_json):
"""do something with this json we think is legit"""
# lets see if the activitypub module can make sense of this json

123
bookwyrm/views/preferences/export.py
View file

@ -1,9 +1,10 @@
""" Let users export their book data """
import csv
import io
from django.contrib.auth.decorators import login_required
from django.db.models import Q
from django.http import StreamingHttpResponse
from django.http import HttpResponse
from django.template.response import TemplateResponse
from django.views import View
from django.utils.decorators import method_decorator
@ -20,76 +21,66 @@ class Export(View):
return TemplateResponse(request, "preferences/export.html")
def post(self, request):
"""Streaming the csv file of a user's book data"""
data = (
models.Edition.viewer_aware_objects(request.user)
.filter(
Q(shelves__user=request.user)
| Q(readthrough__user=request.user)
| Q(review__user=request.user)
| Q(comment__user=request.user)
| Q(quotation__user=request.user)
)
.distinct()
"""Download the csv file of a user's book data"""
books = models.Edition.viewer_aware_objects(request.user)
books_shelves = books.filter(Q(shelves__user=request.user)).distinct()
books_readthrough = books.filter(Q(readthrough__user=request.user)).distinct()
books_review = books.filter(Q(review__user=request.user)).distinct()
books_comment = books.filter(Q(comment__user=request.user)).distinct()
books_quotation = books.filter(Q(quotation__user=request.user)).distinct()
books = set(
list(books_shelves)
+ list(books_readthrough)
+ list(books_review)
+ list(books_comment)
+ list(books_quotation)
)
generator = csv_row_generator(data, request.user)
csv_string = io.StringIO()
writer = csv.writer(csv_string)
pseudo_buffer = Echo()
writer = csv.writer(pseudo_buffer)
# for testing, if you want to see the results in the browser:
# from django.http import JsonResponse
# return JsonResponse(list(generator), safe=False)
return StreamingHttpResponse(
(writer.writerow(row) for row in generator),
deduplication_fields = [
f.name
for f in models.Edition._meta.get_fields() # pylint: disable=protected-access
if getattr(f, "deduplication_field", False)
]
fields = (
["title", "author_text"]
+ deduplication_fields
+ ["rating", "review_name", "review_cw", "review_content"]
)
writer.writerow(fields)
for book in books:
# I think this is more efficient than doing a subquery in the view? but idk
review_rating = (
models.Review.objects.filter(
user=request.user, book=book, rating__isnull=False
)
.order_by("-published_date")
.first()
)
book.rating = review_rating.rating if review_rating else None
review = (
models.Review.objects.filter(
user=request.user, book=book, content__isnull=False
)
.order_by("-published_date")
.first()
)
if review:
book.review_name = review.name
book.review_cw = review.content_warning
book.review_content = review.raw_content
writer.writerow([getattr(book, field, "") or "" for field in fields])
return HttpResponse(
csv_string.getvalue(),
content_type="text/csv",
headers={
"Content-Disposition": 'attachment; filename="bookwyrm-export.csv"'
},
)
def csv_row_generator(books, user):
"""generate a csv entry for the user's book"""
deduplication_fields = [
f.name
for f in models.Edition._meta.get_fields() # pylint: disable=protected-access
if getattr(f, "deduplication_field", False)
]
fields = (
["title", "author_text"]
+ deduplication_fields
+ ["rating", "review_name", "review_cw", "review_content"]
)
yield fields
for book in books:
# I think this is more efficient than doing a subquery in the view? but idk
review_rating = (
models.Review.objects.filter(user=user, book=book, rating__isnull=False)
.order_by("-published_date")
.first()
)
book.rating = review_rating.rating if review_rating else None
review = (
models.Review.objects.filter(user=user, book=book, content__isnull=False)
.order_by("-published_date")
.first()
)
if review:
book.review_name = review.name
book.review_cw = review.content_warning
book.review_content = review.raw_content
yield [getattr(book, field, "") or "" for field in fields]
class Echo:
"""An object that implements just the write method of the file-like
interface. (https://docs.djangoproject.com/en/3.2/howto/outputting-csv/)
"""
# pylint: disable=no-self-use
def write(self, value):
"""Write the value by returning it, instead of storing in a buffer."""
return value

8
bookwyrm/views/server_error.py Normal file
View file

@ -0,0 +1,8 @@
"""custom 500 handler to enable context processors"""
from django.template.response import TemplateResponse
def server_error(request):
"""server error page"""
return TemplateResponse(request, "500.html")

45
bookwyrm/views/status.py
View file

@ -115,6 +115,19 @@ class CreateStatus(View):
if status.reply_parent:
status.mention_users.add(status.reply_parent.user)
# inspect the text for hashtags
for (mention_text, mention_hashtag) in find_or_create_hashtags(content).items():
# add them to status mentions fk
status.mention_hashtags.add(mention_hashtag)
# turn the mention into a link
content = re.sub(
rf"{mention_text}\b(?!@)",
rf'<a href="{mention_hashtag.remote_id}" data-mention="hashtag">'
+ rf"{mention_text}</a>",
content,
)
# deduplicate mentions
status.mention_users.set(set(status.mention_users.all()))
@ -237,6 +250,38 @@ def find_mentions(user, content):
return username_dict
def find_or_create_hashtags(content):
"""detect #hashtags in raw status content
it stores hashtags case-sensitive, but ensures that an existing
hashtag with different case are found and re-used. for example,
an existing #BookWyrm hashtag will be found and used even if the
status content is using #bookwyrm.
"""
if not content:
return {}
found_hashtags = {t.lower(): t for t in re.findall(regex.HASHTAG, content)}
if len(found_hashtags) == 0:
return {}
known_hashtags = {
t.name.lower(): t
for t in models.Hashtag.objects.filter(
Q(name__in=found_hashtags.keys())
).distinct()
}
not_found = found_hashtags.keys() - known_hashtags.keys()
for lower_name in not_found:
tag_name = found_hashtags[lower_name]
mention_hashtag = models.Hashtag(name=tag_name)
mention_hashtag.save()
known_hashtags[lower_name] = mention_hashtag
return {found_hashtags[k]: v for k, v in known_hashtags.items()}
def format_links(content):
"""detect and format links"""
validator = URLValidator()