zedfrigg/bookwyrm
1
0
Fork 0
Code Activity

Merge branch 'main' into tour

Browse source 
Merging in latest changes from main, since I got covid and missed a few weeks.
This commit is contained in:
Hugh Rundle 2022-07-03 16:18:50 +10:00
commit 0e9dc66ffa
56 changed files with 1495 additions and 715 deletions
Download patch file Download diff file Expand all files Collapse all files

9
bookwyrm/connectors/connector_manager.py
View file

@ -152,6 +152,15 @@ def load_more_data(connector_id, book_id):
connector.expand_book_data(book)
@app.task(queue="low_priority")
def create_edition_task(connector_id, work_id, data):
"""separate task for each of the 10,000 editions of LoTR"""
connector_info = models.Connector.objects.get(id=connector_id)
connector = load_connector(connector_info)
work = models.Work.objects.select_subclasses().get(id=work_id)
connector.create_edition_from_data(work, data)
def load_connector(connector_info):
"""instantiate the connector class"""
connector = importlib.import_module(

18
bookwyrm/connectors/inventaire.py
View file

@ -5,7 +5,7 @@ from bookwyrm import models
from bookwyrm.book_search import SearchResult
from .abstract_connector import AbstractConnector, Mapping
from .abstract_connector import get_data
from .connector_manager import ConnectorException
from .connector_manager import ConnectorException, create_edition_task
class Connector(AbstractConnector):
@ -156,12 +156,16 @@ class Connector(AbstractConnector):
for edition_uri in edition_options.get("uris"):
remote_id = self.get_remote_id(edition_uri)
try:
data = self.get_book_data(remote_id)
except ConnectorException:
# who, indeed, knows
continue
self.create_edition_from_data(work, data)
create_edition_task.delay(self.connector.id, work.id, remote_id)
def create_edition_from_data(self, work, edition_data, instance=None):
"""pass in the url as data and then call the version in abstract connector"""
try:
data = self.get_book_data(edition_data)
except ConnectorException:
# who, indeed, knows
return
super().create_edition_from_data(work, data, instance=instance)
def get_cover_url(self, cover_blob, *_):
"""format the relative cover url into an absolute one:

4
bookwyrm/connectors/openlibrary.py
View file

@ -5,7 +5,7 @@ from bookwyrm import models
from bookwyrm.book_search import SearchResult
from .abstract_connector import AbstractConnector, Mapping
from .abstract_connector import get_data, infer_physical_format, unique_physical_format
from .connector_manager import ConnectorException
from .connector_manager import ConnectorException, create_edition_task
from .openlibrary_languages import languages
@ -210,7 +210,7 @@ class Connector(AbstractConnector):
# does this edition have ANY interesting data?
if ignore_edition(edition_data):
continue
self.create_edition_from_data(work, edition_data)
create_edition_task.delay(self.connector.id, work.id, edition_data)
def ignore_edition(edition_data):

2
bookwyrm/importers/calibre_import.py
View file

@ -24,5 +24,5 @@ class CalibreImporter(Importer):
super().__init__(*args, **kwargs)
def get_shelf(self, normalized_row):
# Calibre export does not indicate which shelf to use. Go with a default one for now
# Calibre export does not indicate which shelf to use. Use a default one for now
return Shelf.TO_READ

23
bookwyrm/lists_stream.py
View file

@ -114,12 +114,20 @@ class ListsStream(RedisStore):
@receiver(signals.post_save, sender=models.List)
# pylint: disable=unused-argument
def add_list_on_create(sender, instance, created, *args, **kwargs):
"""add newly created lists streamsstreams"""
if not created:
def add_list_on_create(sender, instance, created, *args, update_fields=None, **kwargs):
"""add newly created lists streams"""
if created:
# when creating new things, gotta wait on the transaction
transaction.on_commit(lambda: add_list_on_create_command(instance.id))
return
# when creating new things, gotta wait on the transaction
transaction.on_commit(lambda: add_list_on_create_command(instance.id))
# if update_fields was specified, we can check if privacy was updated, but if
# it wasn't specified (ie, by an activitypub update), there's no way to know
if update_fields and "privacy" not in update_fields:
return
# the privacy may have changed, so we need to re-do the whole thing
remove_list_task.delay(instance.id, re_add=True)
@receiver(signals.post_delete, sender=models.List)
@ -217,7 +225,7 @@ def populate_lists_task(user_id):
@app.task(queue=MEDIUM)
def remove_list_task(list_id):
def remove_list_task(list_id, re_add=False):
"""remove a list from any stream it might be in"""
stores = models.User.objects.filter(local=True, is_active=True).values_list(
"id", flat=True
@ -227,6 +235,9 @@ def remove_list_task(list_id):
stores = [ListsStream().stream_id(idx) for idx in stores]
ListsStream().remove_object_from_related_stores(list_id, stores=stores)
if re_add:
add_list_task.delay(list_id)
@app.task(queue=HIGH)
def add_list_task(list_id):

4
bookwyrm/models/list.py
View file

@ -129,7 +129,7 @@ class List(OrderedCollectionMixin, BookWyrmModel):
"""on save, update embed_key and avoid clash with existing code"""
if not self.embed_key:
self.embed_key = uuid.uuid4()
return super().save(*args, **kwargs)
super().save(*args, **kwargs)
class ListItem(CollectionItemMixin, BookWyrmModel):
@ -156,7 +156,7 @@ class ListItem(CollectionItemMixin, BookWyrmModel):
super().save(*args, **kwargs)
# tick the updated date on the parent list
self.book_list.updated_date = timezone.now()
self.book_list.save(broadcast=False)
self.book_list.save(broadcast=False, update_fields=["updated_date"])
list_owner = self.book_list.user
model = apps.get_model("bookwyrm.Notification", require_ready=True)

1
bookwyrm/templates/settings/federation/instance_filters.html
View file

@ -2,6 +2,7 @@
{% block filter_fields %}
{% include 'settings/federation/software_filter.html' %}
{% include 'settings/users/server_filter.html' %}
{% endblock %}

14
bookwyrm/tests/lists_stream/test_signals.py
View file

@ -32,9 +32,10 @@ class ListsStreamSignals(TestCase):
def test_add_list_on_create_command(self, _):
"""a new lists has entered"""
book_list = models.List.objects.create(
user=self.remote_user, name="hi", privacy="public"
)
with patch("bookwyrm.lists_stream.remove_list_task.delay"):
book_list = models.List.objects.create(
user=self.remote_user, name="hi", privacy="public"
)
with patch("bookwyrm.lists_stream.add_list_task.delay") as mock:
lists_stream.add_list_on_create_command(book_list.id)
self.assertEqual(mock.call_count, 1)
@ -43,9 +44,10 @@ class ListsStreamSignals(TestCase):
def test_remove_list_on_delete(self, _):
"""delete a list"""
book_list = models.List.objects.create(
user=self.remote_user, name="hi", privacy="public"
)
with patch("bookwyrm.lists_stream.remove_list_task.delay"):
book_list = models.List.objects.create(
user=self.remote_user, name="hi", privacy="public"
)
with patch("bookwyrm.lists_stream.remove_list_task.delay") as mock:
lists_stream.remove_list_on_delete(models.List, book_list)
args = mock.call_args[0]

1
bookwyrm/tests/lists_stream/test_stream.py
View file

@ -11,6 +11,7 @@ from bookwyrm import lists_stream, models
@patch("bookwyrm.activitystreams.add_book_statuses_task.delay")
@patch("bookwyrm.suggested_users.rerank_suggestions_task.delay")
@patch("bookwyrm.activitystreams.populate_stream_task.delay")
@patch("bookwyrm.lists_stream.remove_list_task.delay")
class ListsStream(TestCase):
"""using redis to build activity streams"""

4
bookwyrm/tests/lists_stream/test_tasks.py
View file

@ -35,7 +35,9 @@ class Activitystreams(TestCase):
inbox="https://example.com/users/rat/inbox",
outbox="https://example.com/users/rat/outbox",
)
with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.lists_stream.remove_list_task.delay"):
self.list = models.List.objects.create(
user=self.local_user, name="hi", privacy="public"
)

9
bookwyrm/tests/models/test_group.py
View file

@ -80,7 +80,9 @@ class Group(TestCase):
"""follower-only group booklists should not be excluded from group booklist
listing for group members who do not follower list owner"""
with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.lists_stream.remove_list_task.delay"):
followers_list = models.List.objects.create(
name="Followers List",
curation="group",
@ -101,8 +103,9 @@ class Group(TestCase):
"""private group booklists should not be excluded from group booklist listing
for group members"""
with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.lists_stream.remove_list_task.delay"):
private_list = models.List.objects.create(
name="Private List",
privacy="direct",

20
bookwyrm/tests/models/test_list.py
View file

@ -23,7 +23,9 @@ class List(TestCase):
def test_remote_id(self, _):
"""shelves use custom remote ids"""
with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.lists_stream.remove_list_task.delay"):
book_list = models.List.objects.create(
name="Test List", user=self.local_user
)
@ -32,7 +34,9 @@ class List(TestCase):
def test_to_activity(self, _):
"""jsonify it"""
with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.lists_stream.remove_list_task.delay"):
book_list = models.List.objects.create(
name="Test List", user=self.local_user
)
@ -46,7 +50,9 @@ class List(TestCase):
def test_list_item(self, _):
"""a list entry"""
with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.lists_stream.remove_list_task.delay"):
book_list = models.List.objects.create(
name="Test List", user=self.local_user, privacy="unlisted"
)
@ -64,7 +70,9 @@ class List(TestCase):
def test_list_item_pending(self, _):
"""a list entry"""
with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.lists_stream.remove_list_task.delay"):
book_list = models.List.objects.create(
name="Test List", user=self.local_user
)
@ -84,7 +92,9 @@ class List(TestCase):
def test_embed_key(self, _):
"""embed_key should never be empty"""
with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.lists_stream.remove_list_task.delay"):
book_list = models.List.objects.create(
name="Test List", user=self.local_user
)

1
bookwyrm/tests/test_postgres.py
View file

@ -34,6 +34,7 @@ class PostgresTriggers(TestCase):
)
book.authors.add(author)
book.refresh_from_db()
# pylint: disable=line-too-long
self.assertEqual(
book.search_vector,
"'cool':5B 'goodby':3A 'long':2A 'name':9 'rays':7C 'seri':8 'the':6C 'wow':4B",

4
bookwyrm/tests/views/inbox/test_inbox_remove.py
View file

@ -75,7 +75,9 @@ class InboxRemove(TestCase):
def test_handle_remove_book_from_list(self):
"""listing a book"""
with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.lists_stream.remove_list_task.delay"):
booklist = models.List.objects.create(
name="test list",
user=self.local_user,

7
bookwyrm/tests/views/inbox/test_inbox_update.py
View file

@ -50,7 +50,9 @@ class InboxUpdate(TestCase):
def test_update_list(self):
"""a new list"""
with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.lists_stream.remove_list_task.delay"):
book_list = models.List.objects.create(
name="hi", remote_id="https://example.com/list/22", user=self.local_user
)
@ -69,7 +71,8 @@ class InboxUpdate(TestCase):
"curation": "curated",
"@context": "https://www.w3.org/ns/activitystreams",
}
views.inbox.activity_task(activity)
with patch("bookwyrm.lists_stream.remove_list_task.delay"):
views.inbox.activity_task(activity)
book_list.refresh_from_db()
self.assertEqual(book_list.name, "Test List")
self.assertEqual(book_list.curation, "curated")

4
bookwyrm/tests/views/lists/test_curate.py
View file

@ -36,7 +36,9 @@ class ListViews(TestCase):
parent_work=work,
)
with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.lists_stream.remove_list_task.delay"):
self.list = models.List.objects.create(
name="Test List", user=self.local_user
)

4
bookwyrm/tests/views/lists/test_embed.py
View file

@ -36,7 +36,9 @@ class ListViews(TestCase):
parent_work=work,
)
with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.lists_stream.remove_list_task.delay"):
self.list = models.List.objects.create(
name="Test List", user=self.local_user
)

14
bookwyrm/tests/views/lists/test_list.py
View file

@ -65,7 +65,9 @@ class ListViews(TestCase):
parent_work=work_four,
)
with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.lists_stream.remove_list_task.delay"):
self.list = models.List.objects.create(
name="Test List", user=self.local_user
)
@ -244,7 +246,7 @@ class ListViews(TestCase):
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
) as mock:
) as mock, patch("bookwyrm.lists_stream.remove_list_task.delay"):
result = view(request, self.list.id)
self.assertEqual(mock.call_count, 1)
@ -596,7 +598,7 @@ class ListViews(TestCase):
def test_add_book_outsider(self):
"""put a book on a list"""
self.list.curation = "open"
self.list.save(broadcast=False)
self.list.save(broadcast=False, update_fields=["curation"])
request = self.factory.post(
"",
{
@ -625,7 +627,7 @@ class ListViews(TestCase):
def test_add_book_pending(self):
"""put a book on a list awaiting approval"""
self.list.curation = "curated"
self.list.save(broadcast=False)
self.list.save(broadcast=False, update_fields=["curation"])
request = self.factory.post(
"",
{
@ -658,7 +660,7 @@ class ListViews(TestCase):
def test_add_book_self_curated(self):
"""put a book on a list automatically approved"""
self.list.curation = "curated"
self.list.save(broadcast=False)
self.list.save(broadcast=False, update_fields=["curation"])
request = self.factory.post(
"",
{
@ -687,7 +689,7 @@ class ListViews(TestCase):
def test_add_book_permission_denied(self):
"""you can't add to that list"""
self.list.curation = "closed"
self.list.save(broadcast=False)
self.list.save(broadcast=False, update_fields=["curation"])
request = self.factory.post(
"",
{

4
bookwyrm/tests/views/lists/test_list_item.py
View file

@ -32,7 +32,9 @@ class ListItemViews(TestCase):
remote_id="https://example.com/book/1",
parent_work=work,
)
with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.lists_stream.remove_list_task.delay"):
self.list = models.List.objects.create(
name="Test List", user=self.local_user
)

18
bookwyrm/tests/views/lists/test_lists.py
View file

@ -39,7 +39,9 @@ class ListViews(TestCase):
view = views.Lists.as_view()
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.lists_stream.add_list_task.delay"):
), patch("bookwyrm.lists_stream.add_list_task.delay"), patch(
"bookwyrm.lists_stream.remove_list_task.delay"
):
models.List.objects.create(name="Public list", user=self.local_user)
models.List.objects.create(
name="Private list", privacy="direct", user=self.local_user
@ -62,7 +64,9 @@ class ListViews(TestCase):
def test_saved_lists_page(self):
"""there are so many views, this just makes sure it LOADS"""
view = views.SavedLists.as_view()
with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.lists_stream.remove_list_task.delay"):
booklist = models.List.objects.create(
name="Public list", user=self.local_user
)
@ -82,7 +86,9 @@ class ListViews(TestCase):
def test_saved_lists_page_empty(self):
"""there are so many views, this just makes sure it LOADS"""
view = views.SavedLists.as_view()
with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.lists_stream.remove_list_task.delay"):
models.List.objects.create(name="Public list", user=self.local_user)
models.List.objects.create(
name="Private list", privacy="direct", user=self.local_user
@ -108,7 +114,9 @@ class ListViews(TestCase):
def test_user_lists_page(self):
"""there are so many views, this just makes sure it LOADS"""
view = views.UserLists.as_view()
with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.lists_stream.remove_list_task.delay"):
models.List.objects.create(name="Public list", user=self.local_user)
models.List.objects.create(
name="Private list", privacy="direct", user=self.local_user
@ -146,7 +154,7 @@ class ListViews(TestCase):
request.user = self.local_user
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
) as mock:
) as mock, patch("bookwyrm.lists_stream.remove_list_task.delay"):
result = view(request)
self.assertEqual(mock.call_count, 1)

4
bookwyrm/tests/views/test_search.py
View file

@ -140,7 +140,9 @@ class Views(TestCase):
def test_search_lists(self):
"""searches remote connectors"""
with patch("bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"):
with patch(
"bookwyrm.models.activitypub_mixin.broadcast_task.apply_async"
), patch("bookwyrm.lists_stream.remove_list_task.delay"):
booklist = models.List.objects.create(
user=self.local_user, name="test list"
)

2
bookwyrm/utils/isni.py
View file

@ -106,7 +106,7 @@ def find_authors_by_name(name_string, description=False):
if titles:
# some of the "titles" in ISNI are a little ...iffy
# '@' is used by ISNI/OCLC to index the starting point ignoring stop words
# @ is used by ISNI/OCLC to index the starting point ignoring stop words
# (e.g. "The @Government of no one")
title_elements = [
e

6
bookwyrm/views/admin/federation.py
View file

@ -29,6 +29,8 @@ class Federation(View):
filters = {}
if software := request.GET.get("application_type"):
filters["application_type"] = software
if server := request.GET.get("server"):
filters["server_name"] = server
servers = models.FederatedServer.objects.filter(status=status, **filters)
@ -60,7 +62,9 @@ class Federation(View):
"sort": sort,
"software_options": models.FederatedServer.objects.values_list(
"application_type", flat=True
).distinct(),
)
.distinct()
.order_by("application_type"),
"form": forms.ServerForm(),
}
return TemplateResponse(request, "settings/federation/instance_list.html", data)

9
bookwyrm/views/admin/user_admin.py
View file

@ -22,19 +22,16 @@ class UserAdminList(View):
def get(self, request, status="local"):
"""list of users"""
filters = {}
server = request.GET.get("server")
if server:
if server := request.GET.get("server"):
server = models.FederatedServer.objects.filter(server_name=server).first()
filters["federated_server"] = server
filters["federated_server__isnull"] = False
username = request.GET.get("username")
if username:
if username := request.GET.get("username"):
filters["username__icontains"] = username
scope = request.GET.get("scope")
if scope and scope == "local":
filters["local"] = True
email = request.GET.get("email")
if email:
if email := request.GET.get("email"):
filters["email__endswith"] = email
filters["local"] = status == "local"