diff --git a/.env.example b/.env.example index fb0f7308d..20ce8240b 100644 --- a/.env.example +++ b/.env.example @@ -137,3 +137,6 @@ TWO_FACTOR_LOGIN_MAX_SECONDS=60 # and AWS_S3_CUSTOM_DOMAIN (if used) are added by default. # Value should be a comma-separated list of host names. CSP_ADDITIONAL_HOSTS= +# The last number here means "megabytes" +# Increase if users are having trouble uploading BookWyrm export files. +DATA_UPLOAD_MAX_MEMORY_SIZE = (1024**2 * 100) \ No newline at end of file diff --git a/bookwyrm/migrations/0192_sitesettings_user_exports_enabled.py b/bookwyrm/migrations/0192_sitesettings_user_exports_enabled.py new file mode 100644 index 000000000..ec5b411e2 --- /dev/null +++ b/bookwyrm/migrations/0192_sitesettings_user_exports_enabled.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.23 on 2024-01-16 10:28 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0191_merge_20240102_0326"), + ] + + operations = [ + migrations.AddField( + model_name="sitesettings", + name="user_exports_enabled", + field=models.BooleanField(default=False), + ), + ] diff --git a/bookwyrm/models/site.py b/bookwyrm/models/site.py index bd53f1f07..8075b6434 100644 --- a/bookwyrm/models/site.py +++ b/bookwyrm/models/site.py @@ -96,6 +96,7 @@ class SiteSettings(SiteModel): imports_enabled = models.BooleanField(default=True) import_size_limit = models.IntegerField(default=0) import_limit_reset = models.IntegerField(default=0) + user_exports_enabled = models.BooleanField(default=False) user_import_time_limit = models.IntegerField(default=48) field_tracker = FieldTracker(fields=["name", "instance_tagline", "logo"]) diff --git a/bookwyrm/settings.py b/bookwyrm/settings.py index 7c8947521..3f9665baf 100644 --- a/bookwyrm/settings.py +++ b/bookwyrm/settings.py @@ -442,6 +442,4 @@ if HTTP_X_FORWARDED_PROTO: # Do not change this setting unless you already have an existing # user with the same username - in which case you should change it! INSTANCE_ACTOR_USERNAME = "bookwyrm.instance.actor" -DATA_UPLOAD_MAX_MEMORY_SIZE = ( - 1024**2 * 20 -) # 20MB TEMPORARY FIX WHILST WORKING ON THIS +DATA_UPLOAD_MAX_MEMORY_SIZE = env.int("DATA_UPLOAD_MAX_MEMORY_SIZE", (1024**2 * 100)) diff --git a/bookwyrm/templates/about/about.html b/bookwyrm/templates/about/about.html index 6705793d5..ef3f34037 100644 --- a/bookwyrm/templates/about/about.html +++ b/bookwyrm/templates/about/about.html @@ -31,10 +31,10 @@

-
+
{% if superlatives.top_rated %} {% with book=superlatives.top_rated.default_edition rating=superlatives.top_rated.rating %} -
+
@@ -53,7 +53,7 @@ {% if superlatives.wanted %} {% with book=superlatives.wanted.default_edition %} -
+
@@ -72,7 +72,7 @@ {% if superlatives.controversial %} {% with book=superlatives.controversial.default_edition %} -
+
diff --git a/bookwyrm/templates/moved.html b/bookwyrm/templates/moved.html index 545fc3d87..382b752be 100644 --- a/bookwyrm/templates/moved.html +++ b/bookwyrm/templates/moved.html @@ -23,7 +23,7 @@

- {% id_to_username request.user.moved_to as username %} + {% id_to_username request.user.moved_to as username %} {% blocktrans trimmed with moved_to=user.moved_to %} You have moved your account to {{ username }} {% endblocktrans %} diff --git a/bookwyrm/templates/notifications/items/move_user.html b/bookwyrm/templates/notifications/items/move_user.html index b94d96dc4..3121d3f45 100644 --- a/bookwyrm/templates/notifications/items/move_user.html +++ b/bookwyrm/templates/notifications/items/move_user.html @@ -14,7 +14,7 @@ {% block description %} {% if related_user_moved_to %} - {% id_to_username request.user.moved_to as username %} + {% id_to_username related_user_moved_to as username %} {% blocktrans trimmed %} {{ related_user }} has moved to {{ username }} {% endblocktrans %} diff --git a/bookwyrm/templates/preferences/export-user.html b/bookwyrm/templates/preferences/export-user.html index a468c3f74..cd3119e3e 100644 --- a/bookwyrm/templates/preferences/export-user.html +++ b/bookwyrm/templates/preferences/export-user.html @@ -46,7 +46,11 @@ {% trans "If you wish to migrate any statuses (comments, reviews, or quotes) you must either set the account you are moving to as an alias of this one, or move this account to the new account, before you import your user data." %} {% endspaceless %}

- {% if next_available %} + {% if not site.user_exports_enabled %} +

+ {% trans "New user exports are currently disabled." %} +

+ {% elif next_available %}

{% blocktrans trimmed %} You will be able to create a new export file at {{ next_available }} diff --git a/bookwyrm/templates/settings/imports/imports.html b/bookwyrm/templates/settings/imports/imports.html index 8898aab71..11b3c7e03 100644 --- a/bookwyrm/templates/settings/imports/imports.html +++ b/bookwyrm/templates/settings/imports/imports.html @@ -90,6 +90,33 @@

+ + {% if site.user_exports_enabled %} +
+ + + {% trans "Disable starting new user exports" %} + + + +
+
+ {% trans "This is only intended to be used when things have gone very wrong with exports and you need to pause the feature while addressing issues." %} + {% trans "While exports are disabled, users will not be allowed to start new user exports, but existing exports will not be affected." %} +
+ {% csrf_token %} +
+ +
+
+
@@ -108,7 +135,7 @@ {% trans "Set the value to 0 to not enforce any limit." %}
- + {% csrf_token %} @@ -120,6 +147,28 @@
+ {% else %} +
+
+

{% trans "Users are currently unable to start new user exports. This is the default setting." %}

+ {% if use_s3 %} +

{% trans "It is not currently possible to provide user exports when using s3 storage. The BookWyrm development team are working on a fix for this." %}

+ {% endif %} +
+ {% csrf_token %} +
+ +
+
+ {% endif %}

{% trans "Book Imports" %}

diff --git a/bookwyrm/templatetags/utilities.py b/bookwyrm/templatetags/utilities.py index 754db41dd..a1b3d6cdf 100644 --- a/bookwyrm/templatetags/utilities.py +++ b/bookwyrm/templatetags/utilities.py @@ -125,7 +125,8 @@ def id_to_username(user_id): name = parts[-1] value = f"{name}@{domain}" - return value + return value + return "a new user account" @register.filter(name="get_file_size") diff --git a/bookwyrm/tests/views/preferences/test_export.py b/bookwyrm/tests/views/preferences/test_export.py index 4f498f589..3f758b2f7 100644 --- a/bookwyrm/tests/views/preferences/test_export.py +++ b/bookwyrm/tests/views/preferences/test_export.py @@ -18,7 +18,9 @@ class ExportViews(TestCase): """viewing and creating statuses""" @classmethod - def setUpTestData(self): # pylint: disable=bad-classmethod-argument + def setUpTestData( + self, + ): # pylint: disable=bad-classmethod-argument, disable=invalid-name """we need basic test data and mocks""" with patch("bookwyrm.suggested_users.rerank_suggestions_task.delay"), patch( "bookwyrm.activitystreams.populate_stream_task.delay" @@ -40,6 +42,7 @@ class ExportViews(TestCase): bnf_id="beep", ) + # pylint: disable=invalid-name def setUp(self): """individual test setup""" self.factory = RequestFactory() @@ -53,11 +56,12 @@ class ExportViews(TestCase): def test_export_file(self, *_): """simple export""" - models.ShelfBook.objects.create( + shelfbook = models.ShelfBook.objects.create( shelf=self.local_user.shelf_set.first(), user=self.local_user, book=self.book, ) + book_date = str.encode(f"{shelfbook.shelved_date.date()}") request = self.factory.post("") request.user = self.local_user export = views.Export.as_view()(request) @@ -66,7 +70,7 @@ class ExportViews(TestCase): # pylint: disable=line-too-long self.assertEqual( export.content, - b"title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,rating,review_name,review_cw,review_content\r\nTest Book,," - + self.book.remote_id.encode("utf-8") - + b",,,,,beep,,,,,,123456789X,9781234567890,,,,,\r\n", + b"title,author_text,remote_id,openlibrary_key,inventaire_id,librarything_key,goodreads_key,bnf_id,viaf,wikidata,asin,aasin,isfdb,isbn_10,isbn_13,oclc_number,start_date,finish_date,stopped_date,rating,review_name,review_cw,review_content,review_published,shelf,shelf_name,shelf_date\r\n" + + b"Test Book,,%b,,,,,beep,,,,,,123456789X,9781234567890,,,,,,,,,,to-read,To Read,%b\r\n" + % (self.book.remote_id.encode("utf-8"), book_date), ) diff --git a/bookwyrm/urls.py b/bookwyrm/urls.py index 76e60245b..1a577c84b 100644 --- a/bookwyrm/urls.py +++ b/bookwyrm/urls.py @@ -338,6 +338,16 @@ urlpatterns = [ views.disable_imports, name="settings-imports-disable", ), + re_path( + r"^settings/user-exports/enable/?$", + views.enable_user_exports, + name="settings-user-exports-enable", + ), + re_path( + r"^settings/user-exports/disable/?$", + views.disable_user_exports, + name="settings-user-exports-disable", + ), re_path( r"^settings/imports/enable/?$", views.enable_imports, diff --git a/bookwyrm/views/__init__.py b/bookwyrm/views/__init__.py index 3be813208..f11c11dd6 100644 --- a/bookwyrm/views/__init__.py +++ b/bookwyrm/views/__init__.py @@ -18,6 +18,8 @@ from .admin.imports import ( set_import_size_limit, set_user_import_completed, set_user_import_limit, + enable_user_exports, + disable_user_exports, ) from .admin.ip_blocklist import IPBlocklist from .admin.invite import ManageInvites, Invite, InviteRequest diff --git a/bookwyrm/views/admin/imports.py b/bookwyrm/views/admin/imports.py index a85d6c79e..0924536bf 100644 --- a/bookwyrm/views/admin/imports.py +++ b/bookwyrm/views/admin/imports.py @@ -9,7 +9,7 @@ from django.views.decorators.http import require_POST from bookwyrm import models from bookwyrm.views.helpers import redirect_to_referer -from bookwyrm.settings import PAGE_LENGTH +from bookwyrm.settings import PAGE_LENGTH, USE_S3 # pylint: disable=no-self-use @@ -59,6 +59,7 @@ class ImportList(View): "import_size_limit": site_settings.import_size_limit, "import_limit_reset": site_settings.import_limit_reset, "user_import_time_limit": site_settings.user_import_time_limit, + "use_s3": USE_S3, } return TemplateResponse(request, "settings/imports/imports.html", data) @@ -126,3 +127,25 @@ def set_user_import_limit(request): site.user_import_time_limit = int(request.POST.get("limit")) site.save(update_fields=["user_import_time_limit"]) return redirect("settings-imports") + + +@require_POST +@permission_required("bookwyrm.edit_instance_settings", raise_exception=True) +# pylint: disable=unused-argument +def enable_user_exports(request): + """Allow users to export account data""" + site = models.SiteSettings.objects.get() + site.user_exports_enabled = True + site.save(update_fields=["user_exports_enabled"]) + return redirect("settings-imports") + + +@require_POST +@permission_required("bookwyrm.edit_instance_settings", raise_exception=True) +# pylint: disable=unused-argument +def disable_user_exports(request): + """Don't allow users to export account data""" + site = models.SiteSettings.objects.get() + site.user_exports_enabled = False + site.save(update_fields=["user_exports_enabled"]) + return redirect("settings-imports") diff --git a/bookwyrm/views/preferences/export.py b/bookwyrm/views/preferences/export.py index f54d97ccb..d16f3aaa3 100644 --- a/bookwyrm/views/preferences/export.py +++ b/bookwyrm/views/preferences/export.py @@ -17,7 +17,8 @@ from bookwyrm import models from bookwyrm.models.bookwyrm_export_job import BookwyrmExportJob from bookwyrm.settings import PAGE_LENGTH -# pylint: disable=no-self-use + +# pylint: disable=no-self-use,too-many-locals @method_decorator(login_required, name="dispatch") class Export(View): """Let users export data""" @@ -54,7 +55,19 @@ class Export(View): fields = ( ["title", "author_text"] + deduplication_fields - + ["rating", "review_name", "review_cw", "review_content"] + + [ + "start_date", + "finish_date", + "stopped_date", + "rating", + "review_name", + "review_cw", + "review_content", + "review_published", + "shelf", + "shelf_name", + "shelf_date", + ] ) writer.writerow(fields) @@ -70,6 +83,24 @@ class Export(View): book.rating = review_rating.rating if review_rating else None + readthrough = ( + models.ReadThrough.objects.filter(user=request.user, book=book) + .order_by("-start_date", "-finish_date") + .first() + ) + if readthrough: + book.start_date = ( + readthrough.start_date.date() if readthrough.start_date else None + ) + book.finish_date = ( + readthrough.finish_date.date() if readthrough.finish_date else None + ) + book.stopped_date = ( + readthrough.stopped_date.date() + if readthrough.stopped_date + else None + ) + review = ( models.Review.objects.filter( user=request.user, book=book, content__isnull=False @@ -78,9 +109,27 @@ class Export(View): .first() ) if review: + book.review_published = ( + review.published_date.date() if review.published_date else None + ) book.review_name = review.name book.review_cw = review.content_warning - book.review_content = review.raw_content + book.review_content = ( + review.raw_content if review.raw_content else review.content + ) # GoodReads imported reviews do not have raw_content, but content. + + shelfbook = ( + models.ShelfBook.objects.filter(user=request.user, book=book) + .order_by("-shelved_date", "-created_date", "-updated_date") + .last() + ) + if shelfbook: + book.shelf = shelfbook.shelf.identifier + book.shelf_name = shelfbook.shelf.name + book.shelf_date = ( + shelfbook.shelved_date.date() if shelfbook.shelved_date else None + ) + writer.writerow([getattr(book, field, "") or "" for field in fields]) return HttpResponse( diff --git a/dev-tools/Dockerfile b/dev-tools/Dockerfile index 3b7740a78..6c132944f 100644 --- a/dev-tools/Dockerfile +++ b/dev-tools/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9 +FROM python:3.9-bookworm WORKDIR /app/dev-tools ENV PATH="/app/dev-tools/node_modules/.bin:$PATH" diff --git a/nginx/development b/nginx/development index 841db0124..64cd1b911 100644 --- a/nginx/development +++ b/nginx/development @@ -64,13 +64,18 @@ server { # directly serve images and static files from the # bookwyrm filesystem using sendfile. # make the logs quieter by not reporting these requests - location ~ ^/(images|static)/ { + location ~ \.(bmp|ico|jpg|jpeg|png|tif|tiff|webp|css|js)$ { root /app; try_files $uri =404; add_header X-Cache-Status STATIC; access_log off; } + # block access to any non-image files from images or static + location ~ ^/images/ { + return 403; + } + # monitor the celery queues with flower, no caching enabled location /flower/ { proxy_pass http://flower:8888; diff --git a/nginx/production b/nginx/production index 9018ab9de..76ed19449 100644 --- a/nginx/production +++ b/nginx/production @@ -96,12 +96,17 @@ server { # # directly serve images and static files from the # # bookwyrm filesystem using sendfile. # # make the logs quieter by not reporting these requests -# location ~ ^/(images|static)/ { +# location ~ \.(bmp|ico|jpg|jpeg|png|tif|tiff|webp|css|js)$ { # root /app; # try_files $uri =404; # add_header X-Cache-Status STATIC; # access_log off; # } + +# # block access to any non-image files from images or static +# location ~ ^/images/ { +# return 403; +# } # # # monitor the celery queues with flower, no caching enabled # location /flower/ {