From 4ccf1c4818f2a1524c66d61848d309c2a7254f19 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 12 Nov 2020 12:09:47 -0800 Subject: [PATCH 1/8] Small html style changes on import pages --- bookwyrm/templates/import.html | 3 ++- bookwyrm/templates/import_status.html | 19 +++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/bookwyrm/templates/import.html b/bookwyrm/templates/import.html index c36448127..8e3f5eb49 100644 --- a/bookwyrm/templates/import.html +++ b/bookwyrm/templates/import.html @@ -14,7 +14,8 @@
-
diff --git a/bookwyrm/templates/import_status.html b/bookwyrm/templates/import_status.html index 7db2ba3d5..ffd1fa501 100644 --- a/bookwyrm/templates/import_status.html +++ b/bookwyrm/templates/import_status.html @@ -6,25 +6,24 @@

Import Status

- Import started: {{ job.created_date | naturaltime }} -

+ Import started: {{ job.created_date | naturaltime }} +

{% if task.ready %} - Import completed: {{ task.date_done | naturaltime }} - {% if task.failed %} -

TASK FAILED

+ Import completed: {{ task.date_done | naturaltime }} +

+ {% elif task.failed %} +
TASK FAILED
{{ task.info }} {% endif %}
- {% if job.import_status %} - {% include 'snippets/status.html' with status=job.import_status %} - {% endif %} - {% else %} + {% if not task.ready and not task.failed %} Import still in progress.

- (Hit reload to update!) + (Hit reload to update!) +

{% endif %}
From ffeed6047ed2dea95b8270ed11d45c307d2066e0 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 12 Nov 2020 13:09:29 -0800 Subject: [PATCH 2/8] Fixes reporting import task status --- bookwyrm/templates/import_status.html | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bookwyrm/templates/import_status.html b/bookwyrm/templates/import_status.html index ffd1fa501..e4eb681e1 100644 --- a/bookwyrm/templates/import_status.html +++ b/bookwyrm/templates/import_status.html @@ -8,18 +8,17 @@

Import started: {{ job.created_date | naturaltime }}

- {% if task.ready %} + {% if task.successful %}

Import completed: {{ task.date_done | naturaltime }}

{% elif task.failed %}
TASK FAILED
- {{ task.info }} {% endif %}
- {% if not task.ready and not task.failed %} + {% if task.status == 'PENDING' %} Import still in progress.

(Hit reload to update!) From 0a2d762d3bd4eb9b05d43defe15ff00c121621cd Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 12 Nov 2020 13:16:26 -0800 Subject: [PATCH 3/8] Fixes error state when there are author duplicates there shouldn't be, but it shouldn't crash a goodreads import either --- bookwyrm/connectors/openlibrary.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/bookwyrm/connectors/openlibrary.py b/bookwyrm/connectors/openlibrary.py index 9b8afaa6a..5c26ad45b 100644 --- a/bookwyrm/connectors/openlibrary.py +++ b/bookwyrm/connectors/openlibrary.py @@ -177,10 +177,9 @@ class Connector(AbstractConnector): ''' load that author ''' if not re.match(r'^OL\d+A$', olkey): raise ValueError('Invalid OpenLibrary author ID') - try: - return models.Author.objects.get(openlibrary_key=olkey) - except models.Author.DoesNotExist: - pass + author = models.Author.objects.filter(openlibrary_key=olkey).first() + if author: + return author url = '%s/authors/%s.json' % (self.base_url, olkey) data = get_data(url) From 74236c481089660df694dee3a1557873c67a8406 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 12 Nov 2020 13:20:32 -0800 Subject: [PATCH 4/8] Handle errors tryingt o load covers from instances --- bookwyrm/connectors/bookwyrm_connector.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/bookwyrm/connectors/bookwyrm_connector.py b/bookwyrm/connectors/bookwyrm_connector.py index 5ad795cd3..ecceb4570 100644 --- a/bookwyrm/connectors/bookwyrm_connector.py +++ b/bookwyrm/connectors/bookwyrm_connector.py @@ -79,10 +79,17 @@ class Connector(AbstractConnector): cover_data = data.get('attachment') if not cover_data: return None - cover_url = cover_data[0].get('url') - response = requests.get(cover_url) + try: + cover_url = cover_data[0].get('url') + except IndexError: + return None + try: + response = requests.get(cover_url) + except ConnectionError: + return None + if not response.ok: - response.raise_for_status() + return None image_name = str(uuid4()) + '.' + cover_url.split('.')[-1] image_content = ContentFile(response.content) From 799496bb86ffb6d69d97cfcc1279438bf1f9451a Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 12 Nov 2020 13:33:12 -0800 Subject: [PATCH 5/8] Move past all book loading exceptions during import --- bookwyrm/goodreads_import.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/bookwyrm/goodreads_import.py b/bookwyrm/goodreads_import.py index fe5ac56e2..ee9496306 100644 --- a/bookwyrm/goodreads_import.py +++ b/bookwyrm/goodreads_import.py @@ -20,7 +20,7 @@ def create_job(user, csv_file, include_reviews, privacy): ) for index, entry in enumerate(list(csv.DictReader(csv_file))[:MAX_ENTRIES]): if not all(x in entry for x in ('ISBN13', 'Title', 'Author')): - raise ValueError("Author, title, and isbn must be in data.") + raise ValueError('Author, title, and isbn must be in data.') ImportItem(job=job, index=index, data=entry).save() return job @@ -41,8 +41,9 @@ def import_data(job_id): for item in job.items.all(): try: item.resolve() - except HTTPError: - pass + except: + item.fail_reason = 'Error loading book' + item.save() if item.book: item.save() results.append(item) @@ -51,7 +52,7 @@ def import_data(job_id): outgoing.handle_imported_book( job.user, item, job.include_reviews, job.privacy) else: - item.fail_reason = "Could not find a match for book" + item.fail_reason = 'Could not find a match for book' item.save() finally: create_notification(job.user, 'IMPORT', related_import=job) From 032ce8efbae0b4c6588f785ddec51041aaa9f9de Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 12 Nov 2020 13:52:57 -0800 Subject: [PATCH 6/8] Show failed books --- bookwyrm/templates/import_status.html | 24 +++++++++++++++++++++--- bookwyrm/views.py | 6 +++++- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/bookwyrm/templates/import_status.html b/bookwyrm/templates/import_status.html index e4eb681e1..d091e3033 100644 --- a/bookwyrm/templates/import_status.html +++ b/bookwyrm/templates/import_status.html @@ -26,7 +26,24 @@ {% endif %}

+{% if failed_items %}
+

Failed to load

+
    + {% for item in failed_items %} +
  • + Line {{ item.index }}: + {{ item.data|dict_key:'Title' }} by + {{ item.data|dict_key:'Author' }} + ({{ item.fail_reason }}) +
    • + {% endfor %} +
+
+{% endif %} + +
+

Successfully imported

diff --git a/bookwyrm/views.py b/bookwyrm/views.py index 771e6a85c..b40a75a75 100644 --- a/bookwyrm/views.py +++ b/bookwyrm/views.py @@ -209,10 +209,14 @@ def import_status(request, job_id): if job.user != request.user: raise PermissionDenied task = app.AsyncResult(job.task_id) + items = job.items.order_by('index').all() + failed_items = [i for i in items if i.fail_reason] + items = [i for i in items if not i.fail_reason] return TemplateResponse(request, 'import_status.html', { 'title': 'Import Status', 'job': job, - 'items': job.items.order_by('index').all(), + 'items': items, + 'failed_items': failed_items, 'task': task }) From 932b4f8b81d3bb6b14aabe56bfc3f39d015c9798 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 12 Nov 2020 14:01:17 -0800 Subject: [PATCH 7/8] Small fix in goodreads import error handling --- bookwyrm/goodreads_import.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bookwyrm/goodreads_import.py b/bookwyrm/goodreads_import.py index ee9496306..d5c0ad420 100644 --- a/bookwyrm/goodreads_import.py +++ b/bookwyrm/goodreads_import.py @@ -44,6 +44,8 @@ def import_data(job_id): except: item.fail_reason = 'Error loading book' item.save() + continue + if item.book: item.save() results.append(item) From 1f11ffd64636600233243344da3f63aa2ad81640 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Thu, 12 Nov 2020 14:11:39 -0800 Subject: [PATCH 8/8] Uses task.ready for checking if the job is running --- bookwyrm/templates/import_status.html | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bookwyrm/templates/import_status.html b/bookwyrm/templates/import_status.html index d091e3033..c1dbb26e2 100644 --- a/bookwyrm/templates/import_status.html +++ b/bookwyrm/templates/import_status.html @@ -18,7 +18,7 @@
- {% if task.status == 'PENDING' %} + {% if not task.ready %} Import still in progress.

(Hit reload to update!)

@@ -57,9 +74,10 @@ {{ item.data|dict_key:'Author' }} - {% if item.book %}✓ - {% elif item.fail_reason %} - {{ item.fail_reason }} + {% if item.book %} + + Imported + {% endif %}