From 24e5ea3295cd32f23b0a2f21dee716d4d64f1435 Mon Sep 17 00:00:00 2001 From: Mouse Reeve Date: Sat, 5 Nov 2022 13:29:08 -0700 Subject: [PATCH] Check if import is stopped and create import status field --- bookwyrm/importers/importer.py | 8 ++++- .../migrations/0160_auto_20221105_2030.py | 33 +++++++++++++++++++ bookwyrm/models/import_job.py | 16 ++++++++- 3 files changed, 55 insertions(+), 2 deletions(-) create mode 100644 bookwyrm/migrations/0160_auto_20221105_2030.py diff --git a/bookwyrm/importers/importer.py b/bookwyrm/importers/importer.py index 32800e772..a3c522968 100644 --- a/bookwyrm/importers/importer.py +++ b/bookwyrm/importers/importer.py @@ -132,13 +132,19 @@ def start_import_task(job_id): job = ImportJob.objects.get(id=job_id) # these are sub-tasks so that one big task doesn't use up all the memory in celery for item in job.items.values_list("id", flat=True).all(): - import_item_task.delay(item) + task = import_item_task.delay(item) + item.task_id = task.id + item.save() @app.task(queue="low_priority") def import_item_task(item_id): """resolve a row into a book""" item = models.ImportItem.objects.get(id=item_id) + # make sure the job has not been stopped + if item.job.complete: + return + try: item.resolve() except Exception as err: # pylint: disable=broad-except diff --git a/bookwyrm/migrations/0160_auto_20221105_2030.py b/bookwyrm/migrations/0160_auto_20221105_2030.py new file mode 100644 index 000000000..756787d2a --- /dev/null +++ b/bookwyrm/migrations/0160_auto_20221105_2030.py @@ -0,0 +1,33 @@ +# Generated by Django 3.2.15 on 2022-11-05 20:30 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("bookwyrm", "0159_auto_20220924_0634"), + ] + + operations = [ + migrations.AddField( + model_name="importitem", + name="task_id", + field=models.CharField(blank=True, max_length=200, null=True), + ), + migrations.AddField( + model_name="importjob", + name="status", + field=models.CharField( + choices=[ + ("pending", "Pending"), + ("active", "Active"), + ("complete", "Complete"), + ("stopped", "Stopped"), + ], + default="pending", + max_length=50, + null=True, + ), + ), + ] diff --git a/bookwyrm/models/import_job.py b/bookwyrm/models/import_job.py index b6a6ded41..6db2fd854 100644 --- a/bookwyrm/models/import_job.py +++ b/bookwyrm/models/import_job.py @@ -5,6 +5,7 @@ import dateutil.parser from django.db import models from django.utils import timezone +from django.utils.translation import gettext_lazy as _ from bookwyrm.connectors import connector_manager from bookwyrm.models import ReadThrough, User, Book, Edition @@ -31,6 +32,14 @@ def construct_search_term(title, author): return " ".join([title, author]) +ImportStatuses = [ + ("pending", _("Pending")), + ("active", _("Active")), + ("complete", _("Complete")), + ("stopped", _("Stopped")), +] + + class ImportJob(models.Model): """entry for a specific request for book data import""" @@ -39,11 +48,15 @@ class ImportJob(models.Model): updated_date = models.DateTimeField(default=timezone.now) include_reviews = models.BooleanField(default=True) mappings = models.JSONField() - complete = models.BooleanField(default=False) source = models.CharField(max_length=100) privacy = models.CharField(max_length=255, default="public", choices=PrivacyLevels) retry = models.BooleanField(default=False) + complete = models.BooleanField(default=False) + status = models.CharField( + max_length=50, choices=ImportStatuses, default="pending", null=True + ) + @property def pending_items(self): """items that haven't been processed yet""" @@ -95,6 +108,7 @@ class ImportItem(models.Model): linked_review = models.ForeignKey( "Review", on_delete=models.SET_NULL, null=True, blank=True ) + task_id = models.CharField(max_length=200, null=True, blank=True) def update_job(self): """let the job know when the items get work done"""