Support image uploads in the "notes" markdown fields (#4615)

* Support image uploads in the "notes" markdown fields

- Implemented using the existing EasyMDE library
- Copy / paste support
- Drag / drop support

* Remove debug message

* Updated API version

* Better UX when saving notes

* Pin PIP version (for testing)

* Bug fixes

- Fix typo
- Use correct serializer type

* Add unit testing

* Update role permissions

* Typo fix

* Update migration file

* Adds a notes mixin class to be used for refactoring

* Refactor existing models with notes to use the new mixin

* Add helper function for finding all model types with a certain mixin

* Refactor barcode plugin to use new method

* Typo fix

* Add daily task to delete old / unused notes

* Bug fix for barcode refactoring

* Add unit testing for function

InvenTree/common/api.py

@@ -21,8 +21,8 @@ from InvenTree.api import BulkDeleteMixin
 from InvenTree.config import CONFIG_LOOKUPS
 from InvenTree.filters import ORDER_FILTER, SEARCH_ORDER_FILTER
 from InvenTree.helpers import inheritors
-from InvenTree.mixins import (ListAPI, RetrieveAPI, RetrieveUpdateAPI,
-                              RetrieveUpdateDestroyAPI)
+from InvenTree.mixins import (ListAPI, ListCreateAPI, RetrieveAPI,
+                              RetrieveUpdateAPI, RetrieveUpdateDestroyAPI)
 from InvenTree.permissions import IsSuperuser
 from plugin.models import NotificationUserSetting
 from plugin.serializers import NotificationUserSettingSerializer
@@ -440,6 +440,20 @@ class ConfigDetail(RetrieveAPI):
         return {key: value}
 
 
+class NotesImageList(ListCreateAPI):
+    """List view for all notes images."""
+
+    queryset = common.models.NotesImage.objects.all()
+    serializer_class = common.serializers.NotesImageSerializer
+    permission_classes = [permissions.IsAuthenticated, ]
+
+    def perform_create(self, serializer):
+        """Create (upload) a new notes image"""
+        image = serializer.save()
+        image.user = self.request.user
+        image.save()
+
+
 settings_api_urls = [
     # User settings
     re_path(r'^user/', include([
@@ -473,6 +487,9 @@ common_api_urls = [
     # Webhooks
     path('webhook/<slug:endpoint>/', WebhookView.as_view(), name='api-webhook'),
 
+    # Uploaded images for notes
+    re_path(r'^notes-image-upload/', NotesImageList.as_view(), name='api-notes-image-list'),
+
     # Currencies
     re_path(r'^currency/', include([
         re_path(r'^exchange/', CurrencyExchangeView.as_view(), name='api-currency-exchange'),

InvenTree/common/migrations/0017_notesimage.py

@@ -0,0 +1,27 @@
+# Generated by Django 3.2.18 on 2023-04-17 05:54
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+import common.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('common', '0016_alter_notificationentry_updated'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='NotesImage',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('image', models.ImageField(help_text='Image file', upload_to=common.models.rename_notes_image, verbose_name='Image')),
+                ('date', models.DateTimeField(auto_now_add=True)),
+                ('user', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.SET_NULL, to=settings.AUTH_USER_MODEL)),
+            ],
+        ),
+    ]

InvenTree/common/models.py

@@ -2642,3 +2642,27 @@ class NewsFeedEntry(models.Model):
         help_text=_('Was this news item read?'),
         default=False
     )
+
+
+def rename_notes_image(instance, filename):
+    """Function for renaming uploading image file. Will store in the 'notes' directory."""
+
+    fname = os.path.basename(filename)
+    return os.path.join('notes', fname)
+
+
+class NotesImage(models.Model):
+    """Model for storing uploading images for the 'notes' fields of various models.
+
+    Simply stores the image file, for use in the 'notes' field (of any models which support markdown)
+    """
+
+    image = models.ImageField(
+        upload_to=rename_notes_image,
+        verbose_name=_('Image'),
+        help_text=_('Image file'),
+    )
+
+    user = models.ForeignKey(User, on_delete=models.SET_NULL, null=True, blank=True)
+
+    date = models.DateTimeField(auto_now_add=True)

InvenTree/common/serializers.py

@@ -5,9 +5,10 @@ from django.urls import reverse
 from rest_framework import serializers
 
 from common.models import (InvenTreeSetting, InvenTreeUserSetting,
-                           NewsFeedEntry, NotificationMessage)
+                           NewsFeedEntry, NotesImage, NotificationMessage)
 from InvenTree.helpers import construct_absolute_url, get_objectreference
-from InvenTree.serializers import InvenTreeModelSerializer
+from InvenTree.serializers import (InvenTreeImageSerializerField,
+                                   InvenTreeModelSerializer)
 
 
 class SettingsSerializer(InvenTreeModelSerializer):
@@ -230,3 +231,25 @@ class ConfigSerializer(serializers.Serializer):
         if not isinstance(instance, str):
             instance = list(instance.keys())[0]
         return {'key': instance, **self.instance[instance]}
+
+
+class NotesImageSerializer(InvenTreeModelSerializer):
+    """Serializer for the NotesImage model."""
+
+    class Meta:
+        """Meta options for NotesImageSerializer."""
+
+        model = NotesImage
+        fields = [
+            'pk',
+            'image',
+            'user',
+            'date',
+        ]
+
+        read_only_fields = [
+            'date',
+            'user',
+        ]
+
+    image = InvenTreeImageSerializerField(required=True)

InvenTree/common/tasks.py

@@ -1,14 +1,18 @@
 """Tasks (processes that get offloaded) for common app."""
 
 import logging
+import os
 from datetime import datetime, timedelta
 
 from django.conf import settings
 from django.core.exceptions import AppRegistryNotReady
 from django.db.utils import IntegrityError, OperationalError
+from django.utils import timezone
 
 import feedparser
 
+from InvenTree.helpers import getModelsWithMixin
+from InvenTree.models import InvenTreeNotesMixin
 from InvenTree.tasks import ScheduledTask, scheduled_task
 
 logger = logging.getLogger('inventree')
@@ -26,7 +30,7 @@ def delete_old_notifications():
         logger.info("Could not perform 'delete_old_notifications' - App registry not ready")
         return
 
-    before = datetime.now() - timedelta(days=90)
+    before = timezone.now() - timedelta(days=90)
 
     # Delete notification records before the specified date
     NotificationEntry.objects.filter(updated__lte=before).delete()
@@ -72,3 +76,61 @@ def update_news_feed():
             pass
 
     logger.info('update_news_feed: Sync done')
+
+
+@scheduled_task(ScheduledTask.DAILY)
+def delete_old_notes_images():
+    """Remove old notes images from the database.
+
+    Anything older than ~3 months is removed, unless it is linked to a note
+    """
+    try:
+        from common.models import NotesImage
+    except AppRegistryNotReady:
+        logger.info("Could not perform 'delete_old_notes_images' - App registry not ready")
+        return
+
+    # Remove any notes which point to non-existent image files
+    for note in NotesImage.objects.all():
+        if not os.path.exists(note.image.path):
+            logger.info(f"Deleting note {note.image.path} - image file does not exist")
+            note.delete()
+
+    note_classes = getModelsWithMixin(InvenTreeNotesMixin)
+    before = datetime.now() - timedelta(days=90)
+
+    for note in NotesImage.objects.filter(date__lte=before):
+        # Find any images which are no longer referenced by a note
+
+        found = False
+
+        img = note.image.name
+
+        for model in note_classes:
+            if model.objects.filter(notes__icontains=img).exists():
+                found = True
+                break
+
+        if not found:
+            logger.info(f"Deleting note {img} - image file not linked to a note")
+            note.delete()
+
+    # Finally, remove any images in the notes dir which are not linked to a note
+    notes_dir = os.path.join(settings.MEDIA_ROOT, 'notes')
+
+    images = os.listdir(notes_dir)
+
+    all_notes = NotesImage.objects.all()
+
+    for image in images:
+
+        found = False
+        for note in all_notes:
+            img_path = os.path.basename(note.image.path)
+            if img_path == image:
+                found = True
+                break
+
+        if not found:
+            logger.info(f"Deleting note {image} - image file not linked to a note")
+            os.remove(os.path.join(notes_dir, image))

InvenTree/common/tests.py

@@ -1,5 +1,6 @@
 """Tests for mechanisms in common."""
 
+import io
 import json
 import time
 from datetime import timedelta
@@ -7,9 +8,12 @@ from http import HTTPStatus
 
 from django.contrib.auth import get_user_model
 from django.core.cache import cache
+from django.core.files.uploadedfile import SimpleUploadedFile
 from django.test import Client, TestCase
 from django.urls import reverse
 
+import PIL
+
 from InvenTree.api_tester import InvenTreeAPITestCase, PluginMixin
 from InvenTree.helpers import InvenTreeTestCase, str2bool
 from plugin import registry
@@ -17,8 +21,8 @@ from plugin.models import NotificationUserSetting
 
 from .api import WebhookView
 from .models import (ColorTheme, InvenTreeSetting, InvenTreeUserSetting,
-                     NotificationEntry, NotificationMessage, WebhookEndpoint,
-                     WebhookMessage)
+                     NotesImage, NotificationEntry, NotificationMessage,
+                     WebhookEndpoint, WebhookMessage)
 
 CONTENT_TYPE_JSON = 'application/json'
 
@@ -935,3 +939,65 @@ class CurrencyAPITests(InvenTreeAPITestCase):
             time.sleep(10)
 
         raise TimeoutError("Could not refresh currency exchange data after 5 attempts")
+
+
+class NotesImageTest(InvenTreeAPITestCase):
+    """Tests for uploading images to be used in markdown notes."""
+
+    def test_invalid_files(self):
+        """Test that invalid files are rejected."""
+
+        n = NotesImage.objects.count()
+
+        # Test upload of a simple text file
+        response = self.post(
+            reverse('api-notes-image-list'),
+            data={
+                'image': SimpleUploadedFile('test.txt', b"this is not an image file", content_type='text/plain'),
+            },
+            format='multipart',
+            expected_code=400
+        )
+
+        self.assertIn("Upload a valid image", str(response.data['image']))
+
+        # Test upload of an invalid image file
+        response = self.post(
+            reverse('api-notes-image-list'),
+            data={
+                'image': SimpleUploadedFile('test.png', b"this is not an image file", content_type='image/png'),
+            },
+            format='multipart',
+            expected_code=400,
+        )
+
+        self.assertIn("Upload a valid image", str(response.data['image']))
+
+        # Check that no extra database entries have been created
+        self.assertEqual(NotesImage.objects.count(), n)
+
+    def test_valid_image(self):
+        """Test upload of a valid image file"""
+
+        n = NotesImage.objects.count()
+
+        # Construct a simple image file
+        image = PIL.Image.new('RGB', (100, 100), color='red')
+
+        with io.BytesIO() as output:
+            image.save(output, format='PNG')
+            contents = output.getvalue()
+
+        response = self.post(
+            reverse('api-notes-image-list'),
+            data={
+                'image': SimpleUploadedFile('test.png', contents, content_type='image/png'),
+            },
+            format='multipart',
+            expected_code=201
+        )
+
+        print(response.data)
+
+        # Check that a new file has been created
+        self.assertEqual(NotesImage.objects.count(), n + 1)