From 0c44f738fdff2f16064d782cd15a2a3e3b1021a7 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Thu, 20 Feb 2020 14:42:35 +0200 Subject: [PATCH] More progress - support chunk uploading. --- .../0008_collectionitemchunk_chunkfile.py | 20 ++++++++++ .../migrations/0009_auto_20200220_1220.py | 19 +++++++++ django_etesync/models.py | 16 ++++---- django_etesync/serializers.py | 40 ++++++++++++++----- django_etesync/views.py | 25 ++++-------- 5 files changed, 83 insertions(+), 37 deletions(-) create mode 100644 django_etesync/migrations/0008_collectionitemchunk_chunkfile.py create mode 100644 django_etesync/migrations/0009_auto_20200220_1220.py diff --git a/django_etesync/migrations/0008_collectionitemchunk_chunkfile.py b/django_etesync/migrations/0008_collectionitemchunk_chunkfile.py new file mode 100644 index 0000000..68bb27c --- /dev/null +++ b/django_etesync/migrations/0008_collectionitemchunk_chunkfile.py @@ -0,0 +1,20 @@ +# Generated by Django 3.0.3 on 2020-02-20 12:16 + +from django.db import migrations, models +import django_etesync.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0007_auto_20200220_1144'), + ] + + operations = [ + migrations.AddField( + model_name='collectionitemchunk', + name='chunkFile', + field=models.FileField(default='', upload_to=django_etesync.models.chunk_directory_path), + preserve_default=False, + ), + ] diff --git a/django_etesync/migrations/0009_auto_20200220_1220.py b/django_etesync/migrations/0009_auto_20200220_1220.py new file mode 100644 index 0000000..71e1539 --- /dev/null +++ b/django_etesync/migrations/0009_auto_20200220_1220.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-02-20 12:20 + +from django.db import migrations, models +import django_etesync.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etesync', '0008_collectionitemchunk_chunkfile'), + ] + + operations = [ + migrations.AlterField( + model_name='collectionitemchunk', + name='chunkFile', + field=models.FileField(max_length=150, upload_to=django_etesync.models.chunk_directory_path), + ), + ] diff --git a/django_etesync/models.py b/django_etesync/models.py index 1bd2090..0cefbee 100644 --- a/django_etesync/models.py +++ b/django_etesync/models.py @@ -36,12 +36,6 @@ class Collection(models.Model): return self.uid -def chunk_directory_path(instance, filename): - col = instance.itemSnapshot.item.collection - user_id = col.owner.id - return Path('user_{}'.format(user_id), col.uid, instance.uid) - - class CollectionItem(models.Model): uid = models.CharField(db_index=True, blank=False, null=False, @@ -59,12 +53,19 @@ class CollectionItem(models.Model): return self.snapshots.get(current=True) +def chunk_directory_path(instance, filename): + item = instance.item + col = item.collection + user_id = col.owner.id + return Path('user_{}'.format(user_id), col.uid, item.uid, instance.uid) + + class CollectionItemChunk(models.Model): uid = models.CharField(db_index=True, blank=False, null=False, max_length=44, validators=[UidValidator]) item = models.ForeignKey(CollectionItem, related_name='chunks', on_delete=models.CASCADE) order = models.CharField(max_length=100, blank=False, null=False) - # We probably just want to implement this manually because we can have more than one pointing to a file. chunkFile = models.FileField(upload_to=chunk_directory_path) + chunkFile = models.FileField(upload_to=chunk_directory_path, max_length=150) class Meta: unique_together = ('item', 'order') @@ -87,4 +88,3 @@ class CollectionItemSnapshot(models.Model): def __str__(self): return '{} {} current={}'.format(self.item.uid, self.id, self.current) - diff --git a/django_etesync/serializers.py b/django_etesync/serializers.py index f86d5e1..a4d6fbe 100644 --- a/django_etesync/serializers.py +++ b/django_etesync/serializers.py @@ -61,10 +61,10 @@ class CollectionSerializer(serializers.ModelSerializer): class CollectionItemChunkSerializer(serializers.ModelSerializer): class Meta: model = models.CollectionItemChunk - fields = ('uid', ) + fields = ('uid', 'chunkFile') -class CollectionItemSnapshotSerializer(serializers.ModelSerializer): +class CollectionItemSnapshotBaseSerializer(serializers.ModelSerializer): encryptionKey = BinaryBase64Field() chunks = serializers.SlugRelatedField( slug_field='uid', @@ -77,18 +77,36 @@ class CollectionItemSnapshotSerializer(serializers.ModelSerializer): fields = ('version', 'encryptionKey', 'chunks', 'hmac') -class CollectionItemSnapshotInlineSerializer(CollectionItemSnapshotSerializer): - chunksData = serializers.SerializerMethodField('get_inline_chunks_from_context') +class CollectionItemSnapshotSerializer(CollectionItemSnapshotBaseSerializer): + chunksUrls = serializers.SerializerMethodField('get_chunks_urls') - class Meta(CollectionItemSnapshotSerializer.Meta): - fields = CollectionItemSnapshotSerializer.Meta.fields + ('chunksData', ) + class Meta(CollectionItemSnapshotBaseSerializer.Meta): + fields = CollectionItemSnapshotBaseSerializer.Meta.fields + ('chunksUrls', ) - def get_inline_chunks_from_context(self, obj): - request = self.context.get('request', None) - if request is not None: - return ['SomeInlineData', 'Somemoredata'] - return 'readOnly' + # FIXME: currently the user is exposed in the url. We don't want that, and we can probably avoid that but still save it under the user. + # We would probably be better off just let the user calculate the urls from the uid and a base url for the snapshot. + # E.g. chunkBaseUrl: "/media/bla/bla/" or chunkBaseUrl: "https://media.etesync.com/bla/bla" + def get_chunks_urls(self, obj): + ret = [] + for chunk in obj.chunks.all(): + ret.append(chunk.chunkFile.url) + + return ret + + +class CollectionItemSnapshotInlineSerializer(CollectionItemSnapshotBaseSerializer): + chunksData = serializers.SerializerMethodField('get_chunks_data') + + class Meta(CollectionItemSnapshotBaseSerializer.Meta): + fields = CollectionItemSnapshotBaseSerializer.Meta.fields + ('chunksData', ) + + def get_chunks_data(self, obj): + ret = [] + for chunk in obj.chunks.all(): + with open(chunk.chunkFile.path, 'rb') as f: + ret.append(base64.b64encode(f.read()).decode('ascii')) + return ret class CollectionItemSerializer(serializers.ModelSerializer): content = CollectionItemSnapshotSerializer(read_only=True, many=False) diff --git a/django_etesync/views.py b/django_etesync/views.py index 57136a4..5caa452 100644 --- a/django_etesync/views.py +++ b/django_etesync/views.py @@ -148,20 +148,21 @@ class CollectionItemViewSet(BaseViewSet): class CollectionItemChunkViewSet(viewsets.ViewSet): allowed_methods = ['GET', 'POST'] + parser_classes = (parsers.MultiPartParser, ) authentication_classes = BaseViewSet.authentication_classes permission_classes = BaseViewSet.permission_classes - parser_classes = (parsers.MultiPartParser, ) + serializer_class = CollectionItemChunkSerializer lookup_field = 'uid' - def create(self, request, collection_uid=None): + def create(self, request, collection_uid=None, collection_item_uid=None): # FIXME: we are potentially not getting the correct queryset - collection_object = Collection.objects.get(uid=collection_uid) + col = get_object_or_404(Collection.objects, uid=collection_uid) + col_it = get_object_or_404(col.items, uid=collection_item_uid) - many = isinstance(request.data, list) - serializer = self.serializer_class(data=request.data, many=many) + serializer = self.serializer_class(data=request.data) if serializer.is_valid(): try: - serializer.save(collection=collection_object) + serializer.save(item=col_it, order='abc') except IntegrityError: content = {'code': 'integrity_error'} return Response(content, status=status.HTTP_400_BAD_REQUEST) @@ -169,15 +170,3 @@ class CollectionItemChunkViewSet(viewsets.ViewSet): return Response({}, status=status.HTTP_201_CREATED) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) - - def destroy(self, request, collection_uid=None, uid=None): - # FIXME: implement - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def update(self, request, collection_uid=None, uid=None): - # FIXME: implement, or should it be implemented elsewhere? - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def partial_update(self, request, collection_uid=None, uid=None): - # FIXME: implement, or should it be implemented elsewhere? - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED)