More progress - support chunk uploading.

master
Tom Hacohen 5 years ago
parent 67fb714ddb
commit 0c44f738fd

@ -0,0 +1,20 @@
# Generated by Django 3.0.3 on 2020-02-20 12:16
from django.db import migrations, models
import django_etesync.models
class Migration(migrations.Migration):
dependencies = [
('django_etesync', '0007_auto_20200220_1144'),
]
operations = [
migrations.AddField(
model_name='collectionitemchunk',
name='chunkFile',
field=models.FileField(default='', upload_to=django_etesync.models.chunk_directory_path),
preserve_default=False,
),
]

@ -0,0 +1,19 @@
# Generated by Django 3.0.3 on 2020-02-20 12:20
from django.db import migrations, models
import django_etesync.models
class Migration(migrations.Migration):
dependencies = [
('django_etesync', '0008_collectionitemchunk_chunkfile'),
]
operations = [
migrations.AlterField(
model_name='collectionitemchunk',
name='chunkFile',
field=models.FileField(max_length=150, upload_to=django_etesync.models.chunk_directory_path),
),
]

@ -36,12 +36,6 @@ class Collection(models.Model):
return self.uid return self.uid
def chunk_directory_path(instance, filename):
col = instance.itemSnapshot.item.collection
user_id = col.owner.id
return Path('user_{}'.format(user_id), col.uid, instance.uid)
class CollectionItem(models.Model): class CollectionItem(models.Model):
uid = models.CharField(db_index=True, blank=False, null=False, uid = models.CharField(db_index=True, blank=False, null=False,
@ -59,12 +53,19 @@ class CollectionItem(models.Model):
return self.snapshots.get(current=True) return self.snapshots.get(current=True)
def chunk_directory_path(instance, filename):
item = instance.item
col = item.collection
user_id = col.owner.id
return Path('user_{}'.format(user_id), col.uid, item.uid, instance.uid)
class CollectionItemChunk(models.Model): class CollectionItemChunk(models.Model):
uid = models.CharField(db_index=True, blank=False, null=False, uid = models.CharField(db_index=True, blank=False, null=False,
max_length=44, validators=[UidValidator]) max_length=44, validators=[UidValidator])
item = models.ForeignKey(CollectionItem, related_name='chunks', on_delete=models.CASCADE) item = models.ForeignKey(CollectionItem, related_name='chunks', on_delete=models.CASCADE)
order = models.CharField(max_length=100, blank=False, null=False) order = models.CharField(max_length=100, blank=False, null=False)
# We probably just want to implement this manually because we can have more than one pointing to a file. chunkFile = models.FileField(upload_to=chunk_directory_path) chunkFile = models.FileField(upload_to=chunk_directory_path, max_length=150)
class Meta: class Meta:
unique_together = ('item', 'order') unique_together = ('item', 'order')
@ -87,4 +88,3 @@ class CollectionItemSnapshot(models.Model):
def __str__(self): def __str__(self):
return '{} {} current={}'.format(self.item.uid, self.id, self.current) return '{} {} current={}'.format(self.item.uid, self.id, self.current)

@ -61,10 +61,10 @@ class CollectionSerializer(serializers.ModelSerializer):
class CollectionItemChunkSerializer(serializers.ModelSerializer): class CollectionItemChunkSerializer(serializers.ModelSerializer):
class Meta: class Meta:
model = models.CollectionItemChunk model = models.CollectionItemChunk
fields = ('uid', ) fields = ('uid', 'chunkFile')
class CollectionItemSnapshotSerializer(serializers.ModelSerializer): class CollectionItemSnapshotBaseSerializer(serializers.ModelSerializer):
encryptionKey = BinaryBase64Field() encryptionKey = BinaryBase64Field()
chunks = serializers.SlugRelatedField( chunks = serializers.SlugRelatedField(
slug_field='uid', slug_field='uid',
@ -77,18 +77,36 @@ class CollectionItemSnapshotSerializer(serializers.ModelSerializer):
fields = ('version', 'encryptionKey', 'chunks', 'hmac') fields = ('version', 'encryptionKey', 'chunks', 'hmac')
class CollectionItemSnapshotInlineSerializer(CollectionItemSnapshotSerializer): class CollectionItemSnapshotSerializer(CollectionItemSnapshotBaseSerializer):
chunksData = serializers.SerializerMethodField('get_inline_chunks_from_context') chunksUrls = serializers.SerializerMethodField('get_chunks_urls')
class Meta(CollectionItemSnapshotSerializer.Meta): class Meta(CollectionItemSnapshotBaseSerializer.Meta):
fields = CollectionItemSnapshotSerializer.Meta.fields + ('chunksData', ) fields = CollectionItemSnapshotBaseSerializer.Meta.fields + ('chunksUrls', )
def get_inline_chunks_from_context(self, obj): # FIXME: currently the user is exposed in the url. We don't want that, and we can probably avoid that but still save it under the user.
request = self.context.get('request', None) # We would probably be better off just let the user calculate the urls from the uid and a base url for the snapshot.
if request is not None: # E.g. chunkBaseUrl: "/media/bla/bla/" or chunkBaseUrl: "https://media.etesync.com/bla/bla"
return ['SomeInlineData', 'Somemoredata'] def get_chunks_urls(self, obj):
return 'readOnly' ret = []
for chunk in obj.chunks.all():
ret.append(chunk.chunkFile.url)
return ret
class CollectionItemSnapshotInlineSerializer(CollectionItemSnapshotBaseSerializer):
chunksData = serializers.SerializerMethodField('get_chunks_data')
class Meta(CollectionItemSnapshotBaseSerializer.Meta):
fields = CollectionItemSnapshotBaseSerializer.Meta.fields + ('chunksData', )
def get_chunks_data(self, obj):
ret = []
for chunk in obj.chunks.all():
with open(chunk.chunkFile.path, 'rb') as f:
ret.append(base64.b64encode(f.read()).decode('ascii'))
return ret
class CollectionItemSerializer(serializers.ModelSerializer): class CollectionItemSerializer(serializers.ModelSerializer):
content = CollectionItemSnapshotSerializer(read_only=True, many=False) content = CollectionItemSnapshotSerializer(read_only=True, many=False)

@ -148,20 +148,21 @@ class CollectionItemViewSet(BaseViewSet):
class CollectionItemChunkViewSet(viewsets.ViewSet): class CollectionItemChunkViewSet(viewsets.ViewSet):
allowed_methods = ['GET', 'POST'] allowed_methods = ['GET', 'POST']
parser_classes = (parsers.MultiPartParser, )
authentication_classes = BaseViewSet.authentication_classes authentication_classes = BaseViewSet.authentication_classes
permission_classes = BaseViewSet.permission_classes permission_classes = BaseViewSet.permission_classes
parser_classes = (parsers.MultiPartParser, ) serializer_class = CollectionItemChunkSerializer
lookup_field = 'uid' lookup_field = 'uid'
def create(self, request, collection_uid=None): def create(self, request, collection_uid=None, collection_item_uid=None):
# FIXME: we are potentially not getting the correct queryset # FIXME: we are potentially not getting the correct queryset
collection_object = Collection.objects.get(uid=collection_uid) col = get_object_or_404(Collection.objects, uid=collection_uid)
col_it = get_object_or_404(col.items, uid=collection_item_uid)
many = isinstance(request.data, list) serializer = self.serializer_class(data=request.data)
serializer = self.serializer_class(data=request.data, many=many)
if serializer.is_valid(): if serializer.is_valid():
try: try:
serializer.save(collection=collection_object) serializer.save(item=col_it, order='abc')
except IntegrityError: except IntegrityError:
content = {'code': 'integrity_error'} content = {'code': 'integrity_error'}
return Response(content, status=status.HTTP_400_BAD_REQUEST) return Response(content, status=status.HTTP_400_BAD_REQUEST)
@ -169,15 +170,3 @@ class CollectionItemChunkViewSet(viewsets.ViewSet):
return Response({}, status=status.HTTP_201_CREATED) return Response({}, status=status.HTTP_201_CREATED)
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
def destroy(self, request, collection_uid=None, uid=None):
# FIXME: implement
return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED)
def update(self, request, collection_uid=None, uid=None):
# FIXME: implement, or should it be implemented elsewhere?
return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED)
def partial_update(self, request, collection_uid=None, uid=None):
# FIXME: implement, or should it be implemented elsewhere?
return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED)

Loading…
Cancel
Save