diff --git a/.gitignore b/.gitignore index aa7817e..7f220af 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,15 @@ -/db.sqlite3 +/journal +/db.sqlite3* Session.vim -/local_settings.py /.venv +/assets +/logs /.coverage -/htmlcov -/secret.txt -/static +/tmp +/media __pycache__ .*.swp + + +/etebase_server_settings.py diff --git a/README.md b/README.md index 1de61b6..6e28a43 100644 --- a/README.md +++ b/README.md @@ -1,20 +1,15 @@

-

EteSync - Secure Data Sync

+

Etebase - Encrypt Everything

-A skeleton app for running your own [EteSync](https://www.etesync.com) server +A skeleton app for running your own [Etebase](https://www.etebase.com) server # Installation -## Using pre-built packages - -* Arch Linux : [AUR](https://aur.archlinux.org/packages/etesync-server) -* Fedora : [COPR](https://copr.fedorainfracloud.org/coprs/daftaupe/etesync) - ## From source -Before installing the EteSync server make sure you install `virtualenv` (for **Python 3**): +Before installing the Etebase server make sure you install `virtualenv` (for **Python 3**): * Arch Linux: `pacman -S python-virtualenv` * Debian/Ubuntu: `apt-get install python3-virtualenv` @@ -23,9 +18,10 @@ Before installing the EteSync server make sure you install `virtualenv` (for **P Then just clone the git repo and set up this app: ``` -git clone https://github.com/etesync/server-skeleton.git +git clone https://github.com/etesync/server.git etebase -cd server-skeleton +cd etebase +git checkout etebase # Set up the environment and deps virtualenv -p python3 venv # If doesn't work, try: virtualenv3 venv @@ -37,11 +33,11 @@ pip install -r requirements.txt # Configuration If you are familiar with Django you can just edit the [settings file](etesync_server/settings.py) -according to the [Django deployment checklist](https://docs.djangoproject.com/en/dev/howto/deployment/checklist/) -if you are not, we also provide a simple [configuration file](etesync-server.ini.example) -for easy deployment which you can use. +according to the [Django deployment checklist](https://docs.djangoproject.com/en/dev/howto/deployment/checklist/). +If you are not, we also provide a simple [configuration file](https://github.com/etesync/server/blob/etebase/etebase-server.ini.example) for easy deployment which you can use. +To use the easy configuration file rename it to `etebase-server.ini` and place it either at the root of this repository or in `/etc/etebase-server`. -To use the easy configuration file rename it to `etesync-server.ini` and place it either at the root of this repository or in `/etc/etesync-server`. +There is also a [wikipage](https://github.com/etesync/server/wiki/Basic-Setup-Etebase-(EteSync-v2)) detailing this basic setup. Some particular settings that should be edited are: * [`ALLOWED_HOSTS`](https://docs.djangoproject.com/en/1.11/ref/settings/#std:setting-ALLOWED_HOSTS) @@ -54,7 +50,7 @@ will be served generation purposes. See below for how default configuration of `SECRET_KEY` works for this project. -Now you can initialise our django app +Now you can initialise our django app. ``` ./manage.py migrate @@ -70,16 +66,15 @@ Using the debug server in production is not recommended, so please read the foll # Production deployment -EteSync is based on Django so you should refer to one of the following +There are more details about a proper production setup using Daphne and Nginx in the [wiki](https://github.com/etesync/server/wiki/Production-setup-using-Daphne-and-Nginx). + +Etebase is based on Django so you should refer to one of the following * The instructions of the Django project [here](https://docs.djangoproject.com/en/2.2/howto/deployment/wsgi/). * Instructions from uwsgi [here](http://uwsgi-docs.readthedocs.io/en/latest/tutorials/Django_and_nginx.html). - * The [example configurations](example-configs) in this repo. -There are more details about a proper production setup using uWSGI and Nginx in the [wiki](https://github.com/etesync/server/wiki/Production-setup-using-uWSGI-and-Nginx). +The webserver should also be configured to serve Etebase using TLS. +A guide for doing so can be found in the [wiki](https://github.com/etesync/server/wiki/Setup-HTTPS-for-Etebase) as well. -The webserver should also be configured to serve EteSync using TLS. -A guide for doing so can be found in the [wiki](https://github.com/etesync/server/wiki/Setup-HTTPS-for-EteSync) as well. - # Usage Create yourself an admin user: @@ -88,12 +83,12 @@ Create yourself an admin user: ./manage.py createsuperuser ``` -At this stage you can either just use the admin user, or better yet, go to: ```www.your-etesync-install.com/admin``` -and create a non-privileged user that you can use. - -That's it! +At this stage you need to create accounts to be used with the EteSync apps. To do that, please go to: +`www.your-etesync-install.com/admin` and create a new user to be used with the service. -Now all that's left is to open the EteSync app, add an account, and set your custom server address under the "advance" section. +After this user has been created, you can use any of the EteSync apps to signup (not login!) with the same username and +email in order to set up the account. Please make sure to click "advance" and set your customer server address when you +do. # `SECRET_KEY` and `secret.txt` @@ -117,6 +112,6 @@ Then, inside the virtualenv: You can now restart the server. -# Supporting EteSync +# Supporting Etebase -Please consider registering an account even if you self-host in order to support the development of EteSync, or visit the [contribution](https://www.etesync.com/contribute/) for more information on how to support the service. +Please consider registering an account even if you self-host in order to support the development of Etebase, or visit the [contribution](https://www.etesync.com/contribute/) for more information on how to support the service. diff --git a/django_etebase/__init__.py b/django_etebase/__init__.py new file mode 100644 index 0000000..426fefd --- /dev/null +++ b/django_etebase/__init__.py @@ -0,0 +1 @@ +from .app_settings import app_settings diff --git a/django_etebase/admin.py b/django_etebase/admin.py new file mode 100644 index 0000000..8c38f3f --- /dev/null +++ b/django_etebase/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/django_etebase/app_settings.py b/django_etebase/app_settings.py new file mode 100644 index 0000000..3c580b2 --- /dev/null +++ b/django_etebase/app_settings.py @@ -0,0 +1,83 @@ +# Copyright © 2017 Tom Hacohen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, version 3. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +from django.utils.functional import cached_property + + +class AppSettings: + def __init__(self, prefix): + self.prefix = prefix + + def import_from_str(self, name): + from importlib import import_module + + path, prop = name.rsplit('.', 1) + + mod = import_module(path) + return getattr(mod, prop) + + def _setting(self, name, dflt): + from django.conf import settings + return getattr(settings, self.prefix + name, dflt) + + @cached_property + def API_PERMISSIONS(self): # pylint: disable=invalid-name + perms = self._setting("API_PERMISSIONS", ('rest_framework.permissions.IsAuthenticated', )) + ret = [] + for perm in perms: + ret.append(self.import_from_str(perm)) + return ret + + @cached_property + def API_AUTHENTICATORS(self): # pylint: disable=invalid-name + perms = self._setting("API_AUTHENTICATORS", ('rest_framework.authentication.TokenAuthentication', + 'rest_framework.authentication.SessionAuthentication')) + ret = [] + for perm in perms: + ret.append(self.import_from_str(perm)) + return ret + + @cached_property + def GET_USER_QUERYSET_FUNC(self): # pylint: disable=invalid-name + get_user_queryset = self._setting("GET_USER_QUERYSET_FUNC", None) + if get_user_queryset is not None: + return self.import_from_str(get_user_queryset) + return None + + @cached_property + def CREATE_USER_FUNC(self): # pylint: disable=invalid-name + func = self._setting("CREATE_USER_FUNC", None) + if func is not None: + return self.import_from_str(func) + return None + + @cached_property + def DASHBOARD_URL_FUNC(self): # pylint: disable=invalid-name + func = self._setting("DASHBOARD_URL_FUNC", None) + if func is not None: + return self.import_from_str(func) + return None + + @cached_property + def CHUNK_PATH_FUNC(self): # pylint: disable=invalid-name + func = self._setting("CHUNK_PATH_FUNC", None) + if func is not None: + return self.import_from_str(func) + return None + + @cached_property + def CHALLENGE_VALID_SECONDS(self): # pylint: disable=invalid-name + return self._setting("CHALLENGE_VALID_SECONDS", 60) + + +app_settings = AppSettings('ETEBASE_') diff --git a/django_etebase/apps.py b/django_etebase/apps.py new file mode 100644 index 0000000..286a708 --- /dev/null +++ b/django_etebase/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class DjangoEtebaseConfig(AppConfig): + name = 'django_etebase' diff --git a/django_etebase/drf_msgpack/__init__.py b/django_etebase/drf_msgpack/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_etebase/drf_msgpack/apps.py b/django_etebase/drf_msgpack/apps.py new file mode 100644 index 0000000..619e3e0 --- /dev/null +++ b/django_etebase/drf_msgpack/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class DrfMsgpackConfig(AppConfig): + name = 'drf_msgpack' diff --git a/django_etebase/drf_msgpack/migrations/__init__.py b/django_etebase/drf_msgpack/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_etebase/drf_msgpack/parsers.py b/django_etebase/drf_msgpack/parsers.py new file mode 100644 index 0000000..44cd33b --- /dev/null +++ b/django_etebase/drf_msgpack/parsers.py @@ -0,0 +1,14 @@ +import msgpack + +from rest_framework.parsers import BaseParser +from rest_framework.exceptions import ParseError + + +class MessagePackParser(BaseParser): + media_type = 'application/msgpack' + + def parse(self, stream, media_type=None, parser_context=None): + try: + return msgpack.unpackb(stream.read(), raw=False) + except Exception as exc: + raise ParseError('MessagePack parse error - %s' % str(exc)) diff --git a/django_etebase/drf_msgpack/renderers.py b/django_etebase/drf_msgpack/renderers.py new file mode 100644 index 0000000..9445231 --- /dev/null +++ b/django_etebase/drf_msgpack/renderers.py @@ -0,0 +1,15 @@ +import msgpack + +from rest_framework.renderers import BaseRenderer + + +class MessagePackRenderer(BaseRenderer): + media_type = 'application/msgpack' + format = 'msgpack' + render_style = 'binary' + charset = None + + def render(self, data, media_type=None, renderer_context=None): + if data is None: + return b'' + return msgpack.packb(data, use_bin_type=True) diff --git a/django_etebase/drf_msgpack/views.py b/django_etebase/drf_msgpack/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/django_etebase/drf_msgpack/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/django_etebase/exceptions.py b/django_etebase/exceptions.py new file mode 100644 index 0000000..d05c4e5 --- /dev/null +++ b/django_etebase/exceptions.py @@ -0,0 +1,10 @@ +from rest_framework import serializers, status + + +class EtebaseValidationError(serializers.ValidationError): + def __init__(self, code, detail, status_code=status.HTTP_400_BAD_REQUEST): + super().__init__({ + 'code': code, + 'detail': detail, + }) + self.status_code = status_code diff --git a/django_etebase/migrations/0001_initial.py b/django_etebase/migrations/0001_initial.py new file mode 100644 index 0000000..69a9a91 --- /dev/null +++ b/django_etebase/migrations/0001_initial.py @@ -0,0 +1,91 @@ +# Generated by Django 3.0.3 on 2020-05-13 13:01 + +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import django_etebase.models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Collection', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uid', models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='[a-zA-Z0-9]')])), + ('version', models.PositiveSmallIntegerField()), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('uid', 'owner')}, + }, + ), + migrations.CreateModel( + name='CollectionItem', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uid', models.CharField(db_index=True, max_length=44, null=True, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='[a-zA-Z0-9]')])), + ('version', models.PositiveSmallIntegerField()), + ('encryptionKey', models.BinaryField(editable=True, null=True)), + ('collection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='items', to='django_etebase.Collection')), + ], + options={ + 'unique_together': {('uid', 'collection')}, + }, + ), + migrations.CreateModel( + name='CollectionItemChunk', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uid', models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Expected a 256bit base64url.', regex='^[a-zA-Z0-9\\-_]{43}$')])), + ('chunkFile', models.FileField(max_length=150, unique=True, upload_to=django_etebase.models.chunk_directory_path)), + ('item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='chunks', to='django_etebase.CollectionItem')), + ], + ), + migrations.CreateModel( + name='CollectionItemRevision', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uid', models.CharField(db_index=True, max_length=44, unique=True, validators=[django.core.validators.RegexValidator(message='Expected a 256bit base64url.', regex='^[a-zA-Z0-9\\-_]{43}$')])), + ('meta', models.BinaryField(editable=True)), + ('current', models.BooleanField(db_index=True, default=True, null=True)), + ('deleted', models.BooleanField(default=False)), + ('item', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='revisions', to='django_etebase.CollectionItem')), + ], + options={ + 'unique_together': {('item', 'current')}, + }, + ), + migrations.CreateModel( + name='RevisionChunkRelation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('chunk', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='revisions_relation', to='django_etebase.CollectionItemChunk')), + ('revision', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='chunks_relation', to='django_etebase.CollectionItemRevision')), + ], + options={ + 'ordering': ('id',), + }, + ), + migrations.CreateModel( + name='CollectionMember', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('encryptionKey', models.BinaryField(editable=True)), + ('accessLevel', models.CharField(choices=[('adm', 'Admin'), ('rw', 'Read Write'), ('ro', 'Read Only')], default='ro', max_length=3)), + ('collection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='members', to='django_etebase.Collection')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('user', 'collection')}, + }, + ), + ] diff --git a/django_etebase/migrations/0002_userinfo.py b/django_etebase/migrations/0002_userinfo.py new file mode 100644 index 0000000..6da0bb8 --- /dev/null +++ b/django_etebase/migrations/0002_userinfo.py @@ -0,0 +1,25 @@ +# Generated by Django 3.0.3 on 2020-05-14 09:51 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('myauth', '0001_initial'), + ('django_etebase', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='UserInfo', + fields=[ + ('owner', models.OneToOneField(on_delete=django.db.models.deletion.CASCADE, primary_key=True, serialize=False, to=settings.AUTH_USER_MODEL)), + ('version', models.PositiveSmallIntegerField(default=1)), + ('pubkey', models.BinaryField(editable=True)), + ('salt', models.BinaryField(editable=True)), + ], + ), + ] diff --git a/django_etebase/migrations/0003_collectioninvitation.py b/django_etebase/migrations/0003_collectioninvitation.py new file mode 100644 index 0000000..8fd2066 --- /dev/null +++ b/django_etebase/migrations/0003_collectioninvitation.py @@ -0,0 +1,31 @@ +# Generated by Django 3.0.3 on 2020-05-20 11:03 + +from django.conf import settings +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('django_etebase', '0002_userinfo'), + ] + + operations = [ + migrations.CreateModel( + name='CollectionInvitation', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uid', models.CharField(db_index=True, max_length=44, validators=[django.core.validators.RegexValidator(message='Expected a 256bit base64url.', regex='^[a-zA-Z0-9\\-_]{43}$')])), + ('signedEncryptionKey', models.BinaryField()), + ('accessLevel', models.CharField(choices=[('adm', 'Admin'), ('rw', 'Read Write'), ('ro', 'Read Only')], default='ro', max_length=3)), + ('fromMember', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='django_etebase.CollectionMember')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='incoming_invitations', to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('user', 'fromMember')}, + }, + ), + ] diff --git a/django_etebase/migrations/0004_collectioninvitation_version.py b/django_etebase/migrations/0004_collectioninvitation_version.py new file mode 100644 index 0000000..4052116 --- /dev/null +++ b/django_etebase/migrations/0004_collectioninvitation_version.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-05-21 14:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0003_collectioninvitation'), + ] + + operations = [ + migrations.AddField( + model_name='collectioninvitation', + name='version', + field=models.PositiveSmallIntegerField(default=1), + ), + ] diff --git a/django_etebase/migrations/0005_auto_20200526_1021.py b/django_etebase/migrations/0005_auto_20200526_1021.py new file mode 100644 index 0000000..da0dc33 --- /dev/null +++ b/django_etebase/migrations/0005_auto_20200526_1021.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-05-26 10:21 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0004_collectioninvitation_version'), + ] + + operations = [ + migrations.RenameField( + model_name='userinfo', + old_name='pubkey', + new_name='loginPubkey', + ), + ] diff --git a/django_etebase/migrations/0006_auto_20200526_1040.py b/django_etebase/migrations/0006_auto_20200526_1040.py new file mode 100644 index 0000000..b86a996 --- /dev/null +++ b/django_etebase/migrations/0006_auto_20200526_1040.py @@ -0,0 +1,25 @@ +# Generated by Django 3.0.3 on 2020-05-26 10:40 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0005_auto_20200526_1021'), + ] + + operations = [ + migrations.AddField( + model_name='userinfo', + name='encryptedSeckey', + field=models.BinaryField(default=b'', editable=True), + preserve_default=False, + ), + migrations.AddField( + model_name='userinfo', + name='pubkey', + field=models.BinaryField(default=b'', editable=True), + preserve_default=False, + ), + ] diff --git a/django_etebase/migrations/0007_auto_20200526_1336.py b/django_etebase/migrations/0007_auto_20200526_1336.py new file mode 100644 index 0000000..79978c7 --- /dev/null +++ b/django_etebase/migrations/0007_auto_20200526_1336.py @@ -0,0 +1,39 @@ +# Generated by Django 3.0.3 on 2020-05-26 13:36 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0006_auto_20200526_1040'), + ] + + operations = [ + migrations.AlterField( + model_name='collection', + name='uid', + field=models.CharField(db_index=True, max_length=43, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9]*$')]), + ), + migrations.AlterField( + model_name='collectioninvitation', + name='uid', + field=models.CharField(db_index=True, max_length=43, validators=[django.core.validators.RegexValidator(message='Expected a base64url.', regex='^[a-zA-Z0-9\\-_]{42,43}$')]), + ), + migrations.AlterField( + model_name='collectionitem', + name='uid', + field=models.CharField(db_index=True, max_length=43, null=True, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9]*$')]), + ), + migrations.AlterField( + model_name='collectionitemchunk', + name='uid', + field=models.CharField(db_index=True, max_length=43, validators=[django.core.validators.RegexValidator(message='Expected a base64url.', regex='^[a-zA-Z0-9\\-_]{42,43}$')]), + ), + migrations.AlterField( + model_name='collectionitemrevision', + name='uid', + field=models.CharField(db_index=True, max_length=43, unique=True, validators=[django.core.validators.RegexValidator(message='Expected a base64url.', regex='^[a-zA-Z0-9\\-_]{42,43}$')]), + ), + ] diff --git a/django_etebase/migrations/0008_auto_20200526_1535.py b/django_etebase/migrations/0008_auto_20200526_1535.py new file mode 100644 index 0000000..12656c0 --- /dev/null +++ b/django_etebase/migrations/0008_auto_20200526_1535.py @@ -0,0 +1,28 @@ +# Generated by Django 3.0.3 on 2020-05-26 15:35 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion +import django_etebase.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0007_auto_20200526_1336'), + ] + + operations = [ + migrations.CreateModel( + name='Stoken', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uid', models.CharField(db_index=True, default=django_etebase.models.generate_stoken_uid, max_length=43, unique=True, validators=[django.core.validators.RegexValidator(message='Expected a base64url.', regex='^[a-zA-Z0-9\\-_]{42,43}$')])), + ], + ), + migrations.AddField( + model_name='collectionitemrevision', + name='stoken', + field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.PROTECT, to='django_etebase.Stoken'), + ), + ] diff --git a/django_etebase/migrations/0009_auto_20200526_1535.py b/django_etebase/migrations/0009_auto_20200526_1535.py new file mode 100644 index 0000000..a6ff498 --- /dev/null +++ b/django_etebase/migrations/0009_auto_20200526_1535.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.3 on 2020-05-26 15:35 + +from django.db import migrations + + +def create_stokens(apps, schema_editor): + Stoken = apps.get_model('django_etebase', 'Stoken') + CollectionItemRevision = apps.get_model('django_etebase', 'CollectionItemRevision') + + for rev in CollectionItemRevision.objects.all(): + rev.stoken = Stoken.objects.create() + rev.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0008_auto_20200526_1535'), + ] + + operations = [ + migrations.RunPython(create_stokens), + ] diff --git a/django_etebase/migrations/0010_auto_20200526_1539.py b/django_etebase/migrations/0010_auto_20200526_1539.py new file mode 100644 index 0000000..7ef0eca --- /dev/null +++ b/django_etebase/migrations/0010_auto_20200526_1539.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-05-26 15:39 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0009_auto_20200526_1535'), + ] + + operations = [ + migrations.AlterField( + model_name='collectionitemrevision', + name='stoken', + field=models.OneToOneField(on_delete=django.db.models.deletion.PROTECT, to='django_etebase.Stoken'), + ), + ] diff --git a/django_etebase/migrations/0011_collectionmember_stoken.py b/django_etebase/migrations/0011_collectionmember_stoken.py new file mode 100644 index 0000000..bafaea7 --- /dev/null +++ b/django_etebase/migrations/0011_collectionmember_stoken.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-05-27 07:43 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0010_auto_20200526_1539'), + ] + + operations = [ + migrations.AddField( + model_name='collectionmember', + name='stoken', + field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.PROTECT, to='django_etebase.Stoken'), + ), + ] diff --git a/django_etebase/migrations/0012_auto_20200527_0743.py b/django_etebase/migrations/0012_auto_20200527_0743.py new file mode 100644 index 0000000..ab6adbc --- /dev/null +++ b/django_etebase/migrations/0012_auto_20200527_0743.py @@ -0,0 +1,23 @@ +# Generated by Django 3.0.3 on 2020-05-27 07:43 + +from django.db import migrations + + +def create_stokens(apps, schema_editor): + Stoken = apps.get_model('django_etebase', 'Stoken') + CollectionMember = apps.get_model('django_etebase', 'CollectionMember') + + for member in CollectionMember.objects.all(): + member.stoken = Stoken.objects.create() + member.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0011_collectionmember_stoken'), + ] + + operations = [ + migrations.RunPython(create_stokens), + ] diff --git a/django_etebase/migrations/0013_collectionmemberremoved.py b/django_etebase/migrations/0013_collectionmemberremoved.py new file mode 100644 index 0000000..2641c03 --- /dev/null +++ b/django_etebase/migrations/0013_collectionmemberremoved.py @@ -0,0 +1,28 @@ +# Generated by Django 3.0.3 on 2020-05-27 11:29 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('django_etebase', '0012_auto_20200527_0743'), + ] + + operations = [ + migrations.CreateModel( + name='CollectionMemberRemoved', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('collection', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='removed_members', to='django_etebase.Collection')), + ('stoken', models.OneToOneField(null=True, on_delete=django.db.models.deletion.PROTECT, to='django_etebase.Stoken')), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + options={ + 'unique_together': {('user', 'collection')}, + }, + ), + ] diff --git a/django_etebase/migrations/0014_auto_20200602_1558.py b/django_etebase/migrations/0014_auto_20200602_1558.py new file mode 100644 index 0000000..d1a555d --- /dev/null +++ b/django_etebase/migrations/0014_auto_20200602_1558.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-06-02 15:58 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0013_collectionmemberremoved'), + ] + + operations = [ + migrations.RenameField( + model_name='userinfo', + old_name='encryptedSeckey', + new_name='encryptedContent', + ), + ] diff --git a/django_etebase/migrations/0015_collectionitemrevision_salt.py b/django_etebase/migrations/0015_collectionitemrevision_salt.py new file mode 100644 index 0000000..7f3dd71 --- /dev/null +++ b/django_etebase/migrations/0015_collectionitemrevision_salt.py @@ -0,0 +1,18 @@ +# Generated by Django 3.0.3 on 2020-06-04 12:18 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0014_auto_20200602_1558'), + ] + + operations = [ + migrations.AddField( + model_name='collectionitemrevision', + name='salt', + field=models.BinaryField(default=b'', editable=True), + ), + ] diff --git a/django_etebase/migrations/0016_auto_20200623_0820.py b/django_etebase/migrations/0016_auto_20200623_0820.py new file mode 100644 index 0000000..2c11157 --- /dev/null +++ b/django_etebase/migrations/0016_auto_20200623_0820.py @@ -0,0 +1,31 @@ +# Generated by Django 3.0.3 on 2020-06-23 08:20 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0015_collectionitemrevision_salt'), + ] + + operations = [ + migrations.AddField( + model_name='collection', + name='main_item', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, related_name='parent', to='django_etebase.CollectionItem'), + ), + migrations.AlterUniqueTogether( + name='collection', + unique_together=set(), + ), + migrations.RemoveField( + model_name='collection', + name='uid', + ), + migrations.RemoveField( + model_name='collection', + name='version', + ), + ] diff --git a/django_etebase/migrations/0017_auto_20200623_0958.py b/django_etebase/migrations/0017_auto_20200623_0958.py new file mode 100644 index 0000000..e244b13 --- /dev/null +++ b/django_etebase/migrations/0017_auto_20200623_0958.py @@ -0,0 +1,25 @@ +# Generated by Django 3.0.3 on 2020-06-23 09:58 + +import django.core.validators +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0016_auto_20200623_0820'), + ] + + operations = [ + migrations.AlterField( + model_name='collection', + name='main_item', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='parent', to='django_etebase.CollectionItem'), + ), + migrations.AlterField( + model_name='collectionitem', + name='uid', + field=models.CharField(db_index=True, max_length=43, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9]*$')]), + ), + ] diff --git a/django_etebase/migrations/0018_auto_20200624_0748.py b/django_etebase/migrations/0018_auto_20200624_0748.py new file mode 100644 index 0000000..ec59e0c --- /dev/null +++ b/django_etebase/migrations/0018_auto_20200624_0748.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-06-24 07:48 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0017_auto_20200623_0958'), + ] + + operations = [ + migrations.AlterField( + model_name='collectionitem', + name='uid', + field=models.CharField(db_index=True, max_length=43, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9\\-_]*$')]), + ), + ] diff --git a/django_etebase/migrations/0019_auto_20200626_0748.py b/django_etebase/migrations/0019_auto_20200626_0748.py new file mode 100644 index 0000000..991ca50 --- /dev/null +++ b/django_etebase/migrations/0019_auto_20200626_0748.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-06-26 07:48 + +import django.core.validators +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0018_auto_20200624_0748'), + ] + + operations = [ + migrations.AlterField( + model_name='collectionitemchunk', + name='uid', + field=models.CharField(db_index=True, max_length=60, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9\\-_]*$')]), + ), + ] diff --git a/django_etebase/migrations/0020_remove_collectionitemrevision_salt.py b/django_etebase/migrations/0020_remove_collectionitemrevision_salt.py new file mode 100644 index 0000000..2df32bf --- /dev/null +++ b/django_etebase/migrations/0020_remove_collectionitemrevision_salt.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.3 on 2020-06-26 08:19 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0019_auto_20200626_0748'), + ] + + operations = [ + migrations.RemoveField( + model_name='collectionitemrevision', + name='salt', + ), + ] diff --git a/django_etebase/migrations/0021_auto_20200626_0913.py b/django_etebase/migrations/0021_auto_20200626_0913.py new file mode 100644 index 0000000..b890384 --- /dev/null +++ b/django_etebase/migrations/0021_auto_20200626_0913.py @@ -0,0 +1,40 @@ +# Generated by Django 3.0.3 on 2020-06-26 09:13 + +import django.core.validators +from django.db import migrations, models +import django_etebase.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0020_remove_collectionitemrevision_salt'), + ] + + operations = [ + migrations.AlterField( + model_name='collectioninvitation', + name='uid', + field=models.CharField(db_index=True, max_length=43, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9\\-_]{20,}$')]), + ), + migrations.AlterField( + model_name='collectionitem', + name='uid', + field=models.CharField(db_index=True, max_length=43, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9\\-_]{20,}$')]), + ), + migrations.AlterField( + model_name='collectionitemchunk', + name='uid', + field=models.CharField(db_index=True, max_length=60, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9\\-_]{20,}$')]), + ), + migrations.AlterField( + model_name='collectionitemrevision', + name='uid', + field=models.CharField(db_index=True, max_length=43, unique=True, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9\\-_]{20,}$')]), + ), + migrations.AlterField( + model_name='stoken', + name='uid', + field=models.CharField(db_index=True, default=django_etebase.models.generate_stoken_uid, max_length=43, unique=True, validators=[django.core.validators.RegexValidator(message='Not a valid UID', regex='^[a-zA-Z0-9\\-_]{20,}$')]), + ), + ] diff --git a/django_etebase/migrations/0022_auto_20200804_1059.py b/django_etebase/migrations/0022_auto_20200804_1059.py new file mode 100644 index 0000000..c47e562 --- /dev/null +++ b/django_etebase/migrations/0022_auto_20200804_1059.py @@ -0,0 +1,17 @@ +# Generated by Django 3.0.3 on 2020-08-04 10:59 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0021_auto_20200626_0913'), + ] + + operations = [ + migrations.AlterUniqueTogether( + name='collectionitemchunk', + unique_together={('item', 'uid')}, + ), + ] diff --git a/django_etebase/migrations/0023_collectionitemchunk_collection.py b/django_etebase/migrations/0023_collectionitemchunk_collection.py new file mode 100644 index 0000000..b5d6841 --- /dev/null +++ b/django_etebase/migrations/0023_collectionitemchunk_collection.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-08-04 12:08 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0022_auto_20200804_1059'), + ] + + operations = [ + migrations.AddField( + model_name='collectionitemchunk', + name='collection', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='chunks', to='django_etebase.Collection'), + ), + ] diff --git a/django_etebase/migrations/0024_auto_20200804_1209.py b/django_etebase/migrations/0024_auto_20200804_1209.py new file mode 100644 index 0000000..54c80a3 --- /dev/null +++ b/django_etebase/migrations/0024_auto_20200804_1209.py @@ -0,0 +1,22 @@ +# Generated by Django 3.0.3 on 2020-08-04 12:09 + +from django.db import migrations + + +def change_chunk_to_collections(apps, schema_editor): + CollectionItemChunk = apps.get_model('django_etebase', 'CollectionItemChunk') + + for chunk in CollectionItemChunk.objects.all(): + chunk.collection = chunk.item.collection + chunk.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0023_collectionitemchunk_collection'), + ] + + operations = [ + migrations.RunPython(change_chunk_to_collections), + ] diff --git a/django_etebase/migrations/0025_auto_20200804_1216.py b/django_etebase/migrations/0025_auto_20200804_1216.py new file mode 100644 index 0000000..8849f53 --- /dev/null +++ b/django_etebase/migrations/0025_auto_20200804_1216.py @@ -0,0 +1,27 @@ +# Generated by Django 3.0.3 on 2020-08-04 12:16 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0024_auto_20200804_1209'), + ] + + operations = [ + migrations.AlterField( + model_name='collectionitemchunk', + name='collection', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='chunks', to='django_etebase.Collection'), + ), + migrations.AlterUniqueTogether( + name='collectionitemchunk', + unique_together={('collection', 'uid')}, + ), + migrations.RemoveField( + model_name='collectionitemchunk', + name='item', + ), + ] diff --git a/django_etebase/migrations/0026_auto_20200907_0752.py b/django_etebase/migrations/0026_auto_20200907_0752.py new file mode 100644 index 0000000..38c0b92 --- /dev/null +++ b/django_etebase/migrations/0026_auto_20200907_0752.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1 on 2020-09-07 07:52 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0025_auto_20200804_1216'), + ] + + operations = [ + migrations.RenameField( + model_name='collectioninvitation', + old_name='accessLevel', + new_name='accessLevelOld', + ), + migrations.RenameField( + model_name='collectionmember', + old_name='accessLevel', + new_name='accessLevelOld', + ), + ] diff --git a/django_etebase/migrations/0027_auto_20200907_0752.py b/django_etebase/migrations/0027_auto_20200907_0752.py new file mode 100644 index 0000000..d822d3d --- /dev/null +++ b/django_etebase/migrations/0027_auto_20200907_0752.py @@ -0,0 +1,23 @@ +# Generated by Django 3.1 on 2020-09-07 07:52 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0026_auto_20200907_0752'), + ] + + operations = [ + migrations.AddField( + model_name='collectioninvitation', + name='accessLevel', + field=models.IntegerField(choices=[(0, 'Read Only'), (1, 'Admin'), (2, 'Read Write')], default=0), + ), + migrations.AddField( + model_name='collectionmember', + name='accessLevel', + field=models.IntegerField(choices=[(0, 'Read Only'), (1, 'Admin'), (2, 'Read Write')], default=0), + ), + ] diff --git a/django_etebase/migrations/0028_auto_20200907_0754.py b/django_etebase/migrations/0028_auto_20200907_0754.py new file mode 100644 index 0000000..cb62e63 --- /dev/null +++ b/django_etebase/migrations/0028_auto_20200907_0754.py @@ -0,0 +1,39 @@ +# Generated by Django 3.1 on 2020-09-07 07:54 + +from django.db import migrations + +from django_etebase.models import AccessLevels + + +def change_access_level_to_int(apps, schema_editor): + CollectionMember = apps.get_model('django_etebase', 'CollectionMember') + CollectionInvitation = apps.get_model('django_etebase', 'CollectionInvitation') + + for member in CollectionMember.objects.all(): + if member.accessLevelOld == 'adm': + member.accessLevel = AccessLevels.ADMIN + elif member.accessLevelOld == 'rw': + member.accessLevel = AccessLevels.READ_WRITE + elif member.accessLevelOld == 'ro': + member.accessLevel = AccessLevels.READ_ONLY + member.save() + + for invitation in CollectionInvitation.objects.all(): + if invitation.accessLevelOld == 'adm': + invitation.accessLevel = AccessLevels.ADMIN + elif invitation.accessLevelOld == 'rw': + invitation.accessLevel = AccessLevels.READ_WRITE + elif invitation.accessLevelOld == 'ro': + invitation.accessLevel = AccessLevels.READ_ONLY + invitation.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0027_auto_20200907_0752'), + ] + + operations = [ + migrations.RunPython(change_access_level_to_int), + ] diff --git a/django_etebase/migrations/0029_auto_20200907_0801.py b/django_etebase/migrations/0029_auto_20200907_0801.py new file mode 100644 index 0000000..7cd54d4 --- /dev/null +++ b/django_etebase/migrations/0029_auto_20200907_0801.py @@ -0,0 +1,21 @@ +# Generated by Django 3.1 on 2020-09-07 08:01 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0028_auto_20200907_0754'), + ] + + operations = [ + migrations.RemoveField( + model_name='collectioninvitation', + name='accessLevelOld', + ), + migrations.RemoveField( + model_name='collectionmember', + name='accessLevelOld', + ), + ] diff --git a/django_etebase/migrations/0030_auto_20200922_0832.py b/django_etebase/migrations/0030_auto_20200922_0832.py new file mode 100644 index 0000000..d5fa95d --- /dev/null +++ b/django_etebase/migrations/0030_auto_20200922_0832.py @@ -0,0 +1,19 @@ +# Generated by Django 3.1.1 on 2020-09-22 08:32 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0029_auto_20200907_0801'), + ] + + operations = [ + migrations.AlterField( + model_name='collection', + name='main_item', + field=models.OneToOneField(null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='parent', to='django_etebase.collectionitem'), + ), + ] diff --git a/django_etebase/migrations/0031_auto_20201013_1336.py b/django_etebase/migrations/0031_auto_20201013_1336.py new file mode 100644 index 0000000..ca45dd4 --- /dev/null +++ b/django_etebase/migrations/0031_auto_20201013_1336.py @@ -0,0 +1,29 @@ +# Generated by Django 3.1.1 on 2020-10-13 13:36 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('django_etebase', '0030_auto_20200922_0832'), + ] + + operations = [ + migrations.CreateModel( + name='CollectionType', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('uid', models.BinaryField(db_index=True, editable=True)), + ('owner', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to=settings.AUTH_USER_MODEL)), + ], + ), + migrations.AddField( + model_name='collectionmember', + name='collectionType', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='django_etebase.collectiontype'), + ), + ] diff --git a/django_etebase/migrations/0032_auto_20201013_1409.py b/django_etebase/migrations/0032_auto_20201013_1409.py new file mode 100644 index 0000000..5594006 --- /dev/null +++ b/django_etebase/migrations/0032_auto_20201013_1409.py @@ -0,0 +1,18 @@ +# Generated by Django 3.1.1 on 2020-10-13 14:09 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_etebase', '0031_auto_20201013_1336'), + ] + + operations = [ + migrations.AlterField( + model_name='collectiontype', + name='uid', + field=models.BinaryField(db_index=True, editable=True, unique=True), + ), + ] diff --git a/django_etebase/migrations/__init__.py b/django_etebase/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_etebase/models.py b/django_etebase/models.py new file mode 100644 index 0000000..0036884 --- /dev/null +++ b/django_etebase/models.py @@ -0,0 +1,241 @@ +# Copyright © 2017 Tom Hacohen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, version 3. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from pathlib import Path + +from django.db import models, transaction +from django.conf import settings +from django.core.validators import RegexValidator +from django.db.models import Q +from django.utils.functional import cached_property +from django.utils.crypto import get_random_string + +from rest_framework import status + +from . import app_settings +from .exceptions import EtebaseValidationError + + +UidValidator = RegexValidator(regex=r'^[a-zA-Z0-9\-_]{20,}$', message='Not a valid UID') + + +class CollectionType(models.Model): + owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + uid = models.BinaryField(editable=True, blank=False, null=False, db_index=True, unique=True) + + +class Collection(models.Model): + main_item = models.OneToOneField('CollectionItem', related_name='parent', null=True, on_delete=models.SET_NULL) + owner = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + + def __str__(self): + return self.uid + + @cached_property + def uid(self): + return self.main_item.uid + + @property + def content(self): + return self.main_item.content + + @property + def etag(self): + return self.content.uid + + @cached_property + def stoken(self): + stoken = Stoken.objects.filter( + Q(collectionitemrevision__item__collection=self) | Q(collectionmember__collection=self) + ).order_by('id').last() + + if stoken is None: + raise Exception('stoken is None. Should never happen') + + return stoken.uid + + def validate_unique(self, exclude=None): + super().validate_unique(exclude=exclude) + if exclude is None or 'main_item' in exclude: + return + + if self.__class__.objects.filter(owner=self.owner, main_item__uid=self.main_item.uid) \ + .exclude(id=self.id).exists(): + raise EtebaseValidationError('unique_uid', 'Collection with this uid already exists', + status_code=status.HTTP_409_CONFLICT) + + +class CollectionItem(models.Model): + uid = models.CharField(db_index=True, blank=False, + max_length=43, validators=[UidValidator]) + collection = models.ForeignKey(Collection, related_name='items', on_delete=models.CASCADE) + version = models.PositiveSmallIntegerField() + encryptionKey = models.BinaryField(editable=True, blank=False, null=True) + + class Meta: + unique_together = ('uid', 'collection') + + def __str__(self): + return '{} {}'.format(self.uid, self.collection.uid) + + @cached_property + def content(self): + return self.revisions.get(current=True) + + @property + def etag(self): + return self.content.uid + + +def chunk_directory_path(instance, filename): + custom_func = app_settings.CHUNK_PATH_FUNC + if custom_func is not None: + return custom_func(instance, filename) + + col = instance.collection + user_id = col.owner.id + uid_prefix = instance.uid[:2] + uid_rest = instance.uid[2:] + return Path('user_{}'.format(user_id), col.uid, uid_prefix, uid_rest) + + +class CollectionItemChunk(models.Model): + uid = models.CharField(db_index=True, blank=False, null=False, + max_length=60, validators=[UidValidator]) + collection = models.ForeignKey(Collection, related_name='chunks', on_delete=models.CASCADE) + chunkFile = models.FileField(upload_to=chunk_directory_path, max_length=150, unique=True) + + def __str__(self): + return self.uid + + class Meta: + unique_together = ('collection', 'uid') + + +def generate_stoken_uid(): + return get_random_string(32, allowed_chars='abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_') + + +class Stoken(models.Model): + uid = models.CharField(db_index=True, unique=True, blank=False, null=False, default=generate_stoken_uid, + max_length=43, validators=[UidValidator]) + + +class CollectionItemRevision(models.Model): + stoken = models.OneToOneField(Stoken, on_delete=models.PROTECT) + uid = models.CharField(db_index=True, unique=True, blank=False, null=False, + max_length=43, validators=[UidValidator]) + item = models.ForeignKey(CollectionItem, related_name='revisions', on_delete=models.CASCADE) + meta = models.BinaryField(editable=True, blank=False, null=False) + current = models.BooleanField(db_index=True, default=True, null=True) + deleted = models.BooleanField(default=False) + + class Meta: + unique_together = ('item', 'current') + + def __str__(self): + return '{} {} current={}'.format(self.uid, self.item.uid, self.current) + + +class RevisionChunkRelation(models.Model): + chunk = models.ForeignKey(CollectionItemChunk, related_name='revisions_relation', on_delete=models.CASCADE) + revision = models.ForeignKey(CollectionItemRevision, related_name='chunks_relation', on_delete=models.CASCADE) + + class Meta: + ordering = ('id', ) + + +class AccessLevels(models.IntegerChoices): + READ_ONLY = 0 + ADMIN = 1 + READ_WRITE = 2 + + +class CollectionMember(models.Model): + stoken = models.OneToOneField(Stoken, on_delete=models.PROTECT, null=True) + collection = models.ForeignKey(Collection, related_name='members', on_delete=models.CASCADE) + user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + encryptionKey = models.BinaryField(editable=True, blank=False, null=False) + collectionType = models.ForeignKey(CollectionType, on_delete=models.PROTECT, null=True) + accessLevel = models.IntegerField( + choices=AccessLevels.choices, + default=AccessLevels.READ_ONLY, + ) + + class Meta: + unique_together = ('user', 'collection') + + def __str__(self): + return '{} {}'.format(self.collection.uid, self.user) + + def revoke(self): + with transaction.atomic(): + CollectionMemberRemoved.objects.update_or_create( + collection=self.collection, user=self.user, + defaults={ + 'stoken': Stoken.objects.create(), + }, + ) + + self.delete() + + +class CollectionMemberRemoved(models.Model): + stoken = models.OneToOneField(Stoken, on_delete=models.PROTECT, null=True) + collection = models.ForeignKey(Collection, related_name='removed_members', on_delete=models.CASCADE) + user = models.ForeignKey(settings.AUTH_USER_MODEL, on_delete=models.CASCADE) + + class Meta: + unique_together = ('user', 'collection') + + def __str__(self): + return '{} {}'.format(self.collection.uid, self.user) + + +class CollectionInvitation(models.Model): + uid = models.CharField(db_index=True, blank=False, null=False, + max_length=43, validators=[UidValidator]) + version = models.PositiveSmallIntegerField(default=1) + fromMember = models.ForeignKey(CollectionMember, on_delete=models.CASCADE) + # FIXME: make sure to delete all invitations for the same collection once one is accepted + # Make sure to not allow invitations if already a member + + user = models.ForeignKey(settings.AUTH_USER_MODEL, related_name='incoming_invitations', on_delete=models.CASCADE) + signedEncryptionKey = models.BinaryField(editable=False, blank=False, null=False) + accessLevel = models.IntegerField( + choices=AccessLevels.choices, + default=AccessLevels.READ_ONLY, + ) + + class Meta: + unique_together = ('user', 'fromMember') + + def __str__(self): + return '{} {}'.format(self.fromMember.collection.uid, self.user) + + @cached_property + def collection(self): + return self.fromMember.collection + + +class UserInfo(models.Model): + owner = models.OneToOneField(settings.AUTH_USER_MODEL, on_delete=models.CASCADE, primary_key=True) + version = models.PositiveSmallIntegerField(default=1) + loginPubkey = models.BinaryField(editable=True, blank=False, null=False) + pubkey = models.BinaryField(editable=True, blank=False, null=False) + encryptedContent = models.BinaryField(editable=True, blank=False, null=False) + salt = models.BinaryField(editable=True, blank=False, null=False) + + def __str__(self): + return "UserInfo<{}>".format(self.owner) diff --git a/django_etebase/parsers.py b/django_etebase/parsers.py new file mode 100644 index 0000000..1ca1a70 --- /dev/null +++ b/django_etebase/parsers.py @@ -0,0 +1,15 @@ +from rest_framework.parsers import FileUploadParser + + +class ChunkUploadParser(FileUploadParser): + """ + Parser for chunk upload data. + """ + media_type = 'application/octet-stream' + + def get_filename(self, stream, media_type, parser_context): + """ + Detects the uploaded file name. + """ + view = parser_context['view'] + return parser_context['kwargs'][view.lookup_field] diff --git a/django_etebase/permissions.py b/django_etebase/permissions.py new file mode 100644 index 0000000..c624404 --- /dev/null +++ b/django_etebase/permissions.py @@ -0,0 +1,90 @@ +# Copyright © 2017 Tom Hacohen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, version 3. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from rest_framework import permissions +from django_etebase.models import Collection, AccessLevels + + +def is_collection_admin(collection, user): + member = collection.members.filter(user=user).first() + return (member is not None) and (member.accessLevel == AccessLevels.ADMIN) + + +class IsCollectionAdmin(permissions.BasePermission): + """ + Custom permission to only allow owners of a collection to view it + """ + message = { + 'detail': 'Only collection admins can perform this operation.', + 'code': 'admin_access_required', + } + + def has_permission(self, request, view): + collection_uid = view.kwargs['collection_uid'] + try: + collection = view.get_collection_queryset().get(main_item__uid=collection_uid) + return is_collection_admin(collection, request.user) + except Collection.DoesNotExist: + # If the collection does not exist, we want to 404 later, not permission denied. + return True + + +class IsCollectionAdminOrReadOnly(permissions.BasePermission): + """ + Custom permission to only allow owners of a collection to edit it + """ + message = { + 'detail': 'Only collection admins can edit collections.', + 'code': 'admin_access_required', + } + + def has_permission(self, request, view): + collection_uid = view.kwargs.get('collection_uid', None) + + # Allow creating new collections + if collection_uid is None: + return True + + try: + collection = view.get_collection_queryset().get(main_item__uid=collection_uid) + if request.method in permissions.SAFE_METHODS: + return True + + return is_collection_admin(collection, request.user) + except Collection.DoesNotExist: + # If the collection does not exist, we want to 404 later, not permission denied. + return True + + +class HasWriteAccessOrReadOnly(permissions.BasePermission): + """ + Custom permission to restrict write + """ + message = { + 'detail': 'You need write access to write to this collection', + 'code': 'no_write_access', + } + + def has_permission(self, request, view): + collection_uid = view.kwargs['collection_uid'] + try: + collection = view.get_collection_queryset().get(main_item__uid=collection_uid) + if request.method in permissions.SAFE_METHODS: + return True + else: + member = collection.members.get(user=request.user) + return member.accessLevel != AccessLevels.READ_ONLY + except Collection.DoesNotExist: + # If the collection does not exist, we want to 404 later, not permission denied. + return True diff --git a/django_etebase/renderers.py b/django_etebase/renderers.py new file mode 100644 index 0000000..43c1a0d --- /dev/null +++ b/django_etebase/renderers.py @@ -0,0 +1,18 @@ +from rest_framework.utils.encoders import JSONEncoder as DRFJSONEncoder +from rest_framework.renderers import JSONRenderer as DRFJSONRenderer + +from .serializers import b64encode + + +class JSONEncoder(DRFJSONEncoder): + def default(self, obj): + if isinstance(obj, bytes) or isinstance(obj, memoryview): + return b64encode(obj) + return super().default(obj) + + +class JSONRenderer(DRFJSONRenderer): + """ + Renderer which serializes to JSON with support for our base64 + """ + encoder_class = JSONEncoder diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py new file mode 100644 index 0000000..038e879 --- /dev/null +++ b/django_etebase/serializers.py @@ -0,0 +1,563 @@ +# Copyright © 2017 Tom Hacohen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, version 3. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import base64 + +from django.core.files.base import ContentFile +from django.core import exceptions as django_exceptions +from django.contrib.auth import get_user_model +from django.db import IntegrityError, transaction +from rest_framework import serializers, status +from . import models +from .utils import get_user_queryset, create_user + +from .exceptions import EtebaseValidationError + +User = get_user_model() + + +def process_revisions_for_item(item, revision_data): + chunks_objs = [] + chunks = revision_data.pop('chunks_relation') + for chunk in chunks: + uid = chunk[0] + chunk_obj = models.CollectionItemChunk.objects.filter(uid=uid).first() + if len(chunk) > 1: + content = chunk[1] + # If the chunk already exists we assume it's fine. Otherwise, we upload it. + if chunk_obj is None: + chunk_obj = models.CollectionItemChunk(uid=uid, collection=item.collection) + chunk_obj.chunkFile.save('IGNORED', ContentFile(content)) + chunk_obj.save() + else: + if chunk_obj is None: + raise EtebaseValidationError('chunk_no_content', 'Tried to create a new chunk without content') + + chunks_objs.append(chunk_obj) + + stoken = models.Stoken.objects.create() + + revision = models.CollectionItemRevision.objects.create(**revision_data, item=item, stoken=stoken) + for chunk in chunks_objs: + models.RevisionChunkRelation.objects.create(chunk=chunk, revision=revision) + return revision + + +def b64encode(value): + return base64.urlsafe_b64encode(value).decode('ascii').strip('=') + + +def b64decode(data): + data += "=" * ((4 - len(data) % 4) % 4) + return base64.urlsafe_b64decode(data) + + +def b64decode_or_bytes(data): + if isinstance(data, bytes): + return data + else: + return b64decode(data) + + +class BinaryBase64Field(serializers.Field): + def to_representation(self, value): + return value + + def to_internal_value(self, data): + return b64decode_or_bytes(data) + + +class CollectionEncryptionKeyField(BinaryBase64Field): + def get_attribute(self, instance): + request = self.context.get('request', None) + if request is not None: + return instance.members.get(user=request.user).encryptionKey + return None + + +class CollectionTypeField(BinaryBase64Field): + def get_attribute(self, instance): + request = self.context.get('request', None) + if request is not None: + collection_type = instance.members.get(user=request.user).collectionType + return collection_type and collection_type.uid + return None + + +class UserSlugRelatedField(serializers.SlugRelatedField): + def get_queryset(self): + view = self.context.get('view', None) + return get_user_queryset(super().get_queryset(), view) + + def __init__(self, **kwargs): + super().__init__(slug_field=User.USERNAME_FIELD, **kwargs) + + def to_internal_value(self, data): + return super().to_internal_value(data.lower()) + + +class ChunksField(serializers.RelatedField): + def to_representation(self, obj): + obj = obj.chunk + if self.context.get('prefetch') == 'auto': + with open(obj.chunkFile.path, 'rb') as f: + return (obj.uid, f.read()) + else: + return (obj.uid, ) + + def to_internal_value(self, data): + if data[0] is None or data[1] is None: + raise EtebaseValidationError('no_null', 'null is not allowed') + return (data[0], b64decode_or_bytes(data[1])) + + +class BetterErrorsMixin: + @property + def errors(self): + nice = [] + errors = super().errors + for error_type in errors: + if error_type == 'non_field_errors': + nice.extend( + self.flatten_errors(None, errors[error_type]) + ) + else: + nice.extend( + self.flatten_errors(error_type, errors[error_type]) + ) + if nice: + return {'code': 'field_errors', + 'detail': 'Field validations failed.', + 'errors': nice} + return {} + + def flatten_errors(self, field_name, errors): + ret = [] + if isinstance(errors, dict): + for error_key in errors: + error = errors[error_key] + ret.extend(self.flatten_errors("{}.{}".format(field_name, error_key), error)) + else: + for error in errors: + if hasattr(error, 'detail'): + message = error.detail[0] + elif hasattr(error, 'message'): + message = error.message + else: + message = str(error) + ret.append({ + 'field': field_name, + 'code': error.code, + 'detail': message, + }) + return ret + + def transform_validation_error(self, prefix, err): + if hasattr(err, 'error_dict'): + errors = self.flatten_errors(prefix, err.error_dict) + elif not hasattr(err, 'message'): + errors = self.flatten_errors(prefix, err.error_list) + else: + raise EtebaseValidationError(err.code, err.message) + + raise serializers.ValidationError({ + 'code': 'field_errors', + 'detail': 'Field validations failed.', + 'errors': errors, + }) + + +class CollectionItemChunkSerializer(BetterErrorsMixin, serializers.ModelSerializer): + class Meta: + model = models.CollectionItemChunk + fields = ('uid', 'chunkFile') + + +class CollectionItemRevisionSerializer(BetterErrorsMixin, serializers.ModelSerializer): + chunks = ChunksField( + source='chunks_relation', + queryset=models.RevisionChunkRelation.objects.all(), + style={'base_template': 'input.html'}, + many=True + ) + meta = BinaryBase64Field() + + class Meta: + model = models.CollectionItemRevision + fields = ('chunks', 'meta', 'uid', 'deleted') + + +class CollectionItemSerializer(BetterErrorsMixin, serializers.ModelSerializer): + encryptionKey = BinaryBase64Field(required=False, default=None, allow_null=True) + etag = serializers.CharField(allow_null=True, write_only=True) + content = CollectionItemRevisionSerializer(many=False) + + class Meta: + model = models.CollectionItem + fields = ('uid', 'version', 'encryptionKey', 'content', 'etag') + + def create(self, validated_data): + """Function that's called when this serializer creates an item""" + validate_etag = self.context.get('validate_etag', False) + etag = validated_data.pop('etag') + revision_data = validated_data.pop('content') + uid = validated_data.pop('uid') + + Model = self.__class__.Meta.model + + with transaction.atomic(): + instance, created = Model.objects.get_or_create(uid=uid, defaults=validated_data) + cur_etag = instance.etag if not created else None + + if validate_etag and cur_etag != etag: + raise EtebaseValidationError('wrong_etag', 'Wrong etag. Expected {} got {}'.format(cur_etag, etag), + status_code=status.HTTP_409_CONFLICT) + + if not created: + # We don't have to use select_for_update here because the unique constraint on current guards against + # the race condition. But it's a good idea because it'll lock and wait rather than fail. + current_revision = instance.revisions.filter(current=True).select_for_update().first() + current_revision.current = None + current_revision.save() + + process_revisions_for_item(instance, revision_data) + + return instance + + def update(self, instance, validated_data): + # We never update, we always update in the create method + raise NotImplementedError() + + +class CollectionItemDepSerializer(BetterErrorsMixin, serializers.ModelSerializer): + etag = serializers.CharField() + + class Meta: + model = models.CollectionItem + fields = ('uid', 'etag') + + def validate(self, data): + item = self.__class__.Meta.model.objects.get(uid=data['uid']) + etag = data['etag'] + if item.etag != etag: + raise EtebaseValidationError('wrong_etag', 'Wrong etag. Expected {} got {}'.format(item.etag, etag), + status_code=status.HTTP_409_CONFLICT) + + return data + + +class CollectionItemBulkGetSerializer(BetterErrorsMixin, serializers.ModelSerializer): + etag = serializers.CharField(required=False) + + class Meta: + model = models.CollectionItem + fields = ('uid', 'etag') + + +class CollectionListMultiSerializer(BetterErrorsMixin, serializers.Serializer): + collectionTypes = serializers.ListField( + child=BinaryBase64Field() + ) + + +class CollectionSerializer(BetterErrorsMixin, serializers.ModelSerializer): + collectionKey = CollectionEncryptionKeyField() + # FIXME: make required once "collection-type-migration" is done + collectionType = CollectionTypeField(required=False) + accessLevel = serializers.SerializerMethodField('get_access_level_from_context') + stoken = serializers.CharField(read_only=True) + + item = CollectionItemSerializer(many=False, source='main_item') + + class Meta: + model = models.Collection + fields = ('item', 'accessLevel', 'collectionKey', 'collectionType', 'stoken') + + def get_access_level_from_context(self, obj): + request = self.context.get('request', None) + if request is not None: + return obj.members.get(user=request.user).accessLevel + return None + + def create(self, validated_data): + """Function that's called when this serializer creates an item""" + collection_key = validated_data.pop('collectionKey') + # FIXME: remove the None fallback once "collection-type-migration" is done + collection_type = validated_data.pop('collectionType', None) + + main_item_data = validated_data.pop('main_item') + etag = main_item_data.pop('etag') + revision_data = main_item_data.pop('content') + + instance = self.__class__.Meta.model(**validated_data) + + with transaction.atomic(): + if etag is not None: + raise EtebaseValidationError('bad_etag', 'etag is not null') + + instance.save() + main_item = models.CollectionItem.objects.create(**main_item_data, collection=instance) + + instance.main_item = main_item + + instance.full_clean() + instance.save() + + process_revisions_for_item(main_item, revision_data) + + user = validated_data.get('owner') + + # FIXME: remove the if statement (and else branch) once "collection-type-migration" is done + if collection_type is not None: + collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=collection_type, owner=user) + else: + collection_type_obj = None + + models.CollectionMember(collection=instance, + stoken=models.Stoken.objects.create(), + user=user, + accessLevel=models.AccessLevels.ADMIN, + encryptionKey=collection_key, + collectionType=collection_type_obj, + ).save() + + return instance + + def update(self, instance, validated_data): + raise NotImplementedError() + + +class CollectionMemberSerializer(BetterErrorsMixin, serializers.ModelSerializer): + username = UserSlugRelatedField( + source='user', + read_only=True, + style={'base_template': 'input.html'}, + ) + + class Meta: + model = models.CollectionMember + fields = ('username', 'accessLevel') + + def create(self, validated_data): + raise NotImplementedError() + + def update(self, instance, validated_data): + with transaction.atomic(): + # We only allow updating accessLevel + access_level = validated_data.pop('accessLevel') + if instance.accessLevel != access_level: + instance.stoken = models.Stoken.objects.create() + instance.accessLevel = access_level + instance.save() + + return instance + + +class CollectionInvitationSerializer(BetterErrorsMixin, serializers.ModelSerializer): + username = UserSlugRelatedField( + source='user', + queryset=User.objects, + style={'base_template': 'input.html'}, + ) + collection = serializers.CharField(source='collection.uid') + fromUsername = BinaryBase64Field(source='fromMember.user.username', read_only=True) + fromPubkey = BinaryBase64Field(source='fromMember.user.userinfo.pubkey', read_only=True) + signedEncryptionKey = BinaryBase64Field() + + class Meta: + model = models.CollectionInvitation + fields = ('username', 'uid', 'collection', 'signedEncryptionKey', 'accessLevel', + 'fromUsername', 'fromPubkey', 'version') + + def validate_user(self, value): + request = self.context['request'] + + if request.user.username == value.lower(): + raise EtebaseValidationError('no_self_invite', 'Inviting yourself is not allowed') + return value + + def create(self, validated_data): + request = self.context['request'] + collection = validated_data.pop('collection') + + member = collection.members.get(user=request.user) + + with transaction.atomic(): + try: + return type(self).Meta.model.objects.create(**validated_data, fromMember=member) + except IntegrityError: + raise EtebaseValidationError('invitation_exists', 'Invitation already exists') + + def update(self, instance, validated_data): + with transaction.atomic(): + instance.accessLevel = validated_data.pop('accessLevel') + instance.signedEncryptionKey = validated_data.pop('signedEncryptionKey') + instance.save() + + return instance + + +class InvitationAcceptSerializer(BetterErrorsMixin, serializers.Serializer): + collectionType = BinaryBase64Field() + encryptionKey = BinaryBase64Field() + + def create(self, validated_data): + + with transaction.atomic(): + invitation = self.context['invitation'] + encryption_key = validated_data.get('encryptionKey') + collection_type = validated_data.pop('collectionType') + + user = invitation.user + collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=collection_type, owner=user) + + member = models.CollectionMember.objects.create( + collection=invitation.collection, + stoken=models.Stoken.objects.create(), + user=user, + accessLevel=invitation.accessLevel, + encryptionKey=encryption_key, + collectionType=collection_type_obj, + ) + + models.CollectionMemberRemoved.objects.filter( + user=invitation.user, collection=invitation.collection).delete() + + invitation.delete() + + return member + + def update(self, instance, validated_data): + raise NotImplementedError() + + +class UserSerializer(BetterErrorsMixin, serializers.ModelSerializer): + pubkey = BinaryBase64Field(source='userinfo.pubkey') + encryptedContent = BinaryBase64Field(source='userinfo.encryptedContent') + + class Meta: + model = User + fields = (User.USERNAME_FIELD, User.EMAIL_FIELD, 'pubkey', 'encryptedContent') + + +class UserInfoPubkeySerializer(BetterErrorsMixin, serializers.ModelSerializer): + pubkey = BinaryBase64Field() + + class Meta: + model = models.UserInfo + fields = ('pubkey', ) + + +class UserSignupSerializer(BetterErrorsMixin, serializers.ModelSerializer): + class Meta: + model = User + fields = (User.USERNAME_FIELD, User.EMAIL_FIELD) + extra_kwargs = { + 'username': {'validators': []}, # We specifically validate in SignupSerializer + } + + +class AuthenticationSignupSerializer(BetterErrorsMixin, serializers.Serializer): + """Used both for creating new accounts and setting up existing ones for the first time. + When setting up existing ones the email is ignored." + """ + user = UserSignupSerializer(many=False) + salt = BinaryBase64Field() + loginPubkey = BinaryBase64Field() + pubkey = BinaryBase64Field() + encryptedContent = BinaryBase64Field() + + def create(self, validated_data): + """Function that's called when this serializer creates an item""" + user_data = validated_data.pop('user') + + with transaction.atomic(): + try: + view = self.context.get('view', None) + user_queryset = get_user_queryset(User.objects.all(), view) + instance = user_queryset.get(**{User.USERNAME_FIELD: user_data['username'].lower()}) + except User.DoesNotExist: + # Create the user and save the casing the user chose as the first name + try: + instance = create_user(**user_data, password=None, first_name=user_data['username'], view=view) + instance.clean_fields() + except EtebaseValidationError as e: + raise e + except django_exceptions.ValidationError as e: + self.transform_validation_error("user", e) + except Exception as e: + raise EtebaseValidationError('generic', str(e)) + + if hasattr(instance, 'userinfo'): + raise EtebaseValidationError('user_exists', 'User already exists', status_code=status.HTTP_409_CONFLICT) + + models.UserInfo.objects.create(**validated_data, owner=instance) + + return instance + + def update(self, instance, validated_data): + raise NotImplementedError() + + +class AuthenticationLoginChallengeSerializer(BetterErrorsMixin, serializers.Serializer): + username = serializers.CharField(required=True) + + def create(self, validated_data): + raise NotImplementedError() + + def update(self, instance, validated_data): + raise NotImplementedError() + + +class AuthenticationLoginSerializer(BetterErrorsMixin, serializers.Serializer): + response = BinaryBase64Field() + signature = BinaryBase64Field() + + def create(self, validated_data): + raise NotImplementedError() + + def update(self, instance, validated_data): + raise NotImplementedError() + + +class AuthenticationLoginInnerSerializer(AuthenticationLoginChallengeSerializer): + challenge = BinaryBase64Field() + host = serializers.CharField() + action = serializers.CharField() + + def create(self, validated_data): + raise NotImplementedError() + + def update(self, instance, validated_data): + raise NotImplementedError() + + +class AuthenticationChangePasswordInnerSerializer(AuthenticationLoginInnerSerializer): + loginPubkey = BinaryBase64Field() + encryptedContent = BinaryBase64Field() + + class Meta: + model = models.UserInfo + fields = ('loginPubkey', 'encryptedContent') + + def create(self, validated_data): + raise NotImplementedError() + + def update(self, instance, validated_data): + with transaction.atomic(): + instance.loginPubkey = validated_data.pop('loginPubkey') + instance.encryptedContent = validated_data.pop('encryptedContent') + instance.save() + + return instance diff --git a/django_etebase/signals.py b/django_etebase/signals.py new file mode 100644 index 0000000..03dbed5 --- /dev/null +++ b/django_etebase/signals.py @@ -0,0 +1,3 @@ +from django.dispatch import Signal + +user_signed_up = Signal(providing_args=['request', 'user']) diff --git a/django_etebase/tests.py b/django_etebase/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/django_etebase/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/django_etebase/token_auth/__init__.py b/django_etebase/token_auth/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_etebase/token_auth/admin.py b/django_etebase/token_auth/admin.py new file mode 100644 index 0000000..e69de29 diff --git a/django_etebase/token_auth/apps.py b/django_etebase/token_auth/apps.py new file mode 100644 index 0000000..118b872 --- /dev/null +++ b/django_etebase/token_auth/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class TokenAuthConfig(AppConfig): + name = 'django_etebase.token_auth' diff --git a/django_etebase/token_auth/authentication.py b/django_etebase/token_auth/authentication.py new file mode 100644 index 0000000..432c8cf --- /dev/null +++ b/django_etebase/token_auth/authentication.py @@ -0,0 +1,46 @@ +from django.utils import timezone +from django.utils.translation import gettext_lazy as _ + +from rest_framework import exceptions +from rest_framework.authentication import TokenAuthentication as DRFTokenAuthentication + +from .models import AuthToken, get_default_expiry + + +AUTO_REFRESH = True +MIN_REFRESH_INTERVAL = 60 + + +class TokenAuthentication(DRFTokenAuthentication): + keyword = 'Token' + model = AuthToken + + def authenticate_credentials(self, key): + msg = _('Invalid token.') + model = self.get_model() + try: + token = model.objects.select_related('user').get(key=key) + except model.DoesNotExist: + raise exceptions.AuthenticationFailed(msg) + + if not token.user.is_active: + raise exceptions.AuthenticationFailed(_('User inactive or deleted.')) + + if token.expiry is not None: + if token.expiry < timezone.now(): + token.delete() + raise exceptions.AuthenticationFailed(msg) + + if AUTO_REFRESH: + self.renew_token(token) + + return (token.user, token) + + def renew_token(self, auth_token): + current_expiry = auth_token.expiry + new_expiry = get_default_expiry() + # Throttle refreshing of token to avoid db writes + delta = (new_expiry - current_expiry).total_seconds() + if delta > MIN_REFRESH_INTERVAL: + auth_token.expiry = new_expiry + auth_token.save(update_fields=('expiry',)) diff --git a/django_etebase/token_auth/migrations/0001_initial.py b/django_etebase/token_auth/migrations/0001_initial.py new file mode 100644 index 0000000..5a47366 --- /dev/null +++ b/django_etebase/token_auth/migrations/0001_initial.py @@ -0,0 +1,28 @@ +# Generated by Django 3.0.3 on 2020-06-03 12:49 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion +from django_etebase.token_auth import models as token_auth_models + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='AuthToken', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('key', models.CharField(db_index=True, default=token_auth_models.generate_key, max_length=40, unique=True)), + ('created', models.DateTimeField(auto_now_add=True)), + ('expiry', models.DateTimeField(blank=True, default=token_auth_models.get_default_expiry, null=True)), + ('user', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='auth_token_set', to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/django_etebase/token_auth/migrations/__init__.py b/django_etebase/token_auth/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/django_etebase/token_auth/models.py b/django_etebase/token_auth/models.py new file mode 100644 index 0000000..0fe4766 --- /dev/null +++ b/django_etebase/token_auth/models.py @@ -0,0 +1,26 @@ +from django.contrib.auth import get_user_model +from django.db import models +from django.utils import timezone +from django.utils.crypto import get_random_string + +User = get_user_model() + + +def generate_key(): + return get_random_string(40) + + +def get_default_expiry(): + return timezone.now() + timezone.timedelta(days=30) + + +class AuthToken(models.Model): + + key = models.CharField(max_length=40, unique=True, db_index=True, default=generate_key) + user = models.ForeignKey(User, null=False, blank=False, + related_name='auth_token_set', on_delete=models.CASCADE) + created = models.DateTimeField(auto_now_add=True) + expiry = models.DateTimeField(null=True, blank=True, default=get_default_expiry) + + def __str__(self): + return '{}: {}'.format(self.key, self.user) diff --git a/django_etebase/urls.py b/django_etebase/urls.py new file mode 100644 index 0000000..f6d982e --- /dev/null +++ b/django_etebase/urls.py @@ -0,0 +1,30 @@ +from django.conf import settings +from django.conf.urls import include +from django.urls import path + +from rest_framework_nested import routers + +from django_etebase import views + +router = routers.DefaultRouter() +router.register(r'collection', views.CollectionViewSet) +router.register(r'authentication', views.AuthenticationViewSet, basename='authentication') +router.register(r'invitation/incoming', views.InvitationIncomingViewSet, basename='invitation_incoming') +router.register(r'invitation/outgoing', views.InvitationOutgoingViewSet, basename='invitation_outgoing') + +collections_router = routers.NestedSimpleRouter(router, r'collection', lookup='collection') +collections_router.register(r'item', views.CollectionItemViewSet, basename='collection_item') +collections_router.register(r'member', views.CollectionMemberViewSet, basename='collection_member') + +item_router = routers.NestedSimpleRouter(collections_router, r'item', lookup='collection_item') +item_router.register(r'chunk', views.CollectionItemChunkViewSet, basename='collection_items_chunk') + +if settings.DEBUG: + router.register(r'test/authentication', views.TestAuthenticationViewSet, basename='test_authentication') + +app_name = 'django_etebase' +urlpatterns = [ + path('v1/', include(router.urls)), + path('v1/', include(collections_router.urls)), + path('v1/', include(item_router.urls)), +] diff --git a/django_etebase/utils.py b/django_etebase/utils.py new file mode 100644 index 0000000..1351f9b --- /dev/null +++ b/django_etebase/utils.py @@ -0,0 +1,26 @@ +from django.contrib.auth import get_user_model +from django.core.exceptions import PermissionDenied + +from . import app_settings + + +User = get_user_model() + + +def get_user_queryset(queryset, view): + custom_func = app_settings.GET_USER_QUERYSET_FUNC + if custom_func is not None: + return custom_func(queryset, view) + return queryset + + +def create_user(*args, **kwargs): + custom_func = app_settings.CREATE_USER_FUNC + if custom_func is not None: + return custom_func(*args, **kwargs) + _ = kwargs.pop('view') + return User.objects.create_user(*args, **kwargs) + + +def create_user_blocked(*args, **kwargs): + raise PermissionDenied('Signup is disabled for this server. Please refer to the README for more information.') diff --git a/django_etebase/views.py b/django_etebase/views.py new file mode 100644 index 0000000..5328c84 --- /dev/null +++ b/django_etebase/views.py @@ -0,0 +1,868 @@ +# Copyright © 2017 Tom Hacohen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, version 3. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +import msgpack + +from django.conf import settings +from django.contrib.auth import get_user_model, user_logged_in, user_logged_out +from django.core.exceptions import PermissionDenied +from django.db import transaction, IntegrityError +from django.db.models import Max, Value as V, Q +from django.db.models.functions import Coalesce, Greatest +from django.http import HttpResponseBadRequest, HttpResponse, Http404 +from django.shortcuts import get_object_or_404 + +from rest_framework import status +from rest_framework import viewsets +from rest_framework.decorators import action as action_decorator +from rest_framework.response import Response +from rest_framework.parsers import JSONParser, FormParser, MultiPartParser +from rest_framework.renderers import BrowsableAPIRenderer +from rest_framework.exceptions import AuthenticationFailed +from rest_framework.permissions import IsAuthenticated + +import nacl.encoding +import nacl.signing +import nacl.secret +import nacl.hash + +from .token_auth.models import AuthToken + +from .drf_msgpack.parsers import MessagePackParser +from .drf_msgpack.renderers import MessagePackRenderer + +from . import app_settings, permissions +from .renderers import JSONRenderer +from .models import ( + Collection, + CollectionItem, + CollectionItemRevision, + CollectionMember, + CollectionMemberRemoved, + CollectionInvitation, + Stoken, + UserInfo, + ) +from .serializers import ( + AuthenticationChangePasswordInnerSerializer, + AuthenticationSignupSerializer, + AuthenticationLoginChallengeSerializer, + AuthenticationLoginSerializer, + AuthenticationLoginInnerSerializer, + CollectionSerializer, + CollectionItemSerializer, + CollectionItemBulkGetSerializer, + CollectionItemDepSerializer, + CollectionItemRevisionSerializer, + CollectionItemChunkSerializer, + CollectionListMultiSerializer, + CollectionMemberSerializer, + CollectionInvitationSerializer, + InvitationAcceptSerializer, + UserInfoPubkeySerializer, + UserSerializer, + ) +from .utils import get_user_queryset +from .exceptions import EtebaseValidationError +from .parsers import ChunkUploadParser +from .signals import user_signed_up + +User = get_user_model() + + +def msgpack_encode(content): + return msgpack.packb(content, use_bin_type=True) + + +def msgpack_decode(content): + return msgpack.unpackb(content, raw=False) + + +class BaseViewSet(viewsets.ModelViewSet): + authentication_classes = tuple(app_settings.API_AUTHENTICATORS) + permission_classes = tuple(app_settings.API_PERMISSIONS) + renderer_classes = [JSONRenderer, MessagePackRenderer] + ([BrowsableAPIRenderer] if settings.DEBUG else []) + parser_classes = [JSONParser, MessagePackParser, FormParser, MultiPartParser] + stoken_id_fields = None + + def get_serializer_class(self): + serializer_class = self.serializer_class + + if self.request.method == 'PUT': + serializer_class = getattr(self, 'serializer_update_class', serializer_class) + + return serializer_class + + def get_collection_queryset(self, queryset=Collection.objects): + user = self.request.user + return queryset.filter(members__user=user) + + def get_stoken_obj_id(self, request): + return request.GET.get('stoken', None) + + def get_stoken_obj(self, request): + stoken = self.get_stoken_obj_id(request) + + if stoken is not None: + try: + return Stoken.objects.get(uid=stoken) + except Stoken.DoesNotExist: + raise EtebaseValidationError('bad_stoken', 'Invalid stoken.', status_code=status.HTTP_400_BAD_REQUEST) + + return None + + def filter_by_stoken(self, request, queryset): + stoken_rev = self.get_stoken_obj(request) + + aggr_fields = [Coalesce(Max(field), V(0)) for field in self.stoken_id_fields] + max_stoken = Greatest(*aggr_fields) if len(aggr_fields) > 1 else aggr_fields[0] + queryset = queryset.annotate(max_stoken=max_stoken).order_by('max_stoken') + + if stoken_rev is not None: + queryset = queryset.filter(max_stoken__gt=stoken_rev.id) + + return queryset, stoken_rev + + def get_queryset_stoken(self, queryset): + maxid = -1 + for row in queryset: + rowmaxid = getattr(row, 'max_stoken') or -1 + maxid = max(maxid, rowmaxid) + new_stoken = (maxid >= 0) and Stoken.objects.get(id=maxid) + + return new_stoken or None + + def filter_by_stoken_and_limit(self, request, queryset): + limit = int(request.GET.get('limit', 50)) + + queryset, stoken_rev = self.filter_by_stoken(request, queryset) + + result = list(queryset[:limit + 1]) + if len(result) < limit + 1: + done = True + else: + done = False + result = result[:-1] + + new_stoken_obj = self.get_queryset_stoken(result) or stoken_rev + + return result, new_stoken_obj, done + + # Change how our list works by default + def list(self, request, collection_uid=None, *args, **kwargs): + queryset = self.get_queryset() + serializer = self.get_serializer(queryset, many=True) + + ret = { + 'data': serializer.data, + 'done': True, # we always return all the items, so it's always done + } + + return Response(ret) + + +class CollectionViewSet(BaseViewSet): + allowed_methods = ['GET', 'POST'] + permission_classes = BaseViewSet.permission_classes + (permissions.IsCollectionAdminOrReadOnly, ) + queryset = Collection.objects.all() + serializer_class = CollectionSerializer + lookup_field = 'main_item__uid' + lookup_url_kwarg = 'uid' + stoken_id_fields = ['items__revisions__stoken__id', 'members__stoken__id'] + + def get_queryset(self, queryset=None): + if queryset is None: + queryset = type(self).queryset + return self.get_collection_queryset(queryset) + + def get_serializer_context(self): + context = super().get_serializer_context() + prefetch = self.request.query_params.get('prefetch', 'auto') + context.update({'request': self.request, 'prefetch': prefetch}) + return context + + def destroy(self, request, uid=None, *args, **kwargs): + # FIXME: implement + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + + def partial_update(self, request, uid=None, *args, **kwargs): + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + + def update(self, request, *args, **kwargs): + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + + def create(self, request, *args, **kwargs): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + serializer.save(owner=self.request.user) + + return Response({}, status=status.HTTP_201_CREATED) + + def list(self, request, *args, **kwargs): + queryset = self.get_queryset() + return self.list_common(request, queryset, *args, **kwargs) + + @action_decorator(detail=False, methods=['POST']) + def list_multi(self, request, *args, **kwargs): + serializer = CollectionListMultiSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + + collection_types = serializer.validated_data['collectionTypes'] + + queryset = self.get_queryset() + # FIXME: Remove the isnull part once "collection-type-migration" is done + queryset = queryset.filter( + Q(members__collectionType__uid__in=collection_types) | Q(members__collectionType__isnull=True)) + + return self.list_common(request, queryset, *args, **kwargs) + + def list_common(self, request, queryset, *args, **kwargs): + result, new_stoken_obj, done = self.filter_by_stoken_and_limit(request, queryset) + new_stoken = new_stoken_obj and new_stoken_obj.uid + + serializer = self.get_serializer(result, many=True) + + ret = { + 'data': serializer.data, + 'stoken': new_stoken, + 'done': done, + } + + stoken_obj = self.get_stoken_obj(request) + if stoken_obj is not None: + # FIXME: honour limit? (the limit should be combined for data and this because of stoken) + remed_qs = CollectionMemberRemoved.objects.filter(user=request.user, stoken__id__gt=stoken_obj.id) + if not ret['done']: + # We only filter by the new_stoken if we are not done. This is because if we are done, the new stoken + # can point to the most recent collection change rather than most recent removed membership. + remed_qs = remed_qs.filter(stoken__id__lte=new_stoken_obj.id) + + remed = remed_qs.values_list('collection__main_item__uid', flat=True) + if len(remed) > 0: + ret['removedMemberships'] = [{'uid': x} for x in remed] + + return Response(ret) + + +class CollectionItemViewSet(BaseViewSet): + allowed_methods = ['GET', 'POST', 'PUT'] + permission_classes = BaseViewSet.permission_classes + (permissions.HasWriteAccessOrReadOnly, ) + queryset = CollectionItem.objects.all() + serializer_class = CollectionItemSerializer + lookup_field = 'uid' + stoken_id_fields = ['revisions__stoken__id'] + + def get_queryset(self): + collection_uid = self.kwargs['collection_uid'] + try: + collection = self.get_collection_queryset(Collection.objects).get(main_item__uid=collection_uid) + except Collection.DoesNotExist: + raise Http404("Collection does not exist") + # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') + queryset = type(self).queryset.filter(collection__pk=collection.pk, + revisions__current=True) + + return queryset + + def get_serializer_context(self): + context = super().get_serializer_context() + prefetch = self.request.query_params.get('prefetch', 'auto') + context.update({'request': self.request, 'prefetch': prefetch}) + return context + + def create(self, request, collection_uid=None, *args, **kwargs): + # We create using batch and transaction + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + + def destroy(self, request, collection_uid=None, uid=None, *args, **kwargs): + # We can't have destroy because we need to get data from the user (in the body) such as hmac. + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + + def update(self, request, collection_uid=None, uid=None, *args, **kwargs): + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + + def partial_update(self, request, collection_uid=None, uid=None, *args, **kwargs): + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + + def list(self, request, collection_uid=None, *args, **kwargs): + queryset = self.get_queryset() + + if not self.request.query_params.get('withCollection', False): + queryset = queryset.filter(parent__isnull=True) + + result, new_stoken_obj, done = self.filter_by_stoken_and_limit(request, queryset) + new_stoken = new_stoken_obj and new_stoken_obj.uid + + serializer = self.get_serializer(result, many=True) + + ret = { + 'data': serializer.data, + 'stoken': new_stoken, + 'done': done, + } + return Response(ret) + + @action_decorator(detail=True, methods=['GET']) + def revision(self, request, collection_uid=None, uid=None, *args, **kwargs): + col = get_object_or_404(self.get_collection_queryset(Collection.objects), main_item__uid=collection_uid) + item = get_object_or_404(col.items, uid=uid) + + limit = int(request.GET.get('limit', 50)) + iterator = request.GET.get('iterator', None) + + queryset = item.revisions.order_by('-id') + + if iterator is not None: + iterator = get_object_or_404(queryset, uid=iterator) + queryset = queryset.filter(id__lt=iterator.id) + + result = list(queryset[:limit + 1]) + if len(result) < limit + 1: + done = True + else: + done = False + result = result[:-1] + + serializer = CollectionItemRevisionSerializer(result, context=self.get_serializer_context(), many=True) + + iterator = serializer.data[-1]['uid'] if len(result) > 0 else None + + ret = { + 'data': serializer.data, + 'iterator': iterator, + 'done': done, + } + return Response(ret) + + # FIXME: rename to something consistent with what the clients have - maybe list_updates? + @action_decorator(detail=False, methods=['POST']) + def fetch_updates(self, request, collection_uid=None, *args, **kwargs): + queryset = self.get_queryset() + + serializer = CollectionItemBulkGetSerializer(data=request.data, many=True) + serializer.is_valid(raise_exception=True) + # FIXME: make configurable? + item_limit = 200 + + if len(serializer.validated_data) > item_limit: + content = {'code': 'too_many_items', + 'detail': 'Request has too many items. Limit: {}'. format(item_limit)} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + + queryset, stoken_rev = self.filter_by_stoken(request, queryset) + + uids, etags = zip(*[(item['uid'], item.get('etag')) for item in serializer.validated_data]) + revs = CollectionItemRevision.objects.filter(uid__in=etags, current=True) + queryset = queryset.filter(uid__in=uids).exclude(revisions__in=revs) + + new_stoken_obj = self.get_queryset_stoken(queryset) + new_stoken = new_stoken_obj and new_stoken_obj.uid + stoken = stoken_rev and getattr(stoken_rev, 'uid', None) + new_stoken = new_stoken or stoken + + serializer = self.get_serializer(queryset, many=True) + + ret = { + 'data': serializer.data, + 'stoken': new_stoken, + 'done': True, # we always return all the items, so it's always done + } + return Response(ret) + + @action_decorator(detail=False, methods=['POST']) + def batch(self, request, collection_uid=None, *args, **kwargs): + return self.transaction(request, collection_uid, validate_etag=False) + + @action_decorator(detail=False, methods=['POST']) + def transaction(self, request, collection_uid=None, validate_etag=True, *args, **kwargs): + stoken = request.GET.get('stoken', None) + with transaction.atomic(): # We need this for locking on the collection object + collection_object = get_object_or_404( + self.get_collection_queryset(Collection.objects).select_for_update(), # Lock writes on the collection + main_item__uid=collection_uid) + + if stoken is not None and stoken != collection_object.stoken: + content = {'code': 'stale_stoken', 'detail': 'Stoken is too old'} + return Response(content, status=status.HTTP_409_CONFLICT) + + items = request.data.get('items') + deps = request.data.get('deps', None) + # FIXME: It should just be one serializer + context = self.get_serializer_context() + context.update({'validate_etag': validate_etag}) + serializer = self.get_serializer_class()(data=items, context=context, many=True) + deps_serializer = CollectionItemDepSerializer(data=deps, context=context, many=True) + + ser_valid = serializer.is_valid() + deps_ser_valid = (deps is None or deps_serializer.is_valid()) + if ser_valid and deps_ser_valid: + items = serializer.save(collection=collection_object) + + ret = { + } + return Response(ret, status=status.HTTP_200_OK) + + return Response( + { + "items": serializer.errors, + "deps": deps_serializer.errors if deps is not None else [], + }, + status=status.HTTP_409_CONFLICT) + + +class CollectionItemChunkViewSet(viewsets.ViewSet): + allowed_methods = ['GET', 'PUT'] + authentication_classes = BaseViewSet.authentication_classes + permission_classes = BaseViewSet.permission_classes + renderer_classes = BaseViewSet.renderer_classes + parser_classes = (ChunkUploadParser, ) + serializer_class = CollectionItemChunkSerializer + lookup_field = 'uid' + + def get_serializer_class(self): + return self.serializer_class + + def get_collection_queryset(self, queryset=Collection.objects): + user = self.request.user + return queryset.filter(members__user=user) + + def update(self, request, *args, collection_uid=None, collection_item_uid=None, uid=None, **kwargs): + col = get_object_or_404(self.get_collection_queryset(), main_item__uid=collection_uid) + # IGNORED FOR NOW: col_it = get_object_or_404(col.items, uid=collection_item_uid) + + data = { + "uid": uid, + "chunkFile": request.data["file"], + } + + serializer = self.get_serializer_class()(data=data) + serializer.is_valid(raise_exception=True) + try: + serializer.save(collection=col) + except IntegrityError: + return Response( + {"code": "chunk_exists", "detail": "Chunk already exists."}, + status=status.HTTP_409_CONFLICT + ) + + return Response({}, status=status.HTTP_201_CREATED) + + @action_decorator(detail=True, methods=['GET']) + def download(self, request, collection_uid=None, collection_item_uid=None, uid=None, *args, **kwargs): + import os + from django.views.static import serve + + col = get_object_or_404(self.get_collection_queryset(), main_item__uid=collection_uid) + # IGNORED FOR NOW: col_it = get_object_or_404(col.items, uid=collection_item_uid) + chunk = get_object_or_404(col.chunks, uid=uid) + + filename = chunk.chunkFile.path + dirname = os.path.dirname(filename) + basename = os.path.basename(filename) + + # FIXME: DO NOT USE! Use django-send file or etc instead. + return serve(request, basename, dirname) + + +class CollectionMemberViewSet(BaseViewSet): + allowed_methods = ['GET', 'PUT', 'DELETE'] + our_base_permission_classes = BaseViewSet.permission_classes + permission_classes = our_base_permission_classes + (permissions.IsCollectionAdmin, ) + queryset = CollectionMember.objects.all() + serializer_class = CollectionMemberSerializer + lookup_field = f'user__{User.USERNAME_FIELD}__iexact' + lookup_url_kwarg = 'username' + stoken_id_fields = ['stoken__id'] + + # FIXME: need to make sure that there's always an admin, and maybe also don't let an owner remove adm access + # (if we want to transfer, we need to do that specifically) + + def get_queryset(self, queryset=None): + collection_uid = self.kwargs['collection_uid'] + try: + collection = self.get_collection_queryset(Collection.objects).get(main_item__uid=collection_uid) + except Collection.DoesNotExist: + raise Http404('Collection does not exist') + + if queryset is None: + queryset = type(self).queryset + + return queryset.filter(collection=collection) + + # We override this method because we expect the stoken to be called iterator + def get_stoken_obj_id(self, request): + return request.GET.get('iterator', None) + + def list(self, request, collection_uid=None, *args, **kwargs): + queryset = self.get_queryset().order_by('id') + result, new_stoken_obj, done = self.filter_by_stoken_and_limit(request, queryset) + new_stoken = new_stoken_obj and new_stoken_obj.uid + serializer = self.get_serializer(result, many=True) + + ret = { + 'data': serializer.data, + 'iterator': new_stoken, # Here we call it an iterator, it's only stoken for collection/items + 'done': done, + } + + return Response(ret) + + def create(self, request, *args, **kwargs): + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + + # FIXME: block leaving if we are the last admins - should be deleted / assigned in this case depending if there + # are other memebers. + def perform_destroy(self, instance): + instance.revoke() + + @action_decorator(detail=False, methods=['POST'], permission_classes=our_base_permission_classes) + def leave(self, request, collection_uid=None, *args, **kwargs): + collection_uid = self.kwargs['collection_uid'] + col = get_object_or_404(self.get_collection_queryset(Collection.objects), main_item__uid=collection_uid) + + member = col.members.get(user=request.user) + self.perform_destroy(member) + + return Response({}) + + +class InvitationBaseViewSet(BaseViewSet): + queryset = CollectionInvitation.objects.all() + serializer_class = CollectionInvitationSerializer + lookup_field = 'uid' + lookup_url_kwarg = 'invitation_uid' + + def list(self, request, collection_uid=None, *args, **kwargs): + limit = int(request.GET.get('limit', 50)) + iterator = request.GET.get('iterator', None) + + queryset = self.get_queryset().order_by('id') + + if iterator is not None: + iterator = get_object_or_404(queryset, uid=iterator) + queryset = queryset.filter(id__gt=iterator.id) + + result = list(queryset[:limit + 1]) + if len(result) < limit + 1: + done = True + else: + done = False + result = result[:-1] + + serializer = self.get_serializer(result, many=True) + + iterator = serializer.data[-1]['uid'] if len(result) > 0 else None + + ret = { + 'data': serializer.data, + 'iterator': iterator, + 'done': done, + } + + return Response(ret) + + +class InvitationOutgoingViewSet(InvitationBaseViewSet): + allowed_methods = ['GET', 'POST', 'PUT', 'DELETE'] + + def get_queryset(self, queryset=None): + if queryset is None: + queryset = type(self).queryset + + return queryset.filter(fromMember__user=self.request.user) + + def create(self, request, *args, **kwargs): + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + collection_uid = serializer.validated_data.get('collection', {}).get('uid') + + try: + collection = self.get_collection_queryset(Collection.objects).get(main_item__uid=collection_uid) + except Collection.DoesNotExist: + raise Http404('Collection does not exist') + + if request.user == serializer.validated_data.get('user'): + content = {'code': 'self_invite', 'detail': 'Inviting yourself is invalid'} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + + if not permissions.is_collection_admin(collection, request.user): + raise PermissionDenied({'code': 'admin_access_required', + 'detail': 'User is not an admin of this collection'}) + + serializer.save(collection=collection) + + return Response({}, status=status.HTTP_201_CREATED) + + @action_decorator(detail=False, allowed_methods=['GET'], methods=['GET']) + def fetch_user_profile(self, request, *args, **kwargs): + username = request.GET.get('username') + kwargs = {User.USERNAME_FIELD: username.lower()} + user = get_object_or_404(get_user_queryset(User.objects.all(), self), **kwargs) + user_info = get_object_or_404(UserInfo.objects.all(), owner=user) + serializer = UserInfoPubkeySerializer(user_info) + return Response(serializer.data) + + +class InvitationIncomingViewSet(InvitationBaseViewSet): + allowed_methods = ['GET', 'DELETE'] + + def get_queryset(self, queryset=None): + if queryset is None: + queryset = type(self).queryset + + return queryset.filter(user=self.request.user) + + @action_decorator(detail=True, allowed_methods=['POST'], methods=['POST']) + def accept(self, request, invitation_uid=None, *args, **kwargs): + invitation = get_object_or_404(self.get_queryset(), uid=invitation_uid) + context = self.get_serializer_context() + context.update({'invitation': invitation}) + + serializer = InvitationAcceptSerializer(data=request.data, context=context) + serializer.is_valid(raise_exception=True) + serializer.save() + return Response(status=status.HTTP_201_CREATED) + + +class AuthenticationViewSet(viewsets.ViewSet): + allowed_methods = ['POST'] + authentication_classes = BaseViewSet.authentication_classes + renderer_classes = BaseViewSet.renderer_classes + parser_classes = BaseViewSet.parser_classes + + def get_encryption_key(self, salt): + key = nacl.hash.blake2b(settings.SECRET_KEY.encode(), encoder=nacl.encoding.RawEncoder) + return nacl.hash.blake2b(b'', key=key, salt=salt[:nacl.hash.BLAKE2B_SALTBYTES], person=b'etebase-auth', + encoder=nacl.encoding.RawEncoder) + + def get_queryset(self): + return get_user_queryset(User.objects.all(), self) + + def get_serializer_context(self): + return { + 'request': self.request, + 'format': self.format_kwarg, + 'view': self + } + + def login_response_data(self, user): + return { + 'token': AuthToken.objects.create(user=user).key, + 'user': UserSerializer(user).data, + } + + def list(self, request, *args, **kwargs): + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + + @action_decorator(detail=False, methods=['POST']) + def signup(self, request, *args, **kwargs): + serializer = AuthenticationSignupSerializer(data=request.data, context=self.get_serializer_context()) + serializer.is_valid(raise_exception=True) + user = serializer.save() + + user_signed_up.send(sender=user.__class__, request=request, user=user) + + data = self.login_response_data(user) + return Response(data, status=status.HTTP_201_CREATED) + + def get_login_user(self, username): + kwargs = {User.USERNAME_FIELD + '__iexact': username.lower()} + try: + user = self.get_queryset().get(**kwargs) + if not hasattr(user, 'userinfo'): + raise AuthenticationFailed({'code': 'user_not_init', 'detail': 'User not properly init'}) + return user + except User.DoesNotExist: + raise AuthenticationFailed({'code': 'user_not_found', 'detail': 'User not found'}) + + def validate_login_request(self, request, validated_data, response_raw, signature, expected_action): + from datetime import datetime + + username = validated_data.get('username') + user = self.get_login_user(username) + host = validated_data['host'] + challenge = validated_data['challenge'] + action = validated_data['action'] + + salt = bytes(user.userinfo.salt) + enc_key = self.get_encryption_key(salt) + box = nacl.secret.SecretBox(enc_key) + + challenge_data = msgpack_decode(box.decrypt(challenge)) + now = int(datetime.now().timestamp()) + if action != expected_action: + content = {'code': 'wrong_action', 'detail': 'Expected "{}" but got something else'.format(expected_action)} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + elif now - challenge_data['timestamp'] > app_settings.CHALLENGE_VALID_SECONDS: + content = {'code': 'challenge_expired', 'detail': 'Login challange has expired'} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + elif challenge_data['userId'] != user.id: + content = {'code': 'wrong_user', 'detail': 'This challenge is for the wrong user'} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + elif not settings.DEBUG and host != request.get_host(): + detail = 'Found wrong host name. Got: "{}" expected: "{}"'.format(host, request.get_host()) + content = {'code': 'wrong_host', 'detail': detail} + return Response(content, status=status.HTTP_400_BAD_REQUEST) + + verify_key = nacl.signing.VerifyKey(bytes(user.userinfo.loginPubkey), encoder=nacl.encoding.RawEncoder) + + try: + verify_key.verify(response_raw, signature) + except nacl.exceptions.BadSignatureError: + return Response({'code': 'login_bad_signature', 'detail': 'Wrong password for user.'}, + status=status.HTTP_401_UNAUTHORIZED) + + return None + + @action_decorator(detail=False, methods=['GET']) + def is_etebase(self, request, *args, **kwargs): + return Response({}, status=status.HTTP_200_OK) + + @action_decorator(detail=False, methods=['POST']) + def login_challenge(self, request, *args, **kwargs): + from datetime import datetime + + serializer = AuthenticationLoginChallengeSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + username = serializer.validated_data.get('username') + user = self.get_login_user(username) + + salt = bytes(user.userinfo.salt) + enc_key = self.get_encryption_key(salt) + box = nacl.secret.SecretBox(enc_key) + + challenge_data = { + "timestamp": int(datetime.now().timestamp()), + "userId": user.id, + } + challenge = box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder) + + ret = { + "salt": salt, + "challenge": challenge, + "version": user.userinfo.version, + } + return Response(ret, status=status.HTTP_200_OK) + + @action_decorator(detail=False, methods=['POST']) + def login(self, request, *args, **kwargs): + outer_serializer = AuthenticationLoginSerializer(data=request.data) + outer_serializer.is_valid(raise_exception=True) + + response_raw = outer_serializer.validated_data['response'] + response = msgpack_decode(response_raw) + signature = outer_serializer.validated_data['signature'] + + context = {'host': request.get_host()} + serializer = AuthenticationLoginInnerSerializer(data=response, context=context) + serializer.is_valid(raise_exception=True) + + bad_login_response = self.validate_login_request( + request, serializer.validated_data, response_raw, signature, "login") + if bad_login_response is not None: + return bad_login_response + + username = serializer.validated_data.get('username') + user = self.get_login_user(username) + + data = self.login_response_data(user) + + user_logged_in.send(sender=user.__class__, request=request, user=user) + + return Response(data, status=status.HTTP_200_OK) + + @action_decorator(detail=False, methods=['POST'], permission_classes=[IsAuthenticated]) + def logout(self, request, *args, **kwargs): + request.auth.delete() + user_logged_out.send(sender=request.user.__class__, request=request, user=request.user) + return Response(status=status.HTTP_204_NO_CONTENT) + + @action_decorator(detail=False, methods=['POST'], permission_classes=BaseViewSet.permission_classes) + def change_password(self, request, *args, **kwargs): + outer_serializer = AuthenticationLoginSerializer(data=request.data) + outer_serializer.is_valid(raise_exception=True) + + response_raw = outer_serializer.validated_data['response'] + response = msgpack_decode(response_raw) + signature = outer_serializer.validated_data['signature'] + + context = {'host': request.get_host()} + serializer = AuthenticationChangePasswordInnerSerializer(request.user.userinfo, data=response, context=context) + serializer.is_valid(raise_exception=True) + + bad_login_response = self.validate_login_request( + request, serializer.validated_data, response_raw, signature, "changePassword") + if bad_login_response is not None: + return bad_login_response + + serializer.save() + + return Response({}, status=status.HTTP_200_OK) + + @action_decorator(detail=False, methods=['POST'], permission_classes=[IsAuthenticated]) + def dashboard_url(self, request, *args, **kwargs): + get_dashboard_url = app_settings.DASHBOARD_URL_FUNC + if get_dashboard_url is None: + raise EtebaseValidationError('not_supported', 'This server doesn\'t have a user dashboard.', + status_code=status.HTTP_400_BAD_REQUEST) + + ret = { + 'url': get_dashboard_url(request, *args, **kwargs), + } + return Response(ret) + + +class TestAuthenticationViewSet(viewsets.ViewSet): + allowed_methods = ['POST'] + renderer_classes = BaseViewSet.renderer_classes + parser_classes = BaseViewSet.parser_classes + + def get_serializer_context(self): + return { + 'request': self.request, + 'format': self.format_kwarg, + 'view': self + } + + def list(self, request, *args, **kwargs): + return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) + + @action_decorator(detail=False, methods=['POST']) + def reset(self, request, *args, **kwargs): + # Only run when in DEBUG mode! It's only used for tests + if not settings.DEBUG: + return HttpResponseBadRequest("Only allowed in debug mode.") + + with transaction.atomic(): + user_queryset = get_user_queryset(User.objects.all(), self) + user = get_object_or_404(user_queryset, username=request.data.get('user').get('username')) + + # Only allow test users for extra safety + if not getattr(user, User.USERNAME_FIELD).startswith('test_user'): + return HttpResponseBadRequest("Endpoint not allowed for user.") + + if hasattr(user, 'userinfo'): + user.userinfo.delete() + + serializer = AuthenticationSignupSerializer(data=request.data, context=self.get_serializer_context()) + serializer.is_valid(raise_exception=True) + serializer.save() + + # Delete all of the journal data for this user for a clear test env + user.collection_set.all().delete() + user.incoming_invitations.all().delete() + + # FIXME: also delete chunk files!!! + + return HttpResponse() diff --git a/etebase-server.ini.example b/etebase-server.ini.example new file mode 100644 index 0000000..2b4682a --- /dev/null +++ b/etebase-server.ini.example @@ -0,0 +1,17 @@ +[global] +secret_file = secret.txt +debug = false +;Advanced options, only uncomment if you know what you're doing: +;static_root = /path/to/static +;static_url = /static/ +;media_root = /path/to/media +;media_url = /user-media/ +;language_code = en-us +;time_zone = UTC + +[allowed_hosts] +allowed_host1 = example.com + +[database] +engine = django.db.backends.sqlite3 +name = db.sqlite3 diff --git a/etebase_server/__init__.py b/etebase_server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/etebase_server/asgi.py b/etebase_server/asgi.py new file mode 100644 index 0000000..44f1c53 --- /dev/null +++ b/etebase_server/asgi.py @@ -0,0 +1,16 @@ +""" +ASGI config for etebase_server project. + +It exposes the ASGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ +""" + +import os + +from django.core.asgi import get_asgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'etebase_server.settings') + +application = get_asgi_application() diff --git a/etebase_server/settings.py b/etebase_server/settings.py new file mode 100644 index 0000000..f785cb7 --- /dev/null +++ b/etebase_server/settings.py @@ -0,0 +1,179 @@ +""" +Django settings for etebase_server project. + +Generated by 'django-admin startproject' using Django 3.0.3. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/topics/settings/ + +For the full list of settings and their values, see +https://docs.djangoproject.com/en/3.0/ref/settings/ +""" + +import os +import configparser +from .utils import get_secret_from_file + +# Build paths inside the project like this: os.path.join(BASE_DIR, ...) +BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + +AUTH_USER_MODEL = 'myauth.User' + + +# Quick-start development settings - unsuitable for production +# See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ + +# SECURITY WARNING: keep the secret key used in production secret! +# See secret.py for how this is generated; uses a file 'secret.txt' in the root +# directory +SECRET_FILE = os.path.join(BASE_DIR, "secret.txt") + +# SECURITY WARNING: don't run with debug turned on in production! +DEBUG = True + +ALLOWED_HOSTS = [] + +# Database +# https://docs.djangoproject.com/en/2.0/ref/settings/#databases + +DATABASES = { + 'default': { + 'ENGINE': 'django.db.backends.sqlite3', + 'NAME': os.environ.get('ETEBASE_DB_PATH', + os.path.join(BASE_DIR, 'db.sqlite3')), + } +} + + +# Application definition + +INSTALLED_APPS = [ + 'django.contrib.admin', + 'django.contrib.auth', + 'django.contrib.contenttypes', + 'django.contrib.sessions', + 'django.contrib.messages', + 'django.contrib.staticfiles', + 'corsheaders', + 'rest_framework', + 'myauth.apps.MyauthConfig', + 'django_etebase.apps.DjangoEtebaseConfig', + 'django_etebase.token_auth.apps.TokenAuthConfig', +] + +MIDDLEWARE = [ + 'django.middleware.security.SecurityMiddleware', + 'django.contrib.sessions.middleware.SessionMiddleware', + 'corsheaders.middleware.CorsMiddleware', + 'django.middleware.common.CommonMiddleware', + 'django.middleware.csrf.CsrfViewMiddleware', + 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.messages.middleware.MessageMiddleware', + 'django.middleware.clickjacking.XFrameOptionsMiddleware', +] + +ROOT_URLCONF = 'etebase_server.urls' + +TEMPLATES = [ + { + 'BACKEND': 'django.template.backends.django.DjangoTemplates', + 'DIRS': [ + os.path.join(BASE_DIR, 'templates') + ], + 'APP_DIRS': True, + 'OPTIONS': { + 'context_processors': [ + 'django.template.context_processors.debug', + 'django.template.context_processors.request', + 'django.contrib.auth.context_processors.auth', + 'django.contrib.messages.context_processors.messages', + ], + }, + }, +] + +WSGI_APPLICATION = 'etebase_server.wsgi.application' + + +# Password validation +# https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators + +AUTH_PASSWORD_VALIDATORS = [ + { + 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', + }, + { + 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', + }, +] + + +# Internationalization +# https://docs.djangoproject.com/en/3.0/topics/i18n/ + +LANGUAGE_CODE = 'en-us' + +TIME_ZONE = 'UTC' + +USE_I18N = True + +USE_L10N = True + +USE_TZ = True + +# Cors +CORS_ORIGIN_ALLOW_ALL = True + +# Static files (CSS, JavaScript, Images) +# https://docs.djangoproject.com/en/3.0/howto/static-files/ + +STATIC_URL = '/static/' +STATIC_ROOT = os.environ.get('DJANGO_STATIC_ROOT', os.path.join(BASE_DIR, 'static')) + +MEDIA_ROOT = os.environ.get('DJANGO_MEDIA_ROOT', os.path.join(BASE_DIR, 'media')) +MEDIA_URL = '/user-media/' + + +# Define where to find configuration files +config_locations = ['etebase-server.ini', '/etc/etebase-server/etebase-server.ini'] +# Use config file if present +if any(os.path.isfile(x) for x in config_locations): + config = configparser.ConfigParser() + config.read(config_locations) + + section = config['global'] + + SECRET_FILE = section.get('secret_file', SECRET_FILE) + STATIC_ROOT = section.get('static_root', STATIC_ROOT) + STATIC_URL = section.get('static_url', STATIC_URL) + MEDIA_ROOT = section.get('media_root', MEDIA_ROOT) + MEDIA_URL = section.get('media_url', MEDIA_URL) + LANGUAGE_CODE = section.get('language_code', LANGUAGE_CODE) + TIME_ZONE = section.get('time_zone', TIME_ZONE) + DEBUG = section.getboolean('debug', DEBUG) + + if 'allowed_hosts' in config: + ALLOWED_HOSTS = [y for x, y in config.items('allowed_hosts')] + + if 'database' in config: + DATABASES = { 'default': { x.upper(): y for x, y in config.items('database') } } + +ETEBASE_API_PERMISSIONS = ('rest_framework.permissions.IsAuthenticated', ) +ETEBASE_API_AUTHENTICATORS = ('django_etebase.token_auth.authentication.TokenAuthentication', + 'rest_framework.authentication.SessionAuthentication') +ETEBASE_CREATE_USER_FUNC = 'django_etebase.utils.create_user_blocked' + +# Make an `etebase_server_settings` module available to override settings. +try: + from etebase_server_settings import * +except ImportError: + pass + +if 'SECRET_KEY' not in locals(): + SECRET_KEY = get_secret_from_file(SECRET_FILE) diff --git a/etebase_server/urls.py b/etebase_server/urls.py new file mode 100644 index 0000000..fddc32f --- /dev/null +++ b/etebase_server/urls.py @@ -0,0 +1,17 @@ +from django.conf import settings +from django.conf.urls import include, url +from django.contrib import admin +from django.urls import path +from django.views.generic import TemplateView + +urlpatterns = [ + url(r'^api/', include('django_etebase.urls')), + url(r'^admin/', admin.site.urls), + + path('', TemplateView.as_view(template_name='success.html')), +] + +if settings.DEBUG: + urlpatterns += [ + url(r'^api-auth/', include('rest_framework.urls', namespace='rest_framework')), + ] diff --git a/etebase_server/utils.py b/etebase_server/utils.py new file mode 100644 index 0000000..21c99f2 --- /dev/null +++ b/etebase_server/utils.py @@ -0,0 +1,25 @@ +# Copyright © 2017 Tom Hacohen +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, version 3. +# +# This library is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . + +from django.core.management import utils + +def get_secret_from_file(path): + try: + with open(path, "r") as f: + return f.read().strip() + except EnvironmentError: + with open(path, "w") as f: + secret_key = utils.get_random_secret_key() + f.write(secret_key) + return secret_key diff --git a/etebase_server/wsgi.py b/etebase_server/wsgi.py new file mode 100644 index 0000000..cf449a1 --- /dev/null +++ b/etebase_server/wsgi.py @@ -0,0 +1,16 @@ +""" +WSGI config for etebase_server project. + +It exposes the WSGI callable as a module-level variable named ``application``. + +For more information on this file, see +https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/ +""" + +import os + +from django.core.wsgi import get_wsgi_application + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'etebase_server.settings') + +application = get_wsgi_application() diff --git a/example-configs/nginx-uwsgi/README.md b/example-configs/nginx-uwsgi/README.md new file mode 100644 index 0000000..55b5fa5 --- /dev/null +++ b/example-configs/nginx-uwsgi/README.md @@ -0,0 +1,22 @@ +# Running `etebase` under `nginx` and `uwsgi` + +This configuration assumes that etebase server has been installed in the home folder of a non privileged user +called `EtebaseUser` following the instructions in . Also that static +files have been collected at `/srv/http/etebase_server` by running the following commands: + +```shell +sudo mkdir -p /srv/http/etebase_server/static +sudo chown -R EtebaseUser /srv/http/etebase_server +sudo su EtebaseUser +cd /path/to/etebase +ln -s /srv/http/etebase_server/static static +./manage.py collectstatic +``` + +It is also assumed that `nginx` and `uwsgi` have been installed system wide by `root`, and that `nginx` is running as user/group `www-data`. + +In this setup, `uwsgi` running as a `systemd` service as `root` creates a unix socket with read-write access +to both `EtebaseUser` and `nginx`. It then drops its `root` privilege and runs `etebase` as `EtebaseUser`. + +`nginx` listens on the `https` port (or a non standard port `https` port if desired), delivers static pages directly +and for everything else, communicates with `etebase` over the unix socket. diff --git a/example-configs/nginx-uwsgi/etebase.ini b/example-configs/nginx-uwsgi/etebase.ini new file mode 100644 index 0000000..a2ebe97 --- /dev/null +++ b/example-configs/nginx-uwsgi/etebase.ini @@ -0,0 +1,15 @@ +# uwsgi configuration file +# typical location of this file would be /etc/uwsgi/sites/etebase.ini + +[uwsgi] +socket = /path/to/etebase_server.sock +chown-socket = EtebaseUser:www-data +chmod-socket = 660 +vacuum = true + + +uid = EtebaseUser +chdir = /path/to/etebase +home = %(chdir)/.venv +module = etebase_server.wsgi +master = true diff --git a/example-configs/nginx-uwsgi/my.server.name.conf b/example-configs/nginx-uwsgi/my.server.name.conf index 53b4fb7..6b5de6e 100644 --- a/example-configs/nginx-uwsgi/my.server.name.conf +++ b/example-configs/nginx-uwsgi/my.server.name.conf @@ -1,30 +1,35 @@ -# nginx configuration for etesync server running on https://my.server.name +# nginx configuration for etebase server running on https://my.server.name # typical location of this file would be /etc/nginx/sites-available/my.server.name.conf server { server_name my.server.name; - root /srv/http/etesync_server; + root /srv/http/etebase_server; + + client_max_body_size 20M; - client_max_body_size 5M; - location /static { expires 1y; try_files $uri $uri/ =404; } + location /media { + expires 1y; + try_files $uri $uri/ =404; + } + location / { - uwsgi_pass unix:/path/to/etesync_server.sock; + uwsgi_pass unix:/path/to/etebase_server.sock; include uwsgi_params; } # change 443 to say 9443 to run on a non standard port - listen 443 ssl; - listen [::]:443 ssl; + listen 443 ssl; + listen [::]:443 ssl; # Enable these two instead of the two above if your nginx supports http2 # listen 443 ssl http2; # listen [::]:443 ssl http2; - + ssl_certificate /path/to/certificate-file ssl_certificate_key /path/to/certificate-key-file # other ssl directives as needed diff --git a/manage.py b/manage.py index 56f041a..b793fd2 100755 --- a/manage.py +++ b/manage.py @@ -1,22 +1,21 @@ #!/usr/bin/env python +"""Django's command-line utility for administrative tasks.""" import os import sys -if __name__ == "__main__": - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etesync_server.settings") + +def main(): + os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'etebase_server.settings') try: from django.core.management import execute_from_command_line - except ImportError: - # The above import may fail for some other reason. Ensure that the - # issue is really that Django is missing to avoid masking other - # exceptions on Python 2. - try: - import django - except ImportError: - raise ImportError( - "Couldn't import Django. Are you sure it's installed and " - "available on your PYTHONPATH environment variable? Did you " - "forget to activate a virtual environment?" - ) - raise + except ImportError as exc: + raise ImportError( + "Couldn't import Django. Are you sure it's installed and " + "available on your PYTHONPATH environment variable? Did you " + "forget to activate a virtual environment?" + ) from exc execute_from_command_line(sys.argv) + + +if __name__ == '__main__': + main() diff --git a/myauth/__init__.py b/myauth/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/myauth/admin.py b/myauth/admin.py new file mode 100644 index 0000000..f91be8f --- /dev/null +++ b/myauth/admin.py @@ -0,0 +1,5 @@ +from django.contrib import admin +from django.contrib.auth.admin import UserAdmin +from .models import User + +admin.site.register(User, UserAdmin) diff --git a/myauth/apps.py b/myauth/apps.py new file mode 100644 index 0000000..611e83d --- /dev/null +++ b/myauth/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class MyauthConfig(AppConfig): + name = 'myauth' diff --git a/myauth/migrations/0001_initial.py b/myauth/migrations/0001_initial.py new file mode 100644 index 0000000..1f81e95 --- /dev/null +++ b/myauth/migrations/0001_initial.py @@ -0,0 +1,44 @@ +# Generated by Django 3.0.3 on 2020-05-13 13:00 + +import django.contrib.auth.models +import django.contrib.auth.validators +from django.db import migrations, models +import django.utils.timezone + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + ('auth', '0011_update_proxy_permissions'), + ] + + operations = [ + migrations.CreateModel( + name='User', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('password', models.CharField(max_length=128, verbose_name='password')), + ('last_login', models.DateTimeField(blank=True, null=True, verbose_name='last login')), + ('is_superuser', models.BooleanField(default=False, help_text='Designates that this user has all permissions without explicitly assigning them.', verbose_name='superuser status')), + ('username', models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 150 characters or fewer. Letters, digits and @/./+/-/_ only.', max_length=150, unique=True, validators=[django.contrib.auth.validators.UnicodeUsernameValidator()], verbose_name='username')), + ('first_name', models.CharField(blank=True, max_length=30, verbose_name='first name')), + ('last_name', models.CharField(blank=True, max_length=150, verbose_name='last name')), + ('email', models.EmailField(blank=True, max_length=254, verbose_name='email address')), + ('is_staff', models.BooleanField(default=False, help_text='Designates whether the user can log into this admin site.', verbose_name='staff status')), + ('is_active', models.BooleanField(default=True, help_text='Designates whether this user should be treated as active. Unselect this instead of deleting accounts.', verbose_name='active')), + ('date_joined', models.DateTimeField(default=django.utils.timezone.now, verbose_name='date joined')), + ('groups', models.ManyToManyField(blank=True, help_text='The groups this user belongs to. A user will get all permissions granted to each of their groups.', related_name='user_set', related_query_name='user', to='auth.Group', verbose_name='groups')), + ('user_permissions', models.ManyToManyField(blank=True, help_text='Specific permissions for this user.', related_name='user_set', related_query_name='user', to='auth.Permission', verbose_name='user permissions')), + ], + options={ + 'verbose_name': 'user', + 'verbose_name_plural': 'users', + 'abstract': False, + }, + managers=[ + ('objects', django.contrib.auth.models.UserManager()), + ], + ), + ] diff --git a/myauth/migrations/0002_auto_20200515_0801.py b/myauth/migrations/0002_auto_20200515_0801.py new file mode 100644 index 0000000..3ce02b2 --- /dev/null +++ b/myauth/migrations/0002_auto_20200515_0801.py @@ -0,0 +1,19 @@ +# Generated by Django 3.0.3 on 2020-05-15 08:01 + +from django.db import migrations, models +import myauth.models + + +class Migration(migrations.Migration): + + dependencies = [ + ('myauth', '0001_initial'), + ] + + operations = [ + migrations.AlterField( + model_name='user', + name='username', + field=models.CharField(error_messages={'unique': 'A user with that username already exists.'}, help_text='Required. 150 characters or fewer. Letters, digits and ./+/-/_ only.', max_length=150, unique=True, validators=[myauth.models.UnicodeUsernameValidator()], verbose_name='username'), + ), + ] diff --git a/myauth/migrations/__init__.py b/myauth/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/myauth/models.py b/myauth/models.py new file mode 100644 index 0000000..611555b --- /dev/null +++ b/myauth/models.py @@ -0,0 +1,41 @@ +from django.contrib.auth.models import AbstractUser, UserManager as DjangoUserManager +from django.core import validators +from django.db import models +from django.utils.deconstruct import deconstructible +from django.utils.translation import gettext_lazy as _ + + +@deconstructible +class UnicodeUsernameValidator(validators.RegexValidator): + regex = r'^[\w.-]+\Z' + message = _( + 'Enter a valid username. This value may contain only letters, ' + 'numbers, and ./-/_ characters.' + ) + flags = 0 + + +class UserManager(DjangoUserManager): + def get_by_natural_key(self, username): + return self.get(**{self.model.USERNAME_FIELD + '__iexact': username}) + + +class User(AbstractUser): + username_validator = UnicodeUsernameValidator() + + objects = UserManager() + + username = models.CharField( + _('username'), + max_length=150, + unique=True, + help_text=_('Required. 150 characters or fewer. Letters, digits and ./-/_ only.'), + validators=[username_validator], + error_messages={ + 'unique': _("A user with that username already exists."), + }, + ) + + @classmethod + def normalize_username(cls, username): + return super().normalize_username(username).lower() diff --git a/myauth/tests.py b/myauth/tests.py new file mode 100644 index 0000000..7ce503c --- /dev/null +++ b/myauth/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/myauth/views.py b/myauth/views.py new file mode 100644 index 0000000..91ea44a --- /dev/null +++ b/myauth/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/requirements.in/base.txt b/requirements.in/base.txt new file mode 100644 index 0000000..7d5bf7e --- /dev/null +++ b/requirements.in/base.txt @@ -0,0 +1,7 @@ +django +django-cors-headers +djangorestframework +drf-nested-routers +msgpack +psycopg2-binary +pynacl diff --git a/requirements.in/development.txt b/requirements.in/development.txt new file mode 100644 index 0000000..c752bfb --- /dev/null +++ b/requirements.in/development.txt @@ -0,0 +1,3 @@ +coverage +pip-tools +pywatchman diff --git a/requirements.txt b/requirements.txt index 4195dc6..f6c8ed4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,19 @@ -Django>=2.2.9,<2.2.999 -django-cors-headers==3.2.1 -django-etesync-journal==1.2.0 -djangorestframework>=3.11.0,<3.11.999 -drf-nested-routers==0.91 -pytz +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=requirements.txt requirements.in/base.txt +# +asgiref==3.2.10 # via django +cffi==1.14.0 # via pynacl +django-cors-headers==3.2.1 # via -r requirements.in/base.txt +django==3.1.1 # via -r requirements.in/base.txt, django-cors-headers, djangorestframework, drf-nested-routers +djangorestframework==3.11.0 # via -r requirements.in/base.txt, drf-nested-routers +drf-nested-routers==0.91 # via -r requirements.in/base.txt +msgpack==1.0.0 # via -r requirements.in/base.txt +psycopg2-binary==2.8.4 # via -r requirements.in/base.txt +pycparser==2.20 # via cffi +pynacl==1.3.0 # via -r requirements.in/base.txt +pytz==2019.3 # via django +six==1.14.0 # via pynacl +sqlparse==0.3.0 # via django