From c1534f6587dfc25569fb270b8b7b43984da96f3e Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Wed, 23 Dec 2020 16:29:08 -0500 Subject: [PATCH 001/102] first commit --- etebase_fastapi/__init__.py | 0 etebase_fastapi/app.py | 29 ++++ etebase_fastapi/authentication.py | 251 ++++++++++++++++++++++++++++++ etebase_fastapi/collections.py | 0 etebase_fastapi/execptions.py | 42 +++++ etebase_fastapi/msgpack.py | 63 ++++++++ requirements.in/base.txt | 2 + requirements.txt | 30 ++-- 8 files changed, 405 insertions(+), 12 deletions(-) create mode 100644 etebase_fastapi/__init__.py create mode 100644 etebase_fastapi/app.py create mode 100644 etebase_fastapi/authentication.py create mode 100644 etebase_fastapi/collections.py create mode 100644 etebase_fastapi/execptions.py create mode 100644 etebase_fastapi/msgpack.py diff --git a/etebase_fastapi/__init__.py b/etebase_fastapi/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py new file mode 100644 index 0000000..0ee7aae --- /dev/null +++ b/etebase_fastapi/app.py @@ -0,0 +1,29 @@ +import os + +from django.core.wsgi import get_wsgi_application +from fastapi.middleware.cors import CORSMiddleware + +os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etebase_server.settings") +application = get_wsgi_application() +from fastapi import FastAPI, Request + +from .execptions import CustomHttpException +from .authentication import authentication_router +from .msgpack import MsgpackResponse + +app = FastAPI() +app.include_router(authentication_router, prefix="/api/v1/authentication") +app.add_middleware( + CORSMiddleware, allow_origin_regex="https?://.*", allow_credentials=True, allow_methods=["*"], allow_headers=["*"] +) + + +@app.exception_handler(CustomHttpException) +async def custom_exception_handler(request: Request, exc: CustomHttpException): + return MsgpackResponse(status_code=exc.status_code, content=exc.as_dict) + + +if __name__ == "__main__": + import uvicorn + + uvicorn.run(app, host="0.0.0.0", port=8080) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py new file mode 100644 index 0000000..b1a3272 --- /dev/null +++ b/etebase_fastapi/authentication.py @@ -0,0 +1,251 @@ +import dataclasses +import typing as t +from datetime import datetime +from functools import cached_property + +import nacl +import nacl.encoding +import nacl.hash +import nacl.secret +import nacl.signing +from asgiref.sync import sync_to_async +from django.conf import settings +from django.contrib.auth import get_user_model, user_logged_out, user_logged_in +from django.utils import timezone +from fastapi import APIRouter, Depends, status, Request, Response +from fastapi.security import APIKeyHeader +from pydantic import BaseModel + +from django_etebase import app_settings +from django_etebase.models import UserInfo +from django_etebase.serializers import UserSerializer +from django_etebase.token_auth.models import AuthToken +from django_etebase.token_auth.models import get_default_expiry +from django_etebase.views import msgpack_encode, msgpack_decode +from .execptions import AuthenticationFailed +from .msgpack import MsgpackResponse, MsgpackRoute + +User = get_user_model() +token_scheme = APIKeyHeader(name="Authorization") +AUTO_REFRESH = True +MIN_REFRESH_INTERVAL = 60 +authentication_router = APIRouter(route_class=MsgpackRoute) + + +@dataclasses.dataclass(frozen=True) +class AuthData: + user: User + token: AuthToken + + +class LoginChallengeData(BaseModel): + username: str + + +class LoginResponse(BaseModel): + username: str + challenge: bytes + host: str + action: t.Literal["login", "changePassword"] + + +class Authentication(BaseModel): + response: bytes + signature: bytes + + +class Login(Authentication): + @cached_property + def response_data(self) -> LoginResponse: + return LoginResponse(**msgpack_decode(self.response)) + + +class ChangePasswordResponse(LoginResponse): + loginPubkey: bytes + encryptedContent: bytes + + +class ChangePassword(Authentication): + @cached_property + def response_data(self) -> ChangePasswordResponse: + return ChangePasswordResponse(**msgpack_decode(self.response)) + + +def __renew_token(auth_token: AuthToken): + current_expiry = auth_token.expiry + new_expiry = get_default_expiry() + # Throttle refreshing of token to avoid db writes + delta = (new_expiry - current_expiry).total_seconds() + if delta > MIN_REFRESH_INTERVAL: + auth_token.expiry = new_expiry + auth_token.save(update_fields=("expiry",)) + + +@sync_to_async +def __get_authenticated_user(api_token: str): + api_token = api_token.split()[1] + try: + token: AuthToken = AuthToken.objects.select_related("user").get(key=api_token) + except AuthToken.DoesNotExist: + raise AuthenticationFailed(detail="Invalid token.") + if not token.user.is_active: + raise AuthenticationFailed(detail="User inactive or deleted.") + + if token.expiry is not None: + if token.expiry < timezone.now(): + token.delete() + raise AuthenticationFailed(detail="Invalid token.") + + if AUTO_REFRESH: + __renew_token(token) + + return token.user, token + + +async def get_auth_data(api_token: str = Depends(token_scheme)) -> AuthData: + user, token = await __get_authenticated_user(api_token) + return AuthData(user, token) + + +async def get_authenticated_user(api_token: str = Depends(token_scheme)) -> User: + user, token = await __get_authenticated_user(api_token) + return user + + +@sync_to_async +def __get_login_user(username: str) -> User: + kwargs = {User.USERNAME_FIELD + "__iexact": username.lower()} + try: + user = User.objects.get(**kwargs) + if not hasattr(user, "userinfo"): + raise AuthenticationFailed(code="user_not_init", detail="User not properly init") + return user + except User.DoesNotExist: + raise AuthenticationFailed(code="user_not_found", detail="User not found") + + +async def get_login_user(challenge: LoginChallengeData) -> User: + user = await __get_login_user(challenge.username) + return user + + +def get_encryption_key(salt): + key = nacl.hash.blake2b(settings.SECRET_KEY.encode(), encoder=nacl.encoding.RawEncoder) + return nacl.hash.blake2b( + b"", + key=key, + salt=salt[: nacl.hash.BLAKE2B_SALTBYTES], + person=b"etebase-auth", + encoder=nacl.encoding.RawEncoder, + ) + + +@sync_to_async +def save_changed_password(data: ChangePassword, user: User): + response_data = data.response_data + user_info: UserInfo = user.userinfo + user_info.loginPubkey = response_data.loginPubkey + user_info.encryptedContent = response_data.encryptedContent + user_info.save() + + +@sync_to_async +def login_response_data(user: User): + return { + "token": AuthToken.objects.create(user=user).key, + "user": UserSerializer(user).data, + } + + +@sync_to_async +def send_user_logged_in_async(user: User, request: Request): + user_logged_in.send(sender=user.__class__, request=request, user=user) + + +@sync_to_async +def send_user_logged_out_async(user: User, request: Request): + user_logged_out.send(sender=user.__class__, request=request, user=user) + + +@sync_to_async +def validate_login_request( + validated_data: LoginResponse, + challenge_sent_to_user: Authentication, + user: User, + expected_action: str, + host_from_request: str, +) -> t.Optional[MsgpackResponse]: + + enc_key = get_encryption_key(bytes(user.userinfo.salt)) + box = nacl.secret.SecretBox(enc_key) + challenge_data = msgpack_decode(box.decrypt(validated_data.challenge)) + now = int(datetime.now().timestamp()) + if validated_data.action != expected_action: + content = { + "code": "wrong_action", + "detail": 'Expected "{}" but got something else'.format(challenge_sent_to_user.response), + } + return MsgpackResponse(content, status_code=status.HTTP_400_BAD_REQUEST) + elif now - challenge_data["timestamp"] > app_settings.CHALLENGE_VALID_SECONDS: + content = {"code": "challenge_expired", "detail": "Login challenge has expired"} + return MsgpackResponse(content, status_code=status.HTTP_400_BAD_REQUEST) + elif challenge_data["userId"] != user.id: + content = {"code": "wrong_user", "detail": "This challenge is for the wrong user"} + return MsgpackResponse(content, status_code=status.HTTP_400_BAD_REQUEST) + elif not settings.DEBUG and validated_data.host.split(":", 1)[0] != host_from_request: + detail = 'Found wrong host name. Got: "{}" expected: "{}"'.format(validated_data.host, host_from_request) + content = {"code": "wrong_host", "detail": detail} + return MsgpackResponse(content, status_code=status.HTTP_400_BAD_REQUEST) + + verify_key = nacl.signing.VerifyKey(bytes(user.userinfo.loginPubkey), encoder=nacl.encoding.RawEncoder) + + try: + verify_key.verify(challenge_sent_to_user.response, challenge_sent_to_user.signature) + except nacl.exceptions.BadSignatureError: + return MsgpackResponse( + {"code": "login_bad_signature", "detail": "Wrong password for user."}, + status_code=status.HTTP_401_UNAUTHORIZED, + ) + + return None + + +@authentication_router.post("/login_challenge/") +async def login_challenge(user: User = Depends(get_login_user)): + enc_key = get_encryption_key(user.userinfo.salt) + box = nacl.secret.SecretBox(enc_key) + challenge_data = { + "timestamp": int(datetime.now().timestamp()), + "userId": user.id, + } + challenge = bytes(box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder)) + return MsgpackResponse({"salt": user.userinfo.salt, "version": user.userinfo.version, "challenge": challenge}) + + +@authentication_router.post("/login/") +async def login(data: Login, request: Request): + user = await get_login_user(LoginChallengeData(username=data.response_data.username)) + host = request.headers.get("Host") + bad_login_response = await validate_login_request(data.response_data, data, user, "login", host) + if bad_login_response is not None: + return bad_login_response + data = await login_response_data(user) + await send_user_logged_in_async(user, request) + return MsgpackResponse(data, status_code=status.HTTP_200_OK) + + +@authentication_router.post("/logout/") +async def logout(request: Request, auth_data: AuthData = Depends(get_auth_data)): + await sync_to_async(auth_data.token.delete)() + await send_user_logged_out_async(auth_data.user, request) + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +@authentication_router.post("/change_password/") +async def change_password(data: ChangePassword, request: Request, user: User = Depends(get_authenticated_user)): + host = request.headers.get("Host") + bad_login_response = await validate_login_request(data.response_data, data, user, "changePassword", host) + if bad_login_response is not None: + return bad_login_response + await save_changed_password(data, user) + return Response(status_code=status.HTTP_204_NO_CONTENT) diff --git a/etebase_fastapi/collections.py b/etebase_fastapi/collections.py new file mode 100644 index 0000000..e69de29 diff --git a/etebase_fastapi/execptions.py b/etebase_fastapi/execptions.py new file mode 100644 index 0000000..8808f5d --- /dev/null +++ b/etebase_fastapi/execptions.py @@ -0,0 +1,42 @@ +from fastapi import status + + +class CustomHttpException(Exception): + def __init__(self, code: str, detail: str, status_code: int = status.HTTP_400_BAD_REQUEST): + self.status_code = status_code + self.code = code + self.detail = detail + + @property + def as_dict(self) -> dict: + return {"code": self.code, "detail": self.detail} + + +class AuthenticationFailed(CustomHttpException): + def __init__( + self, + code="authentication_failed", + detail: str = "Incorrect authentication credentials.", + status_code: int = status.HTTP_401_UNAUTHORIZED, + ): + super().__init__(code=code, detail=detail, status_code=status_code) + + +class NotAuthenticated(CustomHttpException): + def __init__( + self, + code="not_authenticated", + detail: str = "Authentication credentials were not provided.", + status_code: int = status.HTTP_401_UNAUTHORIZED, + ): + super().__init__(code=code, detail=detail, status_code=status_code) + + +class PermissionDenied(CustomHttpException): + def __init__( + self, + code="permission_denied", + detail: str = "You do not have permission to perform this action.", + status_code: int = status.HTTP_403_FORBIDDEN, + ): + super().__init__(code=code, detail=detail, status_code=status_code) diff --git a/etebase_fastapi/msgpack.py b/etebase_fastapi/msgpack.py new file mode 100644 index 0000000..53e18cb --- /dev/null +++ b/etebase_fastapi/msgpack.py @@ -0,0 +1,63 @@ +import typing as t +import msgpack +from fastapi.routing import APIRoute, get_request_handler +from starlette.requests import Request +from starlette.responses import Response + + +class MsgpackRequest(Request): + media_type = "application/msgpack" + + async def json(self) -> bytes: + if not hasattr(self, "_json"): + body = await super().body() + self._json = msgpack.unpackb(body, raw=False) + return self._json + + +class MsgpackResponse(Response): + media_type = "application/msgpack" + + def render(self, content: t.Any) -> bytes: + return msgpack.packb(content, use_bin_type=True) + + +class MsgpackRoute(APIRoute): + # keep track of content-type -> request classes + REQUESTS_CLASSES = {MsgpackRequest.media_type: MsgpackRequest} + # keep track of content-type -> response classes + ROUTES_HANDLERS_CLASSES = {MsgpackResponse.media_type: MsgpackResponse} + + def _get_media_type_route_handler(self, media_type): + return get_request_handler( + dependant=self.dependant, + body_field=self.body_field, + status_code=self.status_code, + # use custom response class or fallback on default self.response_class + response_class=self.ROUTES_HANDLERS_CLASSES.get(media_type, self.response_class), + response_field=self.secure_cloned_response_field, + response_model_include=self.response_model_include, + response_model_exclude=self.response_model_exclude, + response_model_by_alias=self.response_model_by_alias, + response_model_exclude_unset=self.response_model_exclude_unset, + response_model_exclude_defaults=self.response_model_exclude_defaults, + response_model_exclude_none=self.response_model_exclude_none, + dependency_overrides_provider=self.dependency_overrides_provider, + ) + + def get_route_handler(self) -> t.Callable: + async def custom_route_handler(request: Request) -> Response: + + content_type = request.headers.get("Content-Type") + try: + request_cls = self.REQUESTS_CLASSES[content_type] + request = request_cls(request.scope, request.receive) + except KeyError: + # nothing registered to handle content_type, process given requests as-is + pass + + accept = request.headers.get("Accept") + route_handler = self._get_media_type_route_handler(accept) + return await route_handler(request) + + return custom_route_handler diff --git a/requirements.in/base.txt b/requirements.in/base.txt index 7d5bf7e..ca8dd94 100644 --- a/requirements.in/base.txt +++ b/requirements.in/base.txt @@ -5,3 +5,5 @@ drf-nested-routers msgpack psycopg2-binary pynacl +fastapi +uvicorn \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index f6c8ed4..3d19eaf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,16 +4,22 @@ # # pip-compile --output-file=requirements.txt requirements.in/base.txt # -asgiref==3.2.10 # via django -cffi==1.14.0 # via pynacl -django-cors-headers==3.2.1 # via -r requirements.in/base.txt -django==3.1.1 # via -r requirements.in/base.txt, django-cors-headers, djangorestframework, drf-nested-routers -djangorestframework==3.11.0 # via -r requirements.in/base.txt, drf-nested-routers -drf-nested-routers==0.91 # via -r requirements.in/base.txt -msgpack==1.0.0 # via -r requirements.in/base.txt -psycopg2-binary==2.8.4 # via -r requirements.in/base.txt +asgiref==3.3.1 # via django +cffi==1.14.4 # via pynacl +click==7.1.2 # via uvicorn +django-cors-headers==3.6.0 # via -r requirements.in/base.txt +django==3.1.4 # via -r requirements.in/base.txt, django-cors-headers, djangorestframework, drf-nested-routers +djangorestframework==3.12.2 # via -r requirements.in/base.txt, drf-nested-routers +drf-nested-routers==0.92.5 # via -r requirements.in/base.txt +fastapi==0.63.0 # via -r requirements.in/base.txt +h11==0.11.0 # via uvicorn +msgpack==1.0.2 # via -r requirements.in/base.txt +psycopg2-binary==2.8.6 # via -r requirements.in/base.txt pycparser==2.20 # via cffi -pynacl==1.3.0 # via -r requirements.in/base.txt -pytz==2019.3 # via django -six==1.14.0 # via pynacl -sqlparse==0.3.0 # via django +pydantic==1.7.3 # via fastapi +pynacl==1.4.0 # via -r requirements.in/base.txt +pytz==2020.4 # via django +six==1.15.0 # via pynacl +sqlparse==0.4.1 # via django +starlette==0.13.6 # via fastapi +uvicorn==0.13.2 # via -r requirements.in/base.txt From 25cb4fec0cbe16d062123460d3ca2ed2669b3738 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 11:10:43 +0200 Subject: [PATCH 002/102] msgpack.py: allow pydantic BaseModel in content --- etebase_fastapi/msgpack.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/etebase_fastapi/msgpack.py b/etebase_fastapi/msgpack.py index 53e18cb..399f3d0 100644 --- a/etebase_fastapi/msgpack.py +++ b/etebase_fastapi/msgpack.py @@ -1,6 +1,7 @@ import typing as t import msgpack from fastapi.routing import APIRoute, get_request_handler +from pydantic import BaseModel from starlette.requests import Request from starlette.responses import Response @@ -19,6 +20,8 @@ class MsgpackResponse(Response): media_type = "application/msgpack" def render(self, content: t.Any) -> bytes: + if isinstance(content, BaseModel): + content = content.dict() return msgpack.packb(content, use_bin_type=True) From 16a99f02ea9465fb5d5690820b38c4055cbc66ea Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 11:12:22 +0200 Subject: [PATCH 003/102] snapshot --- etebase_fastapi/app.py | 8 +++- etebase_fastapi/authentication.py | 3 ++ etebase_fastapi/collection.py | 72 +++++++++++++++++++++++++++++++ etebase_fastapi/collections.py | 0 4 files changed, 81 insertions(+), 2 deletions(-) create mode 100644 etebase_fastapi/collection.py delete mode 100644 etebase_fastapi/collections.py diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index 0ee7aae..acfb42f 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -9,10 +9,14 @@ from fastapi import FastAPI, Request from .execptions import CustomHttpException from .authentication import authentication_router +from .collection import collection_router from .msgpack import MsgpackResponse app = FastAPI() -app.include_router(authentication_router, prefix="/api/v1/authentication") +VERSION = "v1" +BASE_PATH = f"/api/{VERSION}" +app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication") +app.include_router(collection_router, prefix=f"{BASE_PATH}/collection") app.add_middleware( CORSMiddleware, allow_origin_regex="https?://.*", allow_credentials=True, allow_methods=["*"], allow_headers=["*"] ) @@ -26,4 +30,4 @@ async def custom_exception_handler(request: Request, exc: CustomHttpException): if __name__ == "__main__": import uvicorn - uvicorn.run(app, host="0.0.0.0", port=8080) + uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index b1a3272..697f3f4 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -50,6 +50,9 @@ class LoginResponse(BaseModel): class Authentication(BaseModel): + class Config: + keep_untouched = (cached_property,) + response: bytes signature: bytes diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py new file mode 100644 index 0000000..ec0125d --- /dev/null +++ b/etebase_fastapi/collection.py @@ -0,0 +1,72 @@ +import typing as t + +from django.contrib.auth import get_user_model +from django.db.models import Q +from django.db.models import QuerySet +from fastapi import APIRouter, Depends +from pydantic import BaseModel +from asgiref.sync import sync_to_async + +from django_etebase.models import Collection, Stoken, AccessLevels, CollectionMember +from .authentication import get_authenticated_user +from .msgpack import MsgpackRoute, MsgpackResponse + +User = get_user_model() +collection_router = APIRouter(route_class=MsgpackRoute) +default_queryset = Collection.objects.all() + + +class ListMulti(BaseModel): + collectionTypes: t.List[bytes] + + +class CollectionItemOut(BaseModel): + uid: str + + +class CollectionOut(BaseModel): + collectionKey: bytes + collectionType: bytes + accessLevel: AccessLevels + stoken: str + item: CollectionItemOut + + @classmethod + def from_orm_user(cls: t.Type["CollectionOut"], obj: Collection, user: User) -> "CollectionOut": + member: CollectionMember = obj.members.get(user=user) + collection_type = member.collectionType + return cls( + collectionType=collection_type and collection_type.uid, + collectionKey=member.encryptionKey, + accessLevel=member.accessLevel, + stoken=obj.stoken, + item=CollectionItemOut(uid=obj.main_item.uid), + ) + + +class ListResponse(BaseModel): + data: t.List[CollectionOut] + stoken: t.Optional[str] + done: bool + + +@sync_to_async +def list_common(queryset: QuerySet, stoken: t.Optional[str], user: User) -> MsgpackResponse: + data: t.List[CollectionOut] = [CollectionOut.from_orm_user(item, user) for item in queryset] + ret = ListResponse(data=data, stoken=stoken, done=True) + return MsgpackResponse(content=ret) + + +def get_collection_queryset(user: User, queryset: QuerySet) -> QuerySet: + return queryset.filter(members__user=user) + + +@collection_router.post("/list_multi/") +async def list_multi(limit: int, data: ListMulti, user: User = Depends(get_authenticated_user)): + queryset = get_collection_queryset(user, default_queryset) + # FIXME: Remove the isnull part once we attach collection types to all objects ("collection-type-migration") + queryset = queryset.filter( + Q(members__collectionType__uid__in=data.collectionTypes) | Q(members__collectionType__isnull=True) + ) + response = await list_common(queryset, None, user) + return response diff --git a/etebase_fastapi/collections.py b/etebase_fastapi/collections.py deleted file mode 100644 index e69de29..0000000 From f70e2d80a64d33ad485dbb58f0cfa113cdc72f4b Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 11:52:43 +0200 Subject: [PATCH 004/102] stoken_handler.py --- etebase_fastapi/stoken_handler.py | 61 +++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 etebase_fastapi/stoken_handler.py diff --git a/etebase_fastapi/stoken_handler.py b/etebase_fastapi/stoken_handler.py new file mode 100644 index 0000000..c840f0e --- /dev/null +++ b/etebase_fastapi/stoken_handler.py @@ -0,0 +1,61 @@ +import typing as t + +from django.db.models import QuerySet +from fastapi import status + +from django_etebase.exceptions import EtebaseValidationError +from django_etebase.models import Stoken + +# TODO missing stoken_annotation type +StokenAnnotation = t.Any + + +def get_stoken_obj(stoken: t.Optional[str]): + if stoken is not None: + try: + return Stoken.objects.get(uid=stoken) + except Stoken.DoesNotExist: + raise EtebaseValidationError("bad_stoken", "Invalid stoken.", status_code=status.HTTP_400_BAD_REQUEST) + + return None + + +def filter_by_stoken( + stoken: t.Optional[str], queryset: QuerySet, stoken_annotation: StokenAnnotation +) -> t.Tuple[QuerySet, t.Optional[str]]: + stoken_rev = get_stoken_obj(stoken) + + queryset = queryset.annotate(max_stoken=stoken_annotation).order_by("max_stoken") + + if stoken_rev is not None: + queryset = queryset.filter(max_stoken__gt=stoken_rev.id) + + return queryset, stoken_rev + + +def get_queryset_stoken(queryset: list) -> t.Optional[Stoken]: + maxid = -1 + for row in queryset: + rowmaxid = getattr(row, "max_stoken") or -1 + maxid = max(maxid, rowmaxid) + new_stoken = (maxid >= 0) and Stoken.objects.get(id=maxid) + + return new_stoken or None + + +def filter_by_stoken_and_limit( + stoken: t.Optional[str], limit: int, queryset: QuerySet, stoken_annotation: StokenAnnotation +) -> t.Tuple[list, t.Optional[Stoken], bool]: + + queryset, stoken_rev = filter_by_stoken(stoken=stoken, queryset=queryset, stoken_annotation=stoken_annotation) + + result = list(queryset[: limit + 1]) + if len(result) < limit + 1: + done = True + else: + done = False + result = result[:-1] + + new_stoken_obj = get_queryset_stoken(result) or stoken_rev + + return result, new_stoken_obj, done From 7d864594802b5f7f0d8e23ab383f37e68a79df79 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 11:53:11 +0200 Subject: [PATCH 005/102] collection.pyL list_multi --- etebase_fastapi/collection.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index ec0125d..7c9aca8 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -10,6 +10,7 @@ from asgiref.sync import sync_to_async from django_etebase.models import Collection, Stoken, AccessLevels, CollectionMember from .authentication import get_authenticated_user from .msgpack import MsgpackRoute, MsgpackResponse +from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_queryset_stoken User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute) @@ -51,9 +52,11 @@ class ListResponse(BaseModel): @sync_to_async -def list_common(queryset: QuerySet, stoken: t.Optional[str], user: User) -> MsgpackResponse: +def list_common(queryset: QuerySet, user: User, stoken: t.Optional[str], limit: int) -> MsgpackResponse: + result, new_stoken_obj, done = filter_by_stoken_and_limit(stoken, limit, queryset, Collection.stoken_annotation) + new_stoken = new_stoken_obj and new_stoken_obj.uid data: t.List[CollectionOut] = [CollectionOut.from_orm_user(item, user) for item in queryset] - ret = ListResponse(data=data, stoken=stoken, done=True) + ret = ListResponse(data=data, stoken=new_stoken, done=done) return MsgpackResponse(content=ret) @@ -62,11 +65,13 @@ def get_collection_queryset(user: User, queryset: QuerySet) -> QuerySet: @collection_router.post("/list_multi/") -async def list_multi(limit: int, data: ListMulti, user: User = Depends(get_authenticated_user)): +async def list_multi( + data: ListMulti, stoken: t.Optional[str] = None, limit: int = 50, user: User = Depends(get_authenticated_user) +): queryset = get_collection_queryset(user, default_queryset) # FIXME: Remove the isnull part once we attach collection types to all objects ("collection-type-migration") queryset = queryset.filter( Q(members__collectionType__uid__in=data.collectionTypes) | Q(members__collectionType__isnull=True) ) - response = await list_common(queryset, None, user) + response = await list_common(queryset, user, stoken, limit) return response From c90e92b0f00aea03db62edd803925cfb8ceb4ea5 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 12:36:06 +0200 Subject: [PATCH 006/102] snapshot --- etebase_fastapi/authentication.py | 52 +++++++++++++++++++++++++++++++ etebase_fastapi/execptions.py | 46 +++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 697f3f4..3ae6c4b 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -2,6 +2,7 @@ import dataclasses import typing as t from datetime import datetime from functools import cached_property +from django.core import exceptions as django_exceptions import nacl import nacl.encoding @@ -11,16 +12,19 @@ import nacl.signing from asgiref.sync import sync_to_async from django.conf import settings from django.contrib.auth import get_user_model, user_logged_out, user_logged_in +from django.db import transaction from django.utils import timezone from fastapi import APIRouter, Depends, status, Request, Response from fastapi.security import APIKeyHeader from pydantic import BaseModel from django_etebase import app_settings +from django_etebase.exceptions import EtebaseValidationError from django_etebase.models import UserInfo from django_etebase.serializers import UserSerializer from django_etebase.token_auth.models import AuthToken from django_etebase.token_auth.models import get_default_expiry +from django_etebase.utils import create_user from django_etebase.views import msgpack_encode, msgpack_decode from .execptions import AuthenticationFailed from .msgpack import MsgpackResponse, MsgpackRoute @@ -74,6 +78,19 @@ class ChangePassword(Authentication): return ChangePasswordResponse(**msgpack_decode(self.response)) +class UserSignup(BaseModel): + username: str + email: str + + +class SignupIn(BaseModel): + user: UserSignup + salt: bytes + loginPubkey: bytes + pubkey: bytes + encryptedContent: bytes + + def __renew_token(auth_token: AuthToken): current_expiry = auth_token.expiry new_expiry = get_default_expiry() @@ -252,3 +269,38 @@ async def change_password(data: ChangePassword, request: Request, user: User = D return bad_login_response await save_changed_password(data, user) return Response(status_code=status.HTTP_204_NO_CONTENT) + + +@sync_to_async +def signup_save(data: SignupIn): + user_data = data.user + with transaction.atomic(): + try: + # XXX-TOM + # view = self.context.get("view", None) + # user_queryset = get_user_queryset(User.objects.all(), view) + user_queryset = User.objects.all() + instance = user_queryset.get(**{User.USERNAME_FIELD: user_data.username.lower()}) + except User.DoesNotExist: + # Create the user and save the casing the user chose as the first name + try: + # XXX-TOM + instance = create_user(**user_data.dict(), password=None, first_name=user_data.username, view=None) + instance.full_clean() + except EtebaseValidationError as e: + raise e + except django_exceptions.ValidationError as e: + self.transform_validation_error("user", e) + except Exception as e: + raise EtebaseValidationError("generic", str(e)) + + if hasattr(instance, "userinfo"): + raise EtebaseValidationError("user_exists", "User already exists", status_code=status.HTTP_409_CONFLICT) + + models.UserInfo.objects.create(**validated_data, owner=instance) + return instance + + +@authentication_router.post("/signup/") +async def signup(data: SignupIn): + pass diff --git a/etebase_fastapi/execptions.py b/etebase_fastapi/execptions.py index 8808f5d..2b35634 100644 --- a/etebase_fastapi/execptions.py +++ b/etebase_fastapi/execptions.py @@ -1,5 +1,7 @@ from fastapi import status +from django_etebase.exceptions import EtebaseValidationError + class CustomHttpException(Exception): def __init__(self, code: str, detail: str, status_code: int = status.HTTP_400_BAD_REQUEST): @@ -40,3 +42,47 @@ class PermissionDenied(CustomHttpException): status_code: int = status.HTTP_403_FORBIDDEN, ): super().__init__(code=code, detail=detail, status_code=status_code) + + +class ValidationError(CustomHttpException): + def __init__(self, code: str, detail: str, status_code: int = status.HTTP_400_BAD_REQUEST): + super().__init__(code=code, detail=detail, status_code=status_code) + + +def flatten_errors(field_name, errors): + ret = [] + if isinstance(errors, dict): + for error_key in errors: + error = errors[error_key] + ret.extend(flatten_errors("{}.{}".format(field_name, error_key), error)) + else: + for error in errors: + if error.messages: + message = error.messages[0] + else: + message = str(error) + ret.append( + { + "field": field_name, + "code": error.code, + "detail": message, + } + ) + return ret + + +def transform_validation_error(prefix, err): + if hasattr(err, "error_dict"): + errors = flatten_errors(prefix, err.error_dict) + elif not hasattr(err, "message"): + errors = flatten_errors(prefix, err.error_list) + else: + raise EtebaseValidationError(err.code, err.message) + raise ValidationError(code="field_errors", detail="Field validations failed.") + raise serializers.ValidationError( + { + "code": "field_errors", + "detail": "Field validations failed.", + "errors": errors, + } + ) From 72d4a725f5c969bff271b34239363a7c3ee909e3 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 13:21:20 +0200 Subject: [PATCH 007/102] validation errors --- etebase_fastapi/authentication.py | 24 ++++++++++----- etebase_fastapi/execptions.py | 51 ++++++++++++++++++++----------- 2 files changed, 51 insertions(+), 24 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 3ae6c4b..287b46e 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -18,15 +18,16 @@ from fastapi import APIRouter, Depends, status, Request, Response from fastapi.security import APIKeyHeader from pydantic import BaseModel -from django_etebase import app_settings +from django_etebase import app_settings, models from django_etebase.exceptions import EtebaseValidationError from django_etebase.models import UserInfo from django_etebase.serializers import UserSerializer +from django_etebase.signals import user_signed_up from django_etebase.token_auth.models import AuthToken from django_etebase.token_auth.models import get_default_expiry from django_etebase.utils import create_user from django_etebase.views import msgpack_encode, msgpack_decode -from .execptions import AuthenticationFailed +from .execptions import AuthenticationFailed, transform_validation_error, ValidationError from .msgpack import MsgpackResponse, MsgpackRoute User = get_user_model() @@ -272,7 +273,7 @@ async def change_password(data: ChangePassword, request: Request, user: User = D @sync_to_async -def signup_save(data: SignupIn): +def signup_save(data: SignupIn) -> User: user_data = data.user with transaction.atomic(): try: @@ -290,17 +291,26 @@ def signup_save(data: SignupIn): except EtebaseValidationError as e: raise e except django_exceptions.ValidationError as e: - self.transform_validation_error("user", e) + transform_validation_error("user", e) except Exception as e: raise EtebaseValidationError("generic", str(e)) if hasattr(instance, "userinfo"): - raise EtebaseValidationError("user_exists", "User already exists", status_code=status.HTTP_409_CONFLICT) + raise ValidationError("user_exists", "User already exists", status_code=status.HTTP_409_CONFLICT) - models.UserInfo.objects.create(**validated_data, owner=instance) + models.UserInfo.objects.create(**data.dict(exclude={"user"}), owner=instance) return instance +@sync_to_async +def send_user_signed_up_async(user: User, request): + user_signed_up.send(sender=user.__class__, request=request, user=user) + + @authentication_router.post("/signup/") async def signup(data: SignupIn): - pass + user = await signup_save(data) + # XXX-TOM + data = await login_response_data(user) + await send_user_signed_up_async(user, None) + return MsgpackResponse(content=data, status_code=status.HTTP_201_CREATED) diff --git a/etebase_fastapi/execptions.py b/etebase_fastapi/execptions.py index 2b35634..fa76c45 100644 --- a/etebase_fastapi/execptions.py +++ b/etebase_fastapi/execptions.py @@ -1,8 +1,23 @@ from fastapi import status +import typing as t + +from pydantic import BaseModel from django_etebase.exceptions import EtebaseValidationError +class ValidationErrorField(BaseModel): + field: str + code: str + detail: str + + +class ValidationErrorOut(BaseModel): + code: str + detail: str + errors: t.Optional[t.List[ValidationErrorField]] + + class CustomHttpException(Exception): def __init__(self, code: str, detail: str, status_code: int = status.HTTP_400_BAD_REQUEST): self.status_code = status_code @@ -44,12 +59,27 @@ class PermissionDenied(CustomHttpException): super().__init__(code=code, detail=detail, status_code=status_code) +from django_etebase.exceptions import EtebaseValidationError + + class ValidationError(CustomHttpException): - def __init__(self, code: str, detail: str, status_code: int = status.HTTP_400_BAD_REQUEST): + def __init__( + self, + code: str, + detail: str, + status_code: int = status.HTTP_400_BAD_REQUEST, + field: t.Optional[str] = None, + errors: t.Optional[t.List["ValidationError"]] = None, + ): + self.errors = errors super().__init__(code=code, detail=detail, status_code=status_code) + @property + def as_dict(self) -> dict: + return ValidationErrorOut(code=self.code, errors=self.errors, detail=self.detail).dict() + -def flatten_errors(field_name, errors): +def flatten_errors(field_name, errors) -> t.List[ValidationError]: ret = [] if isinstance(errors, dict): for error_key in errors: @@ -61,13 +91,7 @@ def flatten_errors(field_name, errors): message = error.messages[0] else: message = str(error) - ret.append( - { - "field": field_name, - "code": error.code, - "detail": message, - } - ) + ret.append(dict(code=error.code, detail=message, field=field_name)) return ret @@ -78,11 +102,4 @@ def transform_validation_error(prefix, err): errors = flatten_errors(prefix, err.error_list) else: raise EtebaseValidationError(err.code, err.message) - raise ValidationError(code="field_errors", detail="Field validations failed.") - raise serializers.ValidationError( - { - "code": "field_errors", - "detail": "Field validations failed.", - "errors": errors, - } - ) + raise ValidationError(code="field_errors", detail="Field validations failed.", errors=errors) From 2e5dd586574600810dae67501372d254ec917722 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 14:06:35 +0200 Subject: [PATCH 008/102] snapshot --- etebase_fastapi/app.py | 6 +++++ etebase_fastapi/authentication.py | 4 +-- etebase_fastapi/test_reset_view.py | 39 ++++++++++++++++++++++++++++++ 3 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 etebase_fastapi/test_reset_view.py diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index acfb42f..449059a 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -3,6 +3,8 @@ import os from django.core.wsgi import get_wsgi_application from fastapi.middleware.cors import CORSMiddleware +from django.conf import settings + os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etebase_server.settings") application = get_wsgi_application() from fastapi import FastAPI, Request @@ -17,6 +19,10 @@ VERSION = "v1" BASE_PATH = f"/api/{VERSION}" app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication") app.include_router(collection_router, prefix=f"{BASE_PATH}/collection") +if settings.DEBUG: + from .test_reset_view import test_reset_view_router + + app.include_router(test_reset_view_router, prefix=f"{BASE_PATH}/test/authentication") app.add_middleware( CORSMiddleware, allow_origin_regex="https?://.*", allow_credentials=True, allow_methods=["*"], allow_headers=["*"] ) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 287b46e..9c770d4 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -217,7 +217,6 @@ def validate_login_request( detail = 'Found wrong host name. Got: "{}" expected: "{}"'.format(validated_data.host, host_from_request) content = {"code": "wrong_host", "detail": detail} return MsgpackResponse(content, status_code=status.HTTP_400_BAD_REQUEST) - verify_key = nacl.signing.VerifyKey(bytes(user.userinfo.loginPubkey), encoder=nacl.encoding.RawEncoder) try: @@ -272,7 +271,6 @@ async def change_password(data: ChangePassword, request: Request, user: User = D return Response(status_code=status.HTTP_204_NO_CONTENT) -@sync_to_async def signup_save(data: SignupIn) -> User: user_data = data.user with transaction.atomic(): @@ -309,7 +307,7 @@ def send_user_signed_up_async(user: User, request): @authentication_router.post("/signup/") async def signup(data: SignupIn): - user = await signup_save(data) + user = await sync_to_async(signup_save)(data) # XXX-TOM data = await login_response_data(user) await send_user_signed_up_async(user, None) diff --git a/etebase_fastapi/test_reset_view.py b/etebase_fastapi/test_reset_view.py new file mode 100644 index 0000000..ee6a1c3 --- /dev/null +++ b/etebase_fastapi/test_reset_view.py @@ -0,0 +1,39 @@ +from django.conf import settings +from django.contrib.auth import get_user_model +from django.db import transaction +from django.shortcuts import get_object_or_404 +from fastapi import APIRouter, Response, status, Depends + +from django_etebase.utils import get_user_queryset +from etebase_fastapi.authentication import get_authenticated_user, SignupIn, signup_save +from etebase_fastapi.msgpack import MsgpackRoute + +test_reset_view_router = APIRouter(route_class=MsgpackRoute) +User = get_user_model() + + +@test_reset_view_router.post("/reset/") +def reset(data: SignupIn): + # Only run when in DEBUG mode! It's only used for tests + if not settings.DEBUG: + return Response("Only allowed in debug mode.", status_code=status.HTTP_400_BAD_REQUEST) + + with transaction.atomic(): + # XXX-TOM + user_queryset = get_user_queryset(User.objects.all(), None) + user = get_object_or_404(user_queryset, username=data.user.username) + # Only allow test users for extra safety + if not getattr(user, User.USERNAME_FIELD).startswith("test_user"): + return Response("Endpoint not allowed for user.", status_code=status.HTTP_400_BAD_REQUEST) + + if hasattr(user, "userinfo"): + user.userinfo.delete() + signup_save(data) + # Delete all of the journal data for this user for a clear test env + user.collection_set.all().delete() + user.collectionmember_set.all().delete() + user.incoming_invitations.all().delete() + + # FIXME: also delete chunk files!!! + + return Response(status_code=status.HTTP_204_NO_CONTENT) From a0d1d23d2d6a1ba11a84274bc96667f3d21625cf Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 17:22:14 +0200 Subject: [PATCH 009/102] imports --- etebase_fastapi/test_reset_view.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/test_reset_view.py b/etebase_fastapi/test_reset_view.py index ee6a1c3..ea7d8d9 100644 --- a/etebase_fastapi/test_reset_view.py +++ b/etebase_fastapi/test_reset_view.py @@ -2,10 +2,10 @@ from django.conf import settings from django.contrib.auth import get_user_model from django.db import transaction from django.shortcuts import get_object_or_404 -from fastapi import APIRouter, Response, status, Depends +from fastapi import APIRouter, Response, status from django_etebase.utils import get_user_queryset -from etebase_fastapi.authentication import get_authenticated_user, SignupIn, signup_save +from etebase_fastapi.authentication import SignupIn, signup_save from etebase_fastapi.msgpack import MsgpackRoute test_reset_view_router = APIRouter(route_class=MsgpackRoute) From 31e0e0b8323060e8265538ad96666d63724ae1f4 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 17:23:44 +0200 Subject: [PATCH 010/102] change response content to pydantic models and error handling --- etebase_fastapi/authentication.py | 115 +++++++++++++----------------- 1 file changed, 51 insertions(+), 64 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 9c770d4..9ecee4c 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -2,7 +2,6 @@ import dataclasses import typing as t from datetime import datetime from functools import cached_property -from django.core import exceptions as django_exceptions import nacl import nacl.encoding @@ -12,6 +11,7 @@ import nacl.signing from asgiref.sync import sync_to_async from django.conf import settings from django.contrib.auth import get_user_model, user_logged_out, user_logged_in +from django.core import exceptions as django_exceptions from django.db import transaction from django.utils import timezone from fastapi import APIRouter, Depends, status, Request, Response @@ -21,7 +21,6 @@ from pydantic import BaseModel from django_etebase import app_settings, models from django_etebase.exceptions import EtebaseValidationError from django_etebase.models import UserInfo -from django_etebase.serializers import UserSerializer from django_etebase.signals import user_signed_up from django_etebase.token_auth.models import AuthToken from django_etebase.token_auth.models import get_default_expiry @@ -43,10 +42,16 @@ class AuthData: token: AuthToken -class LoginChallengeData(BaseModel): +class LoginChallengeIn(BaseModel): username: str +class LoginChallengeOut(BaseModel): + salt: bytes + challenge: bytes + version: int + + class LoginResponse(BaseModel): username: str challenge: bytes @@ -54,6 +59,26 @@ class LoginResponse(BaseModel): action: t.Literal["login", "changePassword"] +class UserOut(BaseModel): + pubkey: bytes + encryptedContent: bytes + + @classmethod + def from_orm(cls: t.Type["UserOut"], obj: User) -> "UserOut": + return cls(pubkey=obj.userinfo.pubkey, encryptedContent=obj.userinfo.encryptedContent) + + +class LoginOut(BaseModel): + token: str + user: UserOut + + @classmethod + def from_orm(cls: t.Type["LoginOut"], obj: User) -> "LoginOut": + token = AuthToken.objects.create(user=obj).key + user = UserOut.from_orm(obj) + return cls(token=token, user=user) + + class Authentication(BaseModel): class Config: keep_untouched = (cached_property,) @@ -145,7 +170,7 @@ def __get_login_user(username: str) -> User: raise AuthenticationFailed(code="user_not_found", detail="User not found") -async def get_login_user(challenge: LoginChallengeData) -> User: +async def get_login_user(challenge: LoginChallengeIn) -> User: user = await __get_login_user(challenge.username) return user @@ -161,7 +186,6 @@ def get_encryption_key(salt): ) -@sync_to_async def save_changed_password(data: ChangePassword, user: User): response_data = data.response_data user_info: UserInfo = user.userinfo @@ -170,24 +194,6 @@ def save_changed_password(data: ChangePassword, user: User): user_info.save() -@sync_to_async -def login_response_data(user: User): - return { - "token": AuthToken.objects.create(user=user).key, - "user": UserSerializer(user).data, - } - - -@sync_to_async -def send_user_logged_in_async(user: User, request: Request): - user_logged_in.send(sender=user.__class__, request=request, user=user) - - -@sync_to_async -def send_user_logged_out_async(user: User, request: Request): - user_logged_out.send(sender=user.__class__, request=request, user=user) - - @sync_to_async def validate_login_request( validated_data: LoginResponse, @@ -195,39 +201,26 @@ def validate_login_request( user: User, expected_action: str, host_from_request: str, -) -> t.Optional[MsgpackResponse]: - +): enc_key = get_encryption_key(bytes(user.userinfo.salt)) box = nacl.secret.SecretBox(enc_key) challenge_data = msgpack_decode(box.decrypt(validated_data.challenge)) now = int(datetime.now().timestamp()) if validated_data.action != expected_action: - content = { - "code": "wrong_action", - "detail": 'Expected "{}" but got something else'.format(challenge_sent_to_user.response), - } - return MsgpackResponse(content, status_code=status.HTTP_400_BAD_REQUEST) + raise ValidationError("wrong_action", f'Expected "{challenge_sent_to_user.response}" but got something else') elif now - challenge_data["timestamp"] > app_settings.CHALLENGE_VALID_SECONDS: - content = {"code": "challenge_expired", "detail": "Login challenge has expired"} - return MsgpackResponse(content, status_code=status.HTTP_400_BAD_REQUEST) + raise ValidationError("challenge_expired", "Login challenge has expired") elif challenge_data["userId"] != user.id: - content = {"code": "wrong_user", "detail": "This challenge is for the wrong user"} - return MsgpackResponse(content, status_code=status.HTTP_400_BAD_REQUEST) + raise ValidationError("wrong_user", "This challenge is for the wrong user") elif not settings.DEBUG and validated_data.host.split(":", 1)[0] != host_from_request: - detail = 'Found wrong host name. Got: "{}" expected: "{}"'.format(validated_data.host, host_from_request) - content = {"code": "wrong_host", "detail": detail} - return MsgpackResponse(content, status_code=status.HTTP_400_BAD_REQUEST) + raise ValidationError( + "wrong_host", f'Found wrong host name. Got: "{validated_data.host}" expected: "{host_from_request}"' + ) verify_key = nacl.signing.VerifyKey(bytes(user.userinfo.loginPubkey), encoder=nacl.encoding.RawEncoder) - try: verify_key.verify(challenge_sent_to_user.response, challenge_sent_to_user.signature) except nacl.exceptions.BadSignatureError: - return MsgpackResponse( - {"code": "login_bad_signature", "detail": "Wrong password for user."}, - status_code=status.HTTP_401_UNAUTHORIZED, - ) - - return None + raise ValidationError("login_bad_signature", "Wrong password for user.", status.HTTP_401_UNAUTHORIZED) @authentication_router.post("/login_challenge/") @@ -239,35 +232,34 @@ async def login_challenge(user: User = Depends(get_login_user)): "userId": user.id, } challenge = bytes(box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder)) - return MsgpackResponse({"salt": user.userinfo.salt, "version": user.userinfo.version, "challenge": challenge}) + return MsgpackResponse( + LoginChallengeOut(salt=user.userinfo.salt, challenge=challenge, version=user.userinfo.version) + ) @authentication_router.post("/login/") async def login(data: Login, request: Request): - user = await get_login_user(LoginChallengeData(username=data.response_data.username)) + user = await get_login_user(LoginChallengeIn(username=data.response_data.username)) host = request.headers.get("Host") - bad_login_response = await validate_login_request(data.response_data, data, user, "login", host) - if bad_login_response is not None: - return bad_login_response - data = await login_response_data(user) - await send_user_logged_in_async(user, request) - return MsgpackResponse(data, status_code=status.HTTP_200_OK) + await validate_login_request(data.response_data, data, user, "login", host) + data = await sync_to_async(LoginOut.from_orm)(user) + await sync_to_async(user_logged_in.send)(sender=user.__class__, request=None, user=user) + return MsgpackResponse(content=data, status_code=status.HTTP_200_OK) @authentication_router.post("/logout/") async def logout(request: Request, auth_data: AuthData = Depends(get_auth_data)): await sync_to_async(auth_data.token.delete)() - await send_user_logged_out_async(auth_data.user, request) + # XXX-TOM + await sync_to_async(user_logged_out.send)(sender=auth_data.user.__class__, request=None, user=auth_data.user) return Response(status_code=status.HTTP_204_NO_CONTENT) @authentication_router.post("/change_password/") async def change_password(data: ChangePassword, request: Request, user: User = Depends(get_authenticated_user)): host = request.headers.get("Host") - bad_login_response = await validate_login_request(data.response_data, data, user, "changePassword", host) - if bad_login_response is not None: - return bad_login_response - await save_changed_password(data, user) + await validate_login_request(data.response_data, data, user, "changePassword", host) + await sync_to_async(save_changed_password)(data, user) return Response(status_code=status.HTTP_204_NO_CONTENT) @@ -300,15 +292,10 @@ def signup_save(data: SignupIn) -> User: return instance -@sync_to_async -def send_user_signed_up_async(user: User, request): - user_signed_up.send(sender=user.__class__, request=request, user=user) - - @authentication_router.post("/signup/") async def signup(data: SignupIn): user = await sync_to_async(signup_save)(data) # XXX-TOM - data = await login_response_data(user) - await send_user_signed_up_async(user, None) + data = await sync_to_async(LoginOut.from_orm)(user) + await sync_to_async(user_signed_up.send)(sender=user.__class__, request=None, user=user) return MsgpackResponse(content=data, status_code=status.HTTP_201_CREATED) From 4bd826b3bed6fbcbd2b5029094c38e548b38583d Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 19:08:22 +0200 Subject: [PATCH 011/102] remove uvicorn run --- etebase_fastapi/app.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index 449059a..fac2a31 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -31,9 +31,3 @@ app.add_middleware( @app.exception_handler(CustomHttpException) async def custom_exception_handler(request: Request, exc: CustomHttpException): return MsgpackResponse(status_code=exc.status_code, content=exc.as_dict) - - -if __name__ == "__main__": - import uvicorn - - uvicorn.run(app, host="0.0.0.0", port=8000) From be7b830804bbb693ce9abd4e0c095f29ad10a4b9 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Fri, 25 Dec 2020 19:23:46 +0200 Subject: [PATCH 012/102] collection.py: create --- etebase_fastapi/collection.py | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 7c9aca8..ba462c9 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -1,16 +1,16 @@ import typing as t +from asgiref.sync import sync_to_async from django.contrib.auth import get_user_model from django.db.models import Q from django.db.models import QuerySet -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, status from pydantic import BaseModel -from asgiref.sync import sync_to_async -from django_etebase.models import Collection, Stoken, AccessLevels, CollectionMember +from django_etebase.models import Collection, AccessLevels, CollectionMember from .authentication import get_authenticated_user from .msgpack import MsgpackRoute, MsgpackResponse -from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_queryset_stoken +from .stoken_handler import filter_by_stoken_and_limit User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute) @@ -75,3 +75,29 @@ async def list_multi( ) response = await list_common(queryset, user, stoken, limit) return response + + +class CollectionItemContent(BaseModel): + uid: str + meta: bytes + deleted: bool + chunks: t.List[t.List[t.Union[str, bytes]]] + + +class Item(BaseModel): + uid: str + version: int + etag: t.Optional[str] + content: CollectionItemContent + + +class CollectionItemIn(BaseModel): + collectionType: bytes + collectionKey: bytes + item: Item + + +@collection_router.post("/") +def create(data: CollectionItemIn): + # FIXME save actual item + return MsgpackResponse({}, status_code=status.HTTP_201_CREATED) From daac0c163b8b6943d89bd010b68d1c2fdae3ca9d Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Sat, 26 Dec 2020 12:39:20 +0200 Subject: [PATCH 013/102] collection.py: save to db --- etebase_fastapi/collection.py | 75 ++++++++++++++++++++++++++++++++++- 1 file changed, 73 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index ba462c9..1d876e1 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -2,13 +2,18 @@ import typing as t from asgiref.sync import sync_to_async from django.contrib.auth import get_user_model +from django.core import exceptions as django_exceptions +from django.core.files.base import ContentFile +from django.db import transaction from django.db.models import Q from django.db.models import QuerySet from fastapi import APIRouter, Depends, status from pydantic import BaseModel +from django_etebase import models from django_etebase.models import Collection, AccessLevels, CollectionMember from .authentication import get_authenticated_user +from .execptions import ValidationError from .msgpack import MsgpackRoute, MsgpackResponse from .stoken_handler import filter_by_stoken_and_limit @@ -88,6 +93,7 @@ class Item(BaseModel): uid: str version: int etag: t.Optional[str] + encryptionKey: t.Optional[bytes] content: CollectionItemContent @@ -97,7 +103,72 @@ class CollectionItemIn(BaseModel): item: Item +def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemContent): + chunks_objs = [] + + revision = models.CollectionItemRevision(**revision_data.dict(exclude={"chunks"}), item=item) + revision.validate_unique() # Verify there aren't any validation issues + + for chunk in revision_data.chunks: + uid = chunk[0] + chunk_obj = models.CollectionItemChunk.objects.filter(uid=uid).first() + content = chunk[1] if len(chunk) > 1 else None + # If the chunk already exists we assume it's fine. Otherwise, we upload it. + if chunk_obj is None: + if content is not None: + chunk_obj = models.CollectionItemChunk(uid=uid, collection=item.collection) + chunk_obj.chunkFile.save("IGNORED", ContentFile(content)) + chunk_obj.save() + else: + raise ValidationError("chunk_no_content", "Tried to create a new chunk without content") + + chunks_objs.append(chunk_obj) + + stoken = models.Stoken.objects.create() + revision.stoken = stoken + revision.save() + + for chunk in chunks_objs: + models.RevisionChunkRelation.objects.create(chunk=chunk, revision=revision) + return revision + + +def _create(data: CollectionItemIn, user: User): + with transaction.atomic(): + if data.item.etag is not None: + raise ValidationError("bad_etag", "etag is not null") + instance = models.Collection(uid=data.item.uid, owner=user) + try: + instance.validate_unique() + except django_exceptions.ValidationError: + raise ValidationError( + "unique_uid", "Collection with this uid already exists", status_code=status.HTTP_409_CONFLICT + ) + instance.save() + + main_item = models.CollectionItem.objects.create( + uid=data.item.uid, version=data.item.version, encryptionKey=data.item.encryptionKey, collection=instance + ) + + instance.main_item = main_item + instance.save() + + # TODO + process_revisions_for_item(main_item, data.item.content) + + collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=data.collectionType, owner=user) + + models.CollectionMember( + collection=instance, + stoken=models.Stoken.objects.create(), + user=user, + accessLevel=models.AccessLevels.ADMIN, + encryptionKey=data.collectionKey, + collectionType=collection_type_obj, + ).save() + + @collection_router.post("/") -def create(data: CollectionItemIn): - # FIXME save actual item +async def create(data: CollectionItemIn, user: User = Depends(get_authenticated_user)): + await sync_to_async(_create)(data, user) return MsgpackResponse({}, status_code=status.HTTP_201_CREATED) From 8d09e40b3b63e5917a345262e4a28cd728d38a05 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Sat, 26 Dec 2020 18:01:55 +0200 Subject: [PATCH 014/102] rename --- etebase_fastapi/collection.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 1d876e1..b4246ca 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -97,7 +97,7 @@ class Item(BaseModel): content: CollectionItemContent -class CollectionItemIn(BaseModel): +class CollectionIn(BaseModel): collectionType: bytes collectionKey: bytes item: Item @@ -133,7 +133,7 @@ def process_revisions_for_item(item: models.CollectionItem, revision_data: Colle return revision -def _create(data: CollectionItemIn, user: User): +def _create(data: CollectionIn, user: User): with transaction.atomic(): if data.item.etag is not None: raise ValidationError("bad_etag", "etag is not null") @@ -169,6 +169,6 @@ def _create(data: CollectionItemIn, user: User): @collection_router.post("/") -async def create(data: CollectionItemIn, user: User = Depends(get_authenticated_user)): +async def create(data: CollectionIn, user: User = Depends(get_authenticated_user)): await sync_to_async(_create)(data, user) return MsgpackResponse({}, status_code=status.HTTP_201_CREATED) From 1e60938430884fb8a8cf9d0d1710e92894d81bb4 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Sat, 26 Dec 2020 18:02:29 +0200 Subject: [PATCH 015/102] rename --- etebase_fastapi/collection.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index b4246ca..9e64072 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -82,7 +82,7 @@ async def list_multi( return response -class CollectionItemContent(BaseModel): +class CollectionItemRevision(BaseModel): uid: str meta: bytes deleted: bool @@ -94,7 +94,7 @@ class Item(BaseModel): version: int etag: t.Optional[str] encryptionKey: t.Optional[bytes] - content: CollectionItemContent + content: CollectionItemRevision class CollectionIn(BaseModel): @@ -103,7 +103,7 @@ class CollectionIn(BaseModel): item: Item -def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemContent): +def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevision): chunks_objs = [] revision = models.CollectionItemRevision(**revision_data.dict(exclude={"chunks"}), item=item) From 94161943ca019d1e887ce8665e1c0e82df941eee Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Sat, 26 Dec 2020 18:09:46 +0200 Subject: [PATCH 016/102] chunks type hint --- etebase_fastapi/collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 9e64072..77abe0f 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -86,7 +86,7 @@ class CollectionItemRevision(BaseModel): uid: str meta: bytes deleted: bool - chunks: t.List[t.List[t.Union[str, bytes]]] + chunks: t.List[t.Tuple[str, t.Optional[bytes]]] class Item(BaseModel): From 53662451a34e413457beb806564bcd98ac33e743 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Sun, 27 Dec 2020 11:36:18 +0200 Subject: [PATCH 017/102] collection.py: get by uid and fixed create --- etebase_fastapi/collection.py | 115 ++++++++++++++++++++++++---------- 1 file changed, 83 insertions(+), 32 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 77abe0f..0ceb988 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -7,8 +7,8 @@ from django.core.files.base import ContentFile from django.db import transaction from django.db.models import Q from django.db.models import QuerySet -from fastapi import APIRouter, Depends, status -from pydantic import BaseModel +from fastapi import APIRouter, Depends, status, Query +from pydantic import BaseModel, Field from django_etebase import models from django_etebase.models import Collection, AccessLevels, CollectionMember @@ -22,12 +22,57 @@ collection_router = APIRouter(route_class=MsgpackRoute) default_queryset = Collection.objects.all() +Prefetch = t.Literal["auto", "medium"] +PrefetchQuery = Query(default="auto") + + class ListMulti(BaseModel): collectionTypes: t.List[bytes] +class CollectionItemRevisionOut(BaseModel): + uid: str + meta: bytes + deleted: bool + chunks: t.List[t.Tuple[str, t.Optional[bytes]]] + + class Config: + orm_mode = True + + @classmethod + def from_orm_user( + cls: t.Type["CollectionItemRevisionOut"], obj: models.CollectionItemRevision, prefetch: Prefetch + ) -> "CollectionItemRevisionOut": + chunk_obj = obj.chunks_relation.get().chunk + if prefetch == "auto": + with open(chunk_obj.chunkFile.path, "rb") as f: + chunks = chunk_obj.uid, f.read() + else: + chunks = (chunk_obj.uid,) + return cls(uid=obj.uid, meta=obj.meta, deleted=obj.deleted, chunks=[chunks]) + + class CollectionItemOut(BaseModel): uid: str + version: int + encryptionKey: t.Optional[bytes] + etag: t.Optional[str] + content: CollectionItemRevisionOut + + class Config: + orm_mode = True + + @classmethod + def from_orm_user( + cls: t.Type["CollectionItemOut"], obj: models.CollectionItem, prefetch: Prefetch + ) -> "CollectionItemOut": + return cls( + uid=obj.uid, + version=obj.version, + encryptionKey=obj.encryptionKey, + etag=obj.etag, + content=CollectionItemRevisionOut.from_orm_user(obj.content, prefetch), + ) class CollectionOut(BaseModel): @@ -38,16 +83,17 @@ class CollectionOut(BaseModel): item: CollectionItemOut @classmethod - def from_orm_user(cls: t.Type["CollectionOut"], obj: Collection, user: User) -> "CollectionOut": + def from_orm_user(cls: t.Type["CollectionOut"], obj: Collection, user: User, prefetch: Prefetch) -> "CollectionOut": member: CollectionMember = obj.members.get(user=user) collection_type = member.collectionType - return cls( + ret = cls( collectionType=collection_type and collection_type.uid, collectionKey=member.encryptionKey, accessLevel=member.accessLevel, stoken=obj.stoken, - item=CollectionItemOut(uid=obj.main_item.uid), + item=CollectionItemOut.from_orm_user(obj.main_item, prefetch), ) + return ret class ListResponse(BaseModel): @@ -56,11 +102,26 @@ class ListResponse(BaseModel): done: bool +class ItemIn(BaseModel): + uid: str + version: int + etag: t.Optional[str] + content: CollectionItemRevisionOut + + +class CollectionIn(BaseModel): + collectionType: bytes + collectionKey: bytes + item: ItemIn + + @sync_to_async -def list_common(queryset: QuerySet, user: User, stoken: t.Optional[str], limit: int) -> MsgpackResponse: +def list_common( + queryset: QuerySet, user: User, stoken: t.Optional[str], limit: int, prefetch: Prefetch +) -> MsgpackResponse: result, new_stoken_obj, done = filter_by_stoken_and_limit(stoken, limit, queryset, Collection.stoken_annotation) new_stoken = new_stoken_obj and new_stoken_obj.uid - data: t.List[CollectionOut] = [CollectionOut.from_orm_user(item, user) for item in queryset] + data: t.List[CollectionOut] = [CollectionOut.from_orm_user(item, user, prefetch) for item in queryset] ret = ListResponse(data=data, stoken=new_stoken, done=done) return MsgpackResponse(content=ret) @@ -71,39 +132,22 @@ def get_collection_queryset(user: User, queryset: QuerySet) -> QuerySet: @collection_router.post("/list_multi/") async def list_multi( - data: ListMulti, stoken: t.Optional[str] = None, limit: int = 50, user: User = Depends(get_authenticated_user) + data: ListMulti, + stoken: t.Optional[str] = None, + limit: int = 50, + user: User = Depends(get_authenticated_user), + prefetch: Prefetch = PrefetchQuery, ): queryset = get_collection_queryset(user, default_queryset) # FIXME: Remove the isnull part once we attach collection types to all objects ("collection-type-migration") queryset = queryset.filter( Q(members__collectionType__uid__in=data.collectionTypes) | Q(members__collectionType__isnull=True) ) - response = await list_common(queryset, user, stoken, limit) + response = await list_common(queryset, user, stoken, limit, prefetch) return response -class CollectionItemRevision(BaseModel): - uid: str - meta: bytes - deleted: bool - chunks: t.List[t.Tuple[str, t.Optional[bytes]]] - - -class Item(BaseModel): - uid: str - version: int - etag: t.Optional[str] - encryptionKey: t.Optional[bytes] - content: CollectionItemRevision - - -class CollectionIn(BaseModel): - collectionType: bytes - collectionKey: bytes - item: Item - - -def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevision): +def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevisionOut): chunks_objs = [] revision = models.CollectionItemRevision(**revision_data.dict(exclude={"chunks"}), item=item) @@ -147,7 +191,7 @@ def _create(data: CollectionIn, user: User): instance.save() main_item = models.CollectionItem.objects.create( - uid=data.item.uid, version=data.item.version, encryptionKey=data.item.encryptionKey, collection=instance + uid=data.item.uid, version=data.item.version, collection=instance ) instance.main_item = main_item @@ -172,3 +216,10 @@ def _create(data: CollectionIn, user: User): async def create(data: CollectionIn, user: User = Depends(get_authenticated_user)): await sync_to_async(_create)(data, user) return MsgpackResponse({}, status_code=status.HTTP_201_CREATED) + + +@collection_router.get("/{uid}/") +def get_collection(uid: str, user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery): + obj = get_collection_queryset(user, default_queryset).get(uid=uid) + ret = CollectionOut.from_orm_user(obj, user, prefetch) + return MsgpackResponse(ret) From b3c177faa63c47e99411cd56802e75ee84304bad Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Sun, 27 Dec 2020 14:23:19 +0200 Subject: [PATCH 018/102] from_orm_context --- etebase_fastapi/collection.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 0ceb988..2b0b876 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -1,3 +1,4 @@ +import dataclasses import typing as t from asgiref.sync import sync_to_async @@ -26,6 +27,12 @@ Prefetch = t.Literal["auto", "medium"] PrefetchQuery = Query(default="auto") +@dataclasses.dataclass +class Context: + user: t.Optional[User] + prefetch: t.Optional[Prefetch] + + class ListMulti(BaseModel): collectionTypes: t.List[bytes] @@ -40,11 +47,11 @@ class CollectionItemRevisionOut(BaseModel): orm_mode = True @classmethod - def from_orm_user( - cls: t.Type["CollectionItemRevisionOut"], obj: models.CollectionItemRevision, prefetch: Prefetch + def from_orm_context( + cls: t.Type["CollectionItemRevisionOut"], obj: models.CollectionItemRevision, context: Context ) -> "CollectionItemRevisionOut": chunk_obj = obj.chunks_relation.get().chunk - if prefetch == "auto": + if context.prefetch == "auto": with open(chunk_obj.chunkFile.path, "rb") as f: chunks = chunk_obj.uid, f.read() else: @@ -63,15 +70,15 @@ class CollectionItemOut(BaseModel): orm_mode = True @classmethod - def from_orm_user( - cls: t.Type["CollectionItemOut"], obj: models.CollectionItem, prefetch: Prefetch + def from_orm_context( + cls: t.Type["CollectionItemOut"], obj: models.CollectionItem, context: Context ) -> "CollectionItemOut": return cls( uid=obj.uid, version=obj.version, encryptionKey=obj.encryptionKey, etag=obj.etag, - content=CollectionItemRevisionOut.from_orm_user(obj.content, prefetch), + content=CollectionItemRevisionOut.from_orm_context(obj.content, context), ) @@ -83,15 +90,15 @@ class CollectionOut(BaseModel): item: CollectionItemOut @classmethod - def from_orm_user(cls: t.Type["CollectionOut"], obj: Collection, user: User, prefetch: Prefetch) -> "CollectionOut": - member: CollectionMember = obj.members.get(user=user) + def from_orm_context(cls: t.Type["CollectionOut"], obj: Collection, context: Context) -> "CollectionOut": + member: CollectionMember = obj.members.get(user=context.user) collection_type = member.collectionType ret = cls( collectionType=collection_type and collection_type.uid, collectionKey=member.encryptionKey, accessLevel=member.accessLevel, stoken=obj.stoken, - item=CollectionItemOut.from_orm_user(obj.main_item, prefetch), + item=CollectionItemOut.from_orm_context(obj.main_item, context), ) return ret @@ -121,7 +128,8 @@ def list_common( ) -> MsgpackResponse: result, new_stoken_obj, done = filter_by_stoken_and_limit(stoken, limit, queryset, Collection.stoken_annotation) new_stoken = new_stoken_obj and new_stoken_obj.uid - data: t.List[CollectionOut] = [CollectionOut.from_orm_user(item, user, prefetch) for item in queryset] + context = Context(user, prefetch) + data: t.List[CollectionOut] = [CollectionOut.from_orm_context(item, context) for item in queryset] ret = ListResponse(data=data, stoken=new_stoken, done=done) return MsgpackResponse(content=ret) @@ -221,5 +229,5 @@ async def create(data: CollectionIn, user: User = Depends(get_authenticated_user @collection_router.get("/{uid}/") def get_collection(uid: str, user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery): obj = get_collection_queryset(user, default_queryset).get(uid=uid) - ret = CollectionOut.from_orm_user(obj, user, prefetch) + ret = CollectionOut.from_orm_context(obj, Context(user, prefetch)) return MsgpackResponse(ret) From 6e4f8f9917f07b4727adb7e9eb64634ac21b586a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 14:36:16 +0200 Subject: [PATCH 019/102] Fix list_multi to return the filtered queryset. --- etebase_fastapi/collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 2b0b876..ff31e27 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -129,7 +129,7 @@ def list_common( result, new_stoken_obj, done = filter_by_stoken_and_limit(stoken, limit, queryset, Collection.stoken_annotation) new_stoken = new_stoken_obj and new_stoken_obj.uid context = Context(user, prefetch) - data: t.List[CollectionOut] = [CollectionOut.from_orm_context(item, context) for item in queryset] + data: t.List[CollectionOut] = [CollectionOut.from_orm_context(item, context) for item in result] ret = ListResponse(data=data, stoken=new_stoken, done=done) return MsgpackResponse(content=ret) From a0aaf79f42450343dee7c095a9269dabd9a1d39d Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Sun, 27 Dec 2020 15:53:31 +0200 Subject: [PATCH 020/102] item_batch and item_transaction boilerplate only --- etebase_fastapi/collection.py | 52 +++++++++++++++++++++++++++++++++-- 1 file changed, 50 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index ff31e27..093ea8a 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -8,7 +8,7 @@ from django.core.files.base import ContentFile from django.db import transaction from django.db.models import Q from django.db.models import QuerySet -from fastapi import APIRouter, Depends, status, Query +from fastapi import APIRouter, Depends, status, Query, Request from pydantic import BaseModel, Field from django_etebase import models @@ -20,7 +20,7 @@ from .stoken_handler import filter_by_stoken_and_limit User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute) -default_queryset = Collection.objects.all() +default_queryset: QuerySet = Collection.objects.all() Prefetch = t.Literal["auto", "medium"] @@ -122,6 +122,19 @@ class CollectionIn(BaseModel): item: ItemIn +class ItemDepIn(BaseModel): + etag: str + uid: str + + class Config: + orm_mode = True + + +class ItemBatchIn(BaseModel): + items: t.List[ItemIn] + deps: t.Optional[ItemDepIn] + + @sync_to_async def list_common( queryset: QuerySet, user: User, stoken: t.Optional[str], limit: int, prefetch: Prefetch @@ -155,6 +168,14 @@ async def list_multi( return response +@collection_router.post("/list/") +async def collection_list( + req: Request, + user: User = Depends(get_authenticated_user), +): + pass + + def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevisionOut): chunks_objs = [] @@ -231,3 +252,30 @@ def get_collection(uid: str, user: User = Depends(get_authenticated_user), prefe obj = get_collection_queryset(user, default_queryset).get(uid=uid) ret = CollectionOut.from_orm_context(obj, Context(user, prefetch)) return MsgpackResponse(ret) + + +def item_bulk_common(data: ItemBatchIn, user: User, stoken: str, uid: str, validate_etag: bool): + queryset = get_collection_queryset(user, default_queryset) + with transaction.atomic(): # We need this for locking the collection object + collection_object = queryset.select_for_update().get(uid=uid) + if stoken is not None and stoken != collection_object.stoken: + raise ValidationError("stale_stoken", "Stoken is too old", status_code=status.HTTP_409_CONFLICT) + + + +def item_create(): + pass # + + +@collection_router.post("/{uid}/item/transaction/") +def item_transaction( + uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) +): + item_bulk_common(data, user, stoken, uid, validate_etag=True) + + +@collection_router.post("/{uid}/item/batch/") +def item_batch( + uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) +): + item_bulk_common(data, user, stoken, uid, validate_etag=False) From 6f543751a6f511835764cf14a98b016f47cede96 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 15:25:16 +0200 Subject: [PATCH 021/102] Fix and improve typing. --- etebase_fastapi/stoken_handler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/stoken_handler.py b/etebase_fastapi/stoken_handler.py index c840f0e..9ea1500 100644 --- a/etebase_fastapi/stoken_handler.py +++ b/etebase_fastapi/stoken_handler.py @@ -10,7 +10,7 @@ from django_etebase.models import Stoken StokenAnnotation = t.Any -def get_stoken_obj(stoken: t.Optional[str]): +def get_stoken_obj(stoken: t.Optional[str]) -> t.Optional[Stoken]: if stoken is not None: try: return Stoken.objects.get(uid=stoken) @@ -22,7 +22,7 @@ def get_stoken_obj(stoken: t.Optional[str]): def filter_by_stoken( stoken: t.Optional[str], queryset: QuerySet, stoken_annotation: StokenAnnotation -) -> t.Tuple[QuerySet, t.Optional[str]]: +) -> t.Tuple[QuerySet, t.Optional[Stoken]]: stoken_rev = get_stoken_obj(stoken) queryset = queryset.annotate(max_stoken=stoken_annotation).order_by("max_stoken") From df855897f8a4d9fb1d1501d87b7ae3a585aa7256 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 15:26:36 +0200 Subject: [PATCH 022/102] Fix type error. --- etebase_fastapi/stoken_handler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_fastapi/stoken_handler.py b/etebase_fastapi/stoken_handler.py index 9ea1500..fb89651 100644 --- a/etebase_fastapi/stoken_handler.py +++ b/etebase_fastapi/stoken_handler.py @@ -38,7 +38,7 @@ def get_queryset_stoken(queryset: list) -> t.Optional[Stoken]: for row in queryset: rowmaxid = getattr(row, "max_stoken") or -1 maxid = max(maxid, rowmaxid) - new_stoken = (maxid >= 0) and Stoken.objects.get(id=maxid) + new_stoken = Stoken.objects.get(id=maxid) if (maxid >= 0) else None return new_stoken or None From 9d213350e74072aca53167de278692a039d7ad55 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 15:45:29 +0200 Subject: [PATCH 023/102] exceptions.py: fix typo in filename. --- etebase_fastapi/app.py | 2 +- etebase_fastapi/authentication.py | 2 +- etebase_fastapi/collection.py | 2 +- etebase_fastapi/{execptions.py => exceptions.py} | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename etebase_fastapi/{execptions.py => exceptions.py} (100%) diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index fac2a31..6fbdd04 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -9,7 +9,7 @@ os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etebase_server.settings") application = get_wsgi_application() from fastapi import FastAPI, Request -from .execptions import CustomHttpException +from .exceptions import CustomHttpException from .authentication import authentication_router from .collection import collection_router from .msgpack import MsgpackResponse diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 9ecee4c..51ba80d 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -26,7 +26,7 @@ from django_etebase.token_auth.models import AuthToken from django_etebase.token_auth.models import get_default_expiry from django_etebase.utils import create_user from django_etebase.views import msgpack_encode, msgpack_decode -from .execptions import AuthenticationFailed, transform_validation_error, ValidationError +from .exceptions import AuthenticationFailed, transform_validation_error, ValidationError from .msgpack import MsgpackResponse, MsgpackRoute User = get_user_model() diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 093ea8a..b4e9dae 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -14,7 +14,7 @@ from pydantic import BaseModel, Field from django_etebase import models from django_etebase.models import Collection, AccessLevels, CollectionMember from .authentication import get_authenticated_user -from .execptions import ValidationError +from .exceptions import ValidationError from .msgpack import MsgpackRoute, MsgpackResponse from .stoken_handler import filter_by_stoken_and_limit diff --git a/etebase_fastapi/execptions.py b/etebase_fastapi/exceptions.py similarity index 100% rename from etebase_fastapi/execptions.py rename to etebase_fastapi/exceptions.py From 249c3dc2be7dfe2015a19e5758bb9a60e58e6b9f Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 16:04:33 +0200 Subject: [PATCH 024/102] Cleanup models to have common parents. --- etebase_fastapi/collection.py | 49 ++++++++++++++++------------------- 1 file changed, 22 insertions(+), 27 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index b4e9dae..f687127 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -9,7 +9,7 @@ from django.db import transaction from django.db.models import Q from django.db.models import QuerySet from fastapi import APIRouter, Depends, status, Query, Request -from pydantic import BaseModel, Field +from pydantic import BaseModel from django_etebase import models from django_etebase.models import Collection, AccessLevels, CollectionMember @@ -37,7 +37,7 @@ class ListMulti(BaseModel): collectionTypes: t.List[bytes] -class CollectionItemRevisionOut(BaseModel): +class CollectionItemRevision(BaseModel): uid: str meta: bytes deleted: bool @@ -48,8 +48,8 @@ class CollectionItemRevisionOut(BaseModel): @classmethod def from_orm_context( - cls: t.Type["CollectionItemRevisionOut"], obj: models.CollectionItemRevision, context: Context - ) -> "CollectionItemRevisionOut": + cls: t.Type["CollectionItemRevision"], obj: models.CollectionItemRevision, context: Context + ) -> "CollectionItemRevision": chunk_obj = obj.chunks_relation.get().chunk if context.prefetch == "auto": with open(chunk_obj.chunkFile.path, "rb") as f: @@ -59,13 +59,14 @@ class CollectionItemRevisionOut(BaseModel): return cls(uid=obj.uid, meta=obj.meta, deleted=obj.deleted, chunks=[chunks]) -class CollectionItemOut(BaseModel): +class CollectionItemCommon(BaseModel): uid: str version: int encryptionKey: t.Optional[bytes] - etag: t.Optional[str] - content: CollectionItemRevisionOut + content: CollectionItemRevision + +class CollectionItemOut(CollectionItemCommon): class Config: orm_mode = True @@ -82,9 +83,16 @@ class CollectionItemOut(BaseModel): ) -class CollectionOut(BaseModel): - collectionKey: bytes +class CollectionItemIn(CollectionItemCommon): + etag: t.Optional[str] + + +class CollectionCommon(BaseModel): collectionType: bytes + collectionKey: bytes + + +class CollectionOut(CollectionCommon): accessLevel: AccessLevels stoken: str item: CollectionItemOut @@ -103,35 +111,23 @@ class CollectionOut(BaseModel): return ret +class CollectionIn(CollectionCommon): + item: CollectionItemIn + + class ListResponse(BaseModel): data: t.List[CollectionOut] stoken: t.Optional[str] done: bool -class ItemIn(BaseModel): - uid: str - version: int - etag: t.Optional[str] - content: CollectionItemRevisionOut - - -class CollectionIn(BaseModel): - collectionType: bytes - collectionKey: bytes - item: ItemIn - - class ItemDepIn(BaseModel): etag: str uid: str - class Config: - orm_mode = True - class ItemBatchIn(BaseModel): - items: t.List[ItemIn] + items: t.List[CollectionItemIn] deps: t.Optional[ItemDepIn] @@ -262,7 +258,6 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: str, uid: str, valid raise ValidationError("stale_stoken", "Stoken is too old", status_code=status.HTTP_409_CONFLICT) - def item_create(): pass # From b2fe30ac26f9d6380164baa72faa720c493de354 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 16:33:34 +0200 Subject: [PATCH 025/102] Implement item_create, batch and transaction. --- etebase_fastapi/collection.py | 71 ++++++++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 6 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index f687127..5656fb6 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -14,7 +14,7 @@ from pydantic import BaseModel from django_etebase import models from django_etebase.models import Collection, AccessLevels, CollectionMember from .authentication import get_authenticated_user -from .exceptions import ValidationError +from .exceptions import ValidationError, transform_validation_error from .msgpack import MsgpackRoute, MsgpackResponse from .stoken_handler import filter_by_stoken_and_limit @@ -79,7 +79,7 @@ class CollectionItemOut(CollectionItemCommon): version=obj.version, encryptionKey=obj.encryptionKey, etag=obj.etag, - content=CollectionItemRevisionOut.from_orm_context(obj.content, context), + content=CollectionItemRevision.from_orm_context(obj.content, context), ) @@ -125,11 +125,26 @@ class ItemDepIn(BaseModel): etag: str uid: str + def validate_db(self): + item = models.CollectionItem.objects.get(uid=self.uid) + etag = self.etag + if item.etag != etag: + raise ValidationError( + "wrong_etag", + "Wrong etag. Expected {} got {}".format(item.etag, etag), + status_code=status.HTTP_409_CONFLICT, + ) + class ItemBatchIn(BaseModel): items: t.List[CollectionItemIn] deps: t.Optional[ItemDepIn] + def validate_db(self): + if self.deps is not None: + for key, _value in self.deps: + getattr(self.deps, key).validate_db() + @sync_to_async def list_common( @@ -172,7 +187,7 @@ async def collection_list( pass -def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevisionOut): +def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevision): chunks_objs = [] revision = models.CollectionItemRevision(**revision_data.dict(exclude={"chunks"}), item=item) @@ -250,16 +265,60 @@ def get_collection(uid: str, user: User = Depends(get_authenticated_user), prefe return MsgpackResponse(ret) -def item_bulk_common(data: ItemBatchIn, user: User, stoken: str, uid: str, validate_etag: bool): +def item_create(item_model: CollectionItemIn, validate_etag: bool): + """Function that's called when this serializer creates an item""" + etag = item_model.etag + revision_data = item_model.content + uid = item_model.uid + + Model = models.CollectionItem + + with transaction.atomic(): + instance, created = Model.objects.get_or_create( + uid=uid, defaults=item_model.dict(exclude={"uid", "etag", "content"}) + ) + cur_etag = instance.etag if not created else None + + # If we are trying to update an up to date item, abort early and consider it a success + if cur_etag == revision_data.uid: + return instance + + if validate_etag and cur_etag != etag: + raise ValidationError( + "wrong_etag", + "Wrong etag. Expected {} got {}".format(cur_etag, etag), + status_code=status.HTTP_409_CONFLICT, + ) + + if not created: + # We don't have to use select_for_update here because the unique constraint on current guards against + # the race condition. But it's a good idea because it'll lock and wait rather than fail. + current_revision = instance.revisions.filter(current=True).select_for_update().first() + current_revision.current = None + current_revision.save() + + try: + process_revisions_for_item(instance, revision_data) + except django_exceptions.ValidationError as e: + transform_validation_error("content", e) + + return instance + + +def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid: str, validate_etag: bool): queryset = get_collection_queryset(user, default_queryset) with transaction.atomic(): # We need this for locking the collection object collection_object = queryset.select_for_update().get(uid=uid) + if stoken is not None and stoken != collection_object.stoken: raise ValidationError("stale_stoken", "Stoken is too old", status_code=status.HTTP_409_CONFLICT) + # XXX-TOM: make sure we return compatible errors + data.validate_db() + for item in data.items: + item_create(item, validate_etag) -def item_create(): - pass # + return MsgpackResponse({}) @collection_router.post("/{uid}/item/transaction/") From aa483709c33952b91cc9d75b47ef1c68b197b22e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 16:39:20 +0200 Subject: [PATCH 026/102] Fix item creation. --- etebase_fastapi/collection.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 5656fb6..8475f25 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -265,7 +265,7 @@ def get_collection(uid: str, user: User = Depends(get_authenticated_user), prefe return MsgpackResponse(ret) -def item_create(item_model: CollectionItemIn, validate_etag: bool): +def item_create(item_model: CollectionItemIn, collection: models.Collection, validate_etag: bool): """Function that's called when this serializer creates an item""" etag = item_model.etag revision_data = item_model.content @@ -275,7 +275,7 @@ def item_create(item_model: CollectionItemIn, validate_etag: bool): with transaction.atomic(): instance, created = Model.objects.get_or_create( - uid=uid, defaults=item_model.dict(exclude={"uid", "etag", "content"}) + uid=uid, collection=collection, defaults=item_model.dict(exclude={"uid", "etag", "content"}) ) cur_etag = instance.etag if not created else None @@ -316,7 +316,7 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid # XXX-TOM: make sure we return compatible errors data.validate_db() for item in data.items: - item_create(item, validate_etag) + item_create(item, collection_object, validate_etag) return MsgpackResponse({}) From 8afca6ca96c3f15807651be64b476374adeeee98 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 16:44:18 +0200 Subject: [PATCH 027/102] kwarg items: use the same naming as django_etebase. --- etebase_fastapi/collection.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 8475f25..102a7e0 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -321,15 +321,15 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid return MsgpackResponse({}) -@collection_router.post("/{uid}/item/transaction/") +@collection_router.post("/{collection_uid}/item/transaction/") def item_transaction( - uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) + collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): - item_bulk_common(data, user, stoken, uid, validate_etag=True) + item_bulk_common(data, user, stoken, collection_uid, validate_etag=True) -@collection_router.post("/{uid}/item/batch/") +@collection_router.post("/{collection_uid}/item/batch/") def item_batch( - uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) + collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): - item_bulk_common(data, user, stoken, uid, validate_etag=False) + item_bulk_common(data, user, stoken, collection_uid, validate_etag=False) From 92f6ccbc28f99494a464f717c769b98ba56ec8c1 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 17:02:36 +0200 Subject: [PATCH 028/102] Implement item_list and item_get. --- etebase_fastapi/collection.py | 83 ++++++++++++++++++++++++++++++++--- 1 file changed, 77 insertions(+), 6 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 102a7e0..d0a9e61 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -11,16 +11,18 @@ from django.db.models import QuerySet from fastapi import APIRouter, Depends, status, Query, Request from pydantic import BaseModel +# FIXME: it's not good that some things are imported, and some are used from the model including all of the name clashes from django_etebase import models from django_etebase.models import Collection, AccessLevels, CollectionMember from .authentication import get_authenticated_user from .exceptions import ValidationError, transform_validation_error from .msgpack import MsgpackRoute, MsgpackResponse -from .stoken_handler import filter_by_stoken_and_limit +from .stoken_handler import filter_by_stoken_and_limit, StokenAnnotation User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute) default_queryset: QuerySet = Collection.objects.all() +default_item_queryset: QuerySet = models.CollectionItem.objects.all() Prefetch = t.Literal["auto", "medium"] @@ -115,12 +117,18 @@ class CollectionIn(CollectionCommon): item: CollectionItemIn -class ListResponse(BaseModel): +class CollectionListResponse(BaseModel): data: t.List[CollectionOut] stoken: t.Optional[str] done: bool +class CollectionItemListResponse(BaseModel): + data: t.List[CollectionItemOut] + stoken: t.Optional[str] + done: bool + + class ItemDepIn(BaseModel): etag: str uid: str @@ -147,14 +155,18 @@ class ItemBatchIn(BaseModel): @sync_to_async -def list_common( - queryset: QuerySet, user: User, stoken: t.Optional[str], limit: int, prefetch: Prefetch +def collection_list_common( + queryset: QuerySet, + user: User, + stoken: t.Optional[str], + limit: int, + prefetch: Prefetch, ) -> MsgpackResponse: result, new_stoken_obj, done = filter_by_stoken_and_limit(stoken, limit, queryset, Collection.stoken_annotation) new_stoken = new_stoken_obj and new_stoken_obj.uid context = Context(user, prefetch) data: t.List[CollectionOut] = [CollectionOut.from_orm_context(item, context) for item in result] - ret = ListResponse(data=data, stoken=new_stoken, done=done) + ret = CollectionListResponse(data=data, stoken=new_stoken, done=done) return MsgpackResponse(content=ret) @@ -162,6 +174,19 @@ def get_collection_queryset(user: User, queryset: QuerySet) -> QuerySet: return queryset.filter(members__user=user) +def get_item_queryset( + user: User, collection_uid: str, queryset: QuerySet = default_item_queryset +) -> t.Tuple[models.Collection, QuerySet]: + try: + collection = get_collection_queryset(user, Collection.objects).get(uid=collection_uid) + except Collection.DoesNotExist: + raise ValidationError("does_not_exist", "Collection does not exist", status_code=status.HTTP_404_NOT_FOUND) + # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') + queryset = queryset.filter(collection__pk=collection.pk, revisions__current=True) + + return collection, queryset + + @collection_router.post("/list_multi/") async def list_multi( data: ListMulti, @@ -175,7 +200,8 @@ async def list_multi( queryset = queryset.filter( Q(members__collectionType__uid__in=data.collectionTypes) | Q(members__collectionType__isnull=True) ) - response = await list_common(queryset, user, stoken, limit, prefetch) + # XXX-TOM: missing removedMemeberships + response = await collection_list_common(queryset, user, stoken, limit, prefetch) return response @@ -305,6 +331,51 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val return instance +@collection_router.get("/{collection_uid}/item/{uid}/") +def item_get( + collection_uid: str, uid: str, user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery +): + _, queryset = get_item_queryset(user, collection_uid) + obj = queryset.get(uid=uid) + ret = CollectionItemOut.from_orm_context(obj, Context(user, prefetch)) + return MsgpackResponse(ret) + + +@sync_to_async +def item_list_common( + queryset: QuerySet, + user: User, + stoken: t.Optional[str], + limit: int, + prefetch: Prefetch, +) -> MsgpackResponse: + result, new_stoken_obj, done = filter_by_stoken_and_limit( + stoken, limit, queryset, models.CollectionItem.stoken_annotation + ) + new_stoken = new_stoken_obj and new_stoken_obj.uid + context = Context(user, prefetch) + data: t.List[CollectionItemOut] = [CollectionItemOut.from_orm_context(item, context) for item in result] + ret = CollectionItemListResponse(data=data, stoken=new_stoken, done=done) + return MsgpackResponse(content=ret) + + +@collection_router.get("/{collection_uid}/item/") +async def item_list( + collection_uid: str, + stoken: t.Optional[str] = None, + limit: int = 50, + prefetch: Prefetch = PrefetchQuery, + withCollection: bool = False, + user: User = Depends(get_authenticated_user), +): + _, queryset = await sync_to_async(get_item_queryset)(user, collection_uid) + if not withCollection: + queryset = queryset.filter(parent__isnull=True) + + response = await item_list_common(queryset, user, stoken, limit, prefetch) + return response + + def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid: str, validate_etag: bool): queryset = get_collection_queryset(user, default_queryset) with transaction.atomic(): # We need this for locking the collection object From 611c0f3b0a89088aa36afb05e4873281eb971852 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 17:03:17 +0200 Subject: [PATCH 029/102] Conform to naming conventions. --- etebase_fastapi/collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index d0a9e61..91541ce 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -285,7 +285,7 @@ async def create(data: CollectionIn, user: User = Depends(get_authenticated_user @collection_router.get("/{uid}/") -def get_collection(uid: str, user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery): +def collection_get(uid: str, user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery): obj = get_collection_queryset(user, default_queryset).get(uid=uid) ret = CollectionOut.from_orm_context(obj, Context(user, prefetch)) return MsgpackResponse(ret) From e5dbfb57460439cdd28b0d8c9656599ecbe99f03 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 17:07:07 +0200 Subject: [PATCH 030/102] Make the import of models more consistent. --- etebase_fastapi/collection.py | 28 +++++++++++++--------------- 1 file changed, 13 insertions(+), 15 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 91541ce..6e567fd 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -11,17 +11,15 @@ from django.db.models import QuerySet from fastapi import APIRouter, Depends, status, Query, Request from pydantic import BaseModel -# FIXME: it's not good that some things are imported, and some are used from the model including all of the name clashes from django_etebase import models -from django_etebase.models import Collection, AccessLevels, CollectionMember from .authentication import get_authenticated_user from .exceptions import ValidationError, transform_validation_error from .msgpack import MsgpackRoute, MsgpackResponse -from .stoken_handler import filter_by_stoken_and_limit, StokenAnnotation +from .stoken_handler import filter_by_stoken_and_limit User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute) -default_queryset: QuerySet = Collection.objects.all() +default_queryset: QuerySet = models.Collection.objects.all() default_item_queryset: QuerySet = models.CollectionItem.objects.all() @@ -39,7 +37,7 @@ class ListMulti(BaseModel): collectionTypes: t.List[bytes] -class CollectionItemRevision(BaseModel): +class CollectionItemRevisionInOut(BaseModel): uid: str meta: bytes deleted: bool @@ -50,8 +48,8 @@ class CollectionItemRevision(BaseModel): @classmethod def from_orm_context( - cls: t.Type["CollectionItemRevision"], obj: models.CollectionItemRevision, context: Context - ) -> "CollectionItemRevision": + cls: t.Type["CollectionItemRevisionInOut"], obj: models.CollectionItemRevision, context: Context + ) -> "CollectionItemRevisionInOut": chunk_obj = obj.chunks_relation.get().chunk if context.prefetch == "auto": with open(chunk_obj.chunkFile.path, "rb") as f: @@ -65,7 +63,7 @@ class CollectionItemCommon(BaseModel): uid: str version: int encryptionKey: t.Optional[bytes] - content: CollectionItemRevision + content: CollectionItemRevisionInOut class CollectionItemOut(CollectionItemCommon): @@ -81,7 +79,7 @@ class CollectionItemOut(CollectionItemCommon): version=obj.version, encryptionKey=obj.encryptionKey, etag=obj.etag, - content=CollectionItemRevision.from_orm_context(obj.content, context), + content=CollectionItemRevisionInOut.from_orm_context(obj.content, context), ) @@ -95,12 +93,12 @@ class CollectionCommon(BaseModel): class CollectionOut(CollectionCommon): - accessLevel: AccessLevels + accessLevel: models.AccessLevels stoken: str item: CollectionItemOut @classmethod - def from_orm_context(cls: t.Type["CollectionOut"], obj: Collection, context: Context) -> "CollectionOut": + def from_orm_context(cls: t.Type["CollectionOut"], obj: models.Collection, context: Context) -> "CollectionOut": member: CollectionMember = obj.members.get(user=context.user) collection_type = member.collectionType ret = cls( @@ -162,7 +160,7 @@ def collection_list_common( limit: int, prefetch: Prefetch, ) -> MsgpackResponse: - result, new_stoken_obj, done = filter_by_stoken_and_limit(stoken, limit, queryset, Collection.stoken_annotation) + result, new_stoken_obj, done = filter_by_stoken_and_limit(stoken, limit, queryset, models.Collection.stoken_annotation) new_stoken = new_stoken_obj and new_stoken_obj.uid context = Context(user, prefetch) data: t.List[CollectionOut] = [CollectionOut.from_orm_context(item, context) for item in result] @@ -178,8 +176,8 @@ def get_item_queryset( user: User, collection_uid: str, queryset: QuerySet = default_item_queryset ) -> t.Tuple[models.Collection, QuerySet]: try: - collection = get_collection_queryset(user, Collection.objects).get(uid=collection_uid) - except Collection.DoesNotExist: + collection = get_collection_queryset(user, models.Collection.objects).get(uid=collection_uid) + except models.Collection.DoesNotExist: raise ValidationError("does_not_exist", "Collection does not exist", status_code=status.HTTP_404_NOT_FOUND) # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') queryset = queryset.filter(collection__pk=collection.pk, revisions__current=True) @@ -213,7 +211,7 @@ async def collection_list( pass -def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevision): +def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevisionInOut): chunks_objs = [] revision = models.CollectionItemRevision(**revision_data.dict(exclude={"chunks"}), item=item) From 407ce0b7a5222900ff8d2d2c84b8e38726afcfc4 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 17:10:59 +0200 Subject: [PATCH 031/102] Fixed collection_list_common. --- etebase_fastapi/collection.py | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 6e567fd..0f019fa 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -15,7 +15,7 @@ from django_etebase import models from .authentication import get_authenticated_user from .exceptions import ValidationError, transform_validation_error from .msgpack import MsgpackRoute, MsgpackResponse -from .stoken_handler import filter_by_stoken_and_limit +from .stoken_handler import filter_by_stoken_and_limit, get_stoken_obj User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute) @@ -115,11 +115,17 @@ class CollectionIn(CollectionCommon): item: CollectionItemIn +class RemovedMembershipOut(BaseModel): + uid: str + + class CollectionListResponse(BaseModel): data: t.List[CollectionOut] stoken: t.Optional[str] done: bool + removedMemberships: t.Optional[RemovedMembershipOut] + class CollectionItemListResponse(BaseModel): data: t.List[CollectionItemOut] @@ -164,7 +170,22 @@ def collection_list_common( new_stoken = new_stoken_obj and new_stoken_obj.uid context = Context(user, prefetch) data: t.List[CollectionOut] = [CollectionOut.from_orm_context(item, context) for item in result] - ret = CollectionListResponse(data=data, stoken=new_stoken, done=done) + + stoken_obj = get_stoken_obj(stoken) + removedMemberships = None + if stoken_obj is not None: + # FIXME: honour limit? (the limit should be combined for data and this because of stoken) + remed_qs = models.CollectionMemberRemoved.objects.filter(user=user, stoken__id__gt=stoken_obj.id) + if not done and new_stoken_obj is not None: + # We only filter by the new_stoken if we are not done. This is because if we are done, the new stoken + # can point to the most recent collection change rather than most recent removed membership. + remed_qs = remed_qs.filter(stoken__id__lte=new_stoken_obj.id) + + remed = remed_qs.values_list("collection__uid", flat=True) + if len(remed) > 0: + removedMemberships = [{"uid": x} for x in remed] + + ret = CollectionListResponse(data=data, stoken=new_stoken, done=done, removedMemberships=removedMemberships) return MsgpackResponse(content=ret) @@ -194,13 +215,13 @@ async def list_multi( prefetch: Prefetch = PrefetchQuery, ): queryset = get_collection_queryset(user, default_queryset) + # FIXME: Remove the isnull part once we attach collection types to all objects ("collection-type-migration") queryset = queryset.filter( Q(members__collectionType__uid__in=data.collectionTypes) | Q(members__collectionType__isnull=True) ) - # XXX-TOM: missing removedMemeberships - response = await collection_list_common(queryset, user, stoken, limit, prefetch) - return response + + return await collection_list_common(queryset, user, stoken, limit, prefetch) @collection_router.post("/list/") From 7ad98b8d28a70de1074d15f17eadc201720283a2 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 17:30:17 +0200 Subject: [PATCH 032/102] Implement is_etebase. --- etebase_fastapi/authentication.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 51ba80d..e355aa4 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -223,6 +223,11 @@ def validate_login_request( raise ValidationError("login_bad_signature", "Wrong password for user.", status.HTTP_401_UNAUTHORIZED) +@authentication_router.get("/is_etebase/") +async def is_etebase(): + return MsgpackResponse({}) + + @authentication_router.post("/login_challenge/") async def login_challenge(user: User = Depends(get_login_user)): enc_key = get_encryption_key(user.userinfo.salt) From c6c52cfe1100ef401d4941f71f7b28601473f579 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 17:33:01 +0200 Subject: [PATCH 033/102] Implement collection list. --- etebase_fastapi/collection.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 0f019fa..56e2180 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -166,7 +166,9 @@ def collection_list_common( limit: int, prefetch: Prefetch, ) -> MsgpackResponse: - result, new_stoken_obj, done = filter_by_stoken_and_limit(stoken, limit, queryset, models.Collection.stoken_annotation) + result, new_stoken_obj, done = filter_by_stoken_and_limit( + stoken, limit, queryset, models.Collection.stoken_annotation + ) new_stoken = new_stoken_obj and new_stoken_obj.uid context = Context(user, prefetch) data: t.List[CollectionOut] = [CollectionOut.from_orm_context(item, context) for item in result] @@ -227,9 +229,13 @@ async def list_multi( @collection_router.post("/list/") async def collection_list( req: Request, + stoken: t.Optional[str] = None, + limit: int = 50, + prefetch: Prefetch = PrefetchQuery, user: User = Depends(get_authenticated_user), ): - pass + queryset = get_collection_queryset(user, default_queryset) + return await collection_list_common(queryset, user, stoken, limit, prefetch) def process_revisions_for_item(item: models.CollectionItem, revision_data: CollectionItemRevisionInOut): From a9bc08a98d59e60a929476f21bb17551bd65648b Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 17:39:47 +0200 Subject: [PATCH 034/102] Item batch/transaction: fix return data. --- etebase_fastapi/collection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 56e2180..3a9ca8b 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -421,11 +421,11 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid def item_transaction( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): - item_bulk_common(data, user, stoken, collection_uid, validate_etag=True) + return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True) @collection_router.post("/{collection_uid}/item/batch/") def item_batch( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): - item_bulk_common(data, user, stoken, collection_uid, validate_etag=False) + return item_bulk_common(data, user, stoken, collection_uid, validate_etag=False) From a3ae769a2ca3d4a9e3d1f47c67df3350dfd432c7 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 17:58:15 +0200 Subject: [PATCH 035/102] Implement fetch_updates. --- etebase_fastapi/collection.py | 42 ++++++++++++++++++++++++++++++- etebase_fastapi/stoken_handler.py | 2 +- 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 3a9ca8b..7679ed6 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -15,7 +15,7 @@ from django_etebase import models from .authentication import get_authenticated_user from .exceptions import ValidationError, transform_validation_error from .msgpack import MsgpackRoute, MsgpackResponse -from .stoken_handler import filter_by_stoken_and_limit, get_stoken_obj +from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute) @@ -133,6 +133,11 @@ class CollectionItemListResponse(BaseModel): done: bool +class CollectionItemBulkGetIn(BaseModel): + uid: str + etag: t.Optional[str] + + class ItemDepIn(BaseModel): etag: str uid: str @@ -417,6 +422,41 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid return MsgpackResponse({}) +@collection_router.post("/{collection_uid}/item/fetch_updates/") +def fetch_updates( + collection_uid: str, + data: t.List[CollectionItemBulkGetIn], + stoken: t.Optional[str] = None, + prefetch: Prefetch = PrefetchQuery, + user: User = Depends(get_authenticated_user), +): + _, queryset = get_item_queryset(user, collection_uid) + # FIXME: make configurable? + item_limit = 200 + + if len(data) > item_limit: + raise ValidationError("too_many_items", "Request has too many items.", status_code=status.HTTP_400_BAD_REQUEST) + + queryset, stoken_rev = filter_by_stoken(stoken, queryset, models.CollectionItem.stoken_annotation) + + uids, etags = zip(*[(item.uid, item.etag) for item in data]) + revs = models.CollectionItemRevision.objects.filter(uid__in=etags, current=True) + queryset = queryset.filter(uid__in=uids).exclude(revisions__in=revs) + + new_stoken_obj = get_queryset_stoken(queryset) + new_stoken = new_stoken_obj and new_stoken_obj.uid + stoken = stoken_rev and getattr(stoken_rev, "uid", None) + new_stoken = new_stoken or stoken + + context = Context(user, prefetch) + ret = CollectionItemListResponse( + data=[CollectionItemOut.from_orm_context(item, context) for item in queryset], + stoken=new_stoken, + done=True, # we always return all the items, so it's always done + ) + return MsgpackResponse(ret) + + @collection_router.post("/{collection_uid}/item/transaction/") def item_transaction( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) diff --git a/etebase_fastapi/stoken_handler.py b/etebase_fastapi/stoken_handler.py index fb89651..a976830 100644 --- a/etebase_fastapi/stoken_handler.py +++ b/etebase_fastapi/stoken_handler.py @@ -33,7 +33,7 @@ def filter_by_stoken( return queryset, stoken_rev -def get_queryset_stoken(queryset: list) -> t.Optional[Stoken]: +def get_queryset_stoken(queryset: t.Iterable[t.Any]) -> t.Optional[Stoken]: maxid = -1 for row in queryset: rowmaxid = getattr(row, "max_stoken") or -1 From e7721e8fe52fce7667017acafee7a2c0bd8a7143 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 18:12:16 +0200 Subject: [PATCH 036/102] Fix chunk handling. --- etebase_fastapi/collection.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 7679ed6..229b8c4 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -50,13 +50,15 @@ class CollectionItemRevisionInOut(BaseModel): def from_orm_context( cls: t.Type["CollectionItemRevisionInOut"], obj: models.CollectionItemRevision, context: Context ) -> "CollectionItemRevisionInOut": - chunk_obj = obj.chunks_relation.get().chunk - if context.prefetch == "auto": - with open(chunk_obj.chunkFile.path, "rb") as f: - chunks = chunk_obj.uid, f.read() - else: - chunks = (chunk_obj.uid,) - return cls(uid=obj.uid, meta=obj.meta, deleted=obj.deleted, chunks=[chunks]) + chunks = [] + for chunk_relation in obj.chunks_relation.all(): + chunk_obj = chunk_relation.chunk + if context.prefetch == "auto": + with open(chunk_obj.chunkFile.path, "rb") as f: + chunks.append((chunk_obj.uid, f.read())) + else: + chunks.append((chunk_obj.uid,)) + return cls(uid=obj.uid, meta=obj.meta, deleted=obj.deleted, chunks=chunks) class CollectionItemCommon(BaseModel): From e686f016521826c8764d8851bd594cef94848a6a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 18:34:23 +0200 Subject: [PATCH 037/102] Utils: add a utility for getting objects or 404ing. --- etebase_fastapi/collection.py | 6 ++---- etebase_fastapi/utils.py | 13 +++++++++++++ 2 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 etebase_fastapi/utils.py diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 229b8c4..2f8d1ed 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -16,6 +16,7 @@ from .authentication import get_authenticated_user from .exceptions import ValidationError, transform_validation_error from .msgpack import MsgpackRoute, MsgpackResponse from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken +from .utils import get_object_or_404 User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute) @@ -205,10 +206,7 @@ def get_collection_queryset(user: User, queryset: QuerySet) -> QuerySet: def get_item_queryset( user: User, collection_uid: str, queryset: QuerySet = default_item_queryset ) -> t.Tuple[models.Collection, QuerySet]: - try: - collection = get_collection_queryset(user, models.Collection.objects).get(uid=collection_uid) - except models.Collection.DoesNotExist: - raise ValidationError("does_not_exist", "Collection does not exist", status_code=status.HTTP_404_NOT_FOUND) + collection = get_object_or_404(get_collection_queryset(user, models.Collection.objects), uid=collection_uid) # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') queryset = queryset.filter(collection__pk=collection.pk, revisions__current=True) diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py new file mode 100644 index 0000000..d7f8c09 --- /dev/null +++ b/etebase_fastapi/utils.py @@ -0,0 +1,13 @@ +from fastapi import status + +from django.db.models import QuerySet +from django.core.exceptions import ObjectDoesNotExist + +from .exceptions import ValidationError + + +def get_object_or_404(queryset: QuerySet, **kwargs): + try: + return queryset.get(**kwargs) + except ObjectDoesNotExist as e: + raise ValidationError("does_not_exist", str(e), status_code=status.HTTP_404_NOT_FOUND) From 533b2787bb1ac716a9ba4c670d675432b84b3cec Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 18:34:40 +0200 Subject: [PATCH 038/102] Implement item revisions. --- etebase_fastapi/collection.py | 43 +++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 2f8d1ed..8eabf9a 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -136,6 +136,12 @@ class CollectionItemListResponse(BaseModel): done: bool +class CollectionItemRevisionListResponse(BaseModel): + data: t.List[CollectionItemRevisionInOut] + iterator: t.Optional[str] + done: bool + + class CollectionItemBulkGetIn(BaseModel): uid: str etag: t.Optional[str] @@ -422,6 +428,43 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid return MsgpackResponse({}) +@collection_router.get("/{collection_uid}/item/{uid}/revision/") +def item_revisions( + collection_uid: str, + uid: str, + limit: int = 50, + iterator: t.Optional[str] = None, + prefetch: Prefetch = PrefetchQuery, + user: User = Depends(get_authenticated_user), +): + _, items = get_item_queryset(user, collection_uid) + item = get_object_or_404(items, uid=uid) + + queryset = item.revisions.order_by("-id") + + if iterator is not None: + iterator_obj = get_object_or_404(queryset, uid=iterator) + queryset = queryset.filter(id__lt=iterator_obj.id) + + result = list(queryset[: limit + 1]) + if len(result) < limit + 1: + done = True + else: + done = False + result = result[:-1] + + context = Context(user, prefetch) + ret_data = [CollectionItemRevisionInOut.from_orm_context(revision, context) for revision in result] + iterator = ret_data[-1].uid if len(result) > 0 else None + + ret = CollectionItemRevisionListResponse( + data=ret_data, + iterator=iterator, + done=done, + ) + return MsgpackResponse(ret) + + @collection_router.post("/{collection_uid}/item/fetch_updates/") def fetch_updates( collection_uid: str, From 629a84f43243e1ee905ed985458cdd98c0192774 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 18:38:18 +0200 Subject: [PATCH 039/102] app.py: cleanup a bit. --- etebase_fastapi/app.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index 6fbdd04..81772b4 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -3,10 +3,12 @@ import os from django.core.wsgi import get_wsgi_application from fastapi.middleware.cors import CORSMiddleware -from django.conf import settings - os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etebase_server.settings") application = get_wsgi_application() + +from django.conf import settings + +# Not at the top of the file because we first need to setup django from fastapi import FastAPI, Request from .exceptions import CustomHttpException From 13d4121fc275a94c6d2a53d9d894e6754c24f740 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 18:54:06 +0200 Subject: [PATCH 040/102] Move utility functions to utils. --- etebase_fastapi/collection.py | 15 ++------------- etebase_fastapi/utils.py | 17 ++++++++++++++++- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 8eabf9a..25883f8 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -1,4 +1,3 @@ -import dataclasses import typing as t from asgiref.sync import sync_to_async @@ -8,7 +7,7 @@ from django.core.files.base import ContentFile from django.db import transaction from django.db.models import Q from django.db.models import QuerySet -from fastapi import APIRouter, Depends, status, Query, Request +from fastapi import APIRouter, Depends, status, Request from pydantic import BaseModel from django_etebase import models @@ -16,7 +15,7 @@ from .authentication import get_authenticated_user from .exceptions import ValidationError, transform_validation_error from .msgpack import MsgpackRoute, MsgpackResponse from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken -from .utils import get_object_or_404 +from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute) @@ -24,16 +23,6 @@ default_queryset: QuerySet = models.Collection.objects.all() default_item_queryset: QuerySet = models.CollectionItem.objects.all() -Prefetch = t.Literal["auto", "medium"] -PrefetchQuery = Query(default="auto") - - -@dataclasses.dataclass -class Context: - user: t.Optional[User] - prefetch: t.Optional[Prefetch] - - class ListMulti(BaseModel): collectionTypes: t.List[bytes] diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index d7f8c09..d9bef73 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -1,10 +1,25 @@ -from fastapi import status +import dataclasses +import typing as t + +from fastapi import status, Query from django.db.models import QuerySet from django.core.exceptions import ObjectDoesNotExist +from django.contrib.auth import get_user_model from .exceptions import ValidationError +User = get_user_model() + +Prefetch = t.Literal["auto", "medium"] +PrefetchQuery = Query(default="auto") + + +@dataclasses.dataclass +class Context: + user: t.Optional[User] + prefetch: t.Optional[Prefetch] + def get_object_or_404(queryset: QuerySet, **kwargs): try: From ec8c69b3f3ff0c12a31dc941722c86d7ace6d36e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 20:36:11 +0200 Subject: [PATCH 041/102] Fix a few FIXMEs. --- etebase_fastapi/authentication.py | 21 +++++++++++---------- etebase_fastapi/test_reset_view.py | 6 +++--- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index e355aa4..a8fbbfe 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -24,7 +24,7 @@ from django_etebase.models import UserInfo from django_etebase.signals import user_signed_up from django_etebase.token_auth.models import AuthToken from django_etebase.token_auth.models import get_default_expiry -from django_etebase.utils import create_user +from django_etebase.utils import create_user, get_user_queryset, CallbackContext from django_etebase.views import msgpack_encode, msgpack_decode from .exceptions import AuthenticationFailed, transform_validation_error, ValidationError from .msgpack import MsgpackResponse, MsgpackRoute @@ -268,20 +268,21 @@ async def change_password(data: ChangePassword, request: Request, user: User = D return Response(status_code=status.HTTP_204_NO_CONTENT) -def signup_save(data: SignupIn) -> User: +def signup_save(data: SignupIn, request: Request) -> User: user_data = data.user with transaction.atomic(): try: - # XXX-TOM - # view = self.context.get("view", None) - # user_queryset = get_user_queryset(User.objects.all(), view) - user_queryset = User.objects.all() + user_queryset = get_user_queryset(User.objects.all(), CallbackContext(request.path_params)) instance = user_queryset.get(**{User.USERNAME_FIELD: user_data.username.lower()}) except User.DoesNotExist: # Create the user and save the casing the user chose as the first name try: - # XXX-TOM - instance = create_user(**user_data.dict(), password=None, first_name=user_data.username, view=None) + instance = create_user( + **user_data.dict(), + password=None, + first_name=user_data.username, + context=CallbackContext(request.path_params), + ) instance.full_clean() except EtebaseValidationError as e: raise e @@ -298,8 +299,8 @@ def signup_save(data: SignupIn) -> User: @authentication_router.post("/signup/") -async def signup(data: SignupIn): - user = await sync_to_async(signup_save)(data) +async def signup(data: SignupIn, request: Request): + user = await sync_to_async(signup_save)(data, request) # XXX-TOM data = await sync_to_async(LoginOut.from_orm)(user) await sync_to_async(user_signed_up.send)(sender=user.__class__, request=None, user=user) diff --git a/etebase_fastapi/test_reset_view.py b/etebase_fastapi/test_reset_view.py index ea7d8d9..435a56e 100644 --- a/etebase_fastapi/test_reset_view.py +++ b/etebase_fastapi/test_reset_view.py @@ -2,7 +2,7 @@ from django.conf import settings from django.contrib.auth import get_user_model from django.db import transaction from django.shortcuts import get_object_or_404 -from fastapi import APIRouter, Response, status +from fastapi import APIRouter, Request, Response, status from django_etebase.utils import get_user_queryset from etebase_fastapi.authentication import SignupIn, signup_save @@ -13,7 +13,7 @@ User = get_user_model() @test_reset_view_router.post("/reset/") -def reset(data: SignupIn): +def reset(data: SignupIn, request: Request): # Only run when in DEBUG mode! It's only used for tests if not settings.DEBUG: return Response("Only allowed in debug mode.", status_code=status.HTTP_400_BAD_REQUEST) @@ -28,7 +28,7 @@ def reset(data: SignupIn): if hasattr(user, "userinfo"): user.userinfo.delete() - signup_save(data) + signup_save(data, request) # Delete all of the journal data for this user for a clear test env user.collection_set.all().delete() user.collectionmember_set.all().delete() From 7f90edc5114b50fa2ad414a4f2abc18f263fbdf4 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 21:01:14 +0200 Subject: [PATCH 042/102] MsgPack: handle no content. --- etebase_fastapi/msgpack.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/etebase_fastapi/msgpack.py b/etebase_fastapi/msgpack.py index 399f3d0..edffd7e 100644 --- a/etebase_fastapi/msgpack.py +++ b/etebase_fastapi/msgpack.py @@ -19,7 +19,10 @@ class MsgpackRequest(Request): class MsgpackResponse(Response): media_type = "application/msgpack" - def render(self, content: t.Any) -> bytes: + def render(self, content: t.Optional[t.Any]) -> t.Optional[bytes]: + if content is None: + return b"" + if isinstance(content, BaseModel): content = content.dict() return msgpack.packb(content, use_bin_type=True) From b70f2b74705a1ce91ad28c10f643e626b5df0459 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 21:08:00 +0200 Subject: [PATCH 043/102] Invitations: implement invitations endpoints. --- etebase_fastapi/app.py | 3 + etebase_fastapi/invitation.py | 229 ++++++++++++++++++++++++++++++++++ etebase_fastapi/utils.py | 7 ++ 3 files changed, 239 insertions(+) create mode 100644 etebase_fastapi/invitation.py diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index 81772b4..ff50ce5 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -14,6 +14,7 @@ from fastapi import FastAPI, Request from .exceptions import CustomHttpException from .authentication import authentication_router from .collection import collection_router +from .invitation import invitation_incoming_router, invitation_outgoing_router from .msgpack import MsgpackResponse app = FastAPI() @@ -21,6 +22,8 @@ VERSION = "v1" BASE_PATH = f"/api/{VERSION}" app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication") app.include_router(collection_router, prefix=f"{BASE_PATH}/collection") +app.include_router(invitation_incoming_router, prefix=f"{BASE_PATH}/invitation/incoming") +app.include_router(invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing") if settings.DEBUG: from .test_reset_view import test_reset_view_router diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/invitation.py new file mode 100644 index 0000000..077dcfd --- /dev/null +++ b/etebase_fastapi/invitation.py @@ -0,0 +1,229 @@ +import typing as t + +from django.contrib.auth import get_user_model +from django.db import transaction, IntegrityError +from django.db.models import QuerySet +from fastapi import APIRouter, Depends, status, Request +from pydantic import BaseModel + +from django_etebase import models +from django_etebase.utils import get_user_queryset, CallbackContext +from .authentication import get_authenticated_user +from .exceptions import ValidationError, PermissionDenied +from .msgpack import MsgpackRoute, MsgpackResponse +from .utils import get_object_or_404, Context, is_collection_admin + +User = get_user_model() +invitation_incoming_router = APIRouter(route_class=MsgpackRoute) +invitation_outgoing_router = APIRouter(route_class=MsgpackRoute) +default_queryset: QuerySet = models.CollectionInvitation.objects.all() + + +class UserInfoOut(BaseModel): + pubkey: bytes + + class Config: + orm_mode = True + + +class CollectionInvitationAcceptIn(BaseModel): + collectionType: bytes + encryptionKey: bytes + + +class CollectionInvitationCommon(BaseModel): + uid: str + version: int + accessLevel: models.AccessLevels + username: str + collection: str + signedEncryptionKey: bytes + + +class CollectionInvitationIn(CollectionInvitationCommon): + def validate_db(self, context: Context): + if context.user.username == self.username.lower(): + raise ValidationError("no_self_invite", "Inviting yourself is not allowed") + + +class CollectionInvitationOut(CollectionInvitationCommon): + fromUsername: str + fromPubkey: bytes + + class Config: + orm_mode = True + + @classmethod + def from_orm(cls: t.Type["CollectionInvitationOut"], obj: models.CollectionInvitation) -> "CollectionInvitationOut": + return cls( + uid=obj.uid, + version=obj.version, + accessLevel=obj.accessLevel, + username=obj.user.username, + collection=obj.collection.uid, + fromUsername=obj.fromMember.user.username, + fromPubkey=obj.fromMember.user.userinfo.pubkey, + signedEncryptionKey=obj.signedEncryptionKey, + ) + + +class InvitationListResponse(BaseModel): + data: t.List[CollectionInvitationOut] + iterator: t.Optional[str] + done: bool + + +def get_incoming_queryset(user: User, queryset=default_queryset): + return queryset.filter(user=user) + + +def get_outgoing_queryset(user: User, queryset=default_queryset): + return queryset.filter(fromMember__user=user) + + +def list_common( + queryset: QuerySet, + iterator: t.Optional[str], + limit: int, +) -> MsgpackResponse: + queryset = queryset.order_by("id") + + if iterator is not None: + iterator_obj = get_object_or_404(queryset, uid=iterator) + queryset = queryset.filter(id__gt=iterator_obj.id) + + result = list(queryset[: limit + 1]) + if len(result) < limit + 1: + done = True + else: + done = False + result = result[:-1] + + ret_data = result + iterator = ret_data[-1].uid if len(result) > 0 else None + + ret = InvitationListResponse( + data=ret_data, + iterator=iterator, + done=done, + ) + return MsgpackResponse(ret) + + +@invitation_incoming_router.get("/", response_model=InvitationListResponse) +def incoming_list( + iterator: t.Optional[str] = None, + limit: int = 50, + user: User = Depends(get_authenticated_user), +): + return list_common(get_incoming_queryset(user), iterator, limit) + + +@invitation_incoming_router.get("/{invitation_uid}/", response_model=CollectionInvitationOut) +def incoming_get( + invitation_uid: str, + user: User = Depends(get_authenticated_user), +): + queryset = get_incoming_queryset(user) + obj = get_object_or_404(queryset, uid=invitation_uid) + ret = CollectionInvitationOut.from_orm(obj) + return MsgpackResponse(ret) + + +@invitation_incoming_router.delete("/{invitation_uid}/", status_code=status.HTTP_204_NO_CONTENT) +def incoming_delete( + invitation_uid: str, + user: User = Depends(get_authenticated_user), +): + queryset = get_incoming_queryset(user) + obj = get_object_or_404(queryset, uid=invitation_uid) + obj.delete() + + +@invitation_incoming_router.post("/{invitation_uid}/accept/", status_code=status.HTTP_201_CREATED) +def incoming_accept( + invitation_uid: str, + data: CollectionInvitationAcceptIn, + user: User = Depends(get_authenticated_user), +): + queryset = get_incoming_queryset(user) + invitation = get_object_or_404(queryset, uid=invitation_uid) + + with transaction.atomic(): + user = invitation.user + collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=data.collectionType, owner=user) + + models.CollectionMember.objects.create( + collection=invitation.collection, + stoken=models.Stoken.objects.create(), + user=user, + accessLevel=invitation.accessLevel, + encryptionKey=data.encryptionKey, + collectionType=collection_type_obj, + ) + + models.CollectionMemberRemoved.objects.filter(user=invitation.user, collection=invitation.collection).delete() + + invitation.delete() + + +@invitation_outgoing_router.post("/", status_code=status.HTTP_201_CREATED) +def outgoing_create( + data: CollectionInvitationIn, + request: Request, + user: User = Depends(get_authenticated_user), +): + collection = get_object_or_404(models.Collection.objects, uid=data.collection) + to_user = get_object_or_404( + get_user_queryset(User.objects.all(), CallbackContext(request.path_params)), username=data.username + ) + + context = Context(user, None) + data.validate_db(context) + + if not is_collection_admin(collection, user): + raise PermissionDenied("admin_access_required", "User is not an admin of this collection") + + member = collection.members.get(user=user) + + with transaction.atomic(): + try: + ret = models.CollectionInvitation.objects.create( + **data.dict(exclude={"collection", "username"}), user=to_user, fromMember=member + ) + except IntegrityError: + raise ValidationError("invitation_exists", "Invitation already exists") + + return MsgpackResponse(CollectionInvitationOut.from_orm(ret), status_code=status.HTTP_201_CREATED) + + +@invitation_outgoing_router.get("/", response_model=InvitationListResponse) +def outgoing_list( + iterator: t.Optional[str] = None, + limit: int = 50, + user: User = Depends(get_authenticated_user), +): + return list_common(get_outgoing_queryset(user), iterator, limit) + + +@invitation_outgoing_router.delete("/{invitation_uid}/", status_code=status.HTTP_204_NO_CONTENT) +def outgoing_delete( + invitation_uid: str, + user: User = Depends(get_authenticated_user), +): + queryset = get_outgoing_queryset(user) + obj = get_object_or_404(queryset, uid=invitation_uid) + obj.delete() + + +@invitation_outgoing_router.get("/fetch_user_profile/", response_model=UserInfoOut) +def outgoing_fetch_user_profile( + username: str, + request: Request, + user: User = Depends(get_authenticated_user), +): + kwargs = {User.USERNAME_FIELD: username.lower()} + user = get_object_or_404(get_user_queryset(User.objects.all(), CallbackContext(request.path_params)), **kwargs) + user_info = get_object_or_404(models.UserInfo.objects.all(), owner=user) + ret = UserInfoOut.from_orm(user_info) + return MsgpackResponse(ret) diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index d9bef73..150afe8 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -7,6 +7,8 @@ from django.db.models import QuerySet from django.core.exceptions import ObjectDoesNotExist from django.contrib.auth import get_user_model +from django_etebase.models import AccessLevels + from .exceptions import ValidationError User = get_user_model() @@ -26,3 +28,8 @@ def get_object_or_404(queryset: QuerySet, **kwargs): return queryset.get(**kwargs) except ObjectDoesNotExist as e: raise ValidationError("does_not_exist", str(e), status_code=status.HTTP_404_NOT_FOUND) + + +def is_collection_admin(collection, user): + member = collection.members.filter(user=user).first() + return (member is not None) and (member.accessLevel == AccessLevels.ADMIN) From b5a750d6d09ee98fe3f73c9b6766458711a793a1 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 21:41:31 +0200 Subject: [PATCH 044/102] Collection: fix removed memberships. --- etebase_fastapi/collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 25883f8..0af6056 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -116,7 +116,7 @@ class CollectionListResponse(BaseModel): stoken: t.Optional[str] done: bool - removedMemberships: t.Optional[RemovedMembershipOut] + removedMemberships: t.Optional[t.List[RemovedMembershipOut]] class CollectionItemListResponse(BaseModel): From 36e6d3df24628cba764559add4127ab99e57c6f6 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 21:32:48 +0200 Subject: [PATCH 045/102] Members: add member endpoints. --- etebase_fastapi/app.py | 1 + etebase_fastapi/member.py | 83 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) create mode 100644 etebase_fastapi/member.py diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index ff50ce5..755340c 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -14,6 +14,7 @@ from fastapi import FastAPI, Request from .exceptions import CustomHttpException from .authentication import authentication_router from .collection import collection_router +from . import member # noqa from .invitation import invitation_incoming_router, invitation_outgoing_router from .msgpack import MsgpackResponse diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py new file mode 100644 index 0000000..36aa5ce --- /dev/null +++ b/etebase_fastapi/member.py @@ -0,0 +1,83 @@ +import typing as t + +from django.contrib.auth import get_user_model +from django.db.models import QuerySet +from fastapi import Depends, status +from pydantic import BaseModel + +from django_etebase import models +from .authentication import get_authenticated_user +from .msgpack import MsgpackResponse +from .utils import get_object_or_404 +from .stoken_handler import filter_by_stoken_and_limit + +from .collection import collection_router, get_collection_queryset + +User = get_user_model() +default_queryset: QuerySet = models.CollectionMember.objects.all() + + +def get_queryset(user: User, collection_uid: str, queryset=default_queryset) -> t.Tuple[models.Collection, QuerySet]: + collection = get_object_or_404(get_collection_queryset(user, models.Collection.objects), uid=collection_uid) + return collection, queryset.filter(collection=collection) + + +class CollectionMemberOut(BaseModel): + username: str + accessLevel: models.AccessLevels + + class Config: + orm_mode = True + + @classmethod + def from_orm(cls: t.Type["CollectionMemberOut"], obj: models.CollectionMember) -> "CollectionMemberOut": + return cls(username=obj.user.username, accessLevel=obj.accessLevel) + + +class MemberListResponse(BaseModel): + data: t.List[CollectionMemberOut] + iterator: t.Optional[str] + done: bool + + +@collection_router.get("/{collection_uid}/member/", response_model=MemberListResponse) +def member_list( + collection_uid: str, + iterator: t.Optional[str] = None, + limit: int = 50, + user: User = Depends(get_authenticated_user), +): + _, queryset = get_queryset(user, collection_uid) + queryset = queryset.order_by("id") + result, new_stoken_obj, done = filter_by_stoken_and_limit( + iterator, limit, queryset, models.CollectionMember.stoken_annotation + ) + new_stoken = new_stoken_obj and new_stoken_obj.uid + + ret = MemberListResponse( + data=[CollectionMemberOut.from_orm(item) for item in result], + iterator=new_stoken, + done=done, + ) + return MsgpackResponse(ret) + + +@collection_router.delete("/{collection_uid}/member/{username}/", status_code=status.HTTP_204_NO_CONTENT) +def member_delete( + collection_uid: str, + username: str, + user: User = Depends(get_authenticated_user), +): + _, queryset = get_queryset(user, collection_uid) + obj = get_object_or_404(queryset, user__username__iexact=username) + obj.revoke() + + +@collection_router.post("/{collection_uid}/member/leave/", status_code=status.HTTP_204_NO_CONTENT) +def member_leave( + collection_uid: str, + user: User = Depends(get_authenticated_user), +): + collection, _ = get_queryset(user, collection_uid) + obj = get_object_or_404(collection.members, user=user) + obj.revoke() From e8bd8927a01617a5a8841071de30aa0120ad4e4f Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 21:47:30 +0200 Subject: [PATCH 046/102] Implement modifying access level. --- etebase_fastapi/member.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index 36aa5ce..2eeb365 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -1,6 +1,7 @@ import typing as t from django.contrib.auth import get_user_model +from django.db import transaction from django.db.models import QuerySet from fastapi import Depends, status from pydantic import BaseModel @@ -22,6 +23,10 @@ def get_queryset(user: User, collection_uid: str, queryset=default_queryset) -> return collection, queryset.filter(collection=collection) +class CollectionMemberModifyAccessLevelIn(BaseModel): + accessLevel: models.AccessLevels + + class CollectionMemberOut(BaseModel): username: str accessLevel: models.AccessLevels @@ -73,6 +78,24 @@ def member_delete( obj.revoke() +@collection_router.patch("/{collection_uid}/member/{username}/", status_code=status.HTTP_204_NO_CONTENT) +def member_patch( + collection_uid: str, + username: str, + data: CollectionMemberModifyAccessLevelIn, + user: User = Depends(get_authenticated_user), +): + _, queryset = get_queryset(user, collection_uid) + instance = get_object_or_404(queryset, user__username__iexact=username) + + with transaction.atomic(): + # We only allow updating accessLevel + if instance.accessLevel != data.accessLevel: + instance.stoken = models.Stoken.objects.create() + instance.accessLevel = data.accessLevel + instance.save() + + @collection_router.post("/{collection_uid}/member/leave/", status_code=status.HTTP_204_NO_CONTENT) def member_leave( collection_uid: str, From fa0cd01a59095bb781663876f334523ed6f40b07 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 21:50:34 +0200 Subject: [PATCH 047/102] Authentication: implement part of get_dashboard_url. --- etebase_fastapi/authentication.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index a8fbbfe..13af2dd 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -268,6 +268,19 @@ async def change_password(data: ChangePassword, request: Request, user: User = D return Response(status_code=status.HTTP_204_NO_CONTENT) +@authentication_router.post("/dashboard_url/") +def dashboard_url(user: User = Depends(get_authenticated_user)): + # XXX-TOM + get_dashboard_url = app_settings.DASHBOARD_URL_FUNC + if get_dashboard_url is None: + raise ValidationError("not_supported", "This server doesn't have a user dashboard.") + + ret = { + "url": get_dashboard_url(request, *args, **kwargs), + } + return MsgpackResponse(ret) + + def signup_save(data: SignupIn, request: Request) -> User: user_data = data.user with transaction.atomic(): From 403d975934072ff62ed0d147d433578b75a104e8 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 21:58:58 +0200 Subject: [PATCH 048/102] Collection: fix dep handling. --- etebase_fastapi/collection.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 0af6056..5e987d8 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -137,8 +137,8 @@ class CollectionItemBulkGetIn(BaseModel): class ItemDepIn(BaseModel): - etag: str uid: str + etag: str def validate_db(self): item = models.CollectionItem.objects.get(uid=self.uid) @@ -153,12 +153,12 @@ class ItemDepIn(BaseModel): class ItemBatchIn(BaseModel): items: t.List[CollectionItemIn] - deps: t.Optional[ItemDepIn] + deps: t.Optional[t.List[ItemDepIn]] def validate_db(self): if self.deps is not None: - for key, _value in self.deps: - getattr(self.deps, key).validate_db() + for dep in self.deps: + dep.validate_db() @sync_to_async From 8160a333840aa0fe1450808395346f5d62f9793d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 22:13:36 +0200 Subject: [PATCH 049/102] Get collection queryset: remove param. --- etebase_fastapi/collection.py | 14 +++++++------- etebase_fastapi/member.py | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 5e987d8..1fc6f0a 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -194,14 +194,14 @@ def collection_list_common( return MsgpackResponse(content=ret) -def get_collection_queryset(user: User, queryset: QuerySet) -> QuerySet: - return queryset.filter(members__user=user) +def get_collection_queryset(user: User) -> QuerySet: + return default_queryset.filter(members__user=user) def get_item_queryset( user: User, collection_uid: str, queryset: QuerySet = default_item_queryset ) -> t.Tuple[models.Collection, QuerySet]: - collection = get_object_or_404(get_collection_queryset(user, models.Collection.objects), uid=collection_uid) + collection = get_object_or_404(get_collection_queryset(user), uid=collection_uid) # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') queryset = queryset.filter(collection__pk=collection.pk, revisions__current=True) @@ -216,7 +216,7 @@ async def list_multi( user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, ): - queryset = get_collection_queryset(user, default_queryset) + queryset = get_collection_queryset(user) # FIXME: Remove the isnull part once we attach collection types to all objects ("collection-type-migration") queryset = queryset.filter( @@ -234,7 +234,7 @@ async def collection_list( prefetch: Prefetch = PrefetchQuery, user: User = Depends(get_authenticated_user), ): - queryset = get_collection_queryset(user, default_queryset) + queryset = get_collection_queryset(user) return await collection_list_common(queryset, user, stoken, limit, prefetch) @@ -311,7 +311,7 @@ async def create(data: CollectionIn, user: User = Depends(get_authenticated_user @collection_router.get("/{uid}/") def collection_get(uid: str, user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery): - obj = get_collection_queryset(user, default_queryset).get(uid=uid) + obj = get_collection_queryset(user).get(uid=uid) ret = CollectionOut.from_orm_context(obj, Context(user, prefetch)) return MsgpackResponse(ret) @@ -402,7 +402,7 @@ async def item_list( def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid: str, validate_etag: bool): - queryset = get_collection_queryset(user, default_queryset) + queryset = get_collection_queryset(user) with transaction.atomic(): # We need this for locking the collection object collection_object = queryset.select_for_update().get(uid=uid) diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index 2eeb365..534cad1 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -19,7 +19,7 @@ default_queryset: QuerySet = models.CollectionMember.objects.all() def get_queryset(user: User, collection_uid: str, queryset=default_queryset) -> t.Tuple[models.Collection, QuerySet]: - collection = get_object_or_404(get_collection_queryset(user, models.Collection.objects), uid=collection_uid) + collection = get_object_or_404(get_collection_queryset(user), uid=collection_uid) return collection, queryset.filter(collection=collection) From df19887af7df5a78b394ec8d8182bc9cf82bfc4c Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 22:27:33 +0200 Subject: [PATCH 050/102] Use dependency injection for getting collection/item queryset. --- etebase_fastapi/collection.py | 44 +++++++++++++++++------------------ etebase_fastapi/invitation.py | 28 ++++++++++------------ etebase_fastapi/member.py | 34 +++++++++------------------ 3 files changed, 45 insertions(+), 61 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 1fc6f0a..196bb1d 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -194,18 +194,19 @@ def collection_list_common( return MsgpackResponse(content=ret) -def get_collection_queryset(user: User) -> QuerySet: +def get_collection_queryset(user: User = Depends(get_authenticated_user)) -> QuerySet: return default_queryset.filter(members__user=user) -def get_item_queryset( - user: User, collection_uid: str, queryset: QuerySet = default_item_queryset -) -> t.Tuple[models.Collection, QuerySet]: - collection = get_object_or_404(get_collection_queryset(user), uid=collection_uid) +def get_collection(collection_uid: str, queryset: QuerySet = Depends(get_collection_queryset)) -> models.Collection: + return get_object_or_404(queryset, uid=collection_uid) + + +def get_item_queryset(collection: models.Collection = Depends(get_collection)) -> QuerySet: # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') - queryset = queryset.filter(collection__pk=collection.pk, revisions__current=True) + queryset = default_item_queryset.filter(collection__pk=collection.pk, revisions__current=True) - return collection, queryset + return queryset @collection_router.post("/list_multi/") @@ -213,11 +214,10 @@ async def list_multi( data: ListMulti, stoken: t.Optional[str] = None, limit: int = 50, + queryset: QuerySet = Depends(get_collection_queryset), user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, ): - queryset = get_collection_queryset(user) - # FIXME: Remove the isnull part once we attach collection types to all objects ("collection-type-migration") queryset = queryset.filter( Q(members__collectionType__uid__in=data.collectionTypes) | Q(members__collectionType__isnull=True) @@ -228,13 +228,12 @@ async def list_multi( @collection_router.post("/list/") async def collection_list( - req: Request, stoken: t.Optional[str] = None, limit: int = 50, prefetch: Prefetch = PrefetchQuery, user: User = Depends(get_authenticated_user), + queryset: QuerySet = Depends(get_collection_queryset), ): - queryset = get_collection_queryset(user) return await collection_list_common(queryset, user, stoken, limit, prefetch) @@ -309,9 +308,12 @@ async def create(data: CollectionIn, user: User = Depends(get_authenticated_user return MsgpackResponse({}, status_code=status.HTTP_201_CREATED) -@collection_router.get("/{uid}/") -def collection_get(uid: str, user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery): - obj = get_collection_queryset(user).get(uid=uid) +@collection_router.get("/{collection_uid}/") +def collection_get( + obj: models.Collection = Depends(get_collection), + user: User = Depends(get_authenticated_user), + prefetch: Prefetch = PrefetchQuery + ): ret = CollectionOut.from_orm_context(obj, Context(user, prefetch)) return MsgpackResponse(ret) @@ -358,9 +360,10 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val @collection_router.get("/{collection_uid}/item/{uid}/") def item_get( - collection_uid: str, uid: str, user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery + uid: str, + queryset: QuerySet = Depends(get_item_queryset), + user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, ): - _, queryset = get_item_queryset(user, collection_uid) obj = queryset.get(uid=uid) ret = CollectionItemOut.from_orm_context(obj, Context(user, prefetch)) return MsgpackResponse(ret) @@ -386,14 +389,13 @@ def item_list_common( @collection_router.get("/{collection_uid}/item/") async def item_list( - collection_uid: str, + queryset: QuerySet = Depends(get_item_queryset), stoken: t.Optional[str] = None, limit: int = 50, prefetch: Prefetch = PrefetchQuery, withCollection: bool = False, user: User = Depends(get_authenticated_user), ): - _, queryset = await sync_to_async(get_item_queryset)(user, collection_uid) if not withCollection: queryset = queryset.filter(parent__isnull=True) @@ -419,14 +421,13 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid @collection_router.get("/{collection_uid}/item/{uid}/revision/") def item_revisions( - collection_uid: str, uid: str, limit: int = 50, iterator: t.Optional[str] = None, prefetch: Prefetch = PrefetchQuery, user: User = Depends(get_authenticated_user), + items: QuerySet = Depends(get_item_queryset), ): - _, items = get_item_queryset(user, collection_uid) item = get_object_or_404(items, uid=uid) queryset = item.revisions.order_by("-id") @@ -456,13 +457,12 @@ def item_revisions( @collection_router.post("/{collection_uid}/item/fetch_updates/") def fetch_updates( - collection_uid: str, data: t.List[CollectionItemBulkGetIn], stoken: t.Optional[str] = None, prefetch: Prefetch = PrefetchQuery, user: User = Depends(get_authenticated_user), + queryset: QuerySet = Depends(get_item_queryset), ): - _, queryset = get_item_queryset(user, collection_uid) # FIXME: make configurable? item_limit = 200 diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/invitation.py index 077dcfd..cbf0554 100644 --- a/etebase_fastapi/invitation.py +++ b/etebase_fastapi/invitation.py @@ -73,12 +73,12 @@ class InvitationListResponse(BaseModel): done: bool -def get_incoming_queryset(user: User, queryset=default_queryset): - return queryset.filter(user=user) +def get_incoming_queryset(user: User = Depends(get_authenticated_user)): + return default_queryset.filter(user=user) -def get_outgoing_queryset(user: User, queryset=default_queryset): - return queryset.filter(fromMember__user=user) +def get_outgoing_queryset(user: User = Depends(get_authenticated_user)): + return default_queryset.filter(fromMember__user=user) def list_common( @@ -114,17 +114,16 @@ def list_common( def incoming_list( iterator: t.Optional[str] = None, limit: int = 50, - user: User = Depends(get_authenticated_user), + queryset: QuerySet = Depends(get_incoming_queryset), ): - return list_common(get_incoming_queryset(user), iterator, limit) + return list_common(queryset, iterator, limit) @invitation_incoming_router.get("/{invitation_uid}/", response_model=CollectionInvitationOut) def incoming_get( invitation_uid: str, - user: User = Depends(get_authenticated_user), + queryset: QuerySet = Depends(get_incoming_queryset), ): - queryset = get_incoming_queryset(user) obj = get_object_or_404(queryset, uid=invitation_uid) ret = CollectionInvitationOut.from_orm(obj) return MsgpackResponse(ret) @@ -133,9 +132,8 @@ def incoming_get( @invitation_incoming_router.delete("/{invitation_uid}/", status_code=status.HTTP_204_NO_CONTENT) def incoming_delete( invitation_uid: str, - user: User = Depends(get_authenticated_user), + queryset: QuerySet = Depends(get_incoming_queryset), ): - queryset = get_incoming_queryset(user) obj = get_object_or_404(queryset, uid=invitation_uid) obj.delete() @@ -144,9 +142,8 @@ def incoming_delete( def incoming_accept( invitation_uid: str, data: CollectionInvitationAcceptIn, - user: User = Depends(get_authenticated_user), + queryset: QuerySet = Depends(get_incoming_queryset), ): - queryset = get_incoming_queryset(user) invitation = get_object_or_404(queryset, uid=invitation_uid) with transaction.atomic(): @@ -201,17 +198,16 @@ def outgoing_create( def outgoing_list( iterator: t.Optional[str] = None, limit: int = 50, - user: User = Depends(get_authenticated_user), + queryset: QuerySet = Depends(get_outgoing_queryset), ): - return list_common(get_outgoing_queryset(user), iterator, limit) + return list_common(queryset, iterator, limit) @invitation_outgoing_router.delete("/{invitation_uid}/", status_code=status.HTTP_204_NO_CONTENT) def outgoing_delete( invitation_uid: str, - user: User = Depends(get_authenticated_user), + queryset: QuerySet = Depends(get_outgoing_queryset), ): - queryset = get_outgoing_queryset(user) obj = get_object_or_404(queryset, uid=invitation_uid) obj.delete() diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index 534cad1..a491490 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -12,15 +12,18 @@ from .msgpack import MsgpackResponse from .utils import get_object_or_404 from .stoken_handler import filter_by_stoken_and_limit -from .collection import collection_router, get_collection_queryset +from .collection import collection_router, get_collection User = get_user_model() default_queryset: QuerySet = models.CollectionMember.objects.all() -def get_queryset(user: User, collection_uid: str, queryset=default_queryset) -> t.Tuple[models.Collection, QuerySet]: - collection = get_object_or_404(get_collection_queryset(user), uid=collection_uid) - return collection, queryset.filter(collection=collection) +def get_queryset(collection: models.Collection = Depends(get_collection)) -> QuerySet: + return default_queryset.filter(collection=collection) + + +def get_member(username: str, queryset: QuerySet = Depends(get_queryset)) -> QuerySet: + return get_object_or_404(queryset, user__username__iexact=username) class CollectionMemberModifyAccessLevelIn(BaseModel): @@ -47,12 +50,10 @@ class MemberListResponse(BaseModel): @collection_router.get("/{collection_uid}/member/", response_model=MemberListResponse) def member_list( - collection_uid: str, iterator: t.Optional[str] = None, limit: int = 50, - user: User = Depends(get_authenticated_user), + queryset: QuerySet = Depends(get_queryset), ): - _, queryset = get_queryset(user, collection_uid) queryset = queryset.order_by("id") result, new_stoken_obj, done = filter_by_stoken_and_limit( iterator, limit, queryset, models.CollectionMember.stoken_annotation @@ -69,25 +70,16 @@ def member_list( @collection_router.delete("/{collection_uid}/member/{username}/", status_code=status.HTTP_204_NO_CONTENT) def member_delete( - collection_uid: str, - username: str, - user: User = Depends(get_authenticated_user), + obj: models.CollectionMember = Depends(get_member), ): - _, queryset = get_queryset(user, collection_uid) - obj = get_object_or_404(queryset, user__username__iexact=username) obj.revoke() @collection_router.patch("/{collection_uid}/member/{username}/", status_code=status.HTTP_204_NO_CONTENT) def member_patch( - collection_uid: str, - username: str, data: CollectionMemberModifyAccessLevelIn, - user: User = Depends(get_authenticated_user), + instance: models.CollectionMember = Depends(get_member), ): - _, queryset = get_queryset(user, collection_uid) - instance = get_object_or_404(queryset, user__username__iexact=username) - with transaction.atomic(): # We only allow updating accessLevel if instance.accessLevel != data.accessLevel: @@ -97,10 +89,6 @@ def member_patch( @collection_router.post("/{collection_uid}/member/leave/", status_code=status.HTTP_204_NO_CONTENT) -def member_leave( - collection_uid: str, - user: User = Depends(get_authenticated_user), -): - collection, _ = get_queryset(user, collection_uid) +def member_leave(user: User = Depends(get_authenticated_user), collection: models.Collection = Depends(get_collection)): obj = get_object_or_404(collection.members, user=user) obj.revoke() From c7b8b0373a171e114213fa0f5337ac571ea19e3c Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 22:56:23 +0200 Subject: [PATCH 051/102] Add permissions. --- etebase_fastapi/collection.py | 28 ++++++++++++++++++++++++---- etebase_fastapi/member.py | 23 ++++++++++++++++++----- 2 files changed, 42 insertions(+), 9 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 196bb1d..13c53be 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -12,10 +12,10 @@ from pydantic import BaseModel from django_etebase import models from .authentication import get_authenticated_user -from .exceptions import ValidationError, transform_validation_error +from .exceptions import ValidationError, transform_validation_error, PermissionDenied from .msgpack import MsgpackRoute, MsgpackResponse from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken -from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery +from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery, is_collection_admin User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute) @@ -209,6 +209,26 @@ def get_item_queryset(collection: models.Collection = Depends(get_collection)) - return queryset +# permissions + + +def verify_collection_admin( + collection: models.Collection = Depends(get_collection), user: User = Depends(get_authenticated_user) +): + if not is_collection_admin(collection, user): + raise PermissionDenied("admin_access_required", "Only collection admins can perform this operation.") + + +def has_write_access( + collection: models.Collection = Depends(get_collection), user: User = Depends(get_authenticated_user) +): + member = collection.members.get(user=user) + if member.accessLevel == models.AccessLevels.READ_ONLY: + raise PermissionDenied("no_write_access", "You need write access to write to this collection") + + +# paths + @collection_router.post("/list_multi/") async def list_multi( data: ListMulti, @@ -489,14 +509,14 @@ def fetch_updates( return MsgpackResponse(ret) -@collection_router.post("/{collection_uid}/item/transaction/") +@collection_router.post("/{collection_uid}/item/transaction/", dependencies=[Depends(has_write_access)]) def item_transaction( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True) -@collection_router.post("/{collection_uid}/item/batch/") +@collection_router.post("/{collection_uid}/item/batch/", dependencies=[Depends(has_write_access)]) def item_batch( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index a491490..f3c77e5 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -12,7 +12,7 @@ from .msgpack import MsgpackResponse from .utils import get_object_or_404 from .stoken_handler import filter_by_stoken_and_limit -from .collection import collection_router, get_collection +from .collection import collection_router, get_collection, verify_collection_admin User = get_user_model() default_queryset: QuerySet = models.CollectionMember.objects.all() @@ -48,7 +48,9 @@ class MemberListResponse(BaseModel): done: bool -@collection_router.get("/{collection_uid}/member/", response_model=MemberListResponse) +@collection_router.get( + "/{collection_uid}/member/", response_model=MemberListResponse, dependencies=[Depends(verify_collection_admin)] +) def member_list( iterator: t.Optional[str] = None, limit: int = 50, @@ -68,14 +70,22 @@ def member_list( return MsgpackResponse(ret) -@collection_router.delete("/{collection_uid}/member/{username}/", status_code=status.HTTP_204_NO_CONTENT) +@collection_router.delete( + "/{collection_uid}/member/{username}/", + status_code=status.HTTP_204_NO_CONTENT, + dependencies=[Depends(verify_collection_admin)], +) def member_delete( obj: models.CollectionMember = Depends(get_member), ): obj.revoke() -@collection_router.patch("/{collection_uid}/member/{username}/", status_code=status.HTTP_204_NO_CONTENT) +@collection_router.patch( + "/{collection_uid}/member/{username}/", + status_code=status.HTTP_204_NO_CONTENT, + dependencies=[Depends(verify_collection_admin)], +) def member_patch( data: CollectionMemberModifyAccessLevelIn, instance: models.CollectionMember = Depends(get_member), @@ -88,7 +98,10 @@ def member_patch( instance.save() -@collection_router.post("/{collection_uid}/member/leave/", status_code=status.HTTP_204_NO_CONTENT) +@collection_router.post( + "/{collection_uid}/member/leave/", + status_code=status.HTTP_204_NO_CONTENT, +) def member_leave(user: User = Depends(get_authenticated_user), collection: models.Collection = Depends(get_collection)): obj = get_object_or_404(collection.members, user=user) obj.revoke() From 1c8684ee9280856d20d48ca742018dc34ac79995 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 23:06:25 +0200 Subject: [PATCH 052/102] Fix a FIXME. --- etebase_fastapi/test_reset_view.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/etebase_fastapi/test_reset_view.py b/etebase_fastapi/test_reset_view.py index 435a56e..73a803e 100644 --- a/etebase_fastapi/test_reset_view.py +++ b/etebase_fastapi/test_reset_view.py @@ -4,7 +4,7 @@ from django.db import transaction from django.shortcuts import get_object_or_404 from fastapi import APIRouter, Request, Response, status -from django_etebase.utils import get_user_queryset +from django_etebase.utils import get_user_queryset, CallbackContext from etebase_fastapi.authentication import SignupIn, signup_save from etebase_fastapi.msgpack import MsgpackRoute @@ -19,8 +19,7 @@ def reset(data: SignupIn, request: Request): return Response("Only allowed in debug mode.", status_code=status.HTTP_400_BAD_REQUEST) with transaction.atomic(): - # XXX-TOM - user_queryset = get_user_queryset(User.objects.all(), None) + user_queryset = get_user_queryset(User.objects.all(), CallbackContext(request.path_params)) user = get_object_or_404(user_queryset, username=data.user.username) # Only allow test users for extra safety if not getattr(user, User.USERNAME_FIELD).startswith("test_user"): From d63c34693f4a88717353f7e708f8fed58bfe1b48 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Sun, 27 Dec 2020 23:11:12 +0200 Subject: [PATCH 053/102] Change all item_uids to be called item_uids. --- etebase_fastapi/collection.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 13c53be..33f1d64 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -378,13 +378,13 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val return instance -@collection_router.get("/{collection_uid}/item/{uid}/") +@collection_router.get("/{collection_uid}/item/{item_uid}/") def item_get( - uid: str, + item_uid: str, queryset: QuerySet = Depends(get_item_queryset), user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, ): - obj = queryset.get(uid=uid) + obj = queryset.get(uid=item_uid) ret = CollectionItemOut.from_orm_context(obj, Context(user, prefetch)) return MsgpackResponse(ret) @@ -439,16 +439,16 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid return MsgpackResponse({}) -@collection_router.get("/{collection_uid}/item/{uid}/revision/") +@collection_router.get("/{collection_uid}/item/{item_uid}/revision/") def item_revisions( - uid: str, + item_uid: str, limit: int = 50, iterator: t.Optional[str] = None, prefetch: Prefetch = PrefetchQuery, user: User = Depends(get_authenticated_user), items: QuerySet = Depends(get_item_queryset), ): - item = get_object_or_404(items, uid=uid) + item = get_object_or_404(items, uid=item_uid) queryset = item.revisions.order_by("-id") From 15988235f27673398416b244fd40aa39009a89f1 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 08:42:48 +0200 Subject: [PATCH 054/102] Exclude unset fields so fix removedMemberships return value. --- etebase_fastapi/collection.py | 6 +++--- etebase_fastapi/msgpack.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 33f1d64..1c5ca14 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -176,8 +176,9 @@ def collection_list_common( context = Context(user, prefetch) data: t.List[CollectionOut] = [CollectionOut.from_orm_context(item, context) for item in result] + ret = CollectionListResponse(data=data, stoken=new_stoken, done=done) + stoken_obj = get_stoken_obj(stoken) - removedMemberships = None if stoken_obj is not None: # FIXME: honour limit? (the limit should be combined for data and this because of stoken) remed_qs = models.CollectionMemberRemoved.objects.filter(user=user, stoken__id__gt=stoken_obj.id) @@ -188,9 +189,8 @@ def collection_list_common( remed = remed_qs.values_list("collection__uid", flat=True) if len(remed) > 0: - removedMemberships = [{"uid": x} for x in remed] + ret.removedMemberships = [{"uid": x} for x in remed] - ret = CollectionListResponse(data=data, stoken=new_stoken, done=done, removedMemberships=removedMemberships) return MsgpackResponse(content=ret) diff --git a/etebase_fastapi/msgpack.py b/etebase_fastapi/msgpack.py index edffd7e..0c5cc30 100644 --- a/etebase_fastapi/msgpack.py +++ b/etebase_fastapi/msgpack.py @@ -24,7 +24,7 @@ class MsgpackResponse(Response): return b"" if isinstance(content, BaseModel): - content = content.dict() + content = content.dict(exclude_unset=True) return msgpack.packb(content, use_bin_type=True) From 9f26ecf27682b3a0cbea85c10f23bfccdcdacc73 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 09:04:45 +0200 Subject: [PATCH 055/102] API: add documentation tags. --- etebase_fastapi/app.py | 6 ++++-- etebase_fastapi/authentication.py | 2 +- etebase_fastapi/collection.py | 17 +++++++++-------- etebase_fastapi/invitation.py | 4 ++-- etebase_fastapi/member.py | 15 ++++++++------- etebase_fastapi/test_reset_view.py | 2 +- 6 files changed, 25 insertions(+), 21 deletions(-) diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index 755340c..ffea2fb 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -13,8 +13,8 @@ from fastapi import FastAPI, Request from .exceptions import CustomHttpException from .authentication import authentication_router -from .collection import collection_router -from . import member # noqa +from .collection import collection_router, item_router +from .member import member_router from .invitation import invitation_incoming_router, invitation_outgoing_router from .msgpack import MsgpackResponse @@ -23,6 +23,8 @@ VERSION = "v1" BASE_PATH = f"/api/{VERSION}" app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication") app.include_router(collection_router, prefix=f"{BASE_PATH}/collection") +app.include_router(item_router, prefix=f"{BASE_PATH}/collection") +app.include_router(member_router, prefix=f"{BASE_PATH}/collection") app.include_router(invitation_incoming_router, prefix=f"{BASE_PATH}/invitation/incoming") app.include_router(invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing") if settings.DEBUG: diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 13af2dd..f04753b 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -33,7 +33,7 @@ User = get_user_model() token_scheme = APIKeyHeader(name="Authorization") AUTO_REFRESH = True MIN_REFRESH_INTERVAL = 60 -authentication_router = APIRouter(route_class=MsgpackRoute) +authentication_router = APIRouter(route_class=MsgpackRoute, tags=["authentication"]) @dataclasses.dataclass(frozen=True) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 1c5ca14..ead113a 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -7,7 +7,7 @@ from django.core.files.base import ContentFile from django.db import transaction from django.db.models import Q from django.db.models import QuerySet -from fastapi import APIRouter, Depends, status, Request +from fastapi import APIRouter, Depends, status from pydantic import BaseModel from django_etebase import models @@ -18,7 +18,8 @@ from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_st from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery, is_collection_admin User = get_user_model() -collection_router = APIRouter(route_class=MsgpackRoute) +collection_router = APIRouter(route_class=MsgpackRoute, tags=["collection"]) +item_router = APIRouter(route_class=MsgpackRoute, tags=["item"]) default_queryset: QuerySet = models.Collection.objects.all() default_item_queryset: QuerySet = models.CollectionItem.objects.all() @@ -378,7 +379,7 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val return instance -@collection_router.get("/{collection_uid}/item/{item_uid}/") +@item_router.get("/{collection_uid}/item/{item_uid}/") def item_get( item_uid: str, queryset: QuerySet = Depends(get_item_queryset), @@ -407,7 +408,7 @@ def item_list_common( return MsgpackResponse(content=ret) -@collection_router.get("/{collection_uid}/item/") +@item_router.get("/{collection_uid}/item/") async def item_list( queryset: QuerySet = Depends(get_item_queryset), stoken: t.Optional[str] = None, @@ -439,7 +440,7 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid return MsgpackResponse({}) -@collection_router.get("/{collection_uid}/item/{item_uid}/revision/") +@item_router.get("/{collection_uid}/item/{item_uid}/revision/") def item_revisions( item_uid: str, limit: int = 50, @@ -475,7 +476,7 @@ def item_revisions( return MsgpackResponse(ret) -@collection_router.post("/{collection_uid}/item/fetch_updates/") +@item_router.post("/{collection_uid}/item/fetch_updates/") def fetch_updates( data: t.List[CollectionItemBulkGetIn], stoken: t.Optional[str] = None, @@ -509,14 +510,14 @@ def fetch_updates( return MsgpackResponse(ret) -@collection_router.post("/{collection_uid}/item/transaction/", dependencies=[Depends(has_write_access)]) +@item_router.post("/{collection_uid}/item/transaction/", dependencies=[Depends(has_write_access)]) def item_transaction( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True) -@collection_router.post("/{collection_uid}/item/batch/", dependencies=[Depends(has_write_access)]) +@item_router.post("/{collection_uid}/item/batch/", dependencies=[Depends(has_write_access)]) def item_batch( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/invitation.py index cbf0554..9b166ee 100644 --- a/etebase_fastapi/invitation.py +++ b/etebase_fastapi/invitation.py @@ -14,8 +14,8 @@ from .msgpack import MsgpackRoute, MsgpackResponse from .utils import get_object_or_404, Context, is_collection_admin User = get_user_model() -invitation_incoming_router = APIRouter(route_class=MsgpackRoute) -invitation_outgoing_router = APIRouter(route_class=MsgpackRoute) +invitation_incoming_router = APIRouter(route_class=MsgpackRoute, tags=["incoming invitation"]) +invitation_outgoing_router = APIRouter(route_class=MsgpackRoute, tags=["outgoing invitation"]) default_queryset: QuerySet = models.CollectionInvitation.objects.all() diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index f3c77e5..af349d1 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -3,18 +3,19 @@ import typing as t from django.contrib.auth import get_user_model from django.db import transaction from django.db.models import QuerySet -from fastapi import Depends, status +from fastapi import APIRouter, Depends, status from pydantic import BaseModel from django_etebase import models from .authentication import get_authenticated_user -from .msgpack import MsgpackResponse +from .msgpack import MsgpackRoute, MsgpackResponse from .utils import get_object_or_404 from .stoken_handler import filter_by_stoken_and_limit -from .collection import collection_router, get_collection, verify_collection_admin +from .collection import get_collection, verify_collection_admin User = get_user_model() +member_router = APIRouter(route_class=MsgpackRoute, tags=["member"]) default_queryset: QuerySet = models.CollectionMember.objects.all() @@ -48,7 +49,7 @@ class MemberListResponse(BaseModel): done: bool -@collection_router.get( +@member_router.get( "/{collection_uid}/member/", response_model=MemberListResponse, dependencies=[Depends(verify_collection_admin)] ) def member_list( @@ -70,7 +71,7 @@ def member_list( return MsgpackResponse(ret) -@collection_router.delete( +@member_router.delete( "/{collection_uid}/member/{username}/", status_code=status.HTTP_204_NO_CONTENT, dependencies=[Depends(verify_collection_admin)], @@ -81,7 +82,7 @@ def member_delete( obj.revoke() -@collection_router.patch( +@member_router.patch( "/{collection_uid}/member/{username}/", status_code=status.HTTP_204_NO_CONTENT, dependencies=[Depends(verify_collection_admin)], @@ -98,7 +99,7 @@ def member_patch( instance.save() -@collection_router.post( +@member_router.post( "/{collection_uid}/member/leave/", status_code=status.HTTP_204_NO_CONTENT, ) diff --git a/etebase_fastapi/test_reset_view.py b/etebase_fastapi/test_reset_view.py index 73a803e..f21fd84 100644 --- a/etebase_fastapi/test_reset_view.py +++ b/etebase_fastapi/test_reset_view.py @@ -8,7 +8,7 @@ from django_etebase.utils import get_user_queryset, CallbackContext from etebase_fastapi.authentication import SignupIn, signup_save from etebase_fastapi.msgpack import MsgpackRoute -test_reset_view_router = APIRouter(route_class=MsgpackRoute) +test_reset_view_router = APIRouter(route_class=MsgpackRoute, tags=["test helpers"]) User = get_user_model() From ee4e7cf498657ee78d3dbac6f8078085b2abc64d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 09:11:35 +0200 Subject: [PATCH 056/102] Unify the nested router prefix. --- etebase_fastapi/app.py | 5 +++-- etebase_fastapi/collection.py | 12 ++++++------ etebase_fastapi/member.py | 10 ++++------ 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index ffea2fb..a8b12d0 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -21,10 +21,11 @@ from .msgpack import MsgpackResponse app = FastAPI() VERSION = "v1" BASE_PATH = f"/api/{VERSION}" +COLLECTION_UID_MARKER = "{collection_uid}" app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication") app.include_router(collection_router, prefix=f"{BASE_PATH}/collection") -app.include_router(item_router, prefix=f"{BASE_PATH}/collection") -app.include_router(member_router, prefix=f"{BASE_PATH}/collection") +app.include_router(item_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}") +app.include_router(member_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}") app.include_router(invitation_incoming_router, prefix=f"{BASE_PATH}/invitation/incoming") app.include_router(invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing") if settings.DEBUG: diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index ead113a..c0efed1 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -379,7 +379,7 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val return instance -@item_router.get("/{collection_uid}/item/{item_uid}/") +@item_router.get("/item/{item_uid}/") def item_get( item_uid: str, queryset: QuerySet = Depends(get_item_queryset), @@ -408,7 +408,7 @@ def item_list_common( return MsgpackResponse(content=ret) -@item_router.get("/{collection_uid}/item/") +@item_router.get("/item/") async def item_list( queryset: QuerySet = Depends(get_item_queryset), stoken: t.Optional[str] = None, @@ -440,7 +440,7 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid return MsgpackResponse({}) -@item_router.get("/{collection_uid}/item/{item_uid}/revision/") +@item_router.get("/item/{item_uid}/revision/") def item_revisions( item_uid: str, limit: int = 50, @@ -476,7 +476,7 @@ def item_revisions( return MsgpackResponse(ret) -@item_router.post("/{collection_uid}/item/fetch_updates/") +@item_router.post("/item/fetch_updates/") def fetch_updates( data: t.List[CollectionItemBulkGetIn], stoken: t.Optional[str] = None, @@ -510,14 +510,14 @@ def fetch_updates( return MsgpackResponse(ret) -@item_router.post("/{collection_uid}/item/transaction/", dependencies=[Depends(has_write_access)]) +@item_router.post("/item/transaction/", dependencies=[Depends(has_write_access)]) def item_transaction( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True) -@item_router.post("/{collection_uid}/item/batch/", dependencies=[Depends(has_write_access)]) +@item_router.post("/item/batch/", dependencies=[Depends(has_write_access)]) def item_batch( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index af349d1..2c9b631 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -49,9 +49,7 @@ class MemberListResponse(BaseModel): done: bool -@member_router.get( - "/{collection_uid}/member/", response_model=MemberListResponse, dependencies=[Depends(verify_collection_admin)] -) +@member_router.get("/member/", response_model=MemberListResponse, dependencies=[Depends(verify_collection_admin)]) def member_list( iterator: t.Optional[str] = None, limit: int = 50, @@ -72,7 +70,7 @@ def member_list( @member_router.delete( - "/{collection_uid}/member/{username}/", + "/member/{username}/", status_code=status.HTTP_204_NO_CONTENT, dependencies=[Depends(verify_collection_admin)], ) @@ -83,7 +81,7 @@ def member_delete( @member_router.patch( - "/{collection_uid}/member/{username}/", + "/member/{username}/", status_code=status.HTTP_204_NO_CONTENT, dependencies=[Depends(verify_collection_admin)], ) @@ -100,7 +98,7 @@ def member_patch( @member_router.post( - "/{collection_uid}/member/leave/", + "/member/leave/", status_code=status.HTTP_204_NO_CONTENT, ) def member_leave(user: User = Depends(get_authenticated_user), collection: models.Collection = Depends(get_collection)): From 80d69a566325755107d810c6fb6c67147f32c9cc Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 09:25:28 +0200 Subject: [PATCH 057/102] Fix collection list and how we return API responses. --- etebase_fastapi/authentication.py | 28 +++++++++---------- etebase_fastapi/collection.py | 45 +++++++++++++------------------ etebase_fastapi/invitation.py | 18 +++++-------- etebase_fastapi/member.py | 8 +++--- etebase_fastapi/msgpack.py | 2 +- etebase_fastapi/utils.py | 8 ++++++ 6 files changed, 49 insertions(+), 60 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index f04753b..13a8884 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -16,7 +16,6 @@ from django.db import transaction from django.utils import timezone from fastapi import APIRouter, Depends, status, Request, Response from fastapi.security import APIKeyHeader -from pydantic import BaseModel from django_etebase import app_settings, models from django_etebase.exceptions import EtebaseValidationError @@ -27,7 +26,8 @@ from django_etebase.token_auth.models import get_default_expiry from django_etebase.utils import create_user, get_user_queryset, CallbackContext from django_etebase.views import msgpack_encode, msgpack_decode from .exceptions import AuthenticationFailed, transform_validation_error, ValidationError -from .msgpack import MsgpackResponse, MsgpackRoute +from .msgpack import MsgpackRoute +from .utils import BaseModel User = get_user_model() token_scheme = APIKeyHeader(name="Authorization") @@ -225,10 +225,10 @@ def validate_login_request( @authentication_router.get("/is_etebase/") async def is_etebase(): - return MsgpackResponse({}) + pass -@authentication_router.post("/login_challenge/") +@authentication_router.post("/login_challenge/", response_model=LoginChallengeOut) async def login_challenge(user: User = Depends(get_login_user)): enc_key = get_encryption_key(user.userinfo.salt) box = nacl.secret.SecretBox(enc_key) @@ -237,35 +237,31 @@ async def login_challenge(user: User = Depends(get_login_user)): "userId": user.id, } challenge = bytes(box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder)) - return MsgpackResponse( - LoginChallengeOut(salt=user.userinfo.salt, challenge=challenge, version=user.userinfo.version) - ) + return LoginChallengeOut(salt=user.userinfo.salt, challenge=challenge, version=user.userinfo.version) -@authentication_router.post("/login/") +@authentication_router.post("/login/", response_model=LoginOut) async def login(data: Login, request: Request): user = await get_login_user(LoginChallengeIn(username=data.response_data.username)) host = request.headers.get("Host") await validate_login_request(data.response_data, data, user, "login", host) data = await sync_to_async(LoginOut.from_orm)(user) await sync_to_async(user_logged_in.send)(sender=user.__class__, request=None, user=user) - return MsgpackResponse(content=data, status_code=status.HTTP_200_OK) + return data -@authentication_router.post("/logout/") +@authentication_router.post("/logout/", status_code=status.HTTP_204_NO_CONTENT) async def logout(request: Request, auth_data: AuthData = Depends(get_auth_data)): await sync_to_async(auth_data.token.delete)() # XXX-TOM await sync_to_async(user_logged_out.send)(sender=auth_data.user.__class__, request=None, user=auth_data.user) - return Response(status_code=status.HTTP_204_NO_CONTENT) -@authentication_router.post("/change_password/") +@authentication_router.post("/change_password/", status_code=status.HTTP_204_NO_CONTENT) async def change_password(data: ChangePassword, request: Request, user: User = Depends(get_authenticated_user)): host = request.headers.get("Host") await validate_login_request(data.response_data, data, user, "changePassword", host) await sync_to_async(save_changed_password)(data, user) - return Response(status_code=status.HTTP_204_NO_CONTENT) @authentication_router.post("/dashboard_url/") @@ -278,7 +274,7 @@ def dashboard_url(user: User = Depends(get_authenticated_user)): ret = { "url": get_dashboard_url(request, *args, **kwargs), } - return MsgpackResponse(ret) + return ret def signup_save(data: SignupIn, request: Request) -> User: @@ -311,10 +307,10 @@ def signup_save(data: SignupIn, request: Request) -> User: return instance -@authentication_router.post("/signup/") +@authentication_router.post("/signup/", response_model=LoginOut, status_code=status.HTTP_201_CREATED) async def signup(data: SignupIn, request: Request): user = await sync_to_async(signup_save)(data, request) # XXX-TOM data = await sync_to_async(LoginOut.from_orm)(user) await sync_to_async(user_signed_up.send)(sender=user.__class__, request=None, user=user) - return MsgpackResponse(content=data, status_code=status.HTTP_201_CREATED) + return data diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index c0efed1..993d144 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -8,14 +8,13 @@ from django.db import transaction from django.db.models import Q from django.db.models import QuerySet from fastapi import APIRouter, Depends, status -from pydantic import BaseModel from django_etebase import models from .authentication import get_authenticated_user from .exceptions import ValidationError, transform_validation_error, PermissionDenied -from .msgpack import MsgpackRoute, MsgpackResponse +from .msgpack import MsgpackRoute from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken -from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery, is_collection_admin +from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery, is_collection_admin, BaseModel User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute, tags=["collection"]) @@ -169,7 +168,7 @@ def collection_list_common( stoken: t.Optional[str], limit: int, prefetch: Prefetch, -) -> MsgpackResponse: +) -> CollectionListResponse: result, new_stoken_obj, done = filter_by_stoken_and_limit( stoken, limit, queryset, models.Collection.stoken_annotation ) @@ -192,7 +191,7 @@ def collection_list_common( if len(remed) > 0: ret.removedMemberships = [{"uid": x} for x in remed] - return MsgpackResponse(content=ret) + return ret def get_collection_queryset(user: User = Depends(get_authenticated_user)) -> QuerySet: @@ -230,7 +229,7 @@ def has_write_access( # paths -@collection_router.post("/list_multi/") +@collection_router.post("/list_multi/", response_model=CollectionListResponse, response_model_exclude_unset=True) async def list_multi( data: ListMulti, stoken: t.Optional[str] = None, @@ -247,7 +246,7 @@ async def list_multi( return await collection_list_common(queryset, user, stoken, limit, prefetch) -@collection_router.post("/list/") +@collection_router.get("/", response_model=CollectionListResponse) async def collection_list( stoken: t.Optional[str] = None, limit: int = 50, @@ -323,20 +322,18 @@ def _create(data: CollectionIn, user: User): ).save() -@collection_router.post("/") +@collection_router.post("/", status_code=status.HTTP_201_CREATED) async def create(data: CollectionIn, user: User = Depends(get_authenticated_user)): await sync_to_async(_create)(data, user) - return MsgpackResponse({}, status_code=status.HTTP_201_CREATED) -@collection_router.get("/{collection_uid}/") +@collection_router.get("/{collection_uid}/", response_model=CollectionOut) def collection_get( obj: models.Collection = Depends(get_collection), user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery ): - ret = CollectionOut.from_orm_context(obj, Context(user, prefetch)) - return MsgpackResponse(ret) + return CollectionOut.from_orm_context(obj, Context(user, prefetch)) def item_create(item_model: CollectionItemIn, collection: models.Collection, validate_etag: bool): @@ -379,15 +376,14 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val return instance -@item_router.get("/item/{item_uid}/") +@item_router.get("/item/{item_uid}/", response_model=CollectionItemOut) def item_get( item_uid: str, queryset: QuerySet = Depends(get_item_queryset), user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, ): obj = queryset.get(uid=item_uid) - ret = CollectionItemOut.from_orm_context(obj, Context(user, prefetch)) - return MsgpackResponse(ret) + return CollectionItemOut.from_orm_context(obj, Context(user, prefetch)) @sync_to_async @@ -397,18 +393,17 @@ def item_list_common( stoken: t.Optional[str], limit: int, prefetch: Prefetch, -) -> MsgpackResponse: +) -> CollectionItemListResponse: result, new_stoken_obj, done = filter_by_stoken_and_limit( stoken, limit, queryset, models.CollectionItem.stoken_annotation ) new_stoken = new_stoken_obj and new_stoken_obj.uid context = Context(user, prefetch) data: t.List[CollectionItemOut] = [CollectionItemOut.from_orm_context(item, context) for item in result] - ret = CollectionItemListResponse(data=data, stoken=new_stoken, done=done) - return MsgpackResponse(content=ret) + return CollectionItemListResponse(data=data, stoken=new_stoken, done=done) -@item_router.get("/item/") +@item_router.get("/item/", response_model=CollectionItemListResponse) async def item_list( queryset: QuerySet = Depends(get_item_queryset), stoken: t.Optional[str] = None, @@ -437,10 +432,10 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid for item in data.items: item_create(item, collection_object, validate_etag) - return MsgpackResponse({}) + return None -@item_router.get("/item/{item_uid}/revision/") +@item_router.get("/item/{item_uid}/revision/", response_model=CollectionItemRevisionListResponse) def item_revisions( item_uid: str, limit: int = 50, @@ -468,15 +463,14 @@ def item_revisions( ret_data = [CollectionItemRevisionInOut.from_orm_context(revision, context) for revision in result] iterator = ret_data[-1].uid if len(result) > 0 else None - ret = CollectionItemRevisionListResponse( + return CollectionItemRevisionListResponse( data=ret_data, iterator=iterator, done=done, ) - return MsgpackResponse(ret) -@item_router.post("/item/fetch_updates/") +@item_router.post("/item/fetch_updates/", response_model=CollectionItemListResponse) def fetch_updates( data: t.List[CollectionItemBulkGetIn], stoken: t.Optional[str] = None, @@ -502,12 +496,11 @@ def fetch_updates( new_stoken = new_stoken or stoken context = Context(user, prefetch) - ret = CollectionItemListResponse( + return CollectionItemListResponse( data=[CollectionItemOut.from_orm_context(item, context) for item in queryset], stoken=new_stoken, done=True, # we always return all the items, so it's always done ) - return MsgpackResponse(ret) @item_router.post("/item/transaction/", dependencies=[Depends(has_write_access)]) diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/invitation.py index 9b166ee..5c2c338 100644 --- a/etebase_fastapi/invitation.py +++ b/etebase_fastapi/invitation.py @@ -4,14 +4,13 @@ from django.contrib.auth import get_user_model from django.db import transaction, IntegrityError from django.db.models import QuerySet from fastapi import APIRouter, Depends, status, Request -from pydantic import BaseModel from django_etebase import models from django_etebase.utils import get_user_queryset, CallbackContext from .authentication import get_authenticated_user from .exceptions import ValidationError, PermissionDenied -from .msgpack import MsgpackRoute, MsgpackResponse -from .utils import get_object_or_404, Context, is_collection_admin +from .msgpack import MsgpackRoute +from .utils import get_object_or_404, Context, is_collection_admin, BaseModel User = get_user_model() invitation_incoming_router = APIRouter(route_class=MsgpackRoute, tags=["incoming invitation"]) @@ -85,7 +84,7 @@ def list_common( queryset: QuerySet, iterator: t.Optional[str], limit: int, -) -> MsgpackResponse: +) -> InvitationListResponse: queryset = queryset.order_by("id") if iterator is not None: @@ -102,12 +101,11 @@ def list_common( ret_data = result iterator = ret_data[-1].uid if len(result) > 0 else None - ret = InvitationListResponse( + return InvitationListResponse( data=ret_data, iterator=iterator, done=done, ) - return MsgpackResponse(ret) @invitation_incoming_router.get("/", response_model=InvitationListResponse) @@ -125,8 +123,7 @@ def incoming_get( queryset: QuerySet = Depends(get_incoming_queryset), ): obj = get_object_or_404(queryset, uid=invitation_uid) - ret = CollectionInvitationOut.from_orm(obj) - return MsgpackResponse(ret) + return CollectionInvitationOut.from_orm(obj) @invitation_incoming_router.delete("/{invitation_uid}/", status_code=status.HTTP_204_NO_CONTENT) @@ -191,8 +188,6 @@ def outgoing_create( except IntegrityError: raise ValidationError("invitation_exists", "Invitation already exists") - return MsgpackResponse(CollectionInvitationOut.from_orm(ret), status_code=status.HTTP_201_CREATED) - @invitation_outgoing_router.get("/", response_model=InvitationListResponse) def outgoing_list( @@ -221,5 +216,4 @@ def outgoing_fetch_user_profile( kwargs = {User.USERNAME_FIELD: username.lower()} user = get_object_or_404(get_user_queryset(User.objects.all(), CallbackContext(request.path_params)), **kwargs) user_info = get_object_or_404(models.UserInfo.objects.all(), owner=user) - ret = UserInfoOut.from_orm(user_info) - return MsgpackResponse(ret) + return UserInfoOut.from_orm(user_info) diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index 2c9b631..749092c 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -4,12 +4,11 @@ from django.contrib.auth import get_user_model from django.db import transaction from django.db.models import QuerySet from fastapi import APIRouter, Depends, status -from pydantic import BaseModel from django_etebase import models from .authentication import get_authenticated_user -from .msgpack import MsgpackRoute, MsgpackResponse -from .utils import get_object_or_404 +from .msgpack import MsgpackRoute +from .utils import get_object_or_404, BaseModel from .stoken_handler import filter_by_stoken_and_limit from .collection import get_collection, verify_collection_admin @@ -61,12 +60,11 @@ def member_list( ) new_stoken = new_stoken_obj and new_stoken_obj.uid - ret = MemberListResponse( + return MemberListResponse( data=[CollectionMemberOut.from_orm(item) for item in result], iterator=new_stoken, done=done, ) - return MsgpackResponse(ret) @member_router.delete( diff --git a/etebase_fastapi/msgpack.py b/etebase_fastapi/msgpack.py index 0c5cc30..edffd7e 100644 --- a/etebase_fastapi/msgpack.py +++ b/etebase_fastapi/msgpack.py @@ -24,7 +24,7 @@ class MsgpackResponse(Response): return b"" if isinstance(content, BaseModel): - content = content.dict(exclude_unset=True) + content = content.dict() return msgpack.packb(content, use_bin_type=True) diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index 150afe8..7168f87 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -2,6 +2,7 @@ import dataclasses import typing as t from fastapi import status, Query +from pydantic import BaseModel as PyBaseModel from django.db.models import QuerySet from django.core.exceptions import ObjectDoesNotExist @@ -17,6 +18,13 @@ Prefetch = t.Literal["auto", "medium"] PrefetchQuery = Query(default="auto") +class BaseModel(PyBaseModel): + class Config: + json_encoders = { + bytes: lambda x: x, + } + + @dataclasses.dataclass class Context: user: t.Optional[User] From 6517fc5db2dac325028bdb94f937333e9fd42b25 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 09:35:27 +0200 Subject: [PATCH 058/102] More route tags to a central place. --- etebase_fastapi/app.py | 12 ++++++------ etebase_fastapi/authentication.py | 2 +- etebase_fastapi/collection.py | 4 ++-- etebase_fastapi/invitation.py | 4 ++-- etebase_fastapi/member.py | 2 +- 5 files changed, 12 insertions(+), 12 deletions(-) diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index a8b12d0..2bbfc2a 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -22,12 +22,12 @@ app = FastAPI() VERSION = "v1" BASE_PATH = f"/api/{VERSION}" COLLECTION_UID_MARKER = "{collection_uid}" -app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication") -app.include_router(collection_router, prefix=f"{BASE_PATH}/collection") -app.include_router(item_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}") -app.include_router(member_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}") -app.include_router(invitation_incoming_router, prefix=f"{BASE_PATH}/invitation/incoming") -app.include_router(invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing") +app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication", tags=["authentication"]) +app.include_router(collection_router, prefix=f"{BASE_PATH}/collection", tags=["collection"]) +app.include_router(item_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["item"]) +app.include_router(member_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["member"]) +app.include_router(invitation_incoming_router, prefix=f"{BASE_PATH}/invitation/incoming", tags=["incoming invitation"]) +app.include_router(invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing", tags=["outgoing invitation"]) if settings.DEBUG: from .test_reset_view import test_reset_view_router diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 13a8884..742f101 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -33,7 +33,7 @@ User = get_user_model() token_scheme = APIKeyHeader(name="Authorization") AUTO_REFRESH = True MIN_REFRESH_INTERVAL = 60 -authentication_router = APIRouter(route_class=MsgpackRoute, tags=["authentication"]) +authentication_router = APIRouter(route_class=MsgpackRoute) @dataclasses.dataclass(frozen=True) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 993d144..8b69708 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -17,8 +17,8 @@ from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_st from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery, is_collection_admin, BaseModel User = get_user_model() -collection_router = APIRouter(route_class=MsgpackRoute, tags=["collection"]) -item_router = APIRouter(route_class=MsgpackRoute, tags=["item"]) +collection_router = APIRouter(route_class=MsgpackRoute) +item_router = APIRouter(route_class=MsgpackRoute) default_queryset: QuerySet = models.Collection.objects.all() default_item_queryset: QuerySet = models.CollectionItem.objects.all() diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/invitation.py index 5c2c338..ab0bf01 100644 --- a/etebase_fastapi/invitation.py +++ b/etebase_fastapi/invitation.py @@ -13,8 +13,8 @@ from .msgpack import MsgpackRoute from .utils import get_object_or_404, Context, is_collection_admin, BaseModel User = get_user_model() -invitation_incoming_router = APIRouter(route_class=MsgpackRoute, tags=["incoming invitation"]) -invitation_outgoing_router = APIRouter(route_class=MsgpackRoute, tags=["outgoing invitation"]) +invitation_incoming_router = APIRouter(route_class=MsgpackRoute) +invitation_outgoing_router = APIRouter(route_class=MsgpackRoute) default_queryset: QuerySet = models.CollectionInvitation.objects.all() diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index 749092c..26cfcff 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -14,7 +14,7 @@ from .stoken_handler import filter_by_stoken_and_limit from .collection import get_collection, verify_collection_admin User = get_user_model() -member_router = APIRouter(route_class=MsgpackRoute, tags=["member"]) +member_router = APIRouter(route_class=MsgpackRoute) default_queryset: QuerySet = models.CollectionMember.objects.all() From 34c548acda625c4dbfad57e4d6cd780d4d500250 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 09:47:37 +0200 Subject: [PATCH 059/102] Remove extra import. --- etebase_fastapi/exceptions.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/etebase_fastapi/exceptions.py b/etebase_fastapi/exceptions.py index fa76c45..b7bb0e9 100644 --- a/etebase_fastapi/exceptions.py +++ b/etebase_fastapi/exceptions.py @@ -59,9 +59,6 @@ class PermissionDenied(CustomHttpException): super().__init__(code=code, detail=detail, status_code=status_code) -from django_etebase.exceptions import EtebaseValidationError - - class ValidationError(CustomHttpException): def __init__( self, From a75d5479faef37c8d046ae0d5ebebb662d9d3645 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 09:51:27 +0200 Subject: [PATCH 060/102] Rename ValidationError to HttpError. --- etebase_fastapi/authentication.py | 16 ++++++++-------- etebase_fastapi/collection.py | 16 ++++++++-------- etebase_fastapi/exceptions.py | 21 +++++++++------------ etebase_fastapi/invitation.py | 6 +++--- etebase_fastapi/utils.py | 4 ++-- 5 files changed, 30 insertions(+), 33 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 742f101..5650652 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -25,7 +25,7 @@ from django_etebase.token_auth.models import AuthToken from django_etebase.token_auth.models import get_default_expiry from django_etebase.utils import create_user, get_user_queryset, CallbackContext from django_etebase.views import msgpack_encode, msgpack_decode -from .exceptions import AuthenticationFailed, transform_validation_error, ValidationError +from .exceptions import AuthenticationFailed, transform_validation_error, HttpError from .msgpack import MsgpackRoute from .utils import BaseModel @@ -207,20 +207,20 @@ def validate_login_request( challenge_data = msgpack_decode(box.decrypt(validated_data.challenge)) now = int(datetime.now().timestamp()) if validated_data.action != expected_action: - raise ValidationError("wrong_action", f'Expected "{challenge_sent_to_user.response}" but got something else') + raise HttpError("wrong_action", f'Expected "{challenge_sent_to_user.response}" but got something else') elif now - challenge_data["timestamp"] > app_settings.CHALLENGE_VALID_SECONDS: - raise ValidationError("challenge_expired", "Login challenge has expired") + raise HttpError("challenge_expired", "Login challenge has expired") elif challenge_data["userId"] != user.id: - raise ValidationError("wrong_user", "This challenge is for the wrong user") + raise HttpError("wrong_user", "This challenge is for the wrong user") elif not settings.DEBUG and validated_data.host.split(":", 1)[0] != host_from_request: - raise ValidationError( + raise HttpError( "wrong_host", f'Found wrong host name. Got: "{validated_data.host}" expected: "{host_from_request}"' ) verify_key = nacl.signing.VerifyKey(bytes(user.userinfo.loginPubkey), encoder=nacl.encoding.RawEncoder) try: verify_key.verify(challenge_sent_to_user.response, challenge_sent_to_user.signature) except nacl.exceptions.BadSignatureError: - raise ValidationError("login_bad_signature", "Wrong password for user.", status.HTTP_401_UNAUTHORIZED) + raise HttpError("login_bad_signature", "Wrong password for user.", status.HTTP_401_UNAUTHORIZED) @authentication_router.get("/is_etebase/") @@ -269,7 +269,7 @@ def dashboard_url(user: User = Depends(get_authenticated_user)): # XXX-TOM get_dashboard_url = app_settings.DASHBOARD_URL_FUNC if get_dashboard_url is None: - raise ValidationError("not_supported", "This server doesn't have a user dashboard.") + raise HttpError("not_supported", "This server doesn't have a user dashboard.") ret = { "url": get_dashboard_url(request, *args, **kwargs), @@ -301,7 +301,7 @@ def signup_save(data: SignupIn, request: Request) -> User: raise EtebaseValidationError("generic", str(e)) if hasattr(instance, "userinfo"): - raise ValidationError("user_exists", "User already exists", status_code=status.HTTP_409_CONFLICT) + raise HttpError("user_exists", "User already exists", status_code=status.HTTP_409_CONFLICT) models.UserInfo.objects.create(**data.dict(exclude={"user"}), owner=instance) return instance diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 8b69708..7f01682 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -11,7 +11,7 @@ from fastapi import APIRouter, Depends, status from django_etebase import models from .authentication import get_authenticated_user -from .exceptions import ValidationError, transform_validation_error, PermissionDenied +from .exceptions import HttpError, transform_validation_error, PermissionDenied from .msgpack import MsgpackRoute from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery, is_collection_admin, BaseModel @@ -144,7 +144,7 @@ class ItemDepIn(BaseModel): item = models.CollectionItem.objects.get(uid=self.uid) etag = self.etag if item.etag != etag: - raise ValidationError( + raise HttpError( "wrong_etag", "Wrong etag. Expected {} got {}".format(item.etag, etag), status_code=status.HTTP_409_CONFLICT, @@ -274,7 +274,7 @@ def process_revisions_for_item(item: models.CollectionItem, revision_data: Colle chunk_obj.chunkFile.save("IGNORED", ContentFile(content)) chunk_obj.save() else: - raise ValidationError("chunk_no_content", "Tried to create a new chunk without content") + raise HttpError("chunk_no_content", "Tried to create a new chunk without content") chunks_objs.append(chunk_obj) @@ -290,12 +290,12 @@ def process_revisions_for_item(item: models.CollectionItem, revision_data: Colle def _create(data: CollectionIn, user: User): with transaction.atomic(): if data.item.etag is not None: - raise ValidationError("bad_etag", "etag is not null") + raise HttpError("bad_etag", "etag is not null") instance = models.Collection(uid=data.item.uid, owner=user) try: instance.validate_unique() except django_exceptions.ValidationError: - raise ValidationError( + raise HttpError( "unique_uid", "Collection with this uid already exists", status_code=status.HTTP_409_CONFLICT ) instance.save() @@ -355,7 +355,7 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val return instance if validate_etag and cur_etag != etag: - raise ValidationError( + raise HttpError( "wrong_etag", "Wrong etag. Expected {} got {}".format(cur_etag, etag), status_code=status.HTTP_409_CONFLICT, @@ -425,7 +425,7 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid collection_object = queryset.select_for_update().get(uid=uid) if stoken is not None and stoken != collection_object.stoken: - raise ValidationError("stale_stoken", "Stoken is too old", status_code=status.HTTP_409_CONFLICT) + raise HttpError("stale_stoken", "Stoken is too old", status_code=status.HTTP_409_CONFLICT) # XXX-TOM: make sure we return compatible errors data.validate_db() @@ -482,7 +482,7 @@ def fetch_updates( item_limit = 200 if len(data) > item_limit: - raise ValidationError("too_many_items", "Request has too many items.", status_code=status.HTTP_400_BAD_REQUEST) + raise HttpError("too_many_items", "Request has too many items.", status_code=status.HTTP_400_BAD_REQUEST) queryset, stoken_rev = filter_by_stoken(stoken, queryset, models.CollectionItem.stoken_annotation) diff --git a/etebase_fastapi/exceptions.py b/etebase_fastapi/exceptions.py index b7bb0e9..2c1757c 100644 --- a/etebase_fastapi/exceptions.py +++ b/etebase_fastapi/exceptions.py @@ -3,19 +3,17 @@ import typing as t from pydantic import BaseModel -from django_etebase.exceptions import EtebaseValidationError - -class ValidationErrorField(BaseModel): +class HttpErrorField(BaseModel): field: str code: str detail: str -class ValidationErrorOut(BaseModel): +class HttpErrorOut(BaseModel): code: str detail: str - errors: t.Optional[t.List[ValidationErrorField]] + errors: t.Optional[t.List[HttpErrorField]] class CustomHttpException(Exception): @@ -59,24 +57,23 @@ class PermissionDenied(CustomHttpException): super().__init__(code=code, detail=detail, status_code=status_code) -class ValidationError(CustomHttpException): +class HttpError(CustomHttpException): def __init__( self, code: str, detail: str, status_code: int = status.HTTP_400_BAD_REQUEST, - field: t.Optional[str] = None, - errors: t.Optional[t.List["ValidationError"]] = None, + errors: t.Optional[t.List["HttpError"]] = None, ): self.errors = errors super().__init__(code=code, detail=detail, status_code=status_code) @property def as_dict(self) -> dict: - return ValidationErrorOut(code=self.code, errors=self.errors, detail=self.detail).dict() + return HttpErrorOut(code=self.code, errors=self.errors, detail=self.detail).dict() -def flatten_errors(field_name, errors) -> t.List[ValidationError]: +def flatten_errors(field_name, errors) -> t.List[HttpError]: ret = [] if isinstance(errors, dict): for error_key in errors: @@ -98,5 +95,5 @@ def transform_validation_error(prefix, err): elif not hasattr(err, "message"): errors = flatten_errors(prefix, err.error_list) else: - raise EtebaseValidationError(err.code, err.message) - raise ValidationError(code="field_errors", detail="Field validations failed.", errors=errors) + raise HttpError(err.code, err.message) + raise HttpError(code="field_errors", detail="Field validations failed.", errors=errors) diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/invitation.py index ab0bf01..38b74d8 100644 --- a/etebase_fastapi/invitation.py +++ b/etebase_fastapi/invitation.py @@ -8,7 +8,7 @@ from fastapi import APIRouter, Depends, status, Request from django_etebase import models from django_etebase.utils import get_user_queryset, CallbackContext from .authentication import get_authenticated_user -from .exceptions import ValidationError, PermissionDenied +from .exceptions import HttpError, PermissionDenied from .msgpack import MsgpackRoute from .utils import get_object_or_404, Context, is_collection_admin, BaseModel @@ -42,7 +42,7 @@ class CollectionInvitationCommon(BaseModel): class CollectionInvitationIn(CollectionInvitationCommon): def validate_db(self, context: Context): if context.user.username == self.username.lower(): - raise ValidationError("no_self_invite", "Inviting yourself is not allowed") + raise HttpError("no_self_invite", "Inviting yourself is not allowed") class CollectionInvitationOut(CollectionInvitationCommon): @@ -186,7 +186,7 @@ def outgoing_create( **data.dict(exclude={"collection", "username"}), user=to_user, fromMember=member ) except IntegrityError: - raise ValidationError("invitation_exists", "Invitation already exists") + raise HttpError("invitation_exists", "Invitation already exists") @invitation_outgoing_router.get("/", response_model=InvitationListResponse) diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index 7168f87..6ea9513 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -10,7 +10,7 @@ from django.contrib.auth import get_user_model from django_etebase.models import AccessLevels -from .exceptions import ValidationError +from .exceptions import HttpError User = get_user_model() @@ -35,7 +35,7 @@ def get_object_or_404(queryset: QuerySet, **kwargs): try: return queryset.get(**kwargs) except ObjectDoesNotExist as e: - raise ValidationError("does_not_exist", str(e), status_code=status.HTTP_404_NOT_FOUND) + raise HttpError("does_not_exist", str(e), status_code=status.HTTP_404_NOT_FOUND) def is_collection_admin(collection, user): From 4b4be14d32330a6df16b51eb2c93d484402d157a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 10:00:35 +0200 Subject: [PATCH 061/102] Add more responses to the API. --- etebase_fastapi/authentication.py | 10 +++++----- etebase_fastapi/collection.py | 6 +++--- etebase_fastapi/invitation.py | 6 +++--- etebase_fastapi/member.py | 4 ++-- etebase_fastapi/utils.py | 6 +++++- 5 files changed, 18 insertions(+), 14 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 5650652..902b79b 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -14,7 +14,7 @@ from django.contrib.auth import get_user_model, user_logged_out, user_logged_in from django.core import exceptions as django_exceptions from django.db import transaction from django.utils import timezone -from fastapi import APIRouter, Depends, status, Request, Response +from fastapi import APIRouter, Depends, status, Request from fastapi.security import APIKeyHeader from django_etebase import app_settings, models @@ -27,7 +27,7 @@ from django_etebase.utils import create_user, get_user_queryset, CallbackContext from django_etebase.views import msgpack_encode, msgpack_decode from .exceptions import AuthenticationFailed, transform_validation_error, HttpError from .msgpack import MsgpackRoute -from .utils import BaseModel +from .utils import BaseModel, permission_responses User = get_user_model() token_scheme = APIKeyHeader(name="Authorization") @@ -250,21 +250,21 @@ async def login(data: Login, request: Request): return data -@authentication_router.post("/logout/", status_code=status.HTTP_204_NO_CONTENT) +@authentication_router.post("/logout/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses) async def logout(request: Request, auth_data: AuthData = Depends(get_auth_data)): await sync_to_async(auth_data.token.delete)() # XXX-TOM await sync_to_async(user_logged_out.send)(sender=auth_data.user.__class__, request=None, user=auth_data.user) -@authentication_router.post("/change_password/", status_code=status.HTTP_204_NO_CONTENT) +@authentication_router.post("/change_password/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses) async def change_password(data: ChangePassword, request: Request, user: User = Depends(get_authenticated_user)): host = request.headers.get("Host") await validate_login_request(data.response_data, data, user, "changePassword", host) await sync_to_async(save_changed_password)(data, user) -@authentication_router.post("/dashboard_url/") +@authentication_router.post("/dashboard_url/", responses=permission_responses) def dashboard_url(user: User = Depends(get_authenticated_user)): # XXX-TOM get_dashboard_url = app_settings.DASHBOARD_URL_FUNC diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 7f01682..fad49aa 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -14,11 +14,11 @@ from .authentication import get_authenticated_user from .exceptions import HttpError, transform_validation_error, PermissionDenied from .msgpack import MsgpackRoute from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken -from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery, is_collection_admin, BaseModel +from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery, is_collection_admin, BaseModel, permission_responses User = get_user_model() -collection_router = APIRouter(route_class=MsgpackRoute) -item_router = APIRouter(route_class=MsgpackRoute) +collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) +item_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) default_queryset: QuerySet = models.Collection.objects.all() default_item_queryset: QuerySet = models.CollectionItem.objects.all() diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/invitation.py index 38b74d8..1d8df94 100644 --- a/etebase_fastapi/invitation.py +++ b/etebase_fastapi/invitation.py @@ -10,11 +10,11 @@ from django_etebase.utils import get_user_queryset, CallbackContext from .authentication import get_authenticated_user from .exceptions import HttpError, PermissionDenied from .msgpack import MsgpackRoute -from .utils import get_object_or_404, Context, is_collection_admin, BaseModel +from .utils import get_object_or_404, Context, is_collection_admin, BaseModel, permission_responses User = get_user_model() -invitation_incoming_router = APIRouter(route_class=MsgpackRoute) -invitation_outgoing_router = APIRouter(route_class=MsgpackRoute) +invitation_incoming_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) +invitation_outgoing_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) default_queryset: QuerySet = models.CollectionInvitation.objects.all() diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index 26cfcff..8ffed9d 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -8,13 +8,13 @@ from fastapi import APIRouter, Depends, status from django_etebase import models from .authentication import get_authenticated_user from .msgpack import MsgpackRoute -from .utils import get_object_or_404, BaseModel +from .utils import get_object_or_404, BaseModel, permission_responses from .stoken_handler import filter_by_stoken_and_limit from .collection import get_collection, verify_collection_admin User = get_user_model() -member_router = APIRouter(route_class=MsgpackRoute) +member_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) default_queryset: QuerySet = models.CollectionMember.objects.all() diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index 6ea9513..487f03a 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -10,7 +10,7 @@ from django.contrib.auth import get_user_model from django_etebase.models import AccessLevels -from .exceptions import HttpError +from .exceptions import HttpError, HttpErrorOut User = get_user_model() @@ -41,3 +41,7 @@ def get_object_or_404(queryset: QuerySet, **kwargs): def is_collection_admin(collection, user): member = collection.members.filter(user=user).first() return (member is not None) and (member.accessLevel == AccessLevels.ADMIN) + + +response_model_dict = {"model": HttpErrorOut} +permission_responses = {403: response_model_dict} From b39f7951e292588092365964f7c2fc3f317c54d3 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 10:18:35 +0200 Subject: [PATCH 062/102] chunk first-type. --- etebase_fastapi/collection.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index fad49aa..20d79e0 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -515,3 +515,31 @@ def item_batch( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): return item_bulk_common(data, user, stoken, collection_uid, validate_etag=False) + + +# Chunks + + +@item_router.put("/item/{item_uid}/chunk/{chunk_uid}/", dependencies=[Depends(has_write_access)], status_code=status.HTTP_201_CREATED) +def chunk_update( + limit: int = 50, + iterator: t.Optional[str] = None, + prefetch: Prefetch = PrefetchQuery, + user: User = Depends(get_authenticated_user), + collection: models.Collection = Depends(get_collection), +): + # IGNORED FOR NOW: col_it = get_object_or_404(col.items, uid=collection_item_uid) + + data = { + "uid": chunk_uid, + "chunkFile": request.data["file"], + } + + serializer = self.get_serializer_class()(data=data) + serializer.is_valid(raise_exception=True) + try: + serializer.save(collection=col) + except IntegrityError: + return Response( + {"code": "chunk_exists", "detail": "Chunk already exists."}, status=status.HTTP_409_CONFLICT + ) From 959dc9b576ddeeb69de6f367fd381d217a1bccb7 Mon Sep 17 00:00:00 2001 From: Tal Leibman Date: Mon, 28 Dec 2020 10:27:49 +0200 Subject: [PATCH 063/102] minor fix --- etebase_fastapi/collection.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 20d79e0..6757ac3 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -70,7 +70,6 @@ class CollectionItemOut(CollectionItemCommon): uid=obj.uid, version=obj.version, encryptionKey=obj.encryptionKey, - etag=obj.etag, content=CollectionItemRevisionInOut.from_orm_context(obj.content, context), ) @@ -91,7 +90,7 @@ class CollectionOut(CollectionCommon): @classmethod def from_orm_context(cls: t.Type["CollectionOut"], obj: models.Collection, context: Context) -> "CollectionOut": - member: CollectionMember = obj.members.get(user=context.user) + member: models.CollectionMember = obj.members.get(user=context.user) collection_type = member.collectionType ret = cls( collectionType=collection_type and collection_type.uid, From 1a09393dcb425250fc92c73b3584884f4f96e6c8 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 10:29:47 +0200 Subject: [PATCH 064/102] Also add 401 to permission responses. --- etebase_fastapi/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index 487f03a..2ee3700 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -44,4 +44,4 @@ def is_collection_admin(collection, user): response_model_dict = {"model": HttpErrorOut} -permission_responses = {403: response_model_dict} +permission_responses = {401: response_model_dict, 403: response_model_dict} From 37f5a4509f700169cdd60e5f47ccc69034fc9302 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 10:41:22 +0200 Subject: [PATCH 065/102] Improve chunks type. --- etebase_fastapi/collection.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 6757ac3..3af5af0 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -31,7 +31,10 @@ class CollectionItemRevisionInOut(BaseModel): uid: str meta: bytes deleted: bool - chunks: t.List[t.Tuple[str, t.Optional[bytes]]] + chunks: t.List[t.Union[ + t.Tuple[str], + t.Tuple[str, bytes], + ]] class Config: orm_mode = True From cf7690a60f23bbaee363ad15f0415384bf6f037e Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 10:45:34 +0200 Subject: [PATCH 066/102] Remove usages of EtebaseValidationError. --- etebase_fastapi/authentication.py | 2 +- etebase_fastapi/stoken_handler.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 902b79b..a13cc51 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -298,7 +298,7 @@ def signup_save(data: SignupIn, request: Request) -> User: except django_exceptions.ValidationError as e: transform_validation_error("user", e) except Exception as e: - raise EtebaseValidationError("generic", str(e)) + raise HttpError("generic", str(e)) if hasattr(instance, "userinfo"): raise HttpError("user_exists", "User already exists", status_code=status.HTTP_409_CONFLICT) diff --git a/etebase_fastapi/stoken_handler.py b/etebase_fastapi/stoken_handler.py index a976830..76d348a 100644 --- a/etebase_fastapi/stoken_handler.py +++ b/etebase_fastapi/stoken_handler.py @@ -3,9 +3,10 @@ import typing as t from django.db.models import QuerySet from fastapi import status -from django_etebase.exceptions import EtebaseValidationError from django_etebase.models import Stoken +from .exceptions import HttpError + # TODO missing stoken_annotation type StokenAnnotation = t.Any @@ -15,7 +16,7 @@ def get_stoken_obj(stoken: t.Optional[str]) -> t.Optional[Stoken]: try: return Stoken.objects.get(uid=stoken) except Stoken.DoesNotExist: - raise EtebaseValidationError("bad_stoken", "Invalid stoken.", status_code=status.HTTP_400_BAD_REQUEST) + raise HttpError("bad_stoken", "Invalid stoken.", status_code=status.HTTP_400_BAD_REQUEST) return None From 38884fead8cf91783e0213a948459c98973bb7db Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 10:47:07 +0200 Subject: [PATCH 067/102] Revert "Improve chunks type." This reverts commit 37f5a4509f700169cdd60e5f47ccc69034fc9302. --- etebase_fastapi/collection.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 3af5af0..6757ac3 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -31,10 +31,7 @@ class CollectionItemRevisionInOut(BaseModel): uid: str meta: bytes deleted: bool - chunks: t.List[t.Union[ - t.Tuple[str], - t.Tuple[str, bytes], - ]] + chunks: t.List[t.Tuple[str, t.Optional[bytes]]] class Config: orm_mode = True From ad2205e59616c90256afbb908ed95afaa2c24482 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 10:57:40 +0200 Subject: [PATCH 068/102] Add trusted host middleware. --- etebase_fastapi/app.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index 2bbfc2a..75cb099 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -1,7 +1,6 @@ import os from django.core.wsgi import get_wsgi_application -from fastapi.middleware.cors import CORSMiddleware os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etebase_server.settings") application = get_wsgi_application() @@ -10,6 +9,8 @@ from django.conf import settings # Not at the top of the file because we first need to setup django from fastapi import FastAPI, Request +from fastapi.middleware.cors import CORSMiddleware +from fastapi.middleware.trustedhost import TrustedHostMiddleware from .exceptions import CustomHttpException from .authentication import authentication_router @@ -35,6 +36,7 @@ if settings.DEBUG: app.add_middleware( CORSMiddleware, allow_origin_regex="https?://.*", allow_credentials=True, allow_methods=["*"], allow_headers=["*"] ) +app.add_middleware(TrustedHostMiddleware, allowed_hosts=settings.ALLOWED_HOSTS) @app.exception_handler(CustomHttpException) From 295ae6f3d34b6a6732a7789081a59c45a7de4733 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 16:39:16 +0200 Subject: [PATCH 069/102] Update changelog. --- ChangeLog.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/ChangeLog.md b/ChangeLog.md index e3c8232..a74a8af 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,5 +1,13 @@ # Changelog +## Version 0.7.0 +* Chunks: improve the chunk download endpoint to use sendfile extensions +* Chunks: support not passing chunk content if exists +* Chunks: fix chunk uploading media type to accept everything +* Gracefull handle uploading the same revision +* Pass generic context to callbacks instead of the whole view +* Fix handling of some validation errors + ## Version 0.6.1 * Collection: save the UID on the model to use the db for enforcing uniqueness From 63afcc0830170834e59b4db17301743d3c636749 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 11:44:17 +0200 Subject: [PATCH 070/102] Mount the django application. --- etebase_fastapi/app.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/etebase_fastapi/app.py b/etebase_fastapi/app.py index 75cb099..6dffef4 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/app.py @@ -9,6 +9,7 @@ from django.conf import settings # Not at the top of the file because we first need to setup django from fastapi import FastAPI, Request +from fastapi.middleware.wsgi import WSGIMiddleware from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.trustedhost import TrustedHostMiddleware @@ -37,6 +38,7 @@ app.add_middleware( CORSMiddleware, allow_origin_regex="https?://.*", allow_credentials=True, allow_methods=["*"], allow_headers=["*"] ) app.add_middleware(TrustedHostMiddleware, allowed_hosts=settings.ALLOWED_HOSTS) +app.mount("/", WSGIMiddleware(application)) @app.exception_handler(CustomHttpException) From 6c05a7898a14bd81f42ac0fd7c6760785ecb1d78 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 11:49:20 +0200 Subject: [PATCH 071/102] Add functions to split read and write permissions. --- django_etebase/app_settings.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/django_etebase/app_settings.py b/django_etebase/app_settings.py index 7c93f5f..c1e8dc9 100644 --- a/django_etebase/app_settings.py +++ b/django_etebase/app_settings.py @@ -33,14 +33,36 @@ class AppSettings: @cached_property def API_PERMISSIONS(self): # pylint: disable=invalid-name + """ + Deprecated. Do not use. + """ perms = self._setting("API_PERMISSIONS", ("rest_framework.permissions.IsAuthenticated",)) ret = [] for perm in perms: ret.append(self.import_from_str(perm)) return ret + @cached_property + def API_PERMISSIONS_READ(self): # pylint: disable=invalid-name + perms = self._setting("API_PERMISSIONS_READ", tuple()) + ret = [] + for perm in perms: + ret.append(self.import_from_str(perm)) + return ret + + @cached_property + def API_PERMISSIONS_WRITE(self): # pylint: disable=invalid-name + perms = self._setting("API_PERMISSIONS_WRITE", tuple()) + ret = [] + for perm in perms: + ret.append(self.import_from_str(perm)) + return ret + @cached_property def API_AUTHENTICATORS(self): # pylint: disable=invalid-name + """ + Deprecated. Do not use. + """ perms = self._setting( "API_AUTHENTICATORS", ( From b081d0129fafb02434fe94be69e3216f3ad74ac7 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 12:12:00 +0200 Subject: [PATCH 072/102] Add support for read/write permissions. --- etebase_fastapi/collection.py | 61 +++++++++++++++++++++++------------ etebase_fastapi/invitation.py | 34 +++++++++++++------ etebase_fastapi/member.py | 15 ++++----- etebase_fastapi/utils.py | 7 +++- etebase_server/settings.py | 5 --- 5 files changed, 79 insertions(+), 43 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 6757ac3..e6c10c3 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -14,7 +14,17 @@ from .authentication import get_authenticated_user from .exceptions import HttpError, transform_validation_error, PermissionDenied from .msgpack import MsgpackRoute from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken -from .utils import get_object_or_404, Context, Prefetch, PrefetchQuery, is_collection_admin, BaseModel, permission_responses +from .utils import ( + get_object_or_404, + Context, + Prefetch, + PrefetchQuery, + is_collection_admin, + BaseModel, + permission_responses, + PERMISSIONS_READ, + PERMISSIONS_READWRITE, +) User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) @@ -228,7 +238,13 @@ def has_write_access( # paths -@collection_router.post("/list_multi/", response_model=CollectionListResponse, response_model_exclude_unset=True) + +@collection_router.post( + "/list_multi/", + response_model=CollectionListResponse, + response_model_exclude_unset=True, + dependencies=PERMISSIONS_READ, +) async def list_multi( data: ListMulti, stoken: t.Optional[str] = None, @@ -245,7 +261,7 @@ async def list_multi( return await collection_list_common(queryset, user, stoken, limit, prefetch) -@collection_router.get("/", response_model=CollectionListResponse) +@collection_router.get("/", response_model=CollectionListResponse, dependencies=PERMISSIONS_READ) async def collection_list( stoken: t.Optional[str] = None, limit: int = 50, @@ -321,17 +337,17 @@ def _create(data: CollectionIn, user: User): ).save() -@collection_router.post("/", status_code=status.HTTP_201_CREATED) +@collection_router.post("/", status_code=status.HTTP_201_CREATED, dependencies=PERMISSIONS_READWRITE) async def create(data: CollectionIn, user: User = Depends(get_authenticated_user)): await sync_to_async(_create)(data, user) -@collection_router.get("/{collection_uid}/", response_model=CollectionOut) +@collection_router.get("/{collection_uid}/", response_model=CollectionOut, dependencies=PERMISSIONS_READ) def collection_get( - obj: models.Collection = Depends(get_collection), - user: User = Depends(get_authenticated_user), - prefetch: Prefetch = PrefetchQuery - ): + obj: models.Collection = Depends(get_collection), + user: User = Depends(get_authenticated_user), + prefetch: Prefetch = PrefetchQuery, +): return CollectionOut.from_orm_context(obj, Context(user, prefetch)) @@ -375,11 +391,12 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val return instance -@item_router.get("/item/{item_uid}/", response_model=CollectionItemOut) +@item_router.get("/item/{item_uid}/", response_model=CollectionItemOut, dependencies=PERMISSIONS_READ) def item_get( item_uid: str, queryset: QuerySet = Depends(get_item_queryset), - user: User = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, + user: User = Depends(get_authenticated_user), + prefetch: Prefetch = PrefetchQuery, ): obj = queryset.get(uid=item_uid) return CollectionItemOut.from_orm_context(obj, Context(user, prefetch)) @@ -402,7 +419,7 @@ def item_list_common( return CollectionItemListResponse(data=data, stoken=new_stoken, done=done) -@item_router.get("/item/", response_model=CollectionItemListResponse) +@item_router.get("/item/", response_model=CollectionItemListResponse, dependencies=PERMISSIONS_READ) async def item_list( queryset: QuerySet = Depends(get_item_queryset), stoken: t.Optional[str] = None, @@ -434,7 +451,9 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid return None -@item_router.get("/item/{item_uid}/revision/", response_model=CollectionItemRevisionListResponse) +@item_router.get( + "/item/{item_uid}/revision/", response_model=CollectionItemRevisionListResponse, dependencies=PERMISSIONS_READ +) def item_revisions( item_uid: str, limit: int = 50, @@ -469,7 +488,7 @@ def item_revisions( ) -@item_router.post("/item/fetch_updates/", response_model=CollectionItemListResponse) +@item_router.post("/item/fetch_updates/", response_model=CollectionItemListResponse, dependencies=PERMISSIONS_READ) def fetch_updates( data: t.List[CollectionItemBulkGetIn], stoken: t.Optional[str] = None, @@ -502,14 +521,14 @@ def fetch_updates( ) -@item_router.post("/item/transaction/", dependencies=[Depends(has_write_access)]) +@item_router.post("/item/transaction/", dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE]) def item_transaction( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True) -@item_router.post("/item/batch/", dependencies=[Depends(has_write_access)]) +@item_router.post("/item/batch/", dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE]) def item_batch( collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) ): @@ -519,7 +538,11 @@ def item_batch( # Chunks -@item_router.put("/item/{item_uid}/chunk/{chunk_uid}/", dependencies=[Depends(has_write_access)], status_code=status.HTTP_201_CREATED) +@item_router.put( + "/item/{item_uid}/chunk/{chunk_uid}/", + dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE], + status_code=status.HTTP_201_CREATED, +) def chunk_update( limit: int = 50, iterator: t.Optional[str] = None, @@ -539,6 +562,4 @@ def chunk_update( try: serializer.save(collection=col) except IntegrityError: - return Response( - {"code": "chunk_exists", "detail": "Chunk already exists."}, status=status.HTTP_409_CONFLICT - ) + return Response({"code": "chunk_exists", "detail": "Chunk already exists."}, status=status.HTTP_409_CONFLICT) diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/invitation.py index 1d8df94..39460a9 100644 --- a/etebase_fastapi/invitation.py +++ b/etebase_fastapi/invitation.py @@ -10,7 +10,15 @@ from django_etebase.utils import get_user_queryset, CallbackContext from .authentication import get_authenticated_user from .exceptions import HttpError, PermissionDenied from .msgpack import MsgpackRoute -from .utils import get_object_or_404, Context, is_collection_admin, BaseModel, permission_responses +from .utils import ( + get_object_or_404, + Context, + is_collection_admin, + BaseModel, + permission_responses, + PERMISSIONS_READ, + PERMISSIONS_READWRITE, +) User = get_user_model() invitation_incoming_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) @@ -108,7 +116,7 @@ def list_common( ) -@invitation_incoming_router.get("/", response_model=InvitationListResponse) +@invitation_incoming_router.get("/", response_model=InvitationListResponse, dependencies=PERMISSIONS_READ) def incoming_list( iterator: t.Optional[str] = None, limit: int = 50, @@ -117,7 +125,9 @@ def incoming_list( return list_common(queryset, iterator, limit) -@invitation_incoming_router.get("/{invitation_uid}/", response_model=CollectionInvitationOut) +@invitation_incoming_router.get( + "/{invitation_uid}/", response_model=CollectionInvitationOut, dependencies=PERMISSIONS_READ +) def incoming_get( invitation_uid: str, queryset: QuerySet = Depends(get_incoming_queryset), @@ -126,7 +136,9 @@ def incoming_get( return CollectionInvitationOut.from_orm(obj) -@invitation_incoming_router.delete("/{invitation_uid}/", status_code=status.HTTP_204_NO_CONTENT) +@invitation_incoming_router.delete( + "/{invitation_uid}/", status_code=status.HTTP_204_NO_CONTENT, dependencies=PERMISSIONS_READWRITE +) def incoming_delete( invitation_uid: str, queryset: QuerySet = Depends(get_incoming_queryset), @@ -135,7 +147,9 @@ def incoming_delete( obj.delete() -@invitation_incoming_router.post("/{invitation_uid}/accept/", status_code=status.HTTP_201_CREATED) +@invitation_incoming_router.post( + "/{invitation_uid}/accept/", status_code=status.HTTP_201_CREATED, dependencies=PERMISSIONS_READWRITE +) def incoming_accept( invitation_uid: str, data: CollectionInvitationAcceptIn, @@ -161,7 +175,7 @@ def incoming_accept( invitation.delete() -@invitation_outgoing_router.post("/", status_code=status.HTTP_201_CREATED) +@invitation_outgoing_router.post("/", status_code=status.HTTP_201_CREATED, dependencies=PERMISSIONS_READWRITE) def outgoing_create( data: CollectionInvitationIn, request: Request, @@ -189,7 +203,7 @@ def outgoing_create( raise HttpError("invitation_exists", "Invitation already exists") -@invitation_outgoing_router.get("/", response_model=InvitationListResponse) +@invitation_outgoing_router.get("/", response_model=InvitationListResponse, dependencies=PERMISSIONS_READ) def outgoing_list( iterator: t.Optional[str] = None, limit: int = 50, @@ -198,7 +212,9 @@ def outgoing_list( return list_common(queryset, iterator, limit) -@invitation_outgoing_router.delete("/{invitation_uid}/", status_code=status.HTTP_204_NO_CONTENT) +@invitation_outgoing_router.delete( + "/{invitation_uid}/", status_code=status.HTTP_204_NO_CONTENT, dependencies=PERMISSIONS_READWRITE +) def outgoing_delete( invitation_uid: str, queryset: QuerySet = Depends(get_outgoing_queryset), @@ -207,7 +223,7 @@ def outgoing_delete( obj.delete() -@invitation_outgoing_router.get("/fetch_user_profile/", response_model=UserInfoOut) +@invitation_outgoing_router.get("/fetch_user_profile/", response_model=UserInfoOut, dependencies=PERMISSIONS_READ) def outgoing_fetch_user_profile( username: str, request: Request, diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index 8ffed9d..725d44b 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -8,7 +8,7 @@ from fastapi import APIRouter, Depends, status from django_etebase import models from .authentication import get_authenticated_user from .msgpack import MsgpackRoute -from .utils import get_object_or_404, BaseModel, permission_responses +from .utils import get_object_or_404, BaseModel, permission_responses, PERMISSIONS_READ, PERMISSIONS_READWRITE from .stoken_handler import filter_by_stoken_and_limit from .collection import get_collection, verify_collection_admin @@ -48,7 +48,9 @@ class MemberListResponse(BaseModel): done: bool -@member_router.get("/member/", response_model=MemberListResponse, dependencies=[Depends(verify_collection_admin)]) +@member_router.get( + "/member/", response_model=MemberListResponse, dependencies=[Depends(verify_collection_admin), *PERMISSIONS_READ] +) def member_list( iterator: t.Optional[str] = None, limit: int = 50, @@ -70,7 +72,7 @@ def member_list( @member_router.delete( "/member/{username}/", status_code=status.HTTP_204_NO_CONTENT, - dependencies=[Depends(verify_collection_admin)], + dependencies=[Depends(verify_collection_admin), *PERMISSIONS_READWRITE], ) def member_delete( obj: models.CollectionMember = Depends(get_member), @@ -81,7 +83,7 @@ def member_delete( @member_router.patch( "/member/{username}/", status_code=status.HTTP_204_NO_CONTENT, - dependencies=[Depends(verify_collection_admin)], + dependencies=[Depends(verify_collection_admin), *PERMISSIONS_READWRITE], ) def member_patch( data: CollectionMemberModifyAccessLevelIn, @@ -95,10 +97,7 @@ def member_patch( instance.save() -@member_router.post( - "/member/leave/", - status_code=status.HTTP_204_NO_CONTENT, -) +@member_router.post("/member/leave/", status_code=status.HTTP_204_NO_CONTENT, dependencies=PERMISSIONS_READ) def member_leave(user: User = Depends(get_authenticated_user), collection: models.Collection = Depends(get_collection)): obj = get_object_or_404(collection.members, user=user) obj.revoke() diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index 2ee3700..165163a 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -1,13 +1,14 @@ import dataclasses import typing as t -from fastapi import status, Query +from fastapi import status, Query, Depends from pydantic import BaseModel as PyBaseModel from django.db.models import QuerySet from django.core.exceptions import ObjectDoesNotExist from django.contrib.auth import get_user_model +from django_etebase import app_settings from django_etebase.models import AccessLevels from .exceptions import HttpError, HttpErrorOut @@ -43,5 +44,9 @@ def is_collection_admin(collection, user): return (member is not None) and (member.accessLevel == AccessLevels.ADMIN) +PERMISSIONS_READ = [Depends(x) for x in app_settings.API_PERMISSIONS_READ] +PERMISSIONS_READWRITE = PERMISSIONS_READ + [Depends(x) for x in app_settings.API_PERMISSIONS_WRITE] + + response_model_dict = {"model": HttpErrorOut} permission_responses = {401: response_model_dict, 403: response_model_dict} diff --git a/etebase_server/settings.py b/etebase_server/settings.py index 325dca9..46ad3c9 100644 --- a/etebase_server/settings.py +++ b/etebase_server/settings.py @@ -166,11 +166,6 @@ if any(os.path.isfile(x) for x in config_locations): if "database" in config: DATABASES = {"default": {x.upper(): y for x, y in config.items("database")}} -ETEBASE_API_PERMISSIONS = ("rest_framework.permissions.IsAuthenticated",) -ETEBASE_API_AUTHENTICATORS = ( - "django_etebase.token_auth.authentication.TokenAuthentication", - "rest_framework.authentication.SessionAuthentication", -) ETEBASE_CREATE_USER_FUNC = "django_etebase.utils.create_user_blocked" # Efficient file streaming (for large files) From 0fa2f2da3b0c1064c27b5e2851c147843c75a450 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 13:26:12 +0200 Subject: [PATCH 073/102] Make the fastapi application the main asgi one. --- etebase_fastapi/{app.py => main.py} | 13 +++---------- etebase_server/asgi.py | 21 +++++++++++---------- etebase_server/urls.py | 15 ++++++++++++++- etebase_server/wsgi.py | 16 ---------------- 4 files changed, 28 insertions(+), 37 deletions(-) rename etebase_fastapi/{app.py => main.py} (85%) delete mode 100644 etebase_server/wsgi.py diff --git a/etebase_fastapi/app.py b/etebase_fastapi/main.py similarity index 85% rename from etebase_fastapi/app.py rename to etebase_fastapi/main.py index 6dffef4..2c10854 100644 --- a/etebase_fastapi/app.py +++ b/etebase_fastapi/main.py @@ -1,15 +1,7 @@ -import os - -from django.core.wsgi import get_wsgi_application - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etebase_server.settings") -application = get_wsgi_application() - from django.conf import settings # Not at the top of the file because we first need to setup django from fastapi import FastAPI, Request -from fastapi.middleware.wsgi import WSGIMiddleware from fastapi.middleware.cors import CORSMiddleware from fastapi.middleware.trustedhost import TrustedHostMiddleware @@ -30,15 +22,16 @@ app.include_router(item_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_ app.include_router(member_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["member"]) app.include_router(invitation_incoming_router, prefix=f"{BASE_PATH}/invitation/incoming", tags=["incoming invitation"]) app.include_router(invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing", tags=["outgoing invitation"]) + if settings.DEBUG: - from .test_reset_view import test_reset_view_router + from etebase_fastapi.test_reset_view import test_reset_view_router app.include_router(test_reset_view_router, prefix=f"{BASE_PATH}/test/authentication") + app.add_middleware( CORSMiddleware, allow_origin_regex="https?://.*", allow_credentials=True, allow_methods=["*"], allow_headers=["*"] ) app.add_middleware(TrustedHostMiddleware, allowed_hosts=settings.ALLOWED_HOSTS) -app.mount("/", WSGIMiddleware(application)) @app.exception_handler(CustomHttpException) diff --git a/etebase_server/asgi.py b/etebase_server/asgi.py index 0bf63ec..92fad1c 100644 --- a/etebase_server/asgi.py +++ b/etebase_server/asgi.py @@ -1,16 +1,17 @@ -""" -ASGI config for etebase_server project. - -It exposes the ASGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ -""" - import os from django.core.asgi import get_asgi_application os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etebase_server.settings") +django_application = get_asgi_application() + + +def create_application(): + from etebase_fastapi.main import app + + app.mount("/", django_application) + + return app + -application = get_asgi_application() +application = create_application() diff --git a/etebase_server/urls.py b/etebase_server/urls.py index f285977..443763d 100644 --- a/etebase_server/urls.py +++ b/etebase_server/urls.py @@ -1,8 +1,12 @@ +import os + from django.conf import settings from django.conf.urls import include, url from django.contrib import admin -from django.urls import path +from django.urls import path, re_path from django.views.generic import TemplateView +from django.views.static import serve +from django.contrib.staticfiles import finders urlpatterns = [ url(r"^api/", include("django_etebase.urls")), @@ -14,3 +18,12 @@ if settings.DEBUG: urlpatterns += [ url(r"^api-auth/", include("rest_framework.urls", namespace="rest_framework")), ] + + def serve_static(request, path): + filename = finders.find(path) + dirname = os.path.dirname(filename) + basename = os.path.basename(filename) + + return serve(request, basename, dirname) + + urlpatterns += [re_path(r"^static/(?P.*)$", serve_static)] diff --git a/etebase_server/wsgi.py b/etebase_server/wsgi.py deleted file mode 100644 index 908f88c..0000000 --- a/etebase_server/wsgi.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -WSGI config for etebase_server project. - -It exposes the WSGI callable as a module-level variable named ``application``. - -For more information on this file, see -https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/ -""" - -import os - -from django.core.wsgi import get_wsgi_application - -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "etebase_server.settings") - -application = get_wsgi_application() From 4ceb42780ec3d8c475601e2e0b360fb50ceca21b Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 13:56:53 +0200 Subject: [PATCH 074/102] Remove unused django_etebase code. --- README.md | 2 +- django_etebase/admin.py | 3 - django_etebase/drf_msgpack/__init__.py | 0 django_etebase/drf_msgpack/apps.py | 5 - .../drf_msgpack/migrations/__init__.py | 0 django_etebase/drf_msgpack/parsers.py | 14 - django_etebase/drf_msgpack/renderers.py | 15 - django_etebase/drf_msgpack/views.py | 3 - django_etebase/exceptions.py | 12 - django_etebase/parsers.py | 14 - django_etebase/permissions.py | 93 -- django_etebase/renderers.py | 19 - django_etebase/serializers.py | 598 ------------ django_etebase/tests.py | 3 - django_etebase/token_auth/admin.py | 0 django_etebase/token_auth/authentication.py | 46 - django_etebase/urls.py | 30 - django_etebase/views.py | 861 ------------------ etebase_fastapi/authentication.py | 6 +- etebase_fastapi/utils.py | 9 + etebase_server/settings.py | 6 - etebase_server/urls.py | 6 +- requirements.in/base.txt | 6 +- requirements.txt | 6 +- 24 files changed, 15 insertions(+), 1742 deletions(-) delete mode 100644 django_etebase/admin.py delete mode 100644 django_etebase/drf_msgpack/__init__.py delete mode 100644 django_etebase/drf_msgpack/apps.py delete mode 100644 django_etebase/drf_msgpack/migrations/__init__.py delete mode 100644 django_etebase/drf_msgpack/parsers.py delete mode 100644 django_etebase/drf_msgpack/renderers.py delete mode 100644 django_etebase/drf_msgpack/views.py delete mode 100644 django_etebase/exceptions.py delete mode 100644 django_etebase/parsers.py delete mode 100644 django_etebase/permissions.py delete mode 100644 django_etebase/renderers.py delete mode 100644 django_etebase/serializers.py delete mode 100644 django_etebase/tests.py delete mode 100644 django_etebase/token_auth/admin.py delete mode 100644 django_etebase/token_auth/authentication.py delete mode 100644 django_etebase/urls.py delete mode 100644 django_etebase/views.py diff --git a/README.md b/README.md index 3e0bd53..1787a2f 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ Now you can initialise our django app. And you are done! You can now run the debug server just to see everything works as expected by running: ``` -./manage.py runserver 0.0.0.0:8000 +uvicorn etebase_server.asgi:application --port 8000 ``` Using the debug server in production is not recommended, so please read the following section for a proper deployment. diff --git a/django_etebase/admin.py b/django_etebase/admin.py deleted file mode 100644 index 8c38f3f..0000000 --- a/django_etebase/admin.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.contrib import admin - -# Register your models here. diff --git a/django_etebase/drf_msgpack/__init__.py b/django_etebase/drf_msgpack/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/django_etebase/drf_msgpack/apps.py b/django_etebase/drf_msgpack/apps.py deleted file mode 100644 index 22ea2c1..0000000 --- a/django_etebase/drf_msgpack/apps.py +++ /dev/null @@ -1,5 +0,0 @@ -from django.apps import AppConfig - - -class DrfMsgpackConfig(AppConfig): - name = "drf_msgpack" diff --git a/django_etebase/drf_msgpack/migrations/__init__.py b/django_etebase/drf_msgpack/migrations/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/django_etebase/drf_msgpack/parsers.py b/django_etebase/drf_msgpack/parsers.py deleted file mode 100644 index 0504a76..0000000 --- a/django_etebase/drf_msgpack/parsers.py +++ /dev/null @@ -1,14 +0,0 @@ -import msgpack - -from rest_framework.parsers import BaseParser -from rest_framework.exceptions import ParseError - - -class MessagePackParser(BaseParser): - media_type = "application/msgpack" - - def parse(self, stream, media_type=None, parser_context=None): - try: - return msgpack.unpackb(stream.read(), raw=False) - except Exception as exc: - raise ParseError("MessagePack parse error - %s" % str(exc)) diff --git a/django_etebase/drf_msgpack/renderers.py b/django_etebase/drf_msgpack/renderers.py deleted file mode 100644 index 35a4afa..0000000 --- a/django_etebase/drf_msgpack/renderers.py +++ /dev/null @@ -1,15 +0,0 @@ -import msgpack - -from rest_framework.renderers import BaseRenderer - - -class MessagePackRenderer(BaseRenderer): - media_type = "application/msgpack" - format = "msgpack" - render_style = "binary" - charset = None - - def render(self, data, media_type=None, renderer_context=None): - if data is None: - return b"" - return msgpack.packb(data, use_bin_type=True) diff --git a/django_etebase/drf_msgpack/views.py b/django_etebase/drf_msgpack/views.py deleted file mode 100644 index 91ea44a..0000000 --- a/django_etebase/drf_msgpack/views.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.shortcuts import render - -# Create your views here. diff --git a/django_etebase/exceptions.py b/django_etebase/exceptions.py deleted file mode 100644 index 437a71c..0000000 --- a/django_etebase/exceptions.py +++ /dev/null @@ -1,12 +0,0 @@ -from rest_framework import serializers, status - - -class EtebaseValidationError(serializers.ValidationError): - def __init__(self, code, detail, status_code=status.HTTP_400_BAD_REQUEST): - super().__init__( - { - "code": code, - "detail": detail, - } - ) - self.status_code = status_code diff --git a/django_etebase/parsers.py b/django_etebase/parsers.py deleted file mode 100644 index ed1e713..0000000 --- a/django_etebase/parsers.py +++ /dev/null @@ -1,14 +0,0 @@ -from rest_framework.parsers import FileUploadParser - - -class ChunkUploadParser(FileUploadParser): - """ - Parser for chunk upload data. - """ - - def get_filename(self, stream, media_type, parser_context): - """ - Detects the uploaded file name. - """ - view = parser_context["view"] - return parser_context["kwargs"][view.lookup_field] diff --git a/django_etebase/permissions.py b/django_etebase/permissions.py deleted file mode 100644 index 3c77d06..0000000 --- a/django_etebase/permissions.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright © 2017 Tom Hacohen -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, version 3. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -from rest_framework import permissions -from django_etebase.models import Collection, AccessLevels - - -def is_collection_admin(collection, user): - member = collection.members.filter(user=user).first() - return (member is not None) and (member.accessLevel == AccessLevels.ADMIN) - - -class IsCollectionAdmin(permissions.BasePermission): - """ - Custom permission to only allow owners of a collection to view it - """ - - message = { - "detail": "Only collection admins can perform this operation.", - "code": "admin_access_required", - } - - def has_permission(self, request, view): - collection_uid = view.kwargs["collection_uid"] - try: - collection = view.get_collection_queryset().get(main_item__uid=collection_uid) - return is_collection_admin(collection, request.user) - except Collection.DoesNotExist: - # If the collection does not exist, we want to 404 later, not permission denied. - return True - - -class IsCollectionAdminOrReadOnly(permissions.BasePermission): - """ - Custom permission to only allow owners of a collection to edit it - """ - - message = { - "detail": "Only collection admins can edit collections.", - "code": "admin_access_required", - } - - def has_permission(self, request, view): - collection_uid = view.kwargs.get("collection_uid", None) - - # Allow creating new collections - if collection_uid is None: - return True - - try: - collection = view.get_collection_queryset().get(main_item__uid=collection_uid) - if request.method in permissions.SAFE_METHODS: - return True - - return is_collection_admin(collection, request.user) - except Collection.DoesNotExist: - # If the collection does not exist, we want to 404 later, not permission denied. - return True - - -class HasWriteAccessOrReadOnly(permissions.BasePermission): - """ - Custom permission to restrict write - """ - - message = { - "detail": "You need write access to write to this collection", - "code": "no_write_access", - } - - def has_permission(self, request, view): - collection_uid = view.kwargs["collection_uid"] - try: - collection = view.get_collection_queryset().get(main_item__uid=collection_uid) - if request.method in permissions.SAFE_METHODS: - return True - else: - member = collection.members.get(user=request.user) - return member.accessLevel != AccessLevels.READ_ONLY - except Collection.DoesNotExist: - # If the collection does not exist, we want to 404 later, not permission denied. - return True diff --git a/django_etebase/renderers.py b/django_etebase/renderers.py deleted file mode 100644 index 0d359d3..0000000 --- a/django_etebase/renderers.py +++ /dev/null @@ -1,19 +0,0 @@ -from rest_framework.utils.encoders import JSONEncoder as DRFJSONEncoder -from rest_framework.renderers import JSONRenderer as DRFJSONRenderer - -from .serializers import b64encode - - -class JSONEncoder(DRFJSONEncoder): - def default(self, obj): - if isinstance(obj, bytes) or isinstance(obj, memoryview): - return b64encode(obj) - return super().default(obj) - - -class JSONRenderer(DRFJSONRenderer): - """ - Renderer which serializes to JSON with support for our base64 - """ - - encoder_class = JSONEncoder diff --git a/django_etebase/serializers.py b/django_etebase/serializers.py deleted file mode 100644 index 26ac5a7..0000000 --- a/django_etebase/serializers.py +++ /dev/null @@ -1,598 +0,0 @@ -# Copyright © 2017 Tom Hacohen -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, version 3. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import base64 - -from django.core.files.base import ContentFile -from django.core import exceptions as django_exceptions -from django.contrib.auth import get_user_model -from django.db import IntegrityError, transaction -from rest_framework import serializers, status -from . import models -from .utils import get_user_queryset, create_user, CallbackContext - -from .exceptions import EtebaseValidationError - -User = get_user_model() - - -def process_revisions_for_item(item, revision_data): - chunks_objs = [] - chunks = revision_data.pop("chunks_relation") - - revision = models.CollectionItemRevision(**revision_data, item=item) - revision.validate_unique() # Verify there aren't any validation issues - - for chunk in chunks: - uid = chunk[0] - chunk_obj = models.CollectionItemChunk.objects.filter(uid=uid).first() - content = chunk[1] if len(chunk) > 1 else None - # If the chunk already exists we assume it's fine. Otherwise, we upload it. - if chunk_obj is None: - if content is not None: - chunk_obj = models.CollectionItemChunk(uid=uid, collection=item.collection) - chunk_obj.chunkFile.save("IGNORED", ContentFile(content)) - chunk_obj.save() - else: - raise EtebaseValidationError("chunk_no_content", "Tried to create a new chunk without content") - - chunks_objs.append(chunk_obj) - - stoken = models.Stoken.objects.create() - revision.stoken = stoken - revision.save() - - for chunk in chunks_objs: - models.RevisionChunkRelation.objects.create(chunk=chunk, revision=revision) - return revision - - -def b64encode(value): - return base64.urlsafe_b64encode(value).decode("ascii").strip("=") - - -def b64decode(data): - data += "=" * ((4 - len(data) % 4) % 4) - return base64.urlsafe_b64decode(data) - - -def b64decode_or_bytes(data): - if isinstance(data, bytes): - return data - else: - return b64decode(data) - - -class BinaryBase64Field(serializers.Field): - def to_representation(self, value): - return value - - def to_internal_value(self, data): - return b64decode_or_bytes(data) - - -class CollectionEncryptionKeyField(BinaryBase64Field): - def get_attribute(self, instance): - request = self.context.get("request", None) - if request is not None: - return instance.members.get(user=request.user).encryptionKey - return None - - -class CollectionTypeField(BinaryBase64Field): - def get_attribute(self, instance): - request = self.context.get("request", None) - if request is not None: - collection_type = instance.members.get(user=request.user).collectionType - return collection_type and collection_type.uid - return None - - -class UserSlugRelatedField(serializers.SlugRelatedField): - def get_queryset(self): - view = self.context.get("view", None) - return get_user_queryset(super().get_queryset(), context=CallbackContext(view.kwargs)) - - def __init__(self, **kwargs): - super().__init__(slug_field=User.USERNAME_FIELD, **kwargs) - - def to_internal_value(self, data): - return super().to_internal_value(data.lower()) - - -class ChunksField(serializers.RelatedField): - def to_representation(self, obj): - obj = obj.chunk - if self.context.get("prefetch") == "auto": - with open(obj.chunkFile.path, "rb") as f: - return (obj.uid, f.read()) - else: - return (obj.uid,) - - def to_internal_value(self, data): - content = data[1] if len(data) > 1 else None - if data[0] is None: - raise EtebaseValidationError("no_null", "null is not allowed") - return (data[0], b64decode_or_bytes(content) if content is not None else None) - - -class BetterErrorsMixin: - @property - def errors(self): - nice = [] - errors = super().errors - for error_type in errors: - if error_type == "non_field_errors": - nice.extend(self.flatten_errors(None, errors[error_type])) - else: - nice.extend(self.flatten_errors(error_type, errors[error_type])) - if nice: - return {"code": "field_errors", "detail": "Field validations failed.", "errors": nice} - return {} - - def flatten_errors(self, field_name, errors): - ret = [] - if isinstance(errors, dict): - for error_key in errors: - error = errors[error_key] - ret.extend(self.flatten_errors("{}.{}".format(field_name, error_key), error)) - else: - for error in errors: - if getattr(error, "messages", None): - message = error.messages[0] - else: - message = str(error) - ret.append( - { - "field": field_name, - "code": error.code, - "detail": message, - } - ) - return ret - - def transform_validation_error(self, prefix, err): - if hasattr(err, "error_dict"): - errors = self.flatten_errors(prefix, err.error_dict) - elif not hasattr(err, "message"): - errors = self.flatten_errors(prefix, err.error_list) - else: - raise EtebaseValidationError(err.code, err.message) - - raise serializers.ValidationError( - { - "code": "field_errors", - "detail": "Field validations failed.", - "errors": errors, - } - ) - - -class CollectionItemChunkSerializer(BetterErrorsMixin, serializers.ModelSerializer): - class Meta: - model = models.CollectionItemChunk - fields = ("uid", "chunkFile") - - -class CollectionItemRevisionSerializer(BetterErrorsMixin, serializers.ModelSerializer): - chunks = ChunksField( - source="chunks_relation", - queryset=models.RevisionChunkRelation.objects.all(), - style={"base_template": "input.html"}, - many=True, - ) - meta = BinaryBase64Field() - - class Meta: - model = models.CollectionItemRevision - fields = ("chunks", "meta", "uid", "deleted") - extra_kwargs = { - "uid": {"validators": []}, # We deal with it in the serializers - } - - -class CollectionItemSerializer(BetterErrorsMixin, serializers.ModelSerializer): - encryptionKey = BinaryBase64Field(required=False, default=None, allow_null=True) - etag = serializers.CharField(allow_null=True, write_only=True) - content = CollectionItemRevisionSerializer(many=False) - - class Meta: - model = models.CollectionItem - fields = ("uid", "version", "encryptionKey", "content", "etag") - - def create(self, validated_data): - """Function that's called when this serializer creates an item""" - validate_etag = self.context.get("validate_etag", False) - etag = validated_data.pop("etag") - revision_data = validated_data.pop("content") - uid = validated_data.pop("uid") - - Model = self.__class__.Meta.model - - with transaction.atomic(): - instance, created = Model.objects.get_or_create(uid=uid, defaults=validated_data) - cur_etag = instance.etag if not created else None - - # If we are trying to update an up to date item, abort early and consider it a success - if cur_etag == revision_data.get("uid"): - return instance - - if validate_etag and cur_etag != etag: - raise EtebaseValidationError( - "wrong_etag", - "Wrong etag. Expected {} got {}".format(cur_etag, etag), - status_code=status.HTTP_409_CONFLICT, - ) - - if not created: - # We don't have to use select_for_update here because the unique constraint on current guards against - # the race condition. But it's a good idea because it'll lock and wait rather than fail. - current_revision = instance.revisions.filter(current=True).select_for_update().first() - - # If we are just re-uploading the same revision, consider it a succes and return. - if current_revision.uid == revision_data.get("uid"): - return instance - - current_revision.current = None - current_revision.save() - - try: - process_revisions_for_item(instance, revision_data) - except django_exceptions.ValidationError as e: - self.transform_validation_error("content", e) - - return instance - - def update(self, instance, validated_data): - # We never update, we always update in the create method - raise NotImplementedError() - - -class CollectionItemDepSerializer(BetterErrorsMixin, serializers.ModelSerializer): - etag = serializers.CharField() - - class Meta: - model = models.CollectionItem - fields = ("uid", "etag") - - def validate(self, data): - item = self.__class__.Meta.model.objects.get(uid=data["uid"]) - etag = data["etag"] - if item.etag != etag: - raise EtebaseValidationError( - "wrong_etag", - "Wrong etag. Expected {} got {}".format(item.etag, etag), - status_code=status.HTTP_409_CONFLICT, - ) - - return data - - -class CollectionItemBulkGetSerializer(BetterErrorsMixin, serializers.ModelSerializer): - etag = serializers.CharField(required=False) - - class Meta: - model = models.CollectionItem - fields = ("uid", "etag") - - -class CollectionListMultiSerializer(BetterErrorsMixin, serializers.Serializer): - collectionTypes = serializers.ListField(child=BinaryBase64Field()) - - -class CollectionSerializer(BetterErrorsMixin, serializers.ModelSerializer): - collectionKey = CollectionEncryptionKeyField() - collectionType = CollectionTypeField() - accessLevel = serializers.SerializerMethodField("get_access_level_from_context") - stoken = serializers.CharField(read_only=True) - - item = CollectionItemSerializer(many=False, source="main_item") - - class Meta: - model = models.Collection - fields = ("item", "accessLevel", "collectionKey", "collectionType", "stoken") - - def get_access_level_from_context(self, obj): - request = self.context.get("request", None) - if request is not None: - return obj.members.get(user=request.user).accessLevel - return None - - def create(self, validated_data): - """Function that's called when this serializer creates an item""" - collection_key = validated_data.pop("collectionKey") - collection_type = validated_data.pop("collectionType") - - user = validated_data.get("owner") - main_item_data = validated_data.pop("main_item") - uid = main_item_data.get("uid") - etag = main_item_data.pop("etag") - revision_data = main_item_data.pop("content") - - instance = self.__class__.Meta.model(uid=uid, **validated_data) - - with transaction.atomic(): - if etag is not None: - raise EtebaseValidationError("bad_etag", "etag is not null") - - try: - instance.validate_unique() - except django_exceptions.ValidationError: - raise EtebaseValidationError( - "unique_uid", "Collection with this uid already exists", status_code=status.HTTP_409_CONFLICT - ) - instance.save() - - main_item = models.CollectionItem.objects.create(**main_item_data, collection=instance) - - instance.main_item = main_item - instance.save() - - process_revisions_for_item(main_item, revision_data) - - collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=collection_type, owner=user) - - models.CollectionMember( - collection=instance, - stoken=models.Stoken.objects.create(), - user=user, - accessLevel=models.AccessLevels.ADMIN, - encryptionKey=collection_key, - collectionType=collection_type_obj, - ).save() - - return instance - - def update(self, instance, validated_data): - raise NotImplementedError() - - -class CollectionMemberSerializer(BetterErrorsMixin, serializers.ModelSerializer): - username = UserSlugRelatedField( - source="user", - read_only=True, - style={"base_template": "input.html"}, - ) - - class Meta: - model = models.CollectionMember - fields = ("username", "accessLevel") - - def create(self, validated_data): - raise NotImplementedError() - - def update(self, instance, validated_data): - with transaction.atomic(): - # We only allow updating accessLevel - access_level = validated_data.pop("accessLevel") - if instance.accessLevel != access_level: - instance.stoken = models.Stoken.objects.create() - instance.accessLevel = access_level - instance.save() - - return instance - - -class CollectionInvitationSerializer(BetterErrorsMixin, serializers.ModelSerializer): - username = UserSlugRelatedField( - source="user", - queryset=User.objects, - style={"base_template": "input.html"}, - ) - collection = serializers.CharField(source="collection.uid") - fromUsername = serializers.CharField(source="fromMember.user.username", read_only=True) - fromPubkey = BinaryBase64Field(source="fromMember.user.userinfo.pubkey", read_only=True) - signedEncryptionKey = BinaryBase64Field() - - class Meta: - model = models.CollectionInvitation - fields = ( - "username", - "uid", - "collection", - "signedEncryptionKey", - "accessLevel", - "fromUsername", - "fromPubkey", - "version", - ) - - def validate_user(self, value): - request = self.context["request"] - - if request.user.username == value.lower(): - raise EtebaseValidationError("no_self_invite", "Inviting yourself is not allowed") - return value - - def create(self, validated_data): - request = self.context["request"] - collection = validated_data.pop("collection") - - member = collection.members.get(user=request.user) - - with transaction.atomic(): - try: - return type(self).Meta.model.objects.create(**validated_data, fromMember=member) - except IntegrityError: - raise EtebaseValidationError("invitation_exists", "Invitation already exists") - - def update(self, instance, validated_data): - with transaction.atomic(): - instance.accessLevel = validated_data.pop("accessLevel") - instance.signedEncryptionKey = validated_data.pop("signedEncryptionKey") - instance.save() - - return instance - - -class InvitationAcceptSerializer(BetterErrorsMixin, serializers.Serializer): - collectionType = BinaryBase64Field() - encryptionKey = BinaryBase64Field() - - def create(self, validated_data): - - with transaction.atomic(): - invitation = self.context["invitation"] - encryption_key = validated_data.get("encryptionKey") - collection_type = validated_data.pop("collectionType") - - user = invitation.user - collection_type_obj, _ = models.CollectionType.objects.get_or_create(uid=collection_type, owner=user) - - member = models.CollectionMember.objects.create( - collection=invitation.collection, - stoken=models.Stoken.objects.create(), - user=user, - accessLevel=invitation.accessLevel, - encryptionKey=encryption_key, - collectionType=collection_type_obj, - ) - - models.CollectionMemberRemoved.objects.filter( - user=invitation.user, collection=invitation.collection - ).delete() - - invitation.delete() - - return member - - def update(self, instance, validated_data): - raise NotImplementedError() - - -class UserSerializer(BetterErrorsMixin, serializers.ModelSerializer): - pubkey = BinaryBase64Field(source="userinfo.pubkey") - encryptedContent = BinaryBase64Field(source="userinfo.encryptedContent") - - class Meta: - model = User - fields = (User.USERNAME_FIELD, User.EMAIL_FIELD, "pubkey", "encryptedContent") - - -class UserInfoPubkeySerializer(BetterErrorsMixin, serializers.ModelSerializer): - pubkey = BinaryBase64Field() - - class Meta: - model = models.UserInfo - fields = ("pubkey",) - - -class UserSignupSerializer(BetterErrorsMixin, serializers.ModelSerializer): - class Meta: - model = User - fields = (User.USERNAME_FIELD, User.EMAIL_FIELD) - extra_kwargs = { - "username": {"validators": []}, # We specifically validate in SignupSerializer - } - - -class AuthenticationSignupSerializer(BetterErrorsMixin, serializers.Serializer): - """Used both for creating new accounts and setting up existing ones for the first time. - When setting up existing ones the email is ignored." - """ - - user = UserSignupSerializer(many=False) - salt = BinaryBase64Field() - loginPubkey = BinaryBase64Field() - pubkey = BinaryBase64Field() - encryptedContent = BinaryBase64Field() - - def create(self, validated_data): - """Function that's called when this serializer creates an item""" - user_data = validated_data.pop("user") - - with transaction.atomic(): - view = self.context.get("view", None) - try: - user_queryset = get_user_queryset(User.objects.all(), context=CallbackContext(view.kwargs)) - instance = user_queryset.get(**{User.USERNAME_FIELD: user_data["username"].lower()}) - except User.DoesNotExist: - # Create the user and save the casing the user chose as the first name - try: - instance = create_user( - **user_data, - password=None, - first_name=user_data["username"], - context=CallbackContext(view.kwargs) - ) - instance.full_clean() - except EtebaseValidationError as e: - raise e - except django_exceptions.ValidationError as e: - self.transform_validation_error("user", e) - except Exception as e: - raise EtebaseValidationError("generic", str(e)) - - if hasattr(instance, "userinfo"): - raise EtebaseValidationError("user_exists", "User already exists", status_code=status.HTTP_409_CONFLICT) - - models.UserInfo.objects.create(**validated_data, owner=instance) - - return instance - - def update(self, instance, validated_data): - raise NotImplementedError() - - -class AuthenticationLoginChallengeSerializer(BetterErrorsMixin, serializers.Serializer): - username = serializers.CharField(required=True) - - def create(self, validated_data): - raise NotImplementedError() - - def update(self, instance, validated_data): - raise NotImplementedError() - - -class AuthenticationLoginSerializer(BetterErrorsMixin, serializers.Serializer): - response = BinaryBase64Field() - signature = BinaryBase64Field() - - def create(self, validated_data): - raise NotImplementedError() - - def update(self, instance, validated_data): - raise NotImplementedError() - - -class AuthenticationLoginInnerSerializer(AuthenticationLoginChallengeSerializer): - challenge = BinaryBase64Field() - host = serializers.CharField() - action = serializers.CharField() - - def create(self, validated_data): - raise NotImplementedError() - - def update(self, instance, validated_data): - raise NotImplementedError() - - -class AuthenticationChangePasswordInnerSerializer(AuthenticationLoginInnerSerializer): - loginPubkey = BinaryBase64Field() - encryptedContent = BinaryBase64Field() - - class Meta: - model = models.UserInfo - fields = ("loginPubkey", "encryptedContent") - - def create(self, validated_data): - raise NotImplementedError() - - def update(self, instance, validated_data): - with transaction.atomic(): - instance.loginPubkey = validated_data.pop("loginPubkey") - instance.encryptedContent = validated_data.pop("encryptedContent") - instance.save() - - return instance diff --git a/django_etebase/tests.py b/django_etebase/tests.py deleted file mode 100644 index 7ce503c..0000000 --- a/django_etebase/tests.py +++ /dev/null @@ -1,3 +0,0 @@ -from django.test import TestCase - -# Create your tests here. diff --git a/django_etebase/token_auth/admin.py b/django_etebase/token_auth/admin.py deleted file mode 100644 index e69de29..0000000 diff --git a/django_etebase/token_auth/authentication.py b/django_etebase/token_auth/authentication.py deleted file mode 100644 index 7e84956..0000000 --- a/django_etebase/token_auth/authentication.py +++ /dev/null @@ -1,46 +0,0 @@ -from django.utils import timezone -from django.utils.translation import gettext_lazy as _ - -from rest_framework import exceptions -from rest_framework.authentication import TokenAuthentication as DRFTokenAuthentication - -from .models import AuthToken, get_default_expiry - - -AUTO_REFRESH = True -MIN_REFRESH_INTERVAL = 60 - - -class TokenAuthentication(DRFTokenAuthentication): - keyword = "Token" - model = AuthToken - - def authenticate_credentials(self, key): - msg = _("Invalid token.") - model = self.get_model() - try: - token = model.objects.select_related("user").get(key=key) - except model.DoesNotExist: - raise exceptions.AuthenticationFailed(msg) - - if not token.user.is_active: - raise exceptions.AuthenticationFailed(_("User inactive or deleted.")) - - if token.expiry is not None: - if token.expiry < timezone.now(): - token.delete() - raise exceptions.AuthenticationFailed(msg) - - if AUTO_REFRESH: - self.renew_token(token) - - return (token.user, token) - - def renew_token(self, auth_token): - current_expiry = auth_token.expiry - new_expiry = get_default_expiry() - # Throttle refreshing of token to avoid db writes - delta = (new_expiry - current_expiry).total_seconds() - if delta > MIN_REFRESH_INTERVAL: - auth_token.expiry = new_expiry - auth_token.save(update_fields=("expiry",)) diff --git a/django_etebase/urls.py b/django_etebase/urls.py deleted file mode 100644 index 01797c1..0000000 --- a/django_etebase/urls.py +++ /dev/null @@ -1,30 +0,0 @@ -from django.conf import settings -from django.conf.urls import include -from django.urls import path - -from rest_framework_nested import routers - -from django_etebase import views - -router = routers.DefaultRouter() -router.register(r"collection", views.CollectionViewSet) -router.register(r"authentication", views.AuthenticationViewSet, basename="authentication") -router.register(r"invitation/incoming", views.InvitationIncomingViewSet, basename="invitation_incoming") -router.register(r"invitation/outgoing", views.InvitationOutgoingViewSet, basename="invitation_outgoing") - -collections_router = routers.NestedSimpleRouter(router, r"collection", lookup="collection") -collections_router.register(r"item", views.CollectionItemViewSet, basename="collection_item") -collections_router.register(r"member", views.CollectionMemberViewSet, basename="collection_member") - -item_router = routers.NestedSimpleRouter(collections_router, r"item", lookup="collection_item") -item_router.register(r"chunk", views.CollectionItemChunkViewSet, basename="collection_items_chunk") - -if settings.DEBUG: - router.register(r"test/authentication", views.TestAuthenticationViewSet, basename="test_authentication") - -app_name = "django_etebase" -urlpatterns = [ - path("v1/", include(router.urls)), - path("v1/", include(collections_router.urls)), - path("v1/", include(item_router.urls)), -] diff --git a/django_etebase/views.py b/django_etebase/views.py deleted file mode 100644 index 5a03aa4..0000000 --- a/django_etebase/views.py +++ /dev/null @@ -1,861 +0,0 @@ -# Copyright © 2017 Tom Hacohen -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, version 3. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . - -import msgpack - -from django.conf import settings -from django.contrib.auth import get_user_model, user_logged_in, user_logged_out -from django.core.exceptions import PermissionDenied -from django.db import transaction, IntegrityError -from django.db.models import Q -from django.http import HttpResponseBadRequest, HttpResponse, Http404 -from django.shortcuts import get_object_or_404 - -from rest_framework import status -from rest_framework import viewsets -from rest_framework.decorators import action as action_decorator -from rest_framework.response import Response -from rest_framework.parsers import JSONParser, FormParser, MultiPartParser -from rest_framework.renderers import BrowsableAPIRenderer -from rest_framework.exceptions import AuthenticationFailed -from rest_framework.permissions import IsAuthenticated - -import nacl.encoding -import nacl.signing -import nacl.secret -import nacl.hash - -from .sendfile import sendfile -from .token_auth.models import AuthToken - -from .drf_msgpack.parsers import MessagePackParser -from .drf_msgpack.renderers import MessagePackRenderer - -from . import app_settings, permissions -from .renderers import JSONRenderer -from .models import ( - Collection, - CollectionItem, - CollectionItemRevision, - CollectionMember, - CollectionMemberRemoved, - CollectionInvitation, - Stoken, - UserInfo, -) -from .serializers import ( - AuthenticationChangePasswordInnerSerializer, - AuthenticationSignupSerializer, - AuthenticationLoginChallengeSerializer, - AuthenticationLoginSerializer, - AuthenticationLoginInnerSerializer, - CollectionSerializer, - CollectionItemSerializer, - CollectionItemBulkGetSerializer, - CollectionItemDepSerializer, - CollectionItemRevisionSerializer, - CollectionItemChunkSerializer, - CollectionListMultiSerializer, - CollectionMemberSerializer, - CollectionInvitationSerializer, - InvitationAcceptSerializer, - UserInfoPubkeySerializer, - UserSerializer, -) -from .utils import get_user_queryset, CallbackContext -from .exceptions import EtebaseValidationError -from .parsers import ChunkUploadParser -from .signals import user_signed_up - -User = get_user_model() - - -def msgpack_encode(content): - return msgpack.packb(content, use_bin_type=True) - - -def msgpack_decode(content): - return msgpack.unpackb(content, raw=False) - - -class BaseViewSet(viewsets.ModelViewSet): - authentication_classes = tuple(app_settings.API_AUTHENTICATORS) - permission_classes = tuple(app_settings.API_PERMISSIONS) - renderer_classes = [JSONRenderer, MessagePackRenderer] + ([BrowsableAPIRenderer] if settings.DEBUG else []) - parser_classes = [JSONParser, MessagePackParser, FormParser, MultiPartParser] - stoken_annotation = None - - def get_serializer_class(self): - serializer_class = self.serializer_class - - if self.request.method == "PUT": - serializer_class = getattr(self, "serializer_update_class", serializer_class) - - return serializer_class - - def get_collection_queryset(self, queryset=Collection.objects): - user = self.request.user - return queryset.filter(members__user=user) - - def get_stoken_obj_id(self, request): - return request.GET.get("stoken", None) - - def get_stoken_obj(self, request): - stoken = self.get_stoken_obj_id(request) - - if stoken is not None: - try: - return Stoken.objects.get(uid=stoken) - except Stoken.DoesNotExist: - raise EtebaseValidationError("bad_stoken", "Invalid stoken.", status_code=status.HTTP_400_BAD_REQUEST) - - return None - - def filter_by_stoken(self, request, queryset): - stoken_rev = self.get_stoken_obj(request) - - queryset = queryset.annotate(max_stoken=self.stoken_annotation).order_by("max_stoken") - - if stoken_rev is not None: - queryset = queryset.filter(max_stoken__gt=stoken_rev.id) - - return queryset, stoken_rev - - def get_queryset_stoken(self, queryset): - maxid = -1 - for row in queryset: - rowmaxid = getattr(row, "max_stoken") or -1 - maxid = max(maxid, rowmaxid) - new_stoken = (maxid >= 0) and Stoken.objects.get(id=maxid) - - return new_stoken or None - - def filter_by_stoken_and_limit(self, request, queryset): - limit = int(request.GET.get("limit", 50)) - - queryset, stoken_rev = self.filter_by_stoken(request, queryset) - - result = list(queryset[: limit + 1]) - if len(result) < limit + 1: - done = True - else: - done = False - result = result[:-1] - - new_stoken_obj = self.get_queryset_stoken(result) or stoken_rev - - return result, new_stoken_obj, done - - # Change how our list works by default - def list(self, request, collection_uid=None, *args, **kwargs): - queryset = self.get_queryset() - serializer = self.get_serializer(queryset, many=True) - - ret = { - "data": serializer.data, - "done": True, # we always return all the items, so it's always done - } - - return Response(ret) - - -class CollectionViewSet(BaseViewSet): - allowed_methods = ["GET", "POST"] - permission_classes = BaseViewSet.permission_classes + (permissions.IsCollectionAdminOrReadOnly,) - queryset = Collection.objects.all() - serializer_class = CollectionSerializer - lookup_field = "uid" - lookup_url_kwarg = "uid" - stoken_annotation = Collection.stoken_annotation - - def get_queryset(self, queryset=None): - if queryset is None: - queryset = type(self).queryset - return self.get_collection_queryset(queryset) - - def get_serializer_context(self): - context = super().get_serializer_context() - prefetch = self.request.query_params.get("prefetch", "auto") - context.update({"request": self.request, "prefetch": prefetch}) - return context - - def destroy(self, request, uid=None, *args, **kwargs): - # FIXME: implement - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def partial_update(self, request, uid=None, *args, **kwargs): - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def update(self, request, *args, **kwargs): - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def create(self, request, *args, **kwargs): - serializer = self.get_serializer(data=request.data) - serializer.is_valid(raise_exception=True) - serializer.save(owner=self.request.user) - - return Response({}, status=status.HTTP_201_CREATED) - - def list(self, request, *args, **kwargs): - queryset = self.get_queryset() - return self.list_common(request, queryset, *args, **kwargs) - - @action_decorator(detail=False, methods=["POST"]) - def list_multi(self, request, *args, **kwargs): - serializer = CollectionListMultiSerializer(data=request.data) - serializer.is_valid(raise_exception=True) - - collection_types = serializer.validated_data["collectionTypes"] - - queryset = self.get_queryset() - # FIXME: Remove the isnull part once we attach collection types to all objects ("collection-type-migration") - queryset = queryset.filter( - Q(members__collectionType__uid__in=collection_types) | Q(members__collectionType__isnull=True) - ) - - return self.list_common(request, queryset, *args, **kwargs) - - def list_common(self, request, queryset, *args, **kwargs): - result, new_stoken_obj, done = self.filter_by_stoken_and_limit(request, queryset) - new_stoken = new_stoken_obj and new_stoken_obj.uid - - serializer = self.get_serializer(result, many=True) - - ret = { - "data": serializer.data, - "stoken": new_stoken, - "done": done, - } - - stoken_obj = self.get_stoken_obj(request) - if stoken_obj is not None: - # FIXME: honour limit? (the limit should be combined for data and this because of stoken) - remed_qs = CollectionMemberRemoved.objects.filter(user=request.user, stoken__id__gt=stoken_obj.id) - if not ret["done"]: - # We only filter by the new_stoken if we are not done. This is because if we are done, the new stoken - # can point to the most recent collection change rather than most recent removed membership. - remed_qs = remed_qs.filter(stoken__id__lte=new_stoken_obj.id) - - remed = remed_qs.values_list("collection__uid", flat=True) - if len(remed) > 0: - ret["removedMemberships"] = [{"uid": x} for x in remed] - - return Response(ret) - - -class CollectionItemViewSet(BaseViewSet): - allowed_methods = ["GET", "POST", "PUT"] - permission_classes = BaseViewSet.permission_classes + (permissions.HasWriteAccessOrReadOnly,) - queryset = CollectionItem.objects.all() - serializer_class = CollectionItemSerializer - lookup_field = "uid" - stoken_annotation = CollectionItem.stoken_annotation - - def get_queryset(self): - collection_uid = self.kwargs["collection_uid"] - try: - collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) - except Collection.DoesNotExist: - raise Http404("Collection does not exist") - # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') - queryset = type(self).queryset.filter(collection__pk=collection.pk, revisions__current=True) - - return queryset - - def get_serializer_context(self): - context = super().get_serializer_context() - prefetch = self.request.query_params.get("prefetch", "auto") - context.update({"request": self.request, "prefetch": prefetch}) - return context - - def create(self, request, collection_uid=None, *args, **kwargs): - # We create using batch and transaction - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def destroy(self, request, collection_uid=None, uid=None, *args, **kwargs): - # We can't have destroy because we need to get data from the user (in the body) such as hmac. - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def update(self, request, collection_uid=None, uid=None, *args, **kwargs): - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def partial_update(self, request, collection_uid=None, uid=None, *args, **kwargs): - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - def list(self, request, collection_uid=None, *args, **kwargs): - queryset = self.get_queryset() - - if not self.request.query_params.get("withCollection", False): - queryset = queryset.filter(parent__isnull=True) - - result, new_stoken_obj, done = self.filter_by_stoken_and_limit(request, queryset) - new_stoken = new_stoken_obj and new_stoken_obj.uid - - serializer = self.get_serializer(result, many=True) - - ret = { - "data": serializer.data, - "stoken": new_stoken, - "done": done, - } - return Response(ret) - - @action_decorator(detail=True, methods=["GET"]) - def revision(self, request, collection_uid=None, uid=None, *args, **kwargs): - col = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) - item = get_object_or_404(col.items, uid=uid) - - limit = int(request.GET.get("limit", 50)) - iterator = request.GET.get("iterator", None) - - queryset = item.revisions.order_by("-id") - - if iterator is not None: - iterator = get_object_or_404(queryset, uid=iterator) - queryset = queryset.filter(id__lt=iterator.id) - - result = list(queryset[: limit + 1]) - if len(result) < limit + 1: - done = True - else: - done = False - result = result[:-1] - - serializer = CollectionItemRevisionSerializer(result, context=self.get_serializer_context(), many=True) - - iterator = serializer.data[-1]["uid"] if len(result) > 0 else None - - ret = { - "data": serializer.data, - "iterator": iterator, - "done": done, - } - return Response(ret) - - # FIXME: rename to something consistent with what the clients have - maybe list_updates? - @action_decorator(detail=False, methods=["POST"]) - def fetch_updates(self, request, collection_uid=None, *args, **kwargs): - queryset = self.get_queryset() - - serializer = CollectionItemBulkGetSerializer(data=request.data, many=True) - serializer.is_valid(raise_exception=True) - # FIXME: make configurable? - item_limit = 200 - - if len(serializer.validated_data) > item_limit: - content = {"code": "too_many_items", "detail": "Request has too many items. Limit: {}".format(item_limit)} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - - queryset, stoken_rev = self.filter_by_stoken(request, queryset) - - uids, etags = zip(*[(item["uid"], item.get("etag")) for item in serializer.validated_data]) - revs = CollectionItemRevision.objects.filter(uid__in=etags, current=True) - queryset = queryset.filter(uid__in=uids).exclude(revisions__in=revs) - - new_stoken_obj = self.get_queryset_stoken(queryset) - new_stoken = new_stoken_obj and new_stoken_obj.uid - stoken = stoken_rev and getattr(stoken_rev, "uid", None) - new_stoken = new_stoken or stoken - - serializer = self.get_serializer(queryset, many=True) - - ret = { - "data": serializer.data, - "stoken": new_stoken, - "done": True, # we always return all the items, so it's always done - } - return Response(ret) - - @action_decorator(detail=False, methods=["POST"]) - def batch(self, request, collection_uid=None, *args, **kwargs): - return self.transaction(request, collection_uid, validate_etag=False) - - @action_decorator(detail=False, methods=["POST"]) - def transaction(self, request, collection_uid=None, validate_etag=True, *args, **kwargs): - stoken = request.GET.get("stoken", None) - with transaction.atomic(): # We need this for locking on the collection object - collection_object = get_object_or_404( - self.get_collection_queryset(Collection.objects).select_for_update(), # Lock writes on the collection - uid=collection_uid, - ) - - if stoken is not None and stoken != collection_object.stoken: - content = {"code": "stale_stoken", "detail": "Stoken is too old"} - return Response(content, status=status.HTTP_409_CONFLICT) - - items = request.data.get("items") - deps = request.data.get("deps", None) - # FIXME: It should just be one serializer - context = self.get_serializer_context() - context.update({"validate_etag": validate_etag}) - serializer = self.get_serializer_class()(data=items, context=context, many=True) - deps_serializer = CollectionItemDepSerializer(data=deps, context=context, many=True) - - ser_valid = serializer.is_valid() - deps_ser_valid = deps is None or deps_serializer.is_valid() - if ser_valid and deps_ser_valid: - items = serializer.save(collection=collection_object) - - ret = {} - return Response(ret, status=status.HTTP_200_OK) - - return Response( - { - "items": serializer.errors, - "deps": deps_serializer.errors if deps is not None else [], - }, - status=status.HTTP_409_CONFLICT, - ) - - -class CollectionItemChunkViewSet(viewsets.ViewSet): - allowed_methods = ["GET", "PUT"] - authentication_classes = BaseViewSet.authentication_classes - permission_classes = BaseViewSet.permission_classes - renderer_classes = BaseViewSet.renderer_classes - parser_classes = (ChunkUploadParser,) - serializer_class = CollectionItemChunkSerializer - lookup_field = "uid" - - def get_serializer_class(self): - return self.serializer_class - - def get_collection_queryset(self, queryset=Collection.objects): - user = self.request.user - return queryset.filter(members__user=user) - - def update(self, request, *args, collection_uid=None, collection_item_uid=None, uid=None, **kwargs): - col = get_object_or_404(self.get_collection_queryset(), uid=collection_uid) - # IGNORED FOR NOW: col_it = get_object_or_404(col.items, uid=collection_item_uid) - - data = { - "uid": uid, - "chunkFile": request.data["file"], - } - - serializer = self.get_serializer_class()(data=data) - serializer.is_valid(raise_exception=True) - try: - serializer.save(collection=col) - except IntegrityError: - return Response( - {"code": "chunk_exists", "detail": "Chunk already exists."}, status=status.HTTP_409_CONFLICT - ) - - return Response({}, status=status.HTTP_201_CREATED) - - @action_decorator(detail=True, methods=["GET"]) - def download(self, request, collection_uid=None, collection_item_uid=None, uid=None, *args, **kwargs): - col = get_object_or_404(self.get_collection_queryset(), uid=collection_uid) - chunk = get_object_or_404(col.chunks, uid=uid) - - filename = chunk.chunkFile.path - return sendfile(request, filename) - - -class CollectionMemberViewSet(BaseViewSet): - allowed_methods = ["GET", "PUT", "DELETE"] - our_base_permission_classes = BaseViewSet.permission_classes - permission_classes = our_base_permission_classes + (permissions.IsCollectionAdmin,) - queryset = CollectionMember.objects.all() - serializer_class = CollectionMemberSerializer - lookup_field = f"user__{User.USERNAME_FIELD}__iexact" - lookup_url_kwarg = "username" - stoken_annotation = CollectionMember.stoken_annotation - - # FIXME: need to make sure that there's always an admin, and maybe also don't let an owner remove adm access - # (if we want to transfer, we need to do that specifically) - - def get_queryset(self, queryset=None): - collection_uid = self.kwargs["collection_uid"] - try: - collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) - except Collection.DoesNotExist: - raise Http404("Collection does not exist") - - if queryset is None: - queryset = type(self).queryset - - return queryset.filter(collection=collection) - - # We override this method because we expect the stoken to be called iterator - def get_stoken_obj_id(self, request): - return request.GET.get("iterator", None) - - def list(self, request, collection_uid=None, *args, **kwargs): - queryset = self.get_queryset().order_by("id") - result, new_stoken_obj, done = self.filter_by_stoken_and_limit(request, queryset) - new_stoken = new_stoken_obj and new_stoken_obj.uid - serializer = self.get_serializer(result, many=True) - - ret = { - "data": serializer.data, - "iterator": new_stoken, # Here we call it an iterator, it's only stoken for collection/items - "done": done, - } - - return Response(ret) - - def create(self, request, *args, **kwargs): - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - # FIXME: block leaving if we are the last admins - should be deleted / assigned in this case depending if there - # are other memebers. - def perform_destroy(self, instance): - instance.revoke() - - @action_decorator(detail=False, methods=["POST"], permission_classes=our_base_permission_classes) - def leave(self, request, collection_uid=None, *args, **kwargs): - collection_uid = self.kwargs["collection_uid"] - col = get_object_or_404(self.get_collection_queryset(Collection.objects), uid=collection_uid) - - member = col.members.get(user=request.user) - self.perform_destroy(member) - - return Response({}) - - -class InvitationBaseViewSet(BaseViewSet): - queryset = CollectionInvitation.objects.all() - serializer_class = CollectionInvitationSerializer - lookup_field = "uid" - lookup_url_kwarg = "invitation_uid" - - def list(self, request, collection_uid=None, *args, **kwargs): - limit = int(request.GET.get("limit", 50)) - iterator = request.GET.get("iterator", None) - - queryset = self.get_queryset().order_by("id") - - if iterator is not None: - iterator = get_object_or_404(queryset, uid=iterator) - queryset = queryset.filter(id__gt=iterator.id) - - result = list(queryset[: limit + 1]) - if len(result) < limit + 1: - done = True - else: - done = False - result = result[:-1] - - serializer = self.get_serializer(result, many=True) - - iterator = serializer.data[-1]["uid"] if len(result) > 0 else None - - ret = { - "data": serializer.data, - "iterator": iterator, - "done": done, - } - - return Response(ret) - - -class InvitationOutgoingViewSet(InvitationBaseViewSet): - allowed_methods = ["GET", "POST", "PUT", "DELETE"] - - def get_queryset(self, queryset=None): - if queryset is None: - queryset = type(self).queryset - - return queryset.filter(fromMember__user=self.request.user) - - def create(self, request, *args, **kwargs): - serializer = self.get_serializer(data=request.data) - serializer.is_valid(raise_exception=True) - collection_uid = serializer.validated_data.get("collection", {}).get("uid") - - try: - collection = self.get_collection_queryset(Collection.objects).get(uid=collection_uid) - except Collection.DoesNotExist: - raise Http404("Collection does not exist") - - if request.user == serializer.validated_data.get("user"): - content = {"code": "self_invite", "detail": "Inviting yourself is invalid"} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - - if not permissions.is_collection_admin(collection, request.user): - raise PermissionDenied( - {"code": "admin_access_required", "detail": "User is not an admin of this collection"} - ) - - serializer.save(collection=collection) - - return Response({}, status=status.HTTP_201_CREATED) - - @action_decorator(detail=False, allowed_methods=["GET"], methods=["GET"]) - def fetch_user_profile(self, request, *args, **kwargs): - username = request.GET.get("username") - kwargs = {User.USERNAME_FIELD: username.lower()} - user = get_object_or_404(get_user_queryset(User.objects.all(), CallbackContext(self.kwargs)), **kwargs) - user_info = get_object_or_404(UserInfo.objects.all(), owner=user) - serializer = UserInfoPubkeySerializer(user_info) - return Response(serializer.data) - - -class InvitationIncomingViewSet(InvitationBaseViewSet): - allowed_methods = ["GET", "DELETE"] - - def get_queryset(self, queryset=None): - if queryset is None: - queryset = type(self).queryset - - return queryset.filter(user=self.request.user) - - @action_decorator(detail=True, allowed_methods=["POST"], methods=["POST"]) - def accept(self, request, invitation_uid=None, *args, **kwargs): - invitation = get_object_or_404(self.get_queryset(), uid=invitation_uid) - context = self.get_serializer_context() - context.update({"invitation": invitation}) - - serializer = InvitationAcceptSerializer(data=request.data, context=context) - serializer.is_valid(raise_exception=True) - serializer.save() - return Response(status=status.HTTP_201_CREATED) - - -class AuthenticationViewSet(viewsets.ViewSet): - allowed_methods = ["POST"] - authentication_classes = BaseViewSet.authentication_classes - renderer_classes = BaseViewSet.renderer_classes - parser_classes = BaseViewSet.parser_classes - - def get_encryption_key(self, salt): - key = nacl.hash.blake2b(settings.SECRET_KEY.encode(), encoder=nacl.encoding.RawEncoder) - return nacl.hash.blake2b( - b"", - key=key, - salt=salt[: nacl.hash.BLAKE2B_SALTBYTES], - person=b"etebase-auth", - encoder=nacl.encoding.RawEncoder, - ) - - def get_queryset(self): - return get_user_queryset(User.objects.all(), CallbackContext(self.kwargs)) - - def get_serializer_context(self): - return {"request": self.request, "format": self.format_kwarg, "view": self} - - def login_response_data(self, user): - return { - "token": AuthToken.objects.create(user=user).key, - "user": UserSerializer(user).data, - } - - def list(self, request, *args, **kwargs): - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - @action_decorator(detail=False, methods=["POST"]) - def signup(self, request, *args, **kwargs): - serializer = AuthenticationSignupSerializer(data=request.data, context=self.get_serializer_context()) - serializer.is_valid(raise_exception=True) - user = serializer.save() - - user_signed_up.send(sender=user.__class__, request=request, user=user) - - data = self.login_response_data(user) - return Response(data, status=status.HTTP_201_CREATED) - - def get_login_user(self, username): - kwargs = {User.USERNAME_FIELD + "__iexact": username.lower()} - try: - user = self.get_queryset().get(**kwargs) - if not hasattr(user, "userinfo"): - raise AuthenticationFailed({"code": "user_not_init", "detail": "User not properly init"}) - return user - except User.DoesNotExist: - raise AuthenticationFailed({"code": "user_not_found", "detail": "User not found"}) - - def validate_login_request(self, request, validated_data, response_raw, signature, expected_action): - from datetime import datetime - - username = validated_data.get("username") - user = self.get_login_user(username) - host = validated_data["host"] - challenge = validated_data["challenge"] - action = validated_data["action"] - - salt = bytes(user.userinfo.salt) - enc_key = self.get_encryption_key(salt) - box = nacl.secret.SecretBox(enc_key) - - challenge_data = msgpack_decode(box.decrypt(challenge)) - now = int(datetime.now().timestamp()) - if action != expected_action: - content = {"code": "wrong_action", "detail": 'Expected "{}" but got something else'.format(expected_action)} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - elif now - challenge_data["timestamp"] > app_settings.CHALLENGE_VALID_SECONDS: - content = {"code": "challenge_expired", "detail": "Login challange has expired"} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - elif challenge_data["userId"] != user.id: - content = {"code": "wrong_user", "detail": "This challenge is for the wrong user"} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - elif not settings.DEBUG and host.split(":", 1)[0] != request.get_host().split(":", 1)[0]: - detail = 'Found wrong host name. Got: "{}" expected: "{}"'.format(host, request.get_host()) - content = {"code": "wrong_host", "detail": detail} - return Response(content, status=status.HTTP_400_BAD_REQUEST) - - verify_key = nacl.signing.VerifyKey(bytes(user.userinfo.loginPubkey), encoder=nacl.encoding.RawEncoder) - - try: - verify_key.verify(response_raw, signature) - except nacl.exceptions.BadSignatureError: - return Response( - {"code": "login_bad_signature", "detail": "Wrong password for user."}, - status=status.HTTP_401_UNAUTHORIZED, - ) - - return None - - @action_decorator(detail=False, methods=["GET"]) - def is_etebase(self, request, *args, **kwargs): - return Response({}, status=status.HTTP_200_OK) - - @action_decorator(detail=False, methods=["POST"]) - def login_challenge(self, request, *args, **kwargs): - from datetime import datetime - - serializer = AuthenticationLoginChallengeSerializer(data=request.data) - serializer.is_valid(raise_exception=True) - username = serializer.validated_data.get("username") - user = self.get_login_user(username) - - salt = bytes(user.userinfo.salt) - enc_key = self.get_encryption_key(salt) - box = nacl.secret.SecretBox(enc_key) - - challenge_data = { - "timestamp": int(datetime.now().timestamp()), - "userId": user.id, - } - challenge = box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder) - - ret = { - "salt": salt, - "challenge": challenge, - "version": user.userinfo.version, - } - return Response(ret, status=status.HTTP_200_OK) - - @action_decorator(detail=False, methods=["POST"]) - def login(self, request, *args, **kwargs): - outer_serializer = AuthenticationLoginSerializer(data=request.data) - outer_serializer.is_valid(raise_exception=True) - - response_raw = outer_serializer.validated_data["response"] - response = msgpack_decode(response_raw) - signature = outer_serializer.validated_data["signature"] - - context = {"host": request.get_host()} - serializer = AuthenticationLoginInnerSerializer(data=response, context=context) - serializer.is_valid(raise_exception=True) - - bad_login_response = self.validate_login_request( - request, serializer.validated_data, response_raw, signature, "login" - ) - if bad_login_response is not None: - return bad_login_response - - username = serializer.validated_data.get("username") - user = self.get_login_user(username) - - data = self.login_response_data(user) - - user_logged_in.send(sender=user.__class__, request=request, user=user) - - return Response(data, status=status.HTTP_200_OK) - - @action_decorator(detail=False, methods=["POST"], permission_classes=[IsAuthenticated]) - def logout(self, request, *args, **kwargs): - request.auth.delete() - user_logged_out.send(sender=request.user.__class__, request=request, user=request.user) - return Response(status=status.HTTP_204_NO_CONTENT) - - @action_decorator(detail=False, methods=["POST"], permission_classes=BaseViewSet.permission_classes) - def change_password(self, request, *args, **kwargs): - outer_serializer = AuthenticationLoginSerializer(data=request.data) - outer_serializer.is_valid(raise_exception=True) - - response_raw = outer_serializer.validated_data["response"] - response = msgpack_decode(response_raw) - signature = outer_serializer.validated_data["signature"] - - context = {"host": request.get_host()} - serializer = AuthenticationChangePasswordInnerSerializer(request.user.userinfo, data=response, context=context) - serializer.is_valid(raise_exception=True) - - bad_login_response = self.validate_login_request( - request, serializer.validated_data, response_raw, signature, "changePassword" - ) - if bad_login_response is not None: - return bad_login_response - - serializer.save() - - return Response({}, status=status.HTTP_200_OK) - - @action_decorator(detail=False, methods=["POST"], permission_classes=[IsAuthenticated]) - def dashboard_url(self, request, *args, **kwargs): - get_dashboard_url = app_settings.DASHBOARD_URL_FUNC - if get_dashboard_url is None: - raise EtebaseValidationError( - "not_supported", "This server doesn't have a user dashboard.", status_code=status.HTTP_400_BAD_REQUEST - ) - - ret = { - "url": get_dashboard_url(request, *args, **kwargs), - } - return Response(ret) - - -class TestAuthenticationViewSet(viewsets.ViewSet): - allowed_methods = ["POST"] - renderer_classes = BaseViewSet.renderer_classes - parser_classes = BaseViewSet.parser_classes - - def get_serializer_context(self): - return {"request": self.request, "format": self.format_kwarg, "view": self} - - def list(self, request, *args, **kwargs): - return Response(status=status.HTTP_405_METHOD_NOT_ALLOWED) - - @action_decorator(detail=False, methods=["POST"]) - def reset(self, request, *args, **kwargs): - # Only run when in DEBUG mode! It's only used for tests - if not settings.DEBUG: - return HttpResponseBadRequest("Only allowed in debug mode.") - - with transaction.atomic(): - user_queryset = get_user_queryset(User.objects.all(), CallbackContext(self.kwargs)) - user = get_object_or_404(user_queryset, username=request.data.get("user").get("username")) - - # Only allow test users for extra safety - if not getattr(user, User.USERNAME_FIELD).startswith("test_user"): - return HttpResponseBadRequest("Endpoint not allowed for user.") - - if hasattr(user, "userinfo"): - user.userinfo.delete() - - serializer = AuthenticationSignupSerializer(data=request.data, context=self.get_serializer_context()) - serializer.is_valid(raise_exception=True) - serializer.save() - - # Delete all of the journal data for this user for a clear test env - user.collection_set.all().delete() - user.collectionmember_set.all().delete() - user.incoming_invitations.all().delete() - - # FIXME: also delete chunk files!!! - - return HttpResponse() diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index a13cc51..1c262c8 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -18,16 +18,14 @@ from fastapi import APIRouter, Depends, status, Request from fastapi.security import APIKeyHeader from django_etebase import app_settings, models -from django_etebase.exceptions import EtebaseValidationError from django_etebase.models import UserInfo from django_etebase.signals import user_signed_up from django_etebase.token_auth.models import AuthToken from django_etebase.token_auth.models import get_default_expiry from django_etebase.utils import create_user, get_user_queryset, CallbackContext -from django_etebase.views import msgpack_encode, msgpack_decode from .exceptions import AuthenticationFailed, transform_validation_error, HttpError from .msgpack import MsgpackRoute -from .utils import BaseModel, permission_responses +from .utils import BaseModel, permission_responses, msgpack_encode, msgpack_decode User = get_user_model() token_scheme = APIKeyHeader(name="Authorization") @@ -293,7 +291,7 @@ def signup_save(data: SignupIn, request: Request) -> User: context=CallbackContext(request.path_params), ) instance.full_clean() - except EtebaseValidationError as e: + except HttpError as e: raise e except django_exceptions.ValidationError as e: transform_validation_error("user", e) diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index 165163a..3473fa0 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -1,5 +1,6 @@ import dataclasses import typing as t +import msgpack from fastapi import status, Query, Depends from pydantic import BaseModel as PyBaseModel @@ -44,6 +45,14 @@ def is_collection_admin(collection, user): return (member is not None) and (member.accessLevel == AccessLevels.ADMIN) +def msgpack_encode(content): + return msgpack.packb(content, use_bin_type=True) + + +def msgpack_decode(content): + return msgpack.unpackb(content, raw=False) + + PERMISSIONS_READ = [Depends(x) for x in app_settings.API_PERMISSIONS_READ] PERMISSIONS_READWRITE = PERMISSIONS_READ + [Depends(x) for x in app_settings.API_PERMISSIONS_WRITE] diff --git a/etebase_server/settings.py b/etebase_server/settings.py index 46ad3c9..5d57ec0 100644 --- a/etebase_server/settings.py +++ b/etebase_server/settings.py @@ -53,8 +53,6 @@ INSTALLED_APPS = [ "django.contrib.sessions", "django.contrib.messages", "django.contrib.staticfiles", - "corsheaders", - "rest_framework", "myauth.apps.MyauthConfig", "django_etebase.apps.DjangoEtebaseConfig", "django_etebase.token_auth.apps.TokenAuthConfig", @@ -63,7 +61,6 @@ INSTALLED_APPS = [ MIDDLEWARE = [ "django.middleware.security.SecurityMiddleware", "django.contrib.sessions.middleware.SessionMiddleware", - "corsheaders.middleware.CorsMiddleware", "django.middleware.common.CommonMiddleware", "django.middleware.csrf.CsrfViewMiddleware", "django.contrib.auth.middleware.AuthenticationMiddleware", @@ -124,9 +121,6 @@ USE_L10N = True USE_TZ = True -# Cors -CORS_ORIGIN_ALLOW_ALL = True - # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.0/howto/static-files/ diff --git a/etebase_server/urls.py b/etebase_server/urls.py index 443763d..7cf5a60 100644 --- a/etebase_server/urls.py +++ b/etebase_server/urls.py @@ -1,7 +1,7 @@ import os from django.conf import settings -from django.conf.urls import include, url +from django.conf.urls import url from django.contrib import admin from django.urls import path, re_path from django.views.generic import TemplateView @@ -9,15 +9,11 @@ from django.views.static import serve from django.contrib.staticfiles import finders urlpatterns = [ - url(r"^api/", include("django_etebase.urls")), url(r"^admin/", admin.site.urls), path("", TemplateView.as_view(template_name="success.html")), ] if settings.DEBUG: - urlpatterns += [ - url(r"^api-auth/", include("rest_framework.urls", namespace="rest_framework")), - ] def serve_static(request, path): filename = finders.find(path) diff --git a/requirements.in/base.txt b/requirements.in/base.txt index ca8dd94..fee4a56 100644 --- a/requirements.in/base.txt +++ b/requirements.in/base.txt @@ -1,9 +1,5 @@ django -django-cors-headers -djangorestframework -drf-nested-routers msgpack -psycopg2-binary pynacl fastapi -uvicorn \ No newline at end of file +uvicorn diff --git a/requirements.txt b/requirements.txt index 3d19eaf..cfce456 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,14 +7,10 @@ asgiref==3.3.1 # via django cffi==1.14.4 # via pynacl click==7.1.2 # via uvicorn -django-cors-headers==3.6.0 # via -r requirements.in/base.txt -django==3.1.4 # via -r requirements.in/base.txt, django-cors-headers, djangorestframework, drf-nested-routers -djangorestframework==3.12.2 # via -r requirements.in/base.txt, drf-nested-routers -drf-nested-routers==0.92.5 # via -r requirements.in/base.txt +django==3.1.4 # via -r requirements.in/base.txt fastapi==0.63.0 # via -r requirements.in/base.txt h11==0.11.0 # via uvicorn msgpack==1.0.2 # via -r requirements.in/base.txt -psycopg2-binary==2.8.6 # via -r requirements.in/base.txt pycparser==2.20 # via cffi pydantic==1.7.3 # via fastapi pynacl==1.4.0 # via -r requirements.in/base.txt From 2e9caf66f960db6d1177183643e6a4e88e3eba1c Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 14:04:07 +0200 Subject: [PATCH 075/102] Remove deprecated settings. --- django_etebase/app_settings.py | 28 ---------------------------- 1 file changed, 28 deletions(-) diff --git a/django_etebase/app_settings.py b/django_etebase/app_settings.py index c1e8dc9..90225a6 100644 --- a/django_etebase/app_settings.py +++ b/django_etebase/app_settings.py @@ -31,17 +31,6 @@ class AppSettings: return getattr(settings, self.prefix + name, dflt) - @cached_property - def API_PERMISSIONS(self): # pylint: disable=invalid-name - """ - Deprecated. Do not use. - """ - perms = self._setting("API_PERMISSIONS", ("rest_framework.permissions.IsAuthenticated",)) - ret = [] - for perm in perms: - ret.append(self.import_from_str(perm)) - return ret - @cached_property def API_PERMISSIONS_READ(self): # pylint: disable=invalid-name perms = self._setting("API_PERMISSIONS_READ", tuple()) @@ -58,23 +47,6 @@ class AppSettings: ret.append(self.import_from_str(perm)) return ret - @cached_property - def API_AUTHENTICATORS(self): # pylint: disable=invalid-name - """ - Deprecated. Do not use. - """ - perms = self._setting( - "API_AUTHENTICATORS", - ( - "rest_framework.authentication.TokenAuthentication", - "rest_framework.authentication.SessionAuthentication", - ), - ) - ret = [] - for perm in perms: - ret.append(self.import_from_str(perm)) - return ret - @cached_property def GET_USER_QUERYSET_FUNC(self): # pylint: disable=invalid-name get_user_queryset = self._setting("GET_USER_QUERYSET_FUNC", None) From c918d3ed076a799e8e5bced6f48dce8f1de5d5e2 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 14:26:44 +0200 Subject: [PATCH 076/102] Add base64 utils. --- etebase_fastapi/utils.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index 3473fa0..5e45db7 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -1,6 +1,7 @@ import dataclasses import typing as t import msgpack +import base64 from fastapi import status, Query, Depends from pydantic import BaseModel as PyBaseModel @@ -53,6 +54,15 @@ def msgpack_decode(content): return msgpack.unpackb(content, raw=False) +def b64encode(value): + return base64.urlsafe_b64encode(value).decode("ascii").strip("=") + + +def b64decode(data): + data += "=" * ((4 - len(data) % 4) % 4) + return base64.urlsafe_b64decode(data) + + PERMISSIONS_READ = [Depends(x) for x in app_settings.API_PERMISSIONS_READ] PERMISSIONS_READWRITE = PERMISSIONS_READ + [Depends(x) for x in app_settings.API_PERMISSIONS_WRITE] From 313dcf072119a52edd01bc514222185445a7edd7 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 14:27:23 +0200 Subject: [PATCH 077/102] django_etebase utils: add optionl user to context. --- django_etebase/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/django_etebase/utils.py b/django_etebase/utils.py index 1c8654b..09028c4 100644 --- a/django_etebase/utils.py +++ b/django_etebase/utils.py @@ -15,6 +15,7 @@ class CallbackContext: """Class for passing extra context to callbacks""" url_kwargs: t.Dict[str, t.Any] + user: t.Optional[User] def get_user_queryset(queryset, context: CallbackContext): From b3c170e10d6581af2c64f9451ca7ff35e5409cd0 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 14:28:42 +0200 Subject: [PATCH 078/102] fix getting dashboard URL. --- etebase_fastapi/authentication.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 1c262c8..2f8395e 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -264,13 +264,12 @@ async def change_password(data: ChangePassword, request: Request, user: User = D @authentication_router.post("/dashboard_url/", responses=permission_responses) def dashboard_url(user: User = Depends(get_authenticated_user)): - # XXX-TOM get_dashboard_url = app_settings.DASHBOARD_URL_FUNC if get_dashboard_url is None: raise HttpError("not_supported", "This server doesn't have a user dashboard.") ret = { - "url": get_dashboard_url(request, *args, **kwargs), + "url": get_dashboard_url(CallbackContext(request.path_params, user=user)), } return ret From 65cd722616a5b7898ef559387463cdadce656482 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 14:27:23 +0200 Subject: [PATCH 079/102] django_etebase utils: add optionl user to context. --- django_etebase/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_etebase/utils.py b/django_etebase/utils.py index 09028c4..e46cbd0 100644 --- a/django_etebase/utils.py +++ b/django_etebase/utils.py @@ -15,7 +15,7 @@ class CallbackContext: """Class for passing extra context to callbacks""" url_kwargs: t.Dict[str, t.Any] - user: t.Optional[User] + user: t.Optional[User] = None def get_user_queryset(queryset, context: CallbackContext): From c1f171bde0ca3b1a908ae30f2fd24efd83914f54 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 14:47:41 +0200 Subject: [PATCH 080/102] Change how we create applications. --- etebase_fastapi/main.py | 52 +++++++++++++++++++++-------------------- etebase_server/asgi.py | 4 +++- 2 files changed, 30 insertions(+), 26 deletions(-) diff --git a/etebase_fastapi/main.py b/etebase_fastapi/main.py index 2c10854..706081a 100644 --- a/etebase_fastapi/main.py +++ b/etebase_fastapi/main.py @@ -12,28 +12,30 @@ from .member import member_router from .invitation import invitation_incoming_router, invitation_outgoing_router from .msgpack import MsgpackResponse -app = FastAPI() -VERSION = "v1" -BASE_PATH = f"/api/{VERSION}" -COLLECTION_UID_MARKER = "{collection_uid}" -app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication", tags=["authentication"]) -app.include_router(collection_router, prefix=f"{BASE_PATH}/collection", tags=["collection"]) -app.include_router(item_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["item"]) -app.include_router(member_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["member"]) -app.include_router(invitation_incoming_router, prefix=f"{BASE_PATH}/invitation/incoming", tags=["incoming invitation"]) -app.include_router(invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing", tags=["outgoing invitation"]) - -if settings.DEBUG: - from etebase_fastapi.test_reset_view import test_reset_view_router - - app.include_router(test_reset_view_router, prefix=f"{BASE_PATH}/test/authentication") - -app.add_middleware( - CORSMiddleware, allow_origin_regex="https?://.*", allow_credentials=True, allow_methods=["*"], allow_headers=["*"] -) -app.add_middleware(TrustedHostMiddleware, allowed_hosts=settings.ALLOWED_HOSTS) - - -@app.exception_handler(CustomHttpException) -async def custom_exception_handler(request: Request, exc: CustomHttpException): - return MsgpackResponse(status_code=exc.status_code, content=exc.as_dict) +def create_application(prefix=""): + app = FastAPI() + VERSION = "v1" + BASE_PATH = f"{prefix}/api/{VERSION}" + COLLECTION_UID_MARKER = "{collection_uid}" + app.include_router(authentication_router, prefix=f"{BASE_PATH}/authentication", tags=["authentication"]) + app.include_router(collection_router, prefix=f"{BASE_PATH}/collection", tags=["collection"]) + app.include_router(item_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["item"]) + app.include_router(member_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["member"]) + app.include_router(invitation_incoming_router, prefix=f"{BASE_PATH}/invitation/incoming", tags=["incoming invitation"]) + app.include_router(invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing", tags=["outgoing invitation"]) + + if settings.DEBUG: + from etebase_fastapi.test_reset_view import test_reset_view_router + + app.include_router(test_reset_view_router, prefix=f"{BASE_PATH}/test/authentication") + + app.add_middleware( + CORSMiddleware, allow_origin_regex="https?://.*", allow_credentials=True, allow_methods=["*"], allow_headers=["*"] + ) + app.add_middleware(TrustedHostMiddleware, allowed_hosts=settings.ALLOWED_HOSTS) + + + @app.exception_handler(CustomHttpException) + async def custom_exception_handler(request: Request, exc: CustomHttpException): + return MsgpackResponse(status_code=exc.status_code, content=exc.as_dict) + return app diff --git a/etebase_server/asgi.py b/etebase_server/asgi.py index 92fad1c..25dbf77 100644 --- a/etebase_server/asgi.py +++ b/etebase_server/asgi.py @@ -7,7 +7,9 @@ django_application = get_asgi_application() def create_application(): - from etebase_fastapi.main import app + from etebase_fastapi.main import create_application + + app = create_application() app.mount("/", django_application) From 50f89c48e27d0d6d5aecac22d491e53e130a0bba Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 15:07:18 +0200 Subject: [PATCH 081/102] Dashboard url: fix getting dashboard url. --- etebase_fastapi/authentication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 2f8395e..04aec31 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -263,7 +263,7 @@ async def change_password(data: ChangePassword, request: Request, user: User = D @authentication_router.post("/dashboard_url/", responses=permission_responses) -def dashboard_url(user: User = Depends(get_authenticated_user)): +def dashboard_url(request: Request, user: User = Depends(get_authenticated_user)): get_dashboard_url = app_settings.DASHBOARD_URL_FUNC if get_dashboard_url is None: raise HttpError("not_supported", "This server doesn't have a user dashboard.") From ca7f2ec73cb559f0875e580b81a1e0f18b06c21a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 15:08:03 +0200 Subject: [PATCH 082/102] When converting from ORM convert binary fields to bytes. The problem is that some ORMs return memoryview which are more efficient but are not supported by pydantic at the moment. --- etebase_fastapi/authentication.py | 9 +++++---- etebase_fastapi/collection.py | 6 +++--- etebase_fastapi/invitation.py | 8 ++++++-- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 04aec31..2f5a2f1 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -25,7 +25,7 @@ from django_etebase.token_auth.models import get_default_expiry from django_etebase.utils import create_user, get_user_queryset, CallbackContext from .exceptions import AuthenticationFailed, transform_validation_error, HttpError from .msgpack import MsgpackRoute -from .utils import BaseModel, permission_responses, msgpack_encode, msgpack_decode +from .utils import BaseModel, permission_responses, msgpack_encode, msgpack_decode User = get_user_model() token_scheme = APIKeyHeader(name="Authorization") @@ -63,7 +63,7 @@ class UserOut(BaseModel): @classmethod def from_orm(cls: t.Type["UserOut"], obj: User) -> "UserOut": - return cls(pubkey=obj.userinfo.pubkey, encryptedContent=obj.userinfo.encryptedContent) + return cls(pubkey=bytes(obj.userinfo.pubkey), encryptedContent=bytes(obj.userinfo.encryptedContent)) class LoginOut(BaseModel): @@ -228,14 +228,15 @@ async def is_etebase(): @authentication_router.post("/login_challenge/", response_model=LoginChallengeOut) async def login_challenge(user: User = Depends(get_login_user)): - enc_key = get_encryption_key(user.userinfo.salt) + salt = bytes(user.userinfo.salt) + enc_key = get_encryption_key(salt) box = nacl.secret.SecretBox(enc_key) challenge_data = { "timestamp": int(datetime.now().timestamp()), "userId": user.id, } challenge = bytes(box.encrypt(msgpack_encode(challenge_data), encoder=nacl.encoding.RawEncoder)) - return LoginChallengeOut(salt=user.userinfo.salt, challenge=challenge, version=user.userinfo.version) + return LoginChallengeOut(salt=salt, challenge=challenge, version=user.userinfo.version) @authentication_router.post("/login/", response_model=LoginOut) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index e6c10c3..74730ff 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -58,7 +58,7 @@ class CollectionItemRevisionInOut(BaseModel): chunks.append((chunk_obj.uid, f.read())) else: chunks.append((chunk_obj.uid,)) - return cls(uid=obj.uid, meta=obj.meta, deleted=obj.deleted, chunks=chunks) + return cls(uid=obj.uid, meta=bytes(obj.meta), deleted=obj.deleted, chunks=chunks) class CollectionItemCommon(BaseModel): @@ -103,8 +103,8 @@ class CollectionOut(CollectionCommon): member: models.CollectionMember = obj.members.get(user=context.user) collection_type = member.collectionType ret = cls( - collectionType=collection_type and collection_type.uid, - collectionKey=member.encryptionKey, + collectionType=collection_type and bytes(collection_type.uid), + collectionKey=bytes(member.encryptionKey), accessLevel=member.accessLevel, stoken=obj.stoken, item=CollectionItemOut.from_orm_context(obj.main_item, context), diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/invitation.py index 39460a9..9e731bc 100644 --- a/etebase_fastapi/invitation.py +++ b/etebase_fastapi/invitation.py @@ -32,6 +32,10 @@ class UserInfoOut(BaseModel): class Config: orm_mode = True + @classmethod + def from_orm(cls: t.Type["UserInfoOut"], obj: models.UserInfo) -> "UserInfoOut": + return cls(pubkey=bytes(obj.pubkey)) + class CollectionInvitationAcceptIn(BaseModel): collectionType: bytes @@ -69,8 +73,8 @@ class CollectionInvitationOut(CollectionInvitationCommon): username=obj.user.username, collection=obj.collection.uid, fromUsername=obj.fromMember.user.username, - fromPubkey=obj.fromMember.user.userinfo.pubkey, - signedEncryptionKey=obj.signedEncryptionKey, + fromPubkey=bytes(obj.fromMember.user.userinfo.pubkey), + signedEncryptionKey=bytes(obj.signedEncryptionKey), ) From 59e30ed9884990f33408f85cd1c18666e61d5507 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 15:17:13 +0200 Subject: [PATCH 083/102] Signup and logout: make sync. --- etebase_fastapi/authentication.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 2f5a2f1..df5dc62 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -250,10 +250,9 @@ async def login(data: Login, request: Request): @authentication_router.post("/logout/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses) -async def logout(request: Request, auth_data: AuthData = Depends(get_auth_data)): - await sync_to_async(auth_data.token.delete)() - # XXX-TOM - await sync_to_async(user_logged_out.send)(sender=auth_data.user.__class__, request=None, user=auth_data.user) +def logout(request: Request, auth_data: AuthData = Depends(get_auth_data)): + auth_data.token.delete() + user_logged_out.send(sender=auth_data.user.__class__, request=None, user=auth_data.user) @authentication_router.post("/change_password/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses) @@ -306,9 +305,8 @@ def signup_save(data: SignupIn, request: Request) -> User: @authentication_router.post("/signup/", response_model=LoginOut, status_code=status.HTTP_201_CREATED) -async def signup(data: SignupIn, request: Request): - user = await sync_to_async(signup_save)(data, request) - # XXX-TOM - data = await sync_to_async(LoginOut.from_orm)(user) - await sync_to_async(user_signed_up.send)(sender=user.__class__, request=None, user=user) +def signup(data: SignupIn, request: Request): + user = signup_save(data, request) + data = LoginOut.from_orm(user) + user_signed_up.send(sender=user.__class__, request=None, user=user) return data From 1bca435d740f1c279efa2113e1ec9d59edc995b8 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 15:26:34 +0200 Subject: [PATCH 084/102] Workaround typing issue. --- etebase_fastapi/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index 5e45db7..7280018 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -68,4 +68,7 @@ PERMISSIONS_READWRITE = PERMISSIONS_READ + [Depends(x) for x in app_settings.API response_model_dict = {"model": HttpErrorOut} -permission_responses = {401: response_model_dict, 403: response_model_dict} +permission_responses: t.Dict[t.Union[int, str], t.Dict[str, t.Any]] = { + 401: response_model_dict, + 403: response_model_dict, +} From 3e39aa88a15579090286cc8a859c6d0256652caa Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 15:27:29 +0200 Subject: [PATCH 085/102] Remove unused var. --- etebase_fastapi/authentication.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index df5dc62..9c66a7a 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -250,7 +250,7 @@ async def login(data: Login, request: Request): @authentication_router.post("/logout/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses) -def logout(request: Request, auth_data: AuthData = Depends(get_auth_data)): +def logout(auth_data: AuthData = Depends(get_auth_data)): auth_data.token.delete() user_logged_out.send(sender=auth_data.user.__class__, request=None, user=auth_data.user) From c2a2e710c9c9cca7b1bcc9b4523fbe095904c26b Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 15:38:00 +0200 Subject: [PATCH 086/102] Move common dependencies to their own file. --- etebase_fastapi/authentication.py | 57 +-------------------- etebase_fastapi/collection.py | 21 +------- etebase_fastapi/dependencies.py | 82 +++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 74 deletions(-) create mode 100644 etebase_fastapi/dependencies.py diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 9c66a7a..eb54f68 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -1,4 +1,3 @@ -import dataclasses import typing as t from datetime import datetime from functools import cached_property @@ -13,33 +12,22 @@ from django.conf import settings from django.contrib.auth import get_user_model, user_logged_out, user_logged_in from django.core import exceptions as django_exceptions from django.db import transaction -from django.utils import timezone from fastapi import APIRouter, Depends, status, Request -from fastapi.security import APIKeyHeader from django_etebase import app_settings, models +from django_etebase.token_auth.models import AuthToken from django_etebase.models import UserInfo from django_etebase.signals import user_signed_up -from django_etebase.token_auth.models import AuthToken -from django_etebase.token_auth.models import get_default_expiry from django_etebase.utils import create_user, get_user_queryset, CallbackContext from .exceptions import AuthenticationFailed, transform_validation_error, HttpError from .msgpack import MsgpackRoute from .utils import BaseModel, permission_responses, msgpack_encode, msgpack_decode +from .dependencies import AuthData, get_auth_data, get_authenticated_user User = get_user_model() -token_scheme = APIKeyHeader(name="Authorization") -AUTO_REFRESH = True -MIN_REFRESH_INTERVAL = 60 authentication_router = APIRouter(route_class=MsgpackRoute) -@dataclasses.dataclass(frozen=True) -class AuthData: - user: User - token: AuthToken - - class LoginChallengeIn(BaseModel): username: str @@ -115,47 +103,6 @@ class SignupIn(BaseModel): encryptedContent: bytes -def __renew_token(auth_token: AuthToken): - current_expiry = auth_token.expiry - new_expiry = get_default_expiry() - # Throttle refreshing of token to avoid db writes - delta = (new_expiry - current_expiry).total_seconds() - if delta > MIN_REFRESH_INTERVAL: - auth_token.expiry = new_expiry - auth_token.save(update_fields=("expiry",)) - - -@sync_to_async -def __get_authenticated_user(api_token: str): - api_token = api_token.split()[1] - try: - token: AuthToken = AuthToken.objects.select_related("user").get(key=api_token) - except AuthToken.DoesNotExist: - raise AuthenticationFailed(detail="Invalid token.") - if not token.user.is_active: - raise AuthenticationFailed(detail="User inactive or deleted.") - - if token.expiry is not None: - if token.expiry < timezone.now(): - token.delete() - raise AuthenticationFailed(detail="Invalid token.") - - if AUTO_REFRESH: - __renew_token(token) - - return token.user, token - - -async def get_auth_data(api_token: str = Depends(token_scheme)) -> AuthData: - user, token = await __get_authenticated_user(api_token) - return AuthData(user, token) - - -async def get_authenticated_user(api_token: str = Depends(token_scheme)) -> User: - user, token = await __get_authenticated_user(api_token) - return user - - @sync_to_async def __get_login_user(username: str) -> User: kwargs = {User.USERNAME_FIELD + "__iexact": username.lower()} diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 74730ff..a60c7f0 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -5,8 +5,7 @@ from django.contrib.auth import get_user_model from django.core import exceptions as django_exceptions from django.core.files.base import ContentFile from django.db import transaction -from django.db.models import Q -from django.db.models import QuerySet +from django.db.models import Q, QuerySet from fastapi import APIRouter, Depends, status from django_etebase import models @@ -25,12 +24,11 @@ from .utils import ( PERMISSIONS_READ, PERMISSIONS_READWRITE, ) +from .dependencies import get_collection_queryset, get_item_queryset, get_collection User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) item_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) -default_queryset: QuerySet = models.Collection.objects.all() -default_item_queryset: QuerySet = models.CollectionItem.objects.all() class ListMulti(BaseModel): @@ -203,21 +201,6 @@ def collection_list_common( return ret -def get_collection_queryset(user: User = Depends(get_authenticated_user)) -> QuerySet: - return default_queryset.filter(members__user=user) - - -def get_collection(collection_uid: str, queryset: QuerySet = Depends(get_collection_queryset)) -> models.Collection: - return get_object_or_404(queryset, uid=collection_uid) - - -def get_item_queryset(collection: models.Collection = Depends(get_collection)) -> QuerySet: - # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') - queryset = default_item_queryset.filter(collection__pk=collection.pk, revisions__current=True) - - return queryset - - # permissions diff --git a/etebase_fastapi/dependencies.py b/etebase_fastapi/dependencies.py new file mode 100644 index 0000000..ddb9b3b --- /dev/null +++ b/etebase_fastapi/dependencies.py @@ -0,0 +1,82 @@ +import dataclasses + +from fastapi import Depends +from fastapi.security import APIKeyHeader + +from django.contrib.auth import get_user_model +from django.utils import timezone +from django.db.models import QuerySet + +from django_etebase import models +from django_etebase.token_auth.models import AuthToken, get_default_expiry +from .exceptions import AuthenticationFailed +from .utils import get_object_or_404 + + +User = get_user_model() +token_scheme = APIKeyHeader(name="Authorization") +AUTO_REFRESH = True +MIN_REFRESH_INTERVAL = 60 + + +@dataclasses.dataclass(frozen=True) +class AuthData: + user: User + token: AuthToken + + +def __renew_token(auth_token: AuthToken): + current_expiry = auth_token.expiry + new_expiry = get_default_expiry() + # Throttle refreshing of token to avoid db writes + delta = (new_expiry - current_expiry).total_seconds() + if delta > MIN_REFRESH_INTERVAL: + auth_token.expiry = new_expiry + auth_token.save(update_fields=("expiry",)) + + +def __get_authenticated_user(api_token: str): + api_token = api_token.split()[1] + try: + token: AuthToken = AuthToken.objects.select_related("user").get(key=api_token) + except AuthToken.DoesNotExist: + raise AuthenticationFailed(detail="Invalid token.") + if not token.user.is_active: + raise AuthenticationFailed(detail="User inactive or deleted.") + + if token.expiry is not None: + if token.expiry < timezone.now(): + token.delete() + raise AuthenticationFailed(detail="Invalid token.") + + if AUTO_REFRESH: + __renew_token(token) + + return token.user, token + + +def get_auth_data(api_token: str = Depends(token_scheme)) -> AuthData: + user, token = __get_authenticated_user(api_token) + return AuthData(user, token) + + +def get_authenticated_user(api_token: str = Depends(token_scheme)) -> User: + user, _ = __get_authenticated_user(api_token) + return user + + +def get_collection_queryset(user: User = Depends(get_authenticated_user)) -> QuerySet: + default_queryset: QuerySet = models.Collection.objects.all() + return default_queryset.filter(members__user=user) + + +def get_collection(collection_uid: str, queryset: QuerySet = Depends(get_collection_queryset)) -> models.Collection: + return get_object_or_404(queryset, uid=collection_uid) + + +def get_item_queryset(collection: models.Collection = Depends(get_collection)) -> QuerySet: + default_item_queryset: QuerySet = models.CollectionItem.objects.all() + # XXX Potentially add this for performance: .prefetch_related('revisions__chunks') + queryset = default_item_queryset.filter(collection__pk=collection.pk, revisions__current=True) + + return queryset From 586b015eb78a01d40a13c9acad81c274b5be6380 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 16:23:01 +0200 Subject: [PATCH 087/102] Login: also return username and email upon login. --- etebase_fastapi/authentication.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index eb54f68..a211e9b 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -46,12 +46,19 @@ class LoginResponse(BaseModel): class UserOut(BaseModel): + username: str + email: str pubkey: bytes encryptedContent: bytes @classmethod def from_orm(cls: t.Type["UserOut"], obj: User) -> "UserOut": - return cls(pubkey=bytes(obj.userinfo.pubkey), encryptedContent=bytes(obj.userinfo.encryptedContent)) + return cls( + username=obj.username, + email=obj.email, + pubkey=bytes(obj.userinfo.pubkey), + encryptedContent=bytes(obj.userinfo.encryptedContent), + ) class LoginOut(BaseModel): From 151bec0d9e683a0c8da93796a7468c999c6aecd0 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 16:44:13 +0200 Subject: [PATCH 088/102] Fix type error. --- etebase_fastapi/authentication.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index a211e9b..559a60a 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -261,6 +261,6 @@ def signup_save(data: SignupIn, request: Request) -> User: @authentication_router.post("/signup/", response_model=LoginOut, status_code=status.HTTP_201_CREATED) def signup(data: SignupIn, request: Request): user = signup_save(data, request) - data = LoginOut.from_orm(user) + ret = LoginOut.from_orm(user) user_signed_up.send(sender=user.__class__, request=None, user=user) - return data + return ret From fa0979dce17c779321325c6ef173addb47af52f0 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 16:57:09 +0200 Subject: [PATCH 089/102] Test reset: clean reset function. --- etebase_fastapi/test_reset_view.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/etebase_fastapi/test_reset_view.py b/etebase_fastapi/test_reset_view.py index f21fd84..3075290 100644 --- a/etebase_fastapi/test_reset_view.py +++ b/etebase_fastapi/test_reset_view.py @@ -2,31 +2,33 @@ from django.conf import settings from django.contrib.auth import get_user_model from django.db import transaction from django.shortcuts import get_object_or_404 -from fastapi import APIRouter, Request, Response, status +from fastapi import APIRouter, Request, status from django_etebase.utils import get_user_queryset, CallbackContext from etebase_fastapi.authentication import SignupIn, signup_save from etebase_fastapi.msgpack import MsgpackRoute +from etebase_fastapi.exceptions import HttpError test_reset_view_router = APIRouter(route_class=MsgpackRoute, tags=["test helpers"]) User = get_user_model() -@test_reset_view_router.post("/reset/") +@test_reset_view_router.post("/reset/", status_code=status.HTTP_204_NO_CONTENT) def reset(data: SignupIn, request: Request): # Only run when in DEBUG mode! It's only used for tests if not settings.DEBUG: - return Response("Only allowed in debug mode.", status_code=status.HTTP_400_BAD_REQUEST) + raise HttpError(code="generic", detail="Only allowed in debug mode.") with transaction.atomic(): user_queryset = get_user_queryset(User.objects.all(), CallbackContext(request.path_params)) user = get_object_or_404(user_queryset, username=data.user.username) # Only allow test users for extra safety if not getattr(user, User.USERNAME_FIELD).startswith("test_user"): - return Response("Endpoint not allowed for user.", status_code=status.HTTP_400_BAD_REQUEST) + raise HttpError(code="generic", detail="Endpoint not allowed for user.") if hasattr(user, "userinfo"): user.userinfo.delete() + signup_save(data, request) # Delete all of the journal data for this user for a clear test env user.collection_set.all().delete() @@ -34,5 +36,3 @@ def reset(data: SignupIn, request: Request): user.incoming_invitations.all().delete() # FIXME: also delete chunk files!!! - - return Response(status_code=status.HTTP_204_NO_CONTENT) From 10ff303b754f8147acdc4f1740d27bf3dbefa915 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 17:09:20 +0200 Subject: [PATCH 090/102] Fix formatting. --- etebase_fastapi/main.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/etebase_fastapi/main.py b/etebase_fastapi/main.py index 706081a..a55e2fa 100644 --- a/etebase_fastapi/main.py +++ b/etebase_fastapi/main.py @@ -12,6 +12,7 @@ from .member import member_router from .invitation import invitation_incoming_router, invitation_outgoing_router from .msgpack import MsgpackResponse + def create_application(prefix=""): app = FastAPI() VERSION = "v1" @@ -21,8 +22,12 @@ def create_application(prefix=""): app.include_router(collection_router, prefix=f"{BASE_PATH}/collection", tags=["collection"]) app.include_router(item_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["item"]) app.include_router(member_router, prefix=f"{BASE_PATH}/collection/{COLLECTION_UID_MARKER}", tags=["member"]) - app.include_router(invitation_incoming_router, prefix=f"{BASE_PATH}/invitation/incoming", tags=["incoming invitation"]) - app.include_router(invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing", tags=["outgoing invitation"]) + app.include_router( + invitation_incoming_router, prefix=f"{BASE_PATH}/invitation/incoming", tags=["incoming invitation"] + ) + app.include_router( + invitation_outgoing_router, prefix=f"{BASE_PATH}/invitation/outgoing", tags=["outgoing invitation"] + ) if settings.DEBUG: from etebase_fastapi.test_reset_view import test_reset_view_router @@ -30,12 +35,16 @@ def create_application(prefix=""): app.include_router(test_reset_view_router, prefix=f"{BASE_PATH}/test/authentication") app.add_middleware( - CORSMiddleware, allow_origin_regex="https?://.*", allow_credentials=True, allow_methods=["*"], allow_headers=["*"] + CORSMiddleware, + allow_origin_regex="https?://.*", + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], ) app.add_middleware(TrustedHostMiddleware, allowed_hosts=settings.ALLOWED_HOSTS) - @app.exception_handler(CustomHttpException) async def custom_exception_handler(request: Request, exc: CustomHttpException): return MsgpackResponse(status_code=exc.status_code, content=exc.as_dict) + return app From 3d438b9591955e948c71c3737a534392b5b9a0ef Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 17:39:51 +0200 Subject: [PATCH 091/102] Cleanup validation errors. --- etebase_fastapi/collection.py | 41 +++++++++++++++++++++++++++-------- etebase_fastapi/exceptions.py | 19 ++++++++++++++++ 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index a60c7f0..6c7d9a4 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -10,7 +10,7 @@ from fastapi import APIRouter, Depends, status from django_etebase import models from .authentication import get_authenticated_user -from .exceptions import HttpError, transform_validation_error, PermissionDenied +from .exceptions import HttpError, transform_validation_error, PermissionDenied, ValidationError from .msgpack import MsgpackRoute from .stoken_handler import filter_by_stoken_and_limit, filter_by_stoken, get_stoken_obj, get_queryset_stoken from .utils import ( @@ -151,10 +151,11 @@ class ItemDepIn(BaseModel): item = models.CollectionItem.objects.get(uid=self.uid) etag = self.etag if item.etag != etag: - raise HttpError( + raise ValidationError( "wrong_etag", "Wrong etag. Expected {} got {}".format(item.etag, etag), status_code=status.HTTP_409_CONFLICT, + field=self.uid, ) @@ -164,8 +165,19 @@ class ItemBatchIn(BaseModel): def validate_db(self): if self.deps is not None: + errors: t.List[HttpError] = [] for dep in self.deps: - dep.validate_db() + try: + dep.validate_db() + except ValidationError as e: + errors.append(e) + if len(errors) > 0: + raise ValidationError( + code="dep_failed", + detail="Dependencies failed to validate", + errors=errors, + status_code=status.HTTP_409_CONFLICT, + ) @sync_to_async @@ -293,7 +305,7 @@ def _create(data: CollectionIn, user: User): try: instance.validate_unique() except django_exceptions.ValidationError: - raise HttpError( + raise ValidationError( "unique_uid", "Collection with this uid already exists", status_code=status.HTTP_409_CONFLICT ) instance.save() @@ -353,10 +365,11 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val return instance if validate_etag and cur_etag != etag: - raise HttpError( + raise ValidationError( "wrong_etag", "Wrong etag. Expected {} got {}".format(cur_etag, etag), status_code=status.HTTP_409_CONFLICT, + field=uid, ) if not created: @@ -426,12 +439,22 @@ def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid if stoken is not None and stoken != collection_object.stoken: raise HttpError("stale_stoken", "Stoken is too old", status_code=status.HTTP_409_CONFLICT) - # XXX-TOM: make sure we return compatible errors data.validate_db() - for item in data.items: - item_create(item, collection_object, validate_etag) - return None + errors: t.List[HttpError] = [] + for item in data.items: + try: + item_create(item, collection_object, validate_etag) + except ValidationError as e: + errors.append(e) + + if len(errors) > 0: + raise ValidationError( + code="item_failed", + detail="Items failed to validate", + errors=errors, + status_code=status.HTTP_409_CONFLICT, + ) @item_router.get( diff --git a/etebase_fastapi/exceptions.py b/etebase_fastapi/exceptions.py index 2c1757c..72a3faf 100644 --- a/etebase_fastapi/exceptions.py +++ b/etebase_fastapi/exceptions.py @@ -9,12 +9,18 @@ class HttpErrorField(BaseModel): code: str detail: str + class Config: + orm_mode = True + class HttpErrorOut(BaseModel): code: str detail: str errors: t.Optional[t.List[HttpErrorField]] + class Config: + orm_mode = True + class CustomHttpException(Exception): def __init__(self, code: str, detail: str, status_code: int = status.HTTP_400_BAD_REQUEST): @@ -73,6 +79,19 @@ class HttpError(CustomHttpException): return HttpErrorOut(code=self.code, errors=self.errors, detail=self.detail).dict() +class ValidationError(HttpError): + def __init__( + self, + code: str, + detail: str, + status_code: int = status.HTTP_400_BAD_REQUEST, + errors: t.Optional[t.List["HttpError"]] = None, + field: t.Optional[str] = None, + ): + self.field = field + super().__init__(code=code, detail=detail, errors=errors, status_code=status_code) + + def flatten_errors(field_name, errors) -> t.List[HttpError]: ret = [] if isinstance(errors, dict): From f7858a20b7c7165908240aaa2ada4278b17260db Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 17:46:20 +0200 Subject: [PATCH 092/102] Fix user creation. --- django_etebase/utils.py | 2 +- etebase_fastapi/authentication.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/django_etebase/utils.py b/django_etebase/utils.py index e46cbd0..4d36a94 100644 --- a/django_etebase/utils.py +++ b/django_etebase/utils.py @@ -28,7 +28,7 @@ def get_user_queryset(queryset, context: CallbackContext): def create_user(context: CallbackContext, *args, **kwargs): custom_func = app_settings.CREATE_USER_FUNC if custom_func is not None: - return custom_func(*args, **kwargs) + return custom_func(context, *args, **kwargs) return User.objects.create_user(*args, **kwargs) diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index 559a60a..fe522f7 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -181,7 +181,7 @@ async def is_etebase(): @authentication_router.post("/login_challenge/", response_model=LoginChallengeOut) -async def login_challenge(user: User = Depends(get_login_user)): +def login_challenge(user: User = Depends(get_login_user)): salt = bytes(user.userinfo.salt) enc_key = get_encryption_key(salt) box = nacl.secret.SecretBox(enc_key) @@ -238,10 +238,10 @@ def signup_save(data: SignupIn, request: Request) -> User: # Create the user and save the casing the user chose as the first name try: instance = create_user( + CallbackContext(request.path_params), **user_data.dict(), password=None, first_name=user_data.username, - context=CallbackContext(request.path_params), ) instance.full_clean() except HttpError as e: From dcf81aa9ceeca7b56f84f47028ef05eda2980406 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 18:17:41 +0200 Subject: [PATCH 093/102] Fix prefetch medium. --- etebase_fastapi/collection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 6c7d9a4..07780ae 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -55,7 +55,7 @@ class CollectionItemRevisionInOut(BaseModel): with open(chunk_obj.chunkFile.path, "rb") as f: chunks.append((chunk_obj.uid, f.read())) else: - chunks.append((chunk_obj.uid,)) + chunks.append((chunk_obj.uid, None)) return cls(uid=obj.uid, meta=bytes(obj.meta), deleted=obj.deleted, chunks=chunks) From 53b22602b28bf2582284495d736b2b1672d35a9b Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 18:17:57 +0200 Subject: [PATCH 094/102] Implement chunk_update. --- etebase_fastapi/collection.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 07780ae..b3f8c5f 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -4,9 +4,9 @@ from asgiref.sync import sync_to_async from django.contrib.auth import get_user_model from django.core import exceptions as django_exceptions from django.core.files.base import ContentFile -from django.db import transaction +from django.db import transaction, IntegrityError from django.db.models import Q, QuerySet -from fastapi import APIRouter, Depends, status +from fastapi import APIRouter, Depends, status, Request from django_etebase import models from .authentication import get_authenticated_user @@ -544,28 +544,27 @@ def item_batch( # Chunks +@sync_to_async +def chunk_save(chunk_uid: str, collection: models.Collection, content_file: ContentFile): + chunk_obj = models.CollectionItemChunk(uid=chunk_uid, collection=collection) + chunk_obj.chunkFile.save("IGNORED", content_file) + chunk_obj.save() + return chunk_obj + + @item_router.put( "/item/{item_uid}/chunk/{chunk_uid}/", dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE], status_code=status.HTTP_201_CREATED, ) -def chunk_update( - limit: int = 50, - iterator: t.Optional[str] = None, - prefetch: Prefetch = PrefetchQuery, - user: User = Depends(get_authenticated_user), +async def chunk_update( + request: Request, + chunk_uid: str, collection: models.Collection = Depends(get_collection), ): # IGNORED FOR NOW: col_it = get_object_or_404(col.items, uid=collection_item_uid) - - data = { - "uid": chunk_uid, - "chunkFile": request.data["file"], - } - - serializer = self.get_serializer_class()(data=data) - serializer.is_valid(raise_exception=True) + content_file = ContentFile(await request.body()) try: - serializer.save(collection=col) + await chunk_save(chunk_uid, collection, content_file) except IntegrityError: - return Response({"code": "chunk_exists", "detail": "Chunk already exists."}, status=status.HTTP_409_CONFLICT) + raise HttpError("chunk_exists", "Chunk already exists.", status_code=status.HTTP_409_CONFLICT) From c7f09d3fef2935b81ac3a9232a6b88215b73300d Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 18:25:06 +0200 Subject: [PATCH 095/102] implement chunk_download. --- etebase_fastapi/collection.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index b3f8c5f..c5b0801 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -7,6 +7,7 @@ from django.core.files.base import ContentFile from django.db import transaction, IntegrityError from django.db.models import Q, QuerySet from fastapi import APIRouter, Depends, status, Request +from fastapi.responses import FileResponse from django_etebase import models from .authentication import get_authenticated_user @@ -568,3 +569,17 @@ async def chunk_update( await chunk_save(chunk_uid, collection, content_file) except IntegrityError: raise HttpError("chunk_exists", "Chunk already exists.", status_code=status.HTTP_409_CONFLICT) + + +@item_router.get( + "/item/{item_uid}/chunk/{chunk_uid}/download/", + dependencies=PERMISSIONS_READ, +) +def chunk_download( + chunk_uid: str, + collection: models.Collection = Depends(get_collection), +): + chunk = get_object_or_404(collection.chunks, uid=chunk_uid) + + filename = chunk.chunkFile.path + return FileResponse(filename, media_type="application/octet-stream") From f0a8689712601ca2ac5b98b5c3c4aad4f6d58808 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 18:44:55 +0200 Subject: [PATCH 096/102] Implement sendfile. --- .../sendfile/backends/development.py | 17 ------ django_etebase/sendfile/backends/mod_wsgi.py | 17 ------ django_etebase/sendfile/backends/nginx.py | 12 ---- django_etebase/sendfile/backends/simple.py | 60 ------------------- django_etebase/sendfile/backends/xsendfile.py | 9 --- etebase_fastapi/collection.py | 4 +- .../sendfile/LICENSE | 0 .../sendfile/README.md | 0 .../sendfile/__init__.py | 0 .../sendfile/backends/__init__.py | 0 etebase_fastapi/sendfile/backends/mod_wsgi.py | 9 +++ etebase_fastapi/sendfile/backends/nginx.py | 9 +++ etebase_fastapi/sendfile/backends/simple.py | 12 ++++ .../sendfile/backends/xsendfile.py | 6 ++ .../sendfile/utils.py | 15 ++--- 15 files changed, 46 insertions(+), 124 deletions(-) delete mode 100644 django_etebase/sendfile/backends/development.py delete mode 100644 django_etebase/sendfile/backends/mod_wsgi.py delete mode 100644 django_etebase/sendfile/backends/nginx.py delete mode 100644 django_etebase/sendfile/backends/simple.py delete mode 100644 django_etebase/sendfile/backends/xsendfile.py rename {django_etebase => etebase_fastapi}/sendfile/LICENSE (100%) rename {django_etebase => etebase_fastapi}/sendfile/README.md (100%) rename {django_etebase => etebase_fastapi}/sendfile/__init__.py (100%) rename {django_etebase => etebase_fastapi}/sendfile/backends/__init__.py (100%) create mode 100644 etebase_fastapi/sendfile/backends/mod_wsgi.py create mode 100644 etebase_fastapi/sendfile/backends/nginx.py create mode 100644 etebase_fastapi/sendfile/backends/simple.py create mode 100644 etebase_fastapi/sendfile/backends/xsendfile.py rename {django_etebase => etebase_fastapi}/sendfile/utils.py (81%) diff --git a/django_etebase/sendfile/backends/development.py b/django_etebase/sendfile/backends/development.py deleted file mode 100644 index d321932..0000000 --- a/django_etebase/sendfile/backends/development.py +++ /dev/null @@ -1,17 +0,0 @@ -import os.path - -from django.views.static import serve - - -def sendfile(request, filename, **kwargs): - """ - Send file using Django dev static file server. - - .. warning:: - - Do not use in production. This is only to be used when developing and - is provided for convenience only - """ - dirname = os.path.dirname(filename) - basename = os.path.basename(filename) - return serve(request, basename, dirname) diff --git a/django_etebase/sendfile/backends/mod_wsgi.py b/django_etebase/sendfile/backends/mod_wsgi.py deleted file mode 100644 index 07ba3f1..0000000 --- a/django_etebase/sendfile/backends/mod_wsgi.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import absolute_import - -from django.http import HttpResponse - -from ..utils import _convert_file_to_url - - -def sendfile(request, filename, **kwargs): - response = HttpResponse() - response['Location'] = _convert_file_to_url(filename) - # need to destroy get_host() to stop django - # rewriting our location to include http, so that - # mod_wsgi is able to do the internal redirect - request.get_host = lambda: '' - request.build_absolute_uri = lambda location: location - - return response diff --git a/django_etebase/sendfile/backends/nginx.py b/django_etebase/sendfile/backends/nginx.py deleted file mode 100644 index 8764309..0000000 --- a/django_etebase/sendfile/backends/nginx.py +++ /dev/null @@ -1,12 +0,0 @@ -from __future__ import absolute_import - -from django.http import HttpResponse - -from ..utils import _convert_file_to_url - - -def sendfile(request, filename, **kwargs): - response = HttpResponse() - response['X-Accel-Redirect'] = _convert_file_to_url(filename) - - return response diff --git a/django_etebase/sendfile/backends/simple.py b/django_etebase/sendfile/backends/simple.py deleted file mode 100644 index 0549b20..0000000 --- a/django_etebase/sendfile/backends/simple.py +++ /dev/null @@ -1,60 +0,0 @@ -from email.utils import mktime_tz, parsedate_tz -import re - -from django.core.files.base import File -from django.http import HttpResponse, HttpResponseNotModified -from django.utils.http import http_date - - -def sendfile(request, filepath, **kwargs): - '''Use the SENDFILE_ROOT value composed with the path arrived as argument - to build an absolute path with which resolve and return the file contents. - - If the path points to a file out of the root directory (should cover both - situations with '..' and symlinks) then a 404 is raised. - ''' - statobj = filepath.stat() - - # Respect the If-Modified-Since header. - if not was_modified_since(request.META.get('HTTP_IF_MODIFIED_SINCE'), - statobj.st_mtime, statobj.st_size): - return HttpResponseNotModified() - - with File(filepath.open('rb')) as f: - response = HttpResponse(f.chunks()) - - response["Last-Modified"] = http_date(statobj.st_mtime) - return response - - -def was_modified_since(header=None, mtime=0, size=0): - """ - Was something modified since the user last downloaded it? - - header - This is the value of the If-Modified-Since header. If this is None, - I'll just return True. - - mtime - This is the modification time of the item we're talking about. - - size - This is the size of the item we're talking about. - """ - try: - if header is None: - raise ValueError - matches = re.match(r"^([^;]+)(; length=([0-9]+))?$", header, - re.IGNORECASE) - header_date = parsedate_tz(matches.group(1)) - if header_date is None: - raise ValueError - header_mtime = mktime_tz(header_date) - header_len = matches.group(3) - if header_len and int(header_len) != size: - raise ValueError - if mtime > header_mtime: - raise ValueError - except (AttributeError, ValueError, OverflowError): - return True - return False diff --git a/django_etebase/sendfile/backends/xsendfile.py b/django_etebase/sendfile/backends/xsendfile.py deleted file mode 100644 index 74993ee..0000000 --- a/django_etebase/sendfile/backends/xsendfile.py +++ /dev/null @@ -1,9 +0,0 @@ -from django.http import HttpResponse - - -def sendfile(request, filename, **kwargs): - filename = str(filename) - response = HttpResponse() - response['X-Sendfile'] = filename - - return response diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index c5b0801..3b672cc 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -7,7 +7,6 @@ from django.core.files.base import ContentFile from django.db import transaction, IntegrityError from django.db.models import Q, QuerySet from fastapi import APIRouter, Depends, status, Request -from fastapi.responses import FileResponse from django_etebase import models from .authentication import get_authenticated_user @@ -26,6 +25,7 @@ from .utils import ( PERMISSIONS_READWRITE, ) from .dependencies import get_collection_queryset, get_item_queryset, get_collection +from .sendfile import sendfile User = get_user_model() collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) @@ -582,4 +582,4 @@ def chunk_download( chunk = get_object_or_404(collection.chunks, uid=chunk_uid) filename = chunk.chunkFile.path - return FileResponse(filename, media_type="application/octet-stream") + return sendfile(filename) diff --git a/django_etebase/sendfile/LICENSE b/etebase_fastapi/sendfile/LICENSE similarity index 100% rename from django_etebase/sendfile/LICENSE rename to etebase_fastapi/sendfile/LICENSE diff --git a/django_etebase/sendfile/README.md b/etebase_fastapi/sendfile/README.md similarity index 100% rename from django_etebase/sendfile/README.md rename to etebase_fastapi/sendfile/README.md diff --git a/django_etebase/sendfile/__init__.py b/etebase_fastapi/sendfile/__init__.py similarity index 100% rename from django_etebase/sendfile/__init__.py rename to etebase_fastapi/sendfile/__init__.py diff --git a/django_etebase/sendfile/backends/__init__.py b/etebase_fastapi/sendfile/backends/__init__.py similarity index 100% rename from django_etebase/sendfile/backends/__init__.py rename to etebase_fastapi/sendfile/backends/__init__.py diff --git a/etebase_fastapi/sendfile/backends/mod_wsgi.py b/etebase_fastapi/sendfile/backends/mod_wsgi.py new file mode 100644 index 0000000..b8fc6c0 --- /dev/null +++ b/etebase_fastapi/sendfile/backends/mod_wsgi.py @@ -0,0 +1,9 @@ +from __future__ import absolute_import + +from fastapi import Response + +from ..utils import _convert_file_to_url + + +def sendfile(filename, **kwargs): + return Response(headers={"Location": _convert_file_to_url(filename)}) diff --git a/etebase_fastapi/sendfile/backends/nginx.py b/etebase_fastapi/sendfile/backends/nginx.py new file mode 100644 index 0000000..b22e0d0 --- /dev/null +++ b/etebase_fastapi/sendfile/backends/nginx.py @@ -0,0 +1,9 @@ +from __future__ import absolute_import + +from fastapi import Response + +from ..utils import _convert_file_to_url + + +def sendfile(filename, **kwargs): + return Response(headers={"X-Accel-Redirect": _convert_file_to_url(filename)}) diff --git a/etebase_fastapi/sendfile/backends/simple.py b/etebase_fastapi/sendfile/backends/simple.py new file mode 100644 index 0000000..f3a3548 --- /dev/null +++ b/etebase_fastapi/sendfile/backends/simple.py @@ -0,0 +1,12 @@ +from fastapi.responses import FileResponse + + +def sendfile(filename, mimetype, **kwargs): + """Use the SENDFILE_ROOT value composed with the path arrived as argument + to build an absolute path with which resolve and return the file contents. + + If the path points to a file out of the root directory (should cover both + situations with '..' and symlinks) then a 404 is raised. + """ + + return FileResponse(filename, media_type=mimetype) diff --git a/etebase_fastapi/sendfile/backends/xsendfile.py b/etebase_fastapi/sendfile/backends/xsendfile.py new file mode 100644 index 0000000..530f6a1 --- /dev/null +++ b/etebase_fastapi/sendfile/backends/xsendfile.py @@ -0,0 +1,6 @@ +from fastapi import Response + + +def sendfile(filename, **kwargs): + filename = str(filename) + return Response(headers={"X-Sendfile": filename}) diff --git a/django_etebase/sendfile/utils.py b/etebase_fastapi/sendfile/utils.py similarity index 81% rename from django_etebase/sendfile/utils.py rename to etebase_fastapi/sendfile/utils.py index 97c06d7..7c8b1f2 100644 --- a/django_etebase/sendfile/utils.py +++ b/etebase_fastapi/sendfile/utils.py @@ -4,9 +4,11 @@ from pathlib import Path, PurePath from urllib.parse import quote import logging +from fastapi import status +from ..exceptions import HttpError + from django.conf import settings from django.core.exceptions import ImproperlyConfigured -from django.http import Http404 logger = logging.getLogger(__name__) @@ -54,12 +56,12 @@ def _sanitize_path(filepath): try: filepath_abs.relative_to(path_root) except ValueError: - raise Http404("{} wrt {} is impossible".format(filepath_abs, path_root)) + raise HttpError("generic", "{} wrt {} is impossible".format(filepath_abs, path_root), status_code=status.HTTP_404_NOT_FOUND) return filepath_abs -def sendfile(request, filename, mimetype="application/octet-stream", encoding=None): +def sendfile(filename, mimetype="application/octet-stream", encoding=None): """ Create a response to send file using backend configured in ``SENDFILE_BACKEND`` @@ -75,11 +77,10 @@ def sendfile(request, filename, mimetype="application/octet-stream", encoding=No _sendfile = _get_sendfile() if not filepath_obj.exists(): - raise Http404('"%s" does not exist' % filepath_obj) + raise HttpError("does_not_exist", '"%s" does not exist' % filepath_obj, status_code=status.HTTP_404_NOT_FOUND) - response = _sendfile(request, filepath_obj, mimetype=mimetype) + response = _sendfile(filepath_obj, mimetype=mimetype) - response["Content-length"] = filepath_obj.stat().st_size - response["Content-Type"] = mimetype + response.headers["Content-Type"] = mimetype return response From 7714148807a183ef4eb820232d4014662441372a Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Mon, 28 Dec 2020 18:49:05 +0200 Subject: [PATCH 097/102] Use ValidationError when appropriate. --- etebase_fastapi/collection.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 3b672cc..3e4d3e8 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -285,7 +285,7 @@ def process_revisions_for_item(item: models.CollectionItem, revision_data: Colle chunk_obj.chunkFile.save("IGNORED", ContentFile(content)) chunk_obj.save() else: - raise HttpError("chunk_no_content", "Tried to create a new chunk without content") + raise ValidationError("chunk_no_content", "Tried to create a new chunk without content") chunks_objs.append(chunk_obj) @@ -301,7 +301,7 @@ def process_revisions_for_item(item: models.CollectionItem, revision_data: Colle def _create(data: CollectionIn, user: User): with transaction.atomic(): if data.item.etag is not None: - raise HttpError("bad_etag", "etag is not null") + raise ValidationError("bad_etag", "etag is not null") instance = models.Collection(uid=data.item.uid, owner=user) try: instance.validate_unique() From a8b97e60d407659f363d3486cbde1f8ad5d7bdd7 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 09:46:20 +0200 Subject: [PATCH 098/102] Docs: improve metadata. --- etebase_fastapi/main.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/etebase_fastapi/main.py b/etebase_fastapi/main.py index a55e2fa..69303bf 100644 --- a/etebase_fastapi/main.py +++ b/etebase_fastapi/main.py @@ -14,7 +14,15 @@ from .msgpack import MsgpackResponse def create_application(prefix=""): - app = FastAPI() + app = FastAPI( + title="Etebase", + description="The Etebase server API documentation", + externalDocs={ + "url": "https://docs.etebase.com", + "description": "Docs about the API specifications and clients.", + } + # FIXME: version="2.5.0", + ) VERSION = "v1" BASE_PATH = f"{prefix}/api/{VERSION}" COLLECTION_UID_MARKER = "{collection_uid}" From f67730f42d2a78acadf28db253bf76c0f3630ba2 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 10:12:36 +0200 Subject: [PATCH 099/102] Support passing custom middlewares. --- etebase_fastapi/main.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/etebase_fastapi/main.py b/etebase_fastapi/main.py index 69303bf..534798e 100644 --- a/etebase_fastapi/main.py +++ b/etebase_fastapi/main.py @@ -13,7 +13,7 @@ from .invitation import invitation_incoming_router, invitation_outgoing_router from .msgpack import MsgpackResponse -def create_application(prefix=""): +def create_application(prefix="", middlewares=[]): app = FastAPI( title="Etebase", description="The Etebase server API documentation", @@ -51,6 +51,9 @@ def create_application(prefix=""): ) app.add_middleware(TrustedHostMiddleware, allowed_hosts=settings.ALLOWED_HOSTS) + for middleware in middlewares: + app.add_middleware(middleware) + @app.exception_handler(CustomHttpException) async def custom_exception_handler(request: Request, exc: CustomHttpException): return MsgpackResponse(status_code=exc.status_code, content=exc.as_dict) From e13f26ec56f03321ad5f8471f4a5a33f4a6f0be0 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 10:27:35 +0200 Subject: [PATCH 100/102] Fix handling of legacy accounts that don't have collection type. --- etebase_fastapi/collection.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 3e4d3e8..5c6e6b6 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -88,7 +88,8 @@ class CollectionItemIn(CollectionItemCommon): class CollectionCommon(BaseModel): - collectionType: bytes + # FIXME: remove optional once we finish collection-type-migration + collectionType: t.Optional[bytes] collectionKey: bytes From 794b5f398347985d7a354acc16e8fcd536600358 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 13:22:36 +0200 Subject: [PATCH 101/102] Fix many type errors. --- django_etebase/token_auth/models.py | 4 +-- django_etebase/utils.py | 6 ++-- etebase_fastapi/authentication.py | 27 ++++++++--------- etebase_fastapi/collection.py | 45 +++++++++++++++-------------- etebase_fastapi/dependencies.py | 10 +++---- etebase_fastapi/invitation.py | 15 +++++----- etebase_fastapi/member.py | 8 +++-- etebase_fastapi/msgpack.py | 6 ++-- etebase_fastapi/test_reset_view.py | 4 +-- etebase_fastapi/utils.py | 6 ++-- myauth/forms.py | 4 +-- myauth/models.py | 16 +++++++++- 12 files changed, 87 insertions(+), 64 deletions(-) diff --git a/django_etebase/token_auth/models.py b/django_etebase/token_auth/models.py index ac1efff..dd5ae87 100644 --- a/django_etebase/token_auth/models.py +++ b/django_etebase/token_auth/models.py @@ -1,9 +1,9 @@ -from django.contrib.auth import get_user_model from django.db import models from django.utils import timezone from django.utils.crypto import get_random_string +from myauth.models import get_typed_user_model -User = get_user_model() +User = get_typed_user_model() def generate_key(): diff --git a/django_etebase/utils.py b/django_etebase/utils.py index 4d36a94..d812ae3 100644 --- a/django_etebase/utils.py +++ b/django_etebase/utils.py @@ -1,13 +1,13 @@ import typing as t from dataclasses import dataclass -from django.contrib.auth import get_user_model from django.core.exceptions import PermissionDenied +from myauth.models import UserType, get_typed_user_model from . import app_settings -User = get_user_model() +User = get_typed_user_model() @dataclass @@ -15,7 +15,7 @@ class CallbackContext: """Class for passing extra context to callbacks""" url_kwargs: t.Dict[str, t.Any] - user: t.Optional[User] = None + user: t.Optional[UserType] = None def get_user_queryset(queryset, context: CallbackContext): diff --git a/etebase_fastapi/authentication.py b/etebase_fastapi/authentication.py index fe522f7..064d2da 100644 --- a/etebase_fastapi/authentication.py +++ b/etebase_fastapi/authentication.py @@ -9,7 +9,7 @@ import nacl.secret import nacl.signing from asgiref.sync import sync_to_async from django.conf import settings -from django.contrib.auth import get_user_model, user_logged_out, user_logged_in +from django.contrib.auth import user_logged_out, user_logged_in from django.core import exceptions as django_exceptions from django.db import transaction from fastapi import APIRouter, Depends, status, Request @@ -19,12 +19,13 @@ from django_etebase.token_auth.models import AuthToken from django_etebase.models import UserInfo from django_etebase.signals import user_signed_up from django_etebase.utils import create_user, get_user_queryset, CallbackContext +from myauth.models import UserType, get_typed_user_model from .exceptions import AuthenticationFailed, transform_validation_error, HttpError from .msgpack import MsgpackRoute from .utils import BaseModel, permission_responses, msgpack_encode, msgpack_decode from .dependencies import AuthData, get_auth_data, get_authenticated_user -User = get_user_model() +User = get_typed_user_model() authentication_router = APIRouter(route_class=MsgpackRoute) @@ -52,7 +53,7 @@ class UserOut(BaseModel): encryptedContent: bytes @classmethod - def from_orm(cls: t.Type["UserOut"], obj: User) -> "UserOut": + def from_orm(cls: t.Type["UserOut"], obj: UserType) -> "UserOut": return cls( username=obj.username, email=obj.email, @@ -66,7 +67,7 @@ class LoginOut(BaseModel): user: UserOut @classmethod - def from_orm(cls: t.Type["LoginOut"], obj: User) -> "LoginOut": + def from_orm(cls: t.Type["LoginOut"], obj: UserType) -> "LoginOut": token = AuthToken.objects.create(user=obj).key user = UserOut.from_orm(obj) return cls(token=token, user=user) @@ -111,7 +112,7 @@ class SignupIn(BaseModel): @sync_to_async -def __get_login_user(username: str) -> User: +def __get_login_user(username: str) -> UserType: kwargs = {User.USERNAME_FIELD + "__iexact": username.lower()} try: user = User.objects.get(**kwargs) @@ -122,7 +123,7 @@ def __get_login_user(username: str) -> User: raise AuthenticationFailed(code="user_not_found", detail="User not found") -async def get_login_user(challenge: LoginChallengeIn) -> User: +async def get_login_user(challenge: LoginChallengeIn) -> UserType: user = await __get_login_user(challenge.username) return user @@ -138,7 +139,7 @@ def get_encryption_key(salt): ) -def save_changed_password(data: ChangePassword, user: User): +def save_changed_password(data: ChangePassword, user: UserType): response_data = data.response_data user_info: UserInfo = user.userinfo user_info.loginPubkey = response_data.loginPubkey @@ -150,7 +151,7 @@ def save_changed_password(data: ChangePassword, user: User): def validate_login_request( validated_data: LoginResponse, challenge_sent_to_user: Authentication, - user: User, + user: UserType, expected_action: str, host_from_request: str, ): @@ -159,7 +160,7 @@ def validate_login_request( challenge_data = msgpack_decode(box.decrypt(validated_data.challenge)) now = int(datetime.now().timestamp()) if validated_data.action != expected_action: - raise HttpError("wrong_action", f'Expected "{challenge_sent_to_user.response}" but got something else') + raise HttpError("wrong_action", f'Expected "{expected_action}" but got something else') elif now - challenge_data["timestamp"] > app_settings.CHALLENGE_VALID_SECONDS: raise HttpError("challenge_expired", "Login challenge has expired") elif challenge_data["userId"] != user.id: @@ -181,7 +182,7 @@ async def is_etebase(): @authentication_router.post("/login_challenge/", response_model=LoginChallengeOut) -def login_challenge(user: User = Depends(get_login_user)): +def login_challenge(user: UserType = Depends(get_login_user)): salt = bytes(user.userinfo.salt) enc_key = get_encryption_key(salt) box = nacl.secret.SecretBox(enc_key) @@ -210,14 +211,14 @@ def logout(auth_data: AuthData = Depends(get_auth_data)): @authentication_router.post("/change_password/", status_code=status.HTTP_204_NO_CONTENT, responses=permission_responses) -async def change_password(data: ChangePassword, request: Request, user: User = Depends(get_authenticated_user)): +async def change_password(data: ChangePassword, request: Request, user: UserType = Depends(get_authenticated_user)): host = request.headers.get("Host") await validate_login_request(data.response_data, data, user, "changePassword", host) await sync_to_async(save_changed_password)(data, user) @authentication_router.post("/dashboard_url/", responses=permission_responses) -def dashboard_url(request: Request, user: User = Depends(get_authenticated_user)): +def dashboard_url(request: Request, user: UserType = Depends(get_authenticated_user)): get_dashboard_url = app_settings.DASHBOARD_URL_FUNC if get_dashboard_url is None: raise HttpError("not_supported", "This server doesn't have a user dashboard.") @@ -228,7 +229,7 @@ def dashboard_url(request: Request, user: User = Depends(get_authenticated_user) return ret -def signup_save(data: SignupIn, request: Request) -> User: +def signup_save(data: SignupIn, request: Request) -> UserType: user_data = data.user with transaction.atomic(): try: diff --git a/etebase_fastapi/collection.py b/etebase_fastapi/collection.py index 5c6e6b6..9e25b38 100644 --- a/etebase_fastapi/collection.py +++ b/etebase_fastapi/collection.py @@ -1,7 +1,6 @@ import typing as t from asgiref.sync import sync_to_async -from django.contrib.auth import get_user_model from django.core import exceptions as django_exceptions from django.core.files.base import ContentFile from django.db import transaction, IntegrityError @@ -9,6 +8,7 @@ from django.db.models import Q, QuerySet from fastapi import APIRouter, Depends, status, Request from django_etebase import models +from myauth.models import UserType, get_typed_user_model from .authentication import get_authenticated_user from .exceptions import HttpError, transform_validation_error, PermissionDenied, ValidationError from .msgpack import MsgpackRoute @@ -27,7 +27,7 @@ from .utils import ( from .dependencies import get_collection_queryset, get_item_queryset, get_collection from .sendfile import sendfile -User = get_user_model() +User = get_typed_user_model collection_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) item_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) @@ -36,11 +36,14 @@ class ListMulti(BaseModel): collectionTypes: t.List[bytes] +ChunkType = t.Tuple[str, t.Optional[bytes]] + + class CollectionItemRevisionInOut(BaseModel): uid: str meta: bytes deleted: bool - chunks: t.List[t.Tuple[str, t.Optional[bytes]]] + chunks: t.List[ChunkType] class Config: orm_mode = True @@ -49,7 +52,7 @@ class CollectionItemRevisionInOut(BaseModel): def from_orm_context( cls: t.Type["CollectionItemRevisionInOut"], obj: models.CollectionItemRevision, context: Context ) -> "CollectionItemRevisionInOut": - chunks = [] + chunks: t.List[ChunkType] = [] for chunk_relation in obj.chunks_relation.all(): chunk_obj = chunk_relation.chunk if context.prefetch == "auto": @@ -185,7 +188,7 @@ class ItemBatchIn(BaseModel): @sync_to_async def collection_list_common( queryset: QuerySet, - user: User, + user: UserType, stoken: t.Optional[str], limit: int, prefetch: Prefetch, @@ -210,7 +213,7 @@ def collection_list_common( remed = remed_qs.values_list("collection__uid", flat=True) if len(remed) > 0: - ret.removedMemberships = [{"uid": x} for x in remed] + ret.removedMemberships = [RemovedMembershipOut(uid=x) for x in remed] return ret @@ -219,14 +222,14 @@ def collection_list_common( def verify_collection_admin( - collection: models.Collection = Depends(get_collection), user: User = Depends(get_authenticated_user) + collection: models.Collection = Depends(get_collection), user: UserType = Depends(get_authenticated_user) ): if not is_collection_admin(collection, user): raise PermissionDenied("admin_access_required", "Only collection admins can perform this operation.") def has_write_access( - collection: models.Collection = Depends(get_collection), user: User = Depends(get_authenticated_user) + collection: models.Collection = Depends(get_collection), user: UserType = Depends(get_authenticated_user) ): member = collection.members.get(user=user) if member.accessLevel == models.AccessLevels.READ_ONLY: @@ -247,7 +250,7 @@ async def list_multi( stoken: t.Optional[str] = None, limit: int = 50, queryset: QuerySet = Depends(get_collection_queryset), - user: User = Depends(get_authenticated_user), + user: UserType = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, ): # FIXME: Remove the isnull part once we attach collection types to all objects ("collection-type-migration") @@ -263,7 +266,7 @@ async def collection_list( stoken: t.Optional[str] = None, limit: int = 50, prefetch: Prefetch = PrefetchQuery, - user: User = Depends(get_authenticated_user), + user: UserType = Depends(get_authenticated_user), queryset: QuerySet = Depends(get_collection_queryset), ): return await collection_list_common(queryset, user, stoken, limit, prefetch) @@ -299,7 +302,7 @@ def process_revisions_for_item(item: models.CollectionItem, revision_data: Colle return revision -def _create(data: CollectionIn, user: User): +def _create(data: CollectionIn, user: UserType): with transaction.atomic(): if data.item.etag is not None: raise ValidationError("bad_etag", "etag is not null") @@ -335,14 +338,14 @@ def _create(data: CollectionIn, user: User): @collection_router.post("/", status_code=status.HTTP_201_CREATED, dependencies=PERMISSIONS_READWRITE) -async def create(data: CollectionIn, user: User = Depends(get_authenticated_user)): +async def create(data: CollectionIn, user: UserType = Depends(get_authenticated_user)): await sync_to_async(_create)(data, user) @collection_router.get("/{collection_uid}/", response_model=CollectionOut, dependencies=PERMISSIONS_READ) def collection_get( obj: models.Collection = Depends(get_collection), - user: User = Depends(get_authenticated_user), + user: UserType = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, ): return CollectionOut.from_orm_context(obj, Context(user, prefetch)) @@ -393,7 +396,7 @@ def item_create(item_model: CollectionItemIn, collection: models.Collection, val def item_get( item_uid: str, queryset: QuerySet = Depends(get_item_queryset), - user: User = Depends(get_authenticated_user), + user: UserType = Depends(get_authenticated_user), prefetch: Prefetch = PrefetchQuery, ): obj = queryset.get(uid=item_uid) @@ -403,7 +406,7 @@ def item_get( @sync_to_async def item_list_common( queryset: QuerySet, - user: User, + user: UserType, stoken: t.Optional[str], limit: int, prefetch: Prefetch, @@ -424,7 +427,7 @@ async def item_list( limit: int = 50, prefetch: Prefetch = PrefetchQuery, withCollection: bool = False, - user: User = Depends(get_authenticated_user), + user: UserType = Depends(get_authenticated_user), ): if not withCollection: queryset = queryset.filter(parent__isnull=True) @@ -433,7 +436,7 @@ async def item_list( return response -def item_bulk_common(data: ItemBatchIn, user: User, stoken: t.Optional[str], uid: str, validate_etag: bool): +def item_bulk_common(data: ItemBatchIn, user: UserType, stoken: t.Optional[str], uid: str, validate_etag: bool): queryset = get_collection_queryset(user) with transaction.atomic(): # We need this for locking the collection object collection_object = queryset.select_for_update().get(uid=uid) @@ -467,7 +470,7 @@ def item_revisions( limit: int = 50, iterator: t.Optional[str] = None, prefetch: Prefetch = PrefetchQuery, - user: User = Depends(get_authenticated_user), + user: UserType = Depends(get_authenticated_user), items: QuerySet = Depends(get_item_queryset), ): item = get_object_or_404(items, uid=item_uid) @@ -501,7 +504,7 @@ def fetch_updates( data: t.List[CollectionItemBulkGetIn], stoken: t.Optional[str] = None, prefetch: Prefetch = PrefetchQuery, - user: User = Depends(get_authenticated_user), + user: UserType = Depends(get_authenticated_user), queryset: QuerySet = Depends(get_item_queryset), ): # FIXME: make configurable? @@ -531,14 +534,14 @@ def fetch_updates( @item_router.post("/item/transaction/", dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE]) def item_transaction( - collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) + collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: UserType = Depends(get_authenticated_user) ): return item_bulk_common(data, user, stoken, collection_uid, validate_etag=True) @item_router.post("/item/batch/", dependencies=[Depends(has_write_access), *PERMISSIONS_READWRITE]) def item_batch( - collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: User = Depends(get_authenticated_user) + collection_uid: str, data: ItemBatchIn, stoken: t.Optional[str] = None, user: UserType = Depends(get_authenticated_user) ): return item_bulk_common(data, user, stoken, collection_uid, validate_etag=False) diff --git a/etebase_fastapi/dependencies.py b/etebase_fastapi/dependencies.py index ddb9b3b..fb9cec5 100644 --- a/etebase_fastapi/dependencies.py +++ b/etebase_fastapi/dependencies.py @@ -3,17 +3,17 @@ import dataclasses from fastapi import Depends from fastapi.security import APIKeyHeader -from django.contrib.auth import get_user_model from django.utils import timezone from django.db.models import QuerySet from django_etebase import models from django_etebase.token_auth.models import AuthToken, get_default_expiry +from myauth.models import UserType, get_typed_user_model from .exceptions import AuthenticationFailed from .utils import get_object_or_404 -User = get_user_model() +User = get_typed_user_model() token_scheme = APIKeyHeader(name="Authorization") AUTO_REFRESH = True MIN_REFRESH_INTERVAL = 60 @@ -21,7 +21,7 @@ MIN_REFRESH_INTERVAL = 60 @dataclasses.dataclass(frozen=True) class AuthData: - user: User + user: UserType token: AuthToken @@ -60,12 +60,12 @@ def get_auth_data(api_token: str = Depends(token_scheme)) -> AuthData: return AuthData(user, token) -def get_authenticated_user(api_token: str = Depends(token_scheme)) -> User: +def get_authenticated_user(api_token: str = Depends(token_scheme)) -> UserType: user, _ = __get_authenticated_user(api_token) return user -def get_collection_queryset(user: User = Depends(get_authenticated_user)) -> QuerySet: +def get_collection_queryset(user: UserType = Depends(get_authenticated_user)) -> QuerySet: default_queryset: QuerySet = models.Collection.objects.all() return default_queryset.filter(members__user=user) diff --git a/etebase_fastapi/invitation.py b/etebase_fastapi/invitation.py index 9e731bc..eb9f549 100644 --- a/etebase_fastapi/invitation.py +++ b/etebase_fastapi/invitation.py @@ -1,12 +1,12 @@ import typing as t -from django.contrib.auth import get_user_model from django.db import transaction, IntegrityError from django.db.models import QuerySet from fastapi import APIRouter, Depends, status, Request from django_etebase import models from django_etebase.utils import get_user_queryset, CallbackContext +from myauth.models import UserType, get_typed_user_model from .authentication import get_authenticated_user from .exceptions import HttpError, PermissionDenied from .msgpack import MsgpackRoute @@ -20,7 +20,7 @@ from .utils import ( PERMISSIONS_READWRITE, ) -User = get_user_model() +User = get_typed_user_model() invitation_incoming_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) invitation_outgoing_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) default_queryset: QuerySet = models.CollectionInvitation.objects.all() @@ -53,7 +53,8 @@ class CollectionInvitationCommon(BaseModel): class CollectionInvitationIn(CollectionInvitationCommon): def validate_db(self, context: Context): - if context.user.username == self.username.lower(): + user = context.user + if user is not None and (user.username == self.username.lower()): raise HttpError("no_self_invite", "Inviting yourself is not allowed") @@ -84,11 +85,11 @@ class InvitationListResponse(BaseModel): done: bool -def get_incoming_queryset(user: User = Depends(get_authenticated_user)): +def get_incoming_queryset(user: UserType = Depends(get_authenticated_user)): return default_queryset.filter(user=user) -def get_outgoing_queryset(user: User = Depends(get_authenticated_user)): +def get_outgoing_queryset(user: UserType = Depends(get_authenticated_user)): return default_queryset.filter(fromMember__user=user) @@ -183,7 +184,7 @@ def incoming_accept( def outgoing_create( data: CollectionInvitationIn, request: Request, - user: User = Depends(get_authenticated_user), + user: UserType = Depends(get_authenticated_user), ): collection = get_object_or_404(models.Collection.objects, uid=data.collection) to_user = get_object_or_404( @@ -231,7 +232,7 @@ def outgoing_delete( def outgoing_fetch_user_profile( username: str, request: Request, - user: User = Depends(get_authenticated_user), + user: UserType = Depends(get_authenticated_user), ): kwargs = {User.USERNAME_FIELD: username.lower()} user = get_object_or_404(get_user_queryset(User.objects.all(), CallbackContext(request.path_params)), **kwargs) diff --git a/etebase_fastapi/member.py b/etebase_fastapi/member.py index 725d44b..22977ac 100644 --- a/etebase_fastapi/member.py +++ b/etebase_fastapi/member.py @@ -1,11 +1,11 @@ import typing as t -from django.contrib.auth import get_user_model from django.db import transaction from django.db.models import QuerySet from fastapi import APIRouter, Depends, status from django_etebase import models +from myauth.models import UserType, get_typed_user_model from .authentication import get_authenticated_user from .msgpack import MsgpackRoute from .utils import get_object_or_404, BaseModel, permission_responses, PERMISSIONS_READ, PERMISSIONS_READWRITE @@ -13,7 +13,7 @@ from .stoken_handler import filter_by_stoken_and_limit from .collection import get_collection, verify_collection_admin -User = get_user_model() +User = get_typed_user_model() member_router = APIRouter(route_class=MsgpackRoute, responses=permission_responses) default_queryset: QuerySet = models.CollectionMember.objects.all() @@ -98,6 +98,8 @@ def member_patch( @member_router.post("/member/leave/", status_code=status.HTTP_204_NO_CONTENT, dependencies=PERMISSIONS_READ) -def member_leave(user: User = Depends(get_authenticated_user), collection: models.Collection = Depends(get_collection)): +def member_leave( + user: UserType = Depends(get_authenticated_user), collection: models.Collection = Depends(get_collection) +): obj = get_object_or_404(collection.members, user=user) obj.revoke() diff --git a/etebase_fastapi/msgpack.py b/etebase_fastapi/msgpack.py index edffd7e..915e783 100644 --- a/etebase_fastapi/msgpack.py +++ b/etebase_fastapi/msgpack.py @@ -19,13 +19,15 @@ class MsgpackRequest(Request): class MsgpackResponse(Response): media_type = "application/msgpack" - def render(self, content: t.Optional[t.Any]) -> t.Optional[bytes]: + def render(self, content: t.Optional[t.Any]) -> bytes: if content is None: return b"" if isinstance(content, BaseModel): content = content.dict() - return msgpack.packb(content, use_bin_type=True) + ret = msgpack.packb(content, use_bin_type=True) + assert ret is not None + return ret class MsgpackRoute(APIRoute): diff --git a/etebase_fastapi/test_reset_view.py b/etebase_fastapi/test_reset_view.py index 3075290..e328875 100644 --- a/etebase_fastapi/test_reset_view.py +++ b/etebase_fastapi/test_reset_view.py @@ -1,5 +1,4 @@ from django.conf import settings -from django.contrib.auth import get_user_model from django.db import transaction from django.shortcuts import get_object_or_404 from fastapi import APIRouter, Request, status @@ -8,9 +7,10 @@ from django_etebase.utils import get_user_queryset, CallbackContext from etebase_fastapi.authentication import SignupIn, signup_save from etebase_fastapi.msgpack import MsgpackRoute from etebase_fastapi.exceptions import HttpError +from myauth.models import get_typed_user_model test_reset_view_router = APIRouter(route_class=MsgpackRoute, tags=["test helpers"]) -User = get_user_model() +User = get_typed_user_model() @test_reset_view_router.post("/reset/", status_code=status.HTTP_204_NO_CONTENT) diff --git a/etebase_fastapi/utils.py b/etebase_fastapi/utils.py index 7280018..c91c3ec 100644 --- a/etebase_fastapi/utils.py +++ b/etebase_fastapi/utils.py @@ -8,14 +8,14 @@ from pydantic import BaseModel as PyBaseModel from django.db.models import QuerySet from django.core.exceptions import ObjectDoesNotExist -from django.contrib.auth import get_user_model from django_etebase import app_settings from django_etebase.models import AccessLevels +from myauth.models import UserType, get_typed_user_model from .exceptions import HttpError, HttpErrorOut -User = get_user_model() +User = get_typed_user_model() Prefetch = t.Literal["auto", "medium"] PrefetchQuery = Query(default="auto") @@ -30,7 +30,7 @@ class BaseModel(PyBaseModel): @dataclasses.dataclass class Context: - user: t.Optional[User] + user: t.Optional[UserType] prefetch: t.Optional[Prefetch] diff --git a/myauth/forms.py b/myauth/forms.py index 7aacb9b..fc2be74 100644 --- a/myauth/forms.py +++ b/myauth/forms.py @@ -1,8 +1,8 @@ from django import forms -from django.contrib.auth import get_user_model from django.contrib.auth.forms import UsernameField +from myauth.models import get_typed_user_model -User = get_user_model() +User = get_typed_user_model() class AdminUserCreationForm(forms.ModelForm): diff --git a/myauth/models.py b/myauth/models.py index d6585a8..5bc4af7 100644 --- a/myauth/models.py +++ b/myauth/models.py @@ -1,3 +1,5 @@ +import typing as t + from django.contrib.auth.models import AbstractUser, UserManager as DjangoUserManager from django.core import validators from django.db import models @@ -28,9 +30,21 @@ class User(AbstractUser): unique=True, help_text=_("Required. 150 characters or fewer. Letters, digits and ./-/_ only."), validators=[username_validator], - error_messages={"unique": _("A user with that username already exists."),}, + error_messages={ + "unique": _("A user with that username already exists."), + }, ) @classmethod def normalize_username(cls, username): return super().normalize_username(username).lower() + + +UserType = t.Type[User] + + +def get_typed_user_model() -> UserType: + from django.contrib.auth import get_user_model + + ret: t.Any = get_user_model() + return ret From 84b6114e99c206e56292386035dfc7ef0ac84ed4 Mon Sep 17 00:00:00 2001 From: Tom Hacohen Date: Tue, 29 Dec 2020 13:43:11 +0200 Subject: [PATCH 102/102] Requirements: add dev requirements and django-stubs. --- requirements-dev.txt | 28 ++++++++++++++++++++++++++++ requirements.in/development.txt | 3 ++- 2 files changed, 30 insertions(+), 1 deletion(-) create mode 100644 requirements-dev.txt diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..15a8b60 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,28 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --output-file=requirements-dev.txt requirements.in/development.txt +# +appdirs==1.4.4 # via black +asgiref==3.3.1 # via django +black==20.8b1 # via -r requirements.in/development.txt +click==7.1.2 # via black, pip-tools +coverage==5.3.1 # via -r requirements.in/development.txt +django-stubs==1.7.0 # via -r requirements.in/development.txt +django==3.1.4 # via django-stubs +mypy-extensions==0.4.3 # via black, mypy +mypy==0.790 # via django-stubs +pathspec==0.8.1 # via black +pip-tools==5.4.0 # via -r requirements.in/development.txt +pytz==2020.5 # via django +pywatchman==1.4.1 # via -r requirements.in/development.txt +regex==2020.11.13 # via black +six==1.15.0 # via pip-tools +sqlparse==0.4.1 # via django +toml==0.10.2 # via black +typed-ast==1.4.1 # via black, mypy +typing-extensions==3.7.4.3 # via black, django-stubs, mypy + +# The following packages are considered to be unsafe in a requirements file: +# pip diff --git a/requirements.in/development.txt b/requirements.in/development.txt index a956471..fb281d3 100644 --- a/requirements.in/development.txt +++ b/requirements.in/development.txt @@ -1,4 +1,5 @@ coverage pip-tools pywatchman -black \ No newline at end of file +black +django-stubs