diff --git a/Dockerfile b/Dockerfile index c03be65f..736f8f3b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,6 +8,14 @@ COPY . /code/ WORKDIR /code # END gv-base +# BEGIN gv-local +FROM gv-base as gv-local +# install dev and non-dev dependencies: +RUN pip3 install --no-cache-dir -r requirements/dev-requirements.txt +# Start the Django development server +CMD ["python", "manage.py", "runserver", "0.0.0.0:8004"] +# END gv-local + # BEGIN gv-deploy FROM gv-base as gv-deploy # install non-dev and release-only dependencies: @@ -16,11 +24,3 @@ RUN pip3 install --no-cache-dir -r requirements/release.txt RUN python manage.py collectstatic --noinput # note: no CMD in gv-deploy -- depends on deployment # END gv-deploy - -# BEGIN gv-local -FROM gv-base as gv-local -# install dev and non-dev dependencies: -RUN pip3 install --no-cache-dir -r requirements/dev-requirements.txt -# Start the Django development server -CMD ["python", "manage.py", "runserver", "0.0.0.0:8004"] -# END gv-local diff --git a/addon_service/authorized_storage_account/models.py b/addon_service/authorized_storage_account/models.py index 8db07988..bdced720 100644 --- a/addon_service/authorized_storage_account/models.py +++ b/addon_service/authorized_storage_account/models.py @@ -163,7 +163,10 @@ def auth_url(self) -> str | None: Returns None if the ExternalStorageService does not support OAuth2 or if the initial credentials exchange has already ocurred. """ - if self.credentials_format is not CredentialsFormats.OAUTH2: + if ( + self.credentials_format is not CredentialsFormats.OAUTH2 + or self.oauth2_token_metadata_id is None + ): return None state_token = self.oauth2_token_metadata.state_token @@ -189,6 +192,10 @@ def api_base_url(self, value): def imp_cls(self) -> type[AddonImp]: return self.external_service.addon_imp.imp_cls + @property + def credentials_available(self) -> bool: + return self._credentials is not None + @transaction.atomic def initiate_oauth2_flow(self, authorized_scopes=None): if self.credentials_format is not CredentialsFormats.OAUTH2: diff --git a/addon_service/authorized_storage_account/serializers.py b/addon_service/authorized_storage_account/serializers.py index 7598df7c..89b35141 100644 --- a/addon_service/authorized_storage_account/serializers.py +++ b/addon_service/authorized_storage_account/serializers.py @@ -66,6 +66,7 @@ def __init__(self, *args, **kwargs): related_link_view_name=view_names.related_view(RESOURCE_TYPE), ) credentials = CredentialsField(write_only=True, required=False) + initiate_oauth = serializers.BooleanField(write_only=True, required=False) included_serializers = { "account_owner": "addon_service.serializers.UserReferenceSerializer", @@ -91,12 +92,16 @@ def create(self, validated_data): except ModelValidationError as e: raise serializers.ValidationError(e) - if external_service.credentials_format is CredentialsFormats.OAUTH2: + if ( + validated_data.get("initiate_oauth", False) + and external_service.credentials_format is CredentialsFormats.OAUTH2 + ): authorized_account.initiate_oauth2_flow( validated_data.get("authorized_scopes") ) - else: + elif validated_data.get("credentials"): authorized_account.credentials = validated_data["credentials"] + try: authorized_account.save() except ModelValidationError as e: @@ -120,4 +125,6 @@ class Meta: "credentials", "default_root_folder", "external_storage_service", + "initiate_oauth", + "credentials_available", ] diff --git a/addon_service/credentials/encryption.py b/addon_service/credentials/encryption.py index 45090944..e597fd15 100644 --- a/addon_service/credentials/encryption.py +++ b/addon_service/credentials/encryption.py @@ -132,6 +132,12 @@ def _derive_multifernet_key( key_params: KeyParameters, /, # positional-only params for cache-friendliness ) -> fernet.MultiFernet: + if not settings.GRAVYVALET_ENCRYPT_SECRET: + raise RuntimeError( + "gravyvalet can not keep your secrets without a GRAVYVALET_ENCRYPT_SECRET" + " -- ideally chosen by strong randomness, with around 256 bits of entropy" + " (e.g. 64 hex digits; 60 d20 rolls; 20 words of a 10000-word vocabulary)" + ) # https://cryptography.io/en/latest/fernet/#cryptography.fernet.MultiFernet return fernet.MultiFernet( [ diff --git a/addon_service/management/commands/rotate_encryption.py b/addon_service/management/commands/rotate_encryption.py new file mode 100644 index 00000000..ba330030 --- /dev/null +++ b/addon_service/management/commands/rotate_encryption.py @@ -0,0 +1,9 @@ +from django.core.management.base import BaseCommand + +from addon_service.tasks.key_rotation import schedule_encryption_rotation__celery + + +class Command(BaseCommand): + def handle(self, *args, **kwargs): + _task = schedule_encryption_rotation__celery.apply_async() + self.stdout.write(self.style.SUCCESS(f"scheduled task {_task}")) diff --git a/addon_service/oauth/models.py b/addon_service/oauth/models.py index e581a15f..946cb2ce 100644 --- a/addon_service/oauth/models.py +++ b/addon_service/oauth/models.py @@ -1,6 +1,7 @@ from datetime import timedelta from asgiref.sync import sync_to_async +from django.conf import settings from django.contrib.postgres.fields import ArrayField from django.core.exceptions import ValidationError from django.db import ( @@ -28,7 +29,6 @@ class OAuth2ClientConfig(AddonsServiceBaseModel): token_endpoint_url = models.URLField(null=False) # The registered ID of the OAuth client client_id = models.CharField(null=True) - client_secret = models.CharField(null=True) class Meta: verbose_name = "OAuth2 Client Config" @@ -38,6 +38,10 @@ class Meta: def __repr__(self): return f'<{self.__class__.__qualname__}(pk="{self.pk}", auth_uri="{self.auth_uri}", client_id="{self.client_id}")>' + @property + def client_secret(self): + return settings.OAUTH_SECRETS.get(self.client_id) + __str__ = __repr__ diff --git a/addon_service/tests/_factories.py b/addon_service/tests/_factories.py index a0e7ee6f..ab149251 100644 --- a/addon_service/tests/_factories.py +++ b/addon_service/tests/_factories.py @@ -33,7 +33,6 @@ class Meta: auth_callback_url = "https://osf.example/auth/callback" token_endpoint_url = "https://api.example.com/oauth/token" client_id = factory.Faker("word") - client_secret = factory.Faker("word") class AddonOperationInvocationFactory(DjangoModelFactory): diff --git a/addon_service/tests/_helpers.py b/addon_service/tests/_helpers.py index 920f9830..53ba2299 100644 --- a/addon_service/tests/_helpers.py +++ b/addon_service/tests/_helpers.py @@ -187,13 +187,16 @@ def get_test_request(user=None, method="get", path="", cookies=None): return _request +@contextlib.contextmanager def patch_encryption_key_derivation(): - # expect to call scrypt with all the following params: - def _mock_scrypt(secret, salt, n, r, p, dklen, maxmem): - # some random derived key - return b"\xdd\xd1\xdfN9\n\xbb\xa5\x9a|\xc6\x1f\xd6b\xf2\xfc>\x1e\xfe\xfd\x14\xc6n\xd7\x18\xbf'\x04qk\x8c\xfb" + _fake_secret = b"this is fine" + _some_random_key = b"\xdd\xd1\xdfN9\n\xbb\xa5\x9a|\xc6\x1f\xd6b\xf2\xfc>\x1e\xfe\xfd\x14\xc6n\xd7\x18\xbf'\x04qk\x8c\xfb" - return patch( + with patch( + "addon_service.credentials.encryption.settings.GRAVYVALET_ENCRYPT_SECRET", + _fake_secret, + ), patch( "addon_service.credentials.encryption.hashlib.scrypt", - side_effect=_mock_scrypt, - ) + return_value=_some_random_key, + ): + yield diff --git a/addon_service/tests/e2e_tests/test_oauth_flow.py b/addon_service/tests/e2e_tests/test_oauth_flow.py index b2dd8678..356a0338 100644 --- a/addon_service/tests/e2e_tests/test_oauth_flow.py +++ b/addon_service/tests/e2e_tests/test_oauth_flow.py @@ -26,6 +26,7 @@ def _make_post_payload(*, external_service, capabilities=None, credentials=None) "type": "authorized-storage-accounts", "attributes": { "authorized_capabilities": [AddonCapabilities.ACCESS.name], + "initiate_oauth": True, }, "relationships": { "external_storage_service": { diff --git a/addon_service/tests/test_by_type/test_authorized_storage_account.py b/addon_service/tests/test_by_type/test_authorized_storage_account.py index 7076e2f6..f6392c2d 100644 --- a/addon_service/tests/test_by_type/test_authorized_storage_account.py +++ b/addon_service/tests/test_by_type/test_authorized_storage_account.py @@ -56,6 +56,7 @@ def _make_post_payload( "attributes": { "authorized_capabilities": capabilities, "api_base_url": api_root, + "initiate_oauth": True, }, "relationships": { "external_storage_service": { diff --git a/app/celery.py b/app/celery.py index 6010e7f7..75c3d400 100644 --- a/app/celery.py +++ b/app/celery.py @@ -68,4 +68,4 @@ def _enqueue_handler_task(body, message): ] -app.steps["consumer"].add(OsfBackchannelConsumerStep) +# app.steps["consumer"].add(OsfBackchannelConsumerStep) diff --git a/app/env.py b/app/env.py index 8d6edda3..6297cabc 100644 --- a/app/env.py +++ b/app/env.py @@ -4,6 +4,17 @@ import os +DEBUG = bool(os.environ.get("DEBUG")) # any non-empty value enables debug mode +SECRET_KEY = os.environ.get("SECRET_KEY") # used by django for cryptographic signing +ALLOWED_HOSTS = list(filter(bool, os.environ.get("ALLOWED_HOSTS", "").split(","))) +CORS_ALLOWED_ORIGINS = tuple( + filter(bool, os.environ.get("CORS_ALLOWED_ORIGINS", "").split(",")) +) +SENTRY_DSN = os.environ.get("SENTRY_DSN") + +### +# databases + POSTGRES_DB = os.environ.get("POSTGRES_DB") POSTGRES_USER = os.environ.get("POSTGRES_USER") POSTGRES_PASSWORD = os.environ.get("POSTGRES_PASSWORD") @@ -17,17 +28,20 @@ OSFDB_PORT = os.environ.get("OSFDB_PORT", "5432") OSFDB_CONN_MAX_AGE = os.environ.get("OSFDB_CONN_MAX_AGE", 0) OSFDB_SSLMODE = os.environ.get("OSFDB_SSLMODE", "prefer") + +### +# for interacting with osf + OSF_SENSITIVE_DATA_SECRET = os.environ.get("OSF_SENSITIVE_DATA_SECRET", "") OSF_SENSITIVE_DATA_SALT = os.environ.get("OSF_SENSITIVE_DATA_SALT", "") - -SECRET_KEY = os.environ.get("SECRET_KEY") OSF_HMAC_KEY = os.environ.get("OSF_HMAC_KEY") OSF_HMAC_EXPIRATION_SECONDS = int(os.environ.get("OSF_HMAC_EXPIRATION_SECONDS", 110)) - OSF_BASE_URL = os.environ.get("OSF_BASE_URL", "https://osf.example") OSF_API_BASE_URL = os.environ.get("OSF_API_BASE_URL", "https://api.osf.example") +### # amqp/celery + AMQP_BROKER_URL = os.environ.get( "AMQP_BROKER_URL", "amqp://guest:guest@192.168.168.167:5672" ) @@ -36,15 +50,6 @@ "OSF_BACKCHANNEL_QUEUE_NAME", "account_status_changes" ) -# any non-empty value enables debug mode: -DEBUG = bool(os.environ.get("DEBUG")) - -# comma-separated list: -ALLOWED_HOSTS = os.environ.get("ALLOWED_HOSTS", "").split(",") -CORS_ALLOWED_ORIGINS = tuple( - filter(bool, os.environ.get("CORS_ALLOWED_ORIGINS", "").split(",")) -) - ### # credentials encryption secrets and parameters # @@ -56,7 +61,7 @@ # 2. call `.rotate_encryption()` on every `ExternalCredentials` (perhaps via # celery tasks in `addon_service.tasks.key_rotation`) # 3. remove the old secret from GRAVYVALET_ENCRYPT_SECRET_PRIORS -GRAVYVALET_ENCRYPT_SECRET = os.environ.get("GRAVYVALET_ENCRYPT_SECRET") +GRAVYVALET_ENCRYPT_SECRET: str | None = os.environ.get("GRAVYVALET_ENCRYPT_SECRET") GRAVYVALET_ENCRYPT_SECRET_PRIORS = tuple( filter(bool, os.environ.get("GRAVYVALET_ENCRYPT_SECRET_PRIORS", "").split(",")) ) @@ -74,3 +79,11 @@ ) # END credentials encryption secrets and parameters ### + + +### +# OAuth Client Settings +BOX_OAUTH2_CLIENT_ID: str | None = os.environ.get("BOX_CLIENT_ID", "box") +BOX_OAUTH2_CLIENT_SECRET: str | None = os.environ.get("BOX_SECRET", "box_secret") +# END OAuth Client Settings +### diff --git a/app/settings.py b/app/settings.py index 6bc9c8d1..20b08a78 100644 --- a/app/settings.py +++ b/app/settings.py @@ -1,20 +1,30 @@ +import logging from pathlib import Path from app import env +_logger = logging.getLogger(__name__) + + +if env.SENTRY_DSN: + try: + import sentry_sdk + except ImportError: + _logger.warning("SENTRY_DSN defined but sentry_sdk not installed!") + else: + sentry_sdk.init( + dsn=env.SENTRY_DSN, + environment=env.OSF_BASE_URL, + ) + + SECRET_KEY = env.SECRET_KEY OSF_HMAC_KEY = env.OSF_HMAC_KEY or "lmaoooooo" OSF_HMAC_EXPIRATION_SECONDS = env.OSF_HMAC_EXPIRATION_SECONDS -if not env.DEBUG and not env.GRAVYVALET_ENCRYPT_SECRET: - raise RuntimeError( - "pls set `GRAVYVALET_ENCRYPT_SECRET` environment variable to something safely random" - ) -GRAVYVALET_ENCRYPT_SECRET: bytes = ( - env.GRAVYVALET_ENCRYPT_SECRET.encode() - if env.GRAVYVALET_ENCRYPT_SECRET - else b"this is fine" +GRAVYVALET_ENCRYPT_SECRET: bytes | None = ( + env.GRAVYVALET_ENCRYPT_SECRET.encode() if env.GRAVYVALET_ENCRYPT_SECRET else None ) GRAVYVALET_ENCRYPT_SECRET_PRIORS: tuple[bytes, ...] = tuple( _prior.encode() for _prior in env.GRAVYVALET_ENCRYPT_SECRET_PRIORS @@ -192,6 +202,7 @@ # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.1/howto/static-files/ +STATIC_ROOT = BASE_DIR / "static" STATIC_URL = "/static/" OSF_SENSITIVE_DATA_SECRET = env.OSF_SENSITIVE_DATA_SECRET @@ -204,3 +215,9 @@ AMQP_BROKER_URL = env.AMQP_BROKER_URL OSF_BACKCHANNEL_QUEUE_NAME = env.OSF_BACKCHANNEL_QUEUE_NAME GV_QUEUE_NAME_PREFIX = env.GV_QUEUE_NAME_PREFIX + + +### +# Mapping from OAuth Client IDs to Secrets +# DB should store the Client IDs to serve as a shared lookup, but secrets should live in ENVVARS +OAUTH_SECRETS = {env.BOX_OAUTH2_CLIENT_ID: env.BOX_OAUTH2_CLIENT_SECRET} diff --git a/docker-compose.yml b/docker-compose.yml index 63719bc5..b16c170d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,10 +3,12 @@ version: "3.8" services: gravyvalet: - build: . + build: + context: . + target: gv-local restart: unless-stopped command: python manage.py runserver 0.0.0.0:8004 - environment: + environment: &gv_environment DEBUG: 1 DJANGO_SETTINGS_MODULE: app.settings PYTHONUNBUFFERED: 1 @@ -20,6 +22,7 @@ services: OSF_BASE_URL: "http://192.168.168.167:5000" OSF_API_BASE_URL: "http://192.168.168.167:8000" AMQP_BROKER_URL: "amqp://guest:guest@rabbitmq:5672" + GRAVYVALET_ENCRYPT_SECRET: "this is fine" ports: - 8004:8004 stdin_open: true @@ -30,7 +33,9 @@ services: - postgres celeryworker: - build: . + build: + context: . + target: gv-local restart: unless-stopped command: /usr/local/bin/celery --app app worker --uid daemon -l INFO depends_on: @@ -38,16 +43,7 @@ services: - postgres volumes: - ./:/code:cached - environment: - DEBUG: 1 - DJANGO_SETTINGS_MODULE: app.settings - POSTGRES_HOST: postgres - POSTGRES_DB: gravyvalet - POSTGRES_USER: postgres - SECRET_KEY: so-secret - OSF_BASE_URL: "http://192.168.168.167:5000" - OSF_API_BASE_URL: "http://192.168.168.167:8000" - AMQP_BROKER_URL: "amqp://guest:guest@rabbitmq:5672" + environment: *gv_environment stdin_open: true postgres: diff --git a/requirements/dev-requirements.txt b/requirements/dev-requirements.txt index 51f3d502..dc48fcbb 100644 --- a/requirements/dev-requirements.txt +++ b/requirements/dev-requirements.txt @@ -10,7 +10,3 @@ flake8 black==24.* isort pre-commit - -# ASGI server for local use -# https://docs.djangoproject.com/en/4.2/howto/deployment/asgi/daphne/#integration-with-runserver -daphne diff --git a/requirements/release.txt b/requirements/release.txt index 0dbc1238..b150635c 100644 --- a/requirements/release.txt +++ b/requirements/release.txt @@ -1,3 +1,4 @@ -r ./requirements.txt # Requirements to be installed on server deployments +sentry-sdk==2.7.1 diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 2dc1f2b2..f42b7c0e 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,4 +1,5 @@ Django==4.2.7 +daphne==4.1.2 psycopg>=3.1.8 djangorestframework==3.14.0 djangorestframework-jsonapi==6.1.0