more ruggedization and testing

Author: toresbe

app/archive.py

@@ -0,0 +1,33 @@
+import shutil
+from logging import getLogger
+from pathlib import Path
+
+from app.util.file_name_utils import original_file_location
+
+
+class Archive:
+    def __init__(self, archive_base_path: Path = Path("/tmp")):
+        self.archive_base_path = archive_base_path
+        self.logger = getLogger(__name__)
+
+    def validate_destination(self, video_id: str, original_file_name: Path) -> Path:
+        original_file_destination = self.archive_base_path / original_file_location(
+            video_id, Path(original_file_name.name)
+        )
+        assert not original_file_destination.exists(), f"File {original_file_destination} already exists"
+
+        self.logger.info("Destination: %s", original_file_destination)
+        return original_file_destination
+
+    def make_dir_for_original(self, original_file_name: Path):
+        original_file_name.parent.mkdir(exist_ok=True, parents=True)
+        self.logger.info("created destination folder %s", original_file_name.parent)
+        return original_file_name
+
+    def move_original_to_archive(self, video_id: str, original_file_name: Path):
+        original_file_destination = self.validate_destination(video_id, original_file_name)
+        self.make_dir_for_original(original_file_destination)
+        self.logger.info("moving original from %s to %s", original_file_name, original_file_destination)
+
+        shutil.move(original_file_name, original_file_destination)
+        return original_file_destination

app/converter.py

@@ -0,0 +1,38 @@
+import logging
+from pathlib import Path
+
+from frikanalen_django_api_client.models import FormatEnum, VideoFileRequest
+
+from app.django_api.service import DjangoApiService
+from app.ffprobe_schema import FfprobeOutput
+from ffmpeg.command_factory import FfmpegCommandFactory
+from runner import Runner
+
+
+class Converter:
+    ffmpeg_command_factory = FfmpegCommandFactory()
+    django_api: DjangoApiService
+    runner: Runner
+
+    def __init__(self, django_api: DjangoApiService, runner: Runner):
+        self.runner = runner
+        self.django_api = django_api
+
+    async def process_format(
+        self,
+        input_file_path: Path,
+        output_format_name: FormatEnum,
+        metadata: FfprobeOutput,
+        video_id: int,
+    ):
+        logging.info("Producing: %s", output_format_name)
+        ffmpeg_command_factory = FfmpegCommandFactory()
+        cmd_line, target_file_name = ffmpeg_command_factory.build_ffmpeg_command(
+            input_file_path, output_format_name, metadata
+        )
+        await self.runner.run(cmd_line)
+        await self.runner.wait_for_completion()
+
+        # Register it with API
+        req = VideoFileRequest(filename=str(target_file_name), format_=output_format_name, video_id=video_id)
+        await self.django_api.create_video_file(video_file=req)

app/django_api/build_client.py

@@ -0,0 +1,14 @@
+from frikanalen_django_api_client import AuthenticatedClient
+from starlette.requests import Request
+
+from app.config import config
+
+
+def get_client_from_app_state(request: Request) -> AuthenticatedClient:
+    return request.app.state.app_state.api_client  # type: ignore[attr-defined]
+
+
+def build_client():
+    return AuthenticatedClient(
+        base_url=config.django.base_url, token=config.django.token, raise_on_unexpected_status=True
+    )

app/django_api/service.py

@@ -1,22 +1,24 @@
 from datetime import datetime
 
-from fastapi import Depends
 from frikanalen_django_api_client import AuthenticatedClient
 from frikanalen_django_api_client.api.videofiles import videofiles_create, videofiles_list, videofiles_partial_update
 from frikanalen_django_api_client.api.videos import videos_list, videos_partial_update
-from frikanalen_django_api_client.models import PatchedVideoRequest, VideoFile, VideoFileRequest
-
+from frikanalen_django_api_client.models import (
+    PatchedVideoRequest,
+    VideoFile,
+    VideoFileRequest,
+    VideofilesListFormatFsname,
+)
+
+from app.django_api.build_client import build_client
 from app.loudness.loudness_measurement import LoudnessMeasurement
-from app.tus_hook.hook_server import build_client, get_client_from_app_state
 from app.util.pprint_object_list import pprint_object_list
 
 
 class DjangoApiService:
     client: AuthenticatedClient
 
-    def __init__(self, client: AuthenticatedClient = None):
-        if client is None:
-            client = Depends(get_client_from_app_state)
+    def __init__(self, client: AuthenticatedClient):
         self.client = client
 
     async def set_video_duration(self, video_id: str, duration: str):
@@ -36,7 +38,7 @@ async def get_original_files_without_loudness(self, limit=5) -> list[VideoFile]:
         return (
             await videofiles_list.asyncio(
                 client=self.client,
-                format_fsname="original",
+                format_fsname=VideofilesListFormatFsname.ORIGINAL,
                 integrated_lufs_isnull=True,
                 limit=limit,
                 ordering="-video",
@@ -61,8 +63,6 @@ async def get_videos(self, limit=10):
 if __name__ == "__main__":
     import asyncio
 
-    from app.tus_hook.hook_server import get_client_from_app_state
-
     async def main():
         service = DjangoApiService(build_client())
         videos = await service.get_videos()

app/ingest.py

@@ -1,39 +1,49 @@
-import logging
-import shutil
 from datetime import datetime
+from logging import Logger, getLogger
 from pathlib import Path
 
+from frikanalen_django_api_client.models import FormatEnum
+
 from app.django_api.service import DjangoApiService
 from app.ffprobe import do_probe
-from app.tus_hook.hook_server import build_client
 
-from .generate_assets import make_secondaries
+from .archive import Archive
+from .converter import Converter
 from .logging.video_id_filter import VideoIdFilter
-from .util.file_name_utils import original_file_location
 
-django_api = DjangoApiService(build_client())
-archive_base_path = Path("/tmp/archive")
+DESIRED_FORMATS = (FormatEnum("large_thumb"),)
+
 
+class Ingester:
+    video_id: str
+    django_api: DjangoApiService
+    converter_service: Converter
+    archive: Archive
+    logger: Logger
 
-async def ingest(video_id: str, original_file: Path):
-    logger = logging.getLogger(__name__)
-    logger.addFilter(VideoIdFilter(video_id))
-    logger.info("Ingesting file with video_id: %s, original_file: %s", video_id, original_file)
+    def __init__(self, video_id: str, django_api: DjangoApiService, converter_service: Converter, archive: Archive):
+        self.logger = getLogger(__name__)
+        self.logger.addFilter(VideoIdFilter(video_id))
+        self.video_id = video_id
+        self.django_api = django_api
+        self.converter_service = converter_service
+        self.archive = archive
 
-    original_file_destination = archive_base_path / original_file_location(video_id, original_file)
-    assert not original_file_destination.exists(), f"File {original_file_destination} already exists"
-    logger.info("Destination: %s", original_file_destination)
+    async def ingest(self, original_file: Path):
+        self.logger.info("Ingesting file with video_id: %s, original_file: %s", self.video_id, original_file)
 
-    logger.info("Creating destination folder: %s", original_file_destination.parent)
-    original_file_destination.parent.mkdir(exist_ok=True)
+        await self.django_api.set_video_uploaded_time(self.video_id, datetime.now())
+        metadata = await do_probe(original_file)
+        self.logger.info("Got metadata, %d streams", len(metadata.streams))
 
-    await django_api.set_video_uploaded_time(video_id, datetime.now())
-    metadata = await do_probe(original_file)
-    logger.info("Got metadata: %s", metadata)
+        original_file_destination = self.archive.move_original_to_archive(self.video_id, original_file)
+        await self.django_api.set_video_duration(self.video_id, metadata.format.duration)
 
-    logger.info("Moving from %s to %s", original_file, original_file_destination)
-    shutil.move(original_file, original_file_destination)
+        for format_name in DESIRED_FORMATS:
+            self.logger.info("Processing: %s", original_file_destination)
+            await self.converter_service.process_format(
+                original_file_destination, format_name, metadata, int(self.video_id)
+            )
+            self.logger.info("Finished processing: %s", self.video_id)
 
-    await django_api.set_video_duration(video_id, metadata["format"]["duration"])
-    await make_secondaries(video_id, original_file_destination)
-    await django_api.set_video_proper_import(video_id, True)
+        await self.django_api.set_video_proper_import(self.video_id, True)

app/main.py

@@ -0,0 +1,96 @@
+import logging
+from contextlib import asynccontextmanager
+from dataclasses import dataclass
+from pathlib import Path
+from unittest.mock import AsyncMock
+
+from fastapi import FastAPI, HTTPException
+from frikanalen_django_api_client import AuthenticatedClient
+from pydantic import BaseModel, Field, ValidationError
+from werkzeug.utils import secure_filename
+
+from app.archive import Archive
+from app.converter import Converter
+from app.django_api.build_client import build_client
+from app.django_api.service import DjangoApiService
+from app.ingest import Ingester
+from app.tus_hook.hook_schema import HookRequest
+from runner import Runner
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """
+    Lifespan handler for the FastAPI app. This is where we initialize the HTTP client used by
+    the DjangoAPI services. Constructs an authenticated HTTP client based on configuration
+    values config.django.base_url and config.django.token.
+    """
+    client = build_client()
+    await client.__aenter__()
+
+    # ✅ Store it on app state
+    app.state.app_state = IngestAppState(api_client=client)  # type: ignore[attr-defined]
+
+    yield  # App runs here
+
+    # ✅ Cleanup
+    await client.__aexit__(None, None, None)
+
+
+app = FastAPI(lifespan=lifespan)
+
+
+@dataclass
+class IngestAppState:
+    api_client: AuthenticatedClient
+
+
+@app.post("/")
+async def receive_hook(hook: HookRequest):
+    if hook.type == "pre-create":
+        try:
+            UploadMetaData(**hook.event.upload.meta_data.model_dump())
+        except ValidationError as e:
+            raise HTTPException(status_code=422, detail=e.errors()) from e
+        except AttributeError as e:
+            raise HTTPException(status_code=422, detail="Missing required fields") from e
+
+    if hook.type == "post-finish":
+        try:
+            upload_meta = UploadMetaData(**hook.event.upload.meta_data.model_dump())
+            assert hook.event.upload.storage["Type"] == "filestore", "Only filestore storage is supported"
+            print(f"outpath: {hook.event.upload.storage['Path']}")
+
+        except ValidationError as e:
+            logging.error(e.errors)
+            raise HTTPException(status_code=422, detail=e.errors()) from e
+        except AttributeError as e:
+            logging.error(e)
+            raise HTTPException(status_code=422, detail="Missing required fields") from e
+
+        django_api = AsyncMock(spec=DjangoApiService)
+        ingest = Ingester(
+            video_id=upload_meta.video_id,
+            django_api=django_api,
+            converter_service=Converter(django_api, Runner()),
+            archive=Archive(),
+        )
+        out_path = Path(hook.event.upload.storage["Path"])
+        if not out_path.exists():
+            raise HTTPException(status_code=500, detail=f"File not found: {out_path}")
+
+        # rename out_path to original file name
+        out_path.rename(out_path.parent / secure_filename(upload_meta.orig_file_name))
+        await ingest.ingest(out_path)
+    return {}
+
+
+class UploadMetaData(BaseModel):
+    video_id: str = Field(..., alias="VideoID")
+    orig_file_name: str = Field(..., alias="OrigFileName")
+    upload_token: str = Field(..., alias="UploadToken")
+
+
+@app.get("/isAlive")
+async def read_health():
+    return {"status": "ok"}