diff --git a/CHANGELOG.md b/CHANGELOG.md index 5cf803af..0fba3d82 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - We now use `uv` (Universal Virtualenv) to manage python dependencies and run scripts in CI/CD. This should improve dependency resolution and installation times. - We now ship a static ffmpeg binary instead of installing ffmpeg via apt. This should reduce image size and improve compatibility across different host systems. - Added a database migration setup using [Alembic](https://alembic.sqlalchemy.org/) for future database migrations. +- Upgraded `beets` from `v2.5.1` to `v2.6.1` ## [1.2.0] - 25-12-17 diff --git a/backend/alembic/versions/2026_04_12_2038-f06e470b3d1e_match.py b/backend/alembic/versions/2026_04_12_2038-f06e470b3d1e_match.py new file mode 100644 index 00000000..93f0e7d9 --- /dev/null +++ b/backend/alembic/versions/2026_04_12_2038-f06e470b3d1e_match.py @@ -0,0 +1,372 @@ +"""match + +Revision ID: f06e470b3d1e +Revises: 925cf8989fbc +Create Date: 2026-04-12 20:38:28.263069 + +README: +Historically, candidate states included a pickled match item. This approach has proven +to be brittle and difficult to maintain. This migration implements a more refined +database schema for matches. +""" + +from __future__ import annotations +from collections.abc import Sequence +import importlib.util +import io +from pathlib import Path +import pickle +from typing import Any, NamedTuple + +import sqlalchemy as sa +from sqlalchemy.orm import Session +from beets_flask.logger import logging +from beets_flask.database.models import types +from alembic import op + +# We depend on other migrations (no other easy way to import) +BASE_DIR = Path(__file__).resolve().parent +path = BASE_DIR / "2026_04_12_1847-925cf8989fbc_item_pending.py" +spec = importlib.util.spec_from_file_location("item_pending_migration", path) +if not spec or not spec.loader: + raise ImportError +item_migration = importlib.util.module_from_spec(spec) +spec.loader.exec_module(item_migration) + +# revision identifiers, used by Alembic. +revision: str = "f06e470b3d1e" +down_revision: str | Sequence[str] | None = "925cf8989fbc" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +log = logging.getLogger("alembic.runtime.migration") + + +def upgrade() -> None: + """Upgrade schema.""" + # core info table + op.create_table( + "album_info", + sa.Column("data", sa.JSON(), nullable=False), + sa.Column("id", sa.String(), primary_key=True), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("updated_at", sa.DateTime(), nullable=False), + ) + op.create_index("ix_album_info_created_at", "album_info", ["created_at"]) + + op.create_table( + "track_info", + sa.Column("album_id", sa.String(), sa.ForeignKey("album_info.id")), + sa.Column("data", sa.JSON(), nullable=False), + sa.Column("id", sa.String(), primary_key=True), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("updated_at", sa.DateTime(), nullable=False), + ) + op.create_index("ix_track_info_created_at", "track_info", ["created_at"]) + + # distance graph + op.create_table( + "distances", + sa.Column("track_info_id", sa.String(), sa.ForeignKey("track_info.id")), + sa.Column("parent_distance_id", sa.String(), sa.ForeignKey("distances.id")), + sa.Column("raw_distance", sa.Float(), nullable=False), + sa.Column("max_distance", sa.Float(), nullable=False), + sa.Column("id", sa.String(), primary_key=True), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("updated_at", sa.DateTime(), nullable=False), + ) + op.create_index("ix_distances_created_at", "distances", ["created_at"]) + + # matches + op.create_table( + "matches", + sa.Column("id", sa.String(), primary_key=True), + sa.Column("type", sa.String(), nullable=False), + sa.Column( + "distance_id", sa.String(), sa.ForeignKey("distances.id"), nullable=False + ), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("updated_at", sa.DateTime(), nullable=False), + ) + op.create_table( + "matches_album", + sa.Column("id", sa.String(), sa.ForeignKey("matches.id"), primary_key=True), + sa.Column( + "info_id", sa.String(), sa.ForeignKey("album_info.id"), nullable=False + ), + ) + op.create_table( + "matches_track", + sa.Column("id", sa.String(), sa.ForeignKey("matches.id"), primary_key=True), + sa.Column( + "info_id", sa.String(), sa.ForeignKey("track_info.id"), nullable=False + ), + ) + op.create_index("ix_matches_created_at", "matches", ["created_at"]) + + # mappings + op.create_table( + "album_match_track_mappings", + sa.Column( + "album_match_id", + sa.String(), + sa.ForeignKey("matches_album.id"), + nullable=False, + ), + sa.Column("track_info_id", sa.String(), sa.ForeignKey("track_info.id")), + sa.Column("item_id", sa.String(), sa.ForeignKey("items.id")), + sa.Column("id", sa.String(), primary_key=True, nullable=False), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("updated_at", sa.DateTime(), nullable=False), + ) + op.create_index( + "ix_album_match_track_mappings_created_at", + "album_match_track_mappings", + ["created_at"], + ) + + # penalties + op.create_table( + "penalties", + sa.Column("key", sa.String(), nullable=False), + sa.Column("value", types.FloatListType(), nullable=False), + sa.Column( + "distance_id", sa.String(), sa.ForeignKey("distances.id"), nullable=False + ), + sa.Column("id", sa.String(), primary_key=True), + sa.Column("created_at", sa.DateTime(), nullable=False), + sa.Column("updated_at", sa.DateTime(), nullable=False), + ) + op.create_index("ix_penalties_created_at", "penalties", ["created_at"]) + op.create_index("ix_penalties_key", "penalties", ["key"]) + + # Migrate candidate table + with op.batch_alter_table("candidate") as batch_op: + batch_op.add_column(sa.Column("match_id", sa.String(), nullable=True)) + + migrate_data() + + with op.batch_alter_table("candidate") as batch_op: + batch_op.drop_column("match") + batch_op.alter_column("match_id", nullable=False) + batch_op.create_foreign_key( + "fk_candidate_match", + "matches", + ["match_id"], + ["id"], + ) + + +def downgrade() -> None: + """Downgrade schema.""" + + # candidate table (SQLite-safe) + with op.batch_alter_table("candidate") as batch_op: + batch_op.drop_constraint( + "fk_candidate_match", + type_="foreignkey", + ) + batch_op.add_column(sa.Column("match", sa.BLOB(), nullable=True)) + batch_op.drop_column("match_id") + + # independent tables + op.drop_table("matches_track") + op.drop_table("matches_album") + op.drop_table("album_match_track_mappings") + + op.drop_table("penalties") + op.drop_table("matches") + op.drop_table("distances") + op.drop_table("track_info") + op.drop_table("album_info") + + +def migrate_data(): + from beets_flask.database.mapper.match import ( + AlbumMatchMapper, + TrackMatchMapper, + Context, + ) + + conn = op.get_bind() + session = Session(bind=conn) + + result = conn.execution_options(stream_results=True).execute( + sa.text("SELECT id, match FROM candidate WHERE match IS NOT NULL") + ) + total = conn.execute( + sa.text("SELECT COUNT(*) FROM candidate WHERE match IS NOT NULL") + ).scalar() + for i, row in enumerate(result, start=1): + if i % 100 == 0: + log.info("Migrating matches %d / %d rows", i, total) + + candidate_id = row[0] + match_blob = row[1] + + if not match_blob: + continue + + try: + beets_match = load_match(match_blob) + + # A bit of an anti patter here but easiest way out: + # We depend on our mappers here and hope they do not change in the future + db_match: Any + if isinstance(beets_match, AlbumMatchStub): + db_match = AlbumMatchMapper().from_beets( + beets_match, # type: ignore[arg-type] + Context(), + ) + + else: + db_match = TrackMatchMapper().from_beets( + beets_match, # type: ignore[arg-type] + Context(), + ) + + session.add(db_match) + session.flush() # gets db_match.id + + conn.execute( + sa.text("UPDATE candidate SET match_id = :match_id WHERE id = :id"), + {"match_id": db_match.id, "id": candidate_id}, + ) + + except Exception: + log.exception("Failed to migrate candidate %s", candidate_id) + raise + + log.info("Migrated %d / %d matches!", total, total) + + +def load_match(blob: bytes) -> AlbumMatchStub | TrackMatchStub: + return MatchUnpickler(io.BytesIO(blob)).load() + + +# --------------------------- Mocked Beets Classes --------------------------- # + + +class AttributeDictStub: + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + + def __getstate__(self): + return self.__dict__.copy() + + def __setstate__(self, state): + self.__dict__.update(state) + + def __setitem__(self, key, value): + self.__dict__[key] = value + + def __getitem__(self, key): + return self.__dict__[key] + + def keys(self): + return self.__dict__.keys() + + def values(self): + return self.__dict__.values() + + def items(self): + return self.__dict__.items() + + +class DistanceStub: + def __init__(self): + self._penalties = {} + self.tracks = {} + self._raw_distance = 0.0 # Use private backing field + self._max_distance = 0.0 + + @property + def raw_distance(self) -> float: + return self._raw_distance + + @raw_distance.setter + def raw_distance(self, value: float): + self._raw_distance = value + + @property + def max_distance(self) -> float: + return self._max_distance + + @max_distance.setter + def max_distance(self, value: float): + self._max_distance = value + + def __getstate__(self): + return { + "_penalties": self._penalties, + "tracks": self.tracks, + "_raw_distance": self._raw_distance, + "_max_distance": self._max_distance, + } + + def __setstate__(self, state): + self._penalties = state.get("_penalties", {}) + self.tracks = state.get("tracks", {}) + self._raw_distance = state.get("_raw_distance", 0.0) + self._max_distance = state.get("_max_distance", 0.0) + + +class AlbumMatchStub(NamedTuple): + distance: DistanceStub + info: AttributeDictStub + mapping: dict[Any, AttributeDictStub] # Any = item_migration.ModelStub + extra_items: list[Any] + extra_tracks: list[AttributeDictStub] + + +class TrackMatchStub(NamedTuple): + distance: DistanceStub + info: AttributeDictStub + + +class MatchUnpickler(pickle.Unpickler): + CLASS_MAP = { + ("beets.dbcore.db", "LazyConvertDict"): item_migration.LazyConvertDictStub, + ("beets.library", "Item"): item_migration.ModelStub, + ("beets.library.models", "Item"): item_migration.ModelStub, + ("beets.autotag.hooks", "AlbumMatch"): AlbumMatchStub, + ("beets.autotag.hooks", "Distance"): DistanceStub, + ("beets.autotag.hooks", "TrackInfo"): AttributeDictStub, + ("beets.autotag.hooks", "AlbumInfo"): AttributeDictStub, + ("beets.autotag.distance", "Distance"): DistanceStub, + ("beetsplug.discogs", "IntermediateTrackInfo"): AttributeDictStub, + } + + def find_class(self, module, name): + """Override the find_class method to redirect Distance class references.""" + key = (module, name) + if key not in self.CLASS_MAP: + print(f"WARNING: Unknown class not in migration map: {module}.{name}") + return dict # Fallback for unknown classes + return self.CLASS_MAP[key] + + def load(self) -> Any: + object = super().load() + if isinstance(object, DistanceStub): + self._normalize(object) + + if isinstance(object, AlbumMatchStub): + self._normalize(object.distance) + + return object + + def _normalize(self, obj): + if isinstance(obj, DistanceStub): + return self._normalize_distance(obj) + return obj + + def _normalize_distance(self, distance: DistanceStub) -> DistanceStub: + # Beets had a rename at some point which we need to handle here. + if "source" in distance._penalties: + distance._penalties["data_source"] = distance._penalties.pop("source") + + for _, child in distance.tracks.items(): + self._normalize_distance(child) + + return distance diff --git a/backend/beets_flask/database/mapper/base.py b/backend/beets_flask/database/mapper/base.py new file mode 100644 index 00000000..23dfe10a --- /dev/null +++ b/backend/beets_flask/database/mapper/base.py @@ -0,0 +1,63 @@ +from typing import Any, Protocol, TypeVar + +B = TypeVar("B") # beets type +M = TypeVar("M") # model type + + +class Context: + """Shared mapping context used during bidirectional conversion. + + This context provides identity-based caching to avoid duplicate + object reconstruction and to preserve reference consistency + during recursive mappings. + """ + + def __init__(self): + self.from_cache: dict[int, Any] = {} + self.to_cache: dict[int, Any] = {} + + +class BeetsMapper(Protocol[B, M]): + """Protocol for bidirectional mapping between Beets objects and models. + + This mapper provides cached conversion in both directions: + - Beets → Model via `from_beets` + - Model → Beets via `to_beets` + + Identity-based caching (via `id()`) ensures: + - stable object graphs during recursive mapping + - prevention of infinite recursion + - consistent reuse of already-mapped instances + + Subclasses must implement: + - `_from_beets` + - `_to_beets` + """ + + def from_beets(self, obj: B, ctx: Context) -> M: + """Convert a Beets object into a model instance with caching.""" + key = id(obj) + if key in ctx.from_cache: + return ctx.from_cache[key] + + result = self._from_beets(obj, ctx) + ctx.from_cache[key] = result + return result + + def to_beets(self, model: M, ctx: Context) -> B: + """Convert a model instance back into a Beets object with caching.""" + key = id(model) + if key in ctx.to_cache: + return ctx.to_cache[key] + + result = self._to_beets(model, ctx) + ctx.to_cache[key] = result + return result + + def _from_beets(self, obj: B, ctx: Context) -> M: + """Implement Beets → model conversion.""" + raise NotImplementedError + + def _to_beets(self, model: M, ctx: Context) -> B: + """Implement model → Beets conversion.""" + raise NotImplementedError diff --git a/backend/beets_flask/database/mapper/match.py b/backend/beets_flask/database/mapper/match.py new file mode 100644 index 00000000..6e222abc --- /dev/null +++ b/backend/beets_flask/database/mapper/match.py @@ -0,0 +1,265 @@ +"""Converts beets objects to beetsflask database objects. + +Historically beets objects have quite some cross references which tend to +be difficult to map to a structured database. To avoid drilling and handle +deduplication we use mapper classes with a shared context. +""" + +from __future__ import annotations + +import base64 + +from beets_flask.database.models.pending import Item +from beets_flask.importer.types import ( + BeetsAlbumInfo, + BeetsAlbumMatch, + BeetsDistance, + BeetsItem, + BeetsTrackInfo, + BeetsTrackMatch, +) + +from ..models.match import ( + AlbumInfo, + AlbumMatch, + AlbumMatchTrackMapping, + Distance, + Match, + Penalty, + TrackInfo, + TrackMatch, +) +from .base import BeetsMapper, Context + + +class MatchMapper(BeetsMapper[BeetsAlbumMatch | BeetsTrackMatch, Match]): + def __init__(self): + self.album_mapper = AlbumMatchMapper() + self.track_mapper = TrackMatchMapper() + + def _from_beets( + self, obj: BeetsAlbumMatch | BeetsTrackMatch, ctx: Context + ) -> Match: + if isinstance(obj, BeetsAlbumMatch): + return self.album_mapper.from_beets(obj, ctx) + + if isinstance(obj, BeetsTrackMatch): + return self.track_mapper.from_beets(obj, ctx) + + raise TypeError(f"Unsupported beets obj type: {type(obj)}") + + def _to_beets( + self, model: Match, ctx: Context + ) -> BeetsAlbumMatch | BeetsTrackMatch: + if isinstance(model, AlbumMatch): + return self.album_mapper.to_beets(model, ctx) + + if isinstance(model, TrackMatch): + return self.track_mapper.to_beets(model, ctx) + + raise TypeError(f"Unsupported model type: {type(model)}") + + +# ----------------------------------- Info ----------------------------------- # + + +class TrackInfoMapper(BeetsMapper[BeetsTrackInfo, TrackInfo]): + def _from_beets(self, obj: BeetsTrackInfo, ctx: Context) -> TrackInfo: + data = {k: v for k, v in obj.items() if not k.startswith("_")} + model = TrackInfo(data=data) + return model + + def _to_beets(self, model: TrackInfo, ctx: Context) -> BeetsTrackInfo: + beets_obj = BeetsTrackInfo(**model.data) + return beets_obj + + +class AlbumInfoMapper(BeetsMapper[BeetsAlbumInfo, AlbumInfo]): + def __init__(self): + self.track_mapper = TrackInfoMapper() + + def _from_beets(self, obj: BeetsAlbumInfo, ctx: Context) -> AlbumInfo: + data = {k: v for k, v in obj.items()} + data.pop("tracks", None) + return AlbumInfo( + tracks=[self.track_mapper.from_beets(t, ctx) for t in obj.tracks], + data=data, + ) + + def _to_beets(self, model: AlbumInfo, ctx: Context) -> BeetsAlbumInfo: + data = dict(model.data) + data.pop("tracks", None) + return BeetsAlbumInfo( + tracks=[self.track_mapper.to_beets(t, ctx) for t in model.tracks], + **data, + ) + + +class DistanceMapper(BeetsMapper[BeetsDistance, Distance]): + def __init__(self): + self.track_mapper = TrackInfoMapper() + + def _from_beets(self, obj: BeetsDistance, ctx: Context) -> Distance: + penalties = [Penalty(key=k, value=v) for k, v in obj._penalties.items()] + + track_distances: list[Distance] = [] + for beets_track_info, track_distance in obj.tracks.items(): + child = self.from_beets(track_distance, ctx) + child.track_info = self.track_mapper.from_beets(beets_track_info, ctx) + track_distances.append(child) + + return Distance( + raw_distance=obj.raw_distance, + max_distance=obj.max_distance, + penalties=penalties, + track_distances=track_distances, + ) + + def _to_beets(self, model: Distance, ctx: Context) -> BeetsDistance: + distance = BeetsDistance() + + for penalty in model.penalties: + for value in penalty.value: + distance.add(penalty.key, value) + + for track_distance in model.track_distances: + if track_distance.track_info is not None: + distance.tracks[ + self.track_mapper.to_beets(track_distance.track_info, ctx) + ] = self.to_beets(track_distance, ctx) + + return distance + + +# ---------------------------------- Matches --------------------------------- # + + +class TrackMatchMapper(BeetsMapper[BeetsTrackMatch, TrackMatch]): + def __init__(self): + self.track_info_mapper = TrackInfoMapper() + self.distance_mapper = DistanceMapper() + + def _from_beets(self, obj: BeetsTrackMatch, ctx: Context) -> TrackMatch: + return TrackMatch( + info=self.track_info_mapper.from_beets(obj.info, ctx), + distance=self.distance_mapper.from_beets(obj.distance, ctx), + ) + + def _to_beets(self, model: TrackMatch, ctx: Context) -> BeetsTrackMatch: + return BeetsTrackMatch( + info=self.track_info_mapper.to_beets(model.info, ctx), + distance=self.distance_mapper.to_beets(model.distance, ctx), + ) + + +class AlbumMatchMapper(BeetsMapper[BeetsAlbumMatch, AlbumMatch]): + def __init__(self): + self.album_info_mapper = AlbumInfoMapper() + self.distance_mapper = DistanceMapper() + self.track_info_mapper = TrackInfoMapper() + self.item_mapper = ItemMapper() + + def _from_beets(self, obj: BeetsAlbumMatch, ctx: Context) -> AlbumMatch: + model = AlbumMatch( + info=self.album_info_mapper.from_beets(obj.info, ctx), + distance=self.distance_mapper.from_beets(obj.distance, ctx), + ) + + # extra tracks + for extra_track in obj.extra_tracks: + model.track_mappings.append( + AlbumMatchTrackMapping( + track_info=self.track_info_mapper.from_beets(extra_track, ctx), + item=None, + ) + ) + + # extra items + for extra_item in obj.extra_items: + model.track_mappings.append( + AlbumMatchTrackMapping( + track_info=None, + item=self.item_mapper.from_beets(extra_item, ctx), + ) + ) + + # pairs + for item, track in obj.mapping.items(): + model.track_mappings.append( + AlbumMatchTrackMapping( + track_info=self.track_info_mapper.from_beets(track, ctx), + item=self.item_mapper.from_beets(item, ctx), + ) + ) + + return model + + def _to_beets(self, model: AlbumMatch, ctx: Context) -> BeetsAlbumMatch: + mapping: dict[BeetsItem, BeetsTrackInfo] = {} + extra_items: list[BeetsItem] = [] + extra_tracks: list[BeetsTrackInfo] = [] + + for tm in model.track_mappings: + # pairs + if tm.track_info is not None and tm.item is not None: + item = self.item_mapper.to_beets(tm.item, ctx) + track_info = self.track_info_mapper.to_beets(tm.track_info, ctx) + mapping[item] = track_info + + # extra track + elif tm.track_info is not None: + extra_tracks.append(self.track_info_mapper.to_beets(tm.track_info, ctx)) + + # extra item + elif tm.item is not None: + extra_items.append(self.item_mapper.to_beets(tm.item, ctx)) + + return BeetsAlbumMatch( + distance=self.distance_mapper.to_beets(model.distance, ctx), + info=self.album_info_mapper.to_beets(model.info, ctx), + mapping=mapping, + extra_items=extra_items, + extra_tracks=extra_tracks, + ) + + +class ItemMapper(BeetsMapper[BeetsItem, Item]): + def _to_beets(self, model: Item, ctx) -> BeetsItem: + return BeetsItem._awaken( + fixed_values={k: self._decode(v) for k, v in model.fixed_values.items()}, + flex_values={k: self._decode(v) for k, v in model.flex_values.items()}, + ) + + def _from_beets(self, obj: BeetsItem, ctx) -> Item: + return Item( + fixed_values={k: self._encode(v) for k, v in obj._values_fixed.items()}, + flex_values={k: self._encode(v) for k, v in obj._values_flex.items()}, + ) + + @classmethod + def _encode(cls, v): + if isinstance(v, bytes): + return { + "__type__": "bytes", + "data": base64.b64encode(v).decode("ascii"), + } + + if isinstance(v, dict): + return {str(k): cls._encode(val) for k, val in v.items()} + + if isinstance(v, list): + return [cls._encode(x) for x in v] + + return v + + @classmethod + def _decode(cls, v): + if isinstance(v, dict): + if v.get("__type__") == "bytes": + return base64.b64decode(v["data"]) + return {k: cls._decode(val) for k, val in v.items()} + + if isinstance(v, list): + return [cls._decode(x) for x in v] + + return v diff --git a/backend/beets_flask/database/models/base.py b/backend/beets_flask/database/models/base.py index 04a34e32..ffe1a44b 100644 --- a/backend/beets_flask/database/models/base.py +++ b/backend/beets_flask/database/models/base.py @@ -18,7 +18,7 @@ from beets_flask.logger import log -from .types import DictType, IntDictType, StrDictType +from .types import DictType, FloatListType, IntDictType, StrDictType class Base(DeclarativeBase): @@ -30,6 +30,7 @@ class Base(DeclarativeBase): dict[int, int]: IntDictType, dict[str, str]: StrDictType, dict[str, Any]: DictType, + list[float]: FloatListType, } ) diff --git a/backend/beets_flask/database/models/match.py b/backend/beets_flask/database/models/match.py new file mode 100644 index 00000000..49c8398a --- /dev/null +++ b/backend/beets_flask/database/models/match.py @@ -0,0 +1,222 @@ +from __future__ import annotations + +from typing import Any + +from sqlalchemy import ForeignKey +from sqlalchemy.orm import Mapped, mapped_column, relationship +from sqlalchemy.types import JSON + +from .base import Base +from .pending import Item + +# --------------------------------- Distance --------------------------------- # + + +class Distance(Base): + __tablename__ = "distances" + + track_info_id: Mapped[str | None] = mapped_column(ForeignKey("track_info.id")) + parent_distance_id: Mapped[str | None] = mapped_column(ForeignKey("distances.id")) + + # FK columns auto-created from relationships + track_info: Mapped[TrackInfo | None] = relationship() + parent_distance: Mapped[Distance | None] = relationship( + remote_side="Distance.id", + back_populates="track_distances", + ) + + penalties: Mapped[list[Penalty]] = relationship( + back_populates="distance", + cascade="all, delete-orphan", + ) + track_distances: Mapped[list[Distance]] = relationship( + back_populates="parent_distance", + cascade="all, delete-orphan", + ) + + raw_distance: Mapped[float] = mapped_column(default=0.0) + max_distance: Mapped[float] = mapped_column(default=0.0) + + def __init__( + self, + raw_distance: float = 0.0, + max_distance: float = 0.0, + penalties: list[Penalty] | None = None, + track_distances: list[Distance] | None = None, + id: str | None = None, + ): + super().__init__(id) + self.raw_distance = raw_distance + self.max_distance = max_distance + self.penalties = penalties or [] + self.track_distances = track_distances or [] + + +class Penalty(Base): + """Individual penalty entries.""" + + __tablename__ = "penalties" + + key: Mapped[str] = mapped_column(index=True) + value: Mapped[list[float]] + distance_id: Mapped[int] = mapped_column(ForeignKey("distances.id")) + + # Derived + distance: Mapped[Distance] = relationship(back_populates="penalties") + + def __init__( + self, + key: str, + value: list[float], + id: str | None = None, + ): + super().__init__(id) + self.key = key + self.value = value + + +# ----------------------------------- Info ----------------------------------- # + + +class TrackInfo(Base): + __tablename__ = "track_info" + + album_id: Mapped[str | None] = mapped_column(ForeignKey("album_info.id")) + album: Mapped[AlbumInfo] = relationship(back_populates="tracks") + data: Mapped[dict[str, Any]] = mapped_column(JSON(), default=dict) + + def __init__( + self, + *, + data: dict[str, Any] | None = None, + id: str | None = None, + ): + super().__init__(id) + self.data = data or {} + + +class AlbumInfo(Base): + __tablename__ = "album_info" + + tracks: Mapped[list[TrackInfo]] = relationship( + back_populates="album", + cascade="all, delete-orphan", + ) + data: Mapped[dict[str, Any]] = mapped_column(JSON(), default=dict) + + def __init__( + self, + data: dict[str, Any] | None = None, + tracks: list[TrackInfo] | None = None, + id: str | None = None, + ): + super().__init__(id) + self.data = data or {} + self.tracks = tracks or [] + + +# ----------------------------------- Match ---------------------------------- # + + +class Match(Base): + """ + Matches are polymorphic — can be album or track matches. + + This requires us to keep two extra tables. + """ + + __tablename__ = "matches" + + # Needed for polymorphic + id: Mapped[str] = mapped_column(primary_key=True) + type: Mapped[str] = mapped_column() + + distance_id: Mapped[str] = mapped_column(ForeignKey("distances.id")) + distance: Mapped[Distance] = relationship() + + __mapper_args__ = { + "polymorphic_on": "type", + "polymorphic_identity": "matches", + } + + +class AlbumMatch(Match): + __tablename__ = "matches_album" + + id: Mapped[str] = mapped_column(ForeignKey("matches.id"), primary_key=True) + + info_id: Mapped[str] = mapped_column(ForeignKey("album_info.id")) + info: Mapped[AlbumInfo] = relationship() + + track_mappings: Mapped[list[AlbumMatchTrackMapping]] = relationship( + back_populates="album_match", + cascade="all, delete-orphan", + ) + + __mapper_args__ = { + "polymorphic_identity": "album", + } + + def __init__( + self, + info: AlbumInfo, + distance: Distance, + id: str | None = None, + ) -> None: + super().__init__(id) + self.info = info + self.distance = distance + + +class TrackMatch(Match): + __tablename__ = "matches_track" + + id: Mapped[str] = mapped_column(ForeignKey("matches.id"), primary_key=True) + + info_id: Mapped[str] = mapped_column(ForeignKey("track_info.id")) + info: Mapped[TrackInfo] = relationship() + + __mapper_args__ = { + "polymorphic_identity": "track", + } + + def __init__( + self, + info: TrackInfo, + distance: Distance, + id: str | None = None, + ) -> None: + self.info = info + self.distance = distance + super().__init__(id) + + +class AlbumMatchTrackMapping(Base): + """Maps items to track_info for an album_match. + + Filter by album_match_id: + - extra_tracks: track_info is not None and item_id is None + - extra_items: track_info is None and item_id is not None + - mapping: both are set + """ + + __tablename__ = "album_match_track_mappings" + + album_match_id: Mapped[str] = mapped_column(ForeignKey("matches_album.id")) + track_info_id: Mapped[str | None] = mapped_column(ForeignKey("track_info.id")) + item_id: Mapped[str | None] = mapped_column(ForeignKey("items.id")) + + # ID of the beets library Item (not our model, just the raw ID) + album_match: Mapped[AlbumMatch] = relationship(back_populates="track_mappings") + track_info: Mapped[TrackInfo | None] = relationship() + item: Mapped[Item | None] = relationship() + + def __init__( + self, + item: Item | None = None, + track_info: TrackInfo | None = None, + id: str | None = None, + ): + self.track_info = track_info + self.item = item + super().__init__(id) diff --git a/backend/beets_flask/database/models/pending.py b/backend/beets_flask/database/models/pending.py index 4f2e0a63..2c5c8f94 100644 --- a/backend/beets_flask/database/models/pending.py +++ b/backend/beets_flask/database/models/pending.py @@ -1,32 +1,16 @@ from __future__ import annotations -import base64 from typing import TYPE_CHECKING, Any from sqlalchemy import JSON, ForeignKey from sqlalchemy.orm import Mapped, mapped_column, relationship -from beets_flask.importer.types import BeetsItem - from .base import Base if TYPE_CHECKING: from .states import TaskStateInDb -class TasksItems(Base): - __tablename__ = "tasks_items" - - task_id: Mapped[str] = mapped_column(ForeignKey("task.id")) - task: Mapped[TaskStateInDb] = relationship(back_populates="pending_items") - item_id: Mapped[str] = mapped_column(ForeignKey("items.id")) - item: Mapped[Item] = relationship() - - def __init__(self, item: Item, id: str | None = None): - super().__init__(id) - self.item = item - - class Item(Base): __tablename__ = "items" @@ -46,45 +30,15 @@ def __init__( self.fixed_values = fixed_values self.flex_values = flex_values - # FIXME: Move to mapper layer after match migration! - - def to_beets(self): - return BeetsItem._awaken( - fixed_values={k: self._decode(v) for k, v in self.fixed_values.items()}, - flex_values={k: self._decode(v) for k, v in self.flex_values.items()}, - ) - @classmethod - def from_beets(cls, obj: BeetsItem): - return cls( - fixed_values={k: cls._encode(v) for k, v in obj._values_fixed.items()}, - flex_values={k: cls._encode(v) for k, v in obj._values_flex.items()}, - ) - - @classmethod - def _encode(cls, v): - if isinstance(v, bytes): - return { - "__type__": "bytes", - "data": base64.b64encode(v).decode("ascii"), - } - - if isinstance(v, dict): - return {str(k): cls._encode(val) for k, val in v.items()} - - if isinstance(v, list): - return [cls._encode(x) for x in v] - - return v - - @classmethod - def _decode(cls, v): - if isinstance(v, dict): - if v.get("__type__") == "bytes": - return base64.b64decode(v["data"]) - return {k: cls._decode(val) for k, val in v.items()} +class TaskItem(Base): + __tablename__ = "tasks_items" - if isinstance(v, list): - return [cls._decode(x) for x in v] + task_id: Mapped[str] = mapped_column(ForeignKey("task.id")) + task: Mapped[TaskStateInDb] = relationship(back_populates="pending_items") + item_id: Mapped[str] = mapped_column(ForeignKey("items.id")) + item: Mapped[Item] = relationship() - return v + def __init__(self, item: Item, id: str | None = None): + super().__init__(id) + self.item = item diff --git a/backend/beets_flask/database/models/states.py b/backend/beets_flask/database/models/states.py index 2fa045c3..82bff105 100644 --- a/backend/beets_flask/database/models/states.py +++ b/backend/beets_flask/database/models/states.py @@ -13,13 +13,9 @@ from __future__ import annotations -import io import pickle from pathlib import Path -from typing import Any -from beets.autotag import AlbumMatch -from beets.autotag.distance import Distance from beets.importer import Action, ImportTask from sqlalchemy import ( ForeignKey, @@ -33,7 +29,10 @@ relationship, ) +from beets_flask.database.mapper.base import Context +from beets_flask.database.mapper.match import ItemMapper, MatchMapper from beets_flask.database.models.base import Base +from beets_flask.database.models.match import Match from beets_flask.disk import Archive, Folder from beets_flask.importer.progress import Progress from beets_flask.importer.states import ( @@ -44,11 +43,11 @@ SessionState, TaskState, ) -from beets_flask.importer.types import BeetsAlbumMatch, BeetsItem, BeetsTrackMatch +from beets_flask.importer.types import BeetsItem from beets_flask.logger import log from beets_flask.server.exceptions import SerializedException -from .pending import Item, TasksItems +from .pending import TaskItem class FolderInDb(Base): @@ -364,7 +363,7 @@ class TaskStateInDb(Base): old_paths: Mapped[bytes | None] # old_paths contain original file paths, but are only set when files are moved. # (which breaks some deep links that before were identical to paths, but no more!) - pending_items: Mapped[list[TasksItems]] = relationship( + pending_items: Mapped[list[TaskItem]] = relationship( back_populates="task", cascade="all, delete-orphan", ) @@ -380,11 +379,9 @@ class TaskStateInDb(Base): @property def items(self) -> list[BeetsItem]: - return [row.item.to_beets() for row in self.pending_items] - - @items.setter - def items(self, value: list[BeetsItem]): - self.pending_items = [TasksItems(item=Item.from_beets(v)) for v in value] + ctx = Context() + mapper = ItemMapper() + return [mapper.to_beets(row.item, ctx) for row in self.pending_items] def __init__( self, @@ -392,7 +389,7 @@ def __init__( toppath: bytes | None = None, paths: list[bytes] = [], old_paths: list[bytes] | None = None, - items: list[BeetsItem] = [], + pending_items: list[TaskItem] = [], candidates: list[CandidateStateInDb] = [], chosen_candidate_id: str | None = None, progress: Progress = Progress.NOT_STARTED, @@ -405,7 +402,7 @@ def __init__( self.paths = pickle.dumps(paths) self.old_paths = pickle.dumps(old_paths) if old_paths else None - self.items = items + self.pending_items = pending_items self.candidates = candidates self.chosen_candidate_id = chosen_candidate_id self.progress = progress @@ -421,11 +418,16 @@ def from_live_state(cls, state: TaskState) -> TaskStateInDb: else: old_paths = None + ctx = Context() + mapper = ItemMapper() + task = cls( id=state.id, toppath=str(state.toppath).encode("utf-8") if state.toppath else None, paths=state.task.paths, - items=state.items, + pending_items=[ + TaskItem(item=mapper.from_beets(item, ctx)) for item in state.items + ], candidates=[ CandidateStateInDb.from_live_state(c) for c in state.candidate_states ], @@ -490,8 +492,8 @@ class CandidateStateInDb(Base): ) # Should deserialize to AlbumMatch|TrackMatch - # ~4kb per match - match: Mapped[bytes] + match_id: Mapped[str] = mapped_column(ForeignKey("matches.id")) + match: Mapped[Match] = relationship() # Duplicate ids (if any) (beets_id) duplicate_ids: Mapped[str] @@ -501,25 +503,14 @@ class CandidateStateInDb(Base): def __init__( self, - match: BeetsAlbumMatch | BeetsTrackMatch, + match: Match, mapping: dict[int, int], duplicate_ids: list[str] = [], id: str | None = None, ): super().__init__(id) - # Remove db from all items as it can't be pickled - # FIXME: this should go into beets __getstate__ method - # see https://github.com/beetbox/beets/pull/5641 - if isinstance(match, BeetsAlbumMatch): - for item in match.mapping.keys(): - item._db = None - item._Item__album = None - for item in match.extra_items: - item._db = None - item._Item__album = None - - self.match = pickle.dumps(match) + self.match = match self.duplicate_ids = ";".join(map(str, duplicate_ids)) self.mapping = mapping @@ -528,7 +519,7 @@ def from_live_state(cls, state: CandidateState) -> CandidateStateInDb: """Create the DB representation of a live CandidateState.""" return cls( id=state.id, - match=state.match, + match=MatchMapper().from_beets(state.match, Context()), duplicate_ids=state.duplicate_ids, mapping=state._mapping, ) @@ -538,7 +529,7 @@ def to_live_state(self, task_state: TaskState | None) -> CandidateState: if task_state is None: task_state = self.task.to_live_state() live_state = CandidateState( - CustomUnpickler(io.BytesIO(self.match)).load(), + MatchMapper().to_beets(self.match, Context()), task_state, mapping=self.mapping, ) @@ -555,43 +546,4 @@ def to_dict(self) -> SerializedCandidateState: return self.to_live_state(self.task.to_live_state()).serialize() -# Hotfix for match unpickler to resolve beets distance moved -# This is needed because various beets updates changed class implementations -# and we want to rebuild the newer versions of some beets classes from old pickles. -# TODO: We should fix this in general and not pickle beets objects -class CustomUnpickler(pickle.Unpickler): - def find_class(self, module, name): - """Override the find_class method to redirect Distance class references.""" - # Redirect Distance class from beets.autotag.hooks to beets.distance (2.4.0) - if module == "beets.autotag.hooks" and name == "Distance": - return Distance - - # For all other classes, use the default lookup mechanism - return super().find_class(module, name) - - def load(self) -> Any: - object = super().load() - if isinstance(object, Distance): - self.patch_distance(object) - - if isinstance(object, AlbumMatch): - self.patch_distance(object.distance) - - return object - - def patch_distance(self, distance: Distance) -> Distance: - # Rewrite "source" penalty to "data_source" penalty (2.5.0) - if "source" in distance._penalties: - log.debug( - "Converting old distance.source to distance.data_source (changed in beets 2.5.0)" - ) - distance._penalties["data_source"] = distance._penalties["source"] - del distance._penalties["source"] - - # Potential infinite recursion, ah well - for track, d in distance.tracks.items(): - self.patch_distance(d) - return distance - - __all__ = ["SessionStateInDb", "TaskStateInDb", "CandidateStateInDb"] diff --git a/backend/beets_flask/database/models/types.py b/backend/beets_flask/database/models/types.py index 4d558a60..63207b72 100644 --- a/backend/beets_flask/database/models/types.py +++ b/backend/beets_flask/database/models/types.py @@ -1,4 +1,7 @@ +from __future__ import annotations + import json +from array import array from typing import Any from sqlalchemy import types @@ -65,3 +68,34 @@ class StrDictType(DictType): allowed_keys_types = (str,) allowed_values_types = (str,) + + +class FloatListType(types.TypeDecorator): + """Stores a list[float] as binary using array.array ('d' = float64).""" + + impl = types.LargeBinary + cache_ok = True + + def process_bind_param( + self, value: list[float] | None, dialect: Any + ) -> bytes | None: + if value is None: + return None + if not isinstance(value, list): + raise ValueError("Value must be a list") + if not all(isinstance(v, int | float) for v in value): + raise ValueError(f"All values must be float, got: {value}") + arr = array("d", value) + return arr.tobytes() + + def process_result_value( + self, value: bytes | None, dialect: Any + ) -> list[float] | None: + if value is None: + return None + arr = array("d") + arr.frombytes(value) + return arr.tolist() + + def copy(self, **kw: Any) -> FloatListType: + return self.__class__() diff --git a/backend/beets_flask/importer/session.py b/backend/beets_flask/importer/session.py index cd91bbec..b8427429 100644 --- a/backend/beets_flask/importer/session.py +++ b/backend/beets_flask/importer/session.py @@ -36,7 +36,8 @@ from typing import Any, Literal, TypedDict, TypeGuard, TypeVar import nest_asyncio -from beets import autotag, importer, plugins +from beets import autotag, plugins +from beets.importer import ImportAbortError from beets.ui import UserError, _open_library from beets.util import bytestring_path from deprecated import deprecated @@ -46,6 +47,9 @@ from beets_flask.importer.progress import Progress, ProgressState from beets_flask.importer.types import ( BeetsAlbum, + BeetsImportAction, + BeetsImportSession, + BeetsImportTask, BeetsLibrary, DuplicateAction, ) @@ -150,7 +154,7 @@ class Search(TypedDict): search_ids: list[str] search_artist: str | None - search_album: str | None + search_name: str | None def _is_search(d: Any) -> TypeGuard[Search]: @@ -168,7 +172,7 @@ def _is_search(d: Any) -> TypeGuard[Search]: # ---------------------------------------------------------------------------- # -class BaseSession(importer.ImportSession, ABC): +class BaseSession(BeetsImportSession, ABC): """Base class for our GUI-based ImportSessions. Operates on single Albums / files. @@ -191,7 +195,7 @@ class BaseSession(importer.ImportSession, ABC): # are contained in the associated SessionState -> TaskState -> CandidateStates state: SessionState - pipeline: AsyncPipeline[importer.ImportTask, Any] | None = None + pipeline: AsyncPipeline[BeetsImportTask, Any] | None = None config_overlay: dict lib: BeetsLibrary @@ -283,7 +287,7 @@ def get_config_value(self, key: str, type_func: Callable | None = None) -> Any: # -------------------------- State handling helpers -------------------------- # def set_task_progress( - self, task: importer.ImportTask, progress: ProgressState | Progress | str + self, task: BeetsImportTask, progress: ProgressState | Progress | str ): """Set the progress for a task belonging to the session. @@ -296,7 +300,7 @@ def set_task_progress( task_state.set_progress(progress) - def get_task_progress(self, task: importer.ImportTask) -> ProgressState | None: + def get_task_progress(self, task: BeetsImportTask) -> ProgressState | None: """Get the progress of the task, via this sessions state.""" task_state = self.state.get_task_state_for_task(task) return task_state.progress if task_state else None @@ -312,7 +316,7 @@ def stages(self) -> StageOrder: """ raise NotImplementedError("Implement in subclass") - def resolve_duplicate(self, task: importer.ImportTask, found_duplicates): + def resolve_duplicate(self, task: BeetsImportTask, found_duplicates): """Overload default resolve duplicate and skip it. This basically skips this stage. @@ -321,15 +325,15 @@ def resolve_duplicate(self, task: importer.ImportTask, found_duplicates): "Skipping duplicate resolution. " + f"Your session should implement this! -> {self.__class__.__name__}" ) - task.set_choice(importer.Action.SKIP) + task.set_choice(BeetsImportAction.SKIP) - def choose_item(self, task: importer.ImportTask): + def choose_item(self, task: BeetsImportTask): """Overload default choose item and skip it. This session should not reach this stage. """ self.logger.debug(f"skipping choose_item {task}") - return importer.Action.SKIP + return BeetsImportAction.SKIP def should_resume(self, path): """Overload default should_resume and skip it. @@ -340,7 +344,7 @@ def should_resume(self, path): self.logger.debug(f"skipping should_resume {path}") return False - def identify_duplicates(self, task: importer.ImportTask): + def identify_duplicates(self, task: BeetsImportTask): """For all candidates, check if they have duplicates in the library. This stage should only be run for preview sessions, but we still have @@ -350,7 +354,7 @@ def identify_duplicates(self, task: importer.ImportTask): f"This session should not reach this stage. {self.__class__.__name__}" ) - def lookup_candidates(self, task: importer.ImportTask): + def lookup_candidates(self, task: BeetsImportTask): """Lookup candidates for the task. This stage should only be run for preview sessions, but we still have @@ -360,7 +364,7 @@ def lookup_candidates(self, task: importer.ImportTask): f"This session should not reach this stage. {self.__class__.__name__}" ) - def finalize(self, task: importer.ImportTask): + def finalize(self, task: BeetsImportTask): """Last stage called and customizable any session.""" if len(self.config_overlay) > 0: # make sure we dont leave overlays in beets @@ -381,7 +385,7 @@ async def run_async(self) -> SessionState: Take care of this in subclasses. """ # For now, until we improve the upstream beets config logic, - # adhere to importer.ImportSession convention and create a local copy + # adhere to BeetsImportSession convention and create a local copy # of the config. config = get_config().beets_config self.set_config(config["import"]) @@ -403,7 +407,7 @@ async def run_async(self) -> SessionState: try: assert self.pipeline is not None await self.pipeline.run_async() - except importer.ImportAbortError: + except ImportAbortError: log.debug(f"Interactive import session aborted by user") except ApiException as e: if e.persist_in_db: @@ -476,7 +480,7 @@ def stages(self) -> StageOrder: # --------------------------- Stage Definitions -------------------------- # - def identify_duplicates(self, task: importer.ImportTask): + def identify_duplicates(self, task: BeetsImportTask): """For all candidates, check if they have duplicates in the library.""" task_state = self.state.get_task_state_for_task_raise(task) @@ -489,7 +493,7 @@ def identify_duplicates(self, task: importer.ImportTask): if len(duplicates) > 0: log.debug(f"Found duplicates for {cs.id=}: {duplicates}") - def lookup_candidates(self, task: importer.ImportTask): + def lookup_candidates(self, task: BeetsImportTask): """Lookup candidates for the task.""" search_ids = self.config["search_ids"].as_str_seq() # might be an empty list @@ -542,7 +546,7 @@ def __init__( if s != "skip": task.set_progress(Progress.LOOKING_UP_CANDIDATES - 1) - def lookup_candidates(self, task: importer.ImportTask): + def lookup_candidates(self, task: BeetsImportTask): """Amend the found candidate to the already existing candidates (if any).""" # see ref in lookup_candidates in beets/importer.py @@ -560,44 +564,20 @@ def lookup_candidates(self, task: importer.ImportTask): and search["search_artist"].strip() == "" ): search["search_artist"] = None - if search["search_album"] is not None and search["search_album"].strip() == "": - search["search_album"] = None + if search["search_name"] is not None and search["search_name"].strip() == "": + search["search_name"] = None search["search_ids"] = list( filter(lambda x: x.strip() != "", search["search_ids"]) ) log.debug(f"Using {search=} for {task_state.id=}, {task_state.paths=}") - try: - _, _, prop = autotag.tag_album( - task.items, - search_ids=search["search_ids"], - search_album=search["search_album"], - search_artist=search["search_artist"], - ) - except Exception as e: - # TODO: With beets 2.6.0 this should be revisited - # since beets should than be able to handle these exceptions - # gracefully upstream. - # https://github.com/beetbox/beets/pull/5965 - from beetsplug.musicbrainz import MusicBrainzAPIError - from beetsplug.spotify import APIError as SpotifyAPIError - - if isinstance(e, MusicBrainzAPIError): - raise NoCandidatesFoundException( - f"Failed to contact Musicbrainz API: {e.get_message()}", - persist_in_db=False, - ) - elif isinstance(e, SpotifyAPIError): - raise NoCandidatesFoundException( - f"Failed to contact Spotify API: {e}", - persist_in_db=False, - ) - else: - raise NoCandidatesFoundException( - f"Failed to contact online APIs.", - persist_in_db=False, - ) + _, _, prop = autotag.tag_album( + task.items, + search_ids=search["search_ids"], + search_name=search["search_name"], + search_artist=search["search_artist"], + ) task_state.add_candidates(prop.candidates) @@ -610,8 +590,8 @@ def lookup_candidates(self, task: importer.ImportTask): error_text += f"ids: {', '.join(search['search_ids'])}; " if search["search_artist"]: error_text += f"artist: {search['search_artist']}; " - if search["search_album"]: - error_text += f"album: {search['search_album']}; " + if search["search_name"]: + error_text += f"album: {search['search_name']}; " error_text += NoCandidatesFoundException.metadata_plugin_info() raise NoCandidatesFoundException( error_text, @@ -627,7 +607,7 @@ def lookup_candidates(self, task: importer.ImportTask): ) self.state.exc = None - def finalize(self, task: importer.ImportTask): + def finalize(self, task: BeetsImportTask): """Restore initial taks and session states.""" task_state = self.state.get_task_state_for_task_raise(task) @@ -769,7 +749,7 @@ def stages(self): return stages - def finalize(self, task: importer.ImportTask): + def finalize(self, task: BeetsImportTask): """ Reset previous match threshold exceptions. @@ -792,7 +772,7 @@ def finalize(self, task: importer.ImportTask): # --------------------------- Stage Definitions -------------------------- # - def choose_match(self, task: importer.ImportTask): + def choose_match(self, task: BeetsImportTask): self.logger.setLevel(logging.DEBUG) self.logger.debug(f"choose_match {task}") @@ -833,12 +813,12 @@ def choose_match(self, task: importer.ImportTask): # ASIS if candidate_state.id == task_state.asis_candidate.id: log.debug(f"Importing {task} as-is") - return importer.Action.ASIS + return BeetsImportAction.ASIS return candidate_state.match def resolve_duplicate( - self, task: importer.ImportTask, found_duplicates: list[BeetsAlbum] + self, task: BeetsImportTask, found_duplicates: list[BeetsAlbum] ): log.debug( f"Resolving duplicates for {task} with action {self.duplicate_actions}" @@ -853,7 +833,7 @@ def resolve_duplicate( task_state.duplicate_action = task_duplicate_action match task_duplicate_action: case "skip": - task.set_choice(importer.Action.SKIP) + task.set_choice(BeetsImportAction.SKIP) case "keep": pass case "remove": @@ -861,7 +841,7 @@ def resolve_duplicate( case "merge": task.should_merge_duplicates = True case "ask": - # task.set_choice(importer.action.SKIP) + # task.set_choice(BeetsImportAction.SKIP) raise DuplicateException( "You have set the duplicate action to 'ask' in your beets config." ) @@ -985,13 +965,13 @@ def stages(self): stages.insert(before="user_query", stage=match_threshold(self)) return stages - def match_threshold(self, task: importer.ImportTask): + def match_threshold(self, task: BeetsImportTask): """Check if the match quality is good enough to import. Returns true if candidates were found, and the match quality is better than threshlold. - Note: What stops the pipeline is that we set task.choice to importer.action.SKIP, + Note: What stops the pipeline is that we set task.choice to BeetsImportAction.SKIP, or raise an exception. Currently raising, as we do not have a dedicated progress for "not imported". @@ -1018,7 +998,7 @@ def match_threshold(self, task: importer.ImportTask): t = (1 - self.import_threshold) * 100 raise NotImportedException(f"Match below threshold ({d:.0f}% < {t:.0f}%)") # beets would handle this via the task action: - task.set_choice(importer.action.SKIP) + task.set_choice(BeetsImportAction.SKIP) else: log.info( f"Best candidate was better than threshold, importing to library. {distance=} {self.import_threshold=}" diff --git a/backend/beets_flask/importer/states.py b/backend/beets_flask/importer/states.py index 6480332b..6b55cc6c 100644 --- a/backend/beets_flask/importer/states.py +++ b/backend/beets_flask/importer/states.py @@ -10,9 +10,9 @@ from typing import Literal, NotRequired, TypedDict, cast from uuid import uuid4 as uuid -import beets.ui.commands as uicommands from beets import importer from beets.ui import _open_library +from beets.ui.commands.import_.display import show_change from beets.util import bytestring_path, get_most_common_tags from deprecated import deprecated @@ -484,7 +484,7 @@ def type(self) -> Literal["album", "track"]: def diff_preview(self) -> str: """Diff preview of the match to the current meta data.""" out, err, _ = capture_stdout_stderr( - uicommands.show_change, + show_change, self.task_state.task.cur_artist, self.task_state.task.cur_album, self.match, diff --git a/backend/beets_flask/importer/types.py b/backend/beets_flask/importer/types.py index be11f51f..73636ee7 100644 --- a/backend/beets_flask/importer/types.py +++ b/backend/beets_flask/importer/types.py @@ -21,6 +21,8 @@ from beets.autotag.hooks import AlbumMatch as BeetsAlbumMatch from beets.autotag.hooks import TrackInfo as BeetsTrackInfo from beets.autotag.hooks import TrackMatch as BeetsTrackMatch +from beets.importer import Action as BeetsImportAction +from beets.importer import ImportSession as BeetsImportSession from beets.importer import ImportTask as BeetsImportTask from beets.library import Album as BeetsAlbum from beets.library import Item as BeetsItem @@ -42,7 +44,9 @@ "BeetsTrackMatch", "BeetsLibrary", "BeetsDistance", + "BeetsImportAction", "BeetsImportTask", + "BeetsImportSession", ] # to be consistent with beets, here we do not use an enum. diff --git a/backend/beets_flask/invoker/enqueue.py b/backend/beets_flask/invoker/enqueue.py index fa6505f7..cda010b8 100644 --- a/backend/beets_flask/invoker/enqueue.py +++ b/backend/beets_flask/invoker/enqueue.py @@ -209,7 +209,7 @@ def enqueue_preview(hash: str, path: str, extra_meta: ExtraJobMeta, **kwargs) -> def enqueue_preview_add_candidates( hash: str, path: str, extra_meta: ExtraJobMeta, **kwargs ) -> Job: - # May contain search_ids, search_artist, search_album + # May contain search_ids, search_artist, search_name # As always to allow task mapping search: TaskIdMappingArg[Search | Literal["skip"]] = kwargs.pop("search", None) diff --git a/backend/pyproject.toml b/backend/pyproject.toml index e3af6f86..bdb1ed74 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -15,7 +15,7 @@ classifiers = [ dependencies = [ "quart>=0.20.0", "confuse>=2.0.1", - "beets==2.5.1", + "beets==2.6.1", "sqlalchemy>=2.0.35", "rq>=2.0.0", "watchdog>=5.0.3", @@ -44,7 +44,15 @@ dependencies = [ "alembic>=1.18.4", ] -[project.optional-dependencies] +[dependency-groups] +dev = [ + { include-group = "test" }, + { include-group = "docs" }, + { include-group = "typed" }, + "ruff>=0.6.5", + "pre-commit>=3.8.0", +] + # Can be install with e.g. `pip install -e .[dev]` test = [ "pytest>=8.2.2", @@ -53,7 +61,17 @@ test = [ "pytest-cov>=5.0.0", "fakeredis", ] -dev = ["ruff>=0.6.5", "pre-commit>=3.8.0", "beets_flask[typed]"] +docs = [ + "sphinx>=8.0.2", + "furo>=2024.8.6", + "sphinx-copybutton>=0.5.2", + "sphinx-inline-tabs>=2023.4.21", + "sphinxcontrib-typer[html]>=0.5.0", + "sphinxcontrib-mermaid>=2.0.1", + "myst-parser>=4.0.0", + "myst-nb>=1.1.2", + "paracelsus>=0.15.0", +] typed = [ "types-cachetools", "types-requests", @@ -64,16 +82,6 @@ typed = [ "types-pyyaml", "pandas-stubs", ] -all = ["beets_flask[dev,test,typed]"] -docs = [ - "sphinx>=8.0.2", - "furo>=2024.8.6", - "sphinx-copybutton>=0.5.2", - "sphinx-inline-tabs>=2023.4.21", - "sphinxcontrib-typer[html]>=0.5.0", - "myst-parser>=4.0.0", - "myst-nb>=1.1.2", -] [build-system] requires = ["hatchling"] @@ -116,7 +124,6 @@ fixable = ["ALL"] [tool.ruff.lint.pydocstyle] convention = "numpy" - [tool.ruff.lint.isort] known-first-party = ["beets_flask", "tests"] known-third-party = ["alembic", "sqlalchemy"] diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 35a33b4b..88c97d87 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -1,12 +1,17 @@ +import hashlib import logging import os +import pickle import shutil +import tempfile from collections.abc import Callable, Generator from contextlib import _GeneratorContextManager from pathlib import Path import pytest import yaml +from beets import autotag +from beets.autotag import tag_album as _tag_album from quart import Quart from quart.typing import TestClientProtocol from sqlalchemy.orm import Session @@ -46,7 +51,7 @@ def setup_and_teardown(tmpdir_factory): # we have one test that does replacements on this file # and assumes the default 4 workers with open(tmp_dir / "beets/config.yaml", "w") as f: - yaml.dump({"plugins": ["musicbrainz"]}, f) + yaml.dump({"plugins": ["musicbrainz", "spotify"]}, f) yield @@ -205,3 +210,60 @@ def local_redis(monkeypatch): yield log.debug("Unmocking beets_flask.redis") monkeypatch.undo() + + +lookup_cache_dir: Path + + +@pytest.fixture(scope="module", autouse=True) +def mock_tag_album(): + """Fixture that monkeypatches beets tag_album to use cached lookups.""" + # Create temp lookup cache directory once per module + global lookup_cache_dir + + lookup_cache_dir = Path(tempfile.mkdtemp(prefix="beets_lookup_cache_")) + + original_tag_album = autotag.tag_album + autotag.tag_album = tag_album + yield lookup_cache_dir + autotag.tag_album = original_tag_album + + +def tag_album( + items, + search_artist: str | None = None, + search_name: str | None = None, + search_ids: list[str] = [], +): + global lookup_cache_dir + # Compute items hash based on the items + m = hashlib.md5() + for item in items: + m.update(item.path) + if search_artist: + m.update(search_artist.encode("utf-8")) + if search_name: + m.update(search_name.encode("utf-8")) + for search_id in search_ids: + m.update(search_id.encode("utf-8")) + items_hash = m.hexdigest()[:8] + + cache_file = lookup_cache_dir / f"lookup_{items_hash}.pickle" + if cache_file.exists(): + log.debug(f"Using cached lookup from temp dir {cache_file}") + with open(cache_file, "rb") as f: + return pickle.load(f) + else: + # TODO: This pickle contains absolute paths to the files + # while undesired (no use in having them in the git repo) its for now the + # easiest way... and we hope music brainz does not change its data too often! + res = _tag_album(items, search_artist, search_name, search_ids) + + cache_file.parent.mkdir(parents=True, exist_ok=True) + with open(cache_file, "wb") as f: + pickle.dump(res, f) + + return res + + +autotag.tag_album = tag_album diff --git a/backend/tests/integration/test_flows.py b/backend/tests/integration/test_flows.py index 621c74e7..df3a9f33 100644 --- a/backend/tests/integration/test_flows.py +++ b/backend/tests/integration/test_flows.py @@ -41,7 +41,6 @@ from tests.unit.test_importer.conftest import ( VALID_PATHS, album_path_absolute, - use_mock_tag_album, ) @@ -100,7 +99,6 @@ class TestPreview(SendStatusMockMixin, IsolatedDBMixin, IsolatedBeetsLibraryMixi ) def path(self, request) -> Path: path = album_path_absolute(request.param) - use_mock_tag_album(str(path)) return path async def test_preview( @@ -165,7 +163,6 @@ class TestPreviewMultipleTasks( @pytest.fixture() def path(self) -> Path: path = album_path_absolute("multi_flat") - use_mock_tag_album(str(path)) return path @pytest.mark.parametrize( @@ -248,7 +245,6 @@ class TestImportBest(SendStatusMockMixin, IsolatedDBMixin, IsolatedBeetsLibraryM @pytest.fixture() def path(self) -> Path: path = album_path_absolute(VALID_PATHS[0]) - use_mock_tag_album(str(path)) return path def check_mapping_consistency(self, db_session: Session): @@ -310,7 +306,7 @@ async def test_add_candidates(self, db_session: Session, path: Path): id_99_red_balloons, ], # Nena 99 Red Balloons "search_artist": None, - "search_album": None, + "search_name": None, } }, ) @@ -352,7 +348,7 @@ async def test_add_candidates_fails(self, db_session: Session, path: Path): "non_existing_id", ], # Nena 99 Red Balloons "search_artist": None, - "search_album": None, + "search_name": None, } }, ) @@ -392,7 +388,7 @@ async def test_add_candidates_cleared(self, db_session: Session, path: Path): id_99_red_balloons, ], # Nena 99 Red Balloons "search_artist": None, - "search_album": None, + "search_name": None, } }, ) @@ -712,7 +708,6 @@ class TestImportAuto(SendStatusMockMixin, IsolatedDBMixin, IsolatedBeetsLibraryM @pytest.fixture() def path(self) -> Path: path = album_path_absolute(VALID_PATHS[0]) - use_mock_tag_album(str(path)) return path async def test_import_auto_accept(self, db_session: Session, path: Path): @@ -764,7 +759,6 @@ class TestImportAutoFails( @pytest.fixture() def path(self) -> Path: path = album_path_absolute(VALID_PATHS[0]) - use_mock_tag_album(str(path)) return path async def test_import_auto_fails(self, db_session: Session, path: Path): @@ -824,7 +818,6 @@ class TestChooseCandidatesSingleTask( @pytest.fixture() def path_single_task(self) -> Path: path = album_path_absolute(VALID_PATHS[0]) - use_mock_tag_album(str(path)) return path async def test_choose_candidates( @@ -850,6 +843,7 @@ async def test_choose_candidates( assert s_state_indb.folder.full_path == str(path_single_task) assert len(s_state_indb.tasks) == 1 + # Should have to candidates (one from mb and one from spotify) choosen_candidate = s_state_indb.tasks[0].candidates[-2] exc = await run_import_candidate( @@ -886,7 +880,6 @@ class TestMultipleTasks( @pytest.fixture() def path_multiple_tasks(self) -> Path: path = album_path_absolute("multi") - use_mock_tag_album(str(path)) return path async def test_choose_candidates_multiple_tasks( @@ -922,9 +915,7 @@ async def test_choose_candidates_multiple_tasks( candidates: TaskIdMappingArg[CandidateChoice] = {} assert candidates is not None for task in s_state_indb.tasks: - print(task.paths) - print([c.metadata for c in task.candidates]) - assert len(task.candidates) > 2, "Should have candidates" + assert len(task.candidates) >= 2, "Should have candidates" candidates[task.id] = task.candidates[-2].id # Check that we have the same number of candidates as tasks @@ -983,7 +974,7 @@ async def test_duplicate_action( assert duplicate_actions is not None for task in s_state_indb.tasks: - assert len(task.candidates) > 2, "Should have candidates" + assert len(task.candidates) >= 2, "Should have candidates" candidates[task.id] = task.candidates[-2].id duplicate_actions[task.id] = duplicate_action @@ -1013,7 +1004,6 @@ class TestPluginEvents( @pytest.fixture() def path(self) -> Path: path = album_path_absolute(VALID_PATHS[0]) - use_mock_tag_album(str(path)) return path async def test_preview_events(self, db_session: Session, path: Path): @@ -1114,7 +1104,6 @@ class TestImportBootleg( @pytest.fixture() def path(self) -> Path: path = album_path_absolute(VALID_PATHS[0]) - use_mock_tag_album(str(path)) return path async def test_import_bootleg(self, db_session: Session, path: Path): diff --git a/backend/tests/unit/test_database/mapper/test_match.py b/backend/tests/unit/test_database/mapper/test_match.py new file mode 100644 index 00000000..edc1565e --- /dev/null +++ b/backend/tests/unit/test_database/mapper/test_match.py @@ -0,0 +1,371 @@ +from beets.autotag.distance import Distance +from beets.autotag.distance import Distance as BeetsDistance +from beets.autotag.hooks import AlbumInfo as BeetsAlbumInfo +from beets.autotag.hooks import AlbumMatch as BeetsAlbumMatch +from beets.autotag.hooks import TrackInfo as BeetsTrackInfo +from beets.autotag.hooks import TrackMatch as BeetsTrackMatch + +from beets_flask.database.mapper.base import Context +from beets_flask.database.mapper.match import ( + AlbumInfoMapper, + AlbumMatchMapper, + DistanceMapper, + MatchMapper, + TrackInfoMapper, + TrackMatchMapper, +) +from beets_flask.database.models.match import TrackInfo +from tests.conftest import beets_lib_item + + +class TestTrackInfoMapper: + """Tests that we can probably serialize and deserialize + beets TrackInfo objs. + """ + + def test_roundtrip_conversion(self): + """Test that we can convert BeetsTrackInfo to TrackInfo and back.""" + from beets.autotag.hooks import TrackInfo as BeetsTrackInfo + + original = BeetsTrackInfo( + title="Test Track", + artist="Test Artist", + album="Test Album", + length=180.0, + index=1, + ) + + mapper = TrackInfoMapper() + ctx = Context() + + # Test from_beets + model = mapper.from_beets(original, ctx) + assert isinstance(model, TrackInfo) + assert model.data["title"] == "Test Track" + assert model.data["artist"] == "Test Artist" + assert model.data["album"] == "Test Album" + assert model.data["length"] == 180.0 + assert model.data["index"] == 1 + + # Test to_beets + result = mapper.to_beets(model, ctx) + assert result.title == original.title + assert result.artist == original.artist + assert result.album == original.album + assert result.length == original.length + assert result.index == original.index + assert result.genre == original.genre + + +class TestAlbumInfoMatcher: + """Tests that we can probably serialize and deserialize + beets AlbumInfo objs. + """ + + def test_roundtrip_conversion(self): + """Test converting model AlbumInfo to BeetsAlbumInfo.""" + from beets.autotag.hooks import AlbumInfo as BeetsAlbumInfo + from beets.autotag.hooks import TrackInfo as BeetsTrackInfo + + original = BeetsAlbumInfo( + tracks=[ + BeetsTrackInfo(title="a"), + BeetsTrackInfo(title="b"), + ], + year=1, + ) + + mapper = AlbumInfoMapper() + ctx = Context() + + # Test from_beets + model = mapper.from_beets(original, ctx) + assert model.data["year"] == 1 + assert len(model.tracks) == 2 + assert model.tracks[0].data["title"] == "a" + assert model.tracks[1].data["title"] == "b" + + # Test to_beets + result = mapper.to_beets(model, ctx) + assert result.year == original.year + assert len(result.tracks) == len(original.tracks) + assert result.tracks[0].title == original.tracks[0].title + assert result.tracks[1].title == original.tracks[1].title + + +class TestDistanceMapper: + """Tests that we can probably serialize and deserialize + beets Distance objs. + """ + + def test_roundtrip_conversion(self): + """Test converting model Distance to BeetsDistance.""" + + from beets.autotag.distance import Distance as BeetsDistance + + original = BeetsDistance() + original.add("artist", 0.1) + original.add("album", 0.2) + + mapper = DistanceMapper() + ctx = Context() + + # Test from_beets + model = mapper.from_beets(original, ctx) + assert model.max_distance == original.max_distance + assert model.raw_distance == original.raw_distance + + # Test to_beets + result = mapper.to_beets(model, ctx) + assert result.distance == original.distance + assert result.max_distance == original.max_distance + assert result.raw_distance == original.raw_distance + assert result._penalties == original._penalties + + +def create_beets_album_match( + album_id="abc123", + album_name="Test Album", + album_artist="Test Artist", + album_track_count=2, + album_url="https://example.com/album", + album_image_path="/path/to/image.jpg", + album_disambig="", + tracks=None, + distance_penalties=None, + track_distances=None, + mapping=None, + extra_items=None, + extra_tracks=None, +): + """Factory function to generate beets AlbumMatch objects for testing purposes. + + Args: + album_id: The album ID (default: "abc123") + album_name: The album name (default: "Test Album") + album_artist: The album artist (default: "Test Artist") + album_track_count: Number of tracks to generate (default: 2) + album_url: Album URL (default: "https://example.com/album") + album_image_path: Album cover path (default: "/path/to/image.jpg") + album_disambig: Album disambiguation (default: "") + tracks: Custom list of TrackInfo objects. If None, generates from album_track_count. + distance_penalties: Dict of {key: value} penalties. Defaults to {"artist": 0.1, "album": 0.2} + track_distances: Dict of {TrackInfo: Dict of {key: value}} for track-level penalties. + e.g., {track1: {"track_title": 0.05}, track2: {"track_title": 0.0}} + mapping: Dict of {Item: TrackInfo} mappings. If None, generates from album_track_count. + extra_items: List of extra Item objects. If None, generates from album_track_count. + extra_tracks: List of extra TrackInfo objects. If None, generates from album_track_count. + + Returns: + beets.autotag.hooks.AlbumMatch: A test AlbumMatch object + """ + + # Default distance penalties + if distance_penalties is None: + distance_penalties = {"artist": 0.1, "album": 0.2} + + # Generate tracks if not provided + if tracks is None: + tracks = [] + for i in range(album_track_count): + track = BeetsTrackInfo( + title=f"Test Track {i + 1}", + artist=album_artist, + length=180.0 + i * 20, + index=i + 1, + ) + tracks.append(track) + + # Create AlbumInfo with the tracks + album_info = BeetsAlbumInfo( + album=album_name, + artist=album_artist, + tracks=tracks, + album_id=album_id, + album_url=album_url, + album_image_path=album_image_path, + album_disambig=album_disambig, + ) + + # Create Distance with penalties + distance = Distance() + for key, value in distance_penalties.items(): + distance.add(key, value) + + # Add track-level distances + if track_distances is not None: + for track, penalties in track_distances.items(): + track_distance = Distance() + for key, value in penalties.items(): + track_distance.add(key, value) + distance.tracks[track] = track_distance + + # Generate mapping if not provided + if mapping is None: + mapping = {} + for i in range(album_track_count): + item = beets_lib_item(title=f"mapping-{i}") + info = BeetsTrackInfo(title=f"mapping-{i}") + mapping[item] = info + + # Generate extra_tracks if not provided + if extra_tracks is None: + extra_tracks = [] + for i in range(album_track_count): + extra_tracks.append(BeetsTrackInfo(title=f"extra-{i}")) + + # Generate extra_items if not provided + if extra_items is None: + extra_items = [] + for i in range(album_track_count): + extra_items.append(beets_lib_item(title=f"extra-item-{i}")) + + return BeetsAlbumMatch( + distance=distance, + info=album_info, + mapping=mapping, + extra_tracks=extra_tracks, + extra_items=extra_items, + ) + + +class TestAlbumMatchMapper: + def test_roundtrip_conversion(self): + """Test converting model TrackMatch to BeetsTrackMatch.""" + + beets_track1 = BeetsTrackInfo(title="Test Track 1") + beets_track2 = BeetsTrackInfo(title="Test Track 2") + + # Create some extra items using the test fixture + extra_item1 = beets_lib_item(title="extra-item-1") + extra_item2 = beets_lib_item(title="extra-item-2") + + beets_album_match = create_beets_album_match( + album_id="abc123", + tracks=[beets_track1, beets_track2], + distance_penalties={"artist": 0.1, "album": 0.2}, + track_distances={ + beets_track1: {"track_title": 0.05}, + beets_track2: {"track_title": 0.0}, + }, + # We reuse objs here. Is a bit unrealistic + # but fully tests our capabilites + extra_tracks=[beets_track1], + extra_items=[extra_item1, extra_item2], + mapping={extra_item1: beets_track1}, + ) + + mapper = AlbumMatchMapper() + ctx = Context() + + # Test from_beets conversion + model = mapper.from_beets(beets_album_match, ctx) + assert model.info.data["album_id"] == "abc123" + assert model.info.data["album"] == "Test Album" + assert model.info.data["artist"] == "Test Artist" + assert len(model.info.tracks) == 2 + assert model.info.tracks[0].data["title"] == "Test Track 1" + assert model.info.tracks[1].data["title"] == "Test Track 2" + assert model.distance.raw_distance == beets_album_match.distance.raw_distance + penalty_keys = {p.key for p in model.distance.penalties} + assert penalty_keys == {"artist", "album"} + assert len(model.distance.track_distances) == 2 + assert ( + len(model.track_mappings) == 4 # 1 mapping + 1 extra_track + 2 extra_items + ) + + # Test to_beets conversion + result = mapper.to_beets(model, ctx) + assert result.info.album_id == "abc123" + assert result.info.album == "Test Album" + assert result.info.artist == "Test Artist" + assert len(result.info.tracks) == 2 + assert result.info.tracks[0].title == "Test Track 1" + assert result.info.tracks[1].title == "Test Track 2" + assert result.distance.raw_distance == beets_album_match.distance.raw_distance + assert len(result.mapping) == 1 + # Check dedbped worked as expected + assert result.extra_items[0] in result.mapping.keys() + assert result.mapping[result.extra_items[0]].title == beets_track1.title + assert len(result.extra_items) == 2 + assert len(result.extra_tracks) == 1 + + +class TestTrackMatchMapper: + def test_roundtrip_conversion(self): + """Test converting model TrackMatch to BeetsTrackMatch.""" + + track_distance = BeetsDistance() + track_distance.add("artist", 0.1) + track_distance.add("album", 0.2) + beets_track1 = BeetsTrackInfo( + title="Test Track 1", + artist="Test Artist", + length=180.0, + index=1, + ) + original = BeetsTrackMatch(distance=track_distance, info=beets_track1) + + mapper = TrackMatchMapper() + ctx = Context() + + # Test from_beets + model = mapper.from_beets(original, ctx) + assert isinstance(model.info, TrackInfo) + assert model.info.data["title"] == "Test Track 1" + assert model.info.data["artist"] == "Test Artist" + assert model.info.data["length"] == 180.0 + assert model.distance.raw_distance == track_distance.raw_distance + assert len(model.distance.penalties) == 2 + + # Test to_beets + result = mapper.to_beets(model, ctx) + assert isinstance(result, BeetsTrackMatch) + assert result.info.title == beets_track1.title + assert result.info.artist == beets_track1.artist + assert result.info.length == beets_track1.length + assert result.distance.raw_distance == original.distance.raw_distance + + # Verify penalties are preserved + penalty_keys = {p.key for p in model.distance.penalties} + assert penalty_keys == {"artist", "album"} + + def test_roundtrip_album_match(self): + """Test roundtrip conversion for AlbumMatch.""" + beets_track1 = BeetsTrackInfo(title="Test Track 1") + beets_album_match = create_beets_album_match( + album_id="abc123", + album_name="Test Album", + tracks=[beets_track1], + distance_penalties={"artist": 0.1}, + ) + + mapper = MatchMapper() + ctx = Context() + + model = mapper.from_beets(beets_album_match, ctx) + result = mapper.to_beets(model, ctx) + + assert isinstance(result, BeetsAlbumMatch) + assert result.info.album_id == "abc123" + assert result.info.album == "Test Album" + + def test_roundtrip_track_match(self): + """Test roundtrip conversion for TrackMatch.""" + from beets.autotag.distance import Distance as BeetsDistance + + track_distance = BeetsDistance() + track_distance.add("artist", 0.1) + + beets_track = BeetsTrackInfo(title="Test Track") + beets_track_match = BeetsTrackMatch(distance=track_distance, info=beets_track) + + mapper = MatchMapper() + ctx = Context() + + model = mapper.from_beets(beets_track_match, ctx) + result = mapper.to_beets(model, ctx) + + assert isinstance(result, BeetsTrackMatch) + assert result.info.title == "Test Track" + assert result.distance.raw_distance == track_distance.raw_distance diff --git a/backend/tests/unit/test_importer/conftest.py b/backend/tests/unit/test_importer/conftest.py index 5e284a6c..dcc8e28b 100644 --- a/backend/tests/unit/test_importer/conftest.py +++ b/backend/tests/unit/test_importer/conftest.py @@ -66,72 +66,3 @@ def valid_data_for_album_path(path: str | Path) -> dict: } else: raise NotImplementedError(f"Unknown test album path {p=}") - - -# ----------------- Monkeypath beets to use cached responses ----------------- # - -import hashlib -import pickle - -from beets import autotag -from beets.autotag import tag_album as _tag_album - -album_path: str - - -def use_mock_tag_album(a_dir: str): - """Use a cached lookup for the tag_album function in beets - this allows to not make requests to the internet when testing - the importer. - """ - global album_path - album_path = a_dir - - autotag.tag_album = tag_album - - -def tag_album( - items, - search_artist: str | None = None, - search_album: str | None = None, - search_ids: list[str] = [], -): - global album_path - log.debug(f"Using monkey patched lookup {album_path=}") - - # Compute items hash based on the items - - m = hashlib.md5() - for item in items: - m.update(item.path) - if search_artist: - m.update(search_artist.encode("utf-8")) - if search_album: - m.update(search_album.encode("utf-8")) - for search_id in search_ids: - m.update(search_id.encode("utf-8")) - items_hash = m.hexdigest()[:8] - - if (Path(album_path) / f"lookup_{items_hash}.pickle").exists(): - log.debug(f"Using cached lookup {album_path=}") - with open(Path(album_path) / f"lookup_{items_hash}.pickle", "rb") as f: - return pickle.load(f) - - else: - # TODO: This pickle contains absolute paths to the files - # while undesired (no use in having them in the git repo) its for now the - # easiest way... and we hope music brainz does not change its data too often! - log.debug(f"Using default lookup {album_path=}") - res = _tag_album(items, search_artist, search_album, search_ids) - - outdir = Path(album_path) - if not outdir.is_dir(): - outdir = outdir.parent - - with open(outdir / f"lookup_{items_hash}.pickle", "wb") as f: - pickle.dump(res, f) - - return res - - -autotag.tag_album = tag_album diff --git a/backend/tests/unit/test_importer/test_session.py b/backend/tests/unit/test_importer/test_session.py index dba94180..b37029fe 100644 --- a/backend/tests/unit/test_importer/test_session.py +++ b/backend/tests/unit/test_importer/test_session.py @@ -10,7 +10,6 @@ from .conftest import ( VALID_PATHS, album_path_absolute, - use_mock_tag_album, ) log = logging.getLogger(__name__) @@ -26,7 +25,6 @@ def test_generate_lookup(): """ for path in VALID_PATHS: p = Path(__file__).parent.parent.parent / "data" / "audio" / path - use_mock_tag_album(str(p)) state = SessionState(p) session = PreviewSession(state) @@ -48,7 +46,6 @@ class TestPreviewSessions: def get_state(self, path: str): p = album_path_absolute(path) self.session = PreviewSession(SessionState(p)) - use_mock_tag_album(str(p)) return self.session.run_sync() @pytest.mark.parametrize("path", VALID_PATHS) diff --git a/backend/uv.lock b/backend/uv.lock index 3e96ee5c..270668a8 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -175,7 +175,7 @@ wheels = [ [[package]] name = "beets" -version = "2.5.1" +version = "2.6.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, @@ -183,16 +183,19 @@ dependencies = [ { name = "jellyfish" }, { name = "lap" }, { name = "mediafile" }, - { name = "musicbrainzngs" }, + { name = "numba" }, { name = "numpy" }, + { name = "packaging" }, { name = "platformdirs" }, { name = "pyyaml" }, + { name = "requests-ratelimiter" }, + { name = "scipy" }, { name = "typing-extensions" }, { name = "unidecode" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c9/32/2b5ae0038c442e783b4f00b4145b15357a2f2358fd985c60a1f890751bb0/beets-2.5.1.tar.gz", hash = "sha256:7feefd70804fbcf26516089f472bac34c6a77e8e20ec539252fd1bafc91de9a2", size = 2147257, upload-time = "2025-10-14T22:52:55.631Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/f3/f429c70cdf8de80367be64515fe432b1cf44c60a9774cc75ae0aff36e48b/beets-2.6.1.tar.gz", hash = "sha256:1376b992ee18ee1b5de08d3586625e78d4338edb6b47e24f8e74932d8551f672", size = 2175882, upload-time = "2026-02-02T02:30:36.544Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/62/26/c459ae5217a69d1a2c83ddb80b61480764e990049b7d9f6a5b82660457f4/beets-2.5.1-py3-none-any.whl", hash = "sha256:3e58f33d898d007e6bfd385bd145d2c39325ef6b6f831f7269d037bbcb542bf7", size = 573677, upload-time = "2025-10-14T22:52:53.728Z" }, + { url = "https://files.pythonhosted.org/packages/d2/a8/fbdd2eeaea7add3103d52e8c84490ed7db383db5d5227aa8583e2abe284e/beets-2.6.1-py3-none-any.whl", hash = "sha256:d3654fc7fef2c26d35113d40346d873c073ca981e0db1f21e4575ca1860ede02", size = 604352, upload-time = "2026-02-02T02:30:34.303Z" }, ] [[package]] @@ -228,28 +231,26 @@ dependencies = [ { name = "watchdog" }, ] -[package.optional-dependencies] -all = [ +[package.dev-dependencies] +dev = [ { name = "fakeredis" }, + { name = "furo" }, { name = "mypy" }, + { name = "myst-nb" }, + { name = "myst-parser" }, { name = "pandas-stubs" }, + { name = "paracelsus" }, { name = "pre-commit" }, { name = "pytest" }, { name = "pytest-asyncio" }, { name = "pytest-benchmark" }, { name = "pytest-cov" }, { name = "ruff" }, - { name = "types-aiofiles" }, - { name = "types-cachetools" }, - { name = "types-deprecated" }, - { name = "types-pyyaml" }, - { name = "types-requests" }, -] -dev = [ - { name = "mypy" }, - { name = "pandas-stubs" }, - { name = "pre-commit" }, - { name = "ruff" }, + { name = "sphinx" }, + { name = "sphinx-copybutton" }, + { name = "sphinx-inline-tabs" }, + { name = "sphinxcontrib-mermaid" }, + { name = "sphinxcontrib-typer", extra = ["html"] }, { name = "types-aiofiles" }, { name = "types-cachetools" }, { name = "types-deprecated" }, @@ -260,9 +261,11 @@ docs = [ { name = "furo" }, { name = "myst-nb" }, { name = "myst-parser" }, + { name = "paracelsus" }, { name = "sphinx" }, { name = "sphinx-copybutton" }, { name = "sphinx-inline-tabs" }, + { name = "sphinxcontrib-mermaid" }, { name = "sphinxcontrib-typer", extra = ["html"] }, ] test = [ @@ -287,54 +290,84 @@ requires-dist = [ { name = "aiofiles" }, { name = "aiohttp" }, { name = "alembic", specifier = ">=1.18.4" }, - { name = "beets", specifier = "==2.5.1" }, - { name = "beets-flask", extras = ["dev", "test", "typed"], marker = "extra == 'all'" }, - { name = "beets-flask", extras = ["typed"], marker = "extra == 'dev'" }, + { name = "beets", specifier = "==2.6.1" }, { name = "cachetools", specifier = ">=5.3.3" }, { name = "confuse", specifier = ">=2.0.1" }, { name = "deprecated", specifier = ">=1.2.18" }, { name = "eyconf", specifier = ">=0.5.0" }, - { name = "fakeredis", marker = "extra == 'test'" }, - { name = "furo", marker = "extra == 'docs'", specifier = ">=2024.8.6" }, { name = "libtmux", specifier = ">=0.37.0" }, - { name = "mypy", marker = "extra == 'typed'", specifier = ">=1.14.1" }, - { name = "myst-nb", marker = "extra == 'docs'", specifier = ">=1.1.2" }, - { name = "myst-parser", marker = "extra == 'docs'", specifier = ">=4.0.0" }, { name = "natsort" }, { name = "nest-asyncio", specifier = ">=1.6.0" }, { name = "numpy" }, - { name = "pandas-stubs", marker = "extra == 'typed'" }, { name = "pillow", specifier = ">=10.4.0" }, { name = "polars", specifier = ">=1.36.1" }, - { name = "pre-commit", marker = "extra == 'dev'", specifier = ">=3.8.0" }, { name = "pydub" }, { name = "pylast", specifier = ">=5.2.0" }, - { name = "pytest", marker = "extra == 'test'", specifier = ">=8.2.2" }, - { name = "pytest-asyncio", marker = "extra == 'test'", specifier = ">=0.23.8" }, - { name = "pytest-benchmark", marker = "extra == 'test'", specifier = ">=5.2.3" }, - { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=5.0.0" }, { name = "python-socketio", specifier = ">=5.11.4" }, { name = "python2ts", specifier = ">=0.6.1" }, { name = "quart", specifier = ">=0.20.0" }, { name = "requests", specifier = ">=2.32.3" }, { name = "rq", specifier = ">=2.0.0" }, - { name = "ruff", marker = "extra == 'dev'", specifier = ">=0.6.5" }, - { name = "sphinx", marker = "extra == 'docs'", specifier = ">=8.0.2" }, - { name = "sphinx-copybutton", marker = "extra == 'docs'", specifier = ">=0.5.2" }, - { name = "sphinx-inline-tabs", marker = "extra == 'docs'", specifier = ">=2023.4.21" }, - { name = "sphinxcontrib-typer", extras = ["html"], marker = "extra == 'docs'", specifier = ">=0.5.0" }, { name = "sqlalchemy", specifier = ">=2.0.35" }, { name = "tinytag" }, - { name = "types-aiofiles", marker = "extra == 'typed'" }, - { name = "types-cachetools", marker = "extra == 'typed'" }, - { name = "types-deprecated", marker = "extra == 'typed'" }, - { name = "types-pyyaml", marker = "extra == 'typed'" }, - { name = "types-requests", marker = "extra == 'typed'" }, { name = "typing-extensions" }, { name = "uvicorn", specifier = ">=0.36.0" }, { name = "watchdog", specifier = ">=5.0.3" }, ] -provides-extras = ["test", "dev", "typed", "all", "docs"] + +[package.metadata.requires-dev] +dev = [ + { name = "fakeredis" }, + { name = "furo", specifier = ">=2024.8.6" }, + { name = "mypy", specifier = ">=1.14.1" }, + { name = "myst-nb", specifier = ">=1.1.2" }, + { name = "myst-parser", specifier = ">=4.0.0" }, + { name = "pandas-stubs" }, + { name = "paracelsus", specifier = ">=0.15.0" }, + { name = "pre-commit", specifier = ">=3.8.0" }, + { name = "pytest", specifier = ">=8.2.2" }, + { name = "pytest-asyncio", specifier = ">=0.23.8" }, + { name = "pytest-benchmark", specifier = ">=5.2.3" }, + { name = "pytest-cov", specifier = ">=5.0.0" }, + { name = "ruff", specifier = ">=0.6.5" }, + { name = "sphinx", specifier = ">=8.0.2" }, + { name = "sphinx-copybutton", specifier = ">=0.5.2" }, + { name = "sphinx-inline-tabs", specifier = ">=2023.4.21" }, + { name = "sphinxcontrib-mermaid", specifier = ">=2.0.1" }, + { name = "sphinxcontrib-typer", extras = ["html"], specifier = ">=0.5.0" }, + { name = "types-aiofiles" }, + { name = "types-cachetools" }, + { name = "types-deprecated" }, + { name = "types-pyyaml" }, + { name = "types-requests" }, +] +docs = [ + { name = "furo", specifier = ">=2024.8.6" }, + { name = "myst-nb", specifier = ">=1.1.2" }, + { name = "myst-parser", specifier = ">=4.0.0" }, + { name = "paracelsus", specifier = ">=0.15.0" }, + { name = "sphinx", specifier = ">=8.0.2" }, + { name = "sphinx-copybutton", specifier = ">=0.5.2" }, + { name = "sphinx-inline-tabs", specifier = ">=2023.4.21" }, + { name = "sphinxcontrib-mermaid", specifier = ">=2.0.1" }, + { name = "sphinxcontrib-typer", extras = ["html"], specifier = ">=0.5.0" }, +] +test = [ + { name = "fakeredis" }, + { name = "pytest", specifier = ">=8.2.2" }, + { name = "pytest-asyncio", specifier = ">=0.23.8" }, + { name = "pytest-benchmark", specifier = ">=5.2.3" }, + { name = "pytest-cov", specifier = ">=5.0.0" }, +] +typed = [ + { name = "mypy", specifier = ">=1.14.1" }, + { name = "pandas-stubs" }, + { name = "types-aiofiles" }, + { name = "types-cachetools" }, + { name = "types-deprecated" }, + { name = "types-pyyaml" }, + { name = "types-requests" }, +] [[package]] name = "bidict" @@ -1060,6 +1093,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8b/34/b11ab24abb78c73a1b82f6471c2d71bdd1bf2c8f30768ed2f26f1dddc083/libtmux-0.55.0-py3-none-any.whl", hash = "sha256:4b746533856e022c759e5c5cae97f4932e85dae316a2afd4391d6d0e891d6ab8", size = 80094, upload-time = "2026-03-08T00:57:54.141Z" }, ] +[[package]] +name = "llvmlite" +version = "0.47.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/88/a8952b6d5c21e74cbf158515b779666f692846502623e9e3c39d8e8ba25f/llvmlite-0.47.0.tar.gz", hash = "sha256:62031ce968ec74e95092184d4b0e857e444f8fdff0b8f9213707699570c33ccc", size = 193614, upload-time = "2026-03-31T18:29:53.497Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fa/48/4b7fe0e34c169fa2f12532916133e0b219d2823b540733651b34fdac509a/llvmlite-0.47.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:306a265f408c259067257a732c8e159284334018b4083a9e35f67d19792b164f", size = 37232769, upload-time = "2026-03-31T18:28:43.735Z" }, + { url = "https://files.pythonhosted.org/packages/e6/4b/e3f2cd17822cf772a4a51a0a8080b0032e6d37b2dbe8cfb724eac4e31c52/llvmlite-0.47.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:5853bf26160857c0c2573415ff4efe01c4c651e59e2c55c2a088740acfee51cd", size = 56275178, upload-time = "2026-03-31T18:28:48.342Z" }, + { url = "https://files.pythonhosted.org/packages/b6/55/a3b4a543185305a9bdf3d9759d53646ed96e55e7dfd43f53e7a421b8fbae/llvmlite-0.47.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:003bcf7fa579e14db59c1a1e113f93ab8a06b56a4be31c7f08264d1d4072d077", size = 55128632, upload-time = "2026-03-31T18:28:52.901Z" }, + { url = "https://files.pythonhosted.org/packages/2f/f5/d281ae0f79378a5a91f308ea9fdb9f9cc068fddd09629edc0725a5a8fde1/llvmlite-0.47.0-cp312-cp312-win_amd64.whl", hash = "sha256:f3079f25bdc24cd9d27c4b2b5e68f5f60c4fdb7e8ad5ee2b9b006007558f9df7", size = 38138692, upload-time = "2026-03-31T18:28:57.147Z" }, +] + [[package]] name = "mako" version = "1.3.10" @@ -1176,15 +1221,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/81/08/7036c080d7117f28a4af526d794aab6a84463126db031b007717c1a6676e/multidict-6.7.1-py3-none-any.whl", hash = "sha256:55d97cc6dae627efa6a6e548885712d4864b81110ac76fa4e534c03819fa4a56", size = 12319, upload-time = "2026-01-26T02:46:44.004Z" }, ] -[[package]] -name = "musicbrainzngs" -version = "0.7.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0a/67/3e74ae93d90ceeba72ed1a266dd3ca9abd625f315f0afd35f9b034acedd1/musicbrainzngs-0.7.1.tar.gz", hash = "sha256:ab1c0100fd0b305852e65f2ed4113c6de12e68afd55186987b8ed97e0f98e627", size = 117469, upload-time = "2020-01-11T17:38:47.581Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6d/fd/cef7b2580436910ccd2f8d3deec0f3c81743e15c0eb5b97dde3fbf33c0c8/musicbrainzngs-0.7.1-py2.py3-none-any.whl", hash = "sha256:e841a8f975104c0a72290b09f59326050194081a5ae62ee512f41915090e1a10", size = 25289, upload-time = "2020-01-11T17:38:45.469Z" }, -] - [[package]] name = "mutagen" version = "1.47.0" @@ -1319,6 +1355,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/88/b2/d0896bdcdc8d28a7fc5717c305f1a861c26e18c05047949fb371034d98bd/nodeenv-1.10.0-py2.py3-none-any.whl", hash = "sha256:5bb13e3eed2923615535339b3c620e76779af4cb4c6a90deccc9e36b274d3827", size = 23438, upload-time = "2025-12-20T14:08:52.782Z" }, ] +[[package]] +name = "numba" +version = "0.65.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "llvmlite" }, + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/61/7299643b9c18d669e04be7c5bcb64d985070d07553274817b45b049e7bfe/numba-0.65.0.tar.gz", hash = "sha256:edad0d9f6682e93624c00125a471ae4df186175d71fd604c983c377cdc03e68b", size = 2764131, upload-time = "2026-04-01T03:52:01.946Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/2f/8bd31a1ea43c01ac215283d83aa5f8d5acbe7a36c85b82f1757bfe9ccb31/numba-0.65.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:b27ee4847e1bfb17e9604d100417ee7c1d10f15a6711c6213404b3da13a0b2aa", size = 2680705, upload-time = "2026-04-01T03:51:32.597Z" }, + { url = "https://files.pythonhosted.org/packages/73/36/88406bd58600cc696417b8e5dd6a056478da808f3eaf48d18e2421e0c2d9/numba-0.65.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a52d92ffd297c10364bce60cd1fcb88f99284ab5df085f2c6bcd1cb33b529a6f", size = 3801411, upload-time = "2026-04-01T03:51:34.321Z" }, + { url = "https://files.pythonhosted.org/packages/0c/61/ce753a1d7646dd477e16d15e89473703faebb8995d2f71d7ad69a540b565/numba-0.65.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:da8e371e328c06d0010c3d8b44b21858652831b85bcfba78cb22c042e22dbd8e", size = 3501622, upload-time = "2026-04-01T03:51:36.348Z" }, + { url = "https://files.pythonhosted.org/packages/7d/86/db87a5393f1b1fabef53ac3ba4e6b938bb27e40a04ad7cc512098fcae032/numba-0.65.0-cp312-cp312-win_amd64.whl", hash = "sha256:59bb9f2bb9f1238dfd8e927ba50645c18ae769fef4f3d58ea0ea22a2683b91f5", size = 2749979, upload-time = "2026-04-01T03:51:37.88Z" }, +] + [[package]] name = "numpy" version = "2.4.3" @@ -1371,6 +1423,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7c/2f/f91e4eee21585ff548e83358332d5632ee49f6b2dcd96cb5dca4e0468951/pandas_stubs-3.0.0.260204-py3-none-any.whl", hash = "sha256:5ab9e4d55a6e2752e9720828564af40d48c4f709e6a2c69b743014a6fcb6c241", size = 168540, upload-time = "2026-02-04T15:17:15.615Z" }, ] +[[package]] +name = "paracelsus" +version = "0.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging" }, + { name = "pydot" }, + { name = "sqlalchemy" }, + { name = "typer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c1/cc/d545a19967c3bdeba92ca1d8a736576b96b4610154f3bd6dbf01a198e2c3/paracelsus-0.15.0.tar.gz", hash = "sha256:b850b56417eef7b5e301b09ba7d44655f3c76de8681699b93ef6ae410afeb278", size = 92053, upload-time = "2026-01-04T21:38:25.508Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/18/70/3fa8dad530ae181b0a30f9874bababaa3d3781f9ef6c87aeaeed79b3c954/paracelsus-0.15.0-py3-none-any.whl", hash = "sha256:0ed0f97fb5ec09e379e45c1a95e280b1c40ee42af3c77f59f03998477a73fde2", size = 19606, upload-time = "2026-01-04T21:38:24.284Z" }, +] + [[package]] name = "parso" version = "0.8.6" @@ -1579,6 +1646,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, ] +[[package]] +name = "pydot" +version = "4.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyparsing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/50/35/b17cb89ff865484c6a20ef46bf9d95a5f07328292578de0b295f4a6beec2/pydot-4.0.1.tar.gz", hash = "sha256:c2148f681c4a33e08bf0e26a9e5f8e4099a82e0e2a068098f32ce86577364ad5", size = 162594, upload-time = "2025-06-17T20:09:56.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/32/a7125fb28c4261a627f999d5fb4afff25b523800faed2c30979949d6facd/pydot-4.0.1-py3-none-any.whl", hash = "sha256:869c0efadd2708c0be1f916eb669f3d664ca684bc57ffb7ecc08e70d5e93fee6", size = 37087, upload-time = "2025-06-17T20:09:55.25Z" }, +] + [[package]] name = "pydub" version = "0.25.1" @@ -1609,6 +1688,24 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/07/94/677dade2b8ed48631de3fd34b320ebfd59b7a66f831640831112d7a7190b/pylast-7.0.2-py3-none-any.whl", hash = "sha256:c995e078670b3a8e3116a31b17d1f0d89c4d020407f6967ee9ffab2aeecd9de7", size = 26773, upload-time = "2026-01-19T12:40:00.48Z" }, ] +[[package]] +name = "pyparsing" +version = "3.3.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f3/91/9c6ee907786a473bf81c5f53cf703ba0957b23ab84c264080fb5a450416f/pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc", size = 6851574, upload-time = "2026-01-21T03:57:59.36Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, +] + +[[package]] +name = "pyrate-limiter" +version = "4.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/0c/6e78218e6ef726be35a4c0a5e2e281e36ddd940566800219e96d13de99ad/pyrate_limiter-4.1.0.tar.gz", hash = "sha256:be1ac413a263aa410b98757d1b01a880650948a1fc3a959512f15865eb58dbf3", size = 306136, upload-time = "2026-03-22T14:43:03.739Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c7/fd/57181fafae08385d00ea2702be246ab8035352a0a8e1f63391c2bcad74d4/pyrate_limiter-4.1.0-py3-none-any.whl", hash = "sha256:2696b4e4a6cffb3d40fc76662baccb766697893f0979e12bebbfc7d3b6b19603", size = 38197, upload-time = "2026-03-22T14:43:01.975Z" }, +] + [[package]] name = "pysocks" version = "1.7.1" @@ -1842,6 +1939,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/db/4254e3eabe8020b458f1a747140d32277ec7a271daf1d235b70dc0b4e6e3/requests-2.32.5-py3-none-any.whl", hash = "sha256:2462f94637a34fd532264295e186976db0f5d453d1cdd31473c85a6a161affb6", size = 64738, upload-time = "2025-08-18T20:46:00.542Z" }, ] +[[package]] +name = "requests-ratelimiter" +version = "0.9.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pyrate-limiter" }, + { name = "requests" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d5/bf/02d3ddb3a47fc8889a87f96d17b73f4e2550c52b37e9891072fd9995e8f4/requests_ratelimiter-0.9.3.tar.gz", hash = "sha256:a706807210e0a5554be585f0cd6892cb57784343c4d40c7473456b26274a775c", size = 15810, upload-time = "2026-04-02T18:01:42.685Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3d/d1/96098646044b3ec25ab3c14f7a77ebd692dc0b8e305ad25e984b6c6460d2/requests_ratelimiter-0.9.3-py3-none-any.whl", hash = "sha256:f0d2c616891ba84d84aa9940c4251fd516bc6471d44313230653062eedc860d4", size = 11022, upload-time = "2026-04-02T18:01:41.603Z" }, +] + [[package]] name = "rich" version = "14.3.3" @@ -1926,6 +2036,27 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/d0/578c47dd68152ddddddf31cd7fc67dc30b7cdf639a86275fda821b0d9d98/ruff-0.15.6-py3-none-win_arm64.whl", hash = "sha256:c34de3dd0b0ba203be50ae70f5910b17188556630e2178fd7d79fc030eb0d837", size = 11060497, upload-time = "2026-03-12T23:05:25.968Z" }, ] +[[package]] +name = "scipy" +version = "1.17.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7a/97/5a3609c4f8d58b039179648e62dd220f89864f56f7357f5d4f45c29eb2cc/scipy-1.17.1.tar.gz", hash = "sha256:95d8e012d8cb8816c226aef832200b1d45109ed4464303e997c5b13122b297c0", size = 30573822, upload-time = "2026-02-23T00:26:24.851Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/48/b992b488d6f299dbe3f11a20b24d3dda3d46f1a635ede1c46b5b17a7b163/scipy-1.17.1-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:35c3a56d2ef83efc372eaec584314bd0ef2e2f0d2adb21c55e6ad5b344c0dcb8", size = 31610954, upload-time = "2026-02-23T00:17:49.855Z" }, + { url = "https://files.pythonhosted.org/packages/b2/02/cf107b01494c19dc100f1d0b7ac3cc08666e96ba2d64db7626066cee895e/scipy-1.17.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:fcb310ddb270a06114bb64bbe53c94926b943f5b7f0842194d585c65eb4edd76", size = 28172662, upload-time = "2026-02-23T00:18:01.64Z" }, + { url = "https://files.pythonhosted.org/packages/cf/a9/599c28631bad314d219cf9ffd40e985b24d603fc8a2f4ccc5ae8419a535b/scipy-1.17.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:cc90d2e9c7e5c7f1a482c9875007c095c3194b1cfedca3c2f3291cdc2bc7c086", size = 20344366, upload-time = "2026-02-23T00:18:12.015Z" }, + { url = "https://files.pythonhosted.org/packages/35/f5/906eda513271c8deb5af284e5ef0206d17a96239af79f9fa0aebfe0e36b4/scipy-1.17.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:c80be5ede8f3f8eded4eff73cc99a25c388ce98e555b17d31da05287015ffa5b", size = 22704017, upload-time = "2026-02-23T00:18:21.502Z" }, + { url = "https://files.pythonhosted.org/packages/da/34/16f10e3042d2f1d6b66e0428308ab52224b6a23049cb2f5c1756f713815f/scipy-1.17.1-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e19ebea31758fac5893a2ac360fedd00116cbb7628e650842a6691ba7ca28a21", size = 32927842, upload-time = "2026-02-23T00:18:35.367Z" }, + { url = "https://files.pythonhosted.org/packages/01/8e/1e35281b8ab6d5d72ebe9911edcdffa3f36b04ed9d51dec6dd140396e220/scipy-1.17.1-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:02ae3b274fde71c5e92ac4d54bc06c42d80e399fec704383dcd99b301df37458", size = 35235890, upload-time = "2026-02-23T00:18:49.188Z" }, + { url = "https://files.pythonhosted.org/packages/c5/5c/9d7f4c88bea6e0d5a4f1bc0506a53a00e9fcb198de372bfe4d3652cef482/scipy-1.17.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8a604bae87c6195d8b1045eddece0514d041604b14f2727bbc2b3020172045eb", size = 35003557, upload-time = "2026-02-23T00:18:54.74Z" }, + { url = "https://files.pythonhosted.org/packages/65/94/7698add8f276dbab7a9de9fb6b0e02fc13ee61d51c7c3f85ac28b65e1239/scipy-1.17.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:f590cd684941912d10becc07325a3eeb77886fe981415660d9265c4c418d0bea", size = 37625856, upload-time = "2026-02-23T00:19:00.307Z" }, + { url = "https://files.pythonhosted.org/packages/a2/84/dc08d77fbf3d87d3ee27f6a0c6dcce1de5829a64f2eae85a0ecc1f0daa73/scipy-1.17.1-cp312-cp312-win_amd64.whl", hash = "sha256:41b71f4a3a4cab9d366cd9065b288efc4d4f3c0b37a91a8e0947fb5bd7f31d87", size = 36549682, upload-time = "2026-02-23T00:19:07.67Z" }, + { url = "https://files.pythonhosted.org/packages/bc/98/fe9ae9ffb3b54b62559f52dedaebe204b408db8109a8c66fdd04869e6424/scipy-1.17.1-cp312-cp312-win_arm64.whl", hash = "sha256:f4115102802df98b2b0db3cce5cb9b92572633a1197c77b7553e5203f284a5b3", size = 24547340, upload-time = "2026-02-23T00:19:12.024Z" }, +] + [[package]] name = "selenium" version = "4.41.0" @@ -2109,6 +2240,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c2/42/4c8646762ee83602e3fb3fbe774c2fac12f317deb0b5dbeeedd2d3ba4b77/sphinxcontrib_jsmath-1.0.1-py2.py3-none-any.whl", hash = "sha256:2ec2eaebfb78f3f2078e73666b1415417a116cc848b72e5172e596c871103178", size = 5071, upload-time = "2019-01-21T16:10:14.333Z" }, ] +[[package]] +name = "sphinxcontrib-mermaid" +version = "2.0.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jinja2" }, + { name = "pyyaml" }, + { name = "sphinx" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2b/ae/999891de292919b66ea34f2c22fc22c9be90ab3536fbc0fca95716277351/sphinxcontrib_mermaid-2.0.1.tar.gz", hash = "sha256:a21a385a059a6cafd192aa3a586b14bf5c42721e229db67b459dc825d7f0a497", size = 19839, upload-time = "2026-03-05T14:10:41.901Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/46/25d64bcd7821c8d6f1080e1c43d5fcdfc442a18f759a230b5ccdc891093e/sphinxcontrib_mermaid-2.0.1-py3-none-any.whl", hash = "sha256:9dca7fbe827bad5e7e2b97c4047682cfd26e3e07398cfdc96c7a8842ae7f06e7", size = 14064, upload-time = "2026-03-05T14:10:40.533Z" }, +] + [[package]] name = "sphinxcontrib-qthelp" version = "2.0.0" diff --git a/docs/.readthedocs.yaml b/docs/.readthedocs.yaml index 155de2dc..061cae6b 100644 --- a/docs/.readthedocs.yaml +++ b/docs/.readthedocs.yaml @@ -7,28 +7,19 @@ version: 2 # Set the OS, Python version and other tools you might need build: - os: ubuntu-22.04 - tools: - python: "3.11" - # You can also specify other tool versions: - # nodejs: "19" - # rust: "1.64" - # golang: "1.19" + os: ubuntu-24.04 + tools: + python: "3.12" + jobs: + create_environment: + - asdf plugin add uv + - asdf install uv latest + - asdf global uv latest + - cd backend + - UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --all-extras --group docs + install: + - "true" # Build documentation in the "docs/" directory with Sphinx sphinx: - configuration: ./docs/conf.py -# Optionally build your docs in additional formats such as PDF and ePub -# formats: -# - pdf -# - epub - -# Optional but recommended, declare the Python requirements required -# to build your documentation -# See https://docs.readthedocs.io/en/stable/guides/reproducible-builds.html -python: - install: - - method: pip - path: ./backend - extra_requirements: - - docs + configuration: docs/conf.py diff --git a/docs/Makefile b/docs/Makefile index b432ee1a..377f0f27 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -8,6 +8,13 @@ SPHINXBUILD ?= sphinx-build SOURCEDIR = . BUILDDIR = build +DIAGRAMCMD = paracelsus graph beets_flask.database.models.base:Base \ + --config ../backend/pyproject.toml \ + --column-sort preserve-order + +# the grep breaks the charts, causing a slow down, but they still render and are cleaner +DIGRAMGREP = | grep -vE "created_at|updated_at" + # Put it first so that "make" without argument is like "make help". help: @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) @@ -16,10 +23,44 @@ help: # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). + %: Makefile @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) +.PHONY: diagrams + +# to do exact filtering we need to use e.g. --include-tables "^track_info$$" +diagrams: + @$(DIAGRAMCMD) \ + > ./diagrams/all.mmd + + @$(DIAGRAMCMD) \ + --include-tables "task" \ + --include-tables "folder" \ + --include-tables "session" \ + --include-tables "candidate" \ + --include-tables "^matches$$" \ + --include-tables "^items$$" \ + $(DIGRAMGREP) \ + > ./diagrams/high_level.mmd + + @$(DIAGRAMCMD) \ + --include-tables "candidate" \ + --include-tables "matches" \ + $(DIGRAMGREP) \ + > ./diagrams/matches_overview.mmd + + @$(DIAGRAMCMD) \ + --include-tables "matches_album" \ + --include-tables "matches_track" \ + --include-tables "distance" \ + --include-tables "penalties" \ + --include-tables "album" \ + --include-tables "track" \ + --include-tables "^items$$" \ + $(DIGRAMGREP) \ + > ./diagrams/matches_types.mmd clean: rm -rf $(BUILDDIR)/* - rm -rf $(SOURCEDIR)/_autosummary/* \ No newline at end of file + rm -rf $(SOURCEDIR)/_autosummary/* diff --git a/docs/conf.py b/docs/conf.py index 1d9fd499..9332d19f 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -30,6 +30,7 @@ "sphinx_inline_tabs", "sphinxcontrib.typer", "sphinx.ext.napoleon", + "sphinxcontrib.mermaid", # "myst_parser", "myst_nb", ] diff --git a/docs/develop/resources/backend.md b/docs/develop/resources/backend.md index 9a04d954..b58e730e 100644 --- a/docs/develop/resources/backend.md +++ b/docs/develop/resources/backend.md @@ -5,6 +5,8 @@ Beets-Flask provides a quart application with REST API for the beets music libra ```{toctree} :hidden: +./classes +./database ./state_serialize ``` @@ -23,53 +25,3 @@ BEETSDIR="/config/beets" BEETSFLASKDIR="/config/beets-flask" BEETSFLASKLOG="/logs/beets-flask.log" ``` - -## Database Migrations Guide - -We use [Alembic](https://alembic.sqlalchemy.org/) for database migrations. - -### Overview - -- Migrations are stored in `backend/alembic/versions/` -- The database tracks its current version in the `alembic_version` table -- Migration files define `upgrade()` and `downgrade()` functions - -### Quick Reference - -| Task | Command | -|------|---------| -| Create migration | `alembic revision --autogenerate -m "description"` | -| Apply all pending | `alembic upgrade head` | -| Roll back one | `alembic downgrade -1` | -| Check version | `alembic current` | -| See history | `alembic history` | -| Validate | `alembic check` | - -### Workflow: Creating a migration - -We use a local database (`./beets-flask-sqlite.db`) to avoid breaking the docker setup. - -1. Ensure you have a local database: - ```bash - cd backend - alembic upgrade head - ``` - -2. Edit the model (e.g., add a column to `states.py`) - -3. Generate the migration: - ```bash - alembic revision --autogenerate -m "add_column_name" - ``` - -4. Review the generated migration file in `alembic/versions/` - -5. Apply the migration: - ```bash - alembic upgrade head - ``` - -6. Validate with - ```bash - alembic current - ``` diff --git a/docs/develop/resources/classes.md b/docs/develop/resources/classes.md new file mode 100644 index 00000000..f8471247 --- /dev/null +++ b/docs/develop/resources/classes.md @@ -0,0 +1,64 @@ +# Class Overview + +To keep an overview which types come from beets native, we prefix them to `BeetsSomeType` (see `beets_flask/importer/types.py`) + +## Notation + +### Items + +In Beets, an `Item` is a single track. The `Item` can be stored +in the beets database. It represents a tracks metadata on disk. + +### Candidates (TrackInfo & AlbumInfo) + +Retrieved from external sources (e.g. spotify, tidal...). In particular `TrackInfo` is a single tracks metadata from an external source while `AlbumInfo` is information shared but also additional. `AlbumInfo` may contain a list of `TrackInfo`s. + +### Matches (TrackMatch & AlbumMatch) + +Matches are the association between `candidates` and `items`. Historically in beets this was just a list of indice mappings but changed to direct references to objects. + +For the tracks of a candidate we may find the following relationships after trying +to assign items and tracks. + +``` +items ∩ tracks = pairs +items' ∩ tracks = extra_items +items ∩ tracks' = extra_tracks +``` + +Matches are ranked through predefined penalties and using linear assignment problem. This yields a percentage score. + +### Task(s) + +A `Task` is a specific import operation. Tasks need to be started on a folder i.e. `items` and looks up `candidates` online. The goal of task is to assign `items` to `candidates` by finding `matches`. A user can than pick a match. + +## Sessions and Queues + +In Beets and BeetsFlask, folder imports are abstracted into sessions. +In BeetsFlask, each `Session` gets placed in a redis `Queue`, depending on its type: +Previews can take place in parallel, while imports take place one at a time, since this requires file movements on disk and writes into the beets database. + +```{eval-rst} +.. mermaid:: ../../diagrams/sessions.mmd +``` + + +## States + +We keep states of various objects in our own database, mostly to be able to resume imports after generating the initial preview. +This requires us to wrap a lot of the beets objects, to make them persistable. + +The state objects have a hierachy close to the beets internal logic: +- SessionState: Reflects the state of the import session. +- TaskState: Reflects an import task, but they dont have such a precise real-life meaning. +- CandidateState: Reflects a beets match (i.e. a candidate the user might choose) + +```{eval-rst} +.. mermaid:: ../../diagrams/objects_state_relation.mmd +``` + +## PR279 + +```{eval-rst} +.. mermaid:: ../../diagrams/pr279.mmd +``` diff --git a/docs/develop/resources/database.md b/docs/develop/resources/database.md new file mode 100644 index 00000000..e7efdec1 --- /dev/null +++ b/docs/develop/resources/database.md @@ -0,0 +1,79 @@ +# Database + +## Migrations Guide + +We use [Alembic](https://alembic.sqlalchemy.org/) for database migrations. + +### Overview + +- Migrations are stored in `backend/alembic/versions/` +- The database tracks its current version in the `alembic_version` table +- Migration files define `upgrade()` and `downgrade()` functions + +### Quick Reference + +| Task | Command | +|------|---------| +| Create migration | `alembic revision --autogenerate -m "description"` | +| Apply all pending | `alembic upgrade head` | +| Roll back one | `alembic downgrade -1` | +| Check version | `alembic current` | +| See history | `alembic history` | +| Validate | `alembic check` | + +### Workflow: Creating a migration + +We use a local database (`./beets-flask-sqlite.db`) to avoid breaking the docker setup. + +1. Ensure you have a local database: + ```bash + cd backend + alembic upgrade head + ``` + +2. Edit the model (e.g., add a column to `states.py`) + +3. Generate the migration: + ```bash + alembic revision --autogenerate -m "add_column_name" + ``` + +4. Review the generated migration file in `alembic/versions/` + +5. Apply the migration: + ```bash + alembic upgrade head + ``` + +6. Validate with + ```bash + alembic current + ``` + + +## Schema + +Autogenerated database schemas. + +### Overview + +```{eval-rst} +.. mermaid:: ../../diagrams/high_level.mmd +``` +### Matches + +```{eval-rst} +.. mermaid:: ../../diagrams/matches_overview.mmd +``` + +### Matches Types + +```{eval-rst} +.. mermaid:: ../../diagrams/matches_types.mmd +``` + +### Complete + +```{eval-rst} +.. mermaid:: ../../diagrams/all.mmd +``` diff --git a/docs/develop/resources/documentation.md b/docs/develop/resources/documentation.md index 23012e06..67973eb2 100644 --- a/docs/develop/resources/documentation.md +++ b/docs/develop/resources/documentation.md @@ -8,6 +8,10 @@ You may build the documentation locally with. # Install the requirements cd backend pip install -e .[docs] + +# Optionally, create ER-Diargrams +make diagrams + # Build the documentation cd ../docs make html diff --git a/docs/diagrams/all.mmd b/docs/diagrams/all.mmd new file mode 100644 index 00000000..d9719d02 --- /dev/null +++ b/docs/diagrams/all.mmd @@ -0,0 +1,130 @@ +erDiagram + album_info { + JSON data + VARCHAR id PK + DATETIME created_at "indexed" + DATETIME updated_at + } + + album_match_track_mappings { + VARCHAR album_match_id FK + VARCHAR track_info_id FK "nullable" + JSON item "nullable" + VARCHAR id PK + DATETIME created_at "indexed" + DATETIME updated_at + } + + candidate { + VARCHAR task_id FK + VARCHAR match_id FK + VARCHAR duplicate_ids + TEXT mapping + VARCHAR id PK + DATETIME created_at "indexed" + DATETIME updated_at + } + + distances { + VARCHAR track_info_id FK "nullable" + VARCHAR parent_distance_id FK "nullable" + FLOAT raw_distance + FLOAT max_distance + VARCHAR id PK + DATETIME created_at "indexed" + DATETIME updated_at + } + + folder { + VARCHAR full_path PK "indexed" + BOOLEAN is_album "nullable" + VARCHAR id PK + DATETIME created_at "indexed" + DATETIME updated_at + } + + matches { + VARCHAR id PK + VARCHAR type + VARCHAR distance_id FK + DATETIME created_at "indexed" + DATETIME updated_at + } + + matches_album { + VARCHAR id PK,FK + VARCHAR info_id FK + } + + matches_track { + VARCHAR id PK,FK + VARCHAR info_id FK + } + + penalties { + VARCHAR key "indexed" + BLOB value + VARCHAR distance_id FK + VARCHAR id PK + DATETIME created_at "indexed" + DATETIME updated_at + } + + session { + VARCHAR folder_hash FK + INTEGER folder_revision + ENUM progress + BLOB exc "nullable" + VARCHAR id PK + DATETIME created_at "indexed" + DATETIME updated_at + } + + task { + VARCHAR session_id FK + VARCHAR chosen_candidate_id FK "nullable" + BLOB toppath "nullable" + BLOB paths + BLOB old_paths "nullable" + ENUM choice_flag "nullable" + VARCHAR cur_artist "nullable" + VARCHAR cur_album "nullable" + ENUM progress + VARCHAR id PK + DATETIME created_at "indexed" + DATETIME updated_at + } + + task_pending_items { + VARCHAR task_id FK + JSON item + VARCHAR id PK + DATETIME created_at "indexed" + DATETIME updated_at + } + + track_info { + VARCHAR album_id FK "nullable" + JSON data + VARCHAR id PK + DATETIME created_at "indexed" + DATETIME updated_at + } + + matches_album ||--o{ album_match_track_mappings : album_match_id + track_info ||--o{ album_match_track_mappings : track_info_id + task ||--o{ candidate : task_id + matches ||--o{ candidate : match_id + track_info ||--o{ distances : track_info_id + distances ||--o{ distances : parent_distance_id + distances ||--o{ matches : distance_id + matches ||--o| matches_album : id + album_info ||--o{ matches_album : info_id + matches ||--o| matches_track : id + track_info ||--o{ matches_track : info_id + distances ||--o{ penalties : distance_id + folder ||--o{ session : folder_hash + session ||--o{ task : session_id + candidate ||--o{ task : chosen_candidate_id + task ||--o{ task_pending_items : task_id + album_info ||--o{ track_info : album_id diff --git a/docs/diagrams/high_level.mmd b/docs/diagrams/high_level.mmd new file mode 100644 index 00000000..1315b61f --- /dev/null +++ b/docs/diagrams/high_level.mmd @@ -0,0 +1,54 @@ +erDiagram + candidate { + VARCHAR task_id FK + VARCHAR match_id FK + VARCHAR duplicate_ids + TEXT mapping + VARCHAR id PK + } + + folder { + VARCHAR full_path PK "indexed" + BOOLEAN is_album "nullable" + VARCHAR id PK + } + + matches { + VARCHAR id PK + VARCHAR type + VARCHAR distance_id FK + } + + session { + VARCHAR folder_hash FK + INTEGER folder_revision + ENUM progress + BLOB exc "nullable" + VARCHAR id PK + } + + task { + VARCHAR session_id FK + VARCHAR chosen_candidate_id FK "nullable" + BLOB toppath "nullable" + BLOB paths + BLOB old_paths "nullable" + ENUM choice_flag "nullable" + VARCHAR cur_artist "nullable" + VARCHAR cur_album "nullable" + ENUM progress + VARCHAR id PK + } + + task_pending_items { + VARCHAR task_id FK + JSON item + VARCHAR id PK + } + + task ||--o{ candidate : task_id + matches ||--o{ candidate : match_id + folder ||--o{ session : folder_hash + session ||--o{ task : session_id + candidate ||--o{ task : chosen_candidate_id + task ||--o{ task_pending_items : task_id diff --git a/docs/diagrams/matches_overview.mmd b/docs/diagrams/matches_overview.mmd new file mode 100644 index 00000000..6748c23e --- /dev/null +++ b/docs/diagrams/matches_overview.mmd @@ -0,0 +1,28 @@ +erDiagram + candidate { + VARCHAR task_id FK + VARCHAR match_id FK + VARCHAR duplicate_ids + TEXT mapping + VARCHAR id PK + } + + matches { + VARCHAR id PK + VARCHAR type + VARCHAR distance_id FK + } + + matches_album { + VARCHAR id PK,FK + VARCHAR info_id FK + } + + matches_track { + VARCHAR id PK,FK + VARCHAR info_id FK + } + + matches ||--o{ candidate : match_id + matches ||--o| matches_album : id + matches ||--o| matches_track : id diff --git a/docs/diagrams/matches_types.mmd b/docs/diagrams/matches_types.mmd new file mode 100644 index 00000000..5dcb4eb6 --- /dev/null +++ b/docs/diagrams/matches_types.mmd @@ -0,0 +1,52 @@ +erDiagram + album_info { + JSON data + VARCHAR id PK + } + + album_match_track_mappings { + VARCHAR album_match_id FK + VARCHAR track_info_id FK "nullable" + JSON item "nullable" + VARCHAR id PK + } + + distances { + VARCHAR track_info_id FK "nullable" + VARCHAR parent_distance_id FK "nullable" + FLOAT raw_distance + FLOAT max_distance + VARCHAR id PK + } + + matches_album { + VARCHAR id PK,FK + VARCHAR info_id FK + } + + matches_track { + VARCHAR id PK,FK + VARCHAR info_id FK + } + + penalties { + VARCHAR key "indexed" + BLOB value + VARCHAR distance_id FK + VARCHAR id PK + } + + track_info { + VARCHAR album_id FK "nullable" + JSON data + VARCHAR id PK + } + + matches_album ||--o{ album_match_track_mappings : album_match_id + track_info ||--o{ album_match_track_mappings : track_info_id + track_info ||--o{ distances : track_info_id + distances ||--o{ distances : parent_distance_id + album_info ||--o{ matches_album : info_id + track_info ||--o{ matches_track : info_id + distances ||--o{ penalties : distance_id + album_info ||--o{ track_info : album_id diff --git a/docs/diagrams/objects_state_relation.mmd b/docs/diagrams/objects_state_relation.mmd new file mode 100644 index 00000000..47bab45d --- /dev/null +++ b/docs/diagrams/objects_state_relation.mmd @@ -0,0 +1,29 @@ +flowchart LR + subgraph Beets + direction TB + LS[BeetsImportSession] + LT[BeetsImportTask] + LC["BeetsAlbumMatch | BeetsTrackMatch"] + end + + + subgraph BeetsFlask + direction TB + LF["Folder | Archive"] + SS[SessionState] + ST[TaskState] + SC[CandidateState] + end + + subgraph BeetsFlask Database + direction TB + DF[(FolderInDb)] + DS[(SessionStateInDb)] + DT[(TaskStateInDb)] + DC[(CandidateStateInDb)] + end + + LF --> DF + LS --> SS --> DS + LT --> ST --> DT + LC --> SC --> DC diff --git a/docs/diagrams/pr279.mmd b/docs/diagrams/pr279.mmd new file mode 100644 index 00000000..74ba2aaf --- /dev/null +++ b/docs/diagrams/pr279.mmd @@ -0,0 +1,37 @@ + + +flowchart LR + + Folder --> Session --> ImportTask --> BeetsItem --> TaskPendingItem + ImportTask --> AlbumMatch --> AlbumInfo + + AlbumMatch <--> AlbumMatchTrackMapping + AlbumMatchTrackMapping --> BeetsItem + AlbumMatchTrackMapping --> TrackInfo + + + Folder + Session + ImportTask + + subgraph Album + AlbumMatch + AlbumInfo + AlbumMatchTrackMapping + end + + subgraph Track + BeetsItem + TaskPendingItem + TrackInfo + end + + + BeetsItem["`BeetsItem + _(Snapshot + of track on disk)_ + `"] + + AlbumMatch["`AlbumMatch + _(Candidate)_ + `"] diff --git a/docs/diagrams/sessions.mmd b/docs/diagrams/sessions.mmd new file mode 100644 index 00000000..bd1585e8 --- /dev/null +++ b/docs/diagrams/sessions.mmd @@ -0,0 +1,42 @@ +flowchart LR + + BeetsImportSession + BaseSession + subgraph Preview Queue + PreviewSession["` + __PreviewSession__ + _enqueue_preview()_ + _enqueue_import_auto()_ + `"] + AddCandidatesSession["` + __AddCandidatesSession__ + _enqueue_preview_add_candidates()_ + `"] + end + + subgraph Import Queue + ImportSession["` + __ImportSession__ + _enqueue_import_candidate()_ + `"] + BootlegImportSession["` + __BootlegImportSession__ + _enqueue_import_bootleg()_ + `"] + AutoImportSession["` + __AutoImportSession__ + _enqueue_import_auto()_ + `"] + UndoSession["` + __UndoSession__ + _enqueue_import_undo()_ + `"] + end + + BeetsImportSession --> BaseSession + BaseSession --> PreviewSession + BaseSession --> ImportSession + BaseSession --> UndoSession + PreviewSession --> AddCandidatesSession + ImportSession --> BootlegImportSession + ImportSession --> AutoImportSession diff --git a/frontend/src/components/import/candidates/actions.tsx b/frontend/src/components/import/candidates/actions.tsx index 8a313725..b1d9fc16 100644 --- a/frontend/src/components/import/candidates/actions.tsx +++ b/frontend/src/components/import/candidates/actions.tsx @@ -217,7 +217,7 @@ export function CandidateSearch({ task }: { task: SerializedTaskState }) { const [search, setSearch] = useState({ search_ids: [], search_artist: null, - search_album: null, + search_name: null, }); /** Mutation for the search @@ -274,7 +274,7 @@ export function CandidateSearch({ task }: { task: SerializedTaskState }) { setSearch({ search_ids: [], search_artist: '', - search_album: '', + search_name: '', }); } catch (e) { // dont close the dialog @@ -332,11 +332,11 @@ export function CandidateSearch({ task }: { task: SerializedTaskState }) { id="input-search-artist" label="and album" placeholder="Album" - value={search.search_album || ''} + value={search.search_name || ''} onChange={(e) => { setSearch({ ...search, - search_album: e.target.value, + search_name: e.target.value, }); }} /> diff --git a/frontend/src/pythonTypes.ts b/frontend/src/pythonTypes.ts index ca67838a..e343760c 100644 --- a/frontend/src/pythonTypes.ts +++ b/frontend/src/pythonTypes.ts @@ -25,7 +25,7 @@ export interface SerializedProgressState { export interface Search { search_ids: Array; search_artist: null | string; - search_album: null | string; + search_name: null | string; } export interface LibraryStats {