Merge pull request #712 from MannLabs/multiple_config_dicts

allow for multiple config dicts

Author: mschwoer

alphadia/cli.py

@@ -67,7 +67,7 @@
     "--directory",
     "-d",
     type=str,
-    help="Directory containing raw data input files.",
+    help="Directory containing raw data input files. Can be passed multiple times.",
     action="append",
     default=[],
 )
@@ -107,9 +107,10 @@
     "--config-dict",
     type=str,
     help="Python dictionary which will be used to update the default config. Keys and string values need to be surrounded by "
-    'escaped double quotes, e.g. "{\\"key1\\": \\"value1\\"}".',
+    'escaped double quotes, e.g. "{\\"key1\\": \\"value1\\"}". Can be passed multiple times (later items take precedence in case of key clashes).',
     nargs="?",
-    default="{}",
+    action="append",
+    default=[],
 )
 parser.add_argument(
     "--quant-dir",  # TODO deprecate
@@ -121,47 +122,49 @@
 )
 
 
-def _recursive_update(
-    full_dict: dict, update_dict: dict
-):  # TODO merge with Config._update
-    """recursively update a dict with a second dict. The dict is updated inplace.
+def _recursive_update(target_dict: dict, source_dict: dict):
+    """Recursively update a dict with a second dict inplace.
 
     Parameters
     ----------
-    full_dict : dict
+    target_dict : dict
         dict to be updated, is updated inplace.
 
-    update_dict : dict
+    source_dict : dict
         dict with new values
-
     """
-    for key, value in update_dict.items():
-        if key in full_dict:
-            if isinstance(value, dict):
-                _recursive_update(full_dict[key], update_dict[key])
-            else:
-                full_dict[key] = value
+
+    for key, value in source_dict.items():
+        if key not in target_dict:
+            target_dict[key] = {}
+        if isinstance(value, dict):
+            _recursive_update(target_dict[key], value)
         else:
-            full_dict[key] = value
+            target_dict[key] = value
 
 
 def _get_config_from_args(
     args: argparse.Namespace,
-) -> tuple[dict, str | None, str | None]:
+) -> tuple[dict, str | None, dict]:
     """Parse config file from `args.config` if given and update with optional JSON string `args.config_dict`."""
 
     config = {}
     if args.config is not None:
         with open(args.config) as f:
             config = yaml.safe_load(f)
 
+    merged_config_dict = {}
     if args.config_dict:
         try:
-            _recursive_update(config, json.loads(args.config_dict))
+            for config_dict_str in args.config_dict:
+                config_dict = json.loads(config_dict_str)
+                _recursive_update(merged_config_dict, config_dict)
+
+            _recursive_update(config, merged_config_dict)
         except Exception as e:
             raise ValueError(f"Could not parse config dict: {e}") from e
 
-    return config, args.config, args.config_dict
+    return config, args.config, merged_config_dict
 
 
 def _get_from_args_or_config(

docs/methods/command-line.md

@@ -106,6 +106,7 @@ Sometimes, for example when optimizing a single parameter or building multi step
 ```
     --config-dict "{\"library_prediction\":{\"nce\":26}}"
 ```
+Note: this can be passed multiple times, with the later ones taking precedence in case of overlapping keys.
 
 ### 5. Summary
 The full script looks like:
@@ -146,7 +147,7 @@ can be used to specify the directory containing quantification results.
 On startup, the current configuration is dumped as `frozen_config.yaml`, which contains all information to reproduce this run.
 
 Combining these three concepts, here's an example how to reuse an existing quantification (from the `previous_run` directory), but create additional
-output (`peptide_level_lfq`)
+output (`peptide_level_lfq`) through a custom `--config-dict`:
 ```
 alphadia -o ./output_dir --quant-dir ./previous_run/quant --config ./previous_run/frozen_config.yaml --config-dict '{"general": {"reuse_quant": "True"}, "search_output": {"peptide_level_lfq": "True"}}'
 ```

tests/unit_tests/test_cli.py

@@ -33,16 +33,35 @@ def test_get_config_from_args():
 
 def test_get_config_from_config_dict():
     """Test the _get_config_from_args function correctly parses config dict."""
-    mock_args = MagicMock(config=None, config_dict='{"key3": "value3"}')
+    mock_args = MagicMock(config=None, config_dict=['{"key3": "value3"}'])
 
     result = _get_config_from_args(mock_args)
 
-    assert result == ({"key3": "value3"}, None, '{"key3": "value3"}')
+    assert result == ({"key3": "value3"}, None, {"key3": "value3"})
+
+
+def test_get_config_from_config_dict_multiple():
+    """Test the _get_config_from_args function correctly parses multiple config dicts."""
+    mock_args = MagicMock(
+        config=None,
+        config_dict=[
+            '{"key3": "value3", "key4": "value4"}',
+            '{"key4": "value4b", "key5": "value5"}',
+        ],
+    )
+
+    result = _get_config_from_args(mock_args)
+
+    assert result == (
+        {"key3": "value3", "key4": "value4b", "key5": "value5"},
+        None,
+        {"key3": "value3", "key4": "value4b", "key5": "value5"},
+    )
 
 
 def test_get_config_from_args_and_config_dict():
     """Test the _get_config_from_args function correctly merges config file and dict."""
-    mock_args = MagicMock(config="config.yaml", config_dict='{"key3": "value3"}')
+    mock_args = MagicMock(config="config.yaml", config_dict=['{"key3": "value3"}'])
 
     yaml_content = {"key1": "value1", "key2": "value2"}
     mock_yaml = yaml.dump(yaml_content)
@@ -53,7 +72,7 @@ def test_get_config_from_args_and_config_dict():
     assert result == (
         {"key1": "value1", "key2": "value2", "key3": "value3"},
         "config.yaml",
-        '{"key3": "value3"}',
+        {"key3": "value3"},
     )
 
 
@@ -109,7 +128,7 @@ def test_cli_minimal_args(mock_parse_known_args):
         version=None,
         check=None,
         output="/output",
-        config_dict="{}",
+        config_dict=[],
         file=[],
         directory=[],
         regex=".*",
@@ -156,7 +175,7 @@ def test_cli_minimal_args_all_none(mock_parse_known_args):
         fasta=None,
         library=None,
         quant_dir=None,
-        config_dict="{}",
+        config_dict=[],
         file=[],
         directory=[],
         regex=".*",