diff --git a/operator-tools/README_collections.md b/operator-tools/README_collections.md index 1dfee27..12eb5b9 100644 --- a/operator-tools/README_collections.md +++ b/operator-tools/README_collections.md @@ -102,7 +102,31 @@ uv run operator-tools/manage_collections.py batch-create stac/ --pattern "*-stag - Reports success/failure for each file - Summary statistics at the end -#### 4. `info` - Show Collection Information +#### 4. `delete` - Delete a Collection + +Delete a collection from the STAC catalog. Some STAC servers require the collection to be empty before deletion. + +```bash +# Delete a collection (will prompt for confirmation) +uv run operator-tools/manage_collections.py delete sentinel-2-l2a-staging + +# Clean items first, then delete +uv run operator-tools/manage_collections.py delete sentinel-2-l2a-staging --clean-first + +# Skip confirmation prompt +uv run operator-tools/manage_collections.py delete sentinel-2-l2a-staging --clean-first -y +``` + +**Options:** +- `--clean-first`: Remove all items from the collection before deleting it +- `--yes, -y`: Skip confirmation prompt + +**Safety Features:** +- Confirmation prompt before deletion (unless `--yes` is used) +- Option to automatically clean items first +- Handles already-deleted collections gracefully + +#### 5. `info` - Show Collection Information Display detailed information about a collection, including item count. @@ -178,10 +202,11 @@ uv run operator-tools/manage_collections.py batch-create stac/ The tool uses the following STAC Transaction API endpoints: -- `GET /collections/{collection_id}/items` - List items (for cleaning) +- `GET /collections/{collection_id}/items` - List items (for cleaning and info) - `DELETE /collections/{collection_id}/items/{item_id}` - Delete item - `POST /collections` - Create collection - `PUT /collections` - Update collection +- `DELETE /collections/{collection_id}` - Delete collection ## Error Handling @@ -243,6 +268,9 @@ uv run operator-tools/manage_collections.py clean sentinel-2-l2a-staging -y # 4. Update collection metadata uv run operator-tools/manage_collections.py create stac/sentinel-2-l2a.json --update + +# 5. (If needed) Delete collection +uv run operator-tools/manage_collections.py delete sentinel-2-l2a-staging --clean-first -y ``` ### Development Workflow diff --git a/operator-tools/manage_collections.py b/operator-tools/manage_collections.py index 95a398d..2dcb019 100755 --- a/operator-tools/manage_collections.py +++ b/operator-tools/manage_collections.py @@ -6,6 +6,7 @@ Supports: - Cleaning collections (removing all items) - Creating/updating collections from templates +- Deleting collections """ import json @@ -131,6 +132,36 @@ def clean_collection(self, collection_id: str, dry_run: bool = False) -> int: ) return deleted_count + def delete_collection(self, collection_id: str) -> bool: + """ + Delete a collection using Transaction API. + + Args: + collection_id: ID of the collection to delete + + Returns: + True if successful, False otherwise + """ + url = f"{self.api_url}/collections/{collection_id}" + + try: + response = self.session.delete(url, timeout=30) + response.raise_for_status() + click.echo(f"✅ Collection {collection_id} deleted successfully") + return True + except requests.exceptions.HTTPError as e: + if e.response.status_code == 404: + click.echo(f"⚠️ Collection {collection_id} not found", err=True) + return False + click.echo( + f"❌ Failed to delete collection {collection_id}: {e.response.status_code} - {e.response.text}", + err=True, + ) + return False + except Exception as e: + click.echo(f"❌ Error deleting collection {collection_id}: {e}", err=True) + return False + def create_or_update_collection( self, collection_data: dict[str, Any], update: bool = False ) -> bool: @@ -368,6 +399,58 @@ def batch_create(ctx: click.Context, directory: Path, update: bool, pattern: str click.echo(f"❌ Failed: {failed_count}") +@cli.command() +@click.argument("collection_id") +@click.option( + "--clean-first", + is_flag=True, + help="Remove all items from the collection before deleting it", +) +@click.option( + "--yes", + "-y", + is_flag=True, + help="Skip confirmation prompt", +) +@click.pass_context +def delete(ctx: click.Context, collection_id: str, clean_first: bool, yes: bool) -> None: + """ + Delete a collection. + + Note: Some STAC servers require the collection to be empty before deletion. + Use --clean-first to automatically remove all items first. + + Examples: + manage_collections.py delete sentinel-2-l2a-staging + manage_collections.py delete sentinel-2-l2a-staging --clean-first + manage_collections.py delete sentinel-2-l2a-staging --clean-first --yes + """ + manager: STACCollectionManager = ctx.obj["manager"] + + if not yes: + click.confirm( + f"⚠️ This will permanently delete collection '{collection_id}'. Continue?", + abort=True, + ) + + try: + # Clean collection first if requested + if clean_first: + click.echo("\n📋 Cleaning collection before deletion...") + manager.clean_collection(collection_id, dry_run=False) + + # Delete the collection + click.echo(f"\n🗑️ Deleting collection: {collection_id}") + success = manager.delete_collection(collection_id) + + if not success: + raise click.Abort() + + except Exception as e: + click.echo(f"❌ Operation failed: {e}", err=True) + raise click.Abort() from e + + @cli.command() @click.argument("collection_id") @click.pass_context diff --git a/operator-tools/submit_stac_items_notebook.ipynb b/operator-tools/submit_stac_items_notebook.ipynb index dad0a8d..a34cd08 100644 --- a/operator-tools/submit_stac_items_notebook.ipynb +++ b/operator-tools/submit_stac_items_notebook.ipynb @@ -22,7 +22,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 4, "metadata": {}, "outputs": [], "source": [ @@ -56,7 +56,7 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -72,7 +72,7 @@ "STAC_API_URL = \"https://stac.core.eopf.eodc.eu/\"\n", "\n", "# Webhook Configuration\n", - "WEBHOOK_URL = \"http://localhost:12001/samples\"\n", + "WEBHOOK_URL = \"http://localhost:12000/samples\"\n", "\n", "print(\"✅ Configuration loaded\")" ] @@ -86,30 +86,32 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Area of Interest: [12.4, 41.8, 12.6, 42.0]\n", - "Time Range: 2025-07-01T00:00:00Z to 2025-07-31T23:59:59Z\n" + "Area of Interest: [12.0, 44.4, 14.0, 45.0]\n", + "Time Range: 2025-04-01T00:00:00Z to 2025-04-28T23:59:59Z\n" ] } ], "source": [ "# Area of Interest (AOI) - Bounding box: [min_lon, min_lat, max_lon, max_lat]\n", "# Example: Rome area\n", - "aoi_bbox = [12.4, 41.8, 12.6, 42.0]\n", + "# aoi_bbox = [12.4, 41.8, 12.6, 42.0]\n", "# Example 2: Majorca area (2.1697998046875004%2C39.21097520599528%2C3.8177490234375004)\n", "# aoi_bbox = [2.16, 39.21, 3.82, 39.78]\n", "# Example 3: France Full\n", "# aoi_bbox = [-5.14, 41.33, 9.56, 51.09]\n", + "# Example 4: Lagoon From Venice to Trieste\n", + "aoi_bbox = [12.0, 44.4, 14.0, 45.0]\n", "\n", "# Time range\n", - "start_date = \"2025-07-01T00:00:00Z\"\n", - "end_date = \"2025-07-31T23:59:59Z\"\n", + "start_date = \"2025-04-01T00:00:00Z\"\n", + "end_date = \"2025-04-28T23:59:59Z\"\n", "\n", "print(f\"Area of Interest: {aoi_bbox}\")\n", "print(f\"Time Range: {start_date} to {end_date}\")" @@ -124,7 +126,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -137,18 +139,15 @@ " - sentinel-2-l2a\n", " The Sentinel-2 Level-2A Collection 1 product provides orthorectified Surface Reflectance (Bottom-Of-...\n", "\n", - " - sentinel-3-slstr-l1-rbt\n", - " The Sentinel-3 SLSTR Level-1B RBT product provides radiances and brightness temperatures for each pi...\n", - "\n", " - sentinel-3-olci-l2-lrr\n", " The Sentinel-3 OLCI L2 LRR product provides land and atmospheric geophysical parameters computed for...\n", "\n", - " - sentinel-2-l1c\n", - " The Sentinel-2 Level-1C product is composed of 110x110 km2 tiles (ortho-images in UTM/WGS84 projecti...\n", - "\n", " - sentinel-3-olci-l2-lfr\n", " The Sentinel-3 OLCI L2 LFR product provides land and atmospheric geophysical parameters computed for...\n", "\n", + " - sentinel-2-l1c\n", + " The Sentinel-2 Level-1C product is composed of 110x110 km2 tiles (ortho-images in UTM/WGS84 projecti...\n", + "\n", " - sentinel-3-olci-l1-efr\n", " The Sentinel-3 OLCI L1 EFR product provides TOA radiances at full resolution for each pixel in the i...\n", "\n", @@ -158,19 +157,30 @@ " - sentinel-3-slstr-l2-frp\n", " The Sentinel-3 SLSTR Level-2 FRP product provides global (over land and water) fire radiative power.\n", "\n", + " - sentinel-1-l1-slc\n", + " The Sentinel-1 Level-1 Single Look Complex (SLC) products consist of focused SAR data, geo-reference...\n", + "\n", " - sentinel-3-olci-l1-err\n", " The Sentinel-3 OLCI L1 ERR product provides TOA radiances at reduced resolution for each pixel in th...\n", "\n", - " - sentinel-1-l1-slc\n", - " The Sentinel-1 Level-1 Single Look Complex (SLC) products consist of focused SAR data, geo-reference...\n", + " - sentinel-1-l2-ocn\n", + " The Sentinel-1 Level-2 Ocean (OCN) products for wind, wave and currents applications may contain the...\n", "\n", " - sentinel-3-slstr-l2-lst\n", " The Sentinel-3 SLSTR Level-2 LST product provides land surface temperature.\n", "\n", - " - sentinel-1-l2-ocn\n", - " The Sentinel-1 Level-2 Ocean (OCN) products for wind, wave and currents applications may contain the...\n", + " - sentinel-3-slstr-l1-rbt\n", + " The Sentinel-3 SLSTR Level-1B RBT product provides radiances and brightness temperatures for each pi...\n", "\n" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/emathot/Workspace/eopf-explorer/data-pipeline/.venv/lib/python3.13/site-packages/pystac/collection.py:658: UserWarning: Collection is missing extent, setting default spatial and temporal extents\n", + " warnings.warn(\n" + ] } ], "source": [ @@ -201,24 +211,24 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "🔍 Searching collection: sentinel-2-l2a\n", - "🎯 Target collection for processing: sentinel-2-l2a\n" + "🔍 Searching collection: sentinel-2-l1c\n", + "🎯 Target collection for processing: sentinel-2-l1c\n" ] } ], "source": [ "# Choose the source collection to search\n", - "source_collection = \"sentinel-2-l2a\" # Change this to your desired collection\n", + "source_collection = \"sentinel-2-l1c\" # Change this to your desired collection\n", "\n", "# Choose the target collection for processing\n", - "target_collection = \"sentinel-2-l2a-staging\" # Change this to your target collection\n", + "target_collection = \"sentinel-2-l1c\" # Change this to your target collection\n", "\n", "print(f\"🔍 Searching collection: {source_collection}\")\n", "print(f\"🎯 Target collection for processing: {target_collection}\")" @@ -226,7 +236,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -234,7 +244,7 @@ "output_type": "stream", "text": [ "\n", - "✅ Found 10 items (after filtering).\n", + "✅ Found 34 items (after filtering).\n", "\n" ] } @@ -245,7 +255,7 @@ " collections=[source_collection],\n", " bbox=aoi_bbox,\n", " datetime=f\"{start_date}/{end_date}\", # Adjust as needed\n", - " limit=100, # Adjust limit as needed\n", + " limit=200, # Adjust limit as needed\n", ")\n", "\n", "# Collect items paginated results and clean them (workaround for issue #26)\n", @@ -286,7 +296,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -327,7 +337,7 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 17, "metadata": {}, "outputs": [ { @@ -335,23 +345,47 @@ "output_type": "stream", "text": [ "\n", - "📤 Submitting 10 items to pipeline...\n", + "📤 Submitting 34 items to pipeline...\n", "\n", - "✅ Submitted: S2C_MSIL2A_20250728T100051_N0511_R122_T33TTG_20250728T153115\n", - "✅ Submitted: S2C_MSIL2A_20250728T100051_N0511_R122_T32TQM_20250728T153115\n", - "✅ Submitted: S2B_MSIL2A_20250713T100029_N0511_R122_T33TUG_20250713T123724\n", - "✅ Submitted: S2B_MSIL2A_20250713T100029_N0511_R122_T33TTG_20250713T123724\n", - "✅ Submitted: S2B_MSIL2A_20250713T100029_N0511_R122_T32TQM_20250713T123724\n", - "✅ Submitted: S2C_MSIL2A_20250708T100051_N0511_R122_T33TTG_20250708T155705\n", - "✅ Submitted: S2C_MSIL2A_20250708T100051_N0511_R122_T32TQM_20250708T155705\n", - "✅ Submitted: S2B_MSIL2A_20250703T100029_N0511_R122_T33TUG_20250703T122001\n", - "✅ Submitted: S2B_MSIL2A_20250703T100029_N0511_R122_T33TTG_20250703T122001\n", - "✅ Submitted: S2B_MSIL2A_20250703T100029_N0511_R122_T32TQM_20250703T122001\n", + "✅ Submitted: S2B_MSIL1C_20250427T100559_N0511_R022_T33TUK_20250427T123603\n", + "✅ Submitted: S2B_MSIL1C_20250427T100559_N0511_R022_T32TQR_20250427T123603\n", + "✅ Submitted: S2B_MSIL1C_20250427T100559_N0511_R022_T32TQQ_20250427T123603\n", + "✅ Submitted: S2A_MSIL1C_20250424T101041_N0511_R022_T33TUK_20250424T153018\n", + "✅ Submitted: S2A_MSIL1C_20250424T101041_N0511_R022_T32TQR_20250424T153018\n", + "✅ Submitted: S2A_MSIL1C_20250424T101041_N0511_R022_T32TQQ_20250424T170815\n", + "✅ Submitted: S2A_MSIL1C_20250424T101041_N0511_R022_T32TQQ_20250424T153018\n", + "✅ Submitted: S2B_MSIL1C_20250424T100029_N0511_R122_T33TVK_20250424T122548\n", + "✅ Submitted: S2B_MSIL1C_20250424T100029_N0511_R122_T33TUK_20250424T122548\n", + "✅ Submitted: S2B_MSIL1C_20250424T100029_N0511_R122_T32TQR_20250424T122548\n", + "✅ Submitted: S2B_MSIL1C_20250424T100029_N0511_R122_T32TQQ_20250424T122548\n", + "✅ Submitted: S2C_MSIL1C_20250422T101051_N0511_R022_T33TUK_20250422T140109\n", + "✅ Submitted: S2C_MSIL1C_20250422T101051_N0511_R022_T32TQR_20250422T140109\n", + "✅ Submitted: S2C_MSIL1C_20250422T101051_N0511_R022_T32TQQ_20250422T140109\n", + "✅ Submitted: S2C_MSIL1C_20250419T100051_N0511_R122_T33TVK_20250419T134436\n", + "✅ Submitted: S2C_MSIL1C_20250419T100051_N0511_R122_T33TUK_20250419T134436\n", + "✅ Submitted: S2C_MSIL1C_20250419T100051_N0511_R122_T32TQR_20250419T134436\n", + "✅ Submitted: S2C_MSIL1C_20250419T100051_N0511_R122_T32TQQ_20250419T134436\n", + "✅ Submitted: S2B_MSIL1C_20250417T100559_N0511_R022_T33TUK_20250417T123703\n", + "✅ Submitted: S2B_MSIL1C_20250417T100559_N0511_R022_T32TQR_20250417T123703\n", + "✅ Submitted: S2B_MSIL1C_20250417T100559_N0511_R022_T32TQQ_20250417T123703\n", + "✅ Submitted: S2A_MSIL1C_20250414T100701_N0511_R022_T33TUK_20250414T153115\n", + "✅ Submitted: S2A_MSIL1C_20250414T100701_N0511_R022_T32TQR_20250414T153115\n", + "✅ Submitted: S2A_MSIL1C_20250414T100701_N0511_R022_T32TQQ_20250414T153115\n", + "✅ Submitted: S2B_MSIL1C_20250414T100029_N0511_R122_T33TVK_20250414T122616\n", + "✅ Submitted: S2B_MSIL1C_20250414T100029_N0511_R122_T33TUK_20250414T122616\n", + "✅ Submitted: S2B_MSIL1C_20250414T100029_N0511_R122_T32TQR_20250414T122616\n", + "✅ Submitted: S2B_MSIL1C_20250414T100029_N0511_R122_T32TQQ_20250414T122616\n", + "✅ Submitted: S2C_MSIL1C_20250412T101041_N0511_R022_T33TUK_20250412T134850\n", + "✅ Submitted: S2A_MSIL1C_20250411T100041_N0511_R122_T33TUK_20250411T152338\n", + "✅ Submitted: S2A_MSIL1C_20250411T100041_N0511_R122_T32TQR_20250411T152338\n", + "✅ Submitted: S2C_MSIL1C_20250409T100051_N0511_R122_T33TUK_20250409T134445\n", + "✅ Submitted: S2C_MSIL1C_20250409T100051_N0511_R122_T32TQR_20250409T134445\n", + "✅ Submitted: S2C_MSIL1C_20250409T100051_N0511_R122_T32TQQ_20250409T134445\n", "\n", "📊 Summary:\n", - " - Successfully submitted: 10\n", + " - Successfully submitted: 34\n", " - Failed: 0\n", - " - Total: 10\n" + " - Total: 34\n" ] } ], diff --git a/uv.lock b/uv.lock index 95e55e7..7831de0 100644 --- a/uv.lock +++ b/uv.lock @@ -699,7 +699,7 @@ distributed = [ [[package]] name = "data-pipeline" -version = "1.0.0" +version = "1.1.1" source = { editable = "." } dependencies = [ { name = "boto3" },