Add WEKO patch to delay file content task to avoid ES version conflict

Author: yacchin1205

.github/patches/weko-delay-file-content-task.patch

@@ -0,0 +1,14 @@
+diff --git a/modules/weko-deposit/weko_deposit/api.py b/modules/weko-deposit/weko_deposit/api.py
+index XXXXXXX..XXXXXXX 100644
+--- a/modules/weko-deposit/weko_deposit/api.py
++++ b/modules/weko-deposit/weko_deposit/api.py
+@@ -1025,7 +1025,8 @@ class WekoDeposit(Deposit):
+                                                  self.revision_id)
+                     # Upload pdf file content to Elasticsearch
+                     from .tasks import extract_pdf_and_update_file_contents
+-                    extract_pdf_and_update_file_contents.apply_async((reading_targets, str(self.pid.object_uuid)))
++                    # Add countdown to avoid version conflict with web process
++                    extract_pdf_and_update_file_contents.apply_async((reading_targets, str(self.pid.object_uuid)), countdown=10)
+                 except TransportError as err:
+                     if self.jrc.get('content'):
+                         for content in self.jrc['content']:

.github/scripts/setup_weko.sh

@@ -47,6 +47,8 @@ case "$COMMAND" in
     patch -d "${WEKO_ROOT}" -p1 < "${SCRIPT_DIR}/../patches/weko-oauth2-insecure-transport.patch"
     # Apply patch to fix chardet TypeError on ZIP filename handling
     patch -d "${WEKO_ROOT}" -p1 < "${SCRIPT_DIR}/../patches/weko-chardet-fix.patch"
+    # Apply patch to delay file content extraction task to avoid ES version conflict
+    patch -d "${WEKO_ROOT}" -p1 < "${SCRIPT_DIR}/../patches/weko-delay-file-content-task.patch"
     # Generate self-signed certificate for WEKO nginx with SAN for IP address
     mkdir -p "${WEKO_ROOT}/nginx/keys"
     openssl req -x509 -nodes -days 365 -newkey rsa:2048 \
@@ -89,6 +91,44 @@ db.session.commit()
 print('Updated mapping 30002')
 "
 
+    # Grant contributor access to Sample Index
+    echo "=== Granting Contributor Access to Sample Index ==="
+    docker compose -f "${compose_file}" exec -T web invenio shell -c '
+from weko_index_tree.models import Index
+from weko_index_tree.utils import delete_index_trees_from_redis
+from invenio_db import db
+index = Index.query.filter_by(index_name_english="Sample Index").first()
+if not index:
+    raise Exception("Sample Index not found")
+print(f"Before: contribute_role={index.contribute_role}, browsing_role={index.browsing_role}, public_state={index.public_state}")
+index.contribute_role = "1,2,3,4,-98,-99"
+index.browsing_role = "1,2,3,4,-98,-99"
+index.public_state = True
+db.session.commit()
+# Clear Redis cache so API returns fresh data
+for lang in ["en", "ja"]:
+    delete_index_trees_from_redis(lang)
+print(f"After: contribute_role={index.contribute_role}, browsing_role={index.browsing_role}, public_state={index.public_state}")
+'
+
+    # Grant index-tree-access permission to Contributor role
+    echo "=== Granting index-tree-access to Contributor ==="
+    docker compose -f "${compose_file}" exec -T web invenio shell -c '
+from invenio_access.models import ActionRoles, Role
+from invenio_db import db
+role = Role.query.filter_by(name="Contributor").first()
+if not role:
+    raise Exception("Contributor role not found")
+existing = ActionRoles.query.filter_by(action="index-tree-access", role_id=role.id).first()
+if existing:
+    print(f"index-tree-access already granted to Contributor (role_id={role.id})")
+else:
+    ar = ActionRoles(action="index-tree-access", role_id=role.id)
+    db.session.add(ar)
+    db.session.commit()
+    print(f"Granted index-tree-access to Contributor (role_id={role.id})")
+'
+
     # Validate SWORD mapping 30002
     echo "=== SWORD Mapping Validation (30002) ==="
     validation_result=$(docker compose -f "${compose_file}" exec -T web invenio shell -c '

requirements.txt

@@ -9,4 +9,5 @@ requests>=2.26.0
 PyYAML>=5.4.1
 matplotlib>=3.4.0
 seaborn>=0.11.0
-python-dotenv>=0.19.0
+python-dotenv>=0.19.0
+python-docx>=0.8.11

scripts/grdm.py

@@ -283,9 +283,9 @@ def get_select_file_draggable_locator(page, provider):
 def get_select_file_draggable_xpath(name):
     return f'//*[contains(@class, "tb-expand-icon-holder")]//*[contains(@class, "file-extension")]/../../following-sibling::*[contains(@class, "title-text")]//*[text() = "{name}"]/../..'
 
-async def wait_for_uploaded(page, filename):
-    await expect(page.locator(f'//*[text() = "{filename}"]/../following-sibling::*//*[@role = "progressbar"]')).to_have_count(0, timeout=30000)
-    await expect(get_select_file_title_locator(page, filename)).to_be_visible(timeout=1000)    
+async def wait_for_uploaded(page, filename, timeout=30000):
+    await expect(page.locator(f'//*[text() = "{filename}"]/../following-sibling::*//*[@role = "progressbar"]')).to_have_count(0, timeout=timeout)
+    await expect(get_select_file_title_locator(page, filename)).to_be_visible(timeout=timeout)
 
 def _bytes_to_data_url(byte_data, mime_type="application/octet-stream"):
     """バイト配列をDataURLに変換"""
@@ -334,7 +334,6 @@ async def drop_file(page, element_locator, path):
 
 async def drag_and_drop(page, source, dest):
     await expect(source).to_have_class(re.compile('.*ui-draggable.*'))
-    await expect(dest).to_have_class(re.compile('.*ui-droppable.*'))
 
     center_coordinates_source = await source.evaluate('''element => {
         const rect = element.getBoundingClientRect();

scripts/metadata_v2025.py

@@ -17,7 +17,7 @@
 from __future__ import annotations
 
 from enum import Enum
-from typing import Dict
+from typing import Any, Dict
 
 
 class FieldType(Enum):
@@ -48,18 +48,19 @@ class ProjectMetadataForm:
         "プロジェクトの分野": FieldType.POWER_SELECT,
     }
 
-    def __init__(self, page):
+    def __init__(self, page, parent_locator=None):
         self.page = page
+        self._root = parent_locator or page
 
     def _get_locator(self, label: str, field_type: FieldType):
         base = "//*"
         match field_type:
             case FieldType.INPUT:
-                return self.page.locator(
+                return self._root.locator(
                     f'{base}[contains(text(), "{label}")]/../following-sibling::div[1]//input'
                 )
             case FieldType.POWER_SELECT:
-                return self.page.locator(
+                return self._root.locator(
                     f'{base}[contains(text(), "{label}")]/../following-sibling::div[1]'
                 )
             case _:
@@ -81,7 +82,7 @@ async def fill(self, label: str, value: str) -> None:
                 await locator.fill(value)
             case FieldType.POWER_SELECT:
                 await locator.locator(".ember-power-select-trigger").click()
-                option = self.page.locator(
+                option = self._root.locator(
                     f'//li[contains(@class, "ember-power-select-option") and contains(., "{value}")]'
                 )
                 await option.click()
@@ -102,7 +103,7 @@ async def fill_power_select_by_search(self, label: str, value: str) -> None:
         field_type = self.FIELDS[label]
         locator = self._get_locator(label, field_type)
         await locator.locator(".ember-power-select-trigger").click()
-        search = self.page.locator(".ember-power-select-search-input")
+        search = self._root.locator(".ember-power-select-search-input")
         await search.fill(value)
         await search.press("Enter")
 
@@ -115,6 +116,7 @@ class FileMetadataForm:
 
     FIELDS: Dict[str, FieldType] = {
         # Basic info
+        "ファイル種別": FieldType.SELECT,
         "データ No.": FieldType.INPUT,
         "データの名称または論文表題 (日本語)": FieldType.INPUT,
         "Title (English)": FieldType.INPUT,
@@ -139,10 +141,22 @@ class FileMetadataForm:
         "リポジトリURL・DOIリンク": FieldType.INPUT,
         # Creators
         "データ作成者": FieldType.TABLE,
+        "著者名": FieldType.TABLE,
         # Hosting institution
         "データ管理機関 (日本語)": FieldType.INPUT,
         "Hosting institution (English)": FieldType.INPUT,
         "データ管理機関コード": FieldType.INPUT,
+        # Bibliographic specific fields
+        "論文（出版社版）のDOI": FieldType.INPUT,
+        "論文の種類": FieldType.SELECT,
+        "掲載誌名 (日本語)": FieldType.INPUT,
+        "Journal Name (English)": FieldType.INPUT,
+        "発行年月": FieldType.INPUT,
+        "巻": FieldType.INPUT,
+        "号": FieldType.INPUT,
+        "掲載ページ (開始)": FieldType.INPUT,
+        "掲載ページ (終了)": FieldType.INPUT,
+        "学術論文を掲載した「機関リポジトリ等の情報基盤」のDOI": FieldType.INPUT,
         # Data manager
         "データ管理者の種類": FieldType.SELECT,
         "データ管理者の e-Rad 研究者番号": FieldType.INPUT,
@@ -159,42 +173,51 @@ class FileMetadataForm:
         "Remarks (English)": FieldType.TEXTAREA,
         # Metadata access
         "メタデータのアクセス権": FieldType.SELECT,
+        # Publication specific extra
+        "査読の有無": FieldType.SELECT,
+        "版情報": FieldType.SELECT,
     }
 
-    def __init__(self, page):
+    def __init__(self, page, parent_locator=None):
         self.page = page
+        self._root = parent_locator or page
 
     def _get_locator(self, label: str, field_type: FieldType):
         # Special case: 概略データ量 has different xpath
         if label == "概略データ量":
-            return self.page.locator(
+            return self._root.locator(
                 '//label[contains(text(), "概略データ量")]/../..//input[contains(@class, "form-control")]'
             )
+        exact_labels = {"号"}
+        if label in exact_labels:
+            label_xpath = f'//label[normalize-space(.) = "{label}"]'
+        else:
+            label_xpath = f'//label[contains(text(), "{label}")]'
 
         match field_type:
             case FieldType.INPUT:
-                return self.page.locator(
-                    f'//label[contains(text(), "{label}")]/../following-sibling::div[1]//input'
+                return self._root.locator(
+                    f'{label_xpath}/../following-sibling::div[1]//input'
                 )
             case FieldType.INPUT_DIRECT:
-                return self.page.locator(
-                    f'//label[contains(text(), "{label}")]/../following-sibling::input[1]'
+                return self._root.locator(
+                    f'{label_xpath}/../following-sibling::input[1]'
                 )
             case FieldType.TEXTAREA:
-                return self.page.locator(
-                    f'//label[contains(text(), "{label}")]/../following-sibling::textarea[1]'
+                return self._root.locator(
+                    f'{label_xpath}/../following-sibling::textarea[1]'
                 )
             case FieldType.SELECT:
                 if label == "アクセス権":
-                    return self.page.locator(
-                        f'//label[text()="{label}"]/../following-sibling::select[1]'
+                    return self._root.locator(
+                        f'//label[normalize-space(text())="{label}"]/../following-sibling::select[1]'
                     )
-                return self.page.locator(
-                    f'//label[contains(text(), "{label}")]/../following-sibling::select[1]'
+                return self._root.locator(
+                    f'{label_xpath}/../following-sibling::select[1]'
                 )
             case FieldType.TABLE:
-                return self.page.locator(
-                    f'//label[contains(text(), "{label}")]/../following-sibling::div[1]'
+                return self._root.locator(
+                    f'{label_xpath}/../following-sibling::div[1]'
                 )
             case _:
                 raise ValueError(f"Unsupported field type: {field_type}")
@@ -271,3 +294,32 @@ async def get_table_cell(self, label: str, row_index: int, col_index: int) -> st
         row = locator.locator(f"table tbody tr:nth-of-type({row_index + 1})")
         cell_input = row.locator(f"td:nth-of-type({col_index + 1}) input")
         return await cell_input.input_value()
+
+    async def fill_author(self, author: Dict[str, Any]) -> None:
+        """Add an author row and fill all author fields."""
+        container = self.get_locator("著者名")
+        await self.click_table_add_row("著者名")
+
+        edit_rows = container.locator('.metadata-edit-mode')
+        row_count = await edit_rows.count()
+        if row_count == 0:
+            raise AssertionError("No edit rows found for authors")
+        panel = edit_rows.nth(row_count - 1)
+        await panel.wait_for(state="visible")
+
+        await panel.locator('label:has-text("e-Rad 研究者番号") + div input').fill(author['number'])
+
+        ja_inputs = panel.locator('label:has-text("名前(日本語)") + div table input')
+        await ja_inputs.nth(0).fill(author['name_ja']['last'])
+        await ja_inputs.nth(1).fill(author['name_ja']['middle'])
+        await ja_inputs.nth(2).fill(author['name_ja']['first'])
+
+        en_inputs = panel.locator('label:has-text("Name (English)") + div table input')
+        await en_inputs.nth(0).fill(author['name_en']['last'])
+        await en_inputs.nth(1).fill(author['name_en']['middle'])
+        await en_inputs.nth(2).fill(author['name_en']['first'])
+
+        await panel.locator('label:has-text("所属機関名(日本語)") + div input').fill(author['affiliation_ja'])
+        await panel.locator('label:has-text("所属機関名(英語)") + div input').fill(author['affiliation_en'])
+
+        await panel.locator('.hide-edit-row').click()

scripts/playwright.py

@@ -1,4 +1,5 @@
 # ユーティリティ関数群
+import asyncio
 from datetime import datetime
 import os
 import shutil
@@ -134,11 +135,20 @@ async def init_pw_context(close_on_fail=True, last_path=None, browser_type='chro
     console_messages = []
     return (current_session_id, temp_dir)
 
-async def finish_pw_context(screenshot=False, last_path=None):
+async def finish_pw_context(screenshot=False, last_path=None, timeout=180):
     global current_browser
-    await _finish_pw_context(screenshot=screenshot, last_path=last_path)
+    try:
+        await asyncio.wait_for(
+            _finish_pw_context(screenshot=screenshot, last_path=last_path),
+            timeout=timeout
+        )
+    except asyncio.TimeoutError:
+        print(f'finish_pw_context timed out after {timeout} seconds', file=sys.stderr)
     if current_browser is not None:
-        await current_browser.close()
+        try:
+            await asyncio.wait_for(current_browser.close(), timeout=30)
+        except asyncio.TimeoutError:
+            print('browser.close() timed out', file=sys.stderr)
         current_browser = None
 
 async def save_screenshot(path):