Feature/ams w2025 61a metadata

docker-compose.yml

@@ -311,11 +311,14 @@ services:
 
   pgpool:
     restart: "always"
-    image: bitnami/pgpool
+    image: pgpool/pgpool:4.2.2
     environment:
-      - PGPOOL_BACKEND_NODES=0:postgresql:5432
-      - PGPOOL_SR_CHECK_USER=invenio
-      - PGPOOL_SR_CHECK_PASSWORD=dbpass123
+      - PGPOOL_PARAMS_BACKEND_HOSTNAME0=postgresql
+      - PGPOOL_PARAMS_BACKEND_PORT0=5432
+      - PGPOOL_PARAMS_BACKEND_WEIGHT0=1
+      - PGPOOL_PARAMS_SR_CHECK_USER=invenio
+      - PGPOOL_PARAMS_SR_CHECK_PASSWORD=dbpass123
+      - PGPOOL_PARAMS_PORT=5432
       - PGPOOL_ENABLE_LDAP=no
       - PGPOOL_POSTGRES_USERNAME=postgres
       - PGPOOL_POSTGRES_PASSWORD=dbpass123
@@ -452,4 +455,3 @@ volumes:
   mongo_data:
 #  letsencrypt_etc:
 #  letsencrypt_html:
-

docker-compose2.yml

@@ -311,11 +311,14 @@ services:
 
   pgpool:
     restart: "always"
-    image: bitnami/pgpool
+    image: pgpool/pgpool:4.2.2
     environment:
-      - PGPOOL_BACKEND_NODES=0:postgresql:5432
-      - PGPOOL_SR_CHECK_USER=invenio
-      - PGPOOL_SR_CHECK_PASSWORD=dbpass123
+      - PGPOOL_PARAMS_BACKEND_HOSTNAME0=postgresql
+      - PGPOOL_PARAMS_BACKEND_PORT0=5432
+      - PGPOOL_PARAMS_BACKEND_WEIGHT0=1
+      - PGPOOL_PARAMS_SR_CHECK_USER=invenio
+      - PGPOOL_PARAMS_SR_CHECK_PASSWORD=dbpass123
+      - PGPOOL_PARAMS_PORT=5432
       - PGPOOL_ENABLE_LDAP=no
       - PGPOOL_POSTGRES_USERNAME=postgres
       - PGPOOL_POSTGRES_PASSWORD=dbpass123
@@ -452,4 +455,3 @@ volumes:
   mongo_data:
 #  letsencrypt_etc:
 #  letsencrypt_html:
-

modules/weko-accounts/weko_accounts/utils.py

@@ -30,7 +30,6 @@
 import hashlib
 
 from .config import WEKO_API_LIMIT_RATE_DEFAULT
-from weko_admin.models import AdminSettings
 
 limiter = Limiter(
     app=None,
@@ -111,6 +110,7 @@ def parse_attributes():
     error = False
 
     # Get attribute mapping from admin settings
+    from weko_admin.models import AdminSettings
     admin_settings = AdminSettings.get('attribute_mapping', dict_to_object=False)
 
     for header, attr in current_app.config[
@@ -221,7 +221,7 @@ def decorated_view(*args, **kwargs):
 
 def get_sp_info():
     """Get Service Provider (SP) information for Shibboleth login.
-    
+
     Returns:
         dict: A dictionary containing SP entityID, handlerURL, and return URL.
     """
@@ -233,7 +233,7 @@ def get_sp_info():
     sp_entityID = 'https://' + current_app.config['WEB_HOST_NAME'] + '/shibboleth-sp'
     if 'SP_ENTITYID' in current_app.config:
         sp_entityID = current_app.config['SP_ENTITYID']
-    
+
     sp_handlerURL = 'https://' + current_app.config['WEB_HOST_NAME'] + '/Shibboleth.sso'
     if 'SP_HANDLERURL' in current_app.config:
         sp_handlerURL = current_app.config['SP_HANDLERURL']

modules/weko-deposit/requirements2.txt

@@ -287,3 +287,4 @@ xmlschema==0.9.30
 xmltodict==0.12.0
 zipp==3.6.0
 zope.interface==5.5.2
+pypdfium2==4.30.0

modules/weko-deposit/tests/test_utils.py

@@ -19,12 +19,14 @@
 # MA 02111-1307, USA.
 
 from weko_deposit.api import WekoDeposit
-from weko_deposit.utils import update_pdf_contents_es
+from weko_deposit.utils import update_pdf_contents_es, extract_text_from_pdf, extract_text_with_tika
 
 from mock import patch
+from mock import MagicMock
 import uuid
 from tests.helpers import create_record_with_pdf
-
+import os
+import pytest
 
 # .tox/c1/bin/pytest --cov=weko_deposit tests/test_utils.py::test_update_pdf_contents_es -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-deposit/.tox/c1/tmp
 def test_update_pdf_contents_es(app, db, location, mocker):
@@ -46,4 +48,57 @@ def test_update_pdf_contents_es(app, db, location, mocker):
         for args, _ in args_list:
             test = pdf_file_infos[i]
             assert args[0] == (test,str(record_ids[i]))
-            i+=1
+            i+=1
+
+
+# .tox/c1/bin/pytest --cov=weko_deposit tests/test_utils.py::test_extract_text_from_pdf -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-deposit/.tox/c1/tmp
+def test_extract_text_from_pdf():
+    filepath = os.path.join(os.path.dirname(__file__),"data","test_files","test_file_1.2M.pdf")
+
+    # file size > max_size
+    data = extract_text_from_pdf(filepath, 10000)
+    assert len(data.encode('utf-8')) <= 10000
+    assert data.count("test file page") < 1240
+
+    # file size <= max_size
+    data = extract_text_from_pdf(filepath, 100000000)
+    assert len(data.encode('utf-8')) == 19561
+    assert data.count("test file page") == 1240
+
+    # not exist file
+    filepath = "not_exist_file.pdf"
+    with pytest.raises(FileNotFoundError) as e:
+        data = extract_text_from_pdf(filepath, 10000)
+    assert str(e.value) == "/code/modules/weko-deposit/not_exist_file.pdf"
+
+
+# .tox/c1/bin/pytest --cov=weko_deposit tests/test_utils.py::test_extract_text_with_tika -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-deposit/.tox/c1/tmp
+def test_extract_text_with_tika():
+    filepath = os.path.join(os.path.dirname(__file__),"data","test_files","sample_word.docx")
+    # not exist tika jar file.
+    mock_env_not_exist_tika = {"TIKA_JAR_FILE_PATH": "/not/exist/path/tika-server.jar"}
+    with patch.dict(os.environ, mock_env_not_exist_tika, clear=False):
+        with pytest.raises(Exception) as e:
+            data = extract_text_with_tika(filepath, 100)
+        assert str(e.value) == "not exist tika jar file."
+
+    mock_env_not_exist_tika = {"TIKA_JAR_FILE_PATH": "/code/tika/tika-app-2.6.0.jar"}
+    with patch.dict(os.environ, mock_env_not_exist_tika, clear=False):
+        # error with subprocess
+        mock_run = MagicMock()
+        mock_run.returncode.return_value=1
+        mock_run.stderr.decode.return_value="test_error"
+        with patch("weko_deposit.utils.subprocess.run", return_value=mock_run):
+            with pytest.raises(Exception) as e:
+                data = extract_text_with_tika(filepath, 100)
+            assert str(e.value) == "raise in tika: test_error"
+
+        # file size > max_size
+        data = extract_text_with_tika(filepath, 50)
+        assert len(data.encode('utf-8')) < 50
+        assert data == "これはテスト用のサンプルwordファイ"
+
+        # file size <= max_size
+        data = extract_text_with_tika(filepath, 5000)
+        assert len(data.encode('utf-8')) > 50
+        assert data == "これはテスト用のサンプルwordファイルです中身は特に意味がありません"

modules/weko-deposit/weko_deposit/utils.py

@@ -20,6 +20,10 @@
 
 from .tasks import extract_pdf_and_update_file_contents
 from .api import WekoDeposit
+import pypdfium2
+import os
+import subprocess
+
 
 def update_pdf_contents_es(record_ids):
     """register the contents of the record PDF file in elasticsearch
@@ -29,4 +33,73 @@ def update_pdf_contents_es(record_ids):
     deposits = WekoDeposit.get_records(record_ids)
     for dep in deposits:
         file_infos = dep.get_pdf_info()
-        extract_pdf_and_update_file_contents.apply_async((file_infos, str(dep.id)))
+        extract_pdf_and_update_file_contents.apply_async((file_infos, str(dep.id)))
+
+
+def extract_text_from_pdf(filepath, max_size):
+    """Read PDF file and extract text.
+
+    Args:
+        filepath (str): Path to the PDF file.
+        max_size (int): Maximum size of the extracted text in bytes.
+
+    Returns:
+        str: Extracted text from the PDF file.
+
+    """
+    reader = None
+    data = ""
+    try:
+        reader = pypdfium2.PdfDocument(filepath)
+        texts = []
+        total_bytes = 0
+        for page in reader:
+            text = page.get_textpage().get_text_range()
+            encoded = text.encode('utf-8', errors='replace')
+            if total_bytes + len(encoded) > max_size:
+                remain = max_size - total_bytes
+                texts.append(encoded[:remain].decode('utf-8', errors='ignore'))
+                break
+            else:
+                texts.append(text)
+                total_bytes += len(encoded)
+        data = "".join(texts)
+        data = "".join(data.splitlines())
+    finally:
+        if reader is not None:
+            reader.close()
+
+    return data
+
+
+def extract_text_with_tika(filepath, max_size):
+    """Read non-PDF file and extract text.
+
+    Args:
+        filepath (str): Path to the PDF file.
+        max_size (int): Maximum size of the extracted text in bytes.
+
+    Raises:
+        Exception: If Tika jar file does not exist.
+        Exception: If Tika processing fails.
+
+    Returns:
+        str: Extracted text from the non-PDF file.
+    """
+    tika_jar_path = os.environ.get("TIKA_JAR_FILE_PATH")
+    if not tika_jar_path or os.path.isfile(tika_jar_path) is False:
+        raise Exception("not exist tika jar file.")
+    args = ["java", "-jar", tika_jar_path, "-t", filepath]
+    result = subprocess.run(
+        args,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+    )
+    if result.returncode != 0:
+        raise Exception("raise in tika: {}".format(result.stderr.decode("utf-8")))
+    data = "".join(result.stdout.decode("utf-8").splitlines())
+    if len(data.encode('utf-8')) > max_size:
+        encoded = data.encode('utf-8')
+        data = encoded[:max_size].decode('utf-8', errors='ignore')
+
+    return data

modules/weko-records-ui/weko_records_ui/templates/weko_records_ui/output_detail_data.html

@@ -72,7 +72,7 @@
             {%- endfor -%}
         {%- else -%}
             {{ output_attribute_value_mlt_Init( record_detail_data ) }}
-        {%- endif -%}
+        {%- endif -%}        
     {%- endif -%}
 {% endmacro %}
 
@@ -83,7 +83,7 @@
             {% if parrent_name %}
                 {%- set labels = parrent_name.split('.') -%}
                 {%- if labels|length == 1 -%}
-                    {{ child_data(parrent_name, '', level) }}
+                    {{ child_data(parrent_name, '　', level) }}
                 {%- else -%}
                     {%- set displayflag = False -%}
                 {%- endif -%}
@@ -118,7 +118,7 @@
                 <td><a href="{{content | encode_filename }}">{{ content | escape_str }}</a></td>
             {%- elif content|url_to_link -%}
                 <td><a href="{{content}}" target='_blank'>{{ content | escape_str }}</a></td>
-	          {%- else -%}
+            {%- else -%}
                 <td class="multiple-line">{{ content | escape_str }}</td>
             {%- endif -%}
         {% endautoescape %}
@@ -207,7 +207,7 @@
                         {%- set nsflg.dispflg = True -%}
                     {%- endif -%}
                 {%- endfor -%}
-            {%- endif -%}
+            {%- endif -%}        
         {%- endif -%}
     {%- endfor -%}
     {%- for language_value in language_data -%}
@@ -245,7 +245,7 @@
                             {%- if value is string -%}
                                 {%- if attribute_data|length == 2 -%}
                                     {{ output_attribute_value_mlt(attribute_data, level) }}
-                                {%- else -%}
+                                {%- else -%}    
                                     {{ output_attribute_value_mlt_exceptlang(attribute_data,level) }}
                                 {%- endif -%}
                             {%- else -%}

modules/weko-search-ui/requirements2.txt

@@ -287,3 +287,4 @@ xmlschema==0.9.30
 xmltodict==0.12.0
 zipp==3.6.0
 zope.interface==5.5.2
+pypdfium2==4.30.0

modules/weko-search-ui/tests/conftest.py

@@ -136,7 +136,13 @@
 from weko_deposit.api import WekoDeposit
 from weko_deposit.api import WekoDeposit as aWekoDeposit
 from weko_deposit.api import WekoIndexer, WekoRecord
-from weko_deposit.config import WEKO_BUCKET_QUOTA_SIZE, WEKO_MAX_FILE_SIZE
+from weko_deposit.config import (
+    WEKO_BUCKET_QUOTA_SIZE,
+    WEKO_MAX_FILE_SIZE,
+    WEKO_DEPOSIT_FILESIZE_LIMIT,
+    WEKO_MIMETYPE_WHITELIST_FOR_ES,
+    WEKO_DEPOSIT_TEXTMIMETYPE_WHITELIST_FOR_ES
+)
 from weko_groups import WekoGroups
 from weko_index_tree import WekoIndexTree, WekoIndexTreeREST
 from weko_index_tree.api import Indexes
@@ -694,7 +700,10 @@ def base_app(instance_path, search_class, request):
         WEKO_ITEMS_UI_INDEX_PATH_SPLIT = '///',
         WEKO_SEARCH_UI_BULK_EXPORT_RETRY = 5,
         WEKO_SEARCH_UI_BULK_EXPORT_LIMIT = 100,
-        RECORDS_UI_ENDPOINTS = RECORDS_UI_ENDPOINTS
+        RECORDS_UI_ENDPOINTS = RECORDS_UI_ENDPOINTS,
+        WEKO_DEPOSIT_FILESIZE_LIMIT = WEKO_DEPOSIT_FILESIZE_LIMIT,
+        WEKO_MIMETYPE_WHITELIST_FOR_ES = WEKO_MIMETYPE_WHITELIST_FOR_ES,
+        WEKO_DEPOSIT_TEXTMIMETYPE_WHITELIST_FOR_ES = WEKO_DEPOSIT_TEXTMIMETYPE_WHITELIST_FOR_ES
     )
     app_.url_map.converters["pid"] = PIDConverter
     app_.config["RECORDS_REST_ENDPOINTS"]["recid"]["search_class"] = search_class

modules/weko-search-ui/tests/data/ams/sample.txt

@@ -0,0 +1,2 @@
+This is a
+text file.