Skip to content

feat: ワークスペース登録で arXiv ID からの自動補完に対応 #1947

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mhaya wants to merge 1 commit into
base: develop_v2.1.0
Choose a base branch
from
Open

feat: ワークスペース登録で arXiv ID からの自動補完に対応 #1947

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
59 changes: 58 additions & 1 deletion modules/weko-workspace/tests/test_api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from requests.exceptions import RequestException
from requests.models import Response

from weko_workspace.api import JamasURL, CiNiiURL, JALCURL, DATACITEURL, arXivURL
from weko_workspace.api import JamasURL, CiNiiURL, JALCURL, DATACITEURL, arXivURL, extract_arxiv_id


# class JamasURL:
Expand Down Expand Up @@ -471,6 +471,42 @@ def test_get_data(self, app):
result = datacite.get_data()
assert result == {"response":"","error":"request error"}

# def extract_arxiv_id(value):
# .tox/c1/bin/pytest --cov=weko_workspace tests/test_api.py::test_extract_arxiv_id -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-workspace/.tox/c1/tmp
@pytest.mark.parametrize("value, expected", [
# new style
("2301.00001", "2301.00001"),
("0704.0001", "0704.0001"),
("2301.00001v2", "2301.00001v2"),
(" 2301.00001 ", "2301.00001"),
("arXiv:2301.00001", "2301.00001"),
("ARXIV:2301.00001v1", "2301.00001v1"),
# old style
("hep-th/9901001", "hep-th/9901001"),
("math.AG/0601001", "math.AG/0601001"),
("arXiv:hep-th/9901001v3", "hep-th/9901001v3"),
# URL
("https://arxiv.org/abs/2301.00001", "2301.00001"),
("http://arxiv.org/abs/2301.00001v2/", "2301.00001v2"),
("https://arxiv.org/pdf/2301.00001.pdf", "2301.00001"),
("https://export.arxiv.org/abs/hep-th/9901001", "hep-th/9901001"),
# arXiv DOI
("10.48550/arXiv.2301.00001", "2301.00001"),
("https://doi.org/10.48550/arXiv.2301.00001", "2301.00001"),
# not arXiv ID
("10.5109/16119", None),
("https://doi.org/10.5109/16119", None),
("10.1234/5678901", None),
("2301.001", None),
("23010.00001", None),
("https://example.org/abs/2301.00001", None),
("", None),
(None, None),
])
def test_extract_arxiv_id(value, expected):
assert extract_arxiv_id(value) == expected


# class arXivURL:
class TestarXivURL:
# def __init__(self, naid, timeout=None, http_proxy=None, https_proxy=None):
Expand All @@ -492,6 +528,11 @@ def test__init__(self):
assert arXiv._doi == "10.5109/16119"
assert arXiv._timeout == 10
assert arXiv._proxy == {"http":"test_http_proxy","https":"test_https_proxy"}
assert arXiv._arxiv_id is None

# arXiv ID
arXiv = arXivURL("arXiv:2301.00001v2")
assert arXiv._arxiv_id == "2301.00001v2"


# def _create_endpoint(self):
Expand All @@ -501,6 +542,10 @@ def test_create_endpoint(self):
result = arXiv._create_endpoint()
assert result == "10.5109/16119"

arXiv = arXivURL("hep-th/9901001")
result = arXiv._create_endpoint()
assert result == "hep-th/9901001"


# def _create_url(self):
# .tox/c1/bin/pytest --cov=weko_workspace tests/test_api.py::TestarXivURL::test_create_url -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-workspace/.tox/c1/tmp
Expand All @@ -509,6 +554,14 @@ def test_create_url(self):
result = arXiv._create_url()
assert result == "https://export.arxiv.org/api/query?search_query=doi:10.5109/16119"

arXiv = arXivURL("2301.00001")
result = arXiv._create_url()
assert result == "https://export.arxiv.org/api/query?id_list=2301.00001"

arXiv = arXivURL("https://doi.org/10.48550/arXiv.2301.00001")
result = arXiv._create_url()
assert result == "https://export.arxiv.org/api/query?id_list=2301.00001"


# def url(self):
# .tox/c1/bin/pytest --cov=weko_workspace tests/test_api.py::TestarXivURL::test_url -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-workspace/.tox/c1/tmp
Expand All @@ -517,6 +570,10 @@ def test_url(self):
result = arXiv.url
assert result == "https://export.arxiv.org/api/query?search_query=doi:10.5109/16119"

arXiv = arXivURL("hep-th/9901001v1")
result = arXiv.url
assert result == "https://export.arxiv.org/api/query?id_list=hep-th/9901001v1"


# def _do_http_request(self):
# .tox/c1/bin/pytest --cov=weko_workspace tests/test_api.py::TestarXivURL::test_do_http_request -vv -s --cov-branch --cov-report=term --basetemp=/code/modules/weko-workspace/.tox/c1/tmp
Expand Down
30 changes: 29 additions & 1 deletion modules/weko-workspace/tests/test_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3630,7 +3630,7 @@ def test_get_datacite_autofill_item(app):
# .tox/c1/bin/pytest tests/test_utils.py::test_get_arXiv_record_data -vv -s --cov-branch --cov=weko_workspace --cov-report=term --basetemp=/code/modules/weko-workspace/tests/.tox/c1/tmp
def test_get_arXiv_record_data(app, mocker_itemtype):
mock_get_data = {
'response': {"title": [{"@value": "test title", "@language": "ja"}]},
'response': {"feed": {"entry": {"id": "http://arxiv.org/abs/2301.00001v1", "title": "test title"}}},
'error': ''
}
with patch('weko_workspace.utils.arXivURL.get_data', return_value=mock_get_data):
Expand Down Expand Up @@ -3677,6 +3677,34 @@ def test_get_arXiv_record_data(app, mocker_itemtype):
current_cache.delete('arXiv_data10.1234/test-doi1')
result = get_arXiv_record_data('10.1234/test-doi', 1)
assert result == []

# Test with no entry (no hit), error entry (invalid ID) and empty entry list
for response in [
{"feed": {"title": "arXiv Query"}},
{"feed": None},
{},
{"feed": {"entry": {"id": "http://arxiv.org/api/errors#incorrect_id_format_for_abc", "title": "Error"}}},
{"feed": {"entry": []}},
]:
with patch('weko_workspace.utils.arXivURL.get_data', return_value={'response': response, 'error': ''}):

@sourcery-ai sourcery-ai Bot Oct 25, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚪ Low · Added tests fail Black formatting

When the module's black . formatting check runs, black reformats these newly added long patch and fixture lines, so the module's formatting check reports findings instead of passing.

Reformat the added test expressions with Black.

Also at modules/weko-workspace/tests/test_utils.py:3697.

Prompt for AI agents 
In `modules/weko-workspace/tests/test_utils.py` at line 3689:

**Added tests fail Black formatting**

When the module's `black .` formatting check runs, black reformats these newly added long patch and fixture lines, so the module's formatting check reports findings instead of passing.

Reformat the added test expressions with Black.

Also at `modules/weko-workspace/tests/test_utils.py:3697`.

From AGENTS.md L70-73:

- フォーマッター: **Black** を使用（`black .` でソースコードを整形）
- リンター: **Flake8** を使用（`flake8` で静的解析チェック）
- インポート順の整理: **isort** を使用（`isort .` でインポート並び替え）
- これらのフォーマットチェックはコミット前に必ず実行し、指摘がない状態にしてください

with patch('weko_workspace.utils.get_arXiv_data_by_key') as mock_by_key:
current_cache.delete('arXiv_data2301.000011')
result = get_arXiv_record_data('2301.00001', 1)
assert result == []
mock_by_key.assert_not_called()

# Test with multiple entries: use the first entry
response = {"feed": {"entry": [{"id": "http://arxiv.org/abs/2301.00001v1"}, {"id": "http://arxiv.org/abs/2301.00002v1"}]}}
with patch('weko_workspace.utils.arXivURL.get_data', return_value={'response': response, 'error': ''}):
with patch('weko_workspace.utils.get_arXiv_data_by_key', return_value={}) as mock_by_key:
with patch('weko_workspace.utils.get_arXiv_autofill_item'):
with patch('weko_workspace.utils.get_autofill_key_tree'):
with patch('weko_workspace.utils.build_record_model', return_value=[1]):
current_cache.delete('arXiv_data10.1234/test-doi1')
result = get_arXiv_record_data('10.1234/test-doi', 1)
assert result == [1]
api = mock_by_key.call_args[0][0]
assert api['response']['feed']['entry'] == {"id": "http://arxiv.org/abs/2301.00001v1"}

# .tox/c1/bin/pytest tests/test_utils.py::test_get_arXiv_data_by_key -vv -s --cov-branch --cov=weko_workspace --cov-report=term --basetemp=/code/modules/weko-workspace/tests/.tox/c1/tmp
@pytest.mark.parametrize('keyword, expected_result', [
Expand Down
42 changes: 42 additions & 0 deletions modules/weko-workspace/weko_workspace/api.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

"""WEKO3 module docstring."""

import re
import traceback
import requests
import urllib.parse
Expand Down Expand Up @@ -425,6 +426,42 @@ def get_data(self):
response['error'] = str(e)
return response


ARXIV_ID_PATTERN = re.compile(
r'^(?:arxiv:)?'
r'(\d{4}\.\d{4,5}' # new style: 2301.00001
r'|[a-z][a-z-]*(?:\.[a-z]{2})?/\d{7})' # old style: hep-th/9901001
r'(?:v\d+)?$',
re.IGNORECASE)
"""arXiv identifier pattern (with optional "arXiv:" prefix and version)."""

ARXIV_PREFIX_PATTERN = re.compile(
r'^(?:https?://(?:www\.|export\.)?arxiv\.org/(?:abs|pdf)/'
r'|(?:https?://(?:dx\.)?doi\.org/)?10\.48550/arxiv\.)',
re.IGNORECASE)
"""arXiv abstract/PDF URL or arXiv DOI (10.48550/arXiv.) prefix."""


def extract_arxiv_id(value):
"""Extract arXiv identifier from input value.

Accepts a bare arXiv ID (e.g. "2301.00001", "hep-th/9901001"),
"arXiv:" prefixed ID, arxiv.org abs/pdf URL and arXiv DOI
(10.48550/arXiv.xxxx).

:param value: input value.
:return: arXiv identifier, or None if value is not an arXiv ID.
"""
if not isinstance(value, str):
return None
value = ARXIV_PREFIX_PATTERN.sub('', value.strip().rstrip('/'))

@sourcery-ai sourcery-ai Bot Oct 25, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Medium · Valid arXiv URLs fail autofill

When a user pastes an arXiv abstract or PDF URL containing a query string or fragment, extract_arxiv_id removes the URL prefix but leaves the query string or fragment attached, so ARXIV_ID_PATTERN rejects the identifier and the lookup falls back to DOI search without finding arXiv metadata.

Remove the query string or fragment before validating the extracted arXiv identifier.

Also at modules/weko-workspace/weko_workspace/api.py:438-442.

Prompt for AI agents 
In `modules/weko-workspace/weko_workspace/api.py` at line 457:

**Valid arXiv URLs fail autofill**

When a user pastes an arXiv abstract or PDF URL containing a query string or fragment, `extract_arxiv_id` removes the URL prefix but leaves the query string or fragment attached, so `ARXIV_ID_PATTERN` rejects the identifier and the lookup falls back to DOI search without finding arXiv metadata.

Remove the query string or fragment before validating the extracted arXiv identifier.

Also at `modules/weko-workspace/weko_workspace/api.py:438-442`.

if value.lower().endswith('.pdf'):
value = value[:-4]
if not ARXIV_ID_PATTERN.match(value):
return None
return re.sub(r'^arxiv:', '', value, flags=re.IGNORECASE)


class arXivURL:
"""The Class retrieves the metadata from arXiv."""

Expand All @@ -443,6 +480,7 @@ def __init__(self, doi, timeout=None, http_proxy=None, https_proxy=None):
"""
if not doi:
raise ValueError('DOI is required.')
self._arxiv_id = extract_arxiv_id(doi)

@sourcery-ai sourcery-ai Bot Oct 25, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Medium · Cached lookups return wrong metadata

When two ID/item-type combinations whose concatenated values are identical are requested within the cache timeout, cached_api_json builds its cache key by concatenating the ID and item-type ID without a separator, so get_arXiv_record_data can return the first lookup's cached record for the other combination and the caller receives incorrect metadata.

Build the cache key from an unambiguous representation of both the ID and item-type ID.

Prompt for AI agents 
In `modules/weko-workspace/weko_workspace/api.py` at line 483:

**Cached lookups return wrong metadata**

When two ID/item-type combinations whose concatenated values are identical are requested within the cache timeout, `cached_api_json` builds its cache key by concatenating the ID and item-type ID without a separator, so `get_arXiv_record_data` can return the first lookup's cached record for the other combination and the caller receives incorrect metadata.

Build the cache key from an unambiguous representation of both the ID and item-type ID.

@sourcery-ai sourcery-ai Bot Oct 25, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚪ Low · arXiv IDs remain documented as DOIs

arXivURL now accepts arXiv IDs as well as DOIs, but its constructor documentation still describes only a DOI parameter. Callers relying on that contract can overlook the newly supported input forms.

Update the constructor and get_arXiv_record_data docstrings to state that they accept either a DOI or an arXiv identifier/URL.

Prompt for AI agents 
In `modules/weko-workspace/weko_workspace/api.py` at line 483:

**arXiv IDs remain documented as DOIs**

`arXivURL` now accepts arXiv IDs as well as DOIs, but its constructor documentation still describes only a DOI parameter. Callers relying on that contract can overlook the newly supported input forms.

Update the constructor and `get_arXiv_record_data` docstrings to state that they accept either a DOI or an arXiv identifier/URL.

self._doi = "/".join(doi.strip().strip("/").split("/")[-2:])
if timeout:
self._timeout = timeout
Expand All @@ -456,6 +494,8 @@ def _create_endpoint(self):

:return: endpoint string.
"""
if self._arxiv_id:
return self._arxiv_id
endpoint_url = self._doi
return endpoint_url

Expand All @@ -467,6 +507,8 @@ def _create_url(self):
"""
endpoint = self._create_endpoint()

if self._arxiv_id:
return config.WEKO_WORKSPACE_ARXIV_ID_API_URL + endpoint
url = config.WEKO_WORKSPACE_ARXIV_API_URL + endpoint
return url

Expand Down
3 changes: 3 additions & 0 deletions modules/weko-workspace/weko_workspace/config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -318,6 +318,9 @@
WEKO_WORKSPACE_ARXIV_API_URL = 'https://export.arxiv.org/api/query?search_query=doi:'
"""arXiv API URL"""

WEKO_WORKSPACE_ARXIV_ID_API_URL = 'https://export.arxiv.org/api/query?id_list='
"""arXiv API URL for arXiv ID"""

WEKO_WORKSPACE_CINII_REQUIRED_ITEM = [
'title',
'creator',
Expand Down
2 changes: 1 addition & 1 deletion modules/weko-workspace/weko_workspace/static/js/weko_workspace/workspace_register.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2427,7 +2427,7 @@ function toObject(arr) {
}
// $('#meta-search').modal('show');
if (!$('#doiInput').val()) {
alert('please input the DOI');
alert('please input the DOI or arXiv ID');
$scope.isButtonDisabled = false;
return;
}
Expand Down
2 changes: 1 addition & 1 deletion modules/weko-workspace/weko_workspace/templates/weko_workspace/item_register.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,7 @@ <h3 class="panel-title" style="color: #000000; font-weight: bold;" class="col-sm
<div class="row col-sm-12">
<div class="col-sm-12">
<div class="col-sm-11">
<input type="text" id="doiInput" name="doiInput" placeholder="{{ _('Please enter the DOI.') }}"
<input type="text" id="doiInput" name="doiInput" placeholder="{{ _('Please enter the DOI or arXiv ID.') }}"
style="border: 2px solid #ccc; padding: 5px; width: 100%;" ng-model="doiInput">
</div>
<div class="col-sm-1">
Expand Down
Binary file modified modules/weko-workspace/weko_workspace/translations/en/LC_MESSAGES/messages.mo
View file
Open in desktop
Binary file not shown.
2 changes: 1 addition & 1 deletion modules/weko-workspace/weko_workspace/translations/en/LC_MESSAGES/messages.po
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,7 +279,7 @@ msgid "DOI Input"
msgstr ""

#: weko_workspace/templates/weko_workspace/item_register.html:130
msgid "Please enter the DOI."
msgid "Please enter the DOI or arXiv ID."
msgstr ""

#: weko_workspace/templates/weko_workspace/item_register.html:136
Expand Down
Binary file modified modules/weko-workspace/weko_workspace/translations/ja/LC_MESSAGES/messages.mo
View file
Open in desktop
Binary file not shown.
4 changes: 2 additions & 2 deletions modules/weko-workspace/weko_workspace/translations/ja/LC_MESSAGES/messages.po
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -279,8 +279,8 @@ msgid "DOI Input"
msgstr "DOI入力"

#: weko_workspace/templates/weko_workspace/item_register.html:130
msgid "Please enter the DOI."
msgstr "DOIを入力してください。"
msgid "Please enter the DOI or arXiv ID."
msgstr "DOIまたはarXiv IDを入力してください。"

#: weko_workspace/templates/weko_workspace/item_register.html:136
msgid "Get"
Expand Down
2 changes: 1 addition & 1 deletion modules/weko-workspace/weko_workspace/translations/messages.pot
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -278,7 +278,7 @@ msgid "DOI Input"
msgstr ""

#: weko_workspace/templates/weko_workspace/item_register.html:130
msgid "Please enter the DOI."
msgid "Please enter the DOI or arXiv ID."
msgstr ""

#: weko_workspace/templates/weko_workspace/item_register.html:136
Expand Down
9 changes: 9 additions & 0 deletions modules/weko-workspace/weko_workspace/utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2395,6 +2395,15 @@ def get_arXiv_record_data(doi, item_type_id, exclude_duplicate_lang=True):
if api_response["error"] \
or not isinstance(api_response['response'], dict):
return result
# No hit returns a feed without entry. Invalid ID returns an error entry.
feed = api_response['response'].get('feed') or {}
entry = feed.get('entry') if isinstance(feed, dict) else None
if isinstance(entry, list):
entry = entry[0] if entry else None
feed['entry'] = entry
if not isinstance(entry, dict) \
or '/api/errors' in str(entry.get('id', '')):
return result
Comment on lines +2404 to +2406

@sourcery-ai sourcery-ai Bot Oct 25, 2026

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🟡 Medium · API outage appears as no match

When the arXiv service responds with a non-200 status, arXivURL.get_data leaves response empty and error blank for non-200 statuses, so this guard returns []; the view then reports an ordinary no-result response instead of surfacing the service failure.

Log non-200 responses and set an error value so the view can distinguish service failures from genuine no-match results.

Prompt for AI agents 
In `modules/weko-workspace/weko_workspace/utils.py` at lines 2404-2406:

**API outage appears as no match**

When the arXiv service responds with a non-200 status, `arXivURL.get_data` leaves `response` empty and `error` blank for non-200 statuses, so this guard returns `[]`; the view then reports an ordinary no-result response instead of surfacing the service failure.

Log non-200 responses and set an error value so the view can distinguish service failures from genuine no-match results.

api_data = get_arXiv_data_by_key(api_response, 'all')
items = ItemTypes.get_by_id(item_type_id)
if items is None:
Expand Down
Loading