Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
76 changes: 41 additions & 35 deletions ace/scrape.py
Original file line number Diff line number Diff line change
Expand Up @@ -278,29 +278,6 @@ def parse_PMID_xml(xml):
}


def _is_pubmed_landing_page(url):
""" True for the PubMed abstract page, which is what we open when an
article has no LinkOut record. It is not full text. """

return re.match(r'https?://pubmed\.ncbi\.nlm\.nih\.gov/\d+/?$', url or '') is not None


def _extract_citation_doi(html):
""" Pull the DOI out of a page's citation_doi meta tag. """

if not html:
return None

for tag in re.findall(r'(?is)<meta\b[^>]*>', html):
if not re.search(r'''(?i)name\s*=\s*["']?citation_doi["']?''', tag):
continue
match = re.search(r'''(?i)content\s*=\s*["']([^"']+)["']''', tag)
if match:
return match.group(1).strip()

return None


def _plos_doi_from_url(url):
""" Pull the DOI out of a journals.plos.org article URL, if it is one. """

Expand Down Expand Up @@ -589,17 +566,43 @@ def _providers_from_elink_json(json_content):
return [(obj.get('provider', {}).get('nameabbr'), obj['url']['value']) for obj in objurls]

@staticmethod
def _prlinks_target(pmid, providers):
def _prlinks_target(providers):
""" The page that elink's cmd=prlinks&retmode=ref redirects to: the
first LinkOut provider that isn't PMC, else PMC, else the PubMed
abstract page. """
first LinkOut provider that isn't PMC, else PMC. """

for name, url in providers:
if name != 'PMC':
return url
if providers:
return providers[0][1]
return f"https://pubmed.ncbi.nlm.nih.gov/{pmid}/"
return providers[0][1]

def _plos_url_without_linkout(self, pmid):
""" The doi.org URL of a PLoS article that has no LinkOut record, from
its keyed esummary, or None (logged) for any other article. """

try:
response = self._client.get(
'esummary',
params={'db': 'pubmed', 'id': pmid, 'retmode': 'json'},
return_content=False,
)
summary = response.json()['result'][str(pmid)]
# An unknown PMID comes back as HTTP 200 with only an error.
if 'error' in summary:
raise ValueError(summary['error'])
doi = next((a['value'] for a in summary['articleids'] if a['idtype'] == 'doi'), None)
except Exception as e:
logger.error(f"E-utilities summary failed for PMID {pmid}: {self._without_api_key(e)}")
return None

# The caller's journal can be a label of its own, and PLOS_JOURNAL_SITES
# doesn't list every PLOS title. Every PLOS journal DOI has this prefix.
if doi is None or not doi.startswith('10.1371/journal.'):
logger.info("\tNo LinkOut record for PMID %s and not a PLoS article. Skipping...", pmid)
return None
# doi.org lands on the journals.plos.org article page, as LinkOut does.
# The manuscript URL is a download, which the browser saves to disk
# instead of loading.
return f"https://doi.org/{doi}"

def _without_api_key(self, text):
# requests puts the full query string, key included, in its errors.
Expand Down Expand Up @@ -645,7 +648,15 @@ def get_html_by_pmid(self, pmid, journal, mode='browser', prefer_pmc_source=True
)
return

url = self._prlinks_target(pmid, providers)
if providers:
url = self._prlinks_target(providers)
else:
# Without LinkOut, prlinks lands on the PubMed abstract page. That
# is never full text, and NCBI blocks scripts that browse PubMed,
# so only a PLoS article, addressed by its DOI, is opened.
url = self._plos_url_without_linkout(pmid)
if url is None:
return None
logger.info(url)
return self.get_html(
url,
Expand All @@ -666,11 +677,6 @@ def check_for_substitute_url(self, url, html, journal):
try:
if j in PLOS_JOURNAL_SITES:
doi = _plos_doi_from_url(url)
if doi is None and _is_pubmed_landing_page(url):
# E-utilities had no LinkOut for this article, so we
# opened the PubMed abstract page. Its citation_doi
# meta tag is enough to address the full text ourselves.
doi = _extract_citation_doi(html)
if doi is None:
return url
# The legacy 'asset?id=<doi>.XML' endpoint still works, but only
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,134 @@
interactions:
- request:
body: null
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
User-Agent:
- Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like
Gecko) Chrome/124.0.6367.155 Safari/537.36
method: GET
uri: https://eutils.ncbi.nlm.nih.gov/entrez/eutils/elink.fcgi?cmd=prlinks&dbfrom=pubmed&id=99999999&retmode=json
response:
body:
string: '{"header":{"type":"elink","version":"0.3"},"linksets":[{"dbfrom":"pubmed","idurllist":[{"id":"99999999","objurls":[],"info":"No
LinkOut links available for UID: 99999999"}]}]}

'
headers:
Access-Control-Allow-Origin:
- '*'
Access-Control-Expose-Headers:
- X-RateLimit-Limit,X-RateLimit-Remaining
Alt-Svc:
- clear
Cache-Control:
- private
Content-Security-Policy:
- upgrade-insecure-requests
Content-Type:
- application/json; charset=UTF-8
Date:
- Wed, 14 Oct 2026 13:44:10 GMT
Ncbi-Phid:
- 1D322F91B7495C35000028FBE97CA40D.1.1.m_1
Ncbi-Sid:
- 6C918CFA1EBE24E1_28C1SID
Referrer-Policy:
- origin-when-cross-origin
Server:
- Finatra
Set-Cookie:
- ncbi_sid=6C918CFA1EBE24E1_28C1SID; domain=.nih.gov; path=/; expires=Thu, 14
Oct 2027 13:44:10 GMT
Strict-Transport-Security:
- max-age=31536000; includeSubDomains; preload
Transfer-Encoding:
- chunked
Via:
- 1.1 google
X-Ratelimit-Limit:
- '10'
X-Ratelimit-Remaining:
- '9'
X-Ua-Compatible:
- IE=Edge
X-Xss-Protection:
- 1; mode=block
content-length:
- '176'
status:
code: 200
message: OK
- request:
body: null
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Cookie:
- ncbi_sid=6C918CFA1EBE24E1_28C1SID
User-Agent:
- Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like
Gecko) Chrome/124.0.6367.155 Safari/537.36
method: GET
uri: https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id=99999999&retmode=json
response:
body:
string: '{"header":{"type":"esummary","version":"0.3"},"result":{"uids":["99999999"],"99999999":{"uid":"99999999","error":"cannot
get document summary"}}}

'
headers:
Access-Control-Allow-Origin:
- '*'
Access-Control-Expose-Headers:
- X-RateLimit-Limit,X-RateLimit-Remaining
Alt-Svc:
- clear
Cache-Control:
- private
Content-Security-Policy:
- upgrade-insecure-requests
Content-Type:
- application/json; charset=UTF-8
Date:
- Wed, 14 Oct 2026 13:44:10 GMT
Ncbi-Phid:
- 1D3407F95CD407A500003D0661614ABB.1.1.m_1
Ncbi-Sid:
- 6C918CFA1EBE24E1_28C1SID
Referrer-Policy:
- origin-when-cross-origin
Server:
- Finatra
Set-Cookie:
- ncbi_sid=6C918CFA1EBE24E1_28C1SID; domain=.nih.gov; path=/; expires=Thu, 14
Oct 2027 13:44:10 GMT
Strict-Transport-Security:
- max-age=31536000; includeSubDomains; preload
Transfer-Encoding:
- chunked
Via:
- 1.1 google
X-Ratelimit-Limit:
- '10'
X-Ratelimit-Remaining:
- '8'
X-Ua-Compatible:
- IE=Edge
X-Xss-Protection:
- 1; mode=block
content-length:
- '146'
status:
code: 200
message: OK
version: 1
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
interactions:
- request:
body: null
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
User-Agent:
- Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML,
like Gecko) Chrome/123.0.6312.86 Safari/537.36
method: GET
uri: https://eutils.ncbi.nlm.nih.gov/entrez/eutils/elink.fcgi?cmd=prlinks&dbfrom=pubmed&id=42869849&retmode=json
response:
body:
string: '{"header":{"type":"elink","version":"0.3"},"linksets":[{"dbfrom":"pubmed","idurllist":[{"id":"42869849","objurls":[],"info":"No
LinkOut links available for UID: 42869849"}]}]}

'
headers:
Access-Control-Allow-Origin:
- '*'
Access-Control-Expose-Headers:
- X-RateLimit-Limit,X-RateLimit-Remaining
Alt-Svc:
- clear
Cache-Control:
- private
Content-Security-Policy:
- upgrade-insecure-requests
Content-Type:
- application/json; charset=UTF-8
Date:
- Wed, 14 Oct 2026 13:05:21 GMT
Ncbi-Phid:
- 1D33C7E9FE8B2C65000056F4C5D7A4C5.1.1.m_1
Ncbi-Sid:
- DC69A73962DDC2FA_F265SID
Referrer-Policy:
- origin-when-cross-origin
Server:
- Finatra
Set-Cookie:
- ncbi_sid=DC69A73962DDC2FA_F265SID; domain=.nih.gov; path=/; expires=Thu, 14
Oct 2027 13:05:21 GMT
Strict-Transport-Security:
- max-age=31536000; includeSubDomains; preload
Transfer-Encoding:
- chunked
Via:
- 1.1 google
X-Ratelimit-Limit:
- '10'
X-Ratelimit-Remaining:
- '9'
X-Ua-Compatible:
- IE=Edge
X-Xss-Protection:
- 1; mode=block
content-length:
- '176'
status:
code: 200
message: OK
- request:
body: null
headers:
Accept:
- '*/*'
Accept-Encoding:
- gzip, deflate
Connection:
- keep-alive
Cookie:
- ncbi_sid=DC69A73962DDC2FA_F265SID
User-Agent:
- Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML,
like Gecko) Chrome/123.0.6312.86 Safari/537.36
method: GET
uri: https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esummary.fcgi?db=pubmed&id=42869849&retmode=json
response:
body:
string: '{"header":{"type":"esummary","version":"0.3"},"result":{"uids":["42869849"],"42869849":{"uid":"42869849","pubdate":"2026
Oct","epubdate":"","source":"ACR Open Rheumatol","authors":[{"name":"Lerkvaleekul
B","authtype":"Author","clusterid":""},{"name":"Sirachainan N","authtype":"Author","clusterid":""},{"name":"Songdej
D","authtype":"Author","clusterid":""},{"name":"Paisooksantivatana K","authtype":"Author","clusterid":""},{"name":"Chinudomwong
P","authtype":"Author","clusterid":""},{"name":"Vilaiyuk S","authtype":"Author","clusterid":""}],"lastauthor":"Vilaiyuk
S","title":"The Usefulness of White Blood Cell Inflammatory Parameters in
Children With Systemic Lupus Erythematosus.","sorttitle":"usefulness of white
blood cell inflammatory parameters in children with systemic lupus erythematosus","volume":"8","issue":"10","pages":"e90152","lang":["eng"],"nlmuniqueid":"101740025","issn":"","essn":"2578-5745","pubtype":["Journal
Article"],"recordstatus":"PubMed","pubstatus":"4","articleids":[{"idtype":"pubmed","idtypen":1,"value":"42869849"},{"idtype":"doi","idtypen":3,"value":"10.1002/acr2.90152"}],"history":[{"pubstatus":"revised","date":"2026/08/18
00:00"},{"pubstatus":"received","date":"2026/05/15 00:00"},{"pubstatus":"accepted","date":"2026/09/01
00:00"},{"pubstatus":"medline","date":"2026/10/13 23:37"},{"pubstatus":"pubmed","date":"2026/10/13
23:36"},{"pubstatus":"entrez","date":"2026/10/13 22:53"}],"references":[],"attributes":["Has
Abstract"],"pmcrefcount":"","fulljournalname":"ACR open rheumatology","elocationid":"doi:
10.1002/acr2.90152","doctype":"citation","srccontriblist":[],"booktitle":"","medium":"","edition":"","publisherlocation":"","publishername":"","srcdate":"","reportnumber":"","availablefromurl":"","locationlabel":"","doccontriblist":[],"docdate":"","bookname":"","chapter":"","sortpubdate":"2026/10/01
00:00","sortfirstauthor":"Lerkvaleekul B","vernaculartitle":""}}}

'
headers:
Access-Control-Allow-Origin:
- '*'
Access-Control-Expose-Headers:
- X-RateLimit-Limit,X-RateLimit-Remaining
Alt-Svc:
- clear
Cache-Control:
- private
Content-Security-Policy:
- upgrade-insecure-requests
Content-Type:
- application/json; charset=UTF-8
Date:
- Wed, 14 Oct 2026 13:05:20 GMT
Ncbi-Phid:
- 1D33C7E9FE8B2C6500002EF4C66B1DAF.1.1.m_1
Ncbi-Sid:
- DC69A73962DDC2FA_F265SID
Referrer-Policy:
- origin-when-cross-origin
Server:
- Finatra
Set-Cookie:
- ncbi_sid=DC69A73962DDC2FA_F265SID; domain=.nih.gov; path=/; expires=Thu, 14
Oct 2027 13:05:21 GMT
Strict-Transport-Security:
- max-age=31536000; includeSubDomains; preload
Transfer-Encoding:
- chunked
Via:
- 1.1 google
X-Ratelimit-Limit:
- '10'
X-Ratelimit-Remaining:
- '8'
X-Ua-Compatible:
- IE=Edge
X-Xss-Protection:
- 1; mode=block
content-length:
- '1912'
status:
code: 200
message: OK
version: 1
Loading
Loading