Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 18 additions & 3 deletions ace/sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -550,6 +550,23 @@ def decode_html_entities(self, html):
else:
return html

def _page_url(self, soup):
"""The URL the page was served from, as the page itself states it:
<base href>, then the canonical link, then the publisher's URL metas.
Relative links on the page are resolved against it."""
base = soup.find('base', href=True)
if base:
return base['href']
canonical = soup.find('link', rel='canonical', href=True)
if canonical:
return canonical['href']
for attrs in ({'property': 'og:url'},
{'name': 'citation_fulltext_html_url'}):
meta = soup.find('meta', attrs=attrs)
if meta and meta.get('content'):
return meta['content']
return None

def _download_table(self, url):
''' For Sources that have tables in separate files, a helper for
downloading and extracting the table data. Also saves to file if desired.
Expand Down Expand Up @@ -2020,9 +2037,7 @@ def parse_article(self, html, pmid=None, **kwargs):
return False

# Extract table; going to take the approach of opening and parsing the table via links
# To download tables, we need the content URL and the number of tables
content_meta = soup.find('meta', {'name': 'citation_fulltext_html_url'})
content_url = content_meta['content'] if content_meta else None
content_url = self._page_url(soup)

# Find all links that match the Nature table URL pattern.
table_links = soup.find_all('a', href=re.compile(r'/article(s?)/.*/tables/\d+'))
Expand Down
Loading
Loading