Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion ace/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -40,4 +40,4 @@ def _setup_logger(logger):

# Set up logger
logger = logging.getLogger("ace")
_setup_logger(logger)
_setup_logger(logger)
4 changes: 2 additions & 2 deletions ace/export.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def export_database(db, foldername, skip_empty=True, table_html=False):

for t in art.tables:
# Prepare table data row
table_foot = t.footnotes if hasattr(t, 'footnotes') else ''
table_foot = getattr(t, 'notes', '') or ''
n_header_rows = t.header_rows if hasattr(t, 'header_rows') else 1
table_raw_file = (
f"tables/{art.id}/{t.id}.html"
Expand Down Expand Up @@ -175,4 +175,4 @@ def export_database(db, foldername, skip_empty=True, table_html=False):
art_dir.mkdir(parents=True, exist_ok=True)
table_file = art_dir / f"{t.id}.html"
with table_file.open('w', encoding='utf-8') as f:
f.write(t.input_html)
f.write(t.input_html)
71 changes: 71 additions & 0 deletions ace/ingest.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
import multiprocessing as mp
from functools import partial
from tqdm import tqdm
from pathlib import Path
from tempfile import TemporaryDirectory

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -233,3 +235,72 @@ def add_articles(db, files, commit=True, table_dir=None, limit=None,
db.save()

return missing_sources


def extract_and_export(
files,
output_dir,
*,
pmid_filenames=True,
skip_metadata=True,
num_workers=1,
use_readability=False,
):
"""Extract ACE tables and coordinates from HTML files and export them.

The extraction database is temporary. ``tables.csv`` contains every table
accepted by ACE's candidate gate, including tables for which ACE's
deterministic parser extracted no coordinates.
"""
from . import config, database, export

output_dir = Path(output_dir)
file_paths = [str(Path(file_path)) for file_path in files]
previous_config = {
"SAVE_ORIGINAL_HTML": get_config("SAVE_ORIGINAL_HTML"),
"SAVE_ARTICLES_WITHOUT_ACTIVATIONS": get_config(
"SAVE_ARTICLES_WITHOUT_ACTIVATIONS"
),
}

with TemporaryDirectory(prefix="ace-extract-") as temp_dir:
db_path = Path(temp_dir) / "ace.sqlite"
db = database.Database(
adapter="sqlite",
db_name=f"sqlite:///{db_path}",
)
try:
config.update_config(
SAVE_ORIGINAL_HTML=True,
SAVE_ARTICLES_WITHOUT_ACTIVATIONS=True,
)
missing_sources = add_articles(
db,
file_paths,
pmid_filenames=pmid_filenames,
force_ingest=True,
num_workers=num_workers,
skip_metadata=skip_metadata,
use_readability=use_readability,
expand_linked_tables=True,
)

article_count = db.session.query(database.Article).count()
table_count = db.session.query(database.Table).count()
coordinate_count = db.session.query(database.Activation).count()
export.export_database(
db,
output_dir,
skip_empty=False,
table_html=True,
)
finally:
db.session.close()
config.update_config(**previous_config)

return {
"articles": article_count,
"tables": table_count,
"coordinates": coordinate_count,
"missing_sources": [str(path) for path in missing_sources],
}
27 changes: 27 additions & 0 deletions ace/scrape.py
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,33 @@ def exponential_backoff(retries):
logger.error("Exceeded maximum number of retries.")
return None


def get_dynamic_html(url, reconnect_time=8, settle_time=3):
"""Load a JavaScript-protected page with ACE's browser stack.

This is reserved for publisher pages whose saved HTML contains linked
table placeholders rather than the table payload.
"""
try:
with SB(uc=True, xvfb=True) as browser:
browser.uc_open_with_reconnect(
url,
reconnect_time=reconnect_time,
)
html = browser.get_page_source()
if not _validate_scrape(html):
try:
browser.uc_gui_click_captcha()
except Exception:
pass
browser.sleep(settle_time)
html = browser.get_page_source()
return html if _validate_scrape(html) else None
except Exception as exc:
logger.warning("Dynamic retrieval failed for %s: %s", url, exc)
return None


def _convert_pmid_to_pmc(pmids):
url_template = "https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/?ids="
logger.info("Converting PMIDs to PMCIDs...")
Expand Down
Loading
Loading