diff --git a/ace/__init__.py b/ace/__init__.py index 582c968..b635cfa 100644 --- a/ace/__init__.py +++ b/ace/__init__.py @@ -40,4 +40,4 @@ def _setup_logger(logger): # Set up logger logger = logging.getLogger("ace") -_setup_logger(logger) \ No newline at end of file +_setup_logger(logger) diff --git a/ace/export.py b/ace/export.py index b554809..77ba416 100644 --- a/ace/export.py +++ b/ace/export.py @@ -59,7 +59,7 @@ def export_database(db, foldername, skip_empty=True, table_html=False): for t in art.tables: # Prepare table data row - table_foot = t.footnotes if hasattr(t, 'footnotes') else '' + table_foot = getattr(t, 'notes', '') or '' n_header_rows = t.header_rows if hasattr(t, 'header_rows') else 1 table_raw_file = ( f"tables/{art.id}/{t.id}.html" @@ -175,4 +175,4 @@ def export_database(db, foldername, skip_empty=True, table_html=False): art_dir.mkdir(parents=True, exist_ok=True) table_file = art_dir / f"{t.id}.html" with table_file.open('w', encoding='utf-8') as f: - f.write(t.input_html) \ No newline at end of file + f.write(t.input_html) diff --git a/ace/ingest.py b/ace/ingest.py index fd85c9f..74d86d1 100644 --- a/ace/ingest.py +++ b/ace/ingest.py @@ -7,6 +7,8 @@ import multiprocessing as mp from functools import partial from tqdm import tqdm +from pathlib import Path +from tempfile import TemporaryDirectory logger = logging.getLogger(__name__) @@ -233,3 +235,72 @@ def add_articles(db, files, commit=True, table_dir=None, limit=None, db.save() return missing_sources + + +def extract_and_export( + files, + output_dir, + *, + pmid_filenames=True, + skip_metadata=True, + num_workers=1, + use_readability=False, +): + """Extract ACE tables and coordinates from HTML files and export them. + + The extraction database is temporary. ``tables.csv`` contains every table + accepted by ACE's candidate gate, including tables for which ACE's + deterministic parser extracted no coordinates. + """ + from . import config, database, export + + output_dir = Path(output_dir) + file_paths = [str(Path(file_path)) for file_path in files] + previous_config = { + "SAVE_ORIGINAL_HTML": get_config("SAVE_ORIGINAL_HTML"), + "SAVE_ARTICLES_WITHOUT_ACTIVATIONS": get_config( + "SAVE_ARTICLES_WITHOUT_ACTIVATIONS" + ), + } + + with TemporaryDirectory(prefix="ace-extract-") as temp_dir: + db_path = Path(temp_dir) / "ace.sqlite" + db = database.Database( + adapter="sqlite", + db_name=f"sqlite:///{db_path}", + ) + try: + config.update_config( + SAVE_ORIGINAL_HTML=True, + SAVE_ARTICLES_WITHOUT_ACTIVATIONS=True, + ) + missing_sources = add_articles( + db, + file_paths, + pmid_filenames=pmid_filenames, + force_ingest=True, + num_workers=num_workers, + skip_metadata=skip_metadata, + use_readability=use_readability, + expand_linked_tables=True, + ) + + article_count = db.session.query(database.Article).count() + table_count = db.session.query(database.Table).count() + coordinate_count = db.session.query(database.Activation).count() + export.export_database( + db, + output_dir, + skip_empty=False, + table_html=True, + ) + finally: + db.session.close() + config.update_config(**previous_config) + + return { + "articles": article_count, + "tables": table_count, + "coordinates": coordinate_count, + "missing_sources": [str(path) for path in missing_sources], + } diff --git a/ace/scrape.py b/ace/scrape.py index e22e353..00ca9c2 100644 --- a/ace/scrape.py +++ b/ace/scrape.py @@ -45,6 +45,33 @@ def exponential_backoff(retries): logger.error("Exceeded maximum number of retries.") return None + +def get_dynamic_html(url, reconnect_time=8, settle_time=3): + """Load a JavaScript-protected page with ACE's browser stack. + + This is reserved for publisher pages whose saved HTML contains linked + table placeholders rather than the table payload. + """ + try: + with SB(uc=True, xvfb=True) as browser: + browser.uc_open_with_reconnect( + url, + reconnect_time=reconnect_time, + ) + html = browser.get_page_source() + if not _validate_scrape(html): + try: + browser.uc_gui_click_captcha() + except Exception: + pass + browser.sleep(settle_time) + html = browser.get_page_source() + return html if _validate_scrape(html) else None + except Exception as exc: + logger.warning("Dynamic retrieval failed for %s: %s", url, exc) + return None + + def _convert_pmid_to_pmc(pmids): url_template = "https://www.ncbi.nlm.nih.gov/pmc/utils/idconv/v1.0/?ids=" logger.info("Converting PMIDs to PMCIDs...") diff --git a/ace/sources.py b/ace/sources.py index 4f97b0e..df68bf2 100644 --- a/ace/sources.py +++ b/ace/sources.py @@ -234,7 +234,14 @@ def __init__(self, config=None, table_dir=None, use_readability=True): else: self.entities.update(Source.ENTITIES) - def parse_article(self, html, pmid=None, metadata_dir=None, skip_metadata=False): + def parse_article( + self, + html, + pmid=None, + metadata_dir=None, + skip_metadata=False, + **kwargs, + ): ''' Takes HTML article as input and returns an Article. PMID Can also be passed, which prevents having to scrape it from the article and/or look it up in PubMed. ''' @@ -384,6 +391,54 @@ def n_cols_in_row(row): logger.debug("\t\tTrying to parse table...") return tableparser.parse_table(data, html=str(table)) + @staticmethod + def _candidate_table_signature(table): + return re.sub( + r"\s+", + " ", + table.get_text(" ", strip=True), + ).strip().lower() + + def _append_leaf_candidate_tables( + self, + soup, + tables, + seen_signatures=None, + ): + """Append coordinate-like leaf tables not found by a source adapter.""" + seen_signatures = seen_signatures if seen_signatures is not None else set() + for table_html in soup.find_all("table"): + # Publisher pages sometimes use an outer layout table containing the + # real table. Only pass the innermost table to downstream parsers. + if table_html.find("table") is not None: + continue + + signature = self._candidate_table_signature(table_html) + if not signature or signature in seen_signatures: + continue + + caption_node = table_html.find("caption") + caption = caption_node.get_text(" ", strip=True) if caption_node else "" + decision = tableparser.classify_coordinate_table( + html=str(table_html), + caption=caption, + ) + if not decision.candidate: + continue + + parsed = self.parse_table(table_html) + if not parsed: + continue + parsed.position = len(tables) + 1 + parsed.number = str(parsed.position) + parsed.label = f"Table {parsed.position}" + if caption: + parsed.caption = caption + tables.append(parsed) + seen_signatures.add(signature) + + return seen_signatures + def extract_doi(self, soup): ''' Every Source subclass must be able to extract its doi. ''' return @@ -504,11 +559,26 @@ def parse_article(self, html, pmid=None, **kwargs): if signature: seen_signatures.add(signature) - # Try linked table extraction only if embedded parsing produced no usable tables. - if not tables: - linked_tables = self._detect_and_download_table_links(soup, html) - if linked_tables: - tables.extend(linked_tables) + # Source strategies can select an enclosing layout table. Scan all leaf + # candidates as a union so the real coordinate table is not hidden. + self._append_leaf_candidate_tables(soup, tables, seen_signatures) + + # Linked tables are also a union with embedded tables. + linked_tables = self._detect_and_download_table_links(soup, html) + for linked_table in linked_tables or []: + linked_html = getattr(linked_table, "input_html", None) + signature = None + if linked_html: + linked_soup = BeautifulSoup(linked_html, "lxml") + linked_node = linked_soup.find("table") + if linked_node is not None: + signature = self._candidate_table_signature(linked_node) + if signature and signature in seen_signatures: + continue + linked_table.position = len(tables) + 1 + tables.append(linked_table) + if signature: + seen_signatures.add(signature) self.article.tables = tables if not tables: @@ -1373,6 +1443,7 @@ def parse_article(self, html, pmid=None, **kwargs): # Extract tables tables = [] + seen_signatures = set() table_containers = soup.find_all('div', {'class': 'tables'}) if len(table_containers) == 0: # try old method @@ -1381,6 +1452,11 @@ def parse_article(self, html, pmid=None, **kwargs): logger.info(f"Found {len(table_containers)} tables.") for (i, tc) in enumerate(table_containers): table_html = tc.find('table') + if table_html is None: + continue + signature = self._candidate_table_signature(table_html) + if not signature or signature in seen_signatures: + continue t = self.parse_table(table_html) if t: t.position = i + 1 @@ -1398,6 +1474,9 @@ def parse_article(self, html, pmid=None, **kwargs): except: pass tables.append(t) + seen_signatures.add(signature) + + self._append_leaf_candidate_tables(soup, tables, seen_signatures) self.article.tables = tables return self.article @@ -1500,6 +1579,7 @@ def extract_pmid(self, soup): class JournalOfCognitiveNeuroscienceSource(Source): def parse_article(self, html, pmid=None, **kwargs): + expand_linked_tables = kwargs.pop("expand_linked_tables", False) soup = super( JournalOfCognitiveNeuroscienceSource, self).parse_article(html, pmid, **kwargs) if not soup: @@ -1539,35 +1619,34 @@ def parse_article(self, html, pmid=None, **kwargs): tables.append(t) seen_signatures.add(signature) - # Fallback for legacy/JCN mirrors where wrappers are absent and tables are embedded. - if not tables: - for table_html in soup.find_all('table'): - if table_html.get('role') == 'presentation': - continue - raw_text = re.sub(r"\s+", " ", table_html.get_text(" ", strip=True)).strip() - if not raw_text: - continue - signature = raw_text.lower() - if signature in seen_signatures: - continue - header_text = " ".join( - th.get_text(" ", strip=True) for th in table_html.find_all("th") + # Legacy/JCN mirrors often embed tables without modern wrappers. + self._append_leaf_candidate_tables(soup, tables, seen_signatures) + + # Older Atypon pages expose only JavaScript thumbnail placeholders. + # Resolve the DOI to the current article when automatic extraction is + # explicitly allowed to expand linked table content. + has_table_placeholders = bool( + soup.find( + "a", + href=re.compile(r"javascript:popRef\(['\"]T\d+", re.IGNORECASE), + ) + ) + if ( + not tables + and expand_linked_tables + and has_table_placeholders + and doi + ): + linked_html = scrape.get_dynamic_html( + f"https://doi.org/{doi}" + ) + if linked_html: + linked_soup = BeautifulSoup(linked_html, "lxml") + self._append_leaf_candidate_tables( + linked_soup, + tables, + seen_signatures, ) - if not COORD_HEADER_HINT_RE.search(header_text) and not COORD_TRIPLET_HINT_RE.search(raw_text): - continue - - t = self.parse_table(table_html) - if not t: - continue - - t.position = len(tables) + 1 - t.number = str(t.position) - t.label = f"Table {t.position}" - caption = table_html.find("caption") - if caption: - t.caption = caption.get_text().strip() - tables.append(t) - seen_signatures.add(signature) self.article.tables = tables return self.article @@ -1859,6 +1938,7 @@ def parse_article(self, html, pmid=None, **kwargs): logger.info(f"Found {len(table_links)} potential table links.") tables = [] + seen_signatures = set() # Loop through the found links. for i, link in enumerate(table_links): @@ -1874,19 +1954,24 @@ def parse_article(self, html, pmid=None, **kwargs): if not table_soup: continue - # Find the main container first. - tc = table_soup.find('div', class_='c-article-table-container') - if not tc: - # Fallback to finding the first table on the page if container not found - tc = table_soup.find('table') - if not tc: + linked_candidates = table_soup.find_all("table") + linked_candidates = [ + candidate for candidate in linked_candidates + if candidate.find("table") is None + ] + for table_html in linked_candidates: + signature = self._candidate_table_signature(table_html) + if not signature or signature in seen_signatures: continue - - table_html = tc.find('table') if tc.name != 'table' else tc - t = self.parse_table(table_html) - - if t: - t.position = i + 1 + decision = tableparser.classify_coordinate_table( + html=str(table_html) + ) + if not decision.candidate: + continue + t = self.parse_table(table_html) + if not t: + continue + t.position = len(tables) + 1 # Parse metadata from the downloaded page's structure try: @@ -1919,50 +2004,11 @@ def parse_article(self, html, pmid=None, **kwargs): pass tables.append(t) - - # Fallback for Springer pages that already contain inline tables. - if not tables: - seen_signatures = set() - inline_containers = soup.find_all('div', class_=re.compile(r'\bTable\b|table-wrap', re.IGNORECASE)) - if not inline_containers: - inline_containers = soup.find_all('table') - - logger.info(f"SpringerSource fallback: Found {len(inline_containers)} inline table containers.") - for tc in inline_containers: - table_html = tc if getattr(tc, "name", None) == "table" else tc.find("table") - if not table_html: - continue - signature = re.sub(r"\s+", " ", table_html.get_text(" ", strip=True)).strip().lower() - if not signature or signature in seen_signatures: - continue - - t = self.parse_table(table_html) - if not t: - continue - t.position = len(tables) + 1 - - label_source = tc.get("id") or "" - label_match = re.search(r'(tab|table)\s*0*([0-9]+)', label_source, re.IGNORECASE) - if label_match: - t.number = label_match.group(2) - t.label = f"Table {t.number}" - else: - t.number = str(t.position) - t.label = f"Table {t.number}" - - caption_elem = tc.find('div', class_=re.compile(r'caption', re.IGNORECASE)) if getattr(tc, "find", None) else None - if not caption_elem: - caption_elem = table_html.find('caption') - if caption_elem: - t.caption = caption_elem.get_text().strip() - - notes_elem = tc.find(['div', 'p'], class_=re.compile(r'foot|note', re.IGNORECASE)) if getattr(tc, "find", None) else None - if notes_elem: - t.notes = notes_elem.get_text().strip() - - tables.append(t) seen_signatures.add(signature) + # Also retain inline candidates; linked and inline tables are a union. + self._append_leaf_candidate_tables(soup, tables, seen_signatures) + self.article.tables = tables return self.article @@ -2245,35 +2291,8 @@ def parse_article(self, html, pmid=None, **kwargs): tables.append(t) seen_signatures.add(signature) - # Modern PMC-like path: section.tw / div.tbl-box wrappers with direct table content. - if not tables: - fallback_containers = soup.select('section.tw, section[class*=\"table\"], div.tbl-box, div[class*=\"tbl-box\"]') - logger.info(f"Found {len(fallback_containers)} fallback PMC containers.") - for tc in fallback_containers: - table_html = tc.find('table') - if table_html is None: - continue - signature = re.sub(r"\s+", " ", table_html.get_text(" ", strip=True)).strip().lower() - if not signature or signature in seen_signatures: - continue - - t = self.parse_table(table_html) - if t: - t.position = len(tables) + 1 - label_node = tc.find(['h3', 'h4', 'label', 'span'], class_=re.compile(r'label', re.IGNORECASE)) - if label_node: - t.label = label_node.get_text().strip() - m = re.search(r'(\d+)', t.label) - if m: - t.number = m.group(1) - caption_node = tc.find(['div', 'p'], class_=re.compile(r'caption', re.IGNORECASE)) - if caption_node: - t.caption = caption_node.get_text().strip() - notes_node = tc.find(['div', 'p'], class_=re.compile(r'foot|note', re.IGNORECASE)) - if notes_node: - t.notes = notes_node.get_text().strip() - tables.append(t) - seen_signatures.add(signature) + # Modern PMC-like pages and fallback mirrors use several wrapper forms. + self._append_leaf_candidate_tables(soup, tables, seen_signatures) self.article.tables = tables return self.article diff --git a/ace/tableparser.py b/ace/tableparser.py index 73d3cb1..d5aeeb2 100644 --- a/ace/tableparser.py +++ b/ace/tableparser.py @@ -6,11 +6,139 @@ import logging from .config import get_config from .database import Activation, Table -from collections import Counter, defaultdict +from collections import Counter, defaultdict, namedtuple logger = logging.getLogger(__name__) +GATING_VERSION = "2026-07-30.annotations-v2" +CoordinateTableDecision = namedtuple( + "CoordinateTableDecision", ["candidate", "reasons"] +) + +_MINUS_CHARS = "֊‐‑⁃﹣-‒–—﹘−" +_NUMBER_RE = regex.compile( + r"(?= 3: + plausible_numeric_rows += 1 + + reasons = [] + if has_xyz_headers and plausible_numeric_rows: + reasons.append("xyz_headers") + if has_combined_header and triplet_rows: + reasons.append("combined_xyz_header") + if has_coord_context and (triplet_rows or plausible_numeric_rows): + reasons.append("coordinate_context") + if triplet_rows >= 2 and (has_coord_context or has_xyz_headers): + reasons.append("repeated_coordinate_triplets") + + return CoordinateTableDecision(bool(reasons), tuple(reasons)) + + +def is_coordinate_table_candidate(rows=None, html=None, caption=None, footnotes=None): + """Return whether the table should pass ACE's coordinate-table gate.""" + return classify_coordinate_table( + rows=rows, + html=html, + caption=caption, + footnotes=footnotes, + ).candidate + def identify_standard_columns(labels): ''' Takes a set of column labels and returns an equal-length list with names @@ -257,6 +385,7 @@ def _extract_triplet(value): def parse_table(data, html=None): ''' Takes a DataTable as input and returns a Table instance. ''' + gate_decision = classify_coordinate_table(rows=data, html=html) table = Table() # Only store the original HTML if the global config allows it if html is not None and get_config('SAVE_ORIGINAL_HTML'): @@ -421,4 +550,4 @@ def parse_table(data, html=None): table.activations.append(activation) table.finalize() - return table if len(table.activations) else None + return table if len(table.activations) or gate_decision.candidate else None diff --git a/ace/tests/test_ace.py b/ace/tests/test_ace.py index eaac77b..6d62e96 100644 --- a/ace/tests/test_ace.py +++ b/ace/tests/test_ace.py @@ -5,6 +5,7 @@ import pytest from ace import sources, database, export, scrape, ingest +from ace import tableparser @pytest.fixture(scope="module") @@ -46,6 +47,73 @@ def test_pmc_source(test_data_path, source_manager): assert t.n_activations == 11 +def test_candidate_gate_retains_table_without_ace_activations(source_manager): + html = """ + +
+

Table 1

+ + + +
Regionxyz
Midline0010
+
+ + """ + article = source_manager.sources["PMC"].parse_article( + html, pmid="999001", skip_metadata=True + ) + assert len(article.tables) == 1 + assert article.tables[0].n_activations == 0 + + +def test_candidate_gate_handles_combined_coordinates_and_unicode_minus(): + decision = tableparser.classify_coordinate_table(rows=[ + ["Region", "MNI coordinates", "Peak Z"], + ["Amygdala", "−20, -4, -18", "4.2"], + ]) + assert decision.candidate + assert "coordinate_context" in decision.reasons + + +def test_leaf_table_fallback_ignores_layout_and_behavior_tables(source_manager): + html = """ + +
+ +
GroupAge
Control24
+ +
Regionxyz
Insula-32184
+
+ + """ + article = source_manager.default_source.parse_article( + html, pmid="999002", skip_metadata=True + ) + assert len(article.tables) == 1 + assert article.tables[0].n_activations == 1 + + +def test_extract_and_export_public_api(tmp_path): + article_path = tmp_path / "999003.html" + article_path.write_text( + """ + + + +
Regionxyz
Midline0010
+ """, + encoding="utf-8", + ) + output_dir = tmp_path / "processed" + summary = ingest.extract_and_export([article_path], output_dir) + assert summary["articles"] == 1 + assert summary["tables"] == 1 + assert summary["coordinates"] == 0 + assert (output_dir / "coordinates.csv").exists() + assert (output_dir / "tables.csv").exists() + assert len(list((output_dir / "tables" / "999003").glob("*.html"))) == 1 + + @pytest.mark.vcr(record_mode="once") def test_frontiers_source(test_data_path, source_manager): filename = join(test_data_path, 'frontiers.html') @@ -208,7 +276,10 @@ def test_elsivier_table_parse(test_weird_data_path, source_manager): source = source_manager.identify_source(html) article = source.parse_article(html, pmid=pmid) tables = article.tables - assert len(tables) == 4 # should be 6, but the tables are ill formatted + # The candidate gate retains the two malformed coordinate tables even though + # deterministic parsing cannot recover activations from them. + assert len(tables) == 6 + assert sum(table.n_activations == 0 for table in tables) == 2 def test_multi_column_float_conversion(test_weird_data_path, source_manager): @@ -238,7 +309,8 @@ def test_schizophrenia_research_source(test_weird_data_path, source_manager): source = source_manager.identify_source(html) article = source.parse_article(html, pmid=pmid) tables = article.tables - assert len(tables) == 0 + assert len(tables) == 2 + assert all(table.n_activations == 0 for table in tables) def test_find_tables_in_old_sciencedirect(test_weird_data_path, source_manager): @@ -407,7 +479,7 @@ def test_oup_table_wrap_fallback_source(test_weird_data_path, source_manager): assert _count_valid_activations(article.tables) >= 1 -def test_jcn_embedded_table_fallback_source(test_weird_data_path, source_manager): +def test_jcn_thumbnail_placeholder_is_not_exported_as_a_table(test_weird_data_path, source_manager): pmid = '24666131' html = open(join(test_weird_data_path, pmid + '.html')).read() source = source_manager.identify_source(html) @@ -415,8 +487,39 @@ def test_jcn_embedded_table_fallback_source(test_weird_data_path, source_manager assert source.__class__.__name__ == 'JournalOfCognitiveNeuroscienceSource' article = source.parse_article(html, pmid=pmid, skip_metadata=True) assert article is not None - assert len(article.tables) >= 1 - assert _count_valid_activations(article.tables) >= 1 + # This legacy page has only a JavaScript thumbnail link. Treating the + # surrounding article layout as a table creates reference-list coordinates. + assert article.tables == [] + + +def test_jcn_thumbnail_placeholder_expands_linked_article_tables( + test_weird_data_path, + source_manager, + monkeypatch, +): + pmid = '24666131' + html = open(join(test_weird_data_path, pmid + '.html')).read() + source = source_manager.identify_source(html) + linked_html = """ +
+
Table 1. Peak MNI coordinates
+ + + +
Regionxyz
Insula-32184
+
+ """ + monkeypatch.setattr(scrape, "get_dynamic_html", lambda url: linked_html) + + article = source.parse_article( + html, + pmid=pmid, + skip_metadata=True, + expand_linked_tables=True, + ) + + assert len(article.tables) == 1 + assert article.tables[0].n_activations == 1 def test_sciencedirect_combined_coordinate_column_source(test_weird_data_path, source_manager): @@ -524,7 +627,6 @@ def test_validate_scrape_flags_recaptcha_challenge_page(test_weird_data_path): ("26342221", "OUPSource"), ("27623361", "ScienceDirectSource"), ("27319001", "SpringerSource"), - ("20350171", "JournalOfCognitiveNeuroscienceSource"), ("12860777", None), # Unknown source -> DefaultSource fallback ], )