diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d35e89e --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Sample corpus files test byte-level cases (BOM, CRLF, binary): keep them exactly as committed +examples/food_security_project/Text_files/* -text diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000..f4b523a --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,28 @@ +name: tests + +on: + push: + pull_request: + +jobs: + test: + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, windows-latest] + python: ["3.11", "3.12", "3.13"] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python }} + - name: Install + run: | + python -m pip install --upgrade pip + pip install -r requirements.lock + pip install -e . --no-deps + - name: Test + run: python -m pytest -q + - name: Run the example project from the command line + run: python -m ctrlfindr run --config examples/example_project/ctrlfindr.toml --no-progress diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f764be2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +# Generated results and databases +Output/ +*.sqlite +*.sqlite-journal +# LibreOffice / Excel lock files +~$* +.~lock.*# +# Python +__pycache__/ +*.py[cod] +*.egg-info/ +build/ +dist/ +.venv/ +venv/ +.pytest_cache/ +.ipynb_checkpoints/ diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ca5c4d3 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,133 @@ +# Changelog + +## 3.0.0 + +A rewrite of the 2.1 notebook as a Python package (`ctrlfindr`) with a command line, a thin +notebook, tests and CI on Windows and Linux, and a **new framework format**. 2.x frameworks +are not read; the 2.1 example was rewritten in the new format with the same rules. The +effect of each change on the example project is quantified in +[docs/result_changes.md](docs/result_changes.md). + +### New assessment framework (format 1) + +- **Sheets:** `meta`, `settings`, `taxonomy`, `queries`, `variables`, `tests`, `instructions`. + The `co_occurrences`, `doc_conditionals` and `search_strings` sheets are gone: proximity + and document conditions are functions inside queries. +- **File types:** `.ods` or `.xlsx`. Cells are read as displayed, and `ctrlfindr template` + writes a blank framework with instructions and dropdowns. +- **meta:** a required `format_version`, so a framework is never silently reinterpreted. + Optional `title`, `version`, `author`, `language` and `description`, plus any key of your + own, all copied into the codebook and `run_metadata.json`. +- **settings:** sentiment on/off and thresholds, `aggregate_children` and `tests_must_pass`. + These live in the framework, so the framework alone defines the coding; the config only + says where files are and how texts are read. +- **taxonomy:** one row per term (`list | term | note`). New term syntax: + - `~` for a hyphen, space or nothing (`human~wildlife`); + - a standalone `_` for one word (`climate _ risk`); + - exclusion phrases (`-business climate`) and excluded lists (`-@idioms`), which drop only + the matches that lie inside them, not the whole sentence. +- **queries:** + - names with or without quotes; + - `{term}` for inline terms; + - `NEAR`, `BEFORE`, `NOT_NEAR`; + - `DOC`, `DOC_SHARE`, `WINDOW`; + - `COUNT` and `COUNT_DISTINCT` with comparisons; + - queries may use other queries. +- **variables:** + - full expressions instead of comma lists, and `V(n)` for other variables; + - an optional `parent` column (otherwise the hierarchy follows the numbering), with + optional aggregation of children; + - a warning when a child is true where its parent is false. +- **tests:** example sentences with expected results, checked on every run. Groups form + documents, so document-level rules can be tested too. +- **Validation against hand coding:** `ctrlfindr sample` writes a blind, stratified sample + to code. Every analysis then reports precision, estimated recall and F1 per variable, + with the disagreements. +- **Names** follow one rule (a letter first, then letters, digits, `_`, `-`), with suggested + fixes. All problems are reported together with sheet and row, before any text is + processed. +- **New output:** `codebook.md` with the framework in readable form. + +### Text inputs and sentiment + +- **Two input types:** `.txt` files are read as unstructured text, and `.md` files as + Markdown. +- **Structure labels:** every sentence of a Markdown file is labelled `header`, `body`, `list`, + `table` or `footnote`, and every sentence of a text file `unstructured`. Footnote markers + (`[^1]`, `12`, `^12^`) are removed from sentences. The label is stored and + exported in the `structure` column. +- **Excluding sentence types:** the setting `exclude_sentence_types` leaves sentence types + out of the analysis. +- **VADER is optional:** sentiment is off unless a framework sets `sentiment = true`; + vaderSentiment is an optional package; scores are computed during the analysis and cached + in the database. + +### Changes that affect results + +- **Co-occurrence distance** (now `NEAR`) is the number of words *between* the two matches. + 2.1 also counted the first keyword, so a distance of N allowed only N-1 words in between. +- **Self-matches are no longer counted.** In 2.1, a term present in both lists (for example + `encroach*`) co-occurred with itself. Overlapping matches do not count either. +- **Line breaks no longer glue words together.** 2.1 deleted line breaks before capitals and + digits (`climate security\nClimate change` → `securityclimate`). +- **Blank lines end sentences**, so headings are no longer glued onto the following sentence. +- **Sentence splitting** uses pySBD instead of NLTK punkt. +- **Segments without letters**, including citation marks such as `[12]`, are dropped. +- **Unicode normalization and dash/quote folding** apply to text and framework, so + `human–wildlife` (en dash) matches `human-wildlife` (hyphen). +- **Terms ending or starting with punctuation** (`u.s.`) match; 2.1's `\b` boundaries + prevented it. +- **Sentiment:** + - scores come from vaderSentiment (the VADER authors' implementation) instead of NLTK's + port; + - the original-case sentence is scored, not the lowercased one; + - the thresholds are ±0.05 instead of 0, and can be changed in the framework's settings. + +### Running large corpora + +- **Ingest saves each file as soon as it is done.** A stopped or crashed ingest keeps every + finished file, and the next run continues with the rest. +- **The analysis runs in batches of whole documents** (`[analysis] batch_sentences`), so + memory use stays flat. Outputs are replaced only when a run completes, and sentiment + scores are kept across stops. +- **`.txt` cleaning repairs a missing space after a full stop** ("published.The minister"). + +### Changes to inputs and outputs + +- `vadar_compound` is renamed `vader_compound`. +- `results_sentences.tsv` gains `sentence_id`, `structure`, `start` and `end`. +- Variable names keep their capitalization in output headers. +- The database file is `analysis.sqlite`. Source documents, sentences and results are + separate tables. 2.1 overwrote the source table with results, so a second run failed. +- New `run_metadata.json` with versions, settings, test results and file hashes. +- The project configuration is `ctrlfindr.toml`. Ingest and analysis are separate steps, and + ingest only processes new or changed files. +- `.txt` extensions are matched case-insensitively, only the extension is removed from + document names, and files are processed in sorted order. +- Result files that the current settings no longer produce are removed from the output folder. + +### Safety and robustness + +- Expressions are parsed and evaluated without `eval`. +- Every database connection is closed on error, and table names are fixed in code. +- Files that are not UTF-8 stop ingestion with a list of the affected files, or are decoded + with replacement characters if configured. A UTF-8 byte-order mark is accepted. +- TSV files are written as UTF-8 with `\n` line endings on every platform. +- Ingest hashes files without loading the whole corpus into memory. + +### Speed + +- Keyword matching uses one Aho-Corasick pass over all terms. +- Proximity checks only sentences where both sides matched, and only the nearest pairs. +- Expressions are evaluated column-wise. + +Analysis is about 40 times faster than in 2.1. On very long segments, proximity went from +cubic to near-linear cost. + +### Repository + +- Code: `ctrlfindr/`. Example: `examples/example_project/`. Blank frameworks: `template/`. + Reference outputs: `tests/fixtures/`. +- The 2.1 notebook is kept unchanged in `legacy/v2.1/`. Its example framework is at + `tests/fixtures/golden_v2.1/Assessment_framework_v2.1.ods`. +- Outputs and databases are ignored by git. diff --git a/Jupiter Notebook Example/Analysis/Input/~$Assessment_framework.ods b/Jupiter Notebook Example/Analysis/Input/~$Assessment_framework.ods deleted file mode 100644 index 65419c1..0000000 Binary files a/Jupiter Notebook Example/Analysis/Input/~$Assessment_framework.ods and /dev/null differ diff --git a/Jupiter Notebook Example/Analysis/Output/database b/Jupiter Notebook Example/Analysis/Output/database deleted file mode 100644 index 6af2064..0000000 Binary files a/Jupiter Notebook Example/Analysis/Output/database and /dev/null differ diff --git a/Jupiter Notebook/Assessment_framework.ods b/Jupiter Notebook/Assessment_framework.ods deleted file mode 100644 index 1319e9b..0000000 Binary files a/Jupiter Notebook/Assessment_framework.ods and /dev/null differ diff --git a/Jupiter Notebook/CtrlFindr.ipynb b/Jupiter Notebook/CtrlFindr.ipynb deleted file mode 100644 index 9db0708..0000000 --- a/Jupiter Notebook/CtrlFindr.ipynb +++ /dev/null @@ -1,483 +0,0 @@ -{ - "cells": [ - { - "cell_type": "code", - "execution_count": 45, - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "[nltk_data] Downloading package vader_lexicon to C:\\Users\\MSI-\n", - "[nltk_data] CMS\\AppData\\Roaming\\nltk_data...\n", - "[nltk_data] Package vader_lexicon is already up-to-date!\n", - "[nltk_data] Downloading package punkt to C:\\Users\\MSI-\n", - "[nltk_data] CMS\\AppData\\Roaming\\nltk_data...\n", - "[nltk_data] Package punkt is already up-to-date!\n" - ] - } - ], - "source": [ - "# For additional information or assistance with running this code, please reach out to contact [at] scartozzi [dot] eu.\n", - "# This code is released under GNU General Public License v3.0. Feel free to use it as you wish.\n", - "\n", - "import os\n", - "import re\n", - "import time\n", - "import pandas as pd\n", - "import sqlite3\n", - "import nltk\n", - "nltk.download('vader_lexicon')\n", - "nltk.download('punkt')\n", - "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n", - "sent_i = SentimentIntensityAnalyzer()" - ] - }, - { - "cell_type": "code", - "execution_count": 46, - "metadata": {}, - "outputs": [], - "source": [ - "## GLOBAL VARIABLES\n", - "# specify the directory where the text files are stored:\n", - "TEXT_FILES_DIRECTORY = \"./Analysis/Text_files\"\n", - "# specify if you want to load a SQL database or not:\n", - "LOAD_DATABSE = False\n", - "# specify the path to the SQL database:\n", - "DATABASE_PATH = \"./Analysis/Output/database\"\n", - "# specify the name of the table in the SQL database:\n", - "TABLE_NAME = 'documents'\n", - "# specify the directory where the assessment framework is stored:\n", - "ASSESSMENT_FRAMEWORK_DIRECTORY = \"./Analysis/Input/Assessment_framework.ods\"\n", - "# specify the directory where the results will be saved:\n", - "OUTPUT_DIRECTORY = \"./Analysis/Output\"\n", - "# choose the following settings:\n", - "TEXT_CLEANING = True\n", - "SENTIMENT_ANALYSIS = True\n", - "# choose what to export:\n", - "EXPORT_SENTENCE_LEVEL_DATA = True\n", - "EXPORT_DOCUMENT_DATA = True\n", - "# estimate the time needed to run the code:\n", - "PRINT_RUN_TIME = True" - ] - }, - { - "cell_type": "code", - "execution_count": 47, - "metadata": {}, - "outputs": [], - "source": [ - "## LOAD AND PREPARE DATA FOR ANALYSIS\n", - "# IMPORT TXT FILES FROM TEXT_FILES_DIRECTORY\n", - "def txt_to_sql(directory, db_path):\n", - " data = {\"document\": [], \"text\": []}\n", - "\n", - " for filename in os.listdir(directory):\n", - " if filename.endswith(\".txt\"):\n", - " filepath = os.path.join(directory, filename)\n", - " with open(filepath, 'r', encoding='utf-8') as file:\n", - " content = file.read()\n", - " data[\"document\"].append(filename.replace(\".txt\", \"\"))\n", - " data[\"text\"].append(content)\n", - " df = pd.DataFrame(data)\n", - " conn = sqlite3.connect(db_path)\n", - " df.to_sql('documents', conn, if_exists='replace', index=False)\n", - " print(df.shape)\n", - " conn.close()\n", - " return df\n", - "\n", - "# IMPORT ASSESSMENT FRAMEWORK\n", - "def create_dataframes(ods_file):\n", - " variables = process_dataframe(pd.read_excel(ods_file, sheet_name='variables', engine='odf'))\n", - " set_search_strings = process_dataframe(pd.read_excel(ods_file, sheet_name='search_strings', engine='odf'))\n", - " set_co_occurrences = process_dataframe(pd.read_excel(ods_file, sheet_name='co_occurrences', engine='odf'))\n", - " set_doc_conditionals = process_dataframe(pd.read_excel(ods_file, sheet_name='doc_conditionals', engine='odf'))\n", - " set_keywords = process_dataframe(pd.read_excel(ods_file, sheet_name='taxonomy', engine='odf'), drop_na=False)\n", - " return variables, set_search_strings, set_co_occurrences, set_doc_conditionals, set_keywords\n", - "\n", - "# Helper function to process the data from each sheet in the ODS file\n", - "def process_dataframe(df, lowercase_columns=True, drop_na=False):\n", - " if lowercase_columns:\n", - " df.columns = [convert_to_lowercase(col) for col in df.columns]\n", - " df = df.map(convert_to_lowercase)\n", - " if 'query' in df.columns:\n", - " df['query'] = df['query'].apply(format_logical_expression)\n", - " if drop_na:\n", - " df = df.dropna()\n", - " return df\n", - "\n", - "# Helper function to format logical expressions in the search strings' query column\n", - "def format_logical_expression(expression):\n", - " if not isinstance(expression, str):\n", - " return expression\n", - " expression = re.sub(r\"(? 0,\n", - " 'sentiment_neutral': sentiment_scores == 0,\n", - " 'sentiment_neutralpositive': sentiment_scores >= 0,\n", - " 'sentiment_negative': sentiment_scores < 0,\n", - " 'sentiment_neutralnegative': sentiment_scores <= 0\n", - " })\n", - "\n", - "# RUN QUERIES LISTED IN ASSESSMENT_FRAMEWORK\n", - "def run_queries(df, set_search_strings):\n", - " results = df.copy()\n", - " for _, row in set_search_strings.iterrows():\n", - " variable, query = row['query name'], row['query']\n", - " # Convert AND, OR, NOT operations to their Python equivalents\n", - " query = query.replace(\"AND\", \"and\").replace(\"OR\", \"or\").replace(\"NOT\", \"not\")\n", - " # Evaluate the query for each row in df\n", - " results[variable] = df.apply(lambda x: evaluate_query(x, query), axis=1)\n", - " return results\n", - "\n", - "# Helper function to evaluate the query\n", - "def evaluate_query(row, query):\n", - " for column in row.index:\n", - " query = query.replace(f'\"{column}\"', str(row[column]))\n", - " try:\n", - " return eval(query)\n", - " except Exception as e:\n", - " print(f\"Error evaluating query: {query} - {e}\")\n", - " return False\n", - " \n", - "# AGGREGATE VARIABLES LISTED IN ASSESSMENT_FRAMEWORK\n", - "def define_variables(results, variables):\n", - " for _, row in variables.iterrows():\n", - " condition = row['aggregation query']\n", - " child_vars = condition.split(',')\n", - " child_vars = [var.strip(\" ()\\\"'\") for var in child_vars]\n", - " variable_number = str(row['variable number'])\n", - " if '.' not in variable_number:\n", - " variable_number += '.'\n", - " parent_name = f\"{variable_number} {row['variable']}\"\n", - " results[parent_name] = results[child_vars].any(axis=1)\n", - " return results\n", - "\n", - "\n", - "## EXPORT RESULTS\n", - "def get_v_list(df):\n", - " return [col for col in df.columns if bool(re.match(r'^[0-9]+\\.[0-9]*', str(col)))]\n", - "\n", - "def results_document(df, v_list):\n", - " codebook = df.groupby('document')[v_list].sum().astype(int).reset_index()\n", - " return codebook\n", - "\n", - "def results_document_clipped(df, v_list):\n", - " code_bool = df.groupby('document')[v_list].sum().clip(upper=1).reset_index()\n", - " return code_bool\n", - "\n", - "def results_document_percentage_of_sentences(df, v_list):\n", - " grouped = df.groupby('document')\n", - " codebook_count_sent = grouped['sentences'].count().reset_index()\n", - " code_sent_percent = grouped[v_list].sum().reset_index()\n", - " code_sent_percent = code_sent_percent.merge(codebook_count_sent, on='document', how='outer')\n", - " code_sent_percent[v_list] = code_sent_percent[v_list].div(code_sent_percent['sentences'], axis=0).multiply(100)\n", - " code_sent_percent.drop('sentences', axis=1, inplace=True)\n", - " return code_sent_percent\n", - "\n", - "def results_document_sentiment(df, v_list):\n", - " codebook_sentiment = df[['document']].drop_duplicates().set_index('document')\n", - " for var in v_list:\n", - " sentiment = df.loc[df[var], ['document', 'vadar_compound']].groupby('document')['vadar_compound'].mean()\n", - " codebook_sentiment[var] = codebook_sentiment.index.map(sentiment)\n", - " return codebook_sentiment.reset_index()\n", - "\n", - "### SQL FUNCTIONS MANAGEMENT\n", - "def load_data_from_sql(db_path, table_name):\n", - " conn = sqlite3.connect(db_path)\n", - " df = pd.read_sql(f'SELECT * FROM {table_name}', conn)\n", - " conn.close()\n", - " return df\n", - "\n", - "def save_data_to_sql(df, db_path, table_name):\n", - " conn = sqlite3.connect(db_path)\n", - " df.to_sql(table_name, conn, if_exists='replace', index=False)\n", - " conn.close()\n", - "\n", - "def load_data_from_sql(db_path, table_name):\n", - " conn = sqlite3.connect(db_path)\n", - " df = pd.read_sql(f'SELECT * FROM {table_name}', conn)\n", - " conn.close()\n", - " return df\n", - "\n", - "\n", - "## CHECK FOR USER-MADE ERRORS IN ASSESSEMENT FRAMEWORK\n", - "def preliminary_checks(set_search_strings, set_doc_conditionals, set_co_occurrences, taxonomy):\n", - " if not os.path.exists(ASSESSMENT_FRAMEWORK_DIRECTORY):\n", - " print(f\"Error: The Assessment framework file '{ASSESSMENT_FRAMEWORK_DIRECTORY}' not found.\")\n", - " return False\n", - " return True\n", - "\n", - "# Check the search strings for errors\n", - "def check_search_strings(search_strings, taxonomy, doc_conditionals, co_occurrences,variables):\n", - " required_columns = ['query name', 'query']\n", - " if not all(column in search_strings.columns for column in required_columns):\n", - " print(f\"Error: The search_strings tab does not have the required columns. Keep the column headers included in the template.\")\n", - " return False\n", - " valid_terms = set(taxonomy.columns) | set(doc_conditionals['name of document-level conditional']) | set(co_occurrences['name of co-occurrence'])\n", - " if search_strings['query name'].duplicated().any():\n", - " print(\"Error: The 'variable' column in the search_strings tab contains duplicate entries.\")\n", - " return False\n", - " for query in search_strings['query']:\n", - " if query.count(\"(\") != query.count(\")\"):\n", - " print(f\"Error: Mismatched parentheses in '{query}'.\")\n", - " return False\n", - " strings_in_quotes = re.findall(r'\"([^\"]*)\"', query)\n", - " for string in strings_in_quotes:\n", - " if string not in valid_terms and string not in ['sentiment_positive', 'sentiment_neutral', 'sentiment_neutralpositive', 'sentiment_negative', 'sentiment_neutralnegative']:\n", - " print(f\"Error: The string '{string}' inside quotation marks is not found in the provided taxonomy, document-level conditionals, or co-occurrences.\")\n", - " return False\n", - " query_simplified = re.sub(r'\"[^\"]*\"', '', query)\n", - " query_simplified = re.sub(r'\\b(and|or|not)\\b', '', query_simplified, flags=re.IGNORECASE)\n", - " query_simplified = query_simplified.replace(\"(\", \"\").replace(\")\", \"\")\n", - " if query_simplified.strip() != \"\":\n", - " print(f\"Error: Query contains invalid or improperly quoted strings: {query_simplified.strip()}\")\n", - " return False\n", - " if '\"\"' in query or re.search(r'\"\\s+\"', query):\n", - " print(\"Error: Found strings not properly enclosed in quotation marks or missing quotation marks.\")\n", - " return False\n", - " required_co_occurrence_columns = ['name of co-occurrence', 'first list', 'distance between lists', 'second list']\n", - " if not all(column in co_occurrences.columns for column in required_co_occurrence_columns):\n", - " print(\"Error: co_occurrences tab does not have the required columns.\")\n", - " return False\n", - " valid_terms_for_co_occurrences = set(taxonomy.columns)\n", - " for index, row in co_occurrences.iterrows():\n", - " if row['first list'] not in valid_terms_for_co_occurrences or row['second list'] not in valid_terms_for_co_occurrences:\n", - " print(f\"Error: Check if these strings in co_occurrences '{row['first list']}' or '{row['second list']}' are present in the taxonomy as column headers.\")\n", - " return False\n", - " required_doc_conditional_columns = ['name of document-level conditional', 'list']\n", - " if not all(column in doc_conditionals.columns for column in required_doc_conditional_columns):\n", - " print(\"Error: doc_conditionals tab does not have the required columns.\")\n", - " return False\n", - " for index, row in doc_conditionals.iterrows():\n", - " if row['list'] not in valid_terms_for_co_occurrences:\n", - " print(f\"Error: The string '{row['list']}' in doc_conditionals is not present in the taxonomy as column headers.\")\n", - " return False\n", - " all_variables_in_search_strings = set(search_strings['query name'])\n", - " child_variables = set()\n", - " for query in variables['aggregation query']:\n", - " matches = re.findall(r'\"(.*?)\"', query)\n", - " child_variables.update(matches)\n", - " missing_variables = child_variables - all_variables_in_search_strings\n", - " if missing_variables:\n", - " print(f\"Error: The following queries are not present in the search_strings 'query name' column: {missing_variables}. Note that query names cannot be numerical\")\n", - " return False\n", - " return True\n", - "\n", - "## ESTIMATE RUN TIME\n", - "def estimate_run_time(sample_df, full_run=False): \n", - " variables, set_search_strings, set_co_occurrences, set_doc_conditionals, set_keywords = create_dataframes(ASSESSMENT_FRAMEWORK_DIRECTORY)\n", - " key_dict = organize_keywords(set_keywords)\n", - " start_time = time.time()\n", - " sample_df = check_groups(sample_df, key_dict)\n", - " sample_df = initiate_co_occurrences(sample_df, set_co_occurrences, key_dict)\n", - " initiate_document_conditionals(sample_df, set_doc_conditionals)\n", - " if SENTIMENT_ANALYSIS:\n", - " sentiment_df = vadar_sentiment_analysis(sample_df['sentences'])\n", - " sample_df = pd.concat([sample_df, sentiment_df], axis=1)\n", - " sample_results = run_queries(sample_df, set_search_strings)\n", - " sample_results = define_variables(sample_results, variables)\n", - " end_time = time.time()\n", - " return end_time - start_time" - ] - }, - { - "cell_type": "code", - "execution_count": 48, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "(7, 2)\n", - "Estimated run time for 1589 sentences: 1.5410478043556213 seconds (excluding time to export results).\n" - ] - } - ], - "source": [ - "def main(): \n", - " ## LOAD DATA\n", - " if not LOAD_DATABSE:\n", - " df = txt_to_sql(TEXT_FILES_DIRECTORY, DATABASE_PATH)\n", - " else:\n", - " df = load_data_from_sql(DATABASE_PATH, TABLE_NAME) \n", - " variables, set_search_strings, set_co_occurrences, set_doc_conditionals, set_keywords = create_dataframes(ASSESSMENT_FRAMEWORK_DIRECTORY)\n", - " key_dict = organize_keywords(set_keywords)\n", - "\n", - " ## CHECK FOR USER-MADE ERRORS IN ASSESSEMENT FRAMEWORK\n", - " if not preliminary_checks(set_search_strings, set_co_occurrences, set_doc_conditionals, set_keywords):\n", - " print(\"Preliminary checks failed. Exiting...\")\n", - " return \n", - " if not check_search_strings(set_search_strings, set_keywords, set_doc_conditionals, set_co_occurrences, variables):\n", - " print(\"Search strings check failed.\")\n", - " return\n", - " \n", - " ## PREPARE DATA\n", - " if TEXT_CLEANING:\n", - " clean_text(df, 'text')\n", - " df = split_sentences(df)\n", - " df= clean_rows(df)\n", - " df = df.drop('text', axis=1)\n", - "\n", - " ## ESTIMATE RUN TIME\n", - " if PRINT_RUN_TIME:\n", - " total_sentences = len(df)\n", - " if total_sentences < 100:\n", - " print(f\"The dataset has only {total_sentences} sentences, which is insufficient to accurately estimate runtime.\")\n", - " else:\n", - " sample = df.sample(n=100)\n", - " time_for_50 = estimate_run_time(sample, full_run=False)\n", - " estimated_total_time = (time_for_50 / 100) * total_sentences\n", - " print(f\"Estimated run time for {total_sentences} sentences: {estimated_total_time} seconds (excluding time to export results).\")\n", - "\n", - " ## RUN CONTENT ANALYSIS\n", - " df = check_groups(df, key_dict)\n", - " df = initiate_co_occurrences(df, set_co_occurrences, key_dict)\n", - " df = initiate_document_conditionals(df, set_doc_conditionals)\n", - " if SENTIMENT_ANALYSIS:\n", - " sentiment_df = vadar_sentiment_analysis(df['sentences'])\n", - " df = pd.concat([df, sentiment_df], axis=1)\n", - " df = run_queries(df, set_search_strings)\n", - " df = define_variables(df, variables)\n", - "\n", - " ## EXPORT RESULTS\n", - " os.makedirs(OUTPUT_DIRECTORY, exist_ok=True)\n", - " v_list = get_v_list(df)\n", - " if EXPORT_DOCUMENT_DATA:\n", - " codebook = results_document(df, v_list)\n", - " codebook.to_csv(f'{OUTPUT_DIRECTORY}/results_document.tsv', sep='\\t', index=False)\n", - " code_bool = results_document_clipped(df, v_list)\n", - " code_bool.to_csv(f'{OUTPUT_DIRECTORY}/results_document_clipped.tsv', sep='\\t', index=False)\n", - " code_sent_percent = results_document_percentage_of_sentences(df, v_list)\n", - " code_sent_percent.to_csv(f'{OUTPUT_DIRECTORY}/results_document_percentage_of_sentences.tsv', sep='\\t', index=False)\n", - " if SENTIMENT_ANALYSIS:\n", - " codebook_sentiment = results_document_sentiment(df, v_list)\n", - " codebook_sentiment.to_csv(f'{OUTPUT_DIRECTORY}/results_document_sentiment.tsv', sep='\\t', index=False)\n", - " if EXPORT_SENTENCE_LEVEL_DATA:\n", - " df.to_csv(f'{OUTPUT_DIRECTORY}/results_sentences.tsv', sep='\\t', index=False)\n", - " save_data_to_sql(df, DATABASE_PATH, TABLE_NAME)\n", - "\n", - "if __name__ == \"__main__\":\n", - " main_results = main()" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "base", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.10.9" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} diff --git a/README.md b/README.md index 4ac93c7..0af5884 100644 --- a/README.md +++ b/README.md @@ -1,56 +1,457 @@ # CtrlFindr -**CtrlFindr** is a natural language processing and **content analysis toolkit** designed to help you analyze and extract insights from text documents. The toolkit is at the moment implemented in a Jupyter Notebook, providing you with an interactive environment to work with the code and visualize the results. The current version works with English text, but it can easily be adapted for other languages. +CtrlFindr is a toolkit for dictionary-based **content analysis** of English text documents. +It splits each document into sentences and codes every sentence against an *assessment +framework* that you write in a spreadsheet (LibreOffice `.ods` or Excel `.xlsx`). The +framework combines lists of terms, proximity rules, document-level conditions, counts +and sentiment into variables, and the results are exported as tables per +document and per sentence. -In the coming months, I plan to provide a version of the code with a User Interface (UI) and an executable file for easier use. +Version 3.0 is a rewrite of the 2.x notebook with a new framework format; 2.x frameworks +are not read. See [CHANGELOG.md](CHANGELOG.md) and [docs/result_changes.md](docs/result_changes.md). +Version 2.1 is kept in [legacy/](legacy/). -# Features +## How it works -- Text preprocessing (lowercase conversion, sentence splitting), -- Keyword analysis, -- Co-occurrence analysis, -- Sentiment analysis (using VADER SentimentIntensityAnalyzer from nltk), -- Customizable search strings for automating content analysis of natural language (to be filled in the provided Assessment_framework.ots template) -- Export results to TSV files (number of positive findings per document, positive findings per document as booleans, positive findings as percentage of sentences within documents, and sentiment analysis of positive findings) +1. **`check`**: validates the framework, reporting every problem at once with its sheet and + row, and runs the framework's own test sentences. No text is needed. +2. **`ingest`**: reads the `.txt` and `.md` files (see [Text inputs](#text-inputs)), + normalizes the text, splits it into sentences with + [pySBD](https://github.com/nipunsadvilkar/pySBD) and stores everything in an SQLite + database. Later runs only process new or changed files. Each file is saved as soon as it + is done, so you can stop an ingest (Ctrl+C) at any time and run it again to continue. +3. **`analyze`**: codes every sentence and writes the results, a codebook and the run + metadata. When the framework uses sentiment, it also scores sentences with + [VADER](https://github.com/cjhutto/vaderSentiment), once per sentence (scores are kept + for later runs). Run it again after each framework edit; there is no need to ingest again. + Sentences are analyzed in batches of whole documents, so memory use does not grow with + the corpus. A stopped analysis leaves the previous results as they were. -# Dependencies +`run` does ingest and analyze, `template` writes a blank framework, and `sample` draws +sentences to hand-code (see [Validation](#validation-against-hand-coding)). - Python 3.6 or higher - Pandas - Numpy - NLTK +## Installation -# Installation +You need Python 3.11 or newer (tested with 3.11, 3.12 and 3.13), on Windows, macOS or +Linux. No GPU is needed. Download or clone this repository, then run the following in its +folder: - Clone the CtrlFindr repository to your local machine: ``` -git clone https://github.com/username/CtrlFindr.git -``` +pip install -r requirements.lock +pip install -e . --no-deps +``` + +`requirements.lock` pins the exact package versions used to produce the reference results +and to run the tests. A virtual environment (`python -m venv .venv`) keeps these versions +apart from other projects. + +VADER sentiment is optional. `requirements.lock` installs it; with your own installation, +add it with `pip install vaderSentiment` (or `pip install -e ".[sentiment]"`) only if a +framework sets `sentiment = true`. Without it everything else works, and `check` says so +when a framework needs it. + +## Quick start -Navigate to the CtrlFindr directory: ``` -cd CtrlFindr +python -m ctrlfindr run --config examples/example_project/ctrlfindr.toml ``` +The results appear in `examples/example_project/Output/`. To start your own project: + +1. Copy `examples/example_project/ctrlfindr.toml` into a new folder and put your `.txt` or + `.md` files in `Text_files/` next to it. +2. Create a framework: `python -m ctrlfindr template Input/Assessment_framework.xlsx` (or + `.ods`), or copy one from [template/](template/). If you use `.xlsx`, change the + `framework` path in `ctrlfindr.toml`. +3. Fill it in, using the example framework and the template's `instructions` sheet as a + guide, and check it with `python -m ctrlfindr check --config ctrlfindr.toml`. +4. Run `python -m ctrlfindr run --config ctrlfindr.toml`. + +`examples/markdown_project` shows Markdown input, with headings, table rows and footnotes +excluded from the analysis. From Jupyter, use [notebooks/CtrlFindr.ipynb](notebooks/CtrlFindr.ipynb). + +## Text inputs + +CtrlFindr reads every `.txt` and `.md` file directly in the text files folder (not in +subfolders). Hidden and temporary files whose names start with `.` or `~` (such as macOS +`._report.txt` copies or editor lock files) are skipped. Each file is one document, named after the file without its extension; a +`.txt` and an `.md` file with the same name are reported as a clash. Files must be UTF-8. + +| file | treated as | structure label of each sentence | +|---|---|---| +| `.txt` | unstructured plain text | `unstructured` | +| `.md` | Markdown | `header`, `body`, `list`, `table` or `footnote` | + +For Markdown files: + +- **header:** a heading (`# Title`, or a line underlined with `===` or `---`). It is kept + whole, as one sentence. +- **list:** a list item (`-`, `*`, `+`, `•`, `1.`, `1)`), including its continuation lines, + split into sentences. +- **table:** a table row (`| a | b |`). It is kept whole, with its cells joined by ` | `; the + `|---|` separator row is dropped. +- **footnote:** the text of a footnote (`[^1]: …`), including its indented continuation lines + and further paragraphs, split into sentences. +- **body:** everything else (paragraphs, quotes, code blocks), split into sentences. + +Markdown syntax is removed before matching: emphasis markers, link targets (the link text +stays), images, inline code backticks, HTML tags and entities, and footnote markers +(`[^1]`, `12`, `a`, `^12^`). Superscript ordinals and exponents are kept +(`20th` → `20th`, `106` → `10^6`); a number raised after letters, as +in `km2`, cannot be told apart from a footnote marker and is removed. Horizontal +rules and link definitions are dropped. A line starting with a number other than 1 and a +full stop (`2020. Floods…`) inside a paragraph is wrapped text, not a list, as in CommonMark. + +The Markdown structure also helps the sentence splitter: + +- **Block boundaries end sentences.** Every heading, list item, table row, footnote and + paragraph boundary ends a sentence, even without a full stop. +- **Headings and table rows are never split**, whatever full stops they contain + ("Fig. 3 Annual mean", "U.S. | 700"). +- **Markup is removed before splitting.** Link addresses and footnote markers cannot create + false boundaries (`IPCC.[^1] The` is split as "IPCC." | "The"). + +The label is stored with every sentence and appears in the `structure` column of +`results_sentences.tsv`. To leave sentence types out of the analysis altogether, set +`exclude_sentence_types` in the framework's settings sheet, for example +`header, table, footnote`. +Excluded sentences are not coded and do not count in per-document totals or percentages. + +**JupyterLab server.** For long runs, start the command in a JupyterLab terminal rather than +a notebook: it keeps running if the browser tab closes or an idle kernel is shut down. Keep +the database on the server's local disk (set `database` in the config); SQLite is +unreliable on network file systems, which many servers use for home folders. + +## The assessment framework + +A framework is one spreadsheet with these sheets (the template has them all): + +| sheet | columns | purpose | +|---|---|---| +| `meta` | key, value | identifies the framework; `format_version` is required | +| `settings` | key, value, note | coding settings (optional; defaults apply) | +| `taxonomy` | list, term, note | the terms, one per row | +| `queries` | name, query, note | named expressions over lists, queries and sentiment | +| `variables` | number, name, definition, parent, note | the exported results | +| `tests` | sentence, expect_true, expect_false, group, note | example sentences checked on every run (optional) | +| `instructions` | | the rules in short; ignored by CtrlFindr | + +Cells are read as displayed (`1.10` stays `1.10`; a decimal comma `1,1` is read as `1.1`). +Matching ignores upper and lower case. Notes are copied into the codebook. + +### Names + +Lists and queries share one set of names. A name starts with a letter and contains only +letters, digits, `_` and `-` (`fast_onset`, `co-slow`, `a1`); names are case-insensitive and +must be unique. The reserved names are: -Fill the search strings and taxonomy in the Assessment_framework.ots +- the operators `and`, `or`, `not`; +- the function names below; +- the sentiment flags; +- `document`, `sentences`, `sentence_id`, `structure`, `start`, `end`, `vader_compound`. -Open the CtrlFindr.ipynb file in Jupyter Notebook to start analyzing your text files. +Variable names are free text, because variables are referred to by number. + +### meta + +One key per row. `format_version` must be `1`. A framework without it, or with another +version, is rejected rather than reinterpreted. `title`, `version`, `author`, `language` +(only `en` is supported) and `description` are optional, and you may add your own keys. All +of them go into `run_metadata.json` and the codebook. + +### settings + +The framework holds every setting that changes how sentences are coded, so the framework +file alone defines the coding. A missing row uses the default. + +| key | default | meaning | +|---|---|---| +| `sentiment` | false | score sentences with VADER (optional package) and allow the sentiment flags | +| `sentiment_positive_threshold` | 0.05 | compound score at or above which a sentence is positive | +| `sentiment_negative_threshold` | -0.05 | compound score at or below which a sentence is negative | +| `aggregate_children` | false | a variable is also true wherever one of its child variables is true | +| `tests_must_pass` | true | stop the analysis when a test sentence fails | +| `exclude_sentence_types` | (none) | sentence types left out of the analysis, comma-separated: `header`, `body`, `list`, `table`, `footnote`, `unstructured` | + +### taxonomy: terms + +One term per row (`list`, `term`, `note`). A list is true for a sentence when one of its +terms occurs in it, after exclusions. + +| term | matches | +|---|---| +| `flood` | the word "flood" only, never "floods" or "floodplain": terms match whole words | +| `flood*` | `*` stands for any letters or digits, including none: "flood", "floods", "flooding". It may appear anywhere in a term (`*fire`, `biodivers* loss`) but never crosses a space or hyphen. | +| `human~wildlife` | `~` stands for a hyphen, a space or nothing: "human-wildlife", "human wildlife", "humanwildlife" | +| `climate _ risk` | `_` on its own stands for exactly one word: "climate change risk" | +| `flash flood*`, `u.s.` | multi-word terms and punctuation match literally, whatever the line breaks and spacing in the text | +| `-business climate` | an **exclusion phrase**: a match of this list that lies inside "business climate" is ignored. Other matches in the same sentence still count. | +| `-@idioms` | **excludes a list**: every term of the list `idioms` is an exclusion phrase for this list | + +Exclusions work by containment. In "The business climate improved while climate change +accelerated", a list `clim` with `climat*` and `-business climate` ignores the first +"climate" and counts the second. A query `clim AND NOT idioms` would instead drop the +whole sentence. + +Typographic variants are folded in both terms and text: dashes (– — ‐ → -), curly quotes, +ligatures and full-width characters. `human–wildlife` therefore also matches +"human-wildlife". + +### queries: expressions + +A query is a named expression, and its result is a column of the output. + +- **Operators:** `AND`, `OR`, `NOT` and parentheses. `NOT` binds tighter than `AND`, and + `AND` tighter than `OR`, so `a OR b AND c` means `a OR (b AND c)`. +- **Names:** lists, other queries (in any order; loops are reported) and sentiment flags. + Quotes around names are optional. +- **Inline terms:** `{term}` uses a term directly, with the taxonomy syntax: `fast AND {el niño}`. + +Functions: + +| function | true for a sentence when… | +|---|---| +| `NEAR(a, b, n)` | a and b occur at most n words apart, in either order | +| `BEFORE(a, b, n)` | as NEAR, with a first | +| `NOT_NEAR(a, b, n)` | some match of a has no match of b within n words. `NOT NEAR(a, b, n)` instead means no pair is near. | +| `DOC(x)`, `DOC(x) >= n` | x is true in at least one (n) sentence(s) of the same document | +| `DOC_SHARE(x) >= p` | x is true in at least the share p (0–1) of the document's sentences | +| `WINDOW(x, k)` | x is true in this sentence or within k sentences before or after it, in the same document. Distances count the sentences of the document, including any left out by `exclude_sentence_types`. | +| `COUNT(a) >= n` | a has at least n matches in the sentence | +| `COUNT_DISTINCT(a) >= n` | at least n different terms of a match in the sentence | + +Rules for the arguments: + +- **Distance** is the number of words strictly between the two matches; adjacent matches + have distance 0. Only tokens with a letter or digit count as words, so a standalone "-" + does not. Overlapping matches never count, so a word in both lists is not near itself. +- **The `a` and `b` arguments need word positions.** They must be lists, `{terms}` or OR + combinations of these, possibly through a query defined that way. +- **Comparisons** are `>=`, `>`, `=`, `!=`, `<=` and `<`. Without one, `DOC(x)` and + `COUNT(x)` mean ≥ 1, and `DOC_SHARE(x)` means > 0. +- **In COUNT and COUNT_DISTINCT,** a match lying inside a longer match of the same list + counts once ("climate change" is counted once by `climat*` and `climate change`). Each + taxonomy row is one term for COUNT_DISTINCT. + +Example queries: + +``` +co_slow NEAR(slow1, slow2, 5) +a1 fast AND NOT (slow OR co_slow OR exclude_climate) +b2 a1 AND (hwc OR co_hwc) AND NOT exclude_conflict AND DOC(ipcc) +compound COUNT_DISTINCT(hazards) >= 2 AND DOC_SHARE(climate) >= 0.05 +``` + +### variables + +| number | name | definition | parent | +|---|---|---|---| +| 1 | Climate change | a1 OR a2 | | +| 1.1 | Fast onset | a1 | | + +- **number:** `1`, `1.1`, `1.1.1`, unique. In Excel, keep the column formatted as text, as + in the template, so that `1.10` is not turned into `1.1`. +- **definition:** an expression as in queries, which may also use `V(1.1)` for another + variable. +- **parent:** optional. When empty, `1.1` belongs to `1` and `1.1.1` to `1.1`. +- **`aggregate_children = true`:** a parent is also true wherever a child is true, and its + definition may stay empty. +- **Hierarchy check:** with `aggregate_children = false`, the run warns when a child + variable is true in a sentence where its parent is false. + +The output columns are named `1. Climate change`, `1.1 Fast onset` and so on. + +### tests + +| sentence | expect_true | expect_false | group | +|---|---|---|---| +| Floods and storms destroyed the harvest. | fast, a1, 1.1 | slow, a2 | | +| The IPCC published its assessment report. | | | ipcc-doc | +| Floods led to human-wildlife conflict. | b2, 2.1.2 | b1 | ipcc-doc | + +- **When the tests run:** test sentences go through the same normalization and coding as + the corpus on every `check`, `analyze` and `run`. +- **Expectations** name lists, queries, sentiment flags or variable numbers, separated by + commas. +- **Groups:** rows with the same `group` form one document, in order, so `DOC`, `DOC_SHARE` + and `WINDOW` can be tested. A row in a group may carry no expectations and only + provide context, like the IPCC sentence above. +- **Failures** name the row, the lists that matched and the values behind the failing + query. With `tests_must_pass = true` (default) the analysis stops. + +Tests protect a framework from unintended changes when you edit terms or queries. + +## Validation against hand coding + +Tests check that rules work as intended. Validation measures how well the framework agrees +with human judgement on real sentences: + +1. `python -m ctrlfindr sample --config ctrlfindr.toml --n 50` (optionally + `--variables 1,2.1`, `--seed 7`) writes `validation_sample.tsv` to the output folder. For + each variable it contains 50 sentences CtrlFindr flagged and 50 it did not, mixed. It + does not show CtrlFindr's code, so the coding stays blind. +2. Fill `human_code` with 1 or 0 and save the file as the project's validation file + (`[paths] validation`, default `Input/validation.tsv`). +3. Every analysis then writes `validation_report.tsv` and `validation_disagreements.tsv`. + +Codes are matched to sentences by document and sentence number, and checked against the +sentence text in the file. If a document changed and was ingested again, a coded sentence +that moved is found under its new number. A coded sentence that is no longer in the +document is left out and reported, never compared with a different sentence. + +The report gives, per variable, the confusion counts, the precision and an estimated recall. +Flagged and unflagged sentences are sampled separately, so each stratum is weighted by its +size in the corpus: + +- precision = TP / coded flagged; +- recall = (precision × flagged in the corpus) / (precision × flagged in the corpus + FN rate + among coded unflagged × unflagged in the corpus); +- F1 combines the two. + +This assumes random sampling within each stratum, which is what `sample` does. + +## Sentiment + +Sentiment is off unless the framework sets `sentiment = true`, and it needs the optional +package vaderSentiment. Each sentence then gets a VADER compound score between -1 and 1, +computed on the original, not lowercased, sentence. Scores are cached in the database, +so later runs only score new or changed sentences. The five flags use the thresholds from +the settings sheet: + +| flag | true when (default thresholds) | +|---|---| +| `sentiment_positive` | compound ≥ 0.05 | +| `sentiment_negative` | compound ≤ -0.05 | +| `sentiment_neutral` | -0.05 < compound < 0.05 | +| `sentiment_neutralpositive` | compound > -0.05 | +| `sentiment_neutralnegative` | compound < 0.05 | + +A score of exactly 0 is always neutral. Points to keep in mind: + +- **Mixed flags make results asymmetric.** If one query uses `sentiment_neutralpositive` and + its counterpart uses `sentiment_negative`, neutral sentences count for the first and never + for the second. +- **VADER was built for social-media text.** On formal prose (reports, legal texts) its + scores are weak signals: much of the text scores as neutral, and domain words ("risk", + "conflict", "loss") can pull scores negative whatever the stance of the sentence. Check a + sample by hand before relying on sentiment-based variables. + +## Outputs + +All outputs are written to the `output` folder. In the TSV files, booleans are `True` and +`False`. + +| file | content | +|---|---| +| `results_document.tsv` | per document, the number of sentences in which each variable is true | +| `results_document_clipped.tsv` | the same, capped at 1 (does the document contain the variable at all?) | +| `results_document_percentage_of_sentences.tsv` | the same as a percentage of the document's sentences | +| `results_document_sentiment.tsv` | per document, the mean compound score of the sentences in which each variable is true. **An empty cell means the variable never occurred in that document**; it is not a score of 0. Written only when sentiment is on. | +| `results_sentences.tsv` | one row per sentence: `sentence_id`, `structure`, `start`, `end`, `document`, `sentences` (lowercased, normalized), every list, `vader_compound` and the sentiment flags, every query, every variable | +| `codebook.md` | the framework in readable form: meta, settings, variables, queries, lists with exclusions and notes | +| `run_metadata.json` | how the results were made: CtrlFindr and package versions, config, framework meta, settings and SHA-256, test results, SHA-256 of every text file | +| `validation_*.tsv` | see [Validation](#validation-against-hand-coding) | +| `ctrlfindr.log` | the messages of the last command-line run | + +Result files that the current settings no longer produce are removed, so the folder never +mixes runs. The database (`analysis.sqlite`) has these tables: + +- `documents`: path, format (`txt` or `md`), SHA-256 and cleaned text (for Markdown: the + plain text, one block per heading, item, row or paragraph); +- `sentences`: text, structure label and offsets; +- `sentiment`: cached VADER scores (only when a framework used sentiment); +- `results_sentences`: a copy of the sentence results. + +`clean_text[start:end]` of a document is the sentence, so a flagged sentence can be read in +context. + +## Text preparation + +The following is applied to every document (for Markdown, after the Markdown syntax is +removed): + +- **Encoding.** Files must be UTF-8; a byte-order mark, which some Windows editors add, is + ignored. Other encodings stop ingestion with a list of the affected files, unless + `encoding_errors = "replace"` is set. +- **Normalization.** Unicode NFKC plus the dash and quote folding described above. +- **Line breaks.** A single line break becomes a space, so words at line ends are never + glued together. A blank line always ends a sentence, so headings are not glued + onto the next sentence. +- **Citation marks.** A space is inserted between a full stop and a following citation mark + (`2050.[4] The` → `2050. [4] The`) so that the sentence splitter separates the sentences. +- **Missing spaces after a full stop** (`.txt` files only): `published.The minister` becomes + `published. The minister`. This applies when the full stop (or `?`, `!`) follows two + lower-case letters or digits and is followed by a capitalised word, so `U.S.Army`, + `Ph.D.Later`, `St.Louis`, `e.g.The` and `4.2.Results` are left alone. +- **Junk segments.** Segments that contain no letter (page numbers, `[12]`, lone + punctuation) are dropped. +- **Optional cleaning** (`text_cleaning = true`): URLs are removed, and `..` or `...` + becomes `.`. + +Segments longer than 500 words (configurable) are reported. They usually come from tables, +reference lists or text without punctuation, and they distort per-sentence percentages. + +## Configuration + +`ctrlfindr.toml` says where the files are and how the texts are read; how sentences are +coded is in the framework's settings sheet. See +[examples/example_project/ctrlfindr.toml](examples/example_project/ctrlfindr.toml) for +every setting with comments. Relative paths are relative to the config file's folder. + +The next ingest rebuilds the database when you: +- change `text_cleaning` or `encoding_errors`; +- upgrade CtrlFindr or pySBD. + +Upgrading vaderSentiment only clears the cached sentiment scores. + +## Stopping, resuming and memory + +- **Ingest discards outdated analysis results.** When it changes or removes documents, the + results table in the database is dropped, because sentence numbers may have shifted. + Run the analysis again before `sample`. +- **Ingest can be stopped and resumed.** Each file is saved in its own transaction as soon + as it is processed. If an ingest is stopped (Ctrl+C, a closed terminal, a crash or a killed + server job), every finished file stays saved and the next `ingest` or `run` continues with + the rest. A file that was being processed when the run stopped keeps its previous version + until it is processed again. The command line exits with code 130 after Ctrl+C. +- **Analysis runs in batches.** Batches are whole documents of about `batch_sentences` + sentences (`[analysis]` in `ctrlfindr.toml`, default 50,000). `DOC`, `DOC_SHARE` and + `WINDOW` look at whole documents, so batching never changes results. On a test corpus + of 50,850 sentences, peak memory was 170 MB in one batch and 89 MB with batches of 5,000. + Lower the value if memory is tight. +- **A stopped analysis changes nothing.** The new results are written next to the old ones + and swapped in only when the run completes. Sentiment scores computed before the stop are + kept, so the next run does not compute them again. + +## Performance + +On the development machine, the example project (7 documents, about 250,000 characters) +takes about 2 seconds for ingest and analysis together. Analysis of about 32,000 sentences +takes under a second. Sentence splitting during ingest is the slowest step, at about +200,000 characters per second, and runs once per file. + +## Development + +``` +pip install -r requirements.lock +pip install -e . --no-deps +python -m pytest +``` -# Usage +The tests include the following regression checks: -- Place the text files you want to analyze in the TXT folder or adjust the file paths in the txt_to_dataframe() function. -- Prepare the Assessment_framework.ots file containing the variables, search strings, co-occurrences, document conditionals, and taxonomy as specified in the create_dataframes() function. -- Customize the code to meet your specific requirements (e.g., utilize optional functions). -- Execute the cells in the Jupyter Notebook in the order they appear. -- The results will be saved as TSV files . +- the 2.1 code in `legacy/` reproduces its committed outputs; +- the example framework, which is the 2.1 example's rules rewritten in format 1, changes + exactly the rows explained in `docs/result_changes.md` on the 2.1 sentences; +- the full pipeline on the example project reproduces `tests/fixtures/golden_v3`. -- Check the **Jupiter Notebook Example folder** for a sample analysis of few text files. +GitHub Actions runs the tests on Windows and Linux with Python 3.11, 3.12 and 3.13. +`tools/compare_outputs.py OLD NEW` compares two output folders. -# License +## License -This project is licensed under the GNU General Public License v3.0. Please read the LICENSE file for more information. +GNU General Public License v3.0; see [LICENSE](LICENSE). -# How to cite +## How to cite -Scartozzi, Cesare M. (2023.) CtrlFindr (Version no.). Available from https://github.com/CeMSc/CtrlFindr. +Scartozzi, Cesare M. (2023). CtrlFindr (Version 3.0). Available from https://github.com/CeMSc/CtrlFindr. diff --git a/ctrlfindr/__init__.py b/ctrlfindr/__init__.py new file mode 100644 index 0000000..c91c785 --- /dev/null +++ b/ctrlfindr/__init__.py @@ -0,0 +1,7 @@ +"""CtrlFindr: dictionary-based content analysis of text documents, sentence by sentence.""" + +__version__ = "3.0.0" + +from .config import Config, config_from_dict, load_config # noqa: E402 + +__all__ = ["Config", "config_from_dict", "load_config", "__version__"] diff --git a/ctrlfindr/__main__.py b/ctrlfindr/__main__.py new file mode 100644 index 0000000..25e9e4f --- /dev/null +++ b/ctrlfindr/__main__.py @@ -0,0 +1,98 @@ +"""Command line: ``python -m ctrlfindr COMMAND --config path/to/ctrlfindr.toml``. + +Commands: template (blank framework), check (framework only), ingest, analyze, +run (ingest + analyze), sample. +""" + +from __future__ import annotations + +import argparse +import logging +import sys + +from . import __version__ +from .common import CtrlFindrError, setup_logging +from .config import load_config + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="ctrlfindr", description="Sentence-level content analysis of text files.") + parser.add_argument("--version", action="version", version=f"CtrlFindr {__version__}") + sub = parser.add_subparsers(dest="command", required=True) + commands = { + "check": "validate the assessment framework and run its tests (no text needed)", + "ingest": "read the text files into the database (only new or changed files are processed)", + "analyze": "apply the framework to the ingested sentences and write the results", + "run": "ingest, then analyze", + "sample": "write a blind sample of sentences to hand-code for validation", + } + tp = sub.add_parser("template", help="write a blank assessment framework (.ods or .xlsx)") + tp.add_argument("path", help="file to create, e.g. Input/Assessment_framework.xlsx") + for name, help_ in commands.items(): + p = sub.add_parser(name, help=help_) + p.add_argument("--config", "-c", default="ctrlfindr.toml", help="path to ctrlfindr.toml (default: ./ctrlfindr.toml)") + p.add_argument("--verbose", "-v", action="store_true", help="show debug messages") + p.add_argument("--no-progress", action="store_true", help="hide progress bars") + if name == "sample": + p.add_argument("--n", type=int, default=50, help="sentences per stratum (flagged / not flagged) and variable") + p.add_argument("--variables", default="", help="comma-separated variable numbers (default: all)") + p.add_argument("--seed", type=int, default=1, help="random seed, recorded for reproducibility") + args = parser.parse_args(argv) + + log = logging.getLogger("ctrlfindr") + if args.command == "template": + from pathlib import Path + + from .template import write_template + + path = Path(args.path) + if path.exists(): + print(f"ERROR: {path} already exists; choose another file name", file=sys.stderr) + return 2 + if path.suffix.lower() not in (".ods", ".xlsx"): + print("ERROR: the file name must end in .ods or .xlsx", file=sys.stderr) + return 2 + path.parent.mkdir(parents=True, exist_ok=True) + write_template(path) + print(f"Wrote {path}") + return 0 + try: + cfg = load_config(args.config) + setup_logging(logging.DEBUG if args.verbose else logging.INFO, cfg.paths.output / "ctrlfindr.log") + progress = not args.no_progress + if args.command in ("check", "analyze", "run"): + # Validate and test the framework before spending time on ingestion. + from .analyze import check_framework + from .framework import load_framework + + fw = load_framework(cfg.paths.framework) + log.info("Framework %s is valid: %d lists, %d queries, %d variables", + cfg.paths.framework.name, len(fw.lists), len(fw.queries), len(fw.variables)) + check_framework(fw) + if args.command in ("ingest", "run"): + from . import ingest + + ingest.run(cfg, show_progress=progress) + if args.command in ("analyze", "run"): + from . import analyze + + analyze.run(cfg, show_progress=progress, run_tests_first=False) + if args.command == "sample": + from .validation import sample + + targets = [t.strip() for t in args.variables.split(",") if t.strip()] or None + sample(cfg, n=args.n, targets=targets, seed=args.seed) + except KeyboardInterrupt: + print("Stopped. Run the same command again to continue: finished files are kept.", file=sys.stderr) + return 130 + except CtrlFindrError as exc: + if log.handlers: + log.error("%s", exc) + else: # the config could not be loaded, so logging is not set up yet + print(f"ERROR: {exc}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ctrlfindr/analyze.py b/ctrlfindr/analyze.py new file mode 100644 index 0000000..5a9dcbe --- /dev/null +++ b/ctrlfindr/analyze.py @@ -0,0 +1,292 @@ +"""Analysis step: ingested sentences + framework -> coded sentences, five TSVs, codebook.""" + +from __future__ import annotations + +import json +import logging +import platform +import sys +from dataclasses import asdict +from datetime import datetime, timezone +from importlib import metadata +from pathlib import Path + +import pandas as pd + +from . import __version__ +from .codebook import write_codebook +from .common import ( + DOCUMENTS, + RESULTS, + RESULTS_NEW, + SENTENCES, + CtrlFindrError, + ValidationError, + connect, + table_exists, +) +from .config import Config +from .engine import AnalysisStats, analyze_sentences, exclude_sentences, run_tests +from .framework import Framework, load_framework +from .matching import Matcher + +log = logging.getLogger("ctrlfindr") + +OUTPUT_FILES = ( + "results_document", + "results_document_clipped", + "results_document_percentage_of_sentences", + "results_document_sentiment", + "results_sentences", +) + + +class DocumentTables: + """Builds the four document-level tables from batches of results (whole documents).""" + + def __init__(self, fw: Framework): + self.fw = fw + self.columns = [var.column for var in fw.variables] + self.sums: list[pd.DataFrame] = [] + self.counts: list[pd.Series] = [] + self.sent_sum: list[pd.DataFrame] = [] + self.sent_n: list[pd.DataFrame] = [] + + def add(self, results: pd.DataFrame) -> None: + v = self.columns + grouped = results.groupby("document", sort=True) + self.sums.append(grouped[v].sum()) + self.counts.append(grouped["sentences"].count()) + if self.fw.settings.sentiment: + flags = results[v].to_numpy(bool) + compound = results["vader_compound"].to_numpy(float)[:, None] + frame = pd.DataFrame(flags * compound, columns=v) + frame["document"] = results["document"].to_numpy() + self.sent_sum.append(frame.groupby("document", sort=True)[v].sum()) + n = pd.DataFrame(flags.astype(int), columns=v) + n["document"] = results["document"].to_numpy() + self.sent_n.append(n.groupby("document", sort=True)[v].sum()) + + def tables(self) -> dict[str, pd.DataFrame]: + v = self.columns + sums = pd.concat(self.sums) if self.sums else pd.DataFrame(columns=v) + counts = pd.concat(self.counts) if self.counts else pd.Series(dtype=int) + sums.index.name = counts.index.name = "document" + out = { + "results_document": sums.astype(int).reset_index(), + "results_document_clipped": sums.clip(upper=1).reset_index(), + "results_document_percentage_of_sentences": sums.div(counts, axis=0).multiply(100).reset_index(), + } + if self.fw.settings.sentiment: + # Mean compound score of the sentences flagged by each variable; empty (NaN) when + # a variable never fired in a document. + total, n = pd.concat(self.sent_sum), pd.concat(self.sent_n) + means = total.div(n.where(n > 0)) + means.index.name = "document" + out["results_document_sentiment"] = means.reset_index() + return out + + +def document_outputs(results: pd.DataFrame, fw: Framework) -> dict[str, pd.DataFrame]: + """The four document-level tables for a complete results table.""" + acc = DocumentTables(fw) + acc.add(results) + return acc.tables() + + +def write_tsv(df: pd.DataFrame, path: Path) -> None: + df.to_csv(path, sep="\t", index=False, encoding="utf-8", lineterminator="\n") + + +def check_framework(fw: Framework) -> list[str]: + """Run the framework's tests; raise if they fail and the framework requires them to pass.""" + failures = run_tests(fw) + if failures and fw.settings.tests_must_pass: + raise ValidationError(f"{len(failures)} framework test(s) failed (tests sheet)", failures) + for f in failures: + log.warning("%s", f) + if fw.tests and not failures: + log.info("Framework tests: all %d passed", len(fw.tests)) + return failures + + +def _package_versions() -> dict[str, str]: + out = {} + for pkg in ("pandas", "numpy", "pysbd", "vaderSentiment", "pyahocorasick", "openpyxl", "tqdm"): # vader optional + try: + out[pkg] = metadata.version(pkg) + except metadata.PackageNotFoundError: + out[pkg] = "not installed" + return out + + +def run(cfg: Config, show_progress: bool = True, run_tests_first: bool = True) -> dict[str, pd.DataFrame]: + """Validate and test the framework, analyze the ingested sentences, write all outputs. + + The sentences are processed in batches of whole documents ([analysis] batch_sentences), + so memory use does not grow with the corpus. Earlier outputs are only replaced when + the run completes; a stopped run leaves them as they were. Returns the document tables. + + With ``run_tests_first=False`` failing tests are only counted for run_metadata.json, + not reported or enforced (the command line has already done that). + """ + fw = load_framework(cfg.paths.framework) + log.info("Framework OK: %d lists, %d queries, %d variables, %d tests", + len(fw.lists), len(fw.queries), len(fw.variables), len(fw.tests)) + failures = check_framework(fw) if run_tests_first else run_tests(fw) + + db = cfg.paths.database + if not db.is_file(): + raise CtrlFindrError(f"database {db} not found; run the ingest step first") + out_dir = cfg.paths.output + out_dir.mkdir(parents=True, exist_ok=True) + partial = out_dir / "results_sentences.tsv.partial" + tables = DocumentTables(fw) + stats = AnalysisStats() + collector = None + if cfg.paths.validation.is_file(): + from .validation import ValidationCollector + + collector = ValidationCollector(cfg.paths.validation, fw) + matcher = Matcher(fw.all_lists) + n_sentences = n_scored = 0 + + with connect(db) as conn: + if not (table_exists(conn, SENTENCES) and table_exists(conn, DOCUMENTS)): + raise CtrlFindrError(f"database {db} has no ingested documents; run the ingest step first") + documents = pd.read_sql_query(f"SELECT document, path, format, sha256 FROM {DOCUMENTS} ORDER BY document", conn) + counts = conn.execute(f"SELECT document, COUNT(*) FROM {SENTENCES} GROUP BY document ORDER BY document").fetchall() + if not counts: + raise CtrlFindrError("the database contains no sentences; check the text files and run ingest again") + batches = _batches(counts, cfg.analysis.batch_sentences) + total = sum(c for _, c in counts) + log.info("Analyzing %d sentences from %d documents in %d batch(es)", total, len(counts), len(batches)) + conn.execute(f"DROP TABLE IF EXISTS {RESULTS_NEW}") + bar = None + if show_progress: + from tqdm.auto import tqdm + + bar = tqdm(total=total, desc="analyze", unit=" sentences", leave=False, mininterval=0.5) + try: + for first, last, n in batches: + sentences = pd.read_sql_query( + f"SELECT * FROM {SENTENCES} WHERE document BETWEEN ? AND ? ORDER BY document, sentence_id", + conn, params=(first, last)) + sentences = exclude_sentences(sentences, fw.settings, stats) + if not sentences.empty: + if fw.settings.sentiment: + from .sentiment import cached_scores + + sentences["vader_compound"], scored = cached_scores(conn, sentences) + n_scored += scored + results = analyze_sentences(sentences, fw, long_segment_words=cfg.ingest.long_segment_words, + matcher=matcher, stats=stats) + tables.add(results) + if collector is not None: + collector.add(results) + if cfg.export.sentence_level: + results.to_csv(partial, sep="\t", index=False, encoding="utf-8", lineterminator="\n", + mode="w" if n_sentences == 0 else "a", header=n_sentences == 0) + results.to_sql(RESULTS_NEW, conn, if_exists="append", index=False) + n_sentences += len(results) + conn.commit() # sentiment scores of finished batches are kept even if the run stops + if bar is not None: + bar.update(n) + if n_sentences == 0: + raise CtrlFindrError("every sentence is excluded by exclude_sentence_types in the settings sheet") + # Replace the previous results only now that the run is complete, in one transaction + # (sqlite3 would otherwise run each of these statements on its own). + conn.execute("BEGIN") + conn.execute(f"DROP TABLE IF EXISTS {RESULTS}") + conn.execute(f"ALTER TABLE {RESULTS_NEW} RENAME TO {RESULTS}") + conn.commit() + except BaseException: + conn.rollback() + conn.execute(f"DROP TABLE IF EXISTS {RESULTS_NEW}") + conn.commit() + partial.unlink(missing_ok=True) + raise + finally: + if bar is not None: + bar.close() + stats.report() + if n_scored: + log.info("Sentiment: %d sentence(s) scored with VADER, the others came from the cache", n_scored) + + outputs: dict[str, pd.DataFrame] = tables.tables() if cfg.export.document_level else {} + written = [] + for name, df in outputs.items(): + write_tsv(df, out_dir / f"{name}.tsv") + written.append(f"{name}.tsv") + if cfg.export.sentence_level: + partial.replace(out_dir / "results_sentences.tsv") + written.append("results_sentences.tsv") + # Remove result files of an earlier run that this run did not produce, so the folder + # never mixes results of different runs. + stale = [p for p in (out_dir / f"{name}.tsv" for name in OUTPUT_FILES) if p.is_file() and p.name not in written] + for p in stale: + p.unlink() + if stale: + log.info("Removed %s from an earlier run (not produced with the current settings)", + ", ".join(p.name for p in stale)) + write_codebook(fw, out_dir / "codebook.md") + written.append("codebook.md") + + validation = None + validation_files = [out_dir / "validation_report.tsv", out_dir / "validation_disagreements.tsv"] + if collector is not None: + validation = collector.finish(out_dir) + written += [p.name for p in validation_files] + else: # no validation file any more: do not leave an old report behind + for p in validation_files: + if p.is_file(): + p.unlink() + log.info("Removed %s from an earlier run (no validation file now)", p.name) + + meta = { + "ctrlfindr_version": __version__, + "run_at_utc": datetime.now(timezone.utc).isoformat(timespec="seconds"), + "python": sys.version.split()[0], + "platform": platform.platform(), + "packages": _package_versions(), + "config": cfg.to_dict(), + "framework": { + "file": str(fw.path), + "sha256": fw.sha256, + "meta": fw.meta, + "settings": asdict(fw.settings), + "tests": {"total": len(fw.tests), "failed": len(failures)}, + }, + "corpus": { + "documents": len(documents), + "sentences": n_sentences, + "files": documents.to_dict(orient="records"), + }, + "validation": validation, + "outputs": sorted(written), + } + with open(out_dir / "run_metadata.json", "w", encoding="utf-8", newline="\n") as fh: + json.dump(meta, fh, indent=2, ensure_ascii=False) + fh.write("\n") + log.info("Wrote %s to %s", ", ".join(sorted(written) + ["run_metadata.json"]), out_dir) + return outputs + + +def _batches(counts: list[tuple[str, int]], size: int) -> list[tuple[str, str, int]]: + """Group consecutive documents into batches of about ``size`` sentences. + + Returns (first document, last document, sentences) per batch. A document is never split: + DOC, DOC_SHARE and WINDOW need all of its sentences. + """ + out: list[tuple[str, str, int]] = [] + current: list[str] = [] + n = 0 + for doc, c in counts: + if current and n + c > size: + out.append((current[0], current[-1], n)) + current, n = [], 0 + current.append(doc) + n += c + if current: + out.append((current[0], current[-1], n)) + return out diff --git a/ctrlfindr/codebook.py b/ctrlfindr/codebook.py new file mode 100644 index 0000000..6b7c753 --- /dev/null +++ b/ctrlfindr/codebook.py @@ -0,0 +1,59 @@ +"""Human-readable codebook of a framework (Markdown), for appendices and review.""" + +from __future__ import annotations + +from dataclasses import asdict +from pathlib import Path + +from . import query as q +from .framework import Framework + + +def _md(text: str) -> str: + return text.replace("|", "\\|").replace("\n", " ") + + +def codebook(fw: Framework) -> str: + title = fw.meta.get("title") or fw.path.stem + out = [f"# Codebook: {_md(title)}", ""] + for key, value in fw.meta.items(): + if key != "title": + out.append(f"- **{_md(key)}:** {_md(value)}") + out.append(f"- **file:** {fw.path.name} (SHA-256 `{fw.sha256[:16]}…`)") + out += ["", "## Settings", ""] + out += [f"- `{k}` = {(str(v).lower() if isinstance(v, bool) else ', '.join(v) or '(none)' if isinstance(v, tuple) else v)}" + for k, v in asdict(fw.settings).items()] + + out += ["", "## Variables", "", "| number | name | definition | parent | note |", "|---|---|---|---|---|"] + for v in fw.variables: + definition = q.to_text(v.ast) if v.ast is not None else "" + if fw.settings.aggregate_children and v.children: + children = " OR ".join(f"V({c})" for c in v.children) + definition = f"{definition} OR {children}" if definition else children + out.append(f"| {v.number} | {_md(v.name)} | `{_md(definition)}` | {v.parent or ''} | {_md(v.note)} |") + + out += ["", "## Queries", "", "| name | query | note |", "|---|---|---|"] + for qq in fw.queries: + out.append(f"| `{qq.name}` | `{_md(q.to_text(qq.ast))}` | {_md(qq.note)} |") + + out += ["", "## Lists", ""] + for name, spec in fw.lists.items(): + out.append(f"### `{name}`" + (f": {_md(fw.notes[name])}" if name in fw.notes else "")) + out.append("") + out.append("Terms: " + ", ".join(f"`{t}`" for t in spec.terms)) + excl = [f"`{t}`" for t in spec.exclude_terms] + [f"all terms of `{x}`" for x in spec.exclude_lists] + if excl: + out.append("") + out.append("Ignored inside: " + ", ".join(excl)) + notes = [(t, n) for (lst, t), n in fw.term_notes.items() if lst == name] + if notes: + out.append("") + out += [f"- `{t}`: {_md(n)}" for t, n in notes] + out.append("") + if fw.tests: + out += ["## Tests", "", f"{len(fw.tests)} test sentence(s) in the tests sheet.", ""] + return "\n".join(out) + "\n" + + +def write_codebook(fw: Framework, path: Path) -> None: + path.write_text(codebook(fw), encoding="utf-8", newline="\n") diff --git a/ctrlfindr/common.py b/ctrlfindr/common.py new file mode 100644 index 0000000..ee0b18e --- /dev/null +++ b/ctrlfindr/common.py @@ -0,0 +1,130 @@ +"""Shared helpers: errors, text normalization, logging, SQLite access.""" + +from __future__ import annotations + +import contextlib +import hashlib +import logging +import re +import sqlite3 +import unicodedata +from collections.abc import Iterator +from pathlib import Path + + +LOGGER_NAME = "ctrlfindr" +log = logging.getLogger(LOGGER_NAME) + + +class CtrlFindrError(Exception): + """An error caused by the user's input or settings (reported without a traceback).""" + + +class ValidationError(CtrlFindrError): + """One or more problems found while validating inputs; all are reported together.""" + + def __init__(self, title: str, problems: list[str]): + self.title = title + self.problems = problems + lines = [f"{title} ({len(problems)} problem{'s' if len(problems) != 1 else ''}):"] + lines += [f" - {p}" for p in problems] + super().__init__("\n".join(lines)) + + +# --------------------------------------------------------------------------- text + +# Typographic variants folded to their ASCII form, so that a term typed with a plain +# hyphen or straight quote matches text that uses an en dash or curly quote (and the +# reverse). Invisible characters that would split words are removed. +_FOLD = str.maketrans( + { + **{c: "-" for c in "‐‑‒–—―−﹘﹣-"}, + **{c: "'" for c in "‘’‚‛′ʼ"}, + **{c: '"' for c in "“”„‟″«»"}, + "­": None, # soft hyphen + "​": None, # zero-width space + "‌": None, + "‍": None, + "": None, # byte-order mark / zero-width no-break space + } +) + + +def normalize_text(text: str) -> str: + """NFKC-normalize and fold dashes/quotes. Idempotent; applied to corpus and framework.""" + return unicodedata.normalize("NFKC", text).translate(_FOLD) + + +_WS = re.compile(r"\s+") + + +def collapse_ws(text: str) -> str: + return _WS.sub(" ", text).strip() + + +def sha256_file(path: Path) -> str: + h = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + h.update(chunk) + return h.hexdigest() + + +# ------------------------------------------------------------------------ logging + +def setup_logging(level: int = logging.INFO, log_file: Path | None = None) -> None: + """Configure the package logger once (console, plus an optional UTF-8 log file).""" + logger = logging.getLogger(LOGGER_NAME) + logger.setLevel(level) + for h in list(logger.handlers): + logger.removeHandler(h) + h.close() + console = logging.StreamHandler() + console.setFormatter(logging.Formatter("%(levelname)s: %(message)s")) + logger.addHandler(console) + if log_file is not None: + log_file.parent.mkdir(parents=True, exist_ok=True) + fh = logging.FileHandler(log_file, mode="w", encoding="utf-8") + fh.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(message)s")) + logger.addHandler(fh) + logger.propagate = False + + +# ------------------------------------------------------------------------- SQLite + +DOCUMENTS = "documents" +SENTENCES = "sentences" +RESULTS = "results_sentences" +RESULTS_NEW = "results_sentences_new" # filled batch by batch, renamed to RESULTS at the end +SENTIMENT = "sentiment" # cache of VADER scores, filled by the analysis when sentiment is on +META = "meta" +_TABLES = {DOCUMENTS, SENTENCES, RESULTS, RESULTS_NEW, SENTIMENT, META} + + +def _table(name: str) -> str: + """Return a table name after checking it against the fixed whitelist.""" + if name not in _TABLES: + raise ValueError(f"unknown table {name!r}") + return name + + +@contextlib.contextmanager +def connect(db_path: Path) -> Iterator[sqlite3.Connection]: + """Open the database, commit on success, roll back on error, always close.""" + db_path.parent.mkdir(parents=True, exist_ok=True) + conn = sqlite3.connect(db_path) + try: + yield conn + conn.commit() + except BaseException: + conn.rollback() + raise + finally: + conn.close() + + +def table_exists(conn: sqlite3.Connection, name: str) -> bool: + row = conn.execute( + "SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (_table(name),) + ).fetchone() + return row is not None diff --git a/ctrlfindr/config.py b/ctrlfindr/config.py new file mode 100644 index 0000000..18a4562 --- /dev/null +++ b/ctrlfindr/config.py @@ -0,0 +1,136 @@ +"""Project configuration (``ctrlfindr.toml``): where files are and how the corpus is read. + +Everything that changes how sentences are coded (sentiment thresholds and so on) lives in +the framework's settings sheet instead, so a framework fully defines its coding. +Relative paths are resolved against the folder that contains the config file, so the +same project folder works on Windows and Linux and from any working directory. +""" + +from __future__ import annotations + +import tomllib +from dataclasses import asdict, dataclass, field +from pathlib import Path +from typing import Any + +from .common import ValidationError + + +@dataclass +class Paths: + text_files: Path = Path("Text_files") + framework: Path = Path("Input/Assessment_framework.ods") + output: Path = Path("Output") + database: Path = Path("Output/analysis.sqlite") + validation: Path = Path("Input/validation.tsv") # hand-coded sentences; used if the file exists + + +@dataclass +class IngestSettings: + text_cleaning: bool = True # remove URLs, collapse "..." to "." + encoding_errors: str = "strict" # "strict": stop on non-UTF-8 files; "replace": substitute bad bytes + long_segment_words: int = 500 # warn about "sentences" longer than this + + +@dataclass +class AnalysisSettings: + batch_sentences: int = 50000 # sentences per batch (whole documents); lower it to use less memory + + +@dataclass +class ExportSettings: + sentence_level: bool = True + document_level: bool = True + + +@dataclass +class Config: + paths: Paths = field(default_factory=Paths) + ingest: IngestSettings = field(default_factory=IngestSettings) + analysis: AnalysisSettings = field(default_factory=AnalysisSettings) + export: ExportSettings = field(default_factory=ExportSettings) + source: Path | None = None # the config file, if loaded from one + + def to_dict(self) -> dict[str, Any]: + d = asdict(self) + d["source"] = str(self.source) if self.source else None + d["paths"] = {k: str(v) for k, v in d["paths"].items()} + return d + + +_SECTIONS = {"paths": Paths, "ingest": IngestSettings, "analysis": AnalysisSettings, "export": ExportSettings} + + +def _build(raw: dict[str, Any], base: Path, problems: list[str]) -> Config: + cfg = Config() + for section, value in raw.items(): + if section == "sentiment": + problems.append("[sentiment] is not a config setting: sentiment is set in the framework's settings sheet") + continue + if section not in _SECTIONS: + problems.append(f"unknown section [{section}] (allowed: {', '.join(_SECTIONS)})") + continue + if not isinstance(value, dict): + problems.append(f"[{section}] must be a table") + continue + target = getattr(cfg, section) + for key, v in value.items(): + if not hasattr(target, key): + allowed = ", ".join(target.__dataclass_fields__) + problems.append(f"[{section}] unknown setting {key!r} (allowed: {allowed})") + continue + default = getattr(target, key) + if isinstance(default, Path): + if not isinstance(v, str): + problems.append(f"[{section}] {key} must be a path in quotes") + continue + v = Path(v) + elif isinstance(default, bool): + if not isinstance(v, bool): + problems.append(f"[{section}] {key} must be true or false") + continue + elif isinstance(default, (int, float)): + if isinstance(v, bool) or not isinstance(v, (int, float)): + problems.append(f"[{section}] {key} must be a number") + continue + if isinstance(default, int) and v != int(v): + problems.append(f"[{section}] {key} must be a whole number") + continue + v = type(default)(v) + setattr(target, key, v) + for key in cfg.paths.__dataclass_fields__: + p = getattr(cfg.paths, key) + if not p.is_absolute(): + setattr(cfg.paths, key, (base / p).resolve()) + if cfg.analysis.batch_sentences < 1: + problems.append("[analysis] batch_sentences must be at least 1") + if cfg.ingest.encoding_errors not in ("strict", "replace"): + problems.append('[ingest] encoding_errors must be "strict" or "replace"') + return cfg + + +def load_config(path: str | Path) -> Config: + """Read a TOML config file; raises ValidationError listing every problem.""" + path = Path(path) + if not path.is_file(): + raise ValidationError("Configuration not found", [f"file does not exist: {path}"]) + try: + with open(path, "rb") as fh: + raw = tomllib.load(fh) + except tomllib.TOMLDecodeError as exc: + raise ValidationError(f"Configuration {path.name} is not valid TOML", [str(exc)]) from exc + problems: list[str] = [] + cfg = _build(raw, path.parent.resolve(), problems) + if problems: + raise ValidationError(f"Configuration {path.name} is invalid", problems) + cfg.source = path.resolve() + return cfg + + +def config_from_dict(raw: dict[str, Any], base: str | Path = ".") -> Config: + """Build a config from a dict with the same layout as the TOML file (for notebooks/tests).""" + problems: list[str] = [] + cfg = _build(raw, Path(base).resolve(), problems) + if problems: + raise ValidationError("Configuration is invalid", problems) + return cfg diff --git a/ctrlfindr/engine.py b/ctrlfindr/engine.py new file mode 100644 index 0000000..6e1e6fa --- /dev/null +++ b/ctrlfindr/engine.py @@ -0,0 +1,359 @@ +"""Evaluate a framework on a table of sentences, and run the framework's own tests.""" + +from __future__ import annotations + +import logging +import operator +from collections import Counter +from dataclasses import dataclass, field + +import numpy as np +import pandas as pd + +from . import query as q +from .common import CtrlFindrError, normalize_text +from .framework import SENTIMENT_TERMS, Framework, Settings +from .matching import Match, Matcher, count, count_distinct, near, not_near, word_tokens + +log = logging.getLogger("ctrlfindr") + +_COMPARE = {">=": operator.ge, ">": operator.gt, "=": operator.eq, "!=": operator.ne, + "<=": operator.le, "<": operator.lt} + + +def sentiment_flags(compound: np.ndarray, settings: Settings) -> dict[str, np.ndarray]: + """The five sentiment flags for VADER compound scores. + + positive >= positive threshold, negative <= negative threshold, neutral in between. + A score of exactly 0 is always neutral. + """ + c = np.asarray(compound, dtype=float) + pos, neg = settings.sentiment_positive_threshold, settings.sentiment_negative_threshold + neutral = ((c > neg) & (c < pos)) | (c == 0) + positive = (c >= pos) & ~neutral + negative = (c <= neg) & ~neutral + return { + "sentiment_positive": positive, + "sentiment_neutral": neutral, + "sentiment_neutralpositive": positive | neutral, + "sentiment_negative": negative, + "sentiment_neutralnegative": negative | neutral, + } + + +class Evaluator: + """Holds the per-sentence columns and evaluates expressions over them.""" + + def __init__(self, fw: Framework, text: list[str], documents: np.ndarray, positions: np.ndarray | None = None): + """``positions``: sentence numbers within each document (sentence_id). WINDOW measures + distances with them, so sentences left out (exclude_sentence_types) still count as + being in between. Without them, the row order is used.""" + self.fw = fw + self.text = text + self.n = len(text) + self.doc = pd.factorize(documents)[0] + if self.n and np.count_nonzero(np.diff(self.doc)) != self.doc.max(): + # WINDOW and the document functions need the rows of a document to be contiguous + raise CtrlFindrError("the sentences of each document must be consecutive rows") + self.columns: dict[str, np.ndarray] = {} + self.matches: dict[str, list[list[Match] | None]] = {} + self.variables: dict[str, np.ndarray] = {} + self._tokens: dict[int, tuple] = {} + if self.n: + codes = self.doc + first = np.r_[0, np.flatnonzero(np.diff(codes)) + 1] + last = np.r_[first[1:], self.n] - 1 + run = np.repeat(np.arange(len(first)), np.diff(np.r_[first, self.n])) + self.doc_first, self.doc_last = first[run], last[run] + pos = np.arange(self.n) if positions is None else np.asarray(positions, dtype=np.int64) + same_doc = codes[1:] == codes[:-1] + if np.any(np.diff(pos)[same_doc] <= 0): + raise CtrlFindrError("sentence numbers must increase within each document") + self.pos = pos + + # ------------------------------------------------------------------ helpers + + def tokens(self, row: int): + t = self._tokens.get(row) + if t is None: + t = self._tokens[row] = word_tokens(self.text[row]) + return t + + def set_matches(self, arg: q.Node) -> list[list[Match]]: + lists = self.fw.set_lists[id(arg)] + out: list[list[Match]] = [[] for _ in range(self.n)] + for name in lists: + for row, m in enumerate(self.matches[name]): + if m: + out[row].extend(m) + if len(lists) > 1: + out = [sorted(set(m)) for m in out] + return out + + def _group_count(self, codes: np.ndarray, inner: np.ndarray) -> np.ndarray: + if not self.n: + return np.zeros(0) + return np.bincount(codes, weights=inner.astype(float))[codes] + + # --------------------------------------------------------------- evaluation + + def evaluate(self, node: q.Node) -> np.ndarray: + if isinstance(node, q.Name): + return self.columns[node.name] + if isinstance(node, q.InlineTerm): + return self.columns["{" + node.text + "}"] + if isinstance(node, q.Not): + return ~self.evaluate(node.operand) + if isinstance(node, (q.And, q.Or)): + parts = [self.evaluate(o) for o in node.operands] + return (np.logical_and if isinstance(node, q.And) else np.logical_or).reduce(parts) + return self._call(node) + + def _call(self, node: q.Call) -> np.ndarray: + f = node.func + out = np.zeros(self.n, dtype=bool) + if f == "v": + return self.variables[node.args[0]] + if f in ("near", "before", "not_near"): + a, b = self.set_matches(node.args[0]), self.set_matches(node.args[1]) + dist = node.args[2] + for row in range(self.n): + if f == "not_near": + if a[row]: + out[row] = not_near(self.tokens(row), a[row], b[row], dist) + elif a[row] and b[row]: + out[row] = near(self.tokens(row), a[row], b[row], dist, ordered=(f == "before")) + return out + if f == "window": + inner = self.evaluate(node.args[0]) + k = node.args[1] + if not self.n: + return out + # rows of the same document within k sentence numbers: search a key that keeps + # documents apart (gap larger than any position plus k) + span = int(self.pos.max() - self.pos.min()) + k + 1 + key = self.doc.astype(np.int64) * span + (self.pos - self.pos.min()) + lo = np.maximum(np.searchsorted(key, key - k, side="left"), self.doc_first) + hi = np.minimum(np.searchsorted(key, key + k, side="right") - 1, self.doc_last) + cs = np.r_[0, np.cumsum(inner)] + return (cs[hi + 1] - cs[lo]) > 0 + cmp = _COMPARE[node.op] + if f == "doc": + inner = self.evaluate(node.args[0]) + counts = self._group_count(self.doc, inner) + return cmp(counts, node.value) + if f == "doc_share": + inner = self.evaluate(node.args[0]) + counts = self._group_count(self.doc, inner) + sizes = self._group_count(self.doc, np.ones(self.n, dtype=bool)) + return cmp(counts / np.maximum(sizes, 1), node.value) + if f in ("count", "count_distinct"): + fn = count if f == "count" else count_distinct + sets = self.set_matches(node.args[0]) + values = np.fromiter((fn(m) if m else 0 for m in sets), dtype=float, count=self.n) + return cmp(values, node.value) + raise AssertionError(f) + + +@dataclass +class AnalysisStats: + """Counts collected over all batches of an analysis, reported once at the end.""" + + excluded: Counter = field(default_factory=Counter) + emptied_documents: list[str] = field(default_factory=list) + hierarchy: Counter = field(default_factory=Counter) # (child, parent) -> sentences + long_segments: int = 0 + long_examples: list[str] = field(default_factory=list) + long_limit: int = 0 + + def report(self) -> None: + if self.excluded: + log.info("Excluded %d sentence(s) by type: %s", sum(self.excluded.values()), + ", ".join(f"{k} {v}" for k, v in sorted(self.excluded.items()))) + if self.emptied_documents: + gone = self.emptied_documents + log.warning("%d document(s) have no sentences left after the exclusion and are not in the results: %s", + len(gone), ", ".join(gone[:10]) + (" ..." if len(gone) > 10 else "")) + broken = [(c, p, k) for (c, p), k in self.hierarchy.items() if k] + if broken: + log.warning( + "%d child variable(s) are true in sentences where their parent is false (set " + "aggregate_children = true to make parents include their children): %s", + len(broken), "; ".join(f"{c} vs parent {p}: {k} sentences" for c, p, k in broken[:5]), + ) + if self.long_segments: + log.warning( + "%d segment(s) are longer than %d words, which usually means sentence splitting " + "failed (tables, reference lists, text without punctuation). They are slow to " + "process and distort per-sentence percentages. First: %s", + self.long_segments, self.long_limit, ", ".join(self.long_examples), + ) + + +def exclude_sentences(sentences: pd.DataFrame, settings: Settings, stats: AnalysisStats | None = None) -> pd.DataFrame: + """Drop the sentences whose structure label is in ``exclude_sentence_types``.""" + if not settings.exclude_sentence_types or "structure" not in sentences: + return sentences + drop = sentences["structure"].isin(settings.exclude_sentence_types) + own = stats is None + stats = stats or AnalysisStats() + if drop.any(): + stats.excluded.update(sentences.loc[drop, "structure"].value_counts().to_dict()) + stats.emptied_documents += sorted(set(sentences.loc[drop, "document"]) - set(sentences.loc[~drop, "document"])) + if own: + stats.report() + return sentences.loc[~drop].reset_index(drop=True) + + +def analyze_sentences( + sentences: pd.DataFrame, + fw: Framework, + long_segment_words: int = 500, + show_progress: bool = False, + check_hierarchy: bool = True, + matcher: Matcher | None = None, + stats: AnalysisStats | None = None, +) -> pd.DataFrame: + """Code every sentence. + + ``sentences`` needs ``document`` and ``text`` columns, ``vader_compound`` when the + framework uses sentiment, and optionally ``sentence_id``/``structure``/``start``/``end``, + which are carried through. The rows of a document must be consecutive and complete: + DOC, DOC_SHARE and WINDOW look at the whole document. A batch of whole documents can + therefore be analyzed on its own. Pass ``matcher`` to reuse one between batches, and + ``stats`` to collect warnings over batches instead of logging them here. + """ + own_stats = stats is None + stats = stats if stats is not None else AnalysisStats() + sentences = sentences.reset_index(drop=True) + text = [normalize_text(t).lower() for t in sentences["text"].fillna("").astype(str)] + n = len(text) + positions = sentences["sentence_id"].to_numpy() if "sentence_id" in sentences else None + ev = Evaluator(fw, text, sentences["document"].to_numpy(), positions) + + matcher = matcher or Matcher(fw.all_lists) + needed = {name for lists in fw.set_lists.values() for name in lists} + progress = None + if show_progress: + from tqdm.auto import tqdm + + progress = lambda it: tqdm(it, desc="matching", unit=" sentences", leave=False, mininterval=0.5) # noqa: E731 + hits, kept = matcher.match(text, needed, progress) + ev.columns.update(hits) + ev.matches.update(kept) + + compound = None + if fw.settings.sentiment: + if "vader_compound" not in sentences: + raise CtrlFindrError("sentiment = true, but the sentences have no vader_compound scores; analyze.run " + "adds them (or pass them in a vader_compound column)") + compound = pd.to_numeric(sentences["vader_compound"], errors="coerce").to_numpy(float) + if np.isnan(compound).any(): + raise CtrlFindrError("some sentences have no vader_compound score; analyze.run computes them " + "when sentiment = true") + ev.columns.update(sentiment_flags(compound, fw.settings)) + + by_name = {qq.name: qq for qq in fw.queries} + for name in fw.query_order: + ev.columns[name] = ev.evaluate(by_name[name].ast) + + by_number = {v.number: v for v in fw.variables} + for number in fw.variable_order: + v = by_number[number] + value = ev.evaluate(v.ast) if v.ast is not None else np.zeros(n, dtype=bool) + if fw.settings.aggregate_children: + for child in v.children: + value = value | ev.variables[child] + ev.variables[number] = value + + if check_hierarchy and not fw.settings.aggregate_children: + for v in fw.variables: + if v.parent: + stats.hierarchy[(v.number, v.parent)] += int((ev.variables[v.number] & ~ev.variables[v.parent]).sum()) + + if long_segment_words and n: + stats.long_limit = long_segment_words + lengths = np.fromiter((len(t.split()) for t in text), dtype=int, count=n) + long_rows = np.flatnonzero(lengths > long_segment_words) + if len(long_rows): + ids = sentences["sentence_id"] if "sentence_id" in sentences else pd.Series(np.arange(1, n + 1)) + stats.long_segments += len(long_rows) + room = 3 - len(stats.long_examples) + stats.long_examples += [f"{sentences['document'].iat[i]} (sentence {ids.iat[i]}, {lengths[i]} words)" + for i in long_rows[:max(room, 0)]] + if own_stats: + stats.report() + + parts: dict[str, object] = {c: sentences[c].to_numpy() + for c in ("sentence_id", "structure", "start", "end") if c in sentences} + parts["document"] = sentences["document"].to_numpy() + parts["sentences"] = np.array(text, dtype=object) + for name in fw.lists: + parts[name] = ev.columns[name] + if compound is not None: + parts["vader_compound"] = compound + for name in SENTIMENT_TERMS: + parts[name] = ev.columns[name] + for qq in fw.queries: + parts[qq.name] = ev.columns[qq.name] + for v in fw.variables: + parts[v.column] = ev.variables[v.number] + return pd.DataFrame(parts) + + +# ------------------------------------------------------------------------- tests + + +def run_tests(fw: Framework) -> list[str]: + """Run the framework's tests sheet; return a message per failed expectation.""" + if not fw.tests: + return [] + groups = list(dict.fromkeys(t.group for t in fw.tests)) + cases = sorted(fw.tests, key=lambda t: groups.index(t.group)) # rows of a group together + df = pd.DataFrame({ + "document": [t.group for t in cases], + "text": [t.sentence for t in cases], + }) + df["sentence_id"] = df.groupby("document").cumcount() + 1 + if fw.settings.sentiment: + from .sentiment import score_texts + + df["vader_compound"] = score_texts([normalize_text(t) for t in df["text"]]) + res = analyze_sentences(df, fw, long_segment_words=0, check_hierarchy=False) + matcher = Matcher(fw.lists) + columns = {v.number: v.column for v in fw.variables} + failures = [] + for i, t in enumerate(cases): + wrong = [(name, True) for name in t.expect_true if not bool(res.at[i, columns.get(name, name)])] + wrong += [(name, False) for name in t.expect_false if bool(res.at[i, columns.get(name, name)])] + if not wrong: + continue + sentence = res.at[i, "sentences"] + found = matcher.matches(sentence) + matched = "; ".join( + f"{name}: " + ", ".join(sorted({sentence[s:e] for s, e, _ in m})) + for name, m in zip(matcher.lists, found) if m + ) or "no list matched" + for name, expected in wrong: + detail = _explain(fw, res, i, name) + failures.append( + f"tests row {t.row}: expected {name} to be {str(expected).lower()}, got {str(not expected).lower()}. " + f"Sentence: {t.sentence!r}. Matched: {matched}." + (f" {detail}" if detail else "") + ) + return failures + + +def _explain(fw: Framework, res: pd.DataFrame, i: int, name: str) -> str: + """Values of the names used directly by a failing query or variable.""" + node = None + var = next((v for v in fw.variables if v.number == name), None) + if var is not None: + node = var.ast + else: + qq = next((x for x in fw.queries if x.name == name), None) + node = qq.ast if qq else None + if node is None: + return "" + used = sorted(q.names(node)) + vals = [f"{u}={str(bool(res.at[i, u])).lower()}" for u in used if u in res.columns] + return f"{name} = {q.to_text(node)}" + (f" where {', '.join(vals)}" if vals else "") + "." diff --git a/ctrlfindr/framework.py b/ctrlfindr/framework.py new file mode 100644 index 0000000..68d59b2 --- /dev/null +++ b/ctrlfindr/framework.py @@ -0,0 +1,513 @@ +"""Load and validate an assessment framework (.ods or .xlsx), format version 1. + +Sheets (names are case-insensitive; columns marked * are required): + +meta key*, value format_version (required), title, version, author, + language, description, and any other key +settings key*, value, note coding settings (optional sheet; empty value = default) +taxonomy list*, term*, note one row per term; "-phrase" / "-@list" exclude +queries name*, query*, note named expressions (see ctrlfindr.query) +variables number*, name*, definition, parent, note +tests sentence*, expect_true, expect_false, group, note (optional) +instructions (ignored) + +Everything is checked before any text is processed and all problems are reported together, +with sheet and row. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import Path + +from . import query as q +from .common import ValidationError, collapse_ws, normalize_text, sha256_file +from .matching import ListSpec, TermError, parse_term +from .spreadsheet import SUFFIXES, read_workbook + +FORMAT_VERSION = "1" +SENTIMENT_TERMS = ( + "sentiment_positive", + "sentiment_neutral", + "sentiment_neutralpositive", + "sentiment_negative", + "sentiment_neutralnegative", +) +OUTPUT_COLUMNS = {"document", "sentences", "sentence_id", "structure", "start", "end", "vader_compound"} +STRUCTURE_LABELS = ("header", "body", "list", "table", "footnote", "unstructured") # see ctrlfindr.ingest +RESERVED = OUTPUT_COLUMNS | set(SENTIMENT_TERMS) + +SHEETS = { + "meta": (("key",), ("value",)), + "settings": (("key",), ("value", "note")), + "taxonomy": (("list", "term"), ("note",)), + "queries": (("name", "query"), ("note",)), + "variables": (("number", "name"), ("definition", "parent", "note")), + "tests": (("sentence",), ("expect_true", "expect_false", "group", "note")), +} +REQUIRED_SHEETS = ("meta", "taxonomy", "queries", "variables") + +_VAR_NUMBER = re.compile(r"\d+(\.\d+)*") + + +@dataclass +class Settings: + """Coding settings stored in the framework (sheet 'settings').""" + + sentiment: bool = False + sentiment_positive_threshold: float = 0.05 + sentiment_negative_threshold: float = -0.05 + aggregate_children: bool = False + tests_must_pass: bool = True + exclude_sentence_types: tuple[str, ...] = () + + +@dataclass(frozen=True) +class Query: + name: str + text: str + ast: q.Node + note: str = "" + + +@dataclass +class Variable: + number: str + name: str + text: str + ast: q.Node | None + parent: str | None + note: str = "" + children: list[str] = field(default_factory=list) + + @property + def column(self) -> str: + """Output column name: ``"1. name"``, ``"1.1 name"``.""" + num = self.number if "." in self.number else self.number + "." + return f"{num} {self.name}" + + +@dataclass(frozen=True) +class TestCase: + row: int + group: str + sentence: str + expect_true: tuple[str, ...] + expect_false: tuple[str, ...] + note: str = "" + + +@dataclass +class Framework: + path: Path + sha256: str + meta: dict[str, str] + settings: Settings + lists: dict[str, ListSpec] # named lists, in sheet order + inline: dict[str, ListSpec] # {term} used in expressions, keyed "{term}" + notes: dict[str, str] # notes of lists (first note of the list) and queries + term_notes: dict[tuple[str, str], str] + queries: list[Query] + query_order: list[str] + variables: list[Variable] + variable_order: list[str] # variable numbers in evaluation order + tests: list[TestCase] + set_lists: dict[int, tuple[str, ...]] # id(set argument node) -> lists whose matches it uses + + @property + def all_lists(self) -> dict[str, ListSpec]: + return {**self.lists, **self.inline} + + def variable(self, number: str) -> Variable: + return next(v for v in self.variables if v.number == number) + + +def clean_cell(value: str) -> str: + """Normalize a cell like the corpus text (NFKC, dash/quote folding, single spaces).""" + return collapse_ws(normalize_text(value)) + + +def _records(sheet: str, rows: list[list[str]], problems: list[str]) -> list[tuple[int, dict[str, str]]]: + """Rows of a sheet as (spreadsheet row number, {column: cleaned value}). + + Fully empty rows are skipped; rows missing a required value are reported. + """ + required, optional = SHEETS[sheet] + if not rows: + problems.append(f"sheet '{sheet}' has no header row (expected columns: {', '.join(required + optional)})") + return [] + header = [clean_cell(h).lower() for h in rows[0]] + missing = [c for c in required if c not in header] + if missing: + problems.append(f"sheet '{sheet}': missing column(s) {', '.join(repr(m) for m in missing)}") + return [] + index = {c: header.index(c) for c in required + optional if c in header} + out = [] + for i, row in enumerate(rows[1:], start=2): + rec = {c: clean_cell(row[j]) if j < len(row) else "" for c, j in index.items()} + if not any(rec.values()): + continue + empty = [c for c in required if not rec[c]] + if empty: + problems.append(f"sheet '{sheet}', row {i}: empty cell(s) in {', '.join(repr(e) for e in empty)}") + continue + for c in optional: + rec.setdefault(c, "") + out.append((i, rec)) + return out + + +def _bool(value: str) -> bool | None: + v = value.lower() + return True if v in ("true", "yes", "1") else False if v in ("false", "no", "0") else None + + +def _settings(records, problems: list[str]) -> Settings: + s = Settings() + seen: dict[str, int] = {} + for i, rec in records: + key, value = rec["key"].lower(), rec["value"] + where = f"sheet 'settings', row {i}" + if not value: # left empty: keep the default + continue + if key in seen: + problems.append(f"{where}: setting {key!r} is already set in row {seen[key]}") + continue + seen[key] = i + if not hasattr(s, key): + problems.append(f"{where}: unknown setting {key!r} (known: {', '.join(Settings.__dataclass_fields__)})") + continue + default = getattr(s, key) + if isinstance(default, tuple): + items = tuple(dict.fromkeys(x.strip().lower() for x in re.split(r"[,;]", value) if x.strip())) + unknown = [x for x in items if x not in STRUCTURE_LABELS] + if unknown: + problems.append(f"{where}: {key}: unknown sentence type(s) {', '.join(map(repr, unknown))} " + f"(known: {', '.join(STRUCTURE_LABELS)})") + continue + setattr(s, key, items) + elif isinstance(default, bool): + b = _bool(value) + if b is None: + problems.append(f"{where}: {key} must be true or false, not {value!r}") + continue + setattr(s, key, b) + else: + try: + setattr(s, key, float(value.replace(",", "."))) + except ValueError: + problems.append(f"{where}: {key} must be a number, not {value!r}") + if not (-1 <= s.sentiment_negative_threshold <= 0 <= s.sentiment_positive_threshold <= 1): + problems.append("sheet 'settings': thresholds must satisfy " + "-1 <= sentiment_negative_threshold <= 0 <= sentiment_positive_threshold <= 1") + if s.sentiment: + from . import sentiment + + if not sentiment.available(): + problems.append(f"sheet 'settings': {sentiment.MISSING}") + return s + + +def _parent_of(number: str) -> str | None: + return number.rsplit(".", 1)[0] if "." in number else None + + +def load_framework(path: str | Path) -> Framework: + """Read and validate a framework; raises ValidationError listing every problem.""" + path = Path(path) + if not path.is_file(): + raise ValidationError("Assessment framework not found", [f"file does not exist: {path}"]) + if path.suffix.lower() not in SUFFIXES: + raise ValidationError("Assessment framework has an unsupported file type", + [f"{path.name}: use .ods (LibreOffice) or .xlsx (Excel)"]) + try: + raw = read_workbook(path) + except Exception as exc: + raise ValidationError("Assessment framework could not be read", [f"{path}: {exc}"]) from exc + sheets = {clean_cell(k).lower(): v for k, v in raw.items()} + title = f"Assessment framework {path.name} is invalid" + + # --- format version first: without it nothing else can be interpreted (R2) + problems: list[str] = [] + meta: dict[str, str] = {} + if "meta" not in sheets: + raise ValidationError(title, ["sheet 'meta' is missing, so this is not a CtrlFindr framework of format " + f"{FORMAT_VERSION}; start from the template in the template/ folder"]) + for i, rec in _records("meta", sheets["meta"], problems): + if rec["value"]: # keys left empty in the template are ignored + meta[rec["key"].lower()] = rec["value"] + version = meta.get("format_version", "").removesuffix(".0") + if not version: + problems.append("sheet 'meta': the row 'format_version' is missing") + elif version != FORMAT_VERSION: + problems.append(f"sheet 'meta': format_version is {version!r}, but this version of CtrlFindr reads " + f"framework format {FORMAT_VERSION}") + if problems: + raise ValidationError(title, problems) + lang = meta.get("language", "en").lower() + if lang not in ("en", "english"): + problems.append(f"sheet 'meta': language {meta['language']!r} is not supported; CtrlFindr analyzes English (en)") + for sheet in REQUIRED_SHEETS: + if sheet not in sheets: + problems.append(f"sheet '{sheet}' is missing") + if problems: + raise ValidationError(title, problems) + + settings = _settings(_records("settings", sheets.get("settings", [["key", "value"]]), problems), problems) + + # --- names: one namespace for lists and queries (R5) + owners: dict[str, str] = {} + + def claim(name: str, where: str) -> bool: + problem = q.check_name(name) + if problem: + problems.append(f"{where}: {problem}") + return False + if name in RESERVED: + problems.append(f"{where}: name {name!r} is reserved by CtrlFindr") + return False + if name in owners: + problems.append(f"{where}: name {name!r} is already used in {owners[name]}") + return False + owners[name] = where + return True + + # --- taxonomy (long layout) + lists: dict[str, ListSpec] = {} + notes: dict[str, str] = {} + term_notes: dict[tuple[str, str], str] = {} + first_row: dict[str, int] = {} + for i, rec in _records("taxonomy", sheets["taxonomy"], problems): + name, term = rec["list"].lower(), rec["term"].lower() + where = f"sheet 'taxonomy', row {i}" + if name not in lists: + if not claim(name, f"{where} (list)"): + lists.setdefault(f"\x00invalid{i}", ListSpec([])) + continue + lists[name] = ListSpec([]) + first_row[name] = i + spec = lists[name] + if rec["note"] and name not in notes: + notes[name] = rec["note"] + if term.startswith("-@"): + target = term[2:].strip() + if target == name: + problems.append(f"{where}: list {name!r} cannot exclude itself") + elif target not in spec.exclude_lists: + spec.exclude_lists.append(target) + continue + body = term[1:].strip() if term.startswith("-") else term + try: + parse_term(body) + except TermError as exc: + problems.append(f"{where}: {exc}") + continue + target_list = spec.exclude_terms if term.startswith("-") else spec.terms + if body not in target_list: + target_list.append(body) + if rec["note"]: + term_notes[(name, term)] = rec["note"] + lists = {k: v for k, v in lists.items() if not k.startswith("\x00")} + for name, spec in lists.items(): + where = f"sheet 'taxonomy', list {name!r} (row {first_row[name]})" + if not spec.terms: + problems.append(f"{where}: the list has no terms (only exclusions)") + for target in spec.exclude_lists: + if target not in lists: + problems.append(f"{where}: '-@{target}' refers to a list that does not exist") + if not lists: + problems.append("sheet 'taxonomy': no lists defined") + + # --- queries + parsed: list[tuple[int, str, str, q.Node | None, str]] = [] + for i, rec in _records("queries", sheets["queries"], problems): + name = rec["name"].lower() + where = f"sheet 'queries', row {i}" + claim(name, where) + try: + ast: q.Node | None = q.parse(rec["query"]) + except q.QuerySyntaxError as exc: + problems.append(f"{where}: query {name!r}: {exc}") + ast = None + parsed.append((i, name, rec["query"], ast, rec["note"])) + if rec["note"]: + notes.setdefault(name, rec["note"]) + query_names = {name for _, name, _, _, _ in parsed} + query_ast = {name: ast for _, name, _, ast, _ in parsed} + sentiment_names = set(SENTIMENT_TERMS) if settings.sentiment else set() + + inline: dict[str, ListSpec] = {} + set_lists: dict[int, tuple[str, ...]] = {} + + def resolve_set(node: q.Node, seen: frozenset[str] = frozenset()) -> tuple[str, ...] | None: + """Lists whose matches a set argument stands for, or None if it is not a set.""" + if isinstance(node, q.InlineTerm): + return ("{" + node.text + "}",) + if isinstance(node, q.Name): + if node.name in lists: + return (node.name,) + if node.name in query_ast and query_ast[node.name] is not None and node.name not in seen: + return resolve_set(query_ast[node.name], seen | {node.name}) + return None + if isinstance(node, q.Or): + parts = [resolve_set(o, seen) for o in node.operands] + return None if any(p is None for p in parts) else tuple(dict.fromkeys(x for p in parts for x in p)) + return None + + def check_expression(ast: q.Node, where: str, allow_vars: bool, var_numbers: set[str]) -> set[str]: + """Report unknown names and wrong arguments; return the query names used.""" + for n in q.walk(ast): + if isinstance(n, q.Name): + if n.name in SENTIMENT_TERMS and not settings.sentiment: + problems.append(f"{where}: {n.name!r} needs sentiment, which is switched off in the settings sheet") + elif n.name not in lists and n.name not in query_names and n.name not in sentiment_names: + problems.append(f"{where}: {n.name!r} is not a list, query or sentiment flag") + elif isinstance(n, q.InlineTerm): + if n.text.startswith("-"): + problems.append(f"{where}: {{{n.text}}}: exclusions belong in the taxonomy sheet") + continue + try: + parse_term(clean_cell(n.text).lower()) + inline.setdefault("{" + n.text + "}", ListSpec([clean_cell(n.text).lower()])) + except TermError as exc: + problems.append(f"{where}: {exc}") + elif isinstance(n, q.Call) and n.func == "v": + if not allow_vars: + problems.append(f"{where}: V({n.args[0]}) can only be used in the variables sheet") + elif n.args[0] not in var_numbers: + problems.append(f"{where}: V({n.args[0]}) refers to a variable number that does not exist") + for func, arg in q.set_arguments(ast): + lists_used = resolve_set(arg) + if lists_used is None: + problems.append( + f"{where}: {func.upper()} needs word positions, but {q.to_text(arg)!r} is not a list, " + "a {term}, or an OR of those (queries count only if they are defined that way)") + else: + set_lists[id(arg)] = lists_used + return q.names(ast) & query_names + + deps: dict[str, set[str]] = {} + for i, name, _, ast, _ in parsed: + deps.setdefault(name, set()) + if ast is not None: + deps[name] |= check_expression(ast, f"sheet 'queries', row {i}: query {name!r}", False, set()) + query_order, cycles = q.dependency_order(deps) + for cyc in cycles: + problems.append("sheet 'queries': queries refer to each other in a loop: " + " -> ".join(cyc)) + + # --- variables + v_rows = _records("variables", sheets["variables"], problems) + numbers: dict[str, int] = {} + for i, rec in v_rows: + number = rec["number"].replace(",", ".").rstrip(".") + if not _VAR_NUMBER.fullmatch(number): + problems.append(f"sheet 'variables', row {i}: number {rec['number']!r} must look like 1, 1.1 or 1.1.1") + elif number in numbers: + problems.append(f"sheet 'variables', row {i}: number {number!r} is already used in row {numbers[number]}") + else: + numbers[number] = i + variables: list[Variable] = [] + for i, rec in v_rows: + where = f"sheet 'variables', row {i}" + number = rec["number"].replace(",", ".").rstrip(".") + if number not in numbers or numbers[number] != i: + continue + parent = rec["parent"].replace(",", ".").rstrip(".") or None + if parent is not None and parent not in numbers: + problems.append(f"{where}: parent {parent!r} is not a variable number") + parent = None + if parent is None: + implied = _parent_of(number) + while implied and implied not in numbers: + implied = _parent_of(implied) + parent = implied + if parent == number: + problems.append(f"{where}: a variable cannot be its own parent") + parent = None + ast = None + if rec["definition"]: + try: + ast = q.parse(rec["definition"]) + except q.QuerySyntaxError as exc: + problems.append(f"{where}: variable {number}: {exc}") + variables.append(Variable(number, rec["name"], rec["definition"], ast, parent, rec["note"])) + by_number = {v.number: v for v in variables} + for v in variables: + if v.parent in by_number: + by_number[v.parent].children.append(v.number) + var_deps: dict[str, set[str]] = {} + for v in variables: + where = f"sheet 'variables', row {numbers[v.number]}: variable {v.number}" + var_deps[v.number] = set() + if v.ast is not None: + check_expression(v.ast, where, True, set(numbers)) + var_deps[v.number] |= q.variable_refs(v.ast) + elif not v.text: + if not (settings.aggregate_children and v.children): + problems.append(f"sheet 'variables', row {numbers[v.number]}: variable {v.number} has no definition" + + ("" if settings.aggregate_children else + " (it can be left empty only with aggregate_children = true and child variables)")) + if settings.aggregate_children: + var_deps[v.number] |= set(v.children) + # parent chains must not loop + for v in variables: + seen, p = {v.number}, v.parent + while p: + if p in seen: + problems.append(f"sheet 'variables': parent relations form a loop at variable {v.number}") + break + seen.add(p) + p = by_number[p].parent if p in by_number else None + variable_order, vcycles = q.dependency_order(var_deps) + for cyc in vcycles: + problems.append("sheet 'variables': variables refer to each other in a loop: " + " -> ".join(cyc)) + if not variables and not problems: + problems.append("sheet 'variables' defines no variables, so there would be nothing to export") + + # --- tests (R3) + tests: list[TestCase] = [] + known = set(lists) | query_names | sentiment_names | set(numbers) + for i, rec in _records("tests", sheets.get("tests", [["sentence"]]), problems): + where = f"sheet 'tests', row {i}" + expect = {} + for col in ("expect_true", "expect_false"): + items = tuple(x.strip().lower().rstrip(".") if _VAR_NUMBER.fullmatch(x.strip().rstrip(".")) else x.strip().lower() + for x in re.split(r"[,;\n]", rec[col]) if x.strip()) + for x in items: + if x not in known: + problems.append(f"{where}: {col} names {x!r}, which is not a list, query, sentiment flag or variable number") + expect[col] = items + if not expect["expect_true"] and not expect["expect_false"] and not rec["group"]: + # rows in a group may be context only (e.g. the sentence that makes DOC(x) true) + problems.append(f"{where}: the test has no expect_true or expect_false entries (and no group)") + tests.append(TestCase(i, rec["group"] or f"row {i}", rec["sentence"], + expect["expect_true"], expect["expect_false"], rec["note"])) + + # exclusion loops between lists + if not problems: + from .matching import Matcher + + try: + Matcher(lists) + except TermError as exc: + problems.append(f"sheet 'taxonomy': {exc}") + + if problems: + raise ValidationError(title, problems) + return Framework( + path=path, + sha256=sha256_file(path), + meta=meta, + settings=settings, + lists=lists, + inline=inline, + notes=notes, + term_notes=term_notes, + queries=[Query(name, text, ast, note) for _, name, text, ast, note in parsed], + query_order=query_order, + variables=variables, + variable_order=variable_order, + tests=tests, + set_lists=set_lists, + ) diff --git a/ctrlfindr/ingest.py b/ctrlfindr/ingest.py new file mode 100644 index 0000000..5e8b11b --- /dev/null +++ b/ctrlfindr/ingest.py @@ -0,0 +1,343 @@ +"""Ingest step: text files -> cleaned documents and sentences in SQLite. + +Two input types are read from the text files folder: + +* ``.txt`` files are plain, unstructured text. Every sentence gets the structure label + ``unstructured``. +* ``.md`` files are Markdown. Headings, list items, table rows, footnotes and body text are + told apart (see ctrlfindr.markdown), Markdown syntax is removed, and every sentence gets + the label ``header``, ``list``, ``table``, ``footnote`` or ``body``. + +Re-running ingest only processes files that are new or changed (by SHA-256) and removes +documents whose files are gone. Changing the cleaning setting, or upgrading CtrlFindr or +pySBD, rebuilds everything. Sentiment is not computed here (see ctrlfindr.sentiment). + +Each file is saved in its own transaction as soon as it is done, so an ingest that is +stopped (Ctrl+C, a killed process, a crash) keeps every finished file; running it again +continues with the files that are left. +""" + +from __future__ import annotations + +import hashlib +import json +import logging +import re +import sqlite3 +from dataclasses import dataclass +from importlib import metadata +from pathlib import Path + +from . import __version__ +from .common import (DOCUMENTS, META, RESULTS, RESULTS_NEW, SENTENCES, SENTIMENT, CtrlFindrError, ValidationError, + connect, normalize_text, sha256_file, table_exists) +from .config import Config +from .markdown import blocks as markdown_blocks + +log = logging.getLogger("ctrlfindr") + +# Bump when cleaning or segmentation logic changes, so existing databases are rebuilt. +INGEST_REVISION = 6 +SUFFIXES = {".txt": "txt", ".md": "md"} +UNSTRUCTURED = "unstructured" +STRUCTURE_LABELS = ("header", "body", "list", "table", "footnote", UNSTRUCTURED) +_SINGLE_SEGMENT = {"header", "table"} # never split into several sentences + +_URL = re.compile(r"http\S+|www\S+|https\S+") +_ELLIPSIS = re.compile(r"\.{2,}") +_BLANK_LINE = re.compile(r"[^\S\n]*\n(?:[^\S\n]*\n)+[^\S\n]*") # one or more blank lines +_LINE_BREAK = re.compile(r"(? "2050. [4] The" +# "published.The minister" -> "published. The minister": a full stop (or ? !, possibly with a +# closing bracket or quote) after two lower-case letters or digits, directly followed by a +# capitalised word. "U.S.Army", "Ph.D.Later", "St.Louis", "e.g.The" and "4.2.Results" are left alone. +_MISSING_SPACE = re.compile(r"(?<=[a-z0-9]{2})([)\]\"']?[.!?][)\]\"']?)(?=[A-Z][a-z])") + +_SCHEMA = ( + f"""CREATE TABLE IF NOT EXISTS {DOCUMENTS} ( + document TEXT PRIMARY KEY, path TEXT NOT NULL, format TEXT NOT NULL, sha256 TEXT NOT NULL, + clean_text TEXT NOT NULL)""", + f"""CREATE TABLE IF NOT EXISTS {SENTENCES} ( + document TEXT NOT NULL, sentence_id INTEGER NOT NULL, structure TEXT NOT NULL, + start INTEGER NOT NULL, "end" INTEGER NOT NULL, text TEXT NOT NULL, + PRIMARY KEY (document, sentence_id))""", + f"CREATE TABLE IF NOT EXISTS {META} (key TEXT PRIMARY KEY, value TEXT NOT NULL)", +) +_COLUMNS = {DOCUMENTS: ["document", "path", "format", "sha256", "clean_text"], + SENTENCES: ["document", "sentence_id", "structure", "start", "end", "text"]} + + +@dataclass(frozen=True) +class SourceFile: + document: str + path: Path + + @property + def format(self) -> str: + return SUFFIXES[self.path.suffix.lower()] + + +@dataclass(frozen=True) +class Sentence: + sentence_id: int + structure: str + start: int + end: int + text: str + + +def discover(text_dir: Path) -> list[SourceFile]: + """Sorted list of the .txt and .md files in ``text_dir`` (extensions in any case). + + Hidden and temporary files (names starting with ``.`` or ``~``, such as macOS + ``._report.txt`` copies or editor lock files) are skipped. + """ + if not text_dir.is_dir(): + raise CtrlFindrError(f"text files folder not found: {text_dir}") + candidates = [p for p in text_dir.iterdir() if p.is_file() and p.suffix.lower() in SUFFIXES] + skipped = [p for p in candidates if p.name.startswith((".", "~"))] + if skipped: + log.info("Skipped %d hidden or temporary file(s), e.g. %s", len(skipped), skipped[0].name) + files = sorted((p for p in candidates if p not in skipped), key=lambda p: p.name) + if not files: + raise CtrlFindrError(f"no .txt or .md files in {text_dir}") + out = [SourceFile(p.name[: -len(p.suffix)], p) for p in files] + seen: dict[str, Path] = {} + dupes = [] + for f in out: + if f.document in seen: + dupes.append(f"{seen[f.document].name} and {f.path.name} both give document name {f.document!r}") + seen[f.document] = f.path + if dupes: + raise ValidationError("Text files have clashing names", dupes) + return out + + +def decode(data: bytes, errors: str) -> str: + """UTF-8 (a leading byte-order mark is dropped); ``errors`` is 'strict' or 'replace'.""" + return data.decode("utf-8-sig", errors=errors) + + +def clean_document(raw: str, text_cleaning: bool = True, repair_spacing: bool = False) -> str: + """Normalize plain text. Sentence offsets refer to the string returned here. + + Always: NFKC normalization and dash/quote folding; blank lines become ``\\n\\n``, where a + sentence always ends; any other run of whitespace, including a single line break, becomes + one space; a space is inserted between a full stop and a following citation mark + (``2050.[4] The``), which the sentence splitter would otherwise not split. + With ``text_cleaning``: URLs are removed and ``..``/``...`` become ``.``. + With ``repair_spacing`` (used for .txt files): a missing space after a full stop is + added (``published.The minister`` -> ``published. The minister``). + """ + text = normalize_text(raw).replace("\r\n", "\n").replace("\r", "\n") + if text_cleaning: + text = _URL.sub("", text) + text = _ELLIPSIS.sub(".", text) + text = _CITATION_AFTER_STOP.sub(" ", text) + if repair_spacing: + text = _MISSING_SPACE.sub(r"\1 ", text) + text = _BLANK_LINE.sub("\n\n", text) + text = _LINE_BREAK.sub(" ", text) + text = _SPACES.sub(" ", text) + return text.strip() + + +def prepare(raw: str, fmt: str, text_cleaning: bool = True) -> tuple[str, list[str]]: + """Cleaned text of a document and the structure label of each of its blocks. + + Blocks are separated by ``\\n\\n`` in the cleaned text. For Markdown, each heading, list + item, table row and body paragraph is one block, with the Markdown syntax removed. + """ + if fmt == "txt": + text = clean_document(raw, text_cleaning, repair_spacing=True) + return text, [UNSTRUCTURED] * (text.count("\n\n") + 1) + parts, labels = [], [] + for label, block in markdown_blocks(raw): + cleaned = clean_document(block, text_cleaning).replace("\n\n", " ") + if cleaned: + parts.append(cleaned) + labels.append(label) + return "\n\n".join(parts), labels + + +class Segmenter: + """pySBD sentence splitting with character offsets. A block boundary (blank line) always + ends a sentence; headings and table rows are kept whole.""" + + def __init__(self) -> None: + import pysbd + + self._seg = pysbd.Segmenter(language="en", clean=False) + + def split(self, text: str, labels: list[str] | None = None) -> list[Sentence]: + out: list[Sentence] = [] + pos = 0 + for b, block in enumerate(text.split("\n\n")): + label = labels[b] if labels else UNSTRUCTURED + if not block.strip(): + pieces = [] + elif label in _SINGLE_SEGMENT: + pieces = [block] + else: + pieces = self._seg.segment(block) + cursor = 0 + for piece in pieces: + s = piece.strip() + if not s: + continue + k = block.find(s, cursor) + if k < 0: # pySBD altered the text; should not happen with clean=False + raise RuntimeError(f"sentence splitter changed the text near: {s[:60]!r}") + cursor = k + len(s) + if not any(ch.isalpha() for ch in s): # numbers, punctuation, "[12]" citation marks + continue + out.append(Sentence(len(out) + 1, label, pos + k, pos + k + len(s), s)) + pos += len(block) + 2 + return out + + +def _signature(cfg: Config) -> str: + def ver(pkg: str) -> str: + try: + return metadata.version(pkg) + except metadata.PackageNotFoundError: + return "?" + + sig = { + "ctrlfindr": __version__, + "ingest_revision": INGEST_REVISION, + "pysbd": ver("pysbd"), + "text_cleaning": cfg.ingest.text_cleaning, + "encoding_errors": cfg.ingest.encoding_errors, + } + return json.dumps(sig, sort_keys=True) + + +def _prepare_db(conn: sqlite3.Connection, signature: str) -> dict[str, str]: + """Create tables; drop everything if the stored data was made with other settings. + + Returns {document: sha256} for documents that can be kept as they are. + """ + def columns(table: str) -> list[str]: + return [r[1] for r in conn.execute(f"PRAGMA table_info({table})")] + + stored = None + if columns(META) == ["key", "value"]: + row = conn.execute(f"SELECT value FROM {META} WHERE key='ingest_signature'").fetchone() + stored = row[0] if row else None + rebuild = stored != signature or any(columns(t) not in ([], cols) for t, cols in _COLUMNS.items()) + if rebuild: + if stored is not None or columns(DOCUMENTS): + log.info("Ingest settings or versions changed: rebuilding the database") + for table in (RESULTS, RESULTS_NEW, SENTIMENT, SENTENCES, DOCUMENTS, META): + conn.execute(f"DROP TABLE IF EXISTS {table}") + for ddl in _SCHEMA: + conn.execute(ddl) + conn.execute(f"INSERT OR REPLACE INTO {META} (key, value) VALUES ('ingest_signature', ?)", (signature,)) + return dict(conn.execute(f"SELECT document, sha256 FROM {DOCUMENTS}").fetchall()) + + +def run(cfg: Config, show_progress: bool = True) -> dict[str, int]: + """Ingest the text files folder into the database. Returns counts of what was done. + + Files are hashed by streaming them; only new or changed files are read into memory, + one at a time, so memory use does not grow with the size of the corpus. + """ + files = {f.document: f for f in discover(cfg.paths.text_files)} + digests = {d: sha256_file(f.path) for d, f in files.items()} + errors = cfg.ingest.encoding_errors + + db = cfg.paths.database + with connect(db) as conn: + kept = _prepare_db(conn, _signature(cfg)) + removed = [d for d in kept if d not in files] + todo = [d for d in files if kept.get(d) != digests[d]] + + # Check encodings before anything is changed. Unchanged files were checked when + # they were ingested (the encoding setting is part of the stored signature). + bad = [] + for d in todo: + try: + decode(files[d].path.read_bytes(), "strict") + except UnicodeDecodeError as exc: + bad.append(f"{files[d].path.name}: not valid UTF-8 (byte {exc.start})") + if bad and errors == "strict": + raise ValidationError( + "Some text files are not UTF-8. Re-save them as UTF-8, or set " + '[ingest] encoding_errors = "replace" to substitute unreadable characters', + bad, + ) + if bad: + log.warning("%d file(s) are not valid UTF-8; unreadable bytes were replaced: %s", + len(bad), "; ".join(bad[:5]) + (" ..." if len(bad) > 5 else "")) + + cached_sentiment = table_exists(conn, SENTIMENT) + + def forget(d: str) -> None: + conn.execute(f"DELETE FROM {SENTENCES} WHERE document=?", (d,)) + conn.execute(f"DELETE FROM {DOCUMENTS} WHERE document=?", (d,)) + if cached_sentiment: # scores of changed documents no longer match their sentences + conn.execute(f"DELETE FROM {SENTIMENT} WHERE document=?", (d,)) + + for d in removed: + forget(d) + if removed or todo: + # the previous analysis no longer matches the sentences (sentence numbers may shift) + conn.execute(f"DROP TABLE IF EXISTS {RESULTS}") + conn.execute(f"DROP TABLE IF EXISTS {RESULTS_NEW}") + for d, f in files.items(): # files may have moved; keep paths current + conn.execute(f"UPDATE {DOCUMENTS} SET path=? WHERE document=?", (str(f.path), d)) + conn.commit() # settings check, removals and paths are saved before the long part + if todo: + log.info("Ingest: %d file(s) to process, %d unchanged", len(todo), len(files) - len(todo)) + + n_sent = done = 0 + if todo: + segmenter = Segmenter() + iterator = todo + if show_progress: + from tqdm.auto import tqdm + + iterator = tqdm(todo, desc="ingest", unit=" files", leave=False) + empty = [] + try: + for d in iterator: + f = files[d] + data = f.path.read_bytes() + digest = hashlib.sha256(data).hexdigest() # the bytes actually ingested + text, labels = prepare(decode(data, errors), f.format, cfg.ingest.text_cleaning) + del data + sents = segmenter.split(text, labels) + if not sents: + empty.append(f.path.name) + # One transaction per file: until it is committed, the previous version of + # the document (if any) stays in place, so a stop never leaves half a file. + forget(d) + conn.execute( + f"INSERT INTO {DOCUMENTS} (document, path, format, sha256, clean_text) VALUES (?, ?, ?, ?, ?)", + (d, str(f.path), f.format, digest, text), + ) + conn.executemany( + f'INSERT INTO {SENTENCES} (document, sentence_id, structure, start, "end", text) ' + "VALUES (?, ?, ?, ?, ?, ?)", + [(d, s.sentence_id, s.structure, s.start, s.end, s.text) for s in sents], + ) + conn.commit() + done += 1 + n_sent += len(sents) + except KeyboardInterrupt: + conn.rollback() + log.warning("Ingest stopped after %d of %d file(s). The finished files are saved; run ingest " + "again to continue with the %d file(s) left.", done, len(todo), len(todo) - done) + raise + if empty: + log.warning("%d file(s) contain no sentences and will not appear in the results: %s", + len(empty), ", ".join(empty)) + total = conn.execute(f"SELECT COUNT(*) FROM {SENTENCES}").fetchone()[0] + + summary = {"files": len(files), "processed": len(todo), "unchanged": len(files) - len(todo), + "removed": len(removed), "new_sentences": n_sent, "sentences": total} + log.info("Ingest: %(files)d files (%(processed)d processed, %(unchanged)d unchanged, " + "%(removed)d removed); %(sentences)d sentences in the database", summary) + return summary diff --git a/ctrlfindr/markdown.py b/ctrlfindr/markdown.py new file mode 100644 index 0000000..5553bae --- /dev/null +++ b/ctrlfindr/markdown.py @@ -0,0 +1,193 @@ +"""Split a Markdown document into labelled blocks of plain text. + +Each block is ``(structure, text)`` with structure one of: + +* ``header`` a heading (``# Title`` or a line underlined with ``===`` / ``---``); +* ``list`` one list item (``-``, ``*``, ``+``, ``1.`` or ``1)``), with its continuation lines; +* ``table`` one table row (``| a | b |``); the separator row is dropped, cells are + joined with `` | ``; +* ``footnote`` the text of a footnote (``[^1]: ...``), with its indented continuation; +* ``body`` everything else: paragraphs, block quotes, code blocks. + +Markdown syntax is removed from the text: emphasis markers, link targets (the link text is +kept), images, inline code backticks, footnote markers (``[^1]``, ``12``, +``^12^``; superscript ordinals and exponents are kept as ``20th`` and ``10^6``), HTML tags +and entities, and backslash escapes. Horizontal rules are dropped. +""" + +from __future__ import annotations + +import html +import re + +STRUCTURES = ("header", "body", "list", "table", "footnote") + +_ATX = re.compile(r"^ {0,3}(#{1,6})[ \t]+(.*?)[ \t]*#*[ \t]*$|^ {0,3}(#{1,6})[ \t]*$") +_SETEXT = re.compile(r"^ {0,3}(=+|-+)[ \t]*$") +_RULE = re.compile(r"^ {0,3}([-*_])([ \t]*\1){2,}[ \t]*$") +_ITEM = re.compile(r"^([ \t]*)([-*+•]|\d{1,9}[.)])[ \t]+(.*)$") +_FENCE = re.compile(r"^ {0,3}(`{3,}|~{3,})") +_QUOTE = re.compile(r"^ {0,3}>[ \t]?(.*)$") +_TABLE_SEP = re.compile(r"^[ \t]*\|?[ \t]*:?-+:?[ \t]*(\|[ \t]*:?-+:?[ \t]*)*\|?[ \t]*$") +_FOOTNOTE_DEF = re.compile(r"^ {0,3}\[\^[^\]]+\]:[ \t]*(.*)$") +_REF_DEF = re.compile(r"^ {0,3}\[[^\]]+\]:[ \t]*\S+.*$") # [label]: http://... (link definitions) + +_INLINE = [ + (re.compile(r"!\[[^\]]*\]\([^)]*\)"), ""), # images + (re.compile(r"!\[[^\]]*\]\[[^\]]*\]"), ""), + (re.compile(r"\[\^[^\]]+\]"), ""), # footnote markers [^1] + # superscripts (.. and pandoc ^..^): ordinals are joined ("20th"), a number + # raised after a digit is an exponent ("10^6"), other number/symbol marks are footnote + # markers and removed; superscripts with other text keep their text + (re.compile(r"\s*(st|nd|rd|th)\s*|\^(st|nd|rd|th)\^", re.I), r"\1\2"), + (re.compile(r"(?<=\d)(?:\s*(\d{1,3})\s*|\^(\d{1,3})\^)", re.I), r"^\1\2"), + (re.compile(r"\s*[\d,;\-–*†‡ ]{1,12}\s*|\^[\d,\-–*†‡]{1,12}\^", re.I), ""), + (re.compile(r"\s*[a-z](\s*,\s*[a-z])*\s*", re.I), ""), # letter footnote markers a + (re.compile(r"(.*?)", re.I), r"\1"), + (re.compile(r"\[([^\]]+)\]\([^)]*\)"), r"\1"), # [text](url) + (re.compile(r"\[([^\]]+)\]\[[^\]]*\]"), r"\1"), # [text][ref] + (re.compile(r"<(https?://|www\.)[^>]+>"), ""), # + (re.compile(r""), ""), # HTML comments + (re.compile(r"]*>"), " "), # HTML tags + (re.compile(r"`+([^`]*)`+"), r"\1"), # inline code + (re.compile(r"(\*\*|__)(?=\S)(.+?)(?<=\S)\1"), r"\2"), # **bold**, __bold__ + (re.compile(r"(?~])") # \* etc.: the character itself, not markup +_PROTECT = 0xE000 # escaped characters are parked in the Unicode private use area meanwhile + + +def inline_text(text: str) -> str: + """Plain text of a Markdown line or block (syntax removed, whitespace collapsed).""" + text = _ESCAPE.sub(lambda m: chr(_PROTECT + ord(m.group(1))), text) + for rx, repl in _INLINE: + text = rx.sub(repl, text) + text = "".join(chr(ord(c) - _PROTECT) if _PROTECT <= ord(c) < _PROTECT + 128 else c for c in text) + return " ".join(html.unescape(text).split()) + + +def _is_table_row(line: str) -> bool: + return "|" in line and line.strip() not in ("", "|") + + +def _cells(line: str) -> list[str]: + s = line.strip() + if s.startswith("|"): + s = s[1:] + if s.endswith("|") and not s.endswith("\\|"): + s = s[:-1] + return [inline_text(c) for c in re.split(r"(? list[tuple[str, str]]: + """Parse Markdown into ``(structure, plain text)`` blocks, in document order.""" + lines = text.replace("\r\n", "\n").replace("\r", "\n").split("\n") + out: list[tuple[str, str]] = [] + para: list[str] = [] + item: list[str] | None = None # current list item or footnote + item_kind = "list" + + def flush_para() -> None: + nonlocal para + if para: + out.append(("body", " ".join(para))) + para = [] + + def flush_item() -> None: + nonlocal item + if item: + out.append((item_kind, " ".join(item))) + item = None + + i = 0 + while i < len(lines): + line = lines[i] + stripped = line.strip() + + fence = _FENCE.match(line) + if fence: # code block: kept as body text + flush_para() + flush_item() + marker, code = fence.group(1), [] + i += 1 + while i < len(lines) and not lines[i].strip().startswith(marker): + code.append(lines[i].strip()) + i += 1 + if code: + out.append(("body", " ".join(c for c in code if c))) + i += 1 + continue + if not stripped: + flush_para() + flush_item() + i += 1 + continue + if not para and item is None and out and out[-1][0] == "footnote" and re.match(r"( {4}|\t)", line): + item, item_kind = [stripped], "footnote" # a further paragraph of the same footnote + i += 1 + continue + atx = _ATX.match(line) + if atx: + flush_para() + flush_item() + if atx.group(2): + out.append(("header", atx.group(2))) + i += 1 + continue + if para and item is None and _SETEXT.match(line) and not _ITEM.match(line): + out.append(("header", " ".join(para))) # the paragraph above is a setext heading + para = [] + i += 1 + continue + if _RULE.match(line): + flush_para() + flush_item() + i += 1 + continue + if _is_table_row(line) and i + 1 < len(lines) and _TABLE_SEP.match(lines[i + 1]) and "-" in lines[i + 1]: + flush_para() + flush_item() + out.append(("table", " | ".join(c for c in _cells(line) if c))) + i += 2 + while i < len(lines) and _is_table_row(lines[i]) and lines[i].strip(): + if not _TABLE_SEP.match(lines[i]): + row = " | ".join(c for c in _cells(lines[i]) if c) + if row: + out.append(("table", row)) + i += 1 + continue + fn = _FOOTNOTE_DEF.match(line) + if fn: + flush_para() + flush_item() + item, item_kind = [fn.group(1)], "footnote" + i += 1 + continue + m = _ITEM.match(line) + if m and para and item is None and m.group(2)[0].isdigit() and int(m.group(2)[:-1]) != 1: + m = None # "2020. Floods..." inside a paragraph is wrapped text, not a list (CommonMark) + if m: + flush_para() + flush_item() + item, item_kind = [m.group(3)], "list" + i += 1 + continue + if item is not None: # continuation of a list item or footnote (lazy or indented) + item.append(stripped) + i += 1 + continue + q = _QUOTE.match(line) + if q: + para.append(q.group(1)) + i += 1 + continue + if _REF_DEF.match(line): # [label]: http://... link definition + i += 1 + continue + para.append(stripped) + i += 1 + flush_para() + flush_item() + return [(kind, inline_text(t) if kind != "table" else t) for kind, t in out if (inline_text(t) if kind != "table" else t)] diff --git a/ctrlfindr/matching.py b/ctrlfindr/matching.py new file mode 100644 index 0000000..54541c7 --- /dev/null +++ b/ctrlfindr/matching.py @@ -0,0 +1,311 @@ +"""Term syntax, keyword matching (one Aho-Corasick pass for all terms), proximity and counts. + +Term syntax (terms are lowercased and normalized like the corpus): + +* ``flood`` the word "flood" only (whole words: the characters around a match + must not be letters, digits or underscores); +* ``flood*`` ``*`` = any run of letters/digits/underscores, also empty, anywhere in a + term (``*fire``, ``biodivers* loss``); it never crosses a space or hyphen; +* ``human~wildlife`` ``~`` = a hyphen, a space or nothing ("human-wildlife", + "human wildlife", "humanwildlife"); +* ``climate _ risk`` a ``_`` standing alone = exactly one word ("climate change risk"); +* ``-business climate`` an exclusion phrase: a match of the list that lies inside this + phrase is ignored (other matches in the same sentence still count); +* ``-@other_list`` every term of ``other_list`` is an exclusion phrase for this list. +""" + +from __future__ import annotations + +import bisect +import re +from collections import defaultdict +from dataclasses import dataclass, field + +import ahocorasick +import numpy as np + +Span = tuple[int, int] +Match = tuple[int, int, int] # start, end, entry id +Tokens = tuple[list[int], list[int]] # start and end offsets of the words of a sentence + +_WORDCHARS = re.compile(r"\w*") +_TOKEN = re.compile(r"\S+") +_HAS_WORDCHAR = re.compile(r"\w") + + +class TermError(ValueError): + pass + + +def _isw(ch: str) -> bool: + """Python's definition of ``\\w`` for str patterns.""" + return ch.isalnum() or ch == "_" + + +# ------------------------------------------------------------------- term syntax + + +@dataclass(frozen=True) +class Term: + text: str + regex: re.Pattern[str] + literal: str # literal text before the first marker (used by the automaton) + kind: str # "exact" | "prefix" (single trailing *) | "regex" | "scan" (no literal prefix) + + +def parse_term(text: str) -> Term: + """Compile one (already lowercased and normalized) term; raises TermError.""" + words = text.split(" ") + if not _HAS_WORDCHAR.search(text.replace("*", "").replace("~", "").replace(" _ ", " ")) or text == "_": + raise TermError(f"term {text!r} has no letters or digits") + if words[0] == "_" or words[-1] == "_": + raise TermError(f"term {text!r}: a word gap '_' needs words on both sides") + if text.startswith("~") or text.endswith("~") or "~~" in text or "~ " in text or " ~" in text: + raise TermError(f"term {text!r}: '~' must join two parts of a word, as in human~wildlife") + parts = [] + for i, w in enumerate(words): + if i: + parts.append(" ") + if w == "_": + parts.append(r"\S+") + continue + for ch in w: + parts.append(r"\w*" if ch == "*" else "[- ]?" if ch == "~" else re.escape(ch)) + regex = re.compile(r"(?= 0] + cut = min(special) if special else len(text) + literal = text[:cut] + if not special: + kind = "exact" + elif text.count("*") == 1 and text.endswith("*") and "~" not in text and " _ " not in f" {text} ": + kind = "prefix" + elif literal: + kind = "regex" + else: + kind = "scan" + return Term(text, regex, literal, kind) + + +@dataclass +class ListSpec: + """A list of the taxonomy: positive terms plus optional exclusions.""" + + terms: list[str] + exclude_terms: list[str] = field(default_factory=list) + exclude_lists: list[str] = field(default_factory=list) + + +@dataclass(frozen=True) +class _Entry: + slot: int # internal list slot + entry: int # entry id, unique across all lists (for COUNT_DISTINCT over several lists) + kind: str + regex: re.Pattern[str] | None + + +class Matcher: + """Find every list (with exclusions applied) in a sentence with one automaton scan.""" + + def __init__(self, lists: dict[str, ListSpec]): + self.lists = list(lists) + # slots: one per list, plus one per list with inline exclusion phrases + slots: list[str] = list(self.lists) + self._excl_slot: dict[int, int] = {} + for i, name in enumerate(self.lists): + if lists[name].exclude_terms: + self._excl_slot[i] = len(slots) + slots.append(f"{name} (exclusions)") + self._nslots = len(slots) + index = {name: i for i, name in enumerate(self.lists)} + self._excl_lists = {i: [index[x] for x in lists[name].exclude_lists] for i, name in enumerate(self.lists)} + self._order = self._exclusion_order() + + self._automaton = ahocorasick.Automaton() + entries: dict[str, list[_Entry]] = defaultdict(list) + self._scan: list[tuple[int, int, re.Pattern[str]]] = [] + + def add(slot: int, entry: int, text: str) -> None: + t = parse_term(text) + if t.kind == "scan": + self._scan.append((slot, entry, t.regex)) + else: + entries[t.literal].append(_Entry(slot, entry, t.kind, t.regex if t.kind == "regex" else None)) + + next_id = 0 + for i, name in enumerate(self.lists): + for text in lists[name].terms: + add(i, next_id, text) + next_id += 1 + for text in lists[name].exclude_terms: + add(self._excl_slot[i], -1, text) + for literal, items in entries.items(): + self._automaton.add_word(literal, (literal, tuple(items))) + self._empty = not entries + if entries: + self._automaton.make_automaton() + + def _exclusion_order(self) -> list[int]: + """Lists ordered so that exclusion lists are finished before the lists using them.""" + order: list[int] = [] + state: dict[int, int] = {} + + def visit(i: int, path: list[int]) -> None: + if state.get(i) == 2: + return + if state.get(i) == 1: + cyc = " -> ".join(self.lists[k] for k in path[path.index(i):] + [i]) + raise TermError(f"lists exclude each other in a loop: {cyc}") + state[i] = 1 + for j in self._excl_lists[i]: + visit(j, path + [i]) + state[i] = 2 + order.append(i) + + for i in range(len(self.lists)): + visit(i, []) + return order + + def matches(self, sentence: str) -> list[list[Match]]: + """For one (lowercased) sentence: the matches of each list after exclusions.""" + raw: list[set[Match]] = [set() for _ in range(self._nslots)] + n = len(sentence) + if not self._empty: + for end, (literal, items) in self._automaton.iter(sentence): + start = end - len(literal) + 1 + if start > 0 and _isw(sentence[start - 1]): + continue + after = end + 1 + for e in items: + if e.kind == "exact": + if after < n and _isw(sentence[after]): + continue + raw[e.slot].add((start, after, e.entry)) + elif e.kind == "prefix": + raw[e.slot].add((start, _WORDCHARS.match(sentence, after).end(), e.entry)) + else: + m = e.regex.match(sentence, start) + if m: + raw[e.slot].add((m.start(), m.end(), e.entry)) + for slot, entry, rx in self._scan: + for m in rx.finditer(sentence): + raw[slot].add((m.start(), m.end(), entry)) + + final: list[list[Match]] = [[] for _ in self.lists] + for i in self._order: + found = raw[i] + excl: list[Span] = [] + if i in self._excl_slot: + excl += [(s, e) for s, e, _ in raw[self._excl_slot[i]]] + for j in self._excl_lists[i]: + excl += [(s, e) for s, e, _ in final[j]] + if excl and found: + found = {m for m in found if not any(s <= m[0] and m[1] <= e for s, e in excl)} + final[i] = sorted(found) + return final + + def match( + self, sentences: list[str], keep: set[str] | None = None, progress=None + ) -> tuple[dict[str, np.ndarray], dict[str, list[list[Match] | None]]]: + """Boolean column per list, plus the matches of the lists in ``keep``.""" + keep_idx = [i for i, name in enumerate(self.lists) if keep and name in keep] + hits = np.zeros((len(self.lists), len(sentences)), dtype=bool) + kept: dict[str, list[list[Match] | None]] = {self.lists[i]: [None] * len(sentences) for i in keep_idx} + iterator = sentences if progress is None else progress(sentences) + for row, sentence in enumerate(iterator): + found = self.matches(sentence) + for idx, m in enumerate(found): + if m: + hits[idx, row] = True + for idx in keep_idx: + if found[idx]: + kept[self.lists[idx]][row] = found[idx] + return {name: hits[i] for i, name in enumerate(self.lists)}, kept + + +# -------------------------------------------------------------- words and proximity + + +def word_tokens(sentence: str) -> Tokens: + """Offsets of the whitespace-separated tokens that contain a letter or digit.""" + starts: list[int] = [] + ends: list[int] = [] + for m in _TOKEN.finditer(sentence): + if _HAS_WORDCHAR.search(m.group()): + starts.append(m.start()) + ends.append(m.end()) + return starts, ends + + +def words_between(tokens: Tokens, left_end: int, right_start: int) -> int: + """Number of words lying entirely between two character positions.""" + starts, ends = tokens + return max(0, bisect.bisect_right(ends, right_start) - bisect.bisect_left(starts, left_end)) + + +def _nearest_after(tokens: Tokens, end: int, others: list[Match], other_starts: list[int]) -> int | None: + """Words to the nearest match in ``others`` that starts at or after ``end``.""" + j = bisect.bisect_left(other_starts, end) + return words_between(tokens, end, other_starts[j]) if j < len(others) else None + + +def _nearest_before(tokens: Tokens, start: int, other_ends: list[int]) -> int | None: + """Words to the nearest match that ends at or before ``start`` (``other_ends`` sorted).""" + j = bisect.bisect_right(other_ends, start) + return words_between(tokens, other_ends[j - 1], start) if j else None + + +def near(tokens: Tokens, a: list[Match], b: list[Match], distance: int, ordered: bool = False) -> bool: + """A match of ``a`` and a match of ``b`` at most ``distance`` words apart. + + Distance = words strictly between the end of the earlier and the start of the later + match (adjacent = 0). Overlapping matches never count. With ``ordered``, ``a`` must + come first. + """ + b_starts = [m[0] for m in b] + for _, end, _ in a: + d = _nearest_after(tokens, end, b, b_starts) + if d is not None and d <= distance: + return True + if ordered: + return False + a_starts = [m[0] for m in a] + for _, end, _ in b: + d = _nearest_after(tokens, end, a, a_starts) + if d is not None and d <= distance: + return True + return False + + +def not_near(tokens: Tokens, a: list[Match], b: list[Match], distance: int) -> bool: + """Some match of ``a`` has no match of ``b`` within ``distance`` words (on either side).""" + if not a: + return False + if not b: + return True + b_starts = [m[0] for m in b] + b_ends = sorted(m[1] for m in b) + for start, end, _ in a: + after = _nearest_after(tokens, end, b, b_starts) + before = _nearest_before(tokens, start, b_ends) + if not any(d is not None and d <= distance for d in (after, before)): + return True + return False + + +def longest_matches(matches: list[Match]) -> list[Match]: + """Drop matches lying inside a longer match (and duplicates of the same span).""" + out: list[Match] = [] + # longest first, then by position, so a containing match is seen before its parts + for m in sorted(matches, key=lambda x: (x[0] - x[1], x[0])): + if not any(o[0] <= m[0] and m[1] <= o[1] for o in out): + out.append(m) + return sorted(out) + + +def count(matches: list[Match]) -> int: + return len(longest_matches(matches)) + + +def count_distinct(matches: list[Match]) -> int: + return len({m[2] for m in longest_matches(matches)}) diff --git a/ctrlfindr/ods.py b/ctrlfindr/ods.py new file mode 100644 index 0000000..b40873f --- /dev/null +++ b/ctrlfindr/ods.py @@ -0,0 +1,196 @@ +"""Minimal reader for OpenDocument spreadsheets (.ods), standard library only. + +Cells are returned as the text shown in the spreadsheet, never as parsed numbers. +This matters for the framework: a variable number typed as ``1.10`` stays ``"1.10"`` +instead of turning into the float ``1.1`` (which would collide with variable ``1.1``). +""" + +from __future__ import annotations + +import zipfile +import xml.etree.ElementTree as ET +from pathlib import Path + +_TABLE = "urn:oasis:names:tc:opendocument:xmlns:table:1.0" +_TEXT = "urn:oasis:names:tc:opendocument:xmlns:text:1.0" +_T = f"{{{_TABLE}}}" +_X = f"{{{_TEXT}}}" + +# Spreadsheets pad sheets with huge "repeated" blank rows/columns at the end. Blank runs +# are therefore expanded only when content follows them, so that row and column numbers +# stay exact while trailing padding costs nothing. + + +def _paragraph_text(p: ET.Element) -> str: + """Text of a , honouring (spaces), tabs and line breaks.""" + parts: list[str] = [] + + def walk(el: ET.Element) -> None: + if el.text: + parts.append(el.text) + for child in el: + if child.tag == f"{_X}s": + parts.append(" " * int(child.get(f"{_X}c", "1"))) + elif child.tag == f"{_X}tab": + parts.append("\t") + elif child.tag == f"{_X}line-break": + parts.append("\n") + else: + walk(child) + if child.tail: + parts.append(child.tail) + + walk(p) + return "".join(parts) + + +def _cell_text(cell: ET.Element) -> str: + return "\n".join(_paragraph_text(p) for p in cell.findall(f"{_X}p")) + + +def read_ods(path: str | Path) -> dict[str, list[list[str]]]: + """Return ``{sheet name: rows}``; each row is a list of cell strings ('' for empty). + + Trailing empty cells and trailing empty rows are removed. + """ + with zipfile.ZipFile(path) as zf: + root = ET.fromstring(zf.read("content.xml")) + sheets: dict[str, list[list[str]]] = {} + for table in root.iter(f"{_T}table"): + rows: list[list[str]] = [] + blank_rows = 0 + for row in table.iter(f"{_T}table-row"): + cells: list[str] = [] + blank_cells = 0 + for cell in row: + if cell.tag not in (f"{_T}table-cell", f"{_T}covered-table-cell"): + continue + text = _cell_text(cell) + repeat = int(cell.get(f"{_T}number-columns-repeated", "1")) + if not text.strip(): + blank_cells += repeat + continue + cells.extend([""] * blank_cells) + blank_cells = 0 + cells.extend([text] * repeat) + repeat_rows = int(row.get(f"{_T}number-rows-repeated", "1")) + if not cells: + blank_rows += repeat_rows + continue + rows.extend([] for _ in range(blank_rows)) + blank_rows = 0 + rows.extend(list(cells) for _ in range(repeat_rows)) + sheets[table.get(f"{_T}name", "")] = rows + return sheets + + +# ------------------------------------------------------------------------ writing + +_NS = ( + 'xmlns:office="urn:oasis:names:tc:opendocument:xmlns:office:1.0" ' + 'xmlns:style="urn:oasis:names:tc:opendocument:xmlns:style:1.0" ' + 'xmlns:text="urn:oasis:names:tc:opendocument:xmlns:text:1.0" ' + 'xmlns:table="urn:oasis:names:tc:opendocument:xmlns:table:1.0" ' + 'xmlns:fo="urn:oasis:names:tc:opendocument:xmlns:xsl-fo-compatible:1.0" ' + 'xmlns:of="urn:oasis:names:tc:opendocument:xmlns:of:1.2"' +) +_MANIFEST = ( + '' + '' + '' + '' + '' + "" +) +_STYLES = f'' + + +def _esc(s: str) -> str: + return s.replace("&", "&").replace("<", "<").replace(">", ">").replace('"', """) + + +def _para(line: str) -> str: + # Runs of spaces must be written as to survive. + out = [] + for i, part in enumerate(line.split(" ")): + if i: + out.append(' ') + out.append(_esc(part)) + return f"{''.join(out)}" + + +def write_ods(path: Path, sheets) -> None: + """Write ``SheetSpec`` objects (see ctrlfindr.spreadsheet) to an .ods file.""" + styles = [ + '', + '', + '', + ] + widths: dict[float, str] = {} + validations: list[str] = [] + tables: list[str] = [] + for spec in sheets: + cols = [] + for w in spec.widths_cm: + if w not in widths: + widths[w] = f"co{len(widths) + 1}" + styles.append(f'' + f'') + cols.append(f'') + dd_names: dict[int, tuple[str, int]] = {} + for dd in spec.dropdowns: + name = f"val{len(validations) + 1}" + if dd.source: + sheet, col = dd.source + cond = f"of:cell-content-is-in-list([$'{sheet}'.${col}$2:.${col}${dd.rows + 1}])" + else: + cond = "of:cell-content-is-in-list(" + ";".join(f'"{v}"' for v in dd.values) + ")" + validations.append( + f'" + '' + ) + dd_names[dd.column] = (name, dd.rows) + width = max([len(r) for r in spec.rows] + [max(dd_names, default=-1) + 1, len(spec.widths_cm), 1]) + rows_xml = [] + + def cell(value: str, r: int, c: int) -> str: + attrs = "" + if c in dd_names and 1 <= r <= dd_names[c][1]: + attrs += f' table:content-validation-name="{dd_names[c][0]}"' + style = ("ceHW" if spec.wrap else "ceH") if (spec.header and r == 0) else ("ceW" if spec.wrap else "") + if style: + attrs += f' table:style-name="{style}"' + if value == "": + return f"" + body = "".join(_para(line) for line in str(value).split("\n")) + return f'{body}' + + for r, row in enumerate(spec.rows): + cells = [cell(row[c] if c < len(row) else "", r, c) for c in range(width)] + rows_xml.append("" + "".join(cells) + "") + last_dd_row = max((rows for _, rows in dd_names.values()), default=0) + remaining = last_dd_row + 1 - len(spec.rows) + if remaining > 0: # empty rows that still carry the dropdowns + cells = [cell("", len(spec.rows), c) for c in range(width)] + rows_xml.append(f'' + "".join(cells) + "") + tables.append(f'' + "".join(cols) + "".join(rows_xml) + "") + + content = ( + f'' + f"{''.join(styles)}" + "" + + (f"{''.join(validations)}" if validations else "") + + "".join(tables) + + "" + ) + with zipfile.ZipFile(path, "w", zipfile.ZIP_DEFLATED) as zf: + zf.writestr(zipfile.ZipInfo("mimetype"), "application/vnd.oasis.opendocument.spreadsheet", + compress_type=zipfile.ZIP_STORED) + zf.writestr("META-INF/manifest.xml", _MANIFEST) + zf.writestr("content.xml", content) + zf.writestr("styles.xml", _STYLES) diff --git a/ctrlfindr/query.py b/ctrlfindr/query.py new file mode 100644 index 0000000..be891f7 --- /dev/null +++ b/ctrlfindr/query.py @@ -0,0 +1,368 @@ +"""The query language used in the queries and variables sheets: tokenizer and parser. + + expr := and_expr (OR and_expr)* + and_expr := not_expr (AND not_expr)* + not_expr := NOT not_expr | primary + primary := name | "quoted name" | {term} | ( expr ) | call [compare] + call := NEAR(set, set, n) | BEFORE(set, set, n) | NOT_NEAR(set, set, n) + | DOC(expr) | DOC_SHARE(expr) | WINDOW(expr, n) + | COUNT(set) | COUNT_DISTINCT(set) | V(variable number) + compare := (>= | > | = | != | <= | <) number (after DOC, DOC_SHARE, COUNT, + COUNT_DISTINCT only) + +Keywords are case-insensitive. ``not`` binds tighter than ``and``, ``and`` tighter than +``or``. A *set* is something with word positions: a list, a {term}, or an OR of those +(also through a query defined that way). The evaluation lives in ctrlfindr.engine. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from typing import Union + +NAME_RE = re.compile(r"[a-z][a-z0-9_-]*") +OPERATORS = {"and", "or", "not"} +# function name -> (argument kinds, may be followed by a comparison) +FUNCTIONS: dict[str, tuple[tuple[str, ...], bool]] = { + "near": (("set", "set", "int"), False), + "before": (("set", "set", "int"), False), + "not_near": (("set", "set", "int"), False), + "doc": (("expr",), True), + "doc_share": (("expr",), True), + "window": (("expr", "int"), False), + "count": (("set",), True), + "count_distinct": (("set",), True), + "v": (("varnum",), False), +} +RESERVED_WORDS = OPERATORS | set(FUNCTIONS) +COMPARATORS = (">=", "<=", "!=", "=", ">", "<") + + +class QuerySyntaxError(ValueError): + pass + + +# --------------------------------------------------------------------------- AST + + +@dataclass(frozen=True) +class Name: + name: str + + +@dataclass(frozen=True) +class InlineTerm: + text: str + + +@dataclass(frozen=True) +class Not: + operand: "Node" + + +@dataclass(frozen=True) +class And: + operands: tuple["Node", ...] + + +@dataclass(frozen=True) +class Or: + operands: tuple["Node", ...] + + +@dataclass(frozen=True) +class Call: + func: str + args: tuple # Nodes, ints or a variable-number string, as the signature says + op: str | None = None + value: float | None = None + + +Node = Union[Name, InlineTerm, Not, And, Or, Call] + + +def check_name(name: str) -> str | None: + """Problem with a name defined in the framework, or None if it is valid.""" + if not NAME_RE.fullmatch(name): + hint = re.sub(r"[^a-z0-9_-]+", "_", name).strip("_") + if hint and not hint[0].isalpha(): + hint = "x_" + hint + return (f"name {name!r} must start with a letter and contain only letters, digits, '_' or '-'" + + (f" (for example {hint!r})" if hint else "")) + if name in RESERVED_WORDS: + return f"name {name!r} is a reserved word of the query language" + return None + + +# --------------------------------------------------------------------- tokenizer + +_TOKEN = re.compile( + r""" + (?P\s+) + | "(?P[^"]*)" + | \{(?P[^{}]*)\} + | (?P>=|<=|!=|=|>|<) + | (?P\d+(?:\.\d+)*) + | (?P[A-Za-z][A-Za-z0-9_-]*) + | (?P[(),]) + | (?P.) + """, + re.VERBOSE | re.DOTALL, +) + + +def tokenize(text: str) -> list[tuple[str, str, int]]: + tokens: list[tuple[str, str, int]] = [] + pos = 0 + while pos < len(text): + m = _TOKEN.match(text, pos) + kind = m.lastgroup + if kind == "quoted": + name = m.group("quoted").strip() + if not name: + raise QuerySyntaxError(f'empty name "" at position {pos + 1}') + tokens.append(("name", name.lower(), pos)) + elif kind == "term": + term = " ".join(m.group("term").split()).lower() + if not term: + raise QuerySyntaxError(f"empty term {{}} at position {pos + 1}") + tokens.append(("term", term, pos)) + elif kind == "cmp": + tokens.append(("cmp", m.group(), pos)) + elif kind == "num": + tokens.append(("num", m.group(), pos)) + elif kind == "ident": + word = m.group().lower() + if word in OPERATORS: + tokens.append(("op", word, pos)) + elif word in FUNCTIONS and _next_char(text, m.end()) == "(": + tokens.append(("func", word, pos)) + else: + tokens.append(("name", word, pos)) + elif kind == "punct": + tokens.append((m.group(), m.group(), pos)) + elif kind == "other": + ch = m.group() + if ch == '"': + raise QuerySyntaxError(f"unclosed quotation mark at position {pos + 1}") + if ch in "{}": + raise QuerySyntaxError(f"unmatched {ch!r} at position {pos + 1}") + raise QuerySyntaxError(f"unexpected character {ch!r} at position {pos + 1}") + pos = m.end() + return tokens + + +def _next_char(text: str, pos: int) -> str: + while pos < len(text) and text[pos].isspace(): + pos += 1 + return text[pos] if pos < len(text) else "" + + +# ------------------------------------------------------------------------ parser + + +class _Parser: + def __init__(self, text: str): + self.tokens = tokenize(text) + self.i = 0 + + def peek(self): + return self.tokens[self.i] if self.i < len(self.tokens) else None + + def take(self): + tok = self.tokens[self.i] + self.i += 1 + return tok + + def expect(self, kind: str, what: str): + tok = self.peek() + if tok is None or tok[0] != kind: + where = f"at position {tok[2] + 1}" if tok else "at the end" + raise QuerySyntaxError(f"expected {what} {where}") + return self.take() + + def at(self, kind: str, value: str | None = None) -> bool: + tok = self.peek() + return tok is not None and tok[0] == kind and (value is None or tok[1] == value) + + def parse(self) -> Node: + if not self.tokens: + raise QuerySyntaxError("the expression is empty") + node = self.expr() + tok = self.peek() + if tok is not None: + raise QuerySyntaxError(f"unexpected {tok[1]!r} at position {tok[2] + 1}") + return node + + def expr(self) -> Node: + items = [self.and_expr()] + while self.at("op", "or"): + self.take() + items.append(self.and_expr()) + return items[0] if len(items) == 1 else Or(tuple(items)) + + def and_expr(self) -> Node: + items = [self.not_expr()] + while self.at("op", "and"): + self.take() + items.append(self.not_expr()) + return items[0] if len(items) == 1 else And(tuple(items)) + + def not_expr(self) -> Node: + if self.at("op", "not"): + self.take() + return Not(self.not_expr()) + return self.primary() + + def primary(self) -> Node: + tok = self.peek() + if tok is None: + raise QuerySyntaxError("the expression ends where a name or '(' was expected") + kind, value, pos = tok + if kind == "name": + self.take() + return Name(value) + if kind == "term": + self.take() + return InlineTerm(value) + if kind == "(": + self.take() + node = self.expr() + if not self.at(")"): + raise QuerySyntaxError(f"missing ')' for '(' at position {pos + 1}") + self.take() + return node + if kind == "func": + return self.call() + raise QuerySyntaxError(f"unexpected {value!r} at position {pos + 1}") + + def call(self) -> Node: + _, func, pos = self.take() + signature, comparable = FUNCTIONS[func] + self.expect("(", f"'(' after {func.upper()}") + args = [] + for k, kind in enumerate(signature): + if k: + self.expect(",", f"',' ({func.upper()} takes {len(signature)} arguments)") + if kind in ("expr", "set"): + args.append(self.expr()) + elif kind == "int": + tok = self.expect("num", f"a whole number as argument {k + 1} of {func.upper()}") + if "." in tok[1]: + raise QuerySyntaxError(f"{func.upper()} needs a whole number, not {tok[1]}") + args.append(int(tok[1])) + else: # varnum + tok = self.expect("num", "a variable number, e.g. V(2.1)") + args.append(tok[1]) + if not self.at(")"): + tok = self.peek() + raise QuerySyntaxError(f"{func.upper()} takes {len(signature)} argument(s); expected ')'" + + (f" at position {tok[2] + 1}" if tok else "")) + self.take() + op = value = None + if self.at("cmp"): + if not comparable: + raise QuerySyntaxError(f"{func.upper()}(...) cannot be compared with a number") + op = self.take()[1] + tok = self.expect("num", f"a number after {op}") + if tok[1].count(".") > 1: + raise QuerySyntaxError(f"invalid number {tok[1]}") + value = float(tok[1]) + elif comparable: # bare DOC(x), COUNT(x)...: at least one + op, value = (">", 0.0) if func == "doc_share" else (">=", 1.0) + if func == "doc_share" and not 0 <= value <= 1: + raise QuerySyntaxError("DOC_SHARE compares with a share between 0 and 1, e.g. DOC_SHARE(x) >= 0.05") + return Call(func, tuple(args), op, value) + + +def parse(text: str) -> Node: + """Parse an expression; raises QuerySyntaxError.""" + return _Parser(text).parse() + + +# ----------------------------------------------------------------------- helpers + + +def walk(node: Node): + """Yield every node of the tree.""" + yield node + if isinstance(node, Not): + yield from walk(node.operand) + elif isinstance(node, (And, Or)): + for op in node.operands: + yield from walk(op) + elif isinstance(node, Call): + for a in node.args: + if isinstance(a, (Name, InlineTerm, Not, And, Or, Call)): + yield from walk(a) + + +def names(node: Node) -> set[str]: + return {n.name for n in walk(node) if isinstance(n, Name)} + + +def inline_terms(node: Node) -> set[str]: + return {n.text for n in walk(node) if isinstance(n, InlineTerm)} + + +def variable_refs(node: Node) -> set[str]: + return {n.args[0] for n in walk(node) if isinstance(n, Call) and n.func == "v"} + + +def set_arguments(node: Node): + """Yield (function name, argument) for every argument that must be a set.""" + for n in walk(node): + if isinstance(n, Call): + for kind, arg in zip(FUNCTIONS[n.func][0], n.args): + if kind == "set": + yield n.func, arg + + +def to_text(node: Node) -> str: + """Canonical text of an expression (for messages and the codebook).""" + if isinstance(node, Name): + return node.name + if isinstance(node, InlineTerm): + return "{" + node.text + "}" + if isinstance(node, Not): + return "NOT " + _wrap(node.operand) + if isinstance(node, (And, Or)): + sep = " AND " if isinstance(node, And) else " OR " + return sep.join(_wrap(o) for o in node.operands) + args = ", ".join(str(a) if isinstance(a, (int, str)) else to_text(a) for a in node.args) + out = f"{node.func.upper()}({args})" + default = (">", 0.0) if node.func == "doc_share" else (">=", 1.0) + if node.op is not None and (node.op, node.value) != default: + out += f" {node.op} {node.value:g}" + return out + + +def _wrap(node: Node) -> str: + return f"({to_text(node)})" if isinstance(node, (And, Or)) else to_text(node) + + +def dependency_order(deps: dict[str, set[str]]) -> tuple[list[str], list[list[str]]]: + """Order names so each comes after the names it uses; also return any cycles.""" + order: list[str] = [] + state: dict[str, int] = {} + cycles: list[list[str]] = [] + keys = list(deps) + + def visit(name: str, path: list[str]) -> None: + st = state.get(name) + if st == 2: + return + if st == 1: + cycles.append(path[path.index(name):] + [name]) + return + state[name] = 1 + for dep in sorted(deps.get(name, ()), key=lambda d: keys.index(d) if d in keys else -1): + if dep in deps: + visit(dep, path + [name]) + state[name] = 2 + order.append(name) + + for name in keys: + visit(name, []) + return order, cycles diff --git a/ctrlfindr/sentiment.py b/ctrlfindr/sentiment.py new file mode 100644 index 0000000..7b65a45 --- /dev/null +++ b/ctrlfindr/sentiment.py @@ -0,0 +1,82 @@ +"""Optional VADER sentiment scores. + +Sentiment is used only when a framework sets ``sentiment = true``. The scores are then +computed during the analysis, on the original-case sentence, and cached in the database +table ``sentiment``, so a later analysis only scores sentences it has not seen. Ingest +removes the cached scores of documents it re-processes. The package vaderSentiment is an +optional install (``pip install vaderSentiment``). +""" + +from __future__ import annotations + +import importlib.util +import logging +import sqlite3 +from importlib import metadata + +import numpy as np +import pandas as pd + +from .common import META, SENTIMENT, CtrlFindrError + +log = logging.getLogger("ctrlfindr") +MISSING = ("sentiment = true needs the vaderSentiment package, which is not installed: " + "run 'pip install vaderSentiment', or set sentiment = false in the settings sheet") + + +def available() -> bool: + return importlib.util.find_spec("vaderSentiment") is not None + + +def _analyzer(): + if not available(): + raise CtrlFindrError(MISSING) + from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer + + return SentimentIntensityAnalyzer() + + +def version() -> str: + try: + return metadata.version("vaderSentiment") + except metadata.PackageNotFoundError: + return "?" + + +def score_texts(texts) -> list[float]: + """VADER compound score of each text (no caching).""" + sia = _analyzer() + return [sia.polarity_scores(t)["compound"] for t in texts] + + +def cached_scores(conn: sqlite3.Connection, sentences: pd.DataFrame) -> tuple[np.ndarray, int]: + """Compound scores for ``sentences`` (document, sentence_id, text), using and filling the cache. + + Only the cached scores of the documents in ``sentences`` are read, so this can be called + batch by batch. New scores are written, but not committed: the caller commits. + Returns the scores and the number of sentences that had to be scored. + """ + _analyzer() # fail early with a clear message + conn.execute(f"CREATE TABLE IF NOT EXISTS {SENTIMENT} (document TEXT NOT NULL, sentence_id INTEGER NOT NULL, " + "vader_compound REAL NOT NULL, PRIMARY KEY (document, sentence_id))") + conn.execute(f"CREATE TABLE IF NOT EXISTS {META} (key TEXT PRIMARY KEY, value TEXT NOT NULL)") + row = conn.execute(f"SELECT value FROM {META} WHERE key='sentiment_version'").fetchone() + if row is None or row[0] != version(): + conn.execute(f"DELETE FROM {SENTIMENT}") # scores from another vaderSentiment version + conn.execute(f"INSERT OR REPLACE INTO {META} (key, value) VALUES ('sentiment_version', ?)", (version(),)) + if sentences.empty: + return np.zeros(0), 0 + first, last = sentences["document"].min(), sentences["document"].max() + cache = pd.read_sql_query(f"SELECT document, sentence_id, vader_compound FROM {SENTIMENT} " + "WHERE document BETWEEN ? AND ?", conn, params=(first, last)) + merged = sentences[["document", "sentence_id", "text"]].merge(cache, on=["document", "sentence_id"], how="left") + missing = merged["vader_compound"].isna().to_numpy() + if missing.any(): + log.debug("Scoring sentiment of %d sentence(s) with VADER", int(missing.sum())) + new = score_texts(merged.loc[missing, "text"].tolist()) + merged.loc[missing, "vader_compound"] = new + conn.executemany( + f"INSERT OR REPLACE INTO {SENTIMENT} (document, sentence_id, vader_compound) VALUES (?, ?, ?)", + list(zip(merged.loc[missing, "document"], merged.loc[missing, "sentence_id"].astype(int), new)), + ) + return merged["vader_compound"].to_numpy(float), int(missing.sum()) diff --git a/ctrlfindr/spreadsheet.py b/ctrlfindr/spreadsheet.py new file mode 100644 index 0000000..b162511 --- /dev/null +++ b/ctrlfindr/spreadsheet.py @@ -0,0 +1,146 @@ +"""Read and write framework workbooks in OpenDocument (.ods) or Excel (.xlsx) format. + +Reading returns every cell as text, the way it is displayed, so that identifiers such as +variable number ``1.10`` are not turned into numbers. Writing is used for the templates +and the tests; it supports bold headers, column widths, text-formatted columns and +dropdown lists (data validation). +""" + +from __future__ import annotations + +import datetime as _dt +from dataclasses import dataclass, field +from pathlib import Path + +from . import ods + +SUFFIXES = (".ods", ".xlsx") + + +@dataclass +class Dropdown: + """A dropdown on one column: fixed ``values`` or the cells of ``source`` (sheet, column letter).""" + + column: int # 0-based + values: tuple[str, ...] = () + source: tuple[str, str] | None = None + rows: int = 500 # how many rows below the header get the dropdown + + +@dataclass +class SheetSpec: + name: str + rows: list[list[str]] + widths_cm: list[float] = field(default_factory=list) + header: bool = True # first row bold + wrap: bool = False # wrap text in all cells + dropdowns: list[Dropdown] = field(default_factory=list) + + +def read_workbook(path: str | Path) -> dict[str, list[list[str]]]: + """``{sheet name: rows}`` with cells as strings; trailing empty cells/rows removed.""" + path = Path(path) + suffix = path.suffix.lower() + if suffix == ".ods": + return ods.read_ods(path) + if suffix == ".xlsx": + return _read_xlsx(path) + raise ValueError(f"unsupported file type {path.suffix!r}; use .ods or .xlsx") + + +def write_workbook(path: str | Path, sheets: list[SheetSpec]) -> Path: + path = Path(path) + suffix = path.suffix.lower() + if suffix == ".ods": + ods.write_ods(path, sheets) + elif suffix == ".xlsx": + _write_xlsx(path, sheets) + else: + raise ValueError(f"unsupported file type {path.suffix!r}; use .ods or .xlsx") + return path + + +# ------------------------------------------------------------------------- xlsx + +def _xlsx_text(value) -> str: + if value is None: + return "" + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, int): + return str(value) + if isinstance(value, float): + return str(int(value)) if value.is_integer() else repr(value) + if isinstance(value, (_dt.datetime, _dt.date, _dt.time)): + return value.isoformat() + return str(value) + + +def _read_xlsx(path: Path) -> dict[str, list[list[str]]]: + from openpyxl import load_workbook + + wb = load_workbook(path, read_only=True, data_only=True) + try: + sheets: dict[str, list[list[str]]] = {} + for ws in wb.worksheets: + rows: list[list[str]] = [] + pending_blank = 0 + for values in ws.iter_rows(values_only=True): + cells = [_xlsx_text(v) for v in values] + while cells and not cells[-1].strip(): + cells.pop() + if not cells: + pending_blank += 1 + continue + rows.extend([] for _ in range(pending_blank)) + pending_blank = 0 + rows.append(cells) + sheets[ws.title] = rows + return sheets + finally: + wb.close() + + +def _col_letter(index: int) -> str: + from openpyxl.utils import get_column_letter + + return get_column_letter(index + 1) + + +def _write_xlsx(path: Path, sheets: list[SheetSpec]) -> None: + from openpyxl import Workbook + from openpyxl.styles import Alignment, Font + from openpyxl.worksheet.datavalidation import DataValidation + + wb = Workbook() + wb.remove(wb.active) + for spec in sheets: + ws = wb.create_sheet(spec.name) + width = max([len(r) for r in spec.rows] + [len(spec.widths_cm), 1]) + for r, row in enumerate(spec.rows, start=1): + for c, value in enumerate(row, start=1): + cell = ws.cell(row=r, column=c, value=value if value != "" else None) + cell.number_format = "@" # text: Excel must not turn 1.10 into 1.1 + if spec.wrap: + cell.alignment = Alignment(wrap_text=True, vertical="top") + if spec.header and r == 1: + cell.font = Font(bold=True) + # Keep typed values as text in empty rows too. + for c in range(1, width + 1): + for r in range(len(spec.rows) + 1, len(spec.rows) + 501): + ws.cell(row=r, column=c).number_format = "@" + for c, w in enumerate(spec.widths_cm): + ws.column_dimensions[_col_letter(c)].width = w * 5 # ~characters per cm + if spec.header: + ws.freeze_panes = "A2" + for dd in spec.dropdowns: + if dd.source: + sheet, col = dd.source + formula = f"'{sheet}'!${col}$2:${col}${dd.rows + 1}" + else: + formula = '"' + ",".join(dd.values) + '"' + dv = DataValidation(type="list", formula1=formula, allow_blank=True, showErrorMessage=False) + letter = _col_letter(dd.column) + dv.add(f"{letter}2:{letter}{dd.rows + 1}") + ws.add_data_validation(dv) + wb.save(path) diff --git a/ctrlfindr/template.py b/ctrlfindr/template.py new file mode 100644 index 0000000..003117d --- /dev/null +++ b/ctrlfindr/template.py @@ -0,0 +1,106 @@ +"""Blank framework templates (.ods / .xlsx) with instructions and dropdowns.""" + +from __future__ import annotations + +from dataclasses import asdict +from pathlib import Path + +from .framework import FORMAT_VERSION, SENTIMENT_TERMS, Settings +from .spreadsheet import Dropdown, SheetSpec, write_workbook + +META_KEYS = ("format_version", "title", "version", "author", "language", "description") + +def _show(value) -> str: + """A setting value as written in the settings sheet.""" + if isinstance(value, bool): + return str(value).lower() + if isinstance(value, tuple): + return ", ".join(value) + return str(value) + + +SETTING_NOTES = { + "sentiment": "true: score every sentence with VADER (optional package vaderSentiment) and allow the " + "sentiment_* flags in expressions", + "sentiment_positive_threshold": "VADER compound score at or above which a sentence is positive", + "sentiment_negative_threshold": "VADER compound score at or below which a sentence is negative", + "aggregate_children": "true: a variable is also true wherever one of its child variables is true", + "tests_must_pass": "true: stop the analysis when a test in the tests sheet fails", + "exclude_sentence_types": "sentence types left out of the analysis, separated by commas: header, body, " + "list, table, footnote (from .md files), unstructured (all sentences of .txt files)", +} + +INSTRUCTIONS = [ + # (sheet or topic, rule, example) + ("overview", "A framework tells CtrlFindr how to code every sentence. Lists of terms (taxonomy) are combined " + "into queries, and queries into the variables that are exported. Matching ignores upper/lower case.", ""), + ("names", "Lists and queries share one set of names: start with a letter; then letters, digits, _ or -. " + "Each name must be unique.", "fast_onset, co-slow, a1"), + ("names", "Reserved names: and, or, not, the function names below, the sentiment_* flags, document, " + "sentences, sentence_id, start, end, vader_compound.", ""), + ("meta", "One key per row. format_version is required and must stay " + FORMAT_VERSION + ". Other keys are " + "optional and are copied into run_metadata.json and the codebook; you may add your own.", + "title | Climate security frameworks"), + ("settings", "One setting per row; a missing row uses the default shown in this template.", "sentiment | true"), +] + [("settings", f"{k}: {SETTING_NOTES[k]}.", f"default: {_show(v) or 'empty'}") for k, v in asdict(Settings()).items()] + [ + ("taxonomy", "One term per row: list name, term, optional note. Terms match whole words only.", "hazards | drought"), + ("taxonomy", "* stands for any letters or digits, also none. It can appear anywhere in a term but never " + "crosses a space or hyphen.", "flood* -> flood, floods, flooding"), + ("taxonomy", "~ stands for a hyphen, a space or nothing.", "human~wildlife -> human-wildlife, human wildlife"), + ("taxonomy", "_ on its own stands for exactly one word.", "climate _ risk -> climate change risk"), + ("taxonomy", "A term starting with - is an exclusion phrase: a match of the list inside it is ignored; other " + "matches in the same sentence still count.", "-business climate"), + ("taxonomy", "-@list: every term of that list is an exclusion phrase for this list.", "-@climate_idioms"), + ("queries", "One named expression per row. Combine names with AND, OR, NOT and parentheses. NOT binds " + "tighter than AND, AND tighter than OR. Quotes around names are optional.", "fast AND NOT (slow OR co_slow)"), + ("queries", "Names can be lists, other queries or sentiment flags.", "a2 AND sentiment_negative"), + ("queries", "{term} uses a term directly, with the taxonomy term syntax.", "fast AND {el niño}"), + ("queries", "NEAR(a, b, n): a and b at most n words apart, in any order (words in between; adjacent = 0). " + "a and b must be lists, {terms} or OR combinations of those.", "NEAR(increase, capacity, 4)"), + ("queries", "BEFORE(a, b, n): as NEAR, with a first.", "BEFORE(drought, migration, 10)"), + ("queries", "NOT_NEAR(a, b, n): a occurs somewhere with no b within n words.", "NOT_NEAR(conflict, wildlife, 5)"), + ("queries", "DOC(x): x is true in at least one sentence of the same document; add >= n for at least n " + "sentences.", "hwc AND DOC(ipcc) >= 3"), + ("queries", "DOC_SHARE(x) >= p: x is true in at least this share (0 to 1) of the document's sentences.", + "DOC_SHARE(climate) >= 0.05"), + ("queries", "WINDOW(x, k): x is true in this sentence or within k sentences before or after it.", + "drought AND WINDOW(conflict, 1)"), + ("queries", "COUNT(a) >= n: at least n matches of a in the sentence. COUNT_DISTINCT(a) >= n: at least n " + "different terms. A match inside a longer match of the same list counts once.", "COUNT_DISTINCT(hazards) >= 2"), + ("queries", "Comparisons: >= > = != <= <. Sentiment flags: " + ", ".join(SENTIMENT_TERMS) + ".", ""), + ("variables", "The exported results. number: 1, 1.1, 1.1.1 (unique; type them as text). name: free text.", "1.1 | fast onset"), + ("variables", "definition: an expression as in queries; V(n) uses another variable.", "a1 OR V(1.2)"), + ("variables", "parent: optional. When empty, 1.1 belongs to 1 and 1.1.1 to 1.1.", ""), + ("variables", "With aggregate_children = true a parent is also true wherever a child is true, and its definition " + "may stay empty.", ""), + ("tests", "Example sentences checked on every run. expect_true / expect_false: lists, queries, sentiment flags " + "or variable numbers, separated by commas.", "Floods hit the coast. | fast, 1.1 | slow"), + ("tests", "group: rows with the same group form one document, in order (for DOC, DOC_SHARE and WINDOW). " + "A row in a group may have no expectations and only provide context.", "group: doc1"), + ("instructions", "This sheet is ignored by CtrlFindr.", ""), +] + + +def template_sheets() -> list[SheetSpec]: + settings_rows = [["key", "value", "note"]] + [ + [k, _show(v), SETTING_NOTES[k]] for k, v in asdict(Settings()).items() + ] + return [ + SheetSpec("instructions", [["sheet", "rule", "example"]] + [list(r) for r in INSTRUCTIONS], [3, 17, 8], wrap=True), + SheetSpec("meta", [["key", "value"], ["format_version", FORMAT_VERSION], ["title", ""], ["version", ""], + ["author", ""], ["language", "en"], ["description", ""]], [5, 14], + dropdowns=[Dropdown(0, META_KEYS, rows=40)]), + SheetSpec("settings", settings_rows, [7, 3, 16], + dropdowns=[Dropdown(0, tuple(asdict(Settings())), rows=20), Dropdown(1, ("true", "false"), rows=20)]), + SheetSpec("taxonomy", [["list", "term", "note"]], [5, 7, 12]), + SheetSpec("queries", [["name", "query", "note"]], [5, 16, 10]), + SheetSpec("variables", [["number", "name", "definition", "parent", "note"]], [2.5, 8, 12, 2.5, 8], + dropdowns=[Dropdown(3, source=("variables", "A"), rows=500)]), + SheetSpec("tests", [["sentence", "expect_true", "expect_false", "group", "note"]], + [14, 6, 6, 3, 8]), + ] + + +def write_template(path: str | Path) -> Path: + """Write a blank framework (.ods or .xlsx).""" + return write_workbook(path, template_sheets()) diff --git a/ctrlfindr/validation.py b/ctrlfindr/validation.py new file mode 100644 index 0000000..cb24647 --- /dev/null +++ b/ctrlfindr/validation.py @@ -0,0 +1,215 @@ +"""Validation against hand coding: draw a sample to code, then report precision and recall. + +Workflow: + +1. ``python -m ctrlfindr sample --n 50`` writes ``validation_sample.tsv`` to the output + folder: for each variable, ``n`` sentences CtrlFindr flagged and ``n`` it did not, mixed + in random order. CtrlFindr's own code is not shown, so the coding stays blind. +2. A person fills the ``human_code`` column with 1 (the sentence is about the variable) or + 0, and saves the file as the project's validation file (``[paths] validation``). +3. Every analysis then writes ``validation_report.tsv`` and ``validation_disagreements.tsv``. + +Because flagged and unflagged sentences are sampled separately (stratified), recall is +estimated by weighting each stratum by its size in the corpus: + + precision = TP_sample / n_flagged_sample + TP_est = precision * N_flagged + FN_est = (FN_sample / n_unflagged_sample) * N_unflagged + recall = TP_est / (TP_est + FN_est) + +This assumes the coded sentences were drawn at random within each stratum, as ``sample`` +does. Coded sentences are matched to the current results by document and sentence_id, and +their ``sentence`` text is checked against the current sentence: if a document was edited +and ingested again, sentence numbers shift, and such codes are left out (and reported) +rather than compared with a different sentence. +""" + +from __future__ import annotations + +import logging +from pathlib import Path + +import numpy as np +import pandas as pd + +from .common import (DOCUMENTS, RESULTS, SENTENCES, CtrlFindrError, ValidationError, collapse_ws, connect, + normalize_text, table_exists) +from .config import Config +from .framework import Framework, load_framework + +log = logging.getLogger("ctrlfindr") + + +def _target_column(fw: Framework, target: str) -> str | None: + t = str(target).strip().lower().replace(",", ".").rstrip(".") + numbers = {v.number: v.column for v in fw.variables} + if t in numbers: + return numbers[t] + if t.endswith(".0") and t[:-2] in numbers: # a spreadsheet saved variable 2 as 2.0 + return numbers[t[:-2]] + if t in fw.lists or t in {qq.name for qq in fw.queries}: + return t + return None + + +def _quote(column: str) -> str: + return '"' + column.replace('"', '""') + '"' + + +def sample(cfg: Config, n: int = 50, targets: list[str] | None = None, seed: int = 1) -> Path: + """Write a blind validation sample (n flagged + n unflagged sentences per target). + + Only the needed columns of the results are read, so this works for any corpus size. + """ + fw = load_framework(cfg.paths.framework) + db = cfg.paths.database + if not db.is_file(): + raise CtrlFindrError(f"database {db} not found; run ingest and analyze first") + targets = targets or [v.number for v in fw.variables] + rng = np.random.default_rng(seed) + rows = [] + with connect(db) as conn: + if not all(table_exists(conn, t) for t in (RESULTS, SENTENCES, DOCUMENTS)): + raise CtrlFindrError("run the analysis first; the sample is drawn from its results") + columns = {r[1] for r in conn.execute(f"PRAGMA table_info({RESULTS})")} + for t in targets: + col = _target_column(fw, t) + if col is None or col not in columns: + raise CtrlFindrError(f"{t!r} is not a variable number, list or query of the framework") + res = pd.read_sql_query(f"SELECT document, sentence_id, {_quote(col)} AS flag FROM {RESULTS} " + "ORDER BY document, sentence_id", conn) + flag = res["flag"].astype(bool).to_numpy() + for pool in (np.flatnonzero(flag), np.flatnonzero(~flag)): + take = rng.choice(pool, size=min(n, len(pool)), replace=False) if len(pool) else [] + rows += [{"document": res.at[i, "document"], "sentence_id": int(res.at[i, "sentence_id"]), + "variable": t} for i in take] + del res + out = pd.DataFrame(rows, columns=["document", "sentence_id", "variable"]) + out["sentence"] = [conn.execute(f"SELECT text FROM {SENTENCES} WHERE document=? AND sentence_id=?", + (d, int(i))).fetchone()[0] for d, i in zip(out["document"], out["sentence_id"])] + out = out.sample(frac=1, random_state=seed).reset_index(drop=True) + out["human_code"] = "" + out["note"] = "" + path = cfg.paths.output / "validation_sample.tsv" + path.parent.mkdir(parents=True, exist_ok=True) + out.to_csv(path, sep="\t", index=False, encoding="utf-8", lineterminator="\n") + log.info("Wrote %d sentences to code to %s (seed %d)", len(out), path, seed) + return path + + +def _comparable(text: str) -> str: + """Sentence text in the form used for comparison (as in results_sentences.tsv).""" + return collapse_ws(normalize_text(str(text))).lower() + + +def _code(value) -> int | None: + v = str(value).strip().lower() + return 1 if v in ("1", "true", "yes", "y") else 0 if v in ("0", "false", "no", "n") else None + + +class ValidationCollector: + """Compares hand codes with the results, batch by batch (see module docstring).""" + + def __init__(self, path: Path, fw: Framework): + self.path = path + try: + coded = pd.read_csv(path, sep="\t", dtype=str, keep_default_na=False, encoding="utf-8-sig") + except Exception as exc: + raise ValidationError("Validation file could not be read", [f"{path}: {exc}"]) from exc + missing = {"document", "sentence_id", "variable", "human_code"} - set(coded.columns) + if missing: + raise ValidationError("Validation file is missing columns", [f"{path.name}: {', '.join(sorted(missing))}"]) + coded["code"] = coded["human_code"].map(_code) + problems = [f"row {i + 2}: human_code {v!r} must be 1 or 0" for i, v in enumerate(coded["human_code"]) + if v.strip() and _code(v) is None] + if problems: + raise ValidationError(f"Validation file {path.name} has invalid codes", problems[:20]) + coded = coded[coded["code"].notna()].copy() + coded["sentence_id"] = pd.to_numeric(coded["sentence_id"], errors="coerce") + unknown = sorted({t for t in coded["variable"] if _target_column(fw, t) is None}) + if unknown: + raise ValidationError("Validation file names an unknown variable", [repr(t) for t in unknown]) + coded["column"] = [_target_column(fw, t) for t in coded["variable"]] + if "sentence" not in coded: + coded["sentence"] = "" + coded["check"] = [_comparable(t) for t in coded["sentence"]] + self.coded = coded + self.pred: dict[int, bool] = {} # row of `coded` -> CtrlFindr's value + self.mismatched: list[int] = [] # rows whose sentence cannot be found any more + self.moved = 0 # rows found again under a new sentence number + self.flagged = {c: 0 for c in coded["column"].unique()} + self.total = 0 + + def add(self, results: pd.DataFrame) -> None: + self.total += len(results) + for c in self.flagged: + self.flagged[c] += int(results[c].sum()) + docs = set(results["document"]) + mine = self.coded[self.coded["document"].isin(docs)] + if mine.empty: + return + key = results.set_index(["document", "sentence_id"]) + # where each sentence text occurs now (to find coded sentences whose number changed) + by_text: dict[tuple[str, str], list[int]] = {} + for d, i, t in zip(results["document"], results["sentence_id"], results["sentences"]): + by_text.setdefault((d, _comparable(t)), []).append(i) + for row, d, i, c, text in zip(mine.index, mine["document"], mine["sentence_id"], mine["column"], mine["check"]): + if (d, i) in key.index and (not text or _comparable(key.at[(d, i), "sentences"]) == text): + self.pred[row] = bool(key.at[(d, i), c]) + elif text and len(by_text.get((d, text), [])) == 1: # same sentence, new number + self.pred[row] = bool(key.at[(d, by_text[(d, text)][0]), c]) + self.moved += 1 + elif text or (d, i) in key.index: + self.mismatched.append(row) + + def finish(self, out_dir: Path) -> dict: + """Write validation_report.tsv and validation_disagreements.tsv.""" + if self.mismatched: + rows = ", ".join(str(r + 2) for r in self.mismatched[:10]) + (" ..." if len(self.mismatched) > 10 else "") + log.warning("Validation: %d coded sentence(s) left out because the coded sentence is no longer in " + "its document (the document changed since the sample was drawn); rows %s of %s. " + "Draw and code a new sample for those documents.", + len(self.mismatched), rows, self.path.name) + if self.moved: + log.info("Validation: %d coded sentence(s) were found under a new sentence number " + "(their document changed) and are used there", self.moved) + unverified = int((self.coded.loc[sorted(self.pred), "check"] == "").sum()) if self.pred else 0 + if unverified: + log.warning("Validation: %d coded sentence(s) have no sentence text, so it cannot be checked that " + "they still refer to the same sentence", unverified) + report, disagreements = [], [] + coded = self.coded.loc[sorted(self.pred)] + for target, grp in coded.groupby("variable", sort=False): + col = grp["column"].iat[0] + pred = np.array([self.pred[r] for r in grp.index]) + human = grp["code"].to_numpy().astype(int).astype(bool) + n_flag, n_unflag = int(pred.sum()), int((~pred).sum()) + tp, fp = int((pred & human).sum()), int((pred & ~human).sum()) + fn, tn = int((~pred & human).sum()), int((~pred & ~human).sum()) + N_flag = self.flagged[col] + N_unflag = self.total - N_flag + precision = tp / n_flag if n_flag else np.nan + tp_est = precision * N_flag if n_flag else np.nan + fn_est = (fn / n_unflag) * N_unflag if n_unflag else np.nan + recall = tp_est / (tp_est + fn_est) if n_flag and n_unflag and (tp_est + fn_est) > 0 else np.nan + f1 = (2 * precision * recall / (precision + recall) + if precision == precision and recall == recall and (precision + recall) > 0 else np.nan) + report.append({"variable": target, "column": col, "coded": len(grp), "coded_flagged": n_flag, + "coded_unflagged": n_unflag, "tp": tp, "fp": fp, "fn": fn, "tn": tn, + "corpus_flagged": N_flag, "corpus_unflagged": N_unflag, + "precision": precision, "recall_estimated": recall, "f1": f1}) + for (_, r), p, h in zip(grp.iterrows(), pred, human): + if p != h: + disagreements.append({"variable": target, "document": r["document"], + "sentence_id": int(r["sentence_id"]), "ctrlfindr": int(p), "human": int(h), + "sentence": r["sentence"]}) + rep = pd.DataFrame(report) + rep.to_csv(out_dir / "validation_report.tsv", sep="\t", index=False, encoding="utf-8", lineterminator="\n") + pd.DataFrame(disagreements, columns=["variable", "document", "sentence_id", "ctrlfindr", "human", "sentence"] + ).to_csv(out_dir / "validation_disagreements.tsv", sep="\t", index=False, encoding="utf-8", + lineterminator="\n") + coded_n = int(rep["coded"].sum()) if len(rep) else 0 + log.info("Validation: %d coded sentences for %d target(s); see validation_report.tsv", coded_n, len(rep)) + return {"file": str(self.path), "targets": len(rep), "coded_sentences": coded_n, + "left_out_changed": len(self.mismatched), "found_under_new_number": self.moved} + diff --git a/docs/result_changes.md b/docs/result_changes.md new file mode 100644 index 0000000..49a0d63 --- /dev/null +++ b/docs/result_changes.md @@ -0,0 +1,209 @@ +# How results changed from 2.1 to 3.0 + +This report covers the example project (`examples/example_project`). It separates the +causes, so that each difference can be traced to one change. Every difference listed here +is intended; none of them is suppressed in the tests. + +3.0 reads a new framework format. The 3.0 example framework expresses the 2.1 example's +rules in that format: each co-occurrence is a `NEAR(...)` query, the document-level +conditional is `doc_ipcc = DOC(ipcc)`, and each variable is an OR of its queries. The +results below were produced with it. Column names are compared case-insensitively, because +3.0 keeps the capitalization of variable names. + +## A. Same sentences, new analysis + +Here the 3.0 analysis step was run on the sentences committed with 2.1, with the 2.1 +sentiment scores and the 2.1 thresholds (0). Sentence splitting and sentiment scoring are +therefore identical, and only the matching and co-occurrence rules differ. +`tests/test_golden.py` checks that exactly these rows change +(`tests/fixtures/v21_sentences_expected_changes.json`). + +| column | sentences changed | cause | +|---|---:|---| +| `hwc` | 21 | dash folding: taxonomy term "human–wildlife" (en dash) now matches "human-wildlife" (hyphen) | +| `co_slow` | 8 | distance = words strictly between the two matches (v2.1 also counted the first keyword) | +| `co_hwc` | 1 | distance = words strictly between the two matches (v2.1 also counted the first keyword) | +| `co_resourceandcompetition` | 2 | distance = words strictly between the two matches (v2.1 also counted the first keyword) | +| `co_increasecapacity` | 1 | distance = words strictly between the two matches (v2.1 also counted the first keyword) | +| `a1` | 3 | follows from the changes above | +| `a2` | 3 | follows from the changes above | +| `b2` | 1 | follows from the changes above | +| `b3` | 1 | follows from the changes above | +| `c1` | 1 | follows from the changes above | +| `d1` | 1 | follows from the changes above | +| `1.1 fast climate onset` | 3 | follows from the changes above | +| `1.2 slow climate onset` | 3 | follows from the changes above | +| `2.1 human-wildlife conflict and fast climate onset` | 1 | follows from the changes above | +| `2.1.2 human-wildlife conflict and fast climate onsets in articles that mention ipcc` | 1 | follows from the changes above | +| `2.2 human-wildlife conflict and slow climate onsets` | 1 | follows from the changes above | +| `3. resource competition and human-wild life conflict` | 1 | follows from the changes above | +| `4. increase of adaptive capacity` | 1 | follows from the changes above | + +Notes: + +- **hwc (+21):** the taxonomy spells the term with an en dash (`human–wildlife`), while the + text uses a hyphen (`human-wildlife`). In 2.1 the term never matched. All 21 sentences + contain `human-wildlife`. +- **co_\* (+12):** each newly flagged pair is exactly N words apart. 2.1 counted the first + keyword as one of the N words. 3.0 counts only the words between the two matches, which is + what the `How_to_fill` sheet describes. A rule with distance 5 now allows 5 words in between, + not 4. +- **Self-matches:** dropping a co-occurrence of a word with itself (for example `encroach*`, + which is in both `hwc1` and `compete2`) changes no sentence in this corpus. The rule is + covered by unit tests. +- **Queries and variables:** these change only through the columns above. For example, a1 + drops by 3 because it excludes `co_slow`, which gained 8. +- **Sentence text:** the `sentences` column differs only by normalization (dashes, quotes, + NFKC) in 101 sentences. +- **Document tables:** all other cells of the four document tables are identical. + +## B. Full pipeline + +### Sentence splitting and cleaning + +The example corpus goes from 1,589 to 1,695 sentences (+6.7%), for these reasons: + +- **Line breaks.** 2.1 deleted line breaks that were followed by a capital letter or digit, + which glued words together (`securityclimate`, `historyhuman`, `definitionclimate`, ...). + Such words never match. 3.0 turns every line break into a space. +- **Blank lines.** A blank line now always ends a sentence. Headings such as "See also" or + "Terminology" become their own short segments; in 2.1 they were glued onto the next + sentence. +- **Splitter.** pySBD replaces NLTK punkt. Its boundaries differ in places (abbreviations, + decimals). pySBD does not split `2050.[4] The`, so 3.0 inserts a space between a full stop + and a following citation mark before splitting. +- **Missing spaces.** In `.txt` files, a missing space after a full stop is repaired + ("Cold War era.The broadening…" → two sentences). This happens 3 times in + `wikiClimate_security`, which goes from 249 to 252 sentences and gains one sentence each + for "1. climate change" and "1.2 slow climate onset". +- **Junk segments.** Segments without any letter are dropped. This keeps 2.1's rule (numbers + or punctuation only) and adds citation marks such as `[12]` or `[3][4]`, which pySBD splits + off as separate segments. + +Because documents have more sentences, the percentage-of-sentences values fall even where +counts rise. + +### Sentiment + +| change | effect on the example corpus | +|---|---| +| vaderSentiment (the VADER authors' reference implementation) instead of NLTK's port | 57 of 1,589 scores differ on the same lowercased sentences; 13 change sign. The differences are mostly in sentences with "but" or negations, which the two implementations weight differently. | +| VADER scores the original-case sentence instead of the lowercased one | 3 of about 1,700 scores differ (VADER boosts words in capitals) | +| thresholds ±0.05 instead of 0 | on the 3.0 scores: positive 687 → 678, negative 494 → 483, neutral 511 → 531 | + +The `vadar_compound` column is renamed `vader_compound`. + +### New columns + +`results_sentences.tsv` starts with `sentence_id`, `start` and `end`. With the document +name, they locate each sentence in `documents.clean_text` in `analysis.sqlite` +(`clean_text[start:end]`). + +### Comparison of the committed outputs + +Generated with `python tools/compare_outputs.py tests/fixtures/golden_v2.1 tests/fixtures/golden_v3 --rename vadar_compound=vader_compound`. + +#### results_document.tsv +- `1. climate change` (5 of 7 documents): wiki2023_United_Nations_Climate_Change_Conference: 84 → 86; wikiAnthropocene: 51 → 53; wikiClimate_change: 324 → 338; wikiClimate_justice: 115 → 120; wikiClimate_security: 161 → 166 +- `1.1 fast climate onset` (3 of 7 documents): examplefile: 1 → 0; wikiClimate_change: 32 → 31; wikiClimate_justice: 6 → 5 +- `1.2 slow climate onset` (6 of 7 documents): examplefile: 4 → 5; wiki2023_United_Nations_Climate_Change_Conference: 83 → 85; wikiAnthropocene: 51 → 53; wikiClimate_change: 292 → 307; wikiClimate_justice: 109 → 115; wikiClimate_security: 158 → 163 +- `2.1 human-wildlife conflict and fast climate onset` (1 of 7 documents): examplefile: 1 → 0 +- `2.1.2 human-wildlife conflict and fast climate onsets in articles that mention ipcc` (1 of 7 documents): examplefile: 1 → 0 +- `2.2 human-wildlife conflict and slow climate onsets` (1 of 7 documents): examplefile: 1 → 2 +- `3. resource competition and human-wild life conflict` (1 of 7 documents): wikiHuman-wildlife_conflict: 3 → 4 +- `4. increase of adaptive capacity` (1 of 7 documents): wiki2023_United_Nations_Climate_Change_Conference: 0 → 1 + +#### results_document_clipped.tsv +- `1.1 fast climate onset` (1 of 7 documents): examplefile: 1 → 0 +- `2.1 human-wildlife conflict and fast climate onset` (1 of 7 documents): examplefile: 1 → 0 +- `2.1.2 human-wildlife conflict and fast climate onsets in articles that mention ipcc` (1 of 7 documents): examplefile: 1 → 0 +- `4. increase of adaptive capacity` (1 of 7 documents): wiki2023_United_Nations_Climate_Change_Conference: 0 → 1 + +#### results_document_percentage_of_sentences.tsv +- `1. climate change` (6 of 7 documents): wiki2023_United_Nations_Climate_Change_Conference: 37.33 → 35.98; wikiAnthropocene: 20.82 → 20.78; wikiClimate_change: 57.24 → 56.71; wikiClimate_justice: 60.21 → 57.14; wikiClimate_security: 70 → 65.87; wikiHuman-wildlife_conflict: 1.667 → 1.527 +- `1.1 fast climate onset` (6 of 7 documents): examplefile: 8.333 → 0; wiki2023_United_Nations_Climate_Change_Conference: 0.4444 → 0.4184; wikiClimate_change: 5.654 → 5.201; wikiClimate_justice: 3.141 → 2.381; wikiClimate_security: 1.304 → 1.19; wikiHuman-wildlife_conflict: 0.8333 → 0.7634 +- `1.2 slow climate onset` (7 of 7 documents): examplefile: 33.33 → 41.67; wiki2023_United_Nations_Climate_Change_Conference: 36.89 → 35.56; wikiAnthropocene: 20.82 → 20.78; wikiClimate_change: 51.59 → 51.51; wikiClimate_justice: 57.07 → 54.76; wikiClimate_security: 68.7 → 64.68; wikiHuman-wildlife_conflict: 0.8333 → 0.7634 +- `2. climate change and human wild life conflict` (3 of 7 documents): wikiAnthropocene: 0.8163 → 0.7843; wikiClimate_change: 1.237 → 1.174; wikiClimate_justice: 0.5236 → 0.4762 +- `2.1 human-wildlife conflict and fast climate onset` (1 of 7 documents): examplefile: 8.333 → 0 +- `2.1.2 human-wildlife conflict and fast climate onsets in articles that mention ipcc` (1 of 7 documents): examplefile: 8.333 → 0 +- `2.2 human-wildlife conflict and slow climate onsets` (4 of 7 documents): examplefile: 8.333 → 16.67; wikiAnthropocene: 0.8163 → 0.7843; wikiClimate_change: 1.237 → 1.174; wikiClimate_justice: 0.5236 → 0.4762 +- `3. resource competition and human-wild life conflict` (1 of 7 documents): wikiHuman-wildlife_conflict: 2.5 → 3.053 +- `4. increase of adaptive capacity` (2 of 7 documents): wiki2023_United_Nations_Climate_Change_Conference: 0 → 0.4184; wikiClimate_change: 0.1767 → 0.1678 + +#### results_document_sentiment.tsv +- `1. climate change` (5 of 7 documents): wiki2023_United_Nations_Climate_Change_Conference: 0.1162 → 0.1207; wikiAnthropocene: -0.06166 → -0.055; wikiClimate_change: 0.07591 → 0.07456; wikiClimate_justice: 0.1178 → 0.1145; wikiClimate_security: 0.04519 → 0.0316 +- `1.1 fast climate onset` (3 of 7 documents): examplefile: 0.2732 → nan; wikiClimate_change: 0.06157 → 0.06166; wikiClimate_justice: -0.3558 → -0.4816 +- `1.2 slow climate onset` (6 of 7 documents): examplefile: 0 → 0.05464; wiki2023_United_Nations_Climate_Change_Conference: 0.1103 → 0.1151; wikiAnthropocene: -0.06166 → -0.055; wikiClimate_change: 0.07748 → 0.07586; wikiClimate_justice: 0.1439 → 0.1405; wikiClimate_security: 0.05724 → 0.04303 +- `2.1 human-wildlife conflict and fast climate onset` (1 of 7 documents): examplefile: 0.2732 → nan +- `2.1.2 human-wildlife conflict and fast climate onsets in articles that mention ipcc` (1 of 7 documents): examplefile: 0.2732 → nan +- `2.2 human-wildlife conflict and slow climate onsets` (1 of 7 documents): examplefile: -0.3182 → -0.0225 +- `3. resource competition and human-wild life conflict` (1 of 7 documents): wikiHuman-wildlife_conflict: -0.3176 → -0.1026 +- `4. increase of adaptive capacity` (1 of 7 documents): wiki2023_United_Nations_Climate_Change_Conference: nan → 0.9013 + +#### results_sentences.tsv +- sentences: 1589 → 1695 + +| document | old sentences | new sentences | +|---|---:|---:| +| examplefile | 12 | 12 | +| wiki2023_United_Nations_Climate_Change_Conference | 225 | 239 | +| wikiAnthropocene | 245 | 255 | +| wikiClimate_change | 566 | 596 | +| wikiClimate_justice | 191 | 210 | +| wikiClimate_security | 230 | 252 | +| wikiHuman-wildlife_conflict | 120 | 131 | + +| column | old: sentences True | new: sentences True | change | +|---|---:|---:|---:| +| `fast` | 94 | 95 | +1 | +| `slow` | 671 | 700 | +29 | +| `exclude_climate` | 0 | 0 | +0 | +| `slow1` | 191 | 193 | +2 | +| `slow2` | 723 | 747 | +24 | +| `hwc` | 28 | 55 | +27 | +| `exclude_conflict` | 40 | 41 | +1 | +| `hwc1` | 150 | 151 | +1 | +| `compete2` | 162 | 161 | -1 | +| `resource1` | 221 | 224 | +3 | +| `competition2` | 117 | 118 | +1 | +| `ipcc` | 16 | 16 | +0 | +| `increase` | 211 | 209 | -2 | +| `decrease` | 63 | 63 | +0 | +| `capacity` | 29 | 32 | +3 | +| `co_slow` | 84 | 94 | +10 | +| `co_hwc` | 41 | 41 | +0 | +| `co_resourceandcompetition` | 11 | 13 | +2 | +| `co_increasecapacity` | 3 | 4 | +1 | +| `co_decreasecapacity` | 1 | 1 | +0 | +| `doc_ipcc` | 1244 | 1325 | +81 | +| `sentiment_positive` | 678 | 679 | +1 | +| `sentiment_neutral` | 432 | 532 | +100 | +| `sentiment_neutralpositive` | 1110 | 1211 | +101 | +| `sentiment_negative` | 479 | 484 | +5 | +| `sentiment_neutralnegative` | 911 | 1016 | +105 | +| `a1` | 44 | 41 | -3 | +| `a2` | 698 | 729 | +31 | +| `b1` | 0 | 0 | +0 | +| `b2` | 1 | 0 | -1 | +| `b3` | 11 | 12 | +1 | +| `c1` | 4 | 5 | +1 | +| `d1` | 2 | 3 | +1 | +| `d2` | 1 | 1 | +0 | +| `1. climate change` | 742 | 770 | +28 | +| `1.1 fast climate onset` | 44 | 41 | -3 | +| `1.2 slow climate onset` | 698 | 729 | +31 | +| `2. climate change and human wild life conflict` | 12 | 12 | +0 | +| `2.1 human-wildlife conflict and fast climate onset` | 1 | 0 | -1 | +| `2.1.1 human-wildlife conflict and fast climate onsets in articles that do mention ipcc` | 0 | 0 | +0 | +| `2.1.2 human-wildlife conflict and fast climate onsets in articles that mention ipcc` | 1 | 0 | -1 | +| `2.2 human-wildlife conflict and slow climate onsets` | 11 | 12 | +1 | +| `3. resource competition and human-wild life conflict` | 4 | 5 | +1 | +| `4. increase of adaptive capacity` | 2 | 3 | +1 | +| `5. decrease of adaptive capacity` | 1 | 1 | +0 | + +## Regenerating this report + +``` +python -m ctrlfindr run --config examples/example_project/ctrlfindr.toml +python tools/compare_outputs.py tests/fixtures/golden_v2.1 examples/example_project/Output --rename vadar_compound=vader_compound +``` diff --git a/examples/example_project/Input/Assessment_framework.ods b/examples/example_project/Input/Assessment_framework.ods new file mode 100644 index 0000000..cb90276 Binary files /dev/null and b/examples/example_project/Input/Assessment_framework.ods differ diff --git a/Jupiter Notebook Example/Analysis/Text_files/examplefile.txt b/examples/example_project/Text_files/examplefile.txt similarity index 100% rename from Jupiter Notebook Example/Analysis/Text_files/examplefile.txt rename to examples/example_project/Text_files/examplefile.txt diff --git a/Jupiter Notebook Example/Analysis/Text_files/wiki2023_United_Nations_Climate_Change_Conference.txt b/examples/example_project/Text_files/wiki2023_United_Nations_Climate_Change_Conference.txt similarity index 100% rename from Jupiter Notebook Example/Analysis/Text_files/wiki2023_United_Nations_Climate_Change_Conference.txt rename to examples/example_project/Text_files/wiki2023_United_Nations_Climate_Change_Conference.txt diff --git a/Jupiter Notebook Example/Analysis/Text_files/wikiAnthropocene.txt b/examples/example_project/Text_files/wikiAnthropocene.txt similarity index 100% rename from Jupiter Notebook Example/Analysis/Text_files/wikiAnthropocene.txt rename to examples/example_project/Text_files/wikiAnthropocene.txt diff --git a/Jupiter Notebook Example/Analysis/Text_files/wikiClimate_change.txt b/examples/example_project/Text_files/wikiClimate_change.txt similarity index 100% rename from Jupiter Notebook Example/Analysis/Text_files/wikiClimate_change.txt rename to examples/example_project/Text_files/wikiClimate_change.txt diff --git a/Jupiter Notebook Example/Analysis/Text_files/wikiClimate_justice.txt b/examples/example_project/Text_files/wikiClimate_justice.txt similarity index 100% rename from Jupiter Notebook Example/Analysis/Text_files/wikiClimate_justice.txt rename to examples/example_project/Text_files/wikiClimate_justice.txt diff --git a/Jupiter Notebook Example/Analysis/Text_files/wikiClimate_security.txt b/examples/example_project/Text_files/wikiClimate_security.txt similarity index 100% rename from Jupiter Notebook Example/Analysis/Text_files/wikiClimate_security.txt rename to examples/example_project/Text_files/wikiClimate_security.txt diff --git a/Jupiter Notebook Example/Analysis/Text_files/wikiHuman-wildlife_conflict.txt b/examples/example_project/Text_files/wikiHuman-wildlife_conflict.txt similarity index 100% rename from Jupiter Notebook Example/Analysis/Text_files/wikiHuman-wildlife_conflict.txt rename to examples/example_project/Text_files/wikiHuman-wildlife_conflict.txt diff --git a/examples/example_project/ctrlfindr.toml b/examples/example_project/ctrlfindr.toml new file mode 100644 index 0000000..49ad3c8 --- /dev/null +++ b/examples/example_project/ctrlfindr.toml @@ -0,0 +1,22 @@ +# CtrlFindr project settings: where the files are and how the texts are read. +# How sentences are coded (sentiment thresholds etc.) is set in the framework's settings sheet. +# Relative paths are relative to this file's folder. + +[paths] +text_files = "Text_files" # folder with the .txt documents +framework = "Input/Assessment_framework.ods" # assessment framework (.ods or .xlsx) +output = "Output" # results (TSVs, codebook, run_metadata.json, log) +database = "Output/analysis.sqlite" # keep on a local disk, not a network drive +validation = "Input/validation.tsv" # hand-coded sentences; used only if the file exists + +[ingest] +text_cleaning = true # remove URLs; turn ".." and "..." into "." +encoding_errors = "strict" # "strict": stop on files that are not UTF-8; "replace": substitute bad bytes +long_segment_words = 500 # warn about "sentences" longer than this many words + +[analysis] +batch_sentences = 50000 # sentences analyzed at a time (whole documents); lower it to use less memory + +[export] +sentence_level = true # results_sentences.tsv +document_level = true # the four results_document*.tsv files diff --git a/examples/food_security_project/Input/Assessment_framework.ods b/examples/food_security_project/Input/Assessment_framework.ods new file mode 100644 index 0000000..b606872 Binary files /dev/null and b/examples/food_security_project/Input/Assessment_framework.ods differ diff --git a/examples/food_security_project/Input/Assessment_framework.xlsx b/examples/food_security_project/Input/Assessment_framework.xlsx new file mode 100644 index 0000000..05a961c Binary files /dev/null and b/examples/food_security_project/Input/Assessment_framework.xlsx differ diff --git a/examples/food_security_project/README.md b/examples/food_security_project/README.md new file mode 100644 index 0000000..9946a05 --- /dev/null +++ b/examples/food_security_project/README.md @@ -0,0 +1,55 @@ +# Sample project: the four dimensions of food security + +A small project for testing a CtrlFindr installation. The framework +(`Input/Assessment_framework.xlsx`, with an identical `.ods` copy) measures availability, +access, utilization and stability. It uses every term marker, query function and variable +feature at least once. The corpus contains the odd cases listed below. The texts are +invented for the test. + +## Run it + +From the repository folder: + +``` +python -m ctrlfindr check --config examples/food_security_project/ctrlfindr.toml +python -m ctrlfindr run --config examples/food_security_project/ctrlfindr.toml +python tools/compare_outputs.py examples/food_security_project/expected examples/food_security_project/Output +``` + +1. `check` should report the framework as valid and **all 29 tests passed**. +2. `run` should report: 1 hidden file skipped; `empty.txt` with no sentences; 44 sentences + ingested; 6 sentences excluded (5 headers, 1 footnote); 38 analyzed. +3. The comparison should say **all shared cells identical** for the four document tables, + and show the same sentence counts and totals for `results_sentences.tsv`. + +Also open `Input/Assessment_framework.xlsx` in Excel. Variable numbers such as `1.1` must +stay as typed, and the dropdowns in `settings` and in the `parent` column of `variables` +should work. + +## What to look for in `Output/results_sentences.tsv` + +| case | file | expected | +|---|---|---| +| byte-order mark and Windows line endings | drought_report.txt | first sentence starts with "drought update", with no stray character | +| missing space after a full stop | drought_report.txt | "…by a third.The rice crop…" becomes two sentences | +| `...` and a URL | drought_report.txt | "districts." ends a sentence; the URL is gone ("see for details.") | +| citation mark after a full stop | drought_report.txt | "[4] many households…" is its own sentence | +| heading without punctuation in a .txt file | drought_report.txt, market_prices.txt | glued onto the next sentence (`.txt` is unstructured; use `.md` to keep headings apart) | +| `El Niño` (non-ASCII) | drought_report.txt | `shocks` and `stab_seasonal` are true | +| `food~aid`, `cash~transfer*` | drought_report.txt, market_prices.txt | "food-aid" and "cash transfer" match | +| term typed with an en dash (`e–voucher*`) | market_prices.txt | matches "e-voucher" | +| excluded list (`-@idioms`) | market_prices.txt | "market of ideas" does not count as a market | +| excluded phrase (`-shock absorber`) | stability_outlook.md | `shocks` is false in "shock absorber reserves are low." | +| `.TXT` extension, full stops in a term (`u.n.`) | notes.TXT | ingested; `agencies` and `precedence` are true | +| empty file | empty.txt | a warning; not in the results | +| hidden file | ._notes.txt | skipped | +| Markdown headings, list items, table rows | nutrition_brief.md, stability_outlook.md | `structure` is header (excluded), list, table or body | +| footnote and its marker `[^1]` | nutrition_brief.md | the footnote is excluded; no "1" left in the sentence | +| year at the start of a wrapped line | nutrition_brief.md | "undernutrition rates rose between 2019 and 2020." is one body sentence | +| superscript ordinal | nutrition_brief.md | "21st" reads "21st" | +| DOC, DOC_SHARE | nutrition_brief.md, stability_outlook.md | `doc_ipc` true in every sentence of both; `doc_nutrition` only in nutrition_brief | +| WINDOW | stability_outlook.md | `stab_window` true for "prices then spiked." (flood in the previous sentence) | +| NEAR vs NOT NEAR | drought_report.txt | `prices_no_shock` false for "food prices rose very sharply after the drought." (5 words apart) | +| NOT_NEAR | stability_outlook.md | `access_markets` false for "floods and conflict disrupted markets" | + +The settings sheet uses a decimal comma (`0,05`) on purpose: it must be read as 0.05. diff --git a/examples/food_security_project/Text_files/._notes.txt b/examples/food_security_project/Text_files/._notes.txt new file mode 100644 index 0000000..1c484d5 Binary files /dev/null and b/examples/food_security_project/Text_files/._notes.txt differ diff --git a/examples/food_security_project/Text_files/drought_report.txt b/examples/food_security_project/Text_files/drought_report.txt new file mode 100644 index 0000000..48440ea --- /dev/null +++ b/examples/food_security_project/Text_files/drought_report.txt @@ -0,0 +1,8 @@ +DROUGHT UPDATE +The 2023 drought reduced the maize harvest by a third.The rice crop also declined in the north. +Livestock losses were reported in three districts... Farmers sold animals early. +Food prices rose very sharply after the drought.[4] Many households cannot afford a basic diet. +See https://example.org/report for details. + +El Niño is expected to bring erratic rains again next season. The lean season may start early. +“Food for thought”: imports and food-aid deliveries are not enough to close the gap. diff --git a/examples/food_security_project/Text_files/empty.txt b/examples/food_security_project/Text_files/empty.txt new file mode 100644 index 0000000..e69de29 diff --git a/examples/food_security_project/Text_files/market_prices.txt b/examples/food_security_project/Text_files/market_prices.txt new file mode 100644 index 0000000..ac2c47e --- /dev/null +++ b/examples/food_security_project/Text_files/market_prices.txt @@ -0,0 +1,6 @@ +Market prices +Prices in the capital increased by 20 percent year on year. +Traders say road access improved after the rainy season, and the main market reopened. +Incomes stayed flat, so wages buy less food than last year. +The government launched a cash transfer programme and an e–voucher scheme with the WFP. +The market of ideas is lively, but food for thought does not feed anyone. diff --git a/examples/food_security_project/Text_files/notes.TXT b/examples/food_security_project/Text_files/notes.TXT new file mode 100644 index 0000000..4cc11d7 --- /dev/null +++ b/examples/food_security_project/Text_files/notes.TXT @@ -0,0 +1 @@ +Short note. The U.N. visited the market on Monday. diff --git a/examples/food_security_project/Text_files/nutrition_brief.md b/examples/food_security_project/Text_files/nutrition_brief.md new file mode 100644 index 0000000..6719d1b --- /dev/null +++ b/examples/food_security_project/Text_files/nutrition_brief.md @@ -0,0 +1,24 @@ +# Nutrition *brief* + +Child **malnutrition** remains high, and dietary diversity is low.[^1] Stunting affects one child in three. + +Key findings +------------ + +- Diets rely on maize and rice, with few vegetables. +- Clean water and sanitation are lacking; diarrhoea is common. + Hygiene campaigns reached few villages. +1. Micronutrient supplements reduced anaemia. +2. School meals improved nutrition and attendance. + +Undernutrition rates rose between 2019 and +2020. Since then they have stabilised. + +| Indicator | 2022 | 2023 | +|-----------|------|------| +| Stunting (%) | 34 | 33 | +| Food price index | 120 | 145 | + +The IPC Phase 3 classification now covers two districts, a first in the 21st century. + +[^1]: Based on a 2023 household survey of 1,200 families. diff --git a/examples/food_security_project/Text_files/stability_outlook.md b/examples/food_security_project/Text_files/stability_outlook.md new file mode 100644 index 0000000..0ac8749 --- /dev/null +++ b/examples/food_security_project/Text_files/stability_outlook.md @@ -0,0 +1,14 @@ +Stability outlook +================= + +## Shocks + +Floods and conflict disrupted markets in the east. Prices then spiked. +Food prices rose again in the following month. + +> Each shock leaves households less able to cope. + +## Seasonal patterns + +Year after year the lean season brings hunger. Erratic rainfall makes harvests uncertain. +The early warning system flagged a drought risk. Shock absorber reserves are low. diff --git a/examples/food_security_project/ctrlfindr.toml b/examples/food_security_project/ctrlfindr.toml new file mode 100644 index 0000000..790b8f4 --- /dev/null +++ b/examples/food_security_project/ctrlfindr.toml @@ -0,0 +1,22 @@ +# CtrlFindr project settings: where the files are and how the texts are read. +# How sentences are coded (sentiment thresholds etc.) is set in the framework's settings sheet. +# Relative paths are relative to this file's folder. + +[paths] +text_files = "Text_files" # folder with the .txt documents +framework = "Input/Assessment_framework.xlsx" # or Input/Assessment_framework.ods; assessment framework (.ods or .xlsx) +output = "Output" # results (TSVs, codebook, run_metadata.json, log) +database = "Output/analysis.sqlite" # keep on a local disk, not a network drive +validation = "Input/validation.tsv" # hand-coded sentences; used only if the file exists + +[ingest] +text_cleaning = true # remove URLs; turn ".." and "..." into "." +encoding_errors = "strict" # "strict": stop on files that are not UTF-8; "replace": substitute bad bytes +long_segment_words = 500 # warn about "sentences" longer than this many words + +[analysis] +batch_sentences = 50000 # sentences analyzed at a time (whole documents); lower it to use less memory + +[export] +sentence_level = true # results_sentences.tsv +document_level = true # the four results_document*.tsv files diff --git a/examples/food_security_project/expected/results_document.tsv b/examples/food_security_project/expected/results_document.tsv new file mode 100644 index 0000000..b0ba9ed --- /dev/null +++ b/examples/food_security_project/expected/results_document.tsv @@ -0,0 +1,6 @@ +document 1. Availability 1.1 Production and supply 1.2 Falling or disrupted production 2. Access 2.1 Price pressure 2.2 Market access 2.3 Price pressure with negative tone 3. Utilization 3.1 Nutrition, water and health 3.2 Diverse or repeated nutrition terms 4. Stability 4.1 Shocks and seasonality 4.2 Shocks in shock-heavy documents 5. Food security (any dimension) 6. Nutrition focus in IPC documents 7. Calm reporting 8. Prices without shocks +drought_report 4 4 2 1 1 0 0 1 1 0 4 3 3 7 0 5 1 +market_prices 0 0 0 4 1 2 0 0 0 0 1 0 0 4 0 4 1 +notes 0 0 0 1 0 1 0 0 0 0 0 0 0 1 0 2 0 +nutrition_brief 1 1 0 0 0 0 0 9 9 3 1 0 0 10 13 10 1 +stability_outlook 1 1 0 2 2 0 0 0 0 0 5 2 3 6 0 3 2 diff --git a/examples/food_security_project/expected/results_document_clipped.tsv b/examples/food_security_project/expected/results_document_clipped.tsv new file mode 100644 index 0000000..1af74d0 --- /dev/null +++ b/examples/food_security_project/expected/results_document_clipped.tsv @@ -0,0 +1,6 @@ +document 1. Availability 1.1 Production and supply 1.2 Falling or disrupted production 2. Access 2.1 Price pressure 2.2 Market access 2.3 Price pressure with negative tone 3. Utilization 3.1 Nutrition, water and health 3.2 Diverse or repeated nutrition terms 4. Stability 4.1 Shocks and seasonality 4.2 Shocks in shock-heavy documents 5. Food security (any dimension) 6. Nutrition focus in IPC documents 7. Calm reporting 8. Prices without shocks +drought_report 1 1 1 1 1 0 0 1 1 0 1 1 1 1 0 1 1 +market_prices 0 0 0 1 1 1 0 0 0 0 1 0 0 1 0 1 1 +notes 0 0 0 1 0 1 0 0 0 0 0 0 0 1 0 1 0 +nutrition_brief 1 1 0 0 0 0 0 1 1 1 1 0 0 1 1 1 1 +stability_outlook 1 1 0 1 1 0 0 0 0 0 1 1 1 1 0 1 1 diff --git a/examples/food_security_project/expected/results_document_percentage_of_sentences.tsv b/examples/food_security_project/expected/results_document_percentage_of_sentences.tsv new file mode 100644 index 0000000..116f31a --- /dev/null +++ b/examples/food_security_project/expected/results_document_percentage_of_sentences.tsv @@ -0,0 +1,6 @@ +document 1. Availability 1.1 Production and supply 1.2 Falling or disrupted production 2. Access 2.1 Price pressure 2.2 Market access 2.3 Price pressure with negative tone 3. Utilization 3.1 Nutrition, water and health 3.2 Diverse or repeated nutrition terms 4. Stability 4.1 Shocks and seasonality 4.2 Shocks in shock-heavy documents 5. Food security (any dimension) 6. Nutrition focus in IPC documents 7. Calm reporting 8. Prices without shocks +drought_report 40.0 40.0 20.0 10.0 10.0 0.0 0.0 10.0 10.0 0.0 40.0 30.0 30.0 70.0 0.0 50.0 10.0 +market_prices 0.0 0.0 0.0 80.0 20.0 40.0 0.0 0.0 0.0 0.0 20.0 0.0 0.0 80.0 0.0 80.0 20.0 +notes 0.0 0.0 0.0 50.0 0.0 50.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 50.0 0.0 100.0 0.0 +nutrition_brief 7.6923076923076925 7.6923076923076925 0.0 0.0 0.0 0.0 0.0 69.23076923076923 69.23076923076923 23.076923076923077 7.6923076923076925 0.0 0.0 76.92307692307693 100.0 76.92307692307693 7.6923076923076925 +stability_outlook 12.5 12.5 0.0 25.0 25.0 0.0 0.0 0.0 0.0 0.0 62.5 25.0 37.5 75.0 0.0 37.5 25.0 diff --git a/examples/food_security_project/expected/results_document_sentiment.tsv b/examples/food_security_project/expected/results_document_sentiment.tsv new file mode 100644 index 0000000..70e3573 --- /dev/null +++ b/examples/food_security_project/expected/results_document_sentiment.tsv @@ -0,0 +1,6 @@ +document 1. Availability 1.1 Production and supply 1.2 Falling or disrupted production 2. Access 2.1 Price pressure 2.2 Market access 2.3 Price pressure with negative tone 3. Utilization 3.1 Nutrition, water and health 3.2 Diverse or repeated nutrition terms 4. Stability 4.1 Shocks and seasonality 4.2 Shocks in shock-heavy documents 5. Food security (any dimension) 6. Nutrition focus in IPC documents 7. Calm reporting 8. Prices without shocks +drought_report -0.100475 -0.100475 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 -0.05741428571428571 -0.08038 0.0 +market_prices 0.205675 0.2732 0.34735 0.2732 0.205675 0.196925 0.2732 +notes 0.0 0.0 0.0 0.0 +nutrition_brief 0.0 0.0 0.07867777777777778 0.07867777777777778 0.042899999999999994 0.0 0.07081 0.05446923076923076 0.07081 0.0 +stability_outlook -0.296 -0.296 0.0 0.0 -0.24846 0.0 -0.41409999999999997 -0.25638333333333335 -0.37263333333333337 0.0 diff --git a/examples/food_security_project/expected/results_sentences.tsv b/examples/food_security_project/expected/results_sentences.tsv new file mode 100644 index 0000000..4f2d881 --- /dev/null +++ b/examples/food_security_project/expected/results_sentences.tsv @@ -0,0 +1,39 @@ +sentence_id structure start end document sentences production imports prices income markets agencies nutrition water_health shocks trend_up sharp_rise trend_down seasonal ipc idioms vader_compound sentiment_positive sentiment_neutral sentiment_neutralpositive sentiment_negative sentiment_neutralnegative avail_prod avail_down avail_shock access_price access_income access_markets prices_no_shock util_nutrition util_diverse util_many stab_seasonal stab_window stab_doc doc_ipc doc_nutrition neg_price calm precedence 1. Availability 1.1 Production and supply 1.2 Falling or disrupted production 2. Access 2.1 Price pressure 2.2 Market access 2.3 Price pressure with negative tone 3. Utilization 3.1 Nutrition, water and health 3.2 Diverse or repeated nutrition terms 4. Stability 4.1 Shocks and seasonality 4.2 Shocks in shock-heavy documents 5. Food security (any dimension) 6. Nutrition focus in IPC documents 7. Calm reporting 8. Prices without shocks +1 unstructured 0 69 drought_report drought update the 2023 drought reduced the maize harvest by a third. True False False False False False False False True False False True False False False 0.0 False True True False True True True True False False False False False False False False False True False False False False False True True True False False False False False False False True False True True False False False +2 unstructured 70 111 drought_report the rice crop also declined in the north. True False False False False False False False False False False True False False False 0.0 False True True False True True True False False False False False False False False False False False False False False False False True True True False False False False False False False False False False True False False False +3 unstructured 112 162 drought_report livestock losses were reported in three districts. True False False False False False False False False False False False False False False -0.4019 False False False True True True False False False False False False False False False False False False False False False True False True True False False False False False False False False False False False True False True False +4 unstructured 163 190 drought_report farmers sold animals early. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False True False False False False False False False False False False False False False False False False True False +5 unstructured 191 239 drought_report food prices rose very sharply after the drought. False False True False False False False False True True True False False False False 0.0 False True True False True False False False True False False False False False False False True True False False False False False False False False True True False False False False False True True True True False False False +6 unstructured 240 287 drought_report [4] many households cannot afford a basic diet. False False True False False False True False False False False False False False False 0.0 False True True False True False False False False False False True True False False False True False False False False False False False False False False False False False True True False True True False True False False True +7 unstructured 288 304 drought_report see for details. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False True False False False False False False False False False False False False False False False False True False +8 unstructured 306 367 drought_report el niño is expected to bring erratic rains again next season. False False False False False False False False True False False False True False False 0.0 False True True False True False False False False False False False False False False True False True False False False False False False False False False False False False False False False True True True True False False False +9 unstructured 368 400 drought_report the lean season may start early. False False False False False False False False False False False False True False False 0.0 False True True False True False False False False False False False False False False False False False False False False True False False False False False False False False False False False False False False False False True False +10 unstructured 401 485 drought_report """food for thought"": imports and food-aid deliveries are not enough to close the gap." False True False False False False False False False False False False False False True 0.0 False True True False True True False False False False False False False False False False False False False False False True False True True False False False False False False False False False False False True False True False +1 unstructured 0 73 market_prices market prices prices in the capital increased by 20 percent year on year. False False True False True False False False False True False False True False False 0.2732 True False True False False False False False True False True True False False False False False False False False False False False False False False True True True False False False False True False False True False False True +2 unstructured 74 160 market_prices traders say road access improved after the rainy season, and the main market reopened. False False False False True False False False False False False False True False False 0.4215 True False True False False False False False False False True False False False False False False False False False False True False False False False True False True False False False False False False False True False True False +3 unstructured 161 220 market_prices incomes stayed flat, so wages buy less food than last year. False False False True False False False False False False False False False False False 0.0 False True True False True False False False False True False False False False False False False False False False False True False False False False True False False False False False False False False False True False True False +4 unstructured 221 308 market_prices the government launched a cash transfer programme and an e-voucher scheme with the wfp. False False False True False True False False False False False False False False False 0.128 True False True False False False False False False True False False False False False False False False False False False True True False False False True False False False False False False False False False True False True False +5 unstructured 309 382 market_prices the market of ideas is lively, but food for thought does not feed anyone. False False False False False False False False False False False False False False True 0.2382 True False True False False False False False False False False False False False False False False False False False False True False False False False False False False False False False False False False False False False True False +1 unstructured 0 11 notes short note. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False True False False False False False False False False False False False False False False False False True False +2 unstructured 12 50 notes the u.n. visited the market on monday. False False False False True True False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False True True False False False True False True False False False False False False False True False True False +2 body 17 79 nutrition_brief child malnutrition remains high, and dietary diversity is low. False False False False False False True False False False False False False False False -0.2732 False False False True True False False False False False False False True True False False False False True True False True False False False False False False False False True True True False False False True True True False +3 body 80 116 nutrition_brief stunting affects one child in three. False False False False False False True False False False False False False False False 0.0 False True True False True False False False False False False False True False False False False False True True False True False False False False False False False False True True False False False False True True True False +5 list 132 182 nutrition_brief diets rely on maize and rice, with few vegetables. True False False False False False True False False False False False False False False 0.0 False True True False True True False False False False False False True False False False False False True True False True False True True False False False False False True True False False False False True True True False +6 list 184 244 nutrition_brief clean water and sanitation are lacking; diarrhoea is common. False False False False False False False True False False False False False False False 0.4019 True False True False False False False False False False False False True False True False False False True True False True False False False False False False False False True True True False False False True True True False +7 list 245 284 nutrition_brief hygiene campaigns reached few villages. False False False False False False False True False False False False False False False 0.1027 True False True False False False False False False False False False True False False False False False True True False True False False False False False False False False True True False False False False True True True False +8 list 286 328 nutrition_brief micronutrient supplements reduced anaemia. False False False False False False True False False False False True False False False 0.0 False True True False True False False False False False False False True True False False False False True True False False False False False False False False False False True True True False False False True True False False +9 list 330 377 nutrition_brief school meals improved nutrition and attendance. False False False False False False True False False False False False False False False 0.4767 True False True False False False False False False False False False True False False False False False True True False True False False False False False False False False True True False False False False True True True False +10 body 379 427 nutrition_brief undernutrition rates rose between 2019 and 2020. False False False False False False True False False True False False False False False 0.0 False True True False True False False False False False False False True False False False False False True True False False False False False False False False False False True True False False False False True True False False +11 body 428 460 nutrition_brief since then they have stabilised. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False True True False True False False False False False False False False False False False False False False False True True False +12 table 462 485 nutrition_brief indicator | 2022 | 2023 False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False True True False True False False False False False False False False False False False False False False False True True False +13 table 487 509 nutrition_brief stunting (%) | 34 | 33 False False False False False False True False False False False False False False False 0.0 False True True False True False False False False False False False True False False False False False True True False True False False False False False False False False True True False False False False True True True False +14 table 511 539 nutrition_brief food price index | 120 | 145 False False True False False False False False False False False False False False False 0.0 False True True False True False False False False False False True False False False False False False True True False False False False False False False False False False False False False True False False True True False True +15 body 541 626 nutrition_brief the ipc phase 3 classification now covers two districts, a first in the 21st century. False False False False False False False False False False False False False True False 0.0 False True True False True False False False False False False False False False False False False False True True False True False False False False False False False False False False False False False False False True True False +3 body 27 77 stability_outlook floods and conflict disrupted markets in the east. False False False False True False False False True False False False False False False -0.3182 False False False True True False False False False False False False False False False False False True True False False False False False False False False False False False False False False True False True True False False False +4 body 78 97 stability_outlook prices then spiked. False False True False False False False False False True False False False False False 0.0 False True True False True False False False True False False True False False False False True False True False False False False False False False True True False False False False False True True False True False False True +5 body 98 144 stability_outlook food prices rose again in the following month. False False True False False False False False False True False False False False False 0.0 False True True False True False False False True False False True False False False False True False True False False False False False False False True True False False False False False True True False True False False True +6 body 146 193 stability_outlook each shock leaves households less able to cope. False False False False False False False False True False False False False False False -0.3818 False False False True True False False False False False False False False False False False False True True False False False False False False False False False False False False False False True False True True False False False +8 body 214 260 stability_outlook year after year the lean season brings hunger. False False False False False False False False False False False False True False False -0.25 False False False True True False False False False False False False False False False False False False True False False True False False False False False False False False False False False False False False False False True False +9 body 261 303 stability_outlook erratic rainfall makes harvests uncertain. True False False False False False False False False False False False False False False -0.296 False False False True True True False False False False False False False False False False False False True False False True False True True False False False False False False False False False False False True False True False +10 body 304 352 stability_outlook the early warning system flagged a drought risk. False False False False False False False False True False False False False True False -0.5423 False False False True True False False False False False False False False False False False False True True False False False False False False False False False False False False False False True False True True False False False +11 body 353 385 stability_outlook shock absorber reserves are low. False False False False False False False False False False False False False False False -0.5719 False False False True True False False False False False False False False False False False False False True False False True False False False False False False False False False False False False False False False False True False diff --git a/examples/markdown_project/Input/Assessment_framework.xlsx b/examples/markdown_project/Input/Assessment_framework.xlsx new file mode 100644 index 0000000..83f70e5 Binary files /dev/null and b/examples/markdown_project/Input/Assessment_framework.xlsx differ diff --git a/examples/markdown_project/Text_files/drought_brief.md b/examples/markdown_project/Text_files/drought_brief.md new file mode 100644 index 0000000..05ba321 --- /dev/null +++ b/examples/markdown_project/Text_files/drought_brief.md @@ -0,0 +1,39 @@ +# Drought and Conflict in Pastoral Regions + +*Illustrative policy brief written for the CtrlFindr examples.* + +## Summary + +Recurring droughts reduce pasture and water for herders.[^1] Competition over these +resources can turn into **violent conflict** when institutions are weak. This brief +summarises the evidence and lists options for adaptation. + +## Key findings + +- Droughts have become longer in the last two decades. +- Livestock losses push herders to move earlier and further. +- Clashes between herders and farmers rise in drought years, but only where + land rights are contested. +1. Early warning systems reduce losses. +2. Water sharing agreements lower the risk of conflict. + +## Evidence + +| Region | Drought years | Reported clashes | +|--------|---------------|------------------| +| North | 6 | 14 | +| East | 3 | 2 | + +The table shows that clashes are concentrated where droughts are frequent. As one +elder put it: + +> When the wells dry up, old disputes come back. + +## Recommendations + +Governments should invest in water infrastructure and in local mediation. Donors +can support [community-based adaptation](https://example.org/adaptation) programmes +that build resilience. + +[^1]: The figures in this brief are invented for the example; they describe no real + region or conflict. diff --git a/examples/markdown_project/ctrlfindr.toml b/examples/markdown_project/ctrlfindr.toml new file mode 100644 index 0000000..aa6cff1 --- /dev/null +++ b/examples/markdown_project/ctrlfindr.toml @@ -0,0 +1,22 @@ +# CtrlFindr project settings: where the files are and how the texts are read. +# How sentences are coded (sentiment thresholds etc.) is set in the framework's settings sheet. +# Relative paths are relative to this file's folder. + +[paths] +text_files = "Text_files" # folder with the .txt documents +framework = "Input/Assessment_framework.xlsx" # assessment framework (.ods or .xlsx) +output = "Output" # results (TSVs, codebook, run_metadata.json, log) +database = "Output/analysis.sqlite" # keep on a local disk, not a network drive +validation = "Input/validation.tsv" # hand-coded sentences; used only if the file exists + +[ingest] +text_cleaning = true # remove URLs; turn ".." and "..." into "." +encoding_errors = "strict" # "strict": stop on files that are not UTF-8; "replace": substitute bad bytes +long_segment_words = 500 # warn about "sentences" longer than this many words + +[analysis] +batch_sentences = 50000 # sentences analyzed at a time (whole documents); lower it to use less memory + +[export] +sentence_level = true # results_sentences.tsv +document_level = true # the four results_document*.tsv files diff --git a/legacy/README.md b/legacy/README.md new file mode 100644 index 0000000..baf7b82 --- /dev/null +++ b/legacy/README.md @@ -0,0 +1,19 @@ +# CtrlFindr 2.1 (legacy) + +This folder keeps the last notebook version of CtrlFindr for reference and citation. +It is not maintained; use the `ctrlfindr` package (see the main README) for new work. + +- `v2.1/CtrlFindr.ipynb`: the notebook exactly as released in version 2.1. +- `v2.1/ctrlfindr_v21.py`: the notebook's functions extracted unchanged into a module. + The test suite uses it to check that `tests/fixtures/golden_v2.1` (the outputs that were + committed with 2.1) can still be reproduced. Its known bugs are deliberately not fixed; + see `CHANGELOG.md` and `docs/result_changes.md` for what 3.0 changed. + +2.1 frameworks use a different format that CtrlFindr 3 does not read. The 2.1 example +framework is kept at `tests/fixtures/golden_v2.1/Assessment_framework_v2.1.ods`; the 3.0 +example (`examples/example_project/Input/`) expresses the same rules in format 1. + +To run the 2.1 notebook, place it in a folder that contains an `Analysis/` folder with +`Text_files/`, `Input/Assessment_framework.ods` (a 2.1 framework) and `Output/`, as the notebook's settings +cell expects. It needs `pandas`, `odfpy` and `nltk`; on NLTK 3.8.2 or later, replace +`nltk.download('punkt')` with `nltk.download('punkt_tab')`. diff --git a/Jupiter Notebook Example/CtrlFindr.ipynb b/legacy/v2.1/CtrlFindr.ipynb similarity index 100% rename from Jupiter Notebook Example/CtrlFindr.ipynb rename to legacy/v2.1/CtrlFindr.ipynb diff --git a/legacy/v2.1/ctrlfindr_v21.py b/legacy/v2.1/ctrlfindr_v21.py new file mode 100644 index 0000000..8342d62 --- /dev/null +++ b/legacy/v2.1/ctrlfindr_v21.py @@ -0,0 +1,393 @@ +"""CtrlFindr v2.1 analysis functions, extracted verbatim from legacy/v2.1/CtrlFindr.ipynb (cell 3). + +Kept only as a reference implementation: the v3 test suite runs it on the committed +v2.1 sentences to prove the golden fixtures are reproducible, and tools/compare_outputs.py +uses it to show what changed between versions. Do not fix bugs here. + +Differences from the notebook, all outside the analysis logic: +- NLTK is imported lazily so the module loads without NLTK data downloads; +- the notebook's global settings are module-level variables below; +- `run_analysis_from_sentences` replays the analysis half of the notebook's `main()`. +""" +import os +import re +import time + +import pandas as pd +import sqlite3 + + +class _LazyNLTK: + def __getattr__(self, name): + import nltk as _nltk + return getattr(_nltk, name) + + +nltk = _LazyNLTK() + + +class _LazyVader: + _sia = None + + def polarity_scores(self, text): + if self._sia is None: + from nltk.sentiment.vader import SentimentIntensityAnalyzer + self._sia = SentimentIntensityAnalyzer() + return self._sia.polarity_scores(text) + + +sent_i = _LazyVader() + +ASSESSMENT_FRAMEWORK_DIRECTORY = None +SENTIMENT_ANALYSIS = True + +## LOAD AND PREPARE DATA FOR ANALYSIS +# IMPORT TXT FILES FROM TEXT_FILES_DIRECTORY +def txt_to_sql(directory, db_path): + data = {"document": [], "text": []} + + for filename in os.listdir(directory): + if filename.endswith(".txt"): + filepath = os.path.join(directory, filename) + with open(filepath, 'r', encoding='utf-8') as file: + content = file.read() + data["document"].append(filename.replace(".txt", "")) + data["text"].append(content) + df = pd.DataFrame(data) + conn = sqlite3.connect(db_path) + df.to_sql('documents', conn, if_exists='replace', index=False) + print(df.shape) + conn.close() + return df + +# IMPORT ASSESSMENT FRAMEWORK +def create_dataframes(ods_file): + variables = process_dataframe(pd.read_excel(ods_file, sheet_name='variables', engine='odf')) + set_search_strings = process_dataframe(pd.read_excel(ods_file, sheet_name='search_strings', engine='odf')) + set_co_occurrences = process_dataframe(pd.read_excel(ods_file, sheet_name='co_occurrences', engine='odf')) + set_doc_conditionals = process_dataframe(pd.read_excel(ods_file, sheet_name='doc_conditionals', engine='odf')) + set_keywords = process_dataframe(pd.read_excel(ods_file, sheet_name='taxonomy', engine='odf'), drop_na=False) + return variables, set_search_strings, set_co_occurrences, set_doc_conditionals, set_keywords + +# Helper function to process the data from each sheet in the ODS file +def process_dataframe(df, lowercase_columns=True, drop_na=False): + if lowercase_columns: + df.columns = [convert_to_lowercase(col) for col in df.columns] + df = df.map(convert_to_lowercase) + if 'query' in df.columns: + df['query'] = df['query'].apply(format_logical_expression) + if drop_na: + df = df.dropna() + return df + +# Helper function to format logical expressions in the search strings' query column +def format_logical_expression(expression): + if not isinstance(expression, str): + return expression + expression = re.sub(r"(? 0, + 'sentiment_neutral': sentiment_scores == 0, + 'sentiment_neutralpositive': sentiment_scores >= 0, + 'sentiment_negative': sentiment_scores < 0, + 'sentiment_neutralnegative': sentiment_scores <= 0 + }) + +# RUN QUERIES LISTED IN ASSESSMENT_FRAMEWORK +def run_queries(df, set_search_strings): + results = df.copy() + for _, row in set_search_strings.iterrows(): + variable, query = row['query name'], row['query'] + # Convert AND, OR, NOT operations to their Python equivalents + query = query.replace("AND", "and").replace("OR", "or").replace("NOT", "not") + # Evaluate the query for each row in df + results[variable] = df.apply(lambda x: evaluate_query(x, query), axis=1) + return results + +# Helper function to evaluate the query +def evaluate_query(row, query): + for column in row.index: + query = query.replace(f'"{column}"', str(row[column])) + try: + return eval(query) + except Exception as e: + print(f"Error evaluating query: {query} - {e}") + return False + +# AGGREGATE VARIABLES LISTED IN ASSESSMENT_FRAMEWORK +def define_variables(results, variables): + for _, row in variables.iterrows(): + condition = row['aggregation query'] + child_vars = condition.split(',') + child_vars = [var.strip(" ()\"'") for var in child_vars] + variable_number = str(row['variable number']) + if '.' not in variable_number: + variable_number += '.' + parent_name = f"{variable_number} {row['variable']}" + results[parent_name] = results[child_vars].any(axis=1) + return results + + +## EXPORT RESULTS +def get_v_list(df): + return [col for col in df.columns if bool(re.match(r'^[0-9]+\.[0-9]*', str(col)))] + +def results_document(df, v_list): + codebook = df.groupby('document')[v_list].sum().astype(int).reset_index() + return codebook + +def results_document_clipped(df, v_list): + code_bool = df.groupby('document')[v_list].sum().clip(upper=1).reset_index() + return code_bool + +def results_document_percentage_of_sentences(df, v_list): + grouped = df.groupby('document') + codebook_count_sent = grouped['sentences'].count().reset_index() + code_sent_percent = grouped[v_list].sum().reset_index() + code_sent_percent = code_sent_percent.merge(codebook_count_sent, on='document', how='outer') + code_sent_percent[v_list] = code_sent_percent[v_list].div(code_sent_percent['sentences'], axis=0).multiply(100) + code_sent_percent.drop('sentences', axis=1, inplace=True) + return code_sent_percent + +def results_document_sentiment(df, v_list): + codebook_sentiment = df[['document']].drop_duplicates().set_index('document') + for var in v_list: + sentiment = df.loc[df[var], ['document', 'vadar_compound']].groupby('document')['vadar_compound'].mean() + codebook_sentiment[var] = codebook_sentiment.index.map(sentiment) + return codebook_sentiment.reset_index() + +### SQL FUNCTIONS MANAGEMENT +def load_data_from_sql(db_path, table_name): + conn = sqlite3.connect(db_path) + df = pd.read_sql(f'SELECT * FROM {table_name}', conn) + conn.close() + return df + +def save_data_to_sql(df, db_path, table_name): + conn = sqlite3.connect(db_path) + df.to_sql(table_name, conn, if_exists='replace', index=False) + conn.close() + +def load_data_from_sql(db_path, table_name): + conn = sqlite3.connect(db_path) + df = pd.read_sql(f'SELECT * FROM {table_name}', conn) + conn.close() + return df + + +## CHECK FOR USER-MADE ERRORS IN ASSESSEMENT FRAMEWORK +def preliminary_checks(set_search_strings, set_doc_conditionals, set_co_occurrences, taxonomy): + if not os.path.exists(ASSESSMENT_FRAMEWORK_DIRECTORY): + print(f"Error: The Assessment framework file '{ASSESSMENT_FRAMEWORK_DIRECTORY}' not found.") + return False + return True + +# Check the search strings for errors +def check_search_strings(search_strings, taxonomy, doc_conditionals, co_occurrences,variables): + required_columns = ['query name', 'query'] + if not all(column in search_strings.columns for column in required_columns): + print(f"Error: The search_strings tab does not have the required columns. Keep the column headers included in the template.") + return False + valid_terms = set(taxonomy.columns) | set(doc_conditionals['name of document-level conditional']) | set(co_occurrences['name of co-occurrence']) + if search_strings['query name'].duplicated().any(): + print("Error: The 'variable' column in the search_strings tab contains duplicate entries.") + return False + for query in search_strings['query']: + if query.count("(") != query.count(")"): + print(f"Error: Mismatched parentheses in '{query}'.") + return False + strings_in_quotes = re.findall(r'"([^"]*)"', query) + for string in strings_in_quotes: + if string not in valid_terms and string not in ['sentiment_positive', 'sentiment_neutral', 'sentiment_neutralpositive', 'sentiment_negative', 'sentiment_neutralnegative']: + print(f"Error: The string '{string}' inside quotation marks is not found in the provided taxonomy, document-level conditionals, or co-occurrences.") + return False + query_simplified = re.sub(r'"[^"]*"', '', query) + query_simplified = re.sub(r'\b(and|or|not)\b', '', query_simplified, flags=re.IGNORECASE) + query_simplified = query_simplified.replace("(", "").replace(")", "") + if query_simplified.strip() != "": + print(f"Error: Query contains invalid or improperly quoted strings: {query_simplified.strip()}") + return False + if '""' in query or re.search(r'"\s+"', query): + print("Error: Found strings not properly enclosed in quotation marks or missing quotation marks.") + return False + required_co_occurrence_columns = ['name of co-occurrence', 'first list', 'distance between lists', 'second list'] + if not all(column in co_occurrences.columns for column in required_co_occurrence_columns): + print("Error: co_occurrences tab does not have the required columns.") + return False + valid_terms_for_co_occurrences = set(taxonomy.columns) + for index, row in co_occurrences.iterrows(): + if row['first list'] not in valid_terms_for_co_occurrences or row['second list'] not in valid_terms_for_co_occurrences: + print(f"Error: Check if these strings in co_occurrences '{row['first list']}' or '{row['second list']}' are present in the taxonomy as column headers.") + return False + required_doc_conditional_columns = ['name of document-level conditional', 'list'] + if not all(column in doc_conditionals.columns for column in required_doc_conditional_columns): + print("Error: doc_conditionals tab does not have the required columns.") + return False + for index, row in doc_conditionals.iterrows(): + if row['list'] not in valid_terms_for_co_occurrences: + print(f"Error: The string '{row['list']}' in doc_conditionals is not present in the taxonomy as column headers.") + return False + all_variables_in_search_strings = set(search_strings['query name']) + child_variables = set() + for query in variables['aggregation query']: + matches = re.findall(r'"(.*?)"', query) + child_variables.update(matches) + missing_variables = child_variables - all_variables_in_search_strings + if missing_variables: + print(f"Error: The following queries are not present in the search_strings 'query name' column: {missing_variables}. Note that query names cannot be numerical") + return False + return True + +## ESTIMATE RUN TIME +def estimate_run_time(sample_df, full_run=False): + variables, set_search_strings, set_co_occurrences, set_doc_conditionals, set_keywords = create_dataframes(ASSESSMENT_FRAMEWORK_DIRECTORY) + key_dict = organize_keywords(set_keywords) + start_time = time.time() + sample_df = check_groups(sample_df, key_dict) + sample_df = initiate_co_occurrences(sample_df, set_co_occurrences, key_dict) + initiate_document_conditionals(sample_df, set_doc_conditionals) + if SENTIMENT_ANALYSIS: + sentiment_df = vadar_sentiment_analysis(sample_df['sentences']) + sample_df = pd.concat([sample_df, sentiment_df], axis=1) + sample_results = run_queries(sample_df, set_search_strings) + sample_results = define_variables(sample_results, variables) + end_time = time.time() + return end_time - start_time + +def run_analysis_from_sentences(df, ods_file, compound=None): + """Replay the analysis part of the v2.1 ``main()`` on an existing sentence table. + + ``df`` needs ``document`` and ``sentences`` columns (as stored in v2.1's + results_sentences.tsv). If ``compound`` is given, it replaces VADER scoring so the + result does not depend on the installed NLTK version. Returns the sentence-level + frame and the four document-level frames, as v2.1 would export them. + """ + global ASSESSMENT_FRAMEWORK_DIRECTORY + ASSESSMENT_FRAMEWORK_DIRECTORY = ods_file + variables, set_search_strings, set_co_occurrences, set_doc_conditionals, set_keywords = create_dataframes(ods_file) + key_dict = organize_keywords(set_keywords) + assert check_search_strings(set_search_strings, set_keywords, set_doc_conditionals, set_co_occurrences, variables) + df = df[["document", "sentences"]].copy() + df = check_groups(df, key_dict) + df = initiate_co_occurrences(df, set_co_occurrences, key_dict) + df = initiate_document_conditionals(df, set_doc_conditionals) + if SENTIMENT_ANALYSIS: + if compound is None: + sentiment_df = vadar_sentiment_analysis(df["sentences"]) + else: + s = pd.Series(list(compound), index=df.index) + sentiment_df = pd.DataFrame({ + "vadar_compound": s, + "sentiment_positive": s > 0, + "sentiment_neutral": s == 0, + "sentiment_neutralpositive": s >= 0, + "sentiment_negative": s < 0, + "sentiment_neutralnegative": s <= 0, + }) + df = pd.concat([df, sentiment_df], axis=1) + df = run_queries(df, set_search_strings) + df = define_variables(df, variables) + v_list = get_v_list(df) + out = { + "results_document": results_document(df, v_list), + "results_document_clipped": results_document_clipped(df, v_list), + "results_document_percentage_of_sentences": results_document_percentage_of_sentences(df, v_list), + } + if SENTIMENT_ANALYSIS: + out["results_document_sentiment"] = results_document_sentiment(df, v_list) + out["results_sentences"] = df + return out diff --git a/notebooks/CtrlFindr.ipynb b/notebooks/CtrlFindr.ipynb new file mode 100644 index 0000000..2559957 --- /dev/null +++ b/notebooks/CtrlFindr.ipynb @@ -0,0 +1,94 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cell-0", + "metadata": {}, + "source": [ + "# CtrlFindr\n", + "\n", + "Run a CtrlFindr project from a notebook. The same steps are available on the command line:\n", + "\n", + "```\n", + "python -m ctrlfindr run --config path/to/ctrlfindr.toml\n", + "```\n", + "\n", + "For long runs on a JupyterLab server, use the command line in a JupyterLab terminal: it keeps running if the browser tab is closed.\n", + "\n", + "Install once, from the repository folder: `pip install -r requirements.lock`, then `pip install -e . --no-deps`." + ] + }, + { + "cell_type": "code", + "id": "cell-1", + "metadata": {}, + "source": [ + "from pathlib import Path\n", + "\n", + "from ctrlfindr import analyze, ingest, load_config\n", + "from ctrlfindr.common import setup_logging\n", + "\n", + "# Path to the project's settings file (edit this).\n", + "CONFIG = Path(\"../examples/example_project/ctrlfindr.toml\")\n", + "\n", + "cfg = load_config(CONFIG)\n", + "setup_logging(log_file=cfg.paths.output / \"ctrlfindr.log\")" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-2", + "metadata": {}, + "source": [ + "## 1. Ingest\n", + "\n", + "Reads the text files into the database. Only new or changed files are processed, so re-running is cheap." + ] + }, + { + "cell_type": "code", + "id": "cell-3", + "metadata": {}, + "source": [ + "ingest.run(cfg)" + ], + "execution_count": null, + "outputs": [] + }, + { + "cell_type": "markdown", + "id": "cell-4", + "metadata": {}, + "source": [ + "## 2. Analyze\n", + "\n", + "Validates the assessment framework, codes every sentence and writes the five TSV files plus `run_metadata.json` to the output folder. Re-run this cell after editing the framework; there is no need to ingest again." + ] + }, + { + "cell_type": "code", + "id": "cell-5", + "metadata": {}, + "source": [ + "outputs = analyze.run(cfg)\n", + "outputs[\"results_document\"]" + ], + "execution_count": null, + "outputs": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..f641cbc --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,44 @@ +[build-system] +requires = ["setuptools>=77"] +build-backend = "setuptools.build_meta" + +[project] +name = "ctrlfindr" +dynamic = ["version"] +description = "Dictionary-based content analysis of text documents, sentence by sentence" +readme = "README.md" +license = "GPL-3.0-only" +authors = [{ name = "Cesare M. Scartozzi" }] +requires-python = ">=3.11" +dependencies = [ + "pandas>=2.1", + "numpy>=1.26", + "pysbd>=0.3.4", + "pyahocorasick>=2.1", + "openpyxl>=3.1", + "tqdm>=4.60", +] + +[project.optional-dependencies] +sentiment = ["vaderSentiment>=3.3.2"] # needed only for frameworks with sentiment = true +test = ["pytest>=8", "vaderSentiment>=3.3.2"] + +[project.scripts] +ctrlfindr = "ctrlfindr.__main__:main" + +[project.urls] +Homepage = "https://github.com/CeMSc/CtrlFindr" + +[tool.setuptools] +packages = ["ctrlfindr"] + +[tool.setuptools.dynamic] +version = { attr = "ctrlfindr.__version__" } + +[tool.pytest.ini_options] +testpaths = ["tests"] +filterwarnings = [ + # warnings raised by the unmodified v2.1 reference code in legacy/ + "ignore:invalid escape sequence:SyntaxWarning", + "ignore:The inplace keyword", +] diff --git a/requirements.lock b/requirements.lock new file mode 100644 index 0000000..2d5083a --- /dev/null +++ b/requirements.lock @@ -0,0 +1,25 @@ +# Exact versions used to produce tests/fixtures/golden_v3 and run CI. +# Install with: pip install -r requirements.lock && pip install -e . --no-deps +certifi==2026.10.23 +charset-normalizer==3.5.2 +colorama==0.4.6 ; sys_platform == "win32" +defusedxml==0.7.1 # test only: lets the v2.1 reference code (legacy/) read .ods files +et-xmlfile==2.0.0 +idna==3.20 +iniconfig==2.3.1 +numpy==2.4.6 +odfpy==1.4.1 # test only, see above +openpyxl==3.1.5 +packaging==26.3 +pandas==3.1.0 +pluggy==1.7.0 +pyahocorasick==2.3.1 +Pygments==2.21.0 +pysbd==0.3.4 +pytest==9.2.0 +python-dateutil==4.0.0 +requests==2.34.2 # needed by vaderSentiment +tqdm==4.70.1 +tzdata==2026.6 +urllib3==2.8.0 +vaderSentiment==3.3.2 # optional: only for frameworks with sentiment = true diff --git a/template/Assessment_framework.ods b/template/Assessment_framework.ods new file mode 100644 index 0000000..a57c1af Binary files /dev/null and b/template/Assessment_framework.ods differ diff --git a/template/Assessment_framework.xlsx b/template/Assessment_framework.xlsx new file mode 100644 index 0000000..2d074ab Binary files /dev/null and b/template/Assessment_framework.xlsx differ diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..c0721d7 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,18 @@ +from __future__ import annotations + +import sys +from pathlib import Path + +import pandas as pd +import pytest + +ROOT = Path(__file__).resolve().parent.parent +FIXTURES = ROOT / "tests" / "fixtures" +EXAMPLE = ROOT / "examples" / "example_project" +sys.path.insert(0, str(ROOT / "tests")) + + +@pytest.fixture(scope="session") +def golden_v21_sentences() -> pd.DataFrame: + """Sentence table committed with CtrlFindr 2.1 (input for analysis-stage tests).""" + return pd.read_csv(FIXTURES / "golden_v2.1" / "results_sentences.tsv", sep="\t") diff --git a/Jupiter Notebook Example/Analysis/Input/Assessment_framework.ods b/tests/fixtures/golden_v2.1/Assessment_framework_v2.1.ods similarity index 100% rename from Jupiter Notebook Example/Analysis/Input/Assessment_framework.ods rename to tests/fixtures/golden_v2.1/Assessment_framework_v2.1.ods diff --git a/Jupiter Notebook Example/Analysis/Output/results_document.tsv b/tests/fixtures/golden_v2.1/results_document.tsv similarity index 100% rename from Jupiter Notebook Example/Analysis/Output/results_document.tsv rename to tests/fixtures/golden_v2.1/results_document.tsv diff --git a/Jupiter Notebook Example/Analysis/Output/results_document_clipped.tsv b/tests/fixtures/golden_v2.1/results_document_clipped.tsv similarity index 100% rename from Jupiter Notebook Example/Analysis/Output/results_document_clipped.tsv rename to tests/fixtures/golden_v2.1/results_document_clipped.tsv diff --git a/Jupiter Notebook Example/Analysis/Output/results_document_percentage_of_sentences.tsv b/tests/fixtures/golden_v2.1/results_document_percentage_of_sentences.tsv similarity index 100% rename from Jupiter Notebook Example/Analysis/Output/results_document_percentage_of_sentences.tsv rename to tests/fixtures/golden_v2.1/results_document_percentage_of_sentences.tsv diff --git a/Jupiter Notebook Example/Analysis/Output/results_document_sentiment.tsv b/tests/fixtures/golden_v2.1/results_document_sentiment.tsv similarity index 100% rename from Jupiter Notebook Example/Analysis/Output/results_document_sentiment.tsv rename to tests/fixtures/golden_v2.1/results_document_sentiment.tsv diff --git a/Jupiter Notebook Example/Analysis/Output/results_sentences.tsv b/tests/fixtures/golden_v2.1/results_sentences.tsv similarity index 100% rename from Jupiter Notebook Example/Analysis/Output/results_sentences.tsv rename to tests/fixtures/golden_v2.1/results_sentences.tsv diff --git a/tests/fixtures/golden_v3/results_document.tsv b/tests/fixtures/golden_v3/results_document.tsv new file mode 100644 index 0000000..9ff3d56 --- /dev/null +++ b/tests/fixtures/golden_v3/results_document.tsv @@ -0,0 +1,8 @@ +document 1. climate change 1.1 fast climate onset 1.2 slow climate onset 2. climate change and human wild life conflict 2.1 human-wildlife conflict and fast climate onset 2.1.1 human-wildlife conflict and fast climate onsets in articles that do mention IPCC 2.1.2 human-wildlife conflict and fast climate onsets in articles that mention IPCC 2.2 human-wildlife conflict and slow climate onsets 3. resource competition and human-wild life conflict 4. increase of adaptive capacity 5. decrease of adaptive capacity +examplefile 5 0 5 2 0 0 0 2 1 1 1 +wiki2023_United_Nations_Climate_Change_Conference 86 1 85 0 0 0 0 0 0 1 0 +wikiAnthropocene 53 0 53 2 0 0 0 2 0 0 0 +wikiClimate_change 338 31 307 7 0 0 0 7 0 1 0 +wikiClimate_justice 120 5 115 1 0 0 0 1 0 0 0 +wikiClimate_security 166 3 163 0 0 0 0 0 0 0 0 +wikiHuman-wildlife_conflict 2 1 1 0 0 0 0 0 4 0 0 diff --git a/tests/fixtures/golden_v3/results_document_clipped.tsv b/tests/fixtures/golden_v3/results_document_clipped.tsv new file mode 100644 index 0000000..27d5312 --- /dev/null +++ b/tests/fixtures/golden_v3/results_document_clipped.tsv @@ -0,0 +1,8 @@ +document 1. climate change 1.1 fast climate onset 1.2 slow climate onset 2. climate change and human wild life conflict 2.1 human-wildlife conflict and fast climate onset 2.1.1 human-wildlife conflict and fast climate onsets in articles that do mention IPCC 2.1.2 human-wildlife conflict and fast climate onsets in articles that mention IPCC 2.2 human-wildlife conflict and slow climate onsets 3. resource competition and human-wild life conflict 4. increase of adaptive capacity 5. decrease of adaptive capacity +examplefile 1 0 1 1 0 0 0 1 1 1 1 +wiki2023_United_Nations_Climate_Change_Conference 1 1 1 0 0 0 0 0 0 1 0 +wikiAnthropocene 1 0 1 1 0 0 0 1 0 0 0 +wikiClimate_change 1 1 1 1 0 0 0 1 0 1 0 +wikiClimate_justice 1 1 1 1 0 0 0 1 0 0 0 +wikiClimate_security 1 1 1 0 0 0 0 0 0 0 0 +wikiHuman-wildlife_conflict 1 1 1 0 0 0 0 0 1 0 0 diff --git a/tests/fixtures/golden_v3/results_document_percentage_of_sentences.tsv b/tests/fixtures/golden_v3/results_document_percentage_of_sentences.tsv new file mode 100644 index 0000000..36dcd2a --- /dev/null +++ b/tests/fixtures/golden_v3/results_document_percentage_of_sentences.tsv @@ -0,0 +1,8 @@ +document 1. climate change 1.1 fast climate onset 1.2 slow climate onset 2. climate change and human wild life conflict 2.1 human-wildlife conflict and fast climate onset 2.1.1 human-wildlife conflict and fast climate onsets in articles that do mention IPCC 2.1.2 human-wildlife conflict and fast climate onsets in articles that mention IPCC 2.2 human-wildlife conflict and slow climate onsets 3. resource competition and human-wild life conflict 4. increase of adaptive capacity 5. decrease of adaptive capacity +examplefile 41.66666666666667 0.0 41.66666666666667 16.666666666666664 0.0 0.0 0.0 16.666666666666664 8.333333333333332 8.333333333333332 8.333333333333332 +wiki2023_United_Nations_Climate_Change_Conference 35.98326359832636 0.41841004184100417 35.56485355648535 0.0 0.0 0.0 0.0 0.0 0.0 0.41841004184100417 0.0 +wikiAnthropocene 20.784313725490197 0.0 20.784313725490197 0.7843137254901961 0.0 0.0 0.0 0.7843137254901961 0.0 0.0 0.0 +wikiClimate_change 56.711409395973156 5.201342281879195 51.51006711409396 1.174496644295302 0.0 0.0 0.0 1.174496644295302 0.0 0.16778523489932887 0.0 +wikiClimate_justice 57.14285714285714 2.380952380952381 54.761904761904766 0.4761904761904762 0.0 0.0 0.0 0.4761904761904762 0.0 0.0 0.0 +wikiClimate_security 65.87301587301587 1.1904761904761905 64.68253968253968 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 +wikiHuman-wildlife_conflict 1.5267175572519083 0.7633587786259541 0.7633587786259541 0.0 0.0 0.0 0.0 0.0 3.0534351145038165 0.0 0.0 diff --git a/tests/fixtures/golden_v3/results_document_sentiment.tsv b/tests/fixtures/golden_v3/results_document_sentiment.tsv new file mode 100644 index 0000000..8d5b9a4 --- /dev/null +++ b/tests/fixtures/golden_v3/results_document_sentiment.tsv @@ -0,0 +1,8 @@ +document 1. climate change 1.1 fast climate onset 1.2 slow climate onset 2. climate change and human wild life conflict 2.1 human-wildlife conflict and fast climate onset 2.1.1 human-wildlife conflict and fast climate onsets in articles that do mention IPCC 2.1.2 human-wildlife conflict and fast climate onsets in articles that mention IPCC 2.2 human-wildlife conflict and slow climate onsets 3. resource competition and human-wild life conflict 4. increase of adaptive capacity 5. decrease of adaptive capacity +examplefile 0.05464 0.05464 -0.022499999999999992 -0.022499999999999992 -0.3818 0.3182 -0.6908 +wiki2023_United_Nations_Climate_Change_Conference 0.12069534883720932 0.5994 0.11506352941176469 0.9013 +wikiAnthropocene -0.05499622641509434 -0.05499622641509434 0.1806 0.1806 +wikiClimate_change 0.0745603550295858 0.06166451612903226 0.07586254071661239 -0.10034285714285714 -0.10034285714285714 0.6911 +wikiClimate_justice 0.11453833333333334 -0.48156 0.14045565217391304 -0.0258 -0.0258 +wikiClimate_security 0.03159698795180723 -0.5896 0.043030061349693255 +wikiHuman-wildlife_conflict 0.15335000000000001 0.6249 -0.3182 -0.10264999999999999 diff --git a/tests/fixtures/golden_v3/results_sentences.tsv b/tests/fixtures/golden_v3/results_sentences.tsv new file mode 100644 index 0000000..25b50d5 --- /dev/null +++ b/tests/fixtures/golden_v3/results_sentences.tsv @@ -0,0 +1,1696 @@ +sentence_id structure start end document sentences fast slow exclude_climate slow1 slow2 hwc exclude_conflict hwc1 compete2 resource1 competition2 ipcc increase decrease capacity vader_compound sentiment_positive sentiment_neutral sentiment_neutralpositive sentiment_negative sentiment_neutralnegative co_slow co_hwc co_resourceandcompetition co_increasecapacity co_decreasecapacity doc_ipcc a1 a2 b1 b2 b3 c1 d1 d2 1. climate change 1.1 fast climate onset 1.2 slow climate onset 2. climate change and human wild life conflict 2.1 human-wildlife conflict and fast climate onset 2.1.1 human-wildlife conflict and fast climate onsets in articles that do mention IPCC 2.1.2 human-wildlife conflict and fast climate onsets in articles that mention IPCC 2.2 human-wildlife conflict and slow climate onsets 3. resource competition and human-wild life conflict 4. increase of adaptive capacity 5. decrease of adaptive capacity +1 unstructured 0 41 examplefile this file is used for debugging purposes. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +2 unstructured 42 90 examplefile there is going to be an increase in temperature. False False False True True False False False False False False False True False False 0.3182 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +3 unstructured 91 133 examplefile rainfall is expected to rapidly intensify. True False False True True False False False False False False False False False False 0.0 False True True False True True False False False False True False True False False False False False False True False True False False False False False False False False +4 unstructured 134 177 examplefile rainfall, is expected to rapidly intensify. True False False True True False False False False False False False False False False 0.0 False True True False True True False False False False True False True False False False False False False True False True False False False False False False False False +5 unstructured 178 247 examplefile human-wildlife conflict is expected to intensify with climate change. False True False True True True False True True False True False False False False -0.3182 False False False True True True True False False False True False True False False True False False False True False True True False False False True False False False +6 unstructured 248 285 examplefile human-wildlife, conflict of interest. False False False False False True True True True False True False False False False 0.1779 True False True False False False True False False False True False False False False False False False False False False False False False False False False False False False +7 unstructured 286 334 examplefile there has been a violent conflict in the region. False False False False False False False False True False True False False False False -0.7351 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +8 unstructured 335 454 examplefile the latest ipcc study finds that resource competition is escalating conflicts between animals and humans when it rains. False False False False False False False True True True True True False False False -0.3818 False False False True True False True True False False True False False False False False True False False False False False False False False False False True False False +9 unstructured 455 526 examplefile the policy intervention has failed to increase local adaptive capacity. False False False True False False False False False False False False True False True -0.25 False False False True True False False False True False True False False False False False False False False False False False False False False False False False False False +10 unstructured 527 601 examplefile the policy intervention has led to an increase in local adaptive capacity. False False False True False False False False False False False False True False True 0.3182 True False True False False False False False True False True False False False False False False True False False False False False False False False False False True False +11 unstructured 602 675 examplefile if we fail to act, we will undermine the resilience of future generation. False False False False False False False False False False False False False True True -0.6908 False False False True True False False False False True True False False False False False False False True False False False False False False False False False False True +12 unstructured 676 747 examplefile poaching has increased, especially in the aftermath of flooding events. True False False True True True False True False False False False True False False 0.2732 True False True False False True False False False False True False True False False True False False False True False True True False False False True False False False +1 unstructured 0 45 wiki2023_United_Nations_Climate_Change_Conference 2023 united nations climate change conference False True False False True False False False False False False False False False False 0.4215 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +2 unstructured 47 59 wiki2023_United_Nations_Climate_Change_Conference article talk False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +3 unstructured 61 83 wiki2023_United_Nations_Climate_Change_Conference read edit view history False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +4 unstructured 85 90 wiki2023_United_Nations_Climate_Change_Conference tools False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +5 unstructured 92 633 wiki2023_United_Nations_Climate_Change_Conference from wikipedia, the free encyclopedia 2023 united nations climate change conference logo a group photo of national leaders and other dignitaries at the 2023 united nations climate change conference native name مؤتمر الأمم المتحدة للتغير المناخي 2023 date 30 november - 13 december 2023 location expo city, dubai, united arab emirates organised by united arab emirates participants unfccc member countries president sultan al jaber previous event ← sharm el sheikh 2022 next event → baku 2024 website cop28.com al wasl plaza, expo city, dubai False True False False True False False False False False False False False False False 0.9337 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +6 unstructured 635 893 wiki2023_United_Nations_Climate_Change_Conference the 2023 united nations climate change conference or conference of the parties of the unfccc, more commonly known as cop28, was the 28th united nations climate change conference, held from 30 november to 13 december at expo city, dubai, united arab emirates. False True False False True False False False False False False False False False False 0.8779 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +7 unstructured 894 1027 wiki2023_United_Nations_Climate_Change_Conference the cop conference has been held annually (except 2020 due to the covid-19 pandemic)[1] since the first un climate agreement in 1992. False True False False True False False False False False False False False False False 0.4939 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +8 unstructured 1028 1173 wiki2023_United_Nations_Climate_Change_Conference the event is intended for governments to agree on policies to limit global temperature rises and adapt to impacts associated with climate change. False True False False True False False False False False False False False False False 0.3612 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +9 unstructured 1179 1315 wiki2023_United_Nations_Climate_Change_Conference the conference was originally scheduled to end on 12 december, but had to be extended following saudi objections on the final agreement. False False False False False False False False False False False False False False False 0.6486 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +10 unstructured 1316 1470 wiki2023_United_Nations_Climate_Change_Conference [3] on 13 december, the conference president, sultan al jaber announced that a final compromise agreement between the countries involved had been reached. False False False False False False False False False False False False False False False 0.5574 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +11 unstructured 1471 1670 wiki2023_United_Nations_Climate_Change_Conference "the deal commits all signatory countries to move away from carbon energy sources ""in a just, orderly and equitable manner"" to mitigate the worst effects of climate change, and reach net zero by 2050." False True False False True False False False False True False False False False False -0.4215 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +12 unstructured 1671 1975 wiki2023_United_Nations_Climate_Change_Conference [4] the global pact, referred to as the uae consensus, was the first in the history of cop summits to explicitly mention the need to shift away from every type of fossil fuels, but it still received widespread criticism due to the lack of a clear commitment to either fossil fuel phase-out or phase-down. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +13 unstructured 1976 2099 wiki2023_United_Nations_Climate_Change_Conference [4] china and india did not sign the pledge to triple their output of renewable energy and committed to coal power instead. False False False False False False False False False True False False False False False 0.4939 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +14 unstructured 2108 2328 wiki2023_United_Nations_Climate_Change_Conference the conference was widely criticised for its controversial president sultan al jaber, as well as its host country, the uae, which is known for its opaque environmental record and role as a major producer of fossil fuels. False False False False False False False False False False False False False False False -0.3612 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +15 unstructured 2329 2446 wiki2023_United_Nations_Climate_Change_Conference [7] al jaber is the ceo of the abu dhabi national oil company (adnoc), leading to concerns over conflict of interest. False False False False False False True False True False True False False False False 0.1779 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +16 unstructured 2447 2772 wiki2023_United_Nations_Climate_Change_Conference [8] claims of greenwashing of al jaber on wikipedia, twitter and medium;[9] the legal inability to criticise emirati corporations in the uae;[10] alleged covert access to conference emails by adnoc;[11] and the invitation of syrian president bashar al-assad have all raised concerns regarding the integrity of the conference. False False False False False False False False False False False False False False False -0.4404 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +17 unstructured 2773 3042 wiki2023_United_Nations_Climate_Change_Conference "[12] al jaber stated before the beginning of the conference that there was ""no science"" behind fossil fuel phase-out in achieving 1.5 °c;[13] and leaked documents appeared to show the uae planned to use the conference to strike new fossil fuel deals with other nations." False False False False False False False False False False False False False False False -0.4215 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +18 unstructured 3043 3256 wiki2023_United_Nations_Climate_Change_Conference "[14] al jaber claimed that his comments on the phase-out of fossil fuels were ""misinterpreted"" and denied the latter allegation, asserting that the uae does not need the cop presidency to establish business deals." False False False False False False False False False False False False False False False -0.6369 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +19 unstructured 3257 3276 wiki2023_United_Nations_Climate_Change_Conference [15][16] background False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +20 unstructured 3278 3403 wiki2023_United_Nations_Climate_Change_Conference the united arab emirates is one of the most climate-vulnerable countries in the world, due to its very hot and humid climate. False True False False True False False False False False False False False False False 0.4215 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +21 unstructured 3404 3526 wiki2023_United_Nations_Climate_Change_Conference the observed annual average mean surface air temperature in the uae has risen by 1.27 °c (34.29 °f) between 1990 and 2022. False False False False True False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +22 unstructured 3527 3682 wiki2023_United_Nations_Climate_Change_Conference [17] without a reduction in emissions, wet-bulb temperatures in the region are expected to cross 35 °c (95 °f) for a prolonged period of time by the 2070s. False False False False True False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +23 unstructured 3683 3885 wiki2023_United_Nations_Climate_Change_Conference [18][19][20] the red sea and persian gulf are the regions with the highest number of heat-humidity extreme events in the world, and they have exceeded safe wet-bulb temperature thresholds several times. False False False False True False False False False False False False False False False 0.4939 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +24 unstructured 3886 3968 wiki2023_United_Nations_Climate_Change_Conference [21] other impacts felt in the region are dust storms, sea level rise and drought. True True False True True False False False False False False False False False False 0.0 False True True False True True False False False False False False True False False False False False False True False True False False False False False False False False +25 unstructured 3969 4116 wiki2023_United_Nations_Climate_Change_Conference according to the climate & clean air coalition, the uae is making efforts to reduce emissions in many ways across different sectors of its economy. False True False False True False False False False False False False False True False 0.4019 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +26 unstructured 4117 4284 wiki2023_United_Nations_Climate_Change_Conference measures include promoting organic and hydroponic agriculture, building the etihad rail, reducing waste (especially food waste), and promoting a more circular economy. False False False False False False False False False False False False False False False -0.2212 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +27 unstructured 4291 4413 wiki2023_United_Nations_Climate_Change_Conference the uae pledged to reduce carbon emissions to net zero by 2050; the first middle eastern government to make such a pledge. False True False False False False False False False False False False False True False 0.0 False True True False True False False False False False False False True False False False False False False True False True False False False False False False False False +28 unstructured 4414 4507 wiki2023_United_Nations_Climate_Change_Conference [23] it was the first country in the region to sign the paris agreement on 21 september 2016. False False False False False False False False False False False False False False False 0.4939 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +29 unstructured 4508 4635 wiki2023_United_Nations_Climate_Change_Conference [24][25] the country has invested $50 billion into clean energy internationally and promised an additional $50 billion by 2030. False False False False False False False False False True False False False False False 0.743 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +30 unstructured 4636 4759 wiki2023_United_Nations_Climate_Change_Conference [26][27] in november 2022, the uae agreed to partner with the united states to invest another $100 billion in clean energy. False False False False False False False False False True False False False False False 0.8271 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +31 unstructured 4760 4772 wiki2023_United_Nations_Climate_Change_Conference [28] history False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +32 unstructured 4774 5037 wiki2023_United_Nations_Climate_Change_Conference in early 2021, the united arab emirates offered to host the 2023 event,[23] and in november 2021 the prime minister and vice president of the united arab emirates, sheikh mohammed bin rashid al-maktoum, announced that the uae would be hosting the 2023 conference. False False False False False False False False False False False False False False False 0.6808 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +33 unstructured 5038 5136 wiki2023_United_Nations_Climate_Change_Conference it was the third time it was hosted by a member of opec after qatar in 2012 and indonesia in 2007. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +34 unstructured 5143 5357 wiki2023_United_Nations_Climate_Change_Conference "on 1 august 2023, the uae allowed environmental activists to ""assemble peacefully"" at the summit and vowed to provide them a space to ""make their voices heard"", despite its laws that prohibit unauthorised protests." False False False False False False False False False False False False False False False 0.3612 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +35 unstructured 5364 5533 wiki2023_United_Nations_Climate_Change_Conference roughly two months before the conference, some called for an increase in international cooperation which they saw as a necessary condition for successful climate action. False True False True True False False False False False False False True False False 0.7269 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +36 unstructured 5534 6164 wiki2023_United_Nations_Climate_Change_Conference "fatih birol, head of the international energy agency, expressed hope for significant achievements in the summit but noted: ""[the] geopolitical situation, with many nations at loggerheads over the war in ukraine, and still frosty relations between the us and china, would make for a difficult summit [.] the most important challenge [to limiting temperature rises to 1.5 °c above pre-industrial levels] is the lack of international cooperation.""[31][32] the climate envoy from bangladesh described the lack of global solidarity as the main obstacle to stopping climate change, emphasising the need to create a loss and damage fund." False True False False True False True False False True False False False False False -0.9436 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +37 unstructured 6165 6323 wiki2023_United_Nations_Climate_Change_Conference [33] governments have expressed concern that similar to the russian invasion of ukraine, the 2023 israel-hamas war may adversely impact negotiations at cop28. False False False False False False True False False False False False False False False -0.6908 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +38 unstructured 6342 6411 wiki2023_United_Nations_Climate_Change_Conference at the end of november 2023, a pre-cop meeting of ministers was held. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +39 unstructured 6412 6491 wiki2023_United_Nations_Climate_Change_Conference 100 delegations and 70 ministers attended, more than any prior pre-cop meeting. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +40 unstructured 6492 6683 wiki2023_United_Nations_Climate_Change_Conference the general director of the cop, majid al-suwaidi, insisted the conference would deliver in the domain of loss and damage what was agreed at the 2022 united nations climate change conference. False True False False True False False False False False False False False False False -0.1531 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +41 unstructured 6690 6868 wiki2023_United_Nations_Climate_Change_Conference before the conference, pope francis issued an apostolic exhortation called laudate deum, calling for brisk action against the climate crisis and condemning climate change denial. False True False False True False False False False False False False False False False -0.5423 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +42 unstructured 6869 7056 wiki2023_United_Nations_Climate_Change_Conference [39][40] the pope planned to attend the conference, which would have marked the first papal visit to a united nations climate change conference,[41] but had to pull out due to ill health. False True False False True False False False False False False False False False False -0.4215 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +43 unstructured 7057 7178 wiki2023_United_Nations_Climate_Change_Conference [42] charles iii, king of the united kingdom and the 14 other commonwealth realms, made the opening speech at the summit. False False False False False False False False False False False False False False False 0.4215 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +44 unstructured 7179 7340 wiki2023_United_Nations_Climate_Change_Conference [43] united states president joe biden did not attend, with the israel-hamas war and internal us government spending difficulties being cited as possible causes. False False False False False False True False False False False False False False False -0.5106 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +45 unstructured 7347 7492 wiki2023_United_Nations_Climate_Change_Conference china's representative declared that china, the united states and the european union agreed to cooperate to ensure the success of the conference. False False False False False False False False False False False False False False False 0.8807 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +46 unstructured 7493 7724 wiki2023_United_Nations_Climate_Change_Conference [45] at the beginning of november, insiders cautiously expressed hope for a climate agreement between china and the united states ahead of the conference, similar to the agreement of 2014 that paved the way for the paris agreement. False True False False True False False False False False False False False False False 0.936 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +47 unstructured 7725 7856 wiki2023_United_Nations_Climate_Change_Conference china published a plan to reduce methane emissions ahead of the conference, but there was expected contention on coal use in china. False True False False False False False False False False False False False True False 0.0 False True True False True False False False False False False False True False False False False False False True False True False False False False False False False False +48 unstructured 7857 8049 wiki2023_United_Nations_Climate_Change_Conference china has recently characterised coal as essential for its energy security although others say energy security could be improved through upgrades to the energy grid and domestic energy market. False False False False False False False False False True False False True False False 0.9231 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +49 unstructured 8050 8301 wiki2023_United_Nations_Climate_Change_Conference [46] talks between janet yellen and he lifeng yielded a decision to enhance cooperation between the countries on climate related issues and much was expected from the meeting between joe biden and xi jinping later in the month at the 2023 apec summit. False True False False True False False False False False False False True False False 0.0 False True True False True False False False False False False False True False False False False False False True False True False False False False False False False False +50 unstructured 8302 8464 wiki2023_United_Nations_Climate_Change_Conference "according to kate logan from the asia society policy institute, cooperation between the two countries could ""set the stage for a successful outcome at the cop28""." False False False False False False False False False False False False False False False 0.5859 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +51 unstructured 8471 8674 wiki2023_United_Nations_Climate_Change_Conference "on 15 november, the united states and china announced an agreement based on negotiations between climate envoys john kerry and xie zhenhua to ""pursue efforts to triple renewable energy globally by 2030""." False True False False True False False False False True False False False False False 0.7964 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +52 unstructured 8675 8893 wiki2023_United_Nations_Climate_Change_Conference the agreement included a commitment to addressing greenhouse gases, including warming agents other than carbon dioxide, but was criticised for not including a commitment from china to phase out coal-fired power plants. False True False False True False False False False False False False False False False -0.5067 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +53 unstructured 8894 8992 wiki2023_United_Nations_Climate_Change_Conference "the agreement was praised as a ""significant outcome"" by cop28 president-designate sultan al jaber." False False False False False False False False False False False False False False False 0.802 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +54 unstructured 8993 9046 wiki2023_United_Nations_Climate_Change_Conference [48] global stocktake see also: fossil fuel phase-out False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +55 unstructured 9048 9243 wiki2023_United_Nations_Climate_Change_Conference "in september 2023, in advance of the opening of cop28, the united nations published the first two-year assessment of global progress in slowing down climate change, called the ""global stocktake""." False True False False True False False False False False False False False False False 0.6808 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +56 unstructured 9244 9360 wiki2023_United_Nations_Climate_Change_Conference [49] this type of overview was established during cop26 in glasgow and is scheduled to be repeated every five years. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +57 unstructured 9361 9429 wiki2023_United_Nations_Climate_Change_Conference [50][51] the report says that a phase-out of fossil fuels is needed. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +58 unstructured 9430 9494 wiki2023_United_Nations_Climate_Change_Conference previously the united nations avoided making similar statements. False False False False False False False False False False False False False False False 0.1027 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +59 unstructured 9495 9548 wiki2023_United_Nations_Climate_Change_Conference [52] among the 17 key findings of the report are:[53] False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +60 unstructured 9550 9646 wiki2023_United_Nations_Climate_Change_Conference the paris agreement and the resulting climate action significantly helped in reducing emissions. False True False False True False False False False False False False False False False 0.4939 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +61 unstructured 9647 9700 wiki2023_United_Nations_Climate_Change_Conference in 2011 the projected warming by 2100 was 3.7-4.8 °c. False False False False True False False False False False False False False False False 0.1531 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +62 unstructured 9701 9797 wiki2023_United_Nations_Climate_Change_Conference after cop27 it was 2.4-2.6 °c and in the best case, if all pledges are accomplished, 1.7-2.1 °c. False False False False False False False False False False False False False False False 0.7964 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +63 unstructured 9799 9891 wiki2023_United_Nations_Climate_Change_Conference as of september 2023, the world is not on track to reach the targets of the paris agreement. False False False False False False False False False False False False False False False 0.5106 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +64 unstructured 9892 10058 wiki2023_United_Nations_Climate_Change_Conference for having a more than 50% chance of limiting temperature rise to 1.5 °c and more than 67% chance of limiting it to 2 °c, global emissions must peak by the year 2025. False True False True True False False False False False False False False False False 0.5465 True False True False False True False False False False False False True False False False False False False True False True False False False False False False False False +65 unstructured 10060 10123 wiki2023_United_Nations_Climate_Change_Conference trillions of dollars are needed for limiting warming to 1.5 °c. False False False False True False False False False False False False False False False 0.1531 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +66 unstructured 10124 10169 wiki2023_United_Nations_Climate_Change_Conference financial flows need to significantly change. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +67 unstructured 10171 10290 wiki2023_United_Nations_Climate_Change_Conference more effective international cooperation and collaboration are crucial for reaching the targets of the paris agreement. False False False False False False False False False False False False False False False 0.8122 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +68 unstructured 10292 10301 wiki2023_United_Nations_Climate_Change_Conference reception False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +69 unstructured 10303 10402 wiki2023_United_Nations_Climate_Change_Conference as the host country, the uae named sultan al jaber as president-designate of cop28 in january 2023. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +70 unstructured 10403 10527 wiki2023_United_Nations_Climate_Change_Conference [54] his appointment was criticised by western lawmakers due to him being the ceo of abu dhabi national oil company (adnoc). False False False False False False False False False False False False False False False -0.4215 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +71 unstructured 10528 10784 wiki2023_United_Nations_Climate_Change_Conference [55] ninety-nine members of the european parliament and 34 us senators and representatives wrote an open letter, addressed to us president joe biden and european commission president ursula von der leyen, to call the uae to withdraw al jaber's appointment. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +72 unstructured 10785 10858 wiki2023_United_Nations_Climate_Change_Conference [56] signatory us lawmakers included bernie sanders and elizabeth warren. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +73 unstructured 10859 10926 wiki2023_United_Nations_Climate_Change_Conference european signatories mostly hailed from the greens and left groups. False False False False False False False False False False False False False False False 0.2263 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +74 unstructured 10927 11101 wiki2023_United_Nations_Climate_Change_Conference "[57] the letter claimed that ""the private sector polluters were exercising undue influence over the climate summit's process.""[55][56] adnoc is a uae state-owned corporation." False True False False True False False False False False False False False False False -0.4588 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +75 unstructured 11102 11229 wiki2023_United_Nations_Climate_Change_Conference [58] the uae government did not respond to the open letter, neither did the us president nor the european commission president. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +76 unstructured 11230 11312 wiki2023_United_Nations_Climate_Change_Conference [59] us climate envoy john kerry expressed his support for al jaber's appointment. False True False False True False False False False False False False False False False 0.4019 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +77 unstructured 11319 11475 wiki2023_United_Nations_Climate_Change_Conference al jaber's presidency of the cop28 climate change conference contradicted his company's decision to expand fossil fuel production in adnoc drilling company. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False False False True False False False False False False True False True False False False False False False False False +78 unstructured 11476 11688 wiki2023_United_Nations_Climate_Change_Conference "the human rights organisation amnesty international raised concerns, stating, ""sultan al-jaber cannot be an honest broker for climate talks when the company he leads is planning to cause more climate damage.""[61]" False True False False True False False False False False False False False False False -0.4023 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +79 unstructured 11690 11758 wiki2023_United_Nations_Climate_Change_Conference in january 2023, dubai cares became the education partner for cop28. False False False False False False False False False False False False False False False 0.4588 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +80 unstructured 11759 11827 wiki2023_United_Nations_Climate_Change_Conference it had already participated in cop27 held in sharm el sheikh, egypt. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +81 unstructured 11834 11995 wiki2023_United_Nations_Climate_Change_Conference "in april 2023, the emirati organisers of cop28 told the speakers at the climate and health conference to not protest or ""criticise corporations"" in the emirates." False True False False True False False False False False False False False False False 0.4847 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +82 unstructured 11996 12150 wiki2023_United_Nations_Climate_Change_Conference "the warning cited the gulf state's laws, under which the panel members were warned: ""do not criticise islam, uae government, corporations or individuals""." False False False False False False False False False False False False False False False -0.2718 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +83 unstructured 12151 12246 wiki2023_United_Nations_Climate_Change_Conference climate activists raised concerns about how the uae would host cop28 without freedom of speech. False True False False True False False False False False False False False False False -0.5216 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +84 unstructured 12247 12318 wiki2023_United_Nations_Climate_Change_Conference [10] the uae invited the president of syria, bashar al-assad, to cop28. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +85 unstructured 12319 12520 wiki2023_United_Nations_Climate_Change_Conference according to human rights watch (hrw), the uae was assisting assad to improve his image and hrw stated it was outrageous that cop28 was being used to reintroduce assad into the international community. False False False False False False False False False False False False True False False -0.0258 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +86 unstructured 12527 12591 wiki2023_United_Nations_Climate_Change_Conference in june 2023, governments gathered in bonn to prepare for cop28. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +87 unstructured 12592 12702 wiki2023_United_Nations_Climate_Change_Conference hrw highlighted that the uae does not allow freedom of expression and had been aggressively silencing critics. False False False False False False False False False False False False False False False -0.8416 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +88 unstructured 12703 12860 wiki2023_United_Nations_Climate_Change_Conference "the organisation said the governments in bonn should use the opportunity to push the emirati authorities to ease the ""grip on civic space and uphold rights""." False False False False False False False False False False False False False False False 0.6486 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +89 unstructured 12861 13063 wiki2023_United_Nations_Climate_Change_Conference it said the un and other governments must demand the uae end the persecution of rights defenders like ahmed mansoor, and it expressed concern that cop28 participants may stay silent fearing retaliation. False False False False False False False False False False False False False False False -0.34 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +90 unstructured 13070 13249 wiki2023_United_Nations_Climate_Change_Conference on 7 june 2023, a report based on technical analysis by the guardian revealed that sultan al jaber's adnoc was able to read the emails to and from the cop28 climate summit office. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False False False True False False False False False False True False True False False False False False False False False +91 unstructured 13250 13354 wiki2023_United_Nations_Climate_Change_Conference adnoc was also consulted over how to reply to media inquiries by the consultancy firm, gulstan advisory. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +92 unstructured 13355 13436 wiki2023_United_Nations_Climate_Change_Conference the report also stated that the cop28 office shared its email servers with adnoc. False False False False False False False False False False False False False False False 0.34 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +93 unstructured 13437 13517 wiki2023_United_Nations_Climate_Change_Conference the cop28 office shifted to a different server after the the guardian's inquiry. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +94 unstructured 13524 13708 wiki2023_United_Nations_Climate_Change_Conference in august 2023, john kerry urged the oil and gas industry leaders to bring strategies at cop28 to cut their scope 1 (directly caused) and scope 2 (indirectly caused) emissions by 2030. False False False False False False False False False False False False False False False -0.2732 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +95 unstructured 13709 13817 wiki2023_United_Nations_Climate_Change_Conference he also asked them to initiate capital allocation commitments for renewable energy during the same timeline. False False False False False False False False False True False False False False False 0.3818 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +96 unstructured 13818 13978 wiki2023_United_Nations_Climate_Change_Conference "on the other hand, climate experts and campaigners were raising concerns that the cop28 event would become an ""oil cop"", as it was taking place in a petrostate." False True False False True False False False False False False False False False False 0.4939 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +97 unstructured 13979 14113 wiki2023_United_Nations_Climate_Change_Conference central to this criticism was the inclusion of the oil and gas industry in a crucial role in the discussions to combat climate change. False True False False True False False False False False False False False False False -0.698 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +98 unstructured 14120 14244 wiki2023_United_Nations_Climate_Change_Conference pope francis was scheduled to attend the cop28 for three days, but cancelled his trip after a flu-related lung inflammation. False False False False False False False False False False False False False False False -0.3612 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +99 unstructured 14245 14724 wiki2023_United_Nations_Climate_Change_Conference [66] summit proceedings opening ceremony of the world leaders summit at cop28, 1 december 2023 uk foreign secretary david cameron and bill gates at cop28 in dubai on 1 december 2023 indian prime minister narendra modi meeting maldivian president mohamed muizzu at the conference french president emmanuel macron at cop28 in dubai on 2 december 2023 egyptian president abdel fattah el-sisi and us vice president kamala harris, holding a bilateral meeting at cop28, 2 december 2023 False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +100 unstructured 14726 14841 wiki2023_United_Nations_Climate_Change_Conference "over 70,000 people were accredited for the summit, with 400,000 more granted access to the surrounding ""blue zone""." False False False False False False False False False False False False False False False 0.3167 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +101 unstructured 14856 15010 wiki2023_United_Nations_Climate_Change_Conference "on the starting day of the summit on 30 november 2023, a ""loss and damage"" fund to compensate poor states for the effects of climate change was agreed on." False True False False True False False False False False False False False False False -0.7579 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +102 unstructured 15011 15129 wiki2023_United_Nations_Climate_Change_Conference the fund aims to distribute funds to poor states harmed by climate change and is to be administered by the world bank. False True False False True False False False True False False False False False False -0.7351 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +103 unstructured 15130 15330 wiki2023_United_Nations_Climate_Change_Conference initial promises were made by the host (uae) to donate $100 million to the fund, and by the united kingdom ($75 million), united states ($24.5 million), japan ($10 million) and germany ($100 million). False False False False False False False False False False False False False False False 0.802 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +104 unstructured 15341 15514 wiki2023_United_Nations_Climate_Change_Conference "in his opening speech of the conference, charles iii expressed alarm at rising levels of pollution, saying that the world was ""dreadfully far off track"" its climate targets." False True False False True False False False False False False False False False False -0.7269 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +105 unstructured 15515 15712 wiki2023_United_Nations_Climate_Change_Conference "[72] the british monarch warned that ""we are carrying out a vast, frightening experiment of changing every ecological condition, all at once, at a pace that far outstrips nature's ability to cope""." False False False False False False False False False False False False False False False -0.4588 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +106 unstructured 15719 15883 wiki2023_United_Nations_Climate_Change_Conference on 1 december 2023 activists protested outside the venue, calling for ecocide-mass environmental destruction-to be made a crime at the international criminal court. False False False False False False False False False False False False False False False -0.8126 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +107 unstructured 15890 16176 wiki2023_United_Nations_Climate_Change_Conference the head of the international monetary fund, kristalina georgieva, expressed satisfaction from the beginning of the conference because the loss and damage fund was created, but said that for further decarbonisation carbon pricing should be advanced and fossil fuel subsidies eliminated. False False False False False False False False False False False False False False False 0.296 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +108 unstructured 16177 16302 wiki2023_United_Nations_Climate_Change_Conference fossil fuel subsidies reached a record level of 7.1 trillion dollars in the year 2022, due to high fuel prices and inflation. False False False False False False False False False False False False False False False 0.1027 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +109 unstructured 16303 16420 wiki2023_United_Nations_Climate_Change_Conference [75] there are propositions for a creation of a global carbon market managed by the united nations in the conference. False False False False False False False False False False False False False False False 0.5994 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +110 unstructured 16421 16449 wiki2023_United_Nations_Climate_Change_Conference some steps are already done. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +111 unstructured 16464 16578 wiki2023_United_Nations_Climate_Change_Conference on 2 december 2023, cop28 participants launched a declaration to triple nuclear energy capacity from 2020 to 2050. False False False False False False False False False True False False False False False 0.3818 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +112 unstructured 16579 16941 wiki2023_United_Nations_Climate_Change_Conference [79] the declaration was unveiled by us climate envoy john kerry, and signed by 25 countries: armenia, bulgaria, canada, croatia, czech republic, finland, france, ghana, hungary, jamaica, japan, moldova, mongolia, morocco, netherlands, poland, romania, slovakia, slovenia, south korea, sweden, ukraine, united arab emirates, united kingdom and the united states. False True False False True False False False False False False False False False False 0.8126 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +113 unstructured 16942 17115 wiki2023_United_Nations_Climate_Change_Conference the signing countries pledged to implement policies to extend their nuclear capacities, stating that nuclear power played a key role in cutting carbon emissions to net zero. False True False False False False False False False False False False False False False 0.3818 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +114 unstructured 17130 17340 wiki2023_United_Nations_Climate_Change_Conference on 3 december 2023, the guardian revealed that cop28 president sultan al-jaber had dismissed demands for a fossil fuel phase-out, denying its basis in science and claiming it would prevent economic development. False False False False False False False False False False False False False False False -0.3182 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +115 unstructured 17341 17542 wiki2023_United_Nations_Climate_Change_Conference "[83] the following day, al-jaber held a press conference in which he stated he ""respects science"", thinks a phaseout of fossil fuel use is inevitable and claimed his comments were taken out of context." False False False False False False False False False False False False False False False 0.3182 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +116 unstructured 17549 17760 wiki2023_United_Nations_Climate_Change_Conference on 6 december 2023, haitham al-ghais, the current opec secretary-general, urged member nations to focus negotiations on reducing carbon emissions, rather than prohibiting the extraction and sale of fossil fuels. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False True False False False False False False True False True False False False False False False False False +117 unstructured 17761 17907 wiki2023_United_Nations_Climate_Change_Conference [85] as a reaction, greenpeace published a press release calling for arab countries to phase out fossil fuels by 2050, ensuring a just transition. False False False False False False False False False False False False False False False 0.2732 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +118 unstructured 17908 18484 wiki2023_United_Nations_Climate_Change_Conference "greenpeace said that ""the latest research from christian aid and the international institute for applied systems analysis in vienna shows that both the united arab emirates and saudi arabia face a gdp growth reduction of -72% by 2100 if the global temperature rise is allowed to reach 3°c.""[86] the study called mercury rising: the economic impact of climate change on the arabian peninsula predicts a 69% drop in gdp growth for countries at the persian gulf in average, with the highest impact on saudi arabia, kuwait, qatar and the uae if temperatures will reach this level." False True False True True False False False False False False False True False False 0.7269 True False True False False True False False False False False False True False False False False False False True False True False False False False False False False False +119 unstructured 18485 18618 wiki2023_United_Nations_Climate_Change_Conference if temperature rise will be limited to 1.5 degrees, the reduction in economic growth will be limited to 8.2% by 2050 and 36% by 2100. False True False True True False False False False False False False True False False -0.0516 False False False True True True False False False False False False True False False False False False False True False True False False False False False False False False +120 unstructured 18619 18650 wiki2023_United_Nations_Climate_Change_Conference [87] draft and final agreements False False False False False False False False False False False False False False False 0.2732 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +121 unstructured 18652 18748 wiki2023_United_Nations_Climate_Change_Conference the cop28 draft agreement, published on 11 december, avoided calling for a fossil fuel phaseout. False False False False False False False False False False False False False False False 0.2023 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +122 unstructured 18749 18917 wiki2023_United_Nations_Climate_Change_Conference "it was subsequently rejected by the umbrella group and the alliance of small island states, which described the draft as a ""death certificate"" for small island nations." False False False False False False False False False False False False False False False -0.802 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +123 unstructured 18924 19326 wiki2023_United_Nations_Climate_Change_Conference "on the morning of 13 december, shortly after the start of the cop28's final plenary assembly, al-jaber announced that the participating countries had agreed to a compromise deal that encouraged all signatory states to end their dependence on fossil fuels ""in a just, orderly and equitable manner"", in order to prevent the worst outcome of climate change, while also working to achieve net zero by 2050." False True False False True False False False False False False False False False False -0.1027 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +124 unstructured 19327 19753 wiki2023_United_Nations_Climate_Change_Conference "[4][89] the agreement, which had been released to countries for consideration only four hours before being passed[4]-although this aspect did not lead to any formal objections[4]-also called for a tripling of global renewable energy capacity by 2030,[4][89] the development of numerous ""zero- and low-emission technologies"",[4] further efforts ""towards the phase-down of unabated coal power""[4] and a cut in methane emissions." False True False False False False False False False True False False False False False 0.4939 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +125 unstructured 19754 19879 wiki2023_United_Nations_Climate_Change_Conference [89] china and india did not sign the pledge to triple their output of renewable energy, and committed to coal power instead. False False False False False False False False False True False False False False False 0.4939 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +126 unstructured 19888 20366 wiki2023_United_Nations_Climate_Change_Conference "for the first time in the history of cop summits, the global pact explicitly mentioned the necessity to shift away from all kinds of fossil fuels;[4][89] however, the deal was widely criticised for not including a clear commitment to ""phase out"" or ""phase down"" fossil fuels-as requested by many participating countries, civil society groups and scientists[4]-as well as a clear financial plan to help developing countries reach the goal of transitioning away from fossil fuels." False False False False False False False False False False False False False False False 0.6269 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +127 unstructured 20367 20595 wiki2023_United_Nations_Climate_Change_Conference "[4][90] moreover, the choice to include carbon capture and storage in the list of ""zero- and low-emission technologies"" was questioned due to their relative expensiveness and lack of effectiveness in comparison to other methods." False False False False False False False False False False False False False False False -0.4019 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +128 unstructured 20609 20724 wiki2023_United_Nations_Climate_Change_Conference the participants of the conference pledged 85 billion dollars to different climate issues and made ten pledges:[93] False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False False False True False False False False False False True False True False False False False False False False False +129 unstructured 20726 20888 wiki2023_United_Nations_Climate_Change_Conference coalition for high-ambition multilevel partnerships (champ) for climate action pledge was signed by 71 countries including united states, brazil, france, germany. False True False False True False False False False False False False False False False 0.7672 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +130 unstructured 20889 21145 wiki2023_United_Nations_Climate_Change_Conference it includes commitments to promote international and subnational climate action, made new, more ambitious nationally determined contributions by 2025, increase efforts for climate change adaptation, preserving biodiversity for limit warming to 1.5 degrees. False True False True True False False False False False False False True False True 0.9013 True False True False False True False False True False False False True False False False False True False True False True False False False False False False True False +131 unstructured 21146 21297 wiki2023_United_Nations_Climate_Change_Conference [94] uae leaders' declaration on a global climate finance framework was signed by 13 countries including united states, india, united kingdom, germany. False True False False True False False False False False False False False False False 0.6808 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +132 unstructured 21298 21514 wiki2023_United_Nations_Climate_Change_Conference includes targets of mobilising 100 billion dollars by 2025 and 5-7 trillion dollars by 2030 to climate action, make a debt reform, put in place an emissions pricing mechanism, mobilise concession and private capital. False True False False True False False False False False False False False False False -0.3612 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +133 unstructured 21515 21645 wiki2023_United_Nations_Climate_Change_Conference [95] cop28 uae declaration on climate and health was endorsed by 143 countries including the european union, china, united states. False True False False True False False False False False False False False False False 0.8402 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +134 unstructured 21646 21835 wiki2023_United_Nations_Climate_Change_Conference includes commitments to reduce negative health impacts from climate change in collaboration with indigenous peoples, women, local communities and health workers, prevent zoonotic spillover. False True False False True False False False False False False False False True False -0.4767 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +135 unstructured 21836 21983 wiki2023_United_Nations_Climate_Change_Conference [96] cop28 declaration on climate, relief, recovery and peace was adopted by 82 countries, including canada, china, ukraine, united states, rwanda. False True False False True False False False False False False False False False False 0.8971 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +136 unstructured 21984 22206 wiki2023_United_Nations_Climate_Change_Conference the declaration includes commitments to promote peace, environmental policies that support peace, prevent climate change from increasing conflict, help and empower people affected by environmental degradation and conflict. False True False True True False False False True False True False True False False 0.7906 True False True False False True False False False False False False True False False False False False False True False True False False False False False False False False +137 unstructured 22207 22369 wiki2023_United_Nations_Climate_Change_Conference [97] global renewables and energy efficiency pledge was signed by 130 countries, including bhutan, brazil, european union, angola, bangladesh, burkina faso, chad. False False False False False False False False False True False False False False False 0.7241 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +138 unstructured 22370 22537 wiki2023_United_Nations_Climate_Change_Conference "the countries pledge to refer to energy efficiency as ""first fuel"" and double the rate of energy efficiency increase ""from around 2% to over 4% every year until 2030""." False False False True False False False False False True False False True False False 0.8591 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +139 unstructured 22538 22731 wiki2023_United_Nations_Climate_Change_Conference "they also pledged to triple the capacity of renewable energy by 2030, ensuring a just transition, increase renewable energy use and energy efficiency in an ""environmentally responsible manner""." False False False True False False False False False True False False True False False 0.91 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +140 unstructured 22732 22918 wiki2023_United_Nations_Climate_Change_Conference [98] cop28 uae declaration on sustainable agriculture, resilient food systems, and climate action was adopted by 159 countries including australia, belarus, brazil, china, united states. False True False False True False False False False False False False False False True 0.4215 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +141 unstructured 22919 23184 wiki2023_United_Nations_Climate_Change_Conference it includes commitments to make the global food system more sustainable and climate resilient, promote food security by helping vulnerable people, pass to sustainable water management, include food systems into climate action strategies, protect and restore nature. False True False False True False False False False True False False False False True 0.8625 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +142 unstructured 23185 23384 wiki2023_United_Nations_Climate_Change_Conference [99] cop28 gender-responsive just transitions and climate action partnership includes 76 countries, among others canada, china, sweden, democratic republic of the congo, israel, united arab emirates. False True False False True False False False False False False False False False False 0.4215 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +143 unstructured 23385 23532 wiki2023_United_Nations_Climate_Change_Conference its declaration includes a pledge to ensure a just transition, considering the effects on women especially indigenous, rural and with disabilities. False False False False False False False False False False False False False False False 0.3818 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +144 unstructured 23533 23655 wiki2023_United_Nations_Climate_Change_Conference [100] global cooling pledge for cop28 was endorsed by 61 countries including denmark, japan, canada, united states, spain. False False False False False False False False False False False False False False False 0.5859 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +145 unstructured 23656 23826 wiki2023_United_Nations_Climate_Change_Conference the declaration mentions the effects of increasing heat waves on health and economy (including food waste) and the contribution of the cooling industry to climate change. False True False True True False False False False False False False True False False -0.4215 False False False True True True False False False False False False True False False False False False False True False True False False False False False False False False +146 unstructured 23827 24181 wiki2023_United_Nations_Climate_Change_Conference "the signers pledged among others to cut ghg emissions from the cooling industry by 68% by 2050, increase the efficiency rate of air conditioning equipment by half by the year 2030, make a national plan for cooling, increase green spaces and blue spaces in cities and promote passive cooling (that can ""reduce a building's cooling load by more than 25%"")." False True False True False False False False False False False False True True False 0.8126 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +147 unstructured 24182 24362 wiki2023_United_Nations_Climate_Change_Conference [101] cop28 declaration of intent was endorsed by 37 countries and includes different pledges to advance hydrogen engines (including derivatives of hydrogen) as a climate solution. False True False False True False False False False False False False False False False 0.5106 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +148 unstructured 24363 24522 wiki2023_United_Nations_Climate_Change_Conference [102] cop28 joint statement on climate, nature and people was endorsed by 18 countries including china, united states, canada, france, germany, united kingdom. False True False False True False False False False False False False False False False 0.765 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +149 unstructured 24523 24712 wiki2023_United_Nations_Climate_Change_Conference the statement recognises that the problems of climate change, biodiversity loss, land degradation ocean degradation and social inequality are interconnected and cannot be solved separately. False True False False True False False False False True False False False False False -0.9121 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +150 unstructured 24713 24813 wiki2023_United_Nations_Climate_Change_Conference therefore, the signers pledge to unify the plans to solve the problems into one integrated strategy. False False False False False False False False False False False False False False False -0.2263 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +151 unstructured 24821 24849 wiki2023_United_Nations_Climate_Change_Conference public health considerations False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +152 unstructured 24851 24931 wiki2023_United_Nations_Climate_Change_Conference cop28 is the first cop to discuss the impact of climate change on public health. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False False False True False False False False False False True False True False False False False False False False False +153 unstructured 24932 25145 wiki2023_United_Nations_Climate_Change_Conference "[104] in a letter addressed to sultan al jaber, ""organisations representing more than 46 million health professionals"" said a complete phase-out of fossil fuels was the only decisive way to deliver health for all." False False False False False False False False False False False False False False False 0.2263 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +154 unstructured 25146 25418 wiki2023_United_Nations_Climate_Change_Conference [105] the world health organisation called on ministers of health to raise their voices for health as the driving force behind climate action, leading by example with climate-friendly healthcare systems, and advocating for climate finance that safeguards human well-being. False True False False True False False False False False False False True False False 0.34 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +155 unstructured 25419 25560 wiki2023_United_Nations_Climate_Change_Conference [104] it has been estimated that climate change is set to cost up to $580 billion by 2030 in climate related damages to vulnerable countries. False True False False True False False False False False False False False False False -0.5859 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +156 unstructured 25568 25765 wiki2023_United_Nations_Climate_Change_Conference previously, experts such as edmond fernandes urged unfccc to make public health an essential part of all climate meetings and policies, calling human health integral to sustainable climate futures. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False False False True False False False False False False True False True False False False False False False False False +157 unstructured 25766 25784 wiki2023_United_Nations_Climate_Change_Conference [107] food systems False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +158 unstructured 25786 25843 wiki2023_United_Nations_Climate_Change_Conference cop28 is the first cop where food systems were discussed. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +159 unstructured 25844 25982 wiki2023_United_Nations_Climate_Change_Conference during the event, the uae declaration on sustainable agriculture, resilient food systems, and climate action was adopted by 159 countries. False True False False True False False False False False False False False False True 0.0 False True True False True False False False False False False False True False False False False False False True False True False False False False False False False False +160 unstructured 25983 26067 wiki2023_United_Nations_Climate_Change_Conference [99] two-thirds of the estimated 250,000 meals served at the event were plant-based. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +161 unstructured 26068 26208 wiki2023_United_Nations_Climate_Change_Conference the un's food and agriculture organization issued a roadmap report at the event outlining how to reduce agricultural emissions from methane. False False False False False False False False False False False False False True False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +162 unstructured 26209 26440 wiki2023_United_Nations_Climate_Change_Conference "food journalist avery yale kamila said the roadmap ""would require americans to cut food waste and to move to flexitarian diets with far fewer animal products.""[108] the plant based treaty released its safe and just report at cop28." False False False False False False False True False False False False False False False -0.25 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +163 unstructured 26441 26729 wiki2023_United_Nations_Climate_Change_Conference "scientist and treaty proponent peter kalmus said ""rapidly reducing animal agriculture and shifting humanity to a plant-based diet is one of the best, easiest and fastest things we can do to save the planet.""[109] global faith leaders summit main article: global faith leaders cop28 summit" False False False False False False False True False False False False False False False 0.9413 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +164 unstructured 26731 27099 wiki2023_United_Nations_Climate_Change_Conference ahead of the cop28 summit between political leaders of the world, the muslim council of elders, in partnership with the cop28 presidency, the un environment programme and the catholic church, and under the patronage of the uae's president sheikh mohamed bin zayed al-nahyan, organised a global faith leaders summit convening 28 faith leaders to address climate change. False True False False True False False False False False False False False False False 0.6808 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +165 unstructured 27100 27144 wiki2023_United_Nations_Climate_Change_Conference [110] controversies greenwashing accusations False False False False False False False False False False False False False False False -0.4019 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +166 unstructured 27146 27231 wiki2023_United_Nations_Climate_Change_Conference prior to the conference, the uae attempted to greenwash its international reputation. False False False False False False False False False False False False False False False -0.4215 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +167 unstructured 27232 27396 wiki2023_United_Nations_Climate_Change_Conference the country hired pr and lobbying agencies, including akin gump strauss hauer & feld, gulstan advisory and fleishmanhillard, to promote itself as the host of cop28. False False False False False False False False False False False False False False False 0.3182 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +168 unstructured 27397 27534 wiki2023_United_Nations_Climate_Change_Conference [111] the appointment of the uae as host of the conference was controversial due to the country's track record on fossil fuel production. False False False False False False False False False False False False False False False -0.2023 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +169 unstructured 27535 27779 wiki2023_United_Nations_Climate_Change_Conference the president of cop28 is sultan al jaber, who as head of adnoc has overseen a substantial expansion of gas and oil production at the same time that fossil fuel industries are under pressure to reduce output in order to mitigate climate change. False True False False True False False False False False False False False True False -0.1027 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +170 unstructured 27780 27927 wiki2023_United_Nations_Climate_Change_Conference [8][112] in may 2015, cop28 president sultan al jaber was accused of attempting to greenwash his image by paying users to clean his wikipedia page. False False False False False False False False False False False False False False False -0.3182 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +171 unstructured 27928 28055 wiki2023_United_Nations_Climate_Change_Conference a wikipedia user revealed that they were paid by adnoc, while another was paid by masdar to control the narrative on wikipedia. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +172 unstructured 28056 28219 wiki2023_United_Nations_Climate_Change_Conference [113] another report revealed that fake user accounts were being used to run promotional campaigns and defend controversies around cop28 on social media platforms. False False False False False False False False False False False False False False False -0.4767 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +173 unstructured 28220 28290 wiki2023_United_Nations_Climate_Change_Conference hundreds of such pseudo-accounts were operating on twitter and medium. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +174 unstructured 28291 28414 wiki2023_United_Nations_Climate_Change_Conference the campaign aimed at promoting and greenwashing cop28, while also defending criticism against the conference and al jaber. False False False False False False False False False False False False False False False -0.2023 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +175 unstructured 28415 28577 wiki2023_United_Nations_Climate_Change_Conference [9] leading into the summit, users on social media noted a large number of fake social media accounts being used to defend uae's presidency of the climate summit. False True False False True False False False False False False False False False False -0.4215 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +176 unstructured 28578 28668 wiki2023_United_Nations_Climate_Change_Conference the social media accounts were tracked by marc owen jones of hamad bin khalifa university. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +177 unstructured 28676 28921 wiki2023_United_Nations_Climate_Change_Conference campaigning journalist amy westervelt has been critical of lead-up efforts by the un organisers to improve the transparency of cop28 by requiring industry lobbyists to identify and argued this measure is entirely inadequate to control influence. False False False False False False False False False False False False True False False -0.5984 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +178 unstructured 28929 29181 wiki2023_United_Nations_Climate_Change_Conference during the past decade, the uae has spent more than $1 million on direct climate-focused advocacy and paid millions more to advisory firms like gulstan advisory and akin gump strauss hauer & feld and think tanks helping to polish its green credentials. False True False False True False False False False False False False False False False 0.5719 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +179 unstructured 29182 29299 wiki2023_United_Nations_Climate_Change_Conference no other host nation has invested as much time and money to shape its image ahead of the annual climate negotiations. False True False False True False False False False False False False False False False -0.296 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +180 unstructured 29307 29570 wiki2023_United_Nations_Climate_Change_Conference "a report by the guardian based on leaked documents revealed that the uae prepared an all-inclusive list mentioning the ""touchy and sensitive issues"", ranging from the uae's climate issues to human rights violations, such as the war in yemen and human trafficking." False True False False True False True False False False False False False False False -0.8271 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +181 unstructured 29571 29696 wiki2023_United_Nations_Climate_Change_Conference "the document included ""strategic messages"" approved by the emirati government, which were to be used as a reply to the media." False False False False False False False False False False False False False False False 0.4215 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +182 unstructured 29697 29870 wiki2023_United_Nations_Climate_Change_Conference "the document highlighted that the country would defend adnoc's failure to disclose its emissions since 2016 by saying that ""adnoc is currently conducting necessary studies""." False False False False False False False False False False False False False False False -0.5106 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +183 unstructured 29871 30180 wiki2023_United_Nations_Climate_Change_Conference "besides, the document also mentioned that questions on sultan al jaber's presidency will be answered with, ""dr. sultan's full circle career [in energy, climate and diplomacy] gives him the expertise needed to constructively engage, disrupt and unite the very sectors needed to achieve meaningful action.""[117]" False True False False True False False False False True False False False False False 0.7003 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +184 unstructured 30182 30338 wiki2023_United_Nations_Climate_Change_Conference "the uae hired a us-based pr firm, first international resources, to ""counteract all negative press and media reports"" around the gulf state as a cop28 host." False False False False False False False False False True False False False False False -0.5719 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +185 unstructured 30339 30456 wiki2023_United_Nations_Climate_Change_Conference the agreement followed the negative criticism of the uae's decision to assign sultan al jaber as the cop28 president. False False False False False False False False False False False False False False False -0.5267 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +186 unstructured 30457 30574 wiki2023_United_Nations_Climate_Change_Conference on 4 august 2023, the company registered under the united states foreign agents registration act to represent masdar. False False False False False False False False False False False False False False False 0.4215 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +187 unstructured 30575 30807 wiki2023_United_Nations_Climate_Change_Conference "according to the filings, the pr firm was supposed to seek to ""reinforce attitudes among decision-makers in washington, d.c., and across europe regarding the strategic value of the uae in the global fight to address climate change""." False True False False True False False False False False False False False False False -0.0516 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +188 unstructured 30808 30892 wiki2023_United_Nations_Climate_Change_Conference the uae was to pay first international resources a monthly retainer fee of $100,000. False False False False False False False False False True False False False False False -0.1027 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +189 unstructured 30893 31036 wiki2023_United_Nations_Climate_Change_Conference "fossil free media founder and director jamie henn said such an amount is not paid to a pr firm ""when you're confident about your public image""." False False False False False False False False False False False False False False False 0.7579 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +190 unstructured 31037 31238 wiki2023_United_Nations_Climate_Change_Conference "he said that much is spent ""when you want to spin the public to believe the impossible"", such as the claim that the uae and al jaber had been ""really committed to transitioning away from fossil fuels""." False False False False False False False False False False False False False False False 0.4005 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +191 unstructured 31246 31366 wiki2023_United_Nations_Climate_Change_Conference in august 2023, the guardian revealed that the uae failed to report its methane emissions to the un for nearly a decade. False True False False False False False False False False False False False False False -0.5106 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +192 unstructured 31367 31490 wiki2023_United_Nations_Climate_Change_Conference meanwhile, sultan al jaber's adnoc set a much higher methane leak target than the level it claimed to have already reached. False False False False False False False False False False False False True False False -0.25 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +193 unstructured 31491 31649 wiki2023_United_Nations_Climate_Change_Conference cutting methane emissions is believed to be a fast and low-cost method to slow the temperature rise because methane causes almost a quarter of global heating. False True False True True False False False False False False False False False False -0.128 False False False True True True False False False False False False True False False False False False False True False True False False False False False False False False +194 unstructured 31650 31801 wiki2023_United_Nations_Climate_Change_Conference [119] in november of the same year, the centre for research on energy reported that the uae regularly flared methane gas, breaking its own regulations. False False False False False False False False False True False False False False False 0.2732 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +195 unstructured 31802 32006 wiki2023_United_Nations_Climate_Change_Conference [7] minister of foreign affairs abdullah bin zayed al nahyan commissioned a survey of over 20,000 from 20 countries, which was used by the officials to discuss the public attitude towards the arab nation. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +196 unstructured 32007 32054 wiki2023_United_Nations_Climate_Change_Conference the major concerns were all about human rights. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +197 unstructured 32055 32270 wiki2023_United_Nations_Climate_Change_Conference "one of the participants, sconaid mcgeachin said cops have become a platform for activism, and reportedly said: ""we need to preserve the reputation of the uae . and try to minimise those attacks as much as possible""." False False False False False False False False False False False False False False False -0.4404 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +198 unstructured 32271 32393 wiki2023_United_Nations_Climate_Change_Conference "a spokesperson for cop28, called the leaked recording ""unverified"", but stated that cop28 would ""engage all stakeholders""." False False False False False False False False False False False False False False False 0.3506 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +199 unstructured 32394 32428 wiki2023_United_Nations_Climate_Change_Conference [120] treatment of migrant workers False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +200 unstructured 32430 32548 wiki2023_United_Nations_Climate_Change_Conference the uae has been adapting its conference facility, the prior site for expo 2020, to host the cop28 climate conference. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False False False True False False False False False False True False True False False False False False False False False +201 unstructured 32549 32774 wiki2023_United_Nations_Climate_Change_Conference fairsquare, a uk-based human rights group, released a report based on testimonies and photographs, stating that the migrant workers were being exposed to dangerously hot weather and humidity to prepare for the climate summit. True True False False True False False False False False False False False False False -0.5106 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +202 unstructured 32775 32952 wiki2023_United_Nations_Climate_Change_Conference according to the rights group, in september 2023, a dozen migrant workers from asia and africa were working outdoors in temperatures hitting 42 °c (108 °f) despite a midday ban. False False False False True False False False False False False False False False False 0.4449 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +203 unstructured 32953 33048 wiki2023_United_Nations_Climate_Change_Conference cop28 denied the allegations and claimed that no evidence of the midday ban's breach was found. False False False False False False False False False False False False False False False -0.6249 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +204 unstructured 33049 33153 wiki2023_United_Nations_Climate_Change_Conference a worksite supervisor said most of the work is done at night, but some has to be done whenever possible. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +205 unstructured 33154 33408 wiki2023_United_Nations_Climate_Change_Conference a cop28 spokesperson said contractors were required to have worker heat safety plans and weather stations that monitor the thermal work limit, as well as taking into account heat and humidity metrics including wet and dry bulb temperature and wind speed. True False False False True False False False False False False False False False False 0.5994 True False True False False False False False False False False True False False False False False False False True True False False False False False False False False False +206 unstructured 33409 33435 wiki2023_United_Nations_Climate_Change_Conference [121] digital surveillance False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +207 unstructured 33437 33651 wiki2023_United_Nations_Climate_Change_Conference ahead of cop28, amnesty international raised concerns that the uae might continue to use digital surveillance to spy on human rights defenders and civil society members in the uae, including the cop28 participants. False False False False False False False False False False False False False False False 0.0772 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +208 unstructured 33652 33823 wiki2023_United_Nations_Climate_Change_Conference "according to amnesty international's disrupting surveillance team, the uae had a record of using digital surveillance to ""crush dissents and stifle freedom of expression""." False False False False False False False False False False False False False False False 0.5574 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +209 unstructured 33824 34009 wiki2023_United_Nations_Climate_Change_Conference "amnesty's view of the uae's promise to offer a ""platform for activists' voices"" would be unachievable without respect for the human rights of privacy and the right of peaceful assembly." False False False False False False False False False False False False False False False 0.449 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +210 unstructured 34010 34133 wiki2023_United_Nations_Climate_Change_Conference "amnesty opposed the ""unlawful electronic surveillance of conference participants [and of] emirati nationals and residents""." False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +211 unstructured 34134 34331 wiki2023_United_Nations_Climate_Change_Conference "it said that the cop28 attendees should be allowed to download ""privacy-respecting international communications applications"" in the uae that would ensure safe and encrypted means of communication." False False False False False False False False False False False False False False False 0.6705 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +212 unstructured 34332 34368 wiki2023_United_Nations_Climate_Change_Conference [122][123] alleged fossil fuel deals False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +213 unstructured 34370 34623 wiki2023_United_Nations_Climate_Change_Conference on 7 november 2023, an afp investigation revealed in multiple leaked documents that mckinsey was using its position as the primary advisor to cop28 hosts, the united arab emirates, to push the interest of its oil and gas clients (exxonmobil and aramco). False False False False False False False False False False False False False False False 0.5423 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +214 unstructured 34624 34754 wiki2023_United_Nations_Climate_Change_Conference mckinsey has been accused of putting its own interests ahead of the climate by sources involved in preparatory meetings for cop28. False True False False True False False False False False False False False False False -0.0516 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +215 unstructured 34755 35116 wiki2023_United_Nations_Climate_Change_Conference "mckinsey's energy scenario for the cop28 presidency would allow for continued investment in fossil fuels, which would undermine the goals of the paris agreement; an ""energy transition narrative"" recommends oil use to be reduced by only 50% by 2050, and that trillions of dollars should continue to be invested in high-emission assets each year to at least 2050." False False False False False False False False False True False False False True False 0.8271 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +216 unstructured 35124 35447 wiki2023_United_Nations_Climate_Change_Conference "on 27 november 2023, the centre for climate reporting and bbc news reported that based on leaked documents, the uae intended to use cop28 as a platform to discuss fossil fuel deals with fifteen countries, including a deal with china to ""jointly evaluate international lng opportunities"" in mozambique, canada and australia." False True False False True False False False False False False False False False False 0.0772 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +217 unstructured 35448 35664 wiki2023_United_Nations_Climate_Change_Conference the report stated that fossil fuel talks between adnoc and 15 countries were planned, in addition to talks between masdar and 20 countries including the united kingdom, france, germany and others ahead of the summit. False False False False False False False False False False False False False False False 0.4215 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +218 unstructured 35665 35815 wiki2023_United_Nations_Climate_Change_Conference [125][126][127] al jaber denied reports that the uae wanted to seek oil deals at the summit, calling the allegations an attempt to undermine his work. False False False False False False False False False False False False False True False -0.6249 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +219 unstructured 35833 35994 wiki2023_United_Nations_Climate_Change_Conference an analysis found that at least 2,456 cop28 attendees were fossil fuel lobbyists, receiving more passes than the ten countries most vulnerable to climate change. False True False False True False False False False False False False False False False -0.2944 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +220 unstructured 35995 36183 wiki2023_United_Nations_Climate_Change_Conference "[131] analysis by oxfam found that 34 billionaires-together worth roughly $495 billion-attended cop28 as delegates; a quarter of them had made their money in ""highly polluting industries""." False False False False False False False False False False False False False False False 0.2263 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +221 unstructured 36191 36442 wiki2023_United_Nations_Climate_Change_Conference sultan al jaber, the president of cop28, has drawn criticism for asserting there is no scientific basis for phasing out fossil fuels to limit global warming to 1.5 degrees celsius, a claim revealed by the guardian and the centre for climate reporting. False True False False True False False False False False False False False False False -0.5423 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +222 unstructured 36443 36589 wiki2023_United_Nations_Climate_Change_Conference "described as ""incredibly concerning"" and bordering on climate denial, his comments contradict the stance of un secretary-general antónio guterres." False True False False True False False False False False False False False False False -0.3182 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +223 unstructured 36590 36731 wiki2023_United_Nations_Climate_Change_Conference the remarks were made during a confrontational exchange with mary robinson, chair of the elders group, at a live online event on 21 november. False False False False False False False False True False False False False False False -0.3818 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +224 unstructured 36732 36926 wiki2023_United_Nations_Climate_Change_Conference the times noted widespread condemnation of al jaber's statements at cop28, where numerous countries, including the uk and eu members, advocated for the phased reduction of unabated fossil fuels. False False False False False False False False False False False False False False False -0.5859 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +225 unstructured 36927 37133 wiki2023_United_Nations_Climate_Change_Conference in contrast, former us vice president al gore emphasised the historical significance of a potential cop28 agreement to phase out fossil fuels, considering it one of the most crucial events in human history. False False False False False False False False False False False False False False False 0.6486 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +226 unstructured 37134 37304 wiki2023_United_Nations_Climate_Change_Conference simultaneously, bbc news reported a substantial increase in the united arab emirates' oil production, with adnoc, the state oil firm, projected to drill 42% more by 2030. False False False True False False False False False False False False True False False 0.7096 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +227 unstructured 37305 37463 wiki2023_United_Nations_Climate_Change_Conference despite this, adnoc claims it aims to enhance climate-friendliness, including venturing into renewable energy, amid the global call for fossil fuel reduction. False True False False True False False False False True False False True False False 0.2732 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +228 unstructured 37464 37486 wiki2023_United_Nations_Climate_Change_Conference [133] carbon footprint False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +229 unstructured 37488 37651 wiki2023_United_Nations_Climate_Change_Conference several commentators criticised the fact that cop28 had over 70,000 accredited attendees,[67][68][69] as well as the use of private jets by many figures to attend. False False False False False False False False False False False False False False False -0.1779 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +230 unstructured 37652 37752 wiki2023_United_Nations_Climate_Change_Conference [134][135][136] the meeting is estimated to have the largest carbon footprint of any climate summit. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False False False True False False False False False False True False True False False False False False False False False +231 unstructured 37753 37782 wiki2023_United_Nations_Climate_Change_Conference [137] suppression of protests False False False False False False False False False False False False False False False -0.2263 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +232 unstructured 37784 37995 wiki2023_United_Nations_Climate_Change_Conference "on 11 december 2023, licypriya kangujam, a climate activist from india, walked onto the main stage of the conference, held up a sign, ""end fossil fuels. save our planet and our future."", and gave a brief speech." False True False False True False False False False False False False False False False 0.4939 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +233 unstructured 37996 38097 wiki2023_United_Nations_Climate_Change_Conference she was given a round of applause by the audience and removed from the session by security personnel. False False False False False False False False False False False False False False False 0.6369 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +234 unstructured 38098 38172 wiki2023_United_Nations_Climate_Change_Conference according to kangujam, she was banned from further participation in cop28. False False False False False False False False False False False False False False False -0.4588 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +235 unstructured 38173 38291 wiki2023_United_Nations_Climate_Change_Conference "[138] other activists criticised the uae's heavy restrictions on protests, which human rights watch called ""shocking""." False False False False False False False False False False False False False False False -0.7506 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +236 unstructured 38292 38423 wiki2023_United_Nations_Climate_Change_Conference "[139][140] protests at the summit were largely confined to the ""blue zone"", an area under un control where local laws do not apply." False False False False False False False False False False False False False False False -0.2263 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +237 unstructured 38424 38574 wiki2023_United_Nations_Climate_Change_Conference [141] in the blue zone, local activists used cop28 as a platform for human rights demonstrations that are rare and generally not permitted in the uae. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +238 unstructured 38575 38594 wiki2023_United_Nations_Climate_Change_Conference [142][143] see also False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +239 unstructured 38596 38788 wiki2023_United_Nations_Climate_Change_Conference 2023 in climate change 49th g7 summit carbon capture and storage climate finance environmental issues in the united arab emirates global assembly global stocktake greenwashing hydrogen economy False True False False True False False False False False False False False False False 0.34 True False True False False False False False False False False False True False False False False False False True False True False False False False False False False False +1 unstructured 0 12 wikiAnthropocene anthropocene False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +2 unstructured 14 26 wikiAnthropocene article talk False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +3 unstructured 28 50 wikiAnthropocene read edit view history False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +4 unstructured 52 57 wikiAnthropocene tools False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +5 unstructured 59 157 wikiAnthropocene from wikipedia, the free encyclopedia for the documentary film, see anthropocene: the human epoch. False False False False False False False False False False False False False False False 0.5106 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +6 unstructured 158 396 wikiAnthropocene light pollution comes from highly populated areas at night, caused by human activity (electric lights) part of a series on human history human era ↑ prehistory (stone age) (pleistocene epoch) holocene ancient postclassical modern ↓ future False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +7 unstructured 398 401 wikiAnthropocene vte False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +8 unstructured 403 566 wikiAnthropocene the anthropocene (/ˈænθrəpəˌsiːn, ænˈθrɒpə-/ [1][2][3]) is a proposed geological epoch dating from the commencement of significant human impact on earth until now. False False False False False False False False False False False False False False False 0.2023 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +9 unstructured 567 640 wikiAnthropocene it affects earth's geology, landscape, limnology, ecosystems and climate. False True False False True False False True False True False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +10 unstructured 641 753 wikiAnthropocene [4][5] the effects of human activities on earth can be seen for example in biodiversity loss and climate change. False True False False True False False False False False False False False False False -0.3182 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +11 unstructured 754 944 wikiAnthropocene various start dates for the anthropocene have been proposed, ranging from the beginning of the neolithic revolution (12,000-15,000 years ago), to as recently as the 1960s as a starting date. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +12 unstructured 946 1063 wikiAnthropocene in may 2019, the anthropocene working group (awg) voted in favour of submitting a formal proposal to the ics by 2021. False False False False False False False False False False False False False False False 0.4404 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +13 unstructured 1064 1145 wikiAnthropocene [6] the proposal located potential stratigraphic markers to the mid-20th century. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +14 unstructured 1146 1382 wikiAnthropocene [7][6][8] this time period coincides with the start of the great acceleration, a post-world war ii time period during which global population growth, pollution and exploitation of natural resources have all increased at a dramatic rate. False False False True False False True False False True False False True False False 0.7506 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +15 unstructured 1383 1526 wikiAnthropocene [9] the atomic age also started around the mid-20th century, when the risks of nuclear wars, nuclear terrorism and nuclear accidents increased. False False False True False False True False False False False False True False False -0.8885 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +16 unstructured 1528 1761 wikiAnthropocene the anthropocene working group of the subcommission on quaternary stratigraphy (sqs) of the ics voted in april 2016 to proceed towards a formal golden spike (gssp) proposal to define the anthropocene epoch in the geologic time scale. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +17 unstructured 1762 1851 wikiAnthropocene the group presented the proposal to the international geological congress in august 2016. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +18 unstructured 1858 2059 wikiAnthropocene as of march 2024, neither the international commission on stratigraphy (ics) nor the international union of geological sciences (iugs) has approved the term as an official subdivision of geologic time. False False False False False False False False False False False False False False False 0.4215 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +19 unstructured 2060 2279 wikiAnthropocene [11][12] in march 2024, after 15 years of deliberation, a proposal by the iugs to ratify the anthropocene was voted down by a wide margin, primarily due to a dispute over the proposed start time in the mid-20th century. False False False False False False False False False False False False False False False -0.4019 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +20 unstructured 2280 2344 wikiAnthropocene however, an option remains to appeal or challenge these results. False False False False False False False False False False False False False False False 0.0772 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +21 unstructured 2351 2495 wikiAnthropocene although the biologist eugene f. stoermer is often credited with coining the term anthropocene, it was already in informal use in the mid-1970s. False False False False False False False False False False False False False False False 0.3612 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +22 unstructured 2496 2554 wikiAnthropocene [14] paul j. crutzen re-invented and popularised the term. False False False False False False False False False False False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +23 unstructured 2555 2586 wikiAnthropocene [15] development of the concept False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +24 unstructured 2588 2731 wikiAnthropocene "an early concept for the anthropocene was the noosphere by vladimir vernadsky, who in 1938 wrote of ""scientific thought as a geological force""." False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +25 unstructured 2732 2894 wikiAnthropocene [16] scientists in the soviet union appear to have used the term anthropocene as early as the 1960s to refer to the quaternary, the most recent geological period. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +26 unstructured 2901 3241 wikiAnthropocene ecologist eugene f. stoermer subsequently used anthropocene with a different sense in the 1980s[18][19] and the term was widely popularised in 2000 by atmospheric chemist paul j. crutzen,[15][20] who regards the influence of human behavior on earth's atmosphere in recent centuries as so significant as to constitute a new geological epoch. False False False False False False False False False False False False False False False 0.5371 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +27 unstructured 3257 3421 wikiAnthropocene in 2008, the stratigraphy commission of the geological society of london considered a proposal to make the anthropocene a formal unit of geological epoch divisions. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +28 unstructured 3422 3521 wikiAnthropocene [5][23] a majority of the commission decided the proposal had merit and should be examined further. False False False False False False False False False False False False False False False 0.4215 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +29 unstructured 3522 3703 wikiAnthropocene independent working groups of scientists from various geological societies have begun to determine whether the anthropocene will be formally accepted into the geological time scale. False False False False False False False False False False False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +30 unstructured 3710 3903 wikiAnthropocene the pressures we exert on the planet have become so great that scientists are considering whether the earth has entered an entirely new geological epoch: the anthropocene, or the age of humans. False False False False False False False False False False False False False False False 0.6048 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +31 unstructured 3904 4041 wikiAnthropocene it means that we are the first people to live in an age defined by human choice, in which the dominant risk to our survival is ourselves. False False False False False False False False False False False False False False False -0.2732 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +32 unstructured 4043 4081 wikiAnthropocene -achim steiner, undp administrator[25] False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +33 unstructured 4083 4147 wikiAnthropocene the term anthropocene is informally used in scientific contexts. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +34 unstructured 4148 4280 wikiAnthropocene [26] the geological society of america entitled its 2011 annual meeting: archean to anthropocene: the past is the key to the future. False False False False False False False False False False False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +35 unstructured 4281 4428 wikiAnthropocene [27] the new epoch has no agreed start-date, but one proposal, based on atmospheric evidence, is to fix the start with the industrial revolution c. False False False False False False False False False False False False False False False -0.1045 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +36 unstructured 4429 4474 wikiAnthropocene 1780, with the invention of the steam engine. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +37 unstructured 4475 4624 wikiAnthropocene [23][28] other scientists link the new term to earlier events, such as the rise of agriculture and the neolithic revolution (around 12,000 years bp). False False False True False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +38 unstructured 4626 4878 wikiAnthropocene evidence of relative human impact - such as the growing human influence on land use, ecosystems, biodiversity, and species extinction - is substantial; scientists think that human impact has significantly changed (or halted) the growth of biodiversity. False True False False False False False True False True False False True False False 0.6249 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +39 unstructured 4879 5262 wikiAnthropocene "[29][30][31][32] those arguing for earlier dates posit that the proposed anthropocene may have begun as early as 14,000-15,000 years bp, based on geologic evidence; this has led other scientists to suggest that ""the onset of the anthropocene should be extended back many thousand years"";[33]: 1 this would make the anthropocene essentially synonymous with the current term, holocene." False False False False False False False False False False False False False False False -0.4588 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +40 unstructured 5263 5344 wikiAnthropocene the trinity test in july 1945 has been proposed as the start of the anthropocene. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +41 unstructured 5346 5542 wikiAnthropocene in january 2015, 26 of the 38 members of the international anthropocene working group published a paper suggesting the trinity test on 16 july 1945 as the starting point of the proposed new epoch. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +42 unstructured 5543 5622 wikiAnthropocene [34] however, a significant minority supports one of several alternative dates. False False False False False False False False False False False False False False False 0.5106 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +43 unstructured 5623 5715 wikiAnthropocene [34] a march 2015 report suggested either 1610 or 1964 as the beginning of the anthropocene. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +44 unstructured 5716 5922 wikiAnthropocene [35] other scholars point to the diachronous character of the physical strata of the anthropocene, arguing that onset and impact are spread out over time, not reducible to a single instant or date of start. False False False False False False False False False False False False False False False -0.4588 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +45 unstructured 5929 6167 wikiAnthropocene a january 2016 report on the climatic, biological, and geochemical signatures of human activity in sediments and ice cores suggested the era since the mid-20th century should be recognised as a geological epoch distinct from the holocene. False False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +46 unstructured 6174 6325 wikiAnthropocene the anthropocene working group met in oslo in april 2016 to consolidate evidence supporting the argument for the anthropocene as a true geologic epoch. False False False False False False False False False False False False False False False 0.4939 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +47 unstructured 6326 6443 wikiAnthropocene [38] evidence was evaluated and the group voted to recommend anthropocene as the new geological epoch in august 2016. False False False False False False False False False False False False False False False 0.3612 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +48 unstructured 6444 6659 wikiAnthropocene [10] should the international commission on stratigraphy approve the recommendation, the proposal to adopt the term will have to be ratified by the iugs before its formal adoption as part of the geologic time scale. False False False False False False False False False False False False False False False 0.3182 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +49 unstructured 6666 6871 wikiAnthropocene in april 2019, the anthropocene working group (awg) announced that they would vote on a formal proposal to the international commission on stratigraphy, to continue the process started at the 2016 meeting. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +50 unstructured 6872 6986 wikiAnthropocene [8] in may 2019, 29 members of the 34 person awg panel voted in favour of an official proposal to be made by 2021. False False False False False False False False False False False False False False False 0.4404 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +51 unstructured 6987 7073 wikiAnthropocene the awg also voted with 29 votes in favour of a starting date in the mid 20th century. False False False False False False False False False False False False False False False 0.4404 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +52 unstructured 7074 7232 wikiAnthropocene ten candidate sites for a global boundary stratotype section and point have been identified, one of which will be chosen to be included in the final proposal. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +53 unstructured 7233 7357 wikiAnthropocene [6][7] possible markers include microplastics, heavy metals, or radioactive nuclei left by tests from thermonuclear weapons. False False False False False False False False False False False False False False False -0.4404 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +54 unstructured 7364 7514 wikiAnthropocene in november 2021, an alternative proposal that the anthropocene is a geological event, not an epoch, was published[41][42] and later expanded in 2022. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +55 unstructured 7515 7746 wikiAnthropocene [43] this challenged the assumption underlying the case for the anthropocene epoch - the idea that it is possible to accurately assign a precise date of start to highly diachronous processes of human-influenced earth system change. False False False False False False False False False False False False False False False -0.1027 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +56 unstructured 7747 7952 wikiAnthropocene the argument indicated that finding a single gssp would be impractical, given human-induced changes in the earth system occurred at different periods, in different places, and spread under different rates. False False False False False False False False False False False False False False False -0.3612 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +57 unstructured 7953 8279 wikiAnthropocene under this model, the anthropocene would have many events marking human-induced impacts on the planet, including the mass extinction of large vertebrates, the development of early farming, land clearance in the americas, global-scale industrial transformation during the industrial revolution, and the start of the atomic age. False False False False False False False False False True False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +58 unstructured 8280 8497 wikiAnthropocene the authors are members of the awg who had voted against the official proposal of a starting date in the mid-20th century, and sought to reconcile some of the previous models (including ruddiman and maslin proposals). False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +59 unstructured 8498 8737 wikiAnthropocene they cited crutzen's original concept,[44] arguing that the anthropocene is much better and more usefully conceived of as an unfolding geological event, like other major transformations in earth's history such as the great oxidation event. False False False False False False False False False False False False False False False 0.8977 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +60 unstructured 8739 8863 wikiAnthropocene in july 2023, the awg chose crawford lake in ontario, canada as a site representing the beginning of the proposed new epoch. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +61 unstructured 8864 9206 wikiAnthropocene the sediment in that lake shows a spike in levels of plutonium from hydrogen bomb tests, a key marker the group chose to place the start of the anthropocene in the 1950s, along with other elevated markers including carbon particles and nitrates from the burning of fossil fuels and widespread application of chemical fertilizers respectively. False False False False False False False False False False False False False False False -0.2023 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +62 unstructured 9207 9369 wikiAnthropocene if approved, the official declaration of the new anthropocene epoch will take place in august 2024,[45] and its first age may be named crawfordian after the lake. False False False False False False False False False False False False False False False 0.4215 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +63 unstructured 9376 9616 wikiAnthropocene in march 2024, the new york times reported on the results of an internal vote held by the iugs, finding that after nearly 15 years of debate, the proposal to ratify the anthropocene had been defeated by a 12-to-4 margin, with 2 abstentions. False False False False False False False False False False False False False False False -0.4767 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +64 unstructured 9617 9966 wikiAnthropocene these results were not out of a dismissal of human impact on the planet, but rather an inability to constrain the anthropocene in a geological context; the widely-adopted 1950 start date was found to prone to recency bias and to overshadow earlier examples of human impacts, many of which happened in different parts of the world at different times. False False False False False False False False False False False False False False False -0.631 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +65 unstructured 9967 10102 wikiAnthropocene it remains uncertain whether the vote's results functioned as a conclusive rejection or if the results could be challenged or appealed. False False False False False False False False False False False False False False False -0.7269 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +66 unstructured 10103 10224 wikiAnthropocene although the proposal could be raised again, this would require the entire process of debate to start from the beginning. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +67 unstructured 10225 10311 wikiAnthropocene [13] proposed starting point industrial revolution main article: industrial revolution False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +68 unstructured 10313 10385 wikiAnthropocene crutzen proposed the industrial revolution as the start of anthropocene. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +69 unstructured 10386 10507 wikiAnthropocene [47] lovelock proposes that the anthropocene began with the first application of the newcomen atmospheric engine in 1712. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +70 unstructured 10508 10692 wikiAnthropocene [48] the intergovernmental panel on climate change takes the pre-industrial era (chosen as the year 1750) as the baseline related to changes in long-lived, well mixed greenhouse gases. False True False False True False False False False False False True False False False 0.2732 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +71 unstructured 10693 10903 wikiAnthropocene [49] although it is apparent that the industrial revolution ushered in an unprecedented global human impact on the planet,[50] much of earth's landscape already had been profoundly modified by human activities. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +72 unstructured 10904 10991 wikiAnthropocene [51] the human impact on earth has grown progressively, with few substantial slowdowns. False False False False False False False False False False False False True False False 0.2023 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +73 unstructured 10992 11345 wikiAnthropocene "a 2024 scientific perspective paper authored by a group of scientists led by william j. ripple proposed the start of the anthropocene around 1850, stating it is a ""compelling choice . . . from a population, fossil fuel, greenhouse gasses, temperature, and land use perspective.""[52] mid 20th century (great acceleration) main article: great acceleration" False True False False True False False False False True False False False False False 0.8779 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +74 unstructured 11347 11831 wikiAnthropocene "in may 2019 the twenty-nine members of the anthropocene working group (awg) proposed a start date for the epoch in the mid-twentieth century, as that period saw ""a rapidly rising human population accelerated the pace of industrial production, the use of agricultural chemicals and other human activities. at the same time, the first atomic-bomb blasts littered the globe with radioactive debris that became embedded in sediments and glacial ice, becoming part of the geologic record.""" False False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +75 unstructured 11832 12039 wikiAnthropocene the official start-dates, according to the panel, would coincide with either the radionuclides released into the atmosphere from bomb detonations in 1945, or with the limited nuclear test ban treaty of 1963. False False False False False False False False False False False False False False False -0.8271 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +76 unstructured 12040 12069 wikiAnthropocene [53] first atomic bomb (1945) False False False False False False False False False False False False False False False -0.4939 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +77 unstructured 12071 12319 wikiAnthropocene the peak in radionuclides fallout consequential to atomic bomb testing during the 1950s is another possible date for the beginning of the anthropocene (the detonation of the first atomic bomb in 1945 or the partial nuclear test ban treaty in 1963). False False False False False False False False False False False False False False False -0.875 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +78 unstructured 12320 12333 wikiAnthropocene [6] etymology False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +79 unstructured 12335 12504 wikiAnthropocene the name anthropocene is a combination of anthropo- from the ancient greek ἄνθρωπος (ánthropos) meaning 'human' and -cene from καινός (kainós) meaning 'new' or 'recent'. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +80 unstructured 12515 12690 wikiAnthropocene as early as 1873, the italian geologist antonio stoppani acknowledged the increasing power and effect of humanity on the earth's systems and referred to an 'anthropozoic era'. False False False True False False False False False False False False True False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +81 unstructured 12691 12840 wikiAnthropocene [47] nature of human effects main article: human impact on the environment biodiversity loss main articles: holocene extinction and biodiversity loss False True False False False False False False False False False False False False False -0.5574 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +82 unstructured 12842 12931 wikiAnthropocene the human impact on biodiversity forms one of the primary attributes of the anthropocene. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +83 unstructured 12932 13019 wikiAnthropocene [56] humankind has entered what is sometimes called the earth's sixth major extinction. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +84 unstructured 13020 13130 wikiAnthropocene [57][58][59][60][61] most experts agree that human activities have accelerated the rate of species extinction. False True False False False False False True False False False False False False False 0.4173 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +85 unstructured 13131 13246 wikiAnthropocene [31][62] the exact rate remains controversial - perhaps 100 to 1000 times the normal background rate of extinction. False False False False False False False False False False False False False False False -0.2023 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +86 unstructured 13257 13346 wikiAnthropocene anthropogenic extinctions started as humans migrated out of africa over 60,000 years ago. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +87 unstructured 13347 13524 wikiAnthropocene [65] increases in global rates of extinction have been elevated above background rates since at least 1500, and appear to have accelerated in the 19th century and further since. False False False True False False False False False False False False True False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +88 unstructured 13525 13648 wikiAnthropocene [4] rapid economic growth is considered a primary driver of the contemporary displacement and eradication of other species. False False False False False False False True False False False False True False False 0.3818 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +89 unstructured 13655 13990 wikiAnthropocene "according to the 2021 economics of biodiversity review, written by partha dasgupta and published by the uk government, ""biodiversity is declining faster than at any time in human history.""[67][68] a 2022 scientific review published in biological reviews confirms that an anthropogenic sixth mass extinction event is currently underway." False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +90 unstructured 13991 14500 wikiAnthropocene "[69][70] a 2022 study published in frontiers in ecology and the environment, which surveyed more than 3,000 experts, states that the extinction crisis could be worse than previously thought, and estimates that roughly 30% of species ""have been globally threatened or driven extinct since the year 1500.""[71][72] according to a 2023 study published in biological reviews some 48% of 70,000 monitored species are experiencing population declines from human activity, whereas only 3% have increasing populations." False False False True False False False True True False False False True False False -0.8807 False False False True True False True False False False True False False False False False False False False False False False False False False False False False False False +91 unstructured 14501 14564 wikiAnthropocene [73][74][75] this section is an excerpt from biodiversity loss. False True False False False False False False False False False False False False False -0.3182 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +92 unstructured 14564 14570 wikiAnthropocene [edit] False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +93 unstructured 14572 14650 wikiAnthropocene summary of major environmental-change categories that cause biodiversity loss. False True False False False False False False False False False False False False False -0.3182 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +94 unstructured 14651 14749 wikiAnthropocene the data is expressed as a percentage of human-driven change (in red) relative to baseline (blue). False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +95 unstructured 14750 14930 wikiAnthropocene red indicates the percentage of the category that is damaged, lost, or otherwise affected, whereas blue indicates the percentage that is intact, remaining, or otherwise unaffected. False False False False False False False False False False False False False False False -0.6124 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +96 unstructured 14937 15112 wikiAnthropocene biodiversity loss happens when plant or animal species disappear completely from earth (extinction) or when there is a decrease or disappearance of species in a specific area. False True False False False False False True False False False False False True False -0.4939 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +97 unstructured 15113 15203 wikiAnthropocene biodiversity loss means that there is a reduction in biological diversity in a given area. False True False False False False False False False False False False False False False -0.3182 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +98 unstructured 15204 15247 wikiAnthropocene the decrease can be temporary or permanent. False False False False False False False False False False False False False True False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +99 unstructured 15248 15365 wikiAnthropocene it is temporary if the damage that led to the loss is reversible in time, for example through ecological restoration. False False False False False False False False False False False False False False False -0.6705 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +100 unstructured 15366 15422 wikiAnthropocene if this is not possible, then the decrease is permanent. False False False False False False False False False False False False False True False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +101 unstructured 15423 15550 wikiAnthropocene the cause of most of the biodiversity loss is, generally speaking, human activities that push the planetary boundaries too far. False True False False False False False False False False False False False False False -0.3182 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +102 unstructured 15551 15676 wikiAnthropocene [76][77][78] these activities include habitat destruction[79] and land use intensification (for example monoculture farming). False True False True False False False True False True False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +103 unstructured 15677 15827 wikiAnthropocene [80][81] further problem areas are air and water pollution (including nutrient pollution), over-exploitation, invasive species[82] and climate change. False True False False True False False True False True False False False False False -0.4019 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +104 unstructured 15834 16077 wikiAnthropocene many scientists, along with the global assessment report on biodiversity and ecosystem services, say that the main reason for biodiversity loss is a growing human population because this leads to human overpopulation and excessive consumption. False True False False False False False True False True False False True False False -0.1531 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +105 unstructured 16078 16270 wikiAnthropocene "[83][84][85][86][87] others disagree, saying that loss of habitat is caused mainly by ""the growth of commodities for export"" and that population has very little to do with overall consumption." False False False False False False False True False False False False True False False -0.3182 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +106 unstructured 16271 16337 wikiAnthropocene more important are wealth disparities between or within countries. False False False False False False False False False False False False False False False 0.6764 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +107 unstructured 16338 16399 wikiAnthropocene [88] climate change is another threat to global biodiversity. False True False False True False False False True False False False False False False -0.5267 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +108 unstructured 16400 16544 wikiAnthropocene [89][90] for example, coral reefs-which are biodiversity hotspots-will be lost by the year 2100 if global warming continues at the current rate. False True False False True False False False False False False False False False False -0.1779 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +109 unstructured 16545 16714 wikiAnthropocene [91][92] still, it is the general habitat destruction (often for expansion of agriculture), not climate change, that is currently the bigger driver of biodiversity loss. False True False False True False False True False False False False False False False -0.7184 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +110 unstructured 16715 16827 wikiAnthropocene [93][94] invasive species and other disturbances have become more common in forests in the last several decades. False False False False False False False True False True False False False False False -0.34 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +111 unstructured 16828 16947 wikiAnthropocene these tend to be directly or indirectly connected to climate change and can cause a deterioration of forest ecosystems. False True False False True False False True False True False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +112 unstructured 16948 17013 wikiAnthropocene [95][96] biogeography and nocturnality main article: biogeography False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +113 unstructured 17015 17141 wikiAnthropocene studies of urban evolution give an indication of how species may respond to stressors such as temperature change and toxicity. False False False False True False False True False False False False False False False -0.4767 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +114 unstructured 17142 17268 wikiAnthropocene species display varying abilities to respond to altered environments through both phenotypic plasticity and genetic evolution. False False False False False False False True False False False False False False False 0.25 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +115 unstructured 17269 17426 wikiAnthropocene [97][98][99] researchers have documented the movement of many species into regions formerly too cold for them, often at rates faster than initially expected. False False False False False False False True False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +116 unstructured 17434 17554 wikiAnthropocene permanent changes in the distribution of organisms from human influence will become identifiable in the geologic record. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +117 unstructured 17555 17751 wikiAnthropocene this has occurred in part as a result of changing climate, but also in response to farming and fishing, and to the accidental introduction of non-native species to new areas through global travel. False True False False True False False True False False False False False False False -0.1154 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +118 unstructured 17752 17932 wikiAnthropocene [4] the ecosystem of the entire black sea may have changed during the last 2000 years as a result of nutrient and silica input from eroding deforested lands along the danube river. False True False False False False False True False True False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +119 unstructured 17945 18231 wikiAnthropocene researchers have found that the growth of the human population and expansion of human activity has resulted in many species of animals that are normally active during the day, such as elephants, tigers and boars, becoming nocturnal to avoid contact with humans, who are largely diurnal. False False False False False False False True False False False False True False False 0.4767 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +120 unstructured 18232 18356 wikiAnthropocene [103][102] climate change main articles: climate change, effects of climate change, and carbon dioxide in earth's atmosphere False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +121 unstructured 18358 18466 wikiAnthropocene one geological symptom resulting from human activity is increasing atmospheric carbon dioxide (co2) content. False True False True False False False False False False False False True False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +122 unstructured 18467 18618 wikiAnthropocene this signal in the earth's climate system is especially significant because it is occurring much faster,[104] and to a greater extent, than previously. False True False False True False False False False False False False False False False 0.5563 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +123 unstructured 18619 18709 wikiAnthropocene most of this increase is due to the combustion of fossil fuels such as coal, oil, and gas. False False False True False False False False False False False False True False False 0.4506 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +124 unstructured 18710 18779 wikiAnthropocene this section is an excerpt from carbon dioxide in earth's atmosphere. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +125 unstructured 18779 18910 wikiAnthropocene [edit] atmospheric co2 concentration measured at mauna loa observatory in hawaii from 1958 to 2023 (also called the keeling curve). False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +126 unstructured 18911 18968 wikiAnthropocene the rise in co2 over that time period is clearly visible. False True False True False False False False False False False False False False False 0.4019 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +127 unstructured 18969 19026 wikiAnthropocene the concentration is expressed as μmole per mole, or ppm. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +128 unstructured 19027 19188 wikiAnthropocene in earth's atmosphere, carbon dioxide is a trace gas that plays an integral part in the greenhouse effect, carbon cycle, photosynthesis and oceanic carbon cycle. False True False False False False False False False False False False False False False 0.25 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +129 unstructured 19189 19254 wikiAnthropocene it is one of several greenhouse gases in the atmosphere of earth. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +130 unstructured 19255 19372 wikiAnthropocene the current global average concentration of carbon dioxide (co2) in the atmosphere is 421 ppm as of may 2022 (0.04%). False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +131 unstructured 19373 19522 wikiAnthropocene [105] this is an increase of 50% since the start of the industrial revolution, up from 280 ppm during the 10,000 years prior to the mid-18th century. False False False True False False False False False False False False True False False 0.3182 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +132 unstructured 19523 19577 wikiAnthropocene [106][105][107] the increase is due to human activity. False False False True False False False False False False False False True False False 0.3182 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +133 unstructured 19578 19703 wikiAnthropocene [108] burning fossil fuels is the main cause of these increased co2 concentrations and also the main cause of climate change. False True False True True False False False False False False False True False False 0.2732 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +134 unstructured 19704 19821 wikiAnthropocene [109] other large sources of co2 from human activities include cement production, deforestation, and biomass burning. False True False False False True False True False False False False False False False 0.0 False True True False True False False False False False True False True False False True False False False True False True True False False False True False False False +135 unstructured 19822 19880 wikiAnthropocene this section is an excerpt from effects of climate change. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +136 unstructured 19880 19886 wikiAnthropocene [edit] False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +137 unstructured 19888 19968 wikiAnthropocene climate change affects the physical environment, ecosystems and human societies. False True False False True False False True False True False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +138 unstructured 19969 20076 wikiAnthropocene changes in the climate system include an overall warming trend, more extreme weather and rising sea levels. True True False False True False False False False False False False False False False 0.1531 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +139 unstructured 20077 20162 wikiAnthropocene these in turn impact nature and wildlife, as well as human settlements and societies. False False False False False False False True False False False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +140 unstructured 20163 20239 wikiAnthropocene [110] the effects of human-caused climate change are broad and far-reaching. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +141 unstructured 20240 20304 wikiAnthropocene this is especially so if there is no significant climate action. False True False False True False False False False False False False False False False -0.1511 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +142 unstructured 20305 20416 wikiAnthropocene experts sometimes describe the projected and observed negative impacts of climate change as the climate crisis. False True False False True False False False False False False False False False False -0.8316 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +143 unstructured 20417 20473 wikiAnthropocene the changes in climate are not uniform across the earth. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +144 unstructured 20474 20546 wikiAnthropocene in particular, most land areas have warmed faster than most ocean areas. False False False False True False False False False True False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +145 unstructured 20547 20600 wikiAnthropocene the arctic is warming faster than most other regions. False False False False True False False False False False False False False False False 0.1531 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +146 unstructured 20601 20658 wikiAnthropocene [111] there are many effects of climate change on oceans. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +147 unstructured 20659 20769 wikiAnthropocene these include an increase in ocean temperatures, a rise in sea level from ocean warming and ice sheet melting. False True False True True False False False False False False False True False False 0.4404 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +148 unstructured 20770 20814 wikiAnthropocene they include increased ocean stratification. False False False True False False False False False False False False True False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +149 unstructured 20815 20932 wikiAnthropocene they also include changes to ocean currents including a weakening of the atlantic meridional overturning circulation. False False False False False False False False False False False False False False False -0.3182 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +150 unstructured 20933 21003 wikiAnthropocene [112]: 10 carbon dioxide from the atmosphere is acidifiying the ocean. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +151 unstructured 21004 21521 wikiAnthropocene [113] thick orange-brown smoke blocks half a blue sky, with conifers in the foreground a few grey fish swim over grey coral with white spikes desert sand half covers a village of small flat-roofed houses with scattered green trees large areas of still water behind riverside buildings some climate change effects: wildfire caused by heat and dryness, bleached coral caused by ocean acidification and heating, environmental migration caused by desertification, and coastal flooding caused by storms and sea level rise. True True False True True False False False False True False False False False False -0.128 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +152 unstructured 21522 21535 wikiAnthropocene geomorphology False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +153 unstructured 21537 21698 wikiAnthropocene changes in drainage patterns traceable to human activity will persist over geologic time in large parts of the continents where the geologic regime is erosional. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +154 unstructured 21699 21805 wikiAnthropocene this involves, for example, the paths of roads and highways defined by their grading and drainage control. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +155 unstructured 21806 21943 wikiAnthropocene direct changes to the form of the earth's surface by human activities (quarrying and landscaping, for example) also record human impacts. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +156 unstructured 21945 22208 wikiAnthropocene it has been suggested[by whom?] that the deposition of calthemite formations exemplify a natural process which has not previously occurred prior to the human modification of the earth's surface, and which therefore represents a unique process of the anthropocene. False False False False False False False False False False False False False False False 0.3612 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +157 unstructured 22209 22344 wikiAnthropocene [114] calthemite is a secondary deposit, derived from concrete, lime, mortar or other calcareous material outside the cave environment. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +158 unstructured 22345 22530 wikiAnthropocene [115] calthemites grow on or under man-made structures (including mines and tunnels) and mimic the shapes and forms of cave speleothems, such as stalactites, stalagmites, flowstone etc. False False False False False False False False False False False False True False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +159 unstructured 22531 22567 wikiAnthropocene stratigraphy sedimentological record False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +160 unstructured 22569 22714 wikiAnthropocene human activities like deforestation and road construction are believed to have elevated average total sediment fluxes across the earth's surface. False True False False False True False True False False False False False False False 0.3612 True False True False False False False False False False True False True False False True False False False True False True True False False False True False False False +161 unstructured 22715 22892 wikiAnthropocene [4] however, construction of dams on many rivers around the world means the rates of sediment deposition in any given place do not always appear to increase in the anthropocene. False False False True False False False False False False False False True False False 0.3182 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +162 unstructured 22893 23081 wikiAnthropocene for instance, many river deltas around the world are actually currently starved of sediment by such dams, and are subsiding and failing to keep up with sea level rise, rather than growing. False True False True False False False False False False False False True False False -0.7351 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +163 unstructured 23082 23104 wikiAnthropocene [4][116] fossil record False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +164 unstructured 23106 23260 wikiAnthropocene increases in erosion due to farming and other operations will be reflected by changes in sediment composition and increases in deposition rates elsewhere. False False False True False False False False False False False False True False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +165 unstructured 23261 23392 wikiAnthropocene in land areas with a depositional regime, engineered structures will tend to be buried and preserved, along with litter and debris. False False False False False False False False False True False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +166 unstructured 23393 23573 wikiAnthropocene litter and debris thrown from boats or carried by rivers and creeks will accumulate in the marine environment, particularly in coastal areas, but also in mid-ocean garbage patches. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +167 unstructured 23574 23658 wikiAnthropocene "such human-created artifacts preserved in stratigraphy are known as ""technofossils""." False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +168 unstructured 23659 23681 wikiAnthropocene [4][117] technofossils False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +169 unstructured 23683 23782 wikiAnthropocene changes in biodiversity will also be reflected in the fossil record, as will species introductions. False False False False False False False True False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +170 unstructured 23783 24083 wikiAnthropocene an example cited is the domestic chicken, originally the red junglefowl gallus gallus, native to south-east asia but has since become the world's most common bird through human breeding and consumption, with over 60 billion consumed annually and whose bones would become fossilised in landfill sites. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +171 unstructured 24084 24155 wikiAnthropocene "[118] hence, landfills are important resources to find ""technofossils""." False False False False False False False False False True False False False False False 0.2023 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +172 unstructured 24156 24176 wikiAnthropocene [119] trace elements False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +173 unstructured 24178 24261 wikiAnthropocene in terms of trace elements, there are distinct signatures left by modern societies. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +174 unstructured 24262 24481 wikiAnthropocene for example, in the upper fremont glacier in wyoming, there is a layer of chlorine present in ice cores from 1960's atomic weapon testing programs, as well as a layer of mercury associated with coal plants in the 1980s. False False False False True False False False False False False False False False False -0.0258 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +175 unstructured 24499 24662 wikiAnthropocene from the late 1940s, nuclear tests have led to local nuclear fallout and severe contamination of test sites both on land and in the surrounding marine environment. False False False False False False False False False True False False False False False -0.3818 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +176 unstructured 24663 24772 wikiAnthropocene some of the radionuclides that were released during the tests are 137cs, 90sr, 239pu, 240pu, 241am, and 131i. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +177 unstructured 24773 24865 wikiAnthropocene these have been found to have had significant impact on the environment and on human beings. False False False False False False False False False False False False False False False 0.2023 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +178 unstructured 24866 25029 wikiAnthropocene in particular, 137cs and 90sr have been found to have been released into the marine environment and led to bioaccumulation over a period through food chain cycles. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +179 unstructured 25030 25227 wikiAnthropocene the carbon isotope 14c, commonly released during nuclear tests, has also been found to be integrated into the atmospheric co2, and infiltrating the biosphere, through ocean-atmosphere gas exchange. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +180 unstructured 25228 25365 wikiAnthropocene increase in thyroid cancer rates around the world is also surmised to be correlated with increasing proportions of the 131i radionuclide. False False False True False False False False False False False False True False False -0.4767 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +181 unstructured 25373 25575 wikiAnthropocene the highest global concentration of radionuclides was estimated to have been in 1965, one of the dates which has been proposed as a possible benchmark for the start of the formally defined anthropocene. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +182 unstructured 25583 25768 wikiAnthropocene human burning of fossil fuels has also left distinctly elevated concentrations of black carbon, inorganic ash, and spherical carbonaceous particles in recent sediments across the world. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +183 unstructured 25769 25888 wikiAnthropocene concentrations of these components increases markedly and almost simultaneously around the world beginning around 1950. False False False True False False False False False False False False True False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +184 unstructured 25889 25913 wikiAnthropocene [4] anthropocene markers False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +185 unstructured 25915 26163 wikiAnthropocene a marker that accounts for a substantial global impact of humans on the total environment, comparable in scale to those associated with significant perturbations of the geological past, is needed in place of minor changes in atmosphere composition. False False False False False False False False False False False False False False False 0.3818 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +186 unstructured 26176 26261 wikiAnthropocene a useful candidate for holding markers in the geologic time record is the pedosphere. False False False False False False False False False False False False False False False 0.4404 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +187 unstructured 26262 26381 wikiAnthropocene soils retain information about their climatic and geochemical history with features lasting for centuries or millennia. False False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +188 unstructured 26382 26467 wikiAnthropocene [127] human activity is now firmly established as the sixth factor of soil formation. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +189 unstructured 26468 26787 wikiAnthropocene [128] humanity affects pedogenesis directly by, for example, land levelling, trenching and embankment building, landscape-scale control of fire by early humans, organic matter enrichment from additions of manure or other waste, organic matter impoverishment due to continued cultivation and compaction from overgrazing. False False False False False False False False False True False False False False False -0.6124 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +190 unstructured 26788 26882 wikiAnthropocene human activity also affects pedogenesis indirectly by drift of eroded materials or pollutants. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +191 unstructured 26883 27160 wikiAnthropocene anthropogenic soils are those markedly affected by human activities, such as repeated ploughing, the addition of fertilisers, contamination, sealing, or enrichment with artefacts (in the world reference base for soil resources they are classified as anthrosols and technosols). False False False False False False False False False True False False False False False -0.1531 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +192 unstructured 27161 27287 wikiAnthropocene an example from archaeology would be dark earth phenomena when long-term human habitation enriches the soil with black carbon. False False False False False False False True False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +193 unstructured 27289 27479 wikiAnthropocene anthropogenic soils are recalcitrant repositories of artefacts and properties that testify to the dominance of the human impact, and hence appear to be reliable markers for the anthropocene. False False False False False False False False False False False False False False False 0.2023 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +194 unstructured 27480 27750 wikiAnthropocene some anthropogenic soils may be viewed as the 'golden spikes' of geologists (global boundary stratotype section and point), which are locations where there are strata successions with clear evidences of a worldwide event, including the appearance of distinctive fossils. False False False False False False False False False False False False False False False 0.4019 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +195 unstructured 27751 27874 wikiAnthropocene [129] drilling for fossil fuels has also created holes and tubes which are expected to be detectable for millions of years. False False False False False False False False False False False False False False False 0.25 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +196 unstructured 27875 28198 wikiAnthropocene [130] the astrobiologist david grinspoon has proposed that the site of the apollo 11 lunar landing, with the disturbances and artifacts that are so uniquely characteristic of our species' technological activity and which will survive over geological time spans could be considered as the 'golden spike' of the anthropocene. False False False False False False False True False False False False False False False -0.34 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +197 unstructured 28206 28389 wikiAnthropocene an october 2020 study coordinated by university of colorado at boulder found that distinct physical, chemical and biological changes to earth's rock layers began around the year 1950. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +198 unstructured 28390 28964 wikiAnthropocene the research revealed that since about 1950, humans have doubled the amount of fixed nitrogen on the planet through industrial production for agriculture, created a hole in the ozone layer through the industrial scale release of chlorofluorocarbons (cfcs), released enough greenhouse gasses from fossil fuels to cause planetary level climate change, created tens of thousands of synthetic mineral-like compounds that do not naturally occur on earth, and caused almost one-fifth of river sediment worldwide to no longer reach the ocean due to dams, reservoirs and diversions. False True False False True False False False False True False False False False False 0.1842 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +199 unstructured 28965 29144 wikiAnthropocene "humans have produced so many millions of tons of plastic each year since the early 1950s that microplastics are ""forming a near-ubiquitous and unambiguous marker of anthropocene""." False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +200 unstructured 29145 29625 wikiAnthropocene "[132][133] the study highlights a strong correlation between global human population size and growth, global productivity and global energy use and that the ""extraordinary outburst of consumption and productivity demonstrates how the earth system has departed from its holocene state since ~1950 ce, forcing abrupt physical, chemical and biological changes to the earth's stratigraphic record that can be used to justify the proposal for naming a new epoch-the anthropocene.""[133]" False False False False False False False False False True False False True False False 0.7906 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +201 unstructured 29627 30305 wikiAnthropocene "a december 2020 study published in nature found that the total anthropogenic mass, or human-made materials, outweighs all the biomass on earth, and highlighted that ""this quantification of the human enterprise gives a mass-based quantitative and symbolic characterization of the human-induced epoch of the anthropocene.""[134][135] debates ""while we often think of ecological damage as a modern problem our impacts date back millennia to the times in which humans lived as hunter-gatherers. our history with wild animals has been a zero-sum game: either we hunted them to extinction, or we destroyed their habitats with agricultural land."" - hannah ritchie for our world in data." False False False False False False False True False True False False False False False -0.8442 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +202 unstructured 30313 30573 wikiAnthropocene although the validity of anthropocene as a scientific term remains disputed, its underlying premise, i.e., that humans have become a geological force, or rather, the dominant force shaping the earth's climate, has found traction among academics and the public. False True False False True False False False False False False False False False False -0.34 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +203 unstructured 30574 30901 wikiAnthropocene "in an opinion piece for philosophical transactions of the royal society b, rodolfo dirzo, gerardo ceballos, and paul r. ehrlich write that the term is ""increasingly penetrating the lexicon of not only the academic socio-sphere, but also society more generally"", and is now included as an entry in the oxford english dictionary." False False False True False False False False False False False False True False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +204 unstructured 30902 30997 wikiAnthropocene [137] the university of cambridge, as another example, offers a degree in anthropocene studies. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +205 unstructured 30998 31129 wikiAnthropocene [138] in the public sphere, the term anthropocene has become increasingly ubiquitous in activist, pundit, and political discourses. False False False True False False False False False False False False True False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +206 unstructured 31130 31534 wikiAnthropocene "some who are critical of the term anthropocene nevertheless concede that ""for all its problems, [it] carries power.""[139] the popularity and currency of the word has led scholars to label the term a ""charismatic meta-category""[140] or ""charismatic mega-concept.""[141] the term, regardless, has been subject to a variety of criticisms from social scientists, philosophers, indigenous scholars, and others." False False False False False False False False False False False False False False False -0.4215 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +207 unstructured 31536 31924 wikiAnthropocene "the anthropologist john hartigan has argued that due its status as a charismatic meta-category, the term anthropocene marginalizes competing, but less visible, concepts such as that of ""multispecies.""[142] the more salient charge is that the ready acceptance of anthropocene is due to its conceptual proximity to the status quo - that is, to notions of human individuality and centrality." False False False False False False False False True False False False False False False 0.8621 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +208 unstructured 31926 32094 wikiAnthropocene other scholars appreciate the way in which the term anthropocene recognizes humanity as a geological force, but take issue with the indiscriminate way in which it does. False False False False False False False False False False False False False False False 0.2144 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +209 unstructured 32095 32157 wikiAnthropocene not all humans are equally responsible for the climate crisis. False True False False True False False False False False False False False False False -0.4215 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +210 unstructured 32158 32313 wikiAnthropocene to that end, scholars such as the feminist theorist donna haraway and sociologist jason moore, have suggested naming the epoch instead as the capitalocene. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +211 unstructured 32314 32442 wikiAnthropocene [143][144][145] such implies capitalism as the fundamental reason for the ecological crisis, rather than just humans in general. False False False False False False False False False False False False False False False -0.6249 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +212 unstructured 32443 32657 wikiAnthropocene "[146][147][148] however, according to philosopher steven best, humans have created ""hierarchical and growth-addicted societies"" and have demonstrated ""ecocidal proclivities"" long before the emergence of capitalism." False False False False False False False False False False False False True False False 0.7351 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +213 unstructured 32658 32910 wikiAnthropocene [149] hartigan, bould, and haraway all critique what anthropocene does as a term; however, hartigan and bould differ from haraway in that they criticize the utility or validity of a geological framing of the climate crisis, whereas haraway embraces it. False True False False True False False False False False False False False False False -0.7717 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +214 unstructured 32912 33070 wikiAnthropocene "in addition to ""capitalocene,"" other terms have also been proposed by scholars to trace the roots of the epoch to causes other than the human species broadly." False False False False False False False True False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +215 unstructured 33071 33348 wikiAnthropocene "janae davis, for example, has suggested the ""plantationocene"" as a more appropriate term to call attention to the role that plantation agriculture has played in the formation of the epoch, alongside kathryn yusoff's argument that racism as a whole is foundational to the epoch." False False False False False False False False False False False False False False False -0.6369 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +216 unstructured 33349 34018 wikiAnthropocene "the plantationocene concept traces ""the ways that plantation logics organize modern economies, environments, bodies, and social relations.""[150][151][152][153] in a similar vein, indigenous studies scholars such as métis geographer zoe todd have argued that the epoch must be dated back to the colonization of the americas, as this ""names the problem of colonialism as responsible for contemporary environmental crisis.""[154] potawatomi philosopher kyle powys whyte has further argued that the anthropocene has been apparent to indigenous peoples in the americas since the inception of colonialism because of ""colonialism's role in environmental change.""[155][156][157]" False False False False False False False False False False False False False False False -0.6597 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +217 unstructured 34020 34097 wikiAnthropocene other critiques of anthropocene have focused on the genealogy of the concept. False False False False False False False False False False False False False False False 0.3818 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +218 unstructured 34098 34863 wikiAnthropocene "todd also provides a phenomenological account, which draws on the work of the philosopher sara ahmed, writing: ""when discourses and responses to the anthropocene are being generated within institutions and disciplines which are embedded in broader systems that act as de facto 'white public space,' the academy and its power dynamics must be challenged.""[158] other aspects which constitute current understandings of the concept of the anthropocene such as the ontological split between nature and society, the assumption of the centrality and individuality of the human, and the framing of environmental discourse in largely scientific terms have been criticized by scholars as concepts rooted in colonialism and which reinforce systems of postcolonial domination." False False False False False False False False False False False False False False False -0.1531 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +219 unstructured 34864 35055 wikiAnthropocene [159] to that end, todd makes the case that the concept of anthropocene must be indigenized and decolonized if it is to become a vehicle of justice as opposed to white thought and domination. False False False False False False False False False False False False False False False 0.4939 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +220 unstructured 35057 35244 wikiAnthropocene the scholar daniel wildcat, a yuchi member of the muscogee nation of oklahoma, for example, has emphasized spiritual connection to the land as a crucial tenet for any ecological movement. False False False False False False False False False True False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +221 unstructured 35245 35470 wikiAnthropocene [160] similarly, in her study of the ladakhi people in northern india, the anthropologist karine gagné, detailed their understanding of the relation between nonhuman and human agency as one that is deeply intimate and mutual. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +222 unstructured 35471 35845 wikiAnthropocene "for the ladakhi, the nonhuman alters the epistemic, ethical, and affective development of humans - it provides a way of ""being in the world.""[161] the ladakhi, who live in the himalayas, for example, have seen the retreat of the glaciers not just as a physical loss, but also as the loss of entities which generate knowledge, compel ethical reflections, and foster intimacy." False False False False False False False False False False False False False False False 0.5267 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +223 unstructured 35846 35964 wikiAnthropocene other scholars have similarly emphasized the need to return to notions of relatedness and interdependence with nature. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +224 unstructured 35965 36289 wikiAnthropocene "the writer jenny odell has written about what robin wall kimmerer calls ""species loneliness,""[162] the loneliness which occurs from the separation of the human and the nonhuman, and the anthropologist radhika govindrajan has theorized on the ethics of care, or relatedness, which govern relations between humans and animals." False False False False False False False True False False False False False False False 0.1027 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +225 unstructured 36290 36379 wikiAnthropocene [163] scholars are divided on whether to do away with the term anthropocene or co-opt it. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +226 unstructured 36380 36439 wikiAnthropocene """early anthropocene"" model main article: early anthropocene" False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +227 unstructured 36441 36582 wikiAnthropocene william ruddiman has argued that the anthropocene began approximately 8,000 years ago with the development of farming and sedentary cultures. False False False False False False False False False False False False False False False -0.3612 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +228 unstructured 36583 36708 wikiAnthropocene [164] at that point, humans were dispersed across all continents except antarctica, and the neolithic revolution was ongoing. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +229 unstructured 36709 36832 wikiAnthropocene during this period, humans developed agriculture and animal husbandry to supplement or replace hunter-gatherer subsistence. False False False False False False False True False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +230 unstructured 36833 36947 wikiAnthropocene [165] such innovations were followed by a wave of extinctions, beginning with large mammals and terrestrial birds. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +231 unstructured 36948 37087 wikiAnthropocene this wave was driven by both the direct activity of humans (e.g. hunting) and the indirect consequences of land-use change for agriculture. False False False False False False False False False True False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +232 unstructured 37088 37219 wikiAnthropocene landscape-scale burning by prehistoric hunter-gathers may have been an additional early source of anthropogenic atmospheric carbon. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +233 unstructured 37220 37395 wikiAnthropocene [166] ruddiman also claims that the greenhouse gas emissions in-part responsible for the anthropocene began 8,000 years ago when ancient farmers cleared forests to grow crops. False True False False False False False False False True False False True False False 0.4019 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +234 unstructured 37413 37703 wikiAnthropocene "ruddiman's work has been challenged with data from an earlier interglaciation (""stage 11"", approximately 400,000 years ago) which suggests that 16,000 more years must elapse before the current holocene interglaciation comes to an end, and thus the early anthropogenic hypothesis is invalid." False False False False False False False False False False False False False False False -0.1027 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +235 unstructured 37704 37882 wikiAnthropocene "[170] also, the argument that ""something"" is needed to explain the differences in the holocene is challenged by more recent research showing that all interglacials are different." False False False False False False False False False False False False False False False -0.4404 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +236 unstructured 37883 37901 wikiAnthropocene [171] homogenocene False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +237 unstructured 37903 38288 wikiAnthropocene homogenocene (from old greek: homo-, same; geno-, kind; kainos-, new;) is a more specific term used to define our current epoch, in which biodiversity is diminishing and biogeography and ecosystems around the globe seem more and more similar to one another mainly due to invasive species that have been introduced around the globe either on purpose (crops, livestock) or inadvertently. False False False False False False False True False True False False False False False 0.5267 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +238 unstructured 38289 38485 wikiAnthropocene this is due to the newfound globalism that humans participate in, as species traveling across the world to another region was not as easily possible in any point of time in history as it is today. False False False False False False False True False False False False False False False -0.2584 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +239 unstructured 38493 38702 wikiAnthropocene "the term homogenocene was first used by michael samways in his editorial article in the journal of insect conservation from 1999 titled ""translocating fauna to foreign lands: here comes the homogenocene.""[173]" False False False False False False False True False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +240 unstructured 38704 38937 wikiAnthropocene "the term was used again by john l. curnutt in the year 2000 in ecology, in a short list titled ""a guide to the homogenocene"",[174] which reviewed alien species in north america and hawaii: impacts on natural ecosystems by george cox." False False False False False False False True False True False False False False False 0.3612 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +241 unstructured 38938 39113 wikiAnthropocene charles c. mann, in his acclaimed book 1493: uncovering the new world columbus created, gives a bird's-eye view of the mechanisms and ongoing implications of the homogenocene. False False False False False False False False False False False False False False False 0.25 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +242 unstructured 39114 39150 wikiAnthropocene [175] society and culture humanities False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +243 unstructured 39152 39263 wikiAnthropocene the concept of the anthropocene has also been approached via humanities such as philosophy, literature and art. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +244 unstructured 39264 39415 wikiAnthropocene in the scholarly world, it has been the subject of increasing attention through special journals,[176] conferences,[177][178] and disciplinary reports. False False False True False False False False False False False False True False False 0.4019 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +245 unstructured 39416 39679 wikiAnthropocene "[179] the anthropocene, its attendant timescale, and ecological implications prompt questions about death and the end of civilisation,[180] memory and archives,[181] the scope and methods of humanistic inquiry,[182] and emotional responses to the ""end of nature""." False False False False False False False False False False False False False False False -0.5106 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +246 unstructured 39680 39968 wikiAnthropocene "[183] some scholars have posited that the realities of the anthropocene, including ""human-induced biodiversity loss, exponential increases in per-capita resource consumption, and global climate change,"" have made the goal of environmental sustainability largely unattainable and obsolete." False True False True True False False False False True False False True False False -0.5423 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +247 unstructured 39976 40026 wikiAnthropocene historians have actively engaged the anthropocene. False False False False False False False False False False False False False False False 0.6124 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +248 unstructured 40027 40255 wikiAnthropocene in 2000, the same year that paul crutzen coined the term, world historian john mcneill published something new under the sun,[185] tracing the rise of human societies' unprecedented impact on the planet in the twentieth century. False False False True False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +249 unstructured 40256 40476 wikiAnthropocene [185] in 2001, historian of science naomi oreskes revealed the systematic efforts to undermine trust in climate change science and went on to detail the corporate interests delaying action on the environmental challenge. False True False False True False False False False False False False False True False 0.5267 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +250 unstructured 40477 40642 wikiAnthropocene [186][187] both mcneill and oreskes became members of the anthropocene working group because of their work correlating human activities and planetary transformation. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +251 unstructured 40643 40658 wikiAnthropocene popular culture False False False False False False False False False False False False False False False 0.4215 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +252 unstructured 40660 40764 wikiAnthropocene "in 2019, the english musician nick mulvey released a music video on youtube named ""in the anthropocene""." False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +253 unstructured 40765 40900 wikiAnthropocene [188] in cooperation with sharp's brewery, the song was recorded on 105 vinyl records made of washed-up plastic from the cornish coast. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +254 unstructured 40901 41063 wikiAnthropocene "[189] the anthropocene reviewed is a podcast and book by author john green, where he ""reviews different facets of the human-centered planet on a five-star scale""." False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +255 unstructured 41064 41198 wikiAnthropocene [190] in 2015, the american death metal band cattle decapitation released its seventh studio album titled the anthropocene extinction. False False False False False False False False False False False False False False False -0.5994 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +1 unstructured 0 14 wikiClimate_change climate change False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +2 unstructured 16 28 wikiClimate_change article talk False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +3 unstructured 30 59 wikiClimate_change read view source view history False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +4 unstructured 61 66 wikiClimate_change tools False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +5 unstructured 68 198 wikiClimate_change "featured article page semi-protected listen to this article from wikipedia, the free encyclopedia ""global warming"" redirects here." False True False False True False False False False False False False False False False 0.5994 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +6 unstructured 199 287 wikiClimate_change for other uses, see climate change (disambiguation) and global warming (disambiguation). False True False False True False False False False False False False False False False 0.1531 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +7 unstructured 288 338 wikiClimate_change this article is about contemporary climate change. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +8 unstructured 339 405 wikiClimate_change for historical climate trends, see climate variability and change. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +9 unstructured 406 582 wikiClimate_change the global map shows sea temperature rises of 0.5 to 1 degree celsius; land temperature rises of 1 to 2 degree celsius; and arctic temperature rises of up to 4 degrees celsius. False False False False True False False False False True False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +10 unstructured 583 641 wikiClimate_change changes in surface air temperature over the past 50 years. False False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +11 unstructured 642 763 wikiClimate_change [1] the arctic has warmed the most, and temperatures on land have generally increased more than sea surface temperatures. False True False True True False False False False True False False True False False 0.4939 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +12 unstructured 764 879 wikiClimate_change earth's average surface air temperature has increased almost 1.5 °c (about 2.5 °f) since the industrial revolution. False False False True True False False False False False False False True False False 0.2732 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +13 unstructured 880 993 wikiClimate_change natural forces cause some variability, but the 20-year average shows the progressive influence of human activity. False False False False False False False False False False False False False False False 0.1901 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +14 unstructured 999 1149 wikiClimate_change in common usage, climate change describes global warming-the ongoing increase in global average temperature-and its effects on earth's climate system. False True False True True False False False False False False False True False False 0.3182 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +15 unstructured 1150 1244 wikiClimate_change climate change in a broader sense also includes previous long-term changes to earth's climate. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +16 unstructured 1245 1384 wikiClimate_change the current rise in global average temperature is more rapid than previous changes, and is primarily caused by humans burning fossil fuels. False False False True True False False False False False False False False False False 0.0 False True True False True True False False False False True False True False False False False False False True False True False False False False False False False False +17 unstructured 1385 1531 wikiClimate_change [3][4] fossil fuel use, deforestation, and some agricultural and industrial practices add to greenhouse gases, notably carbon dioxide and methane. False True False False False True False True False False False False False False False 0.0 False True True False True False False False False False True False True False False True False False False True False True True False False False True False False False +18 unstructured 1532 1630 wikiClimate_change [5] greenhouse gases absorb some of the heat that the earth radiates after it warms from sunlight. False True False False True False False False False False False False False False False 0.2732 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +19 unstructured 1631 1728 wikiClimate_change larger amounts of these gases trap more heat in earth's lower atmosphere, causing global warming. False True False False True False False False False False False False False False False -0.4404 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +20 unstructured 1730 1797 wikiClimate_change climate change has an increasingly large impact on the environment. False True False True True False False False False False False False True False False 0.0 False True True False True True False False False False True False True False False False False False False True False True False False False False False False False False +21 unstructured 1798 1877 wikiClimate_change deserts are expanding, while heat waves and wildfires are becoming more common. True False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True True False False False False False False False True True False False False False False False False False False +22 unstructured 1878 1993 wikiClimate_change [6] amplified warming in the arctic has contributed to thawing permafrost, retreat of glaciers and sea ice decline. False False False False True False False False False False False False False False False 0.34 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +23 unstructured 1994 2093 wikiClimate_change [7] higher temperatures are also causing more intense storms, droughts, and other weather extremes. True True False False True False False False False False False False True False False 0.1513 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +24 unstructured 2094 2221 wikiClimate_change [8] rapid environmental change in mountains, coral reefs, and the arctic is forcing many species to relocate or become extinct. False False False False False False False True False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +25 unstructured 2222 2326 wikiClimate_change [9] even if efforts to minimise future warming are successful, some effects will continue for centuries. False False False False True False False False False False False False False False False 0.6597 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +26 unstructured 2327 2395 wikiClimate_change these include ocean heating, ocean acidification and sea level rise. False True False True True False False False False False False False False False False 0.0 False True True False True True False False False False True False True False False False False False False True False True False False False False False False False False +27 unstructured 2402 2540 wikiClimate_change climate change threatens people with increased flooding, extreme heat, increased food and water scarcity, more disease, and economic loss. True True False True True False False False True True False False True False False -0.1082 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +28 unstructured 2541 2591 wikiClimate_change human migration and conflict can also be a result. False False False False False False False False True False True False False False False -0.3182 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +29 unstructured 2592 2711 wikiClimate_change [11] the world health organization (who) calls climate change the greatest threat to global health in the 21st century. False True False False True False False False True False False False False False False 0.2023 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +30 unstructured 2712 2808 wikiClimate_change [12] societies and ecosystems will experience more severe risks without action to limit warming. False False False False True False False True False True False False False False False -0.5678 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +31 unstructured 2809 3013 wikiClimate_change [13] adapting to climate change through efforts like flood control measures or drought-resistant crops partially reduces climate change risks, although some limits to adaptation have already been reached. True True False False True False False False False False False False False True True 0.2023 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +32 unstructured 3014 3171 wikiClimate_change [14] poorer communities are responsible for a small share of global emissions, yet have the least ability to adapt and are most vulnerable to climate change. False True False False True False False False False False False False False False False -0.2858 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +33 unstructured 3172 3351 wikiClimate_change [15][16] bobcat fire in monrovia, ca, september 10, 2020 bleached colony of acropora coral a dry lakebed in california, which is experiencing its worst megadrought in 1,200 years. False True False False False False False False False False False False False False False -0.7579 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +34 unstructured 3352 3560 wikiClimate_change [17] examples of some effects of climate change: wildfire intensified by heat and drought, bleaching of corals occurring more often due to marine heatwaves, and worsening droughts compromising water supplies. True True False True True False False False False True False False False False False -0.4588 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +35 unstructured 3562 3676 wikiClimate_change many climate change impacts have been felt in recent years, with 2023 the warmest on record at +1.48 °c (2.66 °f). False True False False True False False False False False False False False False False 0.4019 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +36 unstructured 3677 3808 wikiClimate_change [18] additional warming will increase these impacts and can trigger tipping points, such as melting all of the greenland ice sheet. False False False True True False False False False False False False True False False 0.4404 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +37 unstructured 3809 3908 wikiClimate_change "[19] under the 2015 paris agreement, nations collectively agreed to keep warming ""well under 2 °c""." False False False False True False False False False False False False False False False 0.7906 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +38 unstructured 3909 4038 wikiClimate_change however, with pledges made under the agreement, global warming would still reach about 2.7 °c (4.9 °f) by the end of the century. False True False False True False False False False False False False False False False 0.5994 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +39 unstructured 4039 4151 wikiClimate_change [20] limiting warming to 1.5 °c will require halving emissions by 2030 and achieving net-zero emissions by 2050. False False False False True False False False False False False False False False False 0.1531 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +40 unstructured 4158 4294 wikiClimate_change fossil fuel use can be phased out by conserving energy and switching to energy sources that do not produce significant carbon pollution. False False False False False False False True False True False False False False False 0.3834 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +41 unstructured 4295 4362 wikiClimate_change these energy sources include wind, solar, hydro, and nuclear power. True False False False True False False False False True False False False False False 0.2732 True False True False False False False False False False True True False False False False False False False True True False False False False False False False False False +42 unstructured 4363 4508 wikiClimate_change [22][23] cleanly generated electricity can replace fossil fuels for powering transportation, heating buildings, and running industrial processes. False False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +43 unstructured 4509 4655 wikiClimate_change [24] carbon can also be removed from the atmosphere, for instance by increasing forest cover and farming with methods that capture carbon in soil. False False False True False False False False False True False False True False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +44 unstructured 4656 4676 wikiClimate_change [25][26] terminology False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +45 unstructured 4678 4846 wikiClimate_change before the 1980s it was unclear whether the warming effect of increased greenhouse gases was stronger than the cooling effect of airborne particulates in air pollution. False True False True True False False False False False False False True False False 0.5106 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +46 unstructured 4847 4959 wikiClimate_change scientists used the term inadvertent climate modification to refer to human impacts on the climate at this time. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +47 unstructured 4960 5076 wikiClimate_change [27] in the 1980s, the terms global warming and climate change became more common, often being used interchangeably. False True False False True False False False False False False False False False False 0.1531 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +48 unstructured 5077 5279 wikiClimate_change [28] scientifically, global warming refers only to increased surface warming, while climate change describes both global warming and its effects on earth's climate system, such as precipitation changes. True True False True True False False False False False False False True False False 0.5994 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +49 unstructured 5286 5411 wikiClimate_change climate change can also be used more broadly to include changes to the climate that have happened throughout earth's history. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +50 unstructured 5412 5578 wikiClimate_change [29] global warming-used as early as 1975[30]-became the more popular term after nasa climate scientist james hansen used it in his 1988 testimony in the u.s. senate. False True False False True False False False False False False False False False False 0.4754 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +51 unstructured 5579 5636 wikiClimate_change [31] since the 2000s, climate change has increased usage. False True False True True False False False False False False False True False False 0.2732 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +52 unstructured 5637 5813 wikiClimate_change [32] various scientists, politicians and media now use the terms climate crisis or climate emergency to talk about climate change, and global heating instead of global warming. False True False False True False False False False False False False False False False -0.7269 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +53 unstructured 5814 5974 wikiClimate_change [33] global temperature rise global surface temperature reconstruction over the last 2000 years using proxy data from tree rings, corals, and ice cores in blue. False True False True True False False False False False False False False False False 0.0 False True True False True True False False False False True False True False False False False False False True False True False False False False False False False False +54 unstructured 5975 6013 wikiClimate_change [34] directly observed data is in red. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +55 unstructured 6014 6174 wikiClimate_change [35] temperature records prior to global warming main articles: climate variability and change; temperature record of the last 2,000 years; and paleoclimatology False True False False True False False False False False False False False False False 0.1531 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +56 unstructured 6176 6340 wikiClimate_change prior to human evolution the record includes hotter temperatures and occasional abrupt changes, such as the paleocene-eocene thermal maximum 55.5 million years ago. False False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +57 unstructured 6347 6535 wikiClimate_change over the last few million years human beings evolved in a climate that cycled through ice ages, with global average temperature ranging between current levels and 5-6 °c colder than today. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +58 unstructured 6536 6699 wikiClimate_change [37][38] historical patterns of warming and cooling, like the medieval warm period and the little ice age, did not occur at the same time across different regions. False False False False True False False False False False False False False False False 0.6124 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +59 unstructured 6700 6800 wikiClimate_change temperatures may have reached as high as those of the late 20th century in a limited set of regions. False False False False True False False False False False False False False False False -0.128 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +60 unstructured 6801 6898 wikiClimate_change [39] climate information for that period comes from climate proxies, such as trees and ice cores. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +61 unstructured 6899 7134 wikiClimate_change [40] warming since the industrial revolution main article: instrumental temperature record in recent decades, new high temperature records have substantially outpaced new low temperature records on a growing portion of earth's surface. False False False False True False False False False False False False True False False -0.0164 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +62 unstructured 7135 7273 wikiClimate_change [41] there has been an increase in ocean heat content during recent decades as the oceans absorb over 90% of the heat from global warming. False True False True True False False False False False False False True False False 0.4404 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +63 unstructured 7280 7345 wikiClimate_change around 1850 thermometer records began to provide global coverage. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +64 unstructured 7346 7517 wikiClimate_change [43] between the 18th century and 1970 there was little net warming, as the warming impact of greenhouse gas emissions was offset by cooling from sulfur dioxide emissions. False True False False True False False False False False False False False False False 0.2315 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +65 unstructured 7518 7666 wikiClimate_change sulfur dioxide causes acid rain, but it also produces sulfate aerosols in the atmosphere, which reflect sunlight and cause so-called global dimming. True False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True True False False False False False False False True True False False False False False False False False False +66 unstructured 7667 7800 wikiClimate_change after 1970, the increasing accumulation of greenhouse gases and controls on sulfur pollution led to a marked increase in temperature. False True False True True False False False False False False False True False False 0.3182 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +67 unstructured 7815 7940 wikiClimate_change multiple independent datasets all show worldwide increases in surface temperature,[47] at a rate of around 0.2 °c per decade. False False False True True False False False False False False False True False False 0.0 False True True False True True False False False False True False True False False False False False False True False True False False False False False False False False +68 unstructured 7941 8063 wikiClimate_change [48] the 2013-2022 decade warmed to an average 1.15 °c [1.00-1.25 °c] compared to the pre-industrial baseline (1850-1900). False False False False True False False False False False False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +69 unstructured 8064 8215 wikiClimate_change [49] not every single year was warmer than the last: internal climate variability processes can make any year 0.2 °c warmer or colder than the average. False True False False True False False False False False False False False False False 0.5267 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +70 unstructured 8216 8408 wikiClimate_change "[50] from 1998 to 2013, negative phases of two such processes, pacific decadal oscillation (pdo)[51] and atlantic multidecadal oscillation (amo)[52] caused a so-called ""global warming hiatus""." False True False False True False False False False False False False False False False -0.4767 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +71 unstructured 8409 8535 wikiClimate_change [53] after the hiatus, the opposite occurred, with years like 2023 exhibiting temperatures well above even the recent average. False False False False True False False False False False False False False False False 0.5574 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +72 unstructured 8536 8729 wikiClimate_change [54] this is why the temperature change is defined in terms of a 20-year average, which minimises the noise of hot and cold years and decadal climate patterns, and detects the long-term signal. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +73 unstructured 8744 8813 wikiClimate_change a wide range of other observations reinforce the evidence of warming. False False False False True False False False False False False False False False False 0.1531 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +74 unstructured 8814 8966 wikiClimate_change [57][58] the upper atmosphere is cooling, because greenhouse gases are trapping heat near the earth's surface, and so less heat is radiating into space. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +75 unstructured 8967 9042 wikiClimate_change [59] warming reduces average snow cover and forces the retreat of glaciers. True False False False True False False False False False False False False True False 0.34 True False True False False False False False False False True True False False False False False False False True True False False False False False False False False False +76 unstructured 9043 9187 wikiClimate_change at the same time, warming also causes greater evaporation from the oceans, leading to more atmospheric humidity, more and heavier precipitation. True False False False True False False False False False False False False False False 0.4767 True False True False False False False False False False True True False False False False False False False True True False False False False False False False False False +77 unstructured 9188 9310 wikiClimate_change [60] plants are flowering earlier in spring, and thousands of animal species have been permanently moving to cooler areas. False False False False False False False True False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +78 unstructured 9311 9337 wikiClimate_change [61] differences by region False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +79 unstructured 9339 9394 wikiClimate_change different regions of the world warm at different rates. False False False False True False False False False False False False False False False 0.2263 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +80 unstructured 9395 9528 wikiClimate_change the pattern is independent of where greenhouse gases are emitted, because the gases persist long enough to diffuse across the planet. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +81 unstructured 9529 9689 wikiClimate_change since the pre-industrial period, the average surface temperature over land regions has increased almost twice as fast as the global average surface temperature. False False False True True False False False False True False False True False False 0.2732 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +82 unstructured 9690 9783 wikiClimate_change [62] this is because oceans lose more heat by evaporation and oceans can store a lot of heat. False False False False True False False False False False False False False False False -0.4019 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +83 unstructured 9784 9954 wikiClimate_change [63] the thermal energy in the global climate system has grown with only brief pauses since at least 1970, and over 90% of this extra energy has been stored in the ocean. False True False False True False False False False True False False True False False 0.4939 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +84 unstructured 9955 10038 wikiClimate_change [64][65] the rest has heated the atmosphere, melted ice, and warmed the continents. False False False False True False False False False False False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +85 unstructured 10045 10156 wikiClimate_change the northern hemisphere and the north pole have warmed much faster than the south pole and southern hemisphere. False False False False True False False False False False False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +86 unstructured 10157 10256 wikiClimate_change the northern hemisphere not only has much more land, but also more seasonal snow cover and sea ice. True False False False True False False False False True False False False False False 0.0 False True True False True False False False False False True True False False False False False False False True True False False False False False False False False False +87 unstructured 10257 10382 wikiClimate_change as these surfaces flip from reflecting a lot of light to being dark after the ice has melted, they start absorbing more heat. False False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +88 unstructured 10383 10466 wikiClimate_change [67] local black carbon deposits on snow and ice also contribute to arctic warming. True False False False True False False False False False False False False False False 0.1531 True False True False False False False False False False True True False False False False False False False True True False False False False False False False False False +89 unstructured 10467 10581 wikiClimate_change [68] arctic surface temperatures are increasing between three and four times faster than in the rest of the world. False False False True True False False False False False False False True False False 0.0 False True True False True True False False False False True False True False False False False False False True False True False False False False False False False False +90 unstructured 10582 10792 wikiClimate_change [69][70][71] melting of ice sheets near the poles weakens both the atlantic and the antarctic limb of thermohaline circulation, which further changes the distribution of heat and precipitation around the globe. True False False False True False False False False False False False False False False -0.3182 False False False True True False False False False False True True False False False False False False False True True False False False False False False False False False +91 unstructured 10793 11018 wikiClimate_change [72][73][74][75] future global temperatures further information: carbon budget and earth's energy budget cmip6 multi-model projections of global surface temperature changes for the year 2090 relative to the 1850-1900 average. False False False False True False False False False True False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +92 unstructured 11019 11126 wikiClimate_change the current trajectory for warming by the end of the century is roughly halfway between these two extremes. False False False False True False False False False False False False False False False 0.1531 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +93 unstructured 11141 11326 wikiClimate_change the world meteorological organization estimates a 66% chance of global temperatures exceeding 1.5 °c warming from the preindustrial baseline for at least one year between 2023 and 2027. False False False False True False False False False False False False False False False 0.3818 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +94 unstructured 11327 11470 wikiClimate_change [78][79] because the ipcc uses a 20-year average to define global temperature changes, a single year exceeding 1.5 °c does not break the limit. False False False False True False False False False False False True False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +95 unstructured 11472 11565 wikiClimate_change the ipcc expects the 20-year average global temperature to exceed +1.5 °c in the early 2030s. False False False False True False False False False False False True False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +96 unstructured 11566 11861 wikiClimate_change [80] the ipcc sixth assessment report (2023) included projections that by 2100 global warming is very likely to reach 1.0-1.8 °c under a scenario with very low emissions of greenhouse gases, 2.1-3.5 °c under an intermediate emissions scenario, or 3.3-5.7 °c under a very high emissions scenario. False True False False True False False False False False False True False False False -0.1102 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +97 unstructured 11862 11952 wikiClimate_change [81] in the intermediate and high emission scenarios, the warming will continue past 2100. False False False False True False False False False False False False False False False 0.1531 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +98 unstructured 11963 12129 wikiClimate_change the remaining carbon budget for staying beneath certain temperature increases is determined by modelling the carbon cycle and climate sensitivity to greenhouse gases. False True False True True False False False False False False False True False False 0.5423 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +99 unstructured 12130 12290 wikiClimate_change [84] according to the ipcc, global warming can be kept below 1.5 °c with a two-thirds chance if emissions after 2018 do not exceed 420 or 570 gigatonnes of co2. False True False False True False False False False False False True False False False 0.3818 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +100 unstructured 12291 12347 wikiClimate_change this corresponds to 10 to 13 years of current emissions. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +101 unstructured 12348 12394 wikiClimate_change there are high uncertainties about the budget. False False False False False False False False False False False False False False False -0.34 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +102 unstructured 12395 12520 wikiClimate_change for instance, it may be 100 gigatonnes of co2 equivalent smaller due to co2 and methane release from permafrost and wetlands. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +103 unstructured 12521 12647 wikiClimate_change [85] however, it is clear that fossil fuel resources need to be proactively kept in the ground to prevent substantial warming. False False False False True False False False False True False False False False False 0.6249 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +104 unstructured 12648 12764 wikiClimate_change otherwise, their shortages would not occur until the emissions have already locked in significant long-term impacts. False False False False False False False False False False False False False False False 0.0516 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +105 unstructured 12765 12904 wikiClimate_change [86] causes of recent global temperature rise main article: causes of climate change drivers of climate change from 1850-1900 to 2010-2019. False True False True True False False False False False False False False False False 0.0 False True True False True True False False False False True False True False False False False False False True False True False False False False False False False False +106 unstructured 12905 12999 wikiClimate_change there was no significant contribution from internal variability or solar and volcanic drivers. False False False False False False False False False False False False False False False -0.1511 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +107 unstructured 13001 13110 wikiClimate_change the climate system experiences various cycles on its own which can last for years, decades or even centuries. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +108 unstructured 13111 13232 wikiClimate_change for example, el niño events cause short-term spikes in surface temperature while la niña events cause short term cooling. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +109 unstructured 13233 13323 wikiClimate_change [87] their relative frequency can affect global temperature trends on a decadal timescale. False False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +110 unstructured 13324 13403 wikiClimate_change [88] other changes are caused by an imbalance of energy from external forcings. False False False False False False False False False True False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +111 unstructured 13404 13575 wikiClimate_change [89] examples of these include changes in the concentrations of greenhouse gases, solar luminosity, volcanic eruptions, and variations in the earth's orbit around the sun. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +112 unstructured 13582 13778 wikiClimate_change "to determine the human contribution to climate change, unique ""fingerprints"" for all potential causes are developed and compared with both observed patterns and known internal climate variability." False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +113 unstructured 13779 13926 wikiClimate_change [91] for example, solar forcing-whose fingerprint involves warming the entire atmosphere-is ruled out because only the lower atmosphere has warmed. False False False False True False False False False False False False False False False 0.128 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +114 unstructured 13927 13987 wikiClimate_change [92] atmospheric aerosols produce a smaller, cooling effect. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +115 unstructured 13988 14049 wikiClimate_change other drivers, such as changes in albedo, are less impactful. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +116 unstructured 14050 14316 wikiClimate_change [93] greenhouse gases main articles: greenhouse gas, greenhouse gas emissions, greenhouse effect, and carbon dioxide in earth's atmosphere co2 concentrations over the last 800,000 years as measured from ice cores[94][95][96][97] (blue/green) and directly[98] (black) False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +117 unstructured 14318 14441 wikiClimate_change greenhouse gases are transparent to sunlight, and thus allow it to pass through the atmosphere to heat the earth's surface. False True False False True False False False False False False False False False False 0.2263 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +118 unstructured 14442 14517 wikiClimate_change the earth radiates it as heat, and greenhouse gases absorb a portion of it. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +119 unstructured 14518 14647 wikiClimate_change this absorption slows the rate at which heat escapes into space, trapping heat near the earth's surface and warming it over time. False False False False True False False False False False False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +120 unstructured 14654 14888 wikiClimate_change while water vapour (≈50%) and clouds (≈25%) are the biggest contributors to the greenhouse effect, they primarily change as a function of temperature and are therefore mostly considered to be feedbacks that change climate sensitivity. False True False False True False False False False True False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +121 unstructured 14889 15138 wikiClimate_change on the other hand, concentrations of gases such as co2 (≈20%), tropospheric ozone,[100] cfcs and nitrous oxide are added or removed independently from temperature, and are therefore considered to be external forcings that change global temperatures. False True False False True False False False False False False False False False False 0.4939 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +122 unstructured 15146 15326 wikiClimate_change before the industrial revolution, naturally-occurring amounts of greenhouse gases caused the air near the surface to be about 33 °c warmer than it would have been in their absence. False True False False True False False False False False False False False False False 0.296 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +123 unstructured 15327 15563 wikiClimate_change [102][103] human activity since the industrial revolution, mainly extracting and burning fossil fuels (coal, oil, and natural gas),[104] has increased the amount of greenhouse gases in the atmosphere, resulting in a radiative imbalance. False True False True False False False False False False False False True False False 0.5574 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +124 unstructured 15564 15673 wikiClimate_change in 2019, the concentrations of co2 and methane had increased by about 48% and 160%, respectively, since 1750. False True False True False False False False False False False False True False False 0.5423 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +125 unstructured 15674 15772 wikiClimate_change [105] these co2 levels are higher than they have been at any time during the last 2 million years. False True False False False False False False False False False False True False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +126 unstructured 15773 15857 wikiClimate_change concentrations of methane are far higher than they were over the last 800,000 years. False False False False False False False False False False False False True False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +127 unstructured 15858 15995 wikiClimate_change [106] the global carbon project shows how additions to co2 since 1880 have been caused by different sources ramping up one after another. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +128 unstructured 15997 16095 wikiClimate_change global anthropogenic greenhouse gas emissions in 2019 were equivalent to 59 billion tonnes of co2. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +129 unstructured 16096 16197 wikiClimate_change of these emissions, 75% was co2, 18% was methane, 4% was nitrous oxide, and 2% was fluorinated gases. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +130 unstructured 16198 16332 wikiClimate_change [107] co2 emissions primarily come from burning fossil fuels to provide energy for transport, manufacturing, heating, and electricity. False True False False True False False False False True False False False False False 0.2732 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +131 unstructured 16333 16520 wikiClimate_change [5] additional co2 emissions come from deforestation and industrial processes, which include the co2 released by the chemical reactions for making cement, steel, aluminum, and fertiliser. False True False False False True False True False False False False False False False 0.0 False True True False True False False False False False True False True False False True False False False True False True True False False False True False False False +132 unstructured 16521 16666 wikiClimate_change [108] methane emissions come from livestock, manure, rice cultivation, landfills, wastewater, and coal mining, as well as oil and gas extraction. False True False False False False False False False False False False False False False 0.2732 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +133 unstructured 16667 16757 wikiClimate_change [109] nitrous oxide emissions largely come from the microbial decomposition of fertiliser. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +134 unstructured 16765 16863 wikiClimate_change while methane only lasts in the atmosphere for an average of 12 years,[111] co2 lasts much longer. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +135 unstructured 16864 16924 wikiClimate_change the earth's surface absorbs co2 as part of the carbon cycle. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +136 unstructured 16925 17103 wikiClimate_change while plants on land and in the ocean absorb most excess emissions of co2 every year, that co2 is returned to the atmosphere when biological matter is digested, burns, or decays. False True False False False False False False False True False False False False False -0.3818 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +137 unstructured 17104 17250 wikiClimate_change [112] land-surface carbon sink processes, such as carbon fixation in the soil and photosynthesis, remove about 29% of annual global co2 emissions. False True False False False False False False False True False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +138 unstructured 17251 17329 wikiClimate_change [113] the ocean has absorbed 20 to 30% of emitted co2 over the last 2 decades. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +139 unstructured 17330 17497 wikiClimate_change [114] co2 is only removed from the atmosphere for the long term when it is stored in the earth's crust, which is a process that can take millions of years to complete. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +140 unstructured 17498 17654 wikiClimate_change [112] land surface changes the rate of global tree cover loss has approximately doubled since 2001, to an annual loss approaching an area the size of italy. False False False False False False False False False True False False False False False -0.5574 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +141 unstructured 17662 17862 wikiClimate_change according to food and agriculture organization, around 30% of earth's land area is largely unusable for humans (glaciers, deserts, etc.), 26% is forests, 10% is shrubland and 34% is agricultural land. False False False False False False False False False True False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +142 unstructured 17863 18044 wikiClimate_change [116] deforestation is the main land use change contributor to global warming,[117] as the destroyed trees release co2, and are not replaced by new trees, removing that carbon sink. False True False False True True False True False True False False False False False -0.4939 False False False True True False False False False False True False True False False True False False False True False True True False False False True False False False +143 unstructured 18045 18179 wikiClimate_change [25] between 2001 and 2018, 27% of deforestation was from permanent clearing to enable agricultural expansion for crops and livestock. False True False False False True False True False False False False False False False 0.0 False True True False True False False False False False True False True False False True False False False True False True True False False False True False False False +144 unstructured 18180 18280 wikiClimate_change another 24% has been lost to temporary clearing under the shifting cultivation agricultural systems. False False False False False False False False False False False False False False False -0.3182 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +145 unstructured 18281 18386 wikiClimate_change 26% was due to logging for wood and derived products, and wildfires have accounted for the remaining 23%. True False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True True False False False False False False False True True False False False False False False False False False +146 unstructured 18387 18478 wikiClimate_change [118] some forests have not been fully cleared, but were already degraded by these impacts. False False False False False False False False False True False False False False False -0.6068 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +147 unstructured 18479 18550 wikiClimate_change restoring these forests also recovers their potential as a carbon sink. False False False False False False False False False True False False False False False 0.296 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +148 unstructured 18558 18696 wikiClimate_change local vegetation cover impacts how much of the sunlight gets reflected back into space (albedo), and how much heat is lost by evaporation. False True False False True False False False False False False False False False False -0.3182 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +149 unstructured 18697 18817 wikiClimate_change for instance, the change from a dark forest to grassland makes the surface lighter, causing it to reflect more sunlight. False False False False False False False False False True False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +150 unstructured 18818 18935 wikiClimate_change deforestation can also modify the release of chemical compounds that influence clouds, and by changing wind patterns. True True False False True True False True False False False False False False False 0.0 False True True False True False False False False False True False True False False True False False False True False True True False False False True False False False +151 unstructured 18936 19080 wikiClimate_change [120] in tropic and temperate areas the net effect is to produce significant warming, and forest restoration can make local temperatures cooler. False False False False True False False False False True False False False False False 0.34 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +152 unstructured 19081 19214 wikiClimate_change [119] at latitudes closer to the poles, there is a cooling effect as forest is replaced by snow-covered (and more reflective) plains. True False False False False False False False False True False False False False False 0.0 False True True False True False False False False False True True False False False False False False False True True False False False False False False False False False +153 unstructured 19215 19341 wikiClimate_change [120] globally, these increases in surface albedo have been the dominant direct influence on temperature from land use change. False True False True True False False False False True False False True False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +154 unstructured 19342 19417 wikiClimate_change thus, land use change to date is estimated to have a slight cooling effect. False False False False False False False False False True False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +155 unstructured 19418 19457 wikiClimate_change [121] other factors aerosols and clouds False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +156 unstructured 19459 19536 wikiClimate_change air pollution, in the form of aerosols, affects the climate on a large scale. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +157 unstructured 19537 19587 wikiClimate_change [122] aerosols scatter and absorb solar radiation. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +158 unstructured 19588 19695 wikiClimate_change from 1961 to 1990, a gradual reduction in the amount of sunlight reaching the earth's surface was observed. False False False False False False False False False False False False False False False 0.2023 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +159 unstructured 19696 19905 wikiClimate_change this phenomenon is popularly known as global dimming,[123] and is primarily attributed to sulfate aerosols produced by the combustion of fossil fuels with heavy sulfur concentrations like coal and bunker fuel. False False False False False False False False False False False False False False False 0.6486 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +160 unstructured 19906 20046 wikiClimate_change [46] smaller contributions come from black carbon, organic carbon from combustion of fossil fuels and biofuels, and from anthropogenic dust. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +161 unstructured 20047 20213 wikiClimate_change [124][45][125][126][127] globally, aerosols have been declining since 1990 due to pollution controls, meaning that they no longer mask greenhouse gas warming as much. False True False False True False False False False False False False False False False -0.1531 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +162 unstructured 20225 20290 wikiClimate_change aerosols also have indirect effects on the earth's energy budget. False False False False False False False False False True False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +163 unstructured 20291 20402 wikiClimate_change sulfate aerosols act as cloud condensation nuclei and lead to clouds that have more and smaller cloud droplets. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +164 unstructured 20403 20500 wikiClimate_change these clouds reflect solar radiation more efficiently than clouds with fewer and larger droplets. False False False False False False False False False False False False False False False 0.4576 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +165 unstructured 20501 20605 wikiClimate_change [129] they also reduce the growth of raindrops, which makes clouds more reflective to incoming sunlight. False False False False False False False False False False False False True True False 0.3818 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +166 unstructured 20606 20690 wikiClimate_change [130] indirect effects of aerosols are the largest uncertainty in radiative forcing. False False False False False False False False False False False False False False False -0.34 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +167 unstructured 20698 20848 wikiClimate_change while aerosols typically limit global warming by reflecting sunlight, black carbon in soot that falls on snow or ice can contribute to global warming. True True False False True False False False False False False False False False False 0.296 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +168 unstructured 20849 20950 wikiClimate_change not only does this increase the absorption of sunlight, it also increases melting and sea-level rise. False True False True False False False False False False False False True False False 0.3869 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +169 unstructured 20951 21052 wikiClimate_change [132] limiting new black carbon deposits in the arctic could reduce global warming by 0.2 °c by 2050. False True False False True False False False False False False False False True False 0.1531 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +170 unstructured 21053 21225 wikiClimate_change [133] the effect of decreasing sulfur content of fuel oil for ships since 2020[134] is estimated to cause an additional 0.05 °c increase in global mean temperature by 2050. False False False True True False False False False False False False True True False 0.3182 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +171 unstructured 21226 21473 wikiClimate_change "[135] solar and volcanic activity further information: solar activity and climate the fourth national climate assessment (""nca4"", usgcrp, 2017) includes charts illustrating that neither solar nor volcanic activity can explain the observed warming." False True False False True False False False False False False False False False False 0.1531 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +172 unstructured 21486 21599 wikiClimate_change as the sun is the earth's primary energy source, changes in incoming sunlight directly affect the climate system. False True False False True False False False False True False False False False False 0.2732 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +173 unstructured 21600 21739 wikiClimate_change [131] solar irradiance has been measured directly by satellites,[138] and indirect measurements are available from the early 1600s onwards. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +174 unstructured 21740 21912 wikiClimate_change [131] since 1880, there has been no upward trend in the amount of the sun's energy reaching the earth, in contrast to the warming of the lower atmosphere (the troposphere). False False False False True False False False False True False False False False False 0.0258 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +175 unstructured 21913 22059 wikiClimate_change [139] the upper atmosphere (the stratosphere) would also be warming if the sun was sending more energy to earth, but instead, it has been cooling. False False False False True False False False False True False False False False False 0.2492 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +176 unstructured 22060 22158 wikiClimate_change [92] this is consistent with greenhouse gases preventing heat from leaving the earth's atmosphere. False True False False True False False False False False False False False False False -0.0258 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +177 unstructured 22166 22390 wikiClimate_change explosive volcanic eruptions can release gases, dust and ash that partially block sunlight and reduce temperatures, or they can send water vapor into the atmosphere, which adds to greenhouse gases and increases temperatures. False True False True True False False False False True False False True True False -0.4404 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +178 unstructured 22391 22541 wikiClimate_change [141] these impacts on temperature only last for several years, because both water vapor and volcanic material have low persistence in the atmosphere. False False False False True False False False False True False False False False False -0.2732 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +179 unstructured 22542 22670 wikiClimate_change [142] volcanic co2 emissions are more persistent, but they are equivalent to less than 1% of current human-caused co2 emissions. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +180 unstructured 22671 22793 wikiClimate_change [143] volcanic activity still represents the single largest natural impact (forcing) on temperature in the industrial era. False False False False True False False False False False False False False False False 0.7717 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +181 unstructured 22794 22923 wikiClimate_change yet, like the other natural forcings, it has had negligible impacts on global temperature trends since the industrial revolution. False False False False True False False False False False False False False False False 0.6124 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +182 unstructured 22924 23117 wikiClimate_change [142] climate change feedbacks main articles: climate change feedbacks and climate sensitivity sea ice reflects 50% to 70% of incoming sunlight, while the ocean, being darker, reflects only 6%. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +183 unstructured 23118 23252 wikiClimate_change as an area of sea ice melts and exposes more ocean, more heat is absorbed by the ocean, raising temperatures that melt still more ice. False False False False True False False False False False False False False False False -0.128 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +184 unstructured 23253 23289 wikiClimate_change this is a positive feedback process. False False False False False False False False False False False False False False False 0.5574 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +185 unstructured 23297 23487 wikiClimate_change "the response of the climate system to an initial forcing is modified by feedbacks: increased by ""self-reinforcing"" or ""positive"" feedbacks and reduced by ""balancing"" or ""negative"" feedbacks." False True False True True False False False False False False False True True False 0.25 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +186 unstructured 23488 23610 wikiClimate_change [145] the main reinforcing feedbacks are the water-vapour feedback, the ice-albedo feedback, and the net effect of clouds. False True False False True False False False False True False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +187 unstructured 23611 23758 wikiClimate_change [146][147] the primary balancing mechanism is radiative cooling, as earth's surface gives off more heat to space in response to rising temperature. False False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +188 unstructured 23759 23898 wikiClimate_change [148] in addition to temperature feedbacks, there are feedbacks in the carbon cycle, such as the fertilising effect of co2 on plant growth. False True False False True False False False False False False False True False False 0.3818 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +189 unstructured 23906 24086 wikiClimate_change uncertainty over feedbacks, particularly cloud cover,[150] is the major reason why different climate models project different magnitudes of warming for a given amount of emissions. False True False False True False False False False False False False False False False -0.2023 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +190 unstructured 24087 24133 wikiClimate_change [151] as air warms, it can hold more moisture. False False False False True False False False False False False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +191 unstructured 24134 24205 wikiClimate_change water vapour, as a potent greenhouse gas, holds heat in the atmosphere. False True False False True False False False False True False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +192 unstructured 24206 24306 wikiClimate_change [146] if cloud cover increases, more sunlight will be reflected back into space, cooling the planet. False False False True False False False False False False False False True False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +193 unstructured 24307 24435 wikiClimate_change if clouds become higher and thinner, they act as an insulator, reflecting heat from below back downwards and warming the planet. False False False False True False False False False False False False True False False 0.1531 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +194 unstructured 24443 24578 wikiClimate_change another major feedback is the reduction of snow cover and sea ice in the arctic, which reduces the reflectivity of the earth's surface. True False False False True False False False False False False False False True False 0.0 False True True False True False False False False False True True False False False False False False False True True False False False False False False False False False +195 unstructured 24579 24704 wikiClimate_change [153] more of the sun's energy is now absorbed in these regions, contributing to amplification of arctic temperature changes. False False False False True False False False False True False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +196 unstructured 24705 24811 wikiClimate_change [154] arctic amplification is also thawing permafrost, which releases methane and co2 into the atmosphere. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +197 unstructured 24812 24919 wikiClimate_change [155] climate change can also cause methane releases from wetlands, marine systems, and freshwater systems. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +198 unstructured 24920 24998 wikiClimate_change [156] overall, climate feedbacks are expected to become increasingly positive. False True False True True False False False False False False False True False False 0.5574 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +199 unstructured 25006 25100 wikiClimate_change around half of human-caused co2 emissions have been absorbed by land plants and by the oceans. False True False False False False False False False True False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +200 unstructured 25101 25223 wikiClimate_change [158] this fraction is not static and if future co2 emissions decrease, the earth will be able to absorb up to around 70%. False True False False False False False False False False False False False True False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +201 unstructured 25224 25349 wikiClimate_change if they increase substantially, it'll still absorb more carbon than now, but the overall fraction will decrease to below 40%. False False False True False False False False False False False False True True False 0.1655 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +202 unstructured 25350 25540 wikiClimate_change [159] this is because climate change increases droughts and heat waves that eventually inhibit plant growth on land, and soils will release more carbon from dead plants when they are warmer. True True False True True False False False False True False False True False False -0.5209 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +203 unstructured 25541 25728 wikiClimate_change [160][161] the rate at which oceans absorb atmospheric carbon will be lowered as they become more acidic and experience changes in thermohaline circulation and phytoplankton distribution. False False False False False False False False False False False False False False False -0.128 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +204 unstructured 25729 25881 wikiClimate_change [162][163][73] modelling further information: climate model and climate change scenario energy flows between space, the atmosphere, and earth's surface. False True False False True False False False False True False False False False False 0.2732 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +205 unstructured 25882 26032 wikiClimate_change most sunlight passes through the atmosphere to heat the earth's surface, then greenhouse gases absorb most of the heat the earth radiates in response. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +206 unstructured 26033 26147 wikiClimate_change adding to greenhouse gases increases this insulating effect, causing an energy imbalance that heats the planet up. False True False True True False False False False True False False True False False 0.2732 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +207 unstructured 26149 26267 wikiClimate_change a climate model is a representation of the physical, chemical and biological processes that affect the climate system. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +208 unstructured 26268 26405 wikiClimate_change [164] models include natural processes like changes in the earth's orbit, historical changes in the sun's activity, and volcanic forcing. False False False False False False False False False False False False False False False 0.6124 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +209 unstructured 26406 26544 wikiClimate_change [165] models are used to estimate the degree of warming future emissions will cause when accounting for the strength of climate feedbacks. False True False False True False False False False False False False False False False 0.5859 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +210 unstructured 26545 26708 wikiClimate_change [166][167] models also predict the circulation of the oceans, the annual cycle of the seasons, and the flows of carbon between the land surface and the atmosphere. False False False False False False False False False True False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +211 unstructured 26716 26826 wikiClimate_change the physical realism of models is tested by examining their ability to simulate contemporary or past climates. False False False False True False False False False False False False False False False 0.3182 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +212 unstructured 26827 26953 wikiClimate_change [169] past models have underestimated the rate of arctic shrinkage[170] and underestimated the rate of precipitation increase. True False False True True False False False False False False False True False False -0.2263 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +213 unstructured 26954 27074 wikiClimate_change [171] sea level rise since 1990 was underestimated in older models, but more recent models agree well with observations. False True False True False False False False False False False False False False False 0.6952 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +214 unstructured 27075 27238 wikiClimate_change "[172] the 2017 united states-published national climate assessment notes that ""climate models may still be underestimating or missing relevant feedback processes""." False True False False True False False False False False False False False False False 0.1531 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +215 unstructured 27239 27346 wikiClimate_change [173] additionally, climate models may be unable to adequately predict short-term regional climatic shifts. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +216 unstructured 27354 27430 wikiClimate_change a subset of climate models add societal factors to a physical climate model. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +217 unstructured 27431 27547 wikiClimate_change these models simulate how population, economic growth, and energy use affect-and interact with-the physical climate. False True False False True False False False False True False False True False False 0.5719 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +218 unstructured 27548 27641 wikiClimate_change with this information, these models can produce scenarios of future greenhouse gas emissions. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +219 unstructured 27642 27796 wikiClimate_change this is then used as input for physical climate models and carbon cycle models to predict how atmospheric concentrations of greenhouse gases might change. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +220 unstructured 27797 27966 wikiClimate_change [175][176] depending on the socioeconomic scenario and the mitigation scenario, models produce atmospheric co2 concentrations that range widely between 380 and 1400 ppm. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +221 unstructured 27967 28140 wikiClimate_change [177] impacts main article: effects of climate change the sixth ipcc assessment report projects changes in average soil moisture that can disrupt agriculture and ecosystems. False True False False True False False True False True False True False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +222 unstructured 28141 28315 wikiClimate_change a reduction in soil moisture by one standard deviation means that average soil moisture will approximately match the ninth driest year between 1850 and 1900 at that location. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +223 unstructured 28316 28443 wikiClimate_change environmental effects further information: effects of climate change on oceans and effects of climate change on the water cycle False True False False True False False False False True False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +224 unstructured 28445 28552 wikiClimate_change the environmental effects of climate change are broad and far-reaching, affecting oceans, ice, and weather. True True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +225 unstructured 28553 28592 wikiClimate_change changes may occur gradually or rapidly. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +226 unstructured 28593 28713 wikiClimate_change evidence for these effects comes from studying climate change in the past, from modelling, and from modern observations. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +227 unstructured 28714 28816 wikiClimate_change [178] since the 1950s, droughts and heat waves have appeared simultaneously with increasing frequency. True False False True True False False False False False False False True False False 0.0 False True True False True True False False False False True False True False False False False False False True False True False False False False False False False False +228 unstructured 28817 28915 wikiClimate_change [179] extremely wet or dry events within the monsoon period have increased in india and east asia. True False False True False False False False False False False False True False False 0.2732 True False True False False False False False False False True True False False False False False False False True True False False False False False False False False False +229 unstructured 28916 29000 wikiClimate_change [180] monsoonal precipitation over the northern hemisphere has increased since 1980. True False False True True False False False False False False False True False False 0.2732 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +230 unstructured 29001 29175 wikiClimate_change [181] the rainfall rate and intensity of hurricanes and typhoons is likely increasing,[182] and the geographic range likely expanding poleward in response to climate warming. True True False True True False False False False False False False True False False 0.1531 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +231 unstructured 29176 29261 wikiClimate_change [183] frequency of tropical cyclones has not increased as a result of climate change. True True False True True False False False False False False False True False False -0.2057 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +232 unstructured 29262 29396 wikiClimate_change [184] historical sea level reconstruction and projections up to 2100 published in 2017 by the u.s. global change research program[185] False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +233 unstructured 29398 29506 wikiClimate_change global sea level is rising as a consequence of thermal expansion and the melting of glaciers and ice sheets. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +234 unstructured 29507 29592 wikiClimate_change between 1993 and 2020, the rise increased over time, averaging 3.3 ± 0.3 mm per year. False False False True False False False False False False False False True False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +235 unstructured 29593 29783 wikiClimate_change [186] over the 21st century, the ipcc projects 32-62 cm of sea level rise under a low emission scenario, 44-76 cm under an intermediate one and 65-101 cm under a very high emission scenario. False True False True False False False False False False False True False False False -0.2732 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +236 unstructured 29784 29970 wikiClimate_change [187] marine ice sheet instability processes in antarctica may add substantially to these values,[188] including the possibility of a 2-meter sea level rise by 2100 under high emissions. False True False True True False False False False False False False False False False -0.6249 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +237 unstructured 29978 30060 wikiClimate_change climate change has led to decades of shrinking and thinning of the arctic sea ice. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +238 unstructured 30061 30223 wikiClimate_change [190] while ice-free summers are expected to be rare at 1.5 °c degrees of warming, they are set to occur once every three to ten years at a warming level of 2 °c. False False False False True False False False False False False False False False False 0.296 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +239 unstructured 30224 30343 wikiClimate_change [191] higher atmospheric co2 concentrations cause more co2 to dissolve in the oceans, which is making them more acidic. False True False False False False False False False False False False True False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +240 unstructured 30344 30480 wikiClimate_change [192] because oxygen is less soluble in warmer water,[193] its concentrations in the ocean are decreasing, and dead zones are expanding. False False False False True False False False False True False False False True False -0.5209 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +241 unstructured 30481 30678 wikiClimate_change [194] tipping points and long-term impacts different levels of global warming may cause different parts of earth's climate system to reach tipping points that cause transitions to different states. False True False False True False False False False False False False False False False 0.1779 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +242 unstructured 30679 30740 wikiClimate_change [195][196] main article: tipping points in the climate system False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +243 unstructured 30742 30953 wikiClimate_change greater degrees of global warming increase the risk of passing through 'tipping points'-thresholds beyond which certain major impacts can no longer be avoided even if temperatures return to their previous state. False True False True True False False False False False False False True False False 0.1372 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +244 unstructured 30954 31139 wikiClimate_change [197][198] for instance, the greenland ice sheet is already melting, but if global warming reaches levels between 1.7 °c and 2.3 °c, its melting will continue until it fully disappears. False True False False True False False False False False False False False False False -0.3269 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +245 unstructured 31140 31305 wikiClimate_change if the warming is later reduced to 1.5 °c or less, it will still lose a lot more ice than if the warming was never allowed to reach the threshold in the first place. False False False False True False False False False False False False False True False -0.1466 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +246 unstructured 31306 31440 wikiClimate_change [199] while the ice sheets would melt over millennia, other tipping points would occur faster and give societies less time to respond. False False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +247 unstructured 31441 31658 wikiClimate_change the collapse of major ocean currents like the atlantic meridional overturning circulation (amoc), and irreversible damage to key ecosystems like the amazon rainforest and coral reefs can unfold in a matter of decades. False False False False False False False True False True False False False False False -0.2815 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +248 unstructured 31666 31817 wikiClimate_change the long-term effects of climate change on oceans include further ice melt, ocean warming, sea level rise, ocean acidification and ocean deoxygenation. False True False True True False False False False False False False False False False 0.1531 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +249 unstructured 31818 31925 wikiClimate_change [200] the timescale of long-term impacts are centuries to millennia due to co2's long atmospheric lifetime. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +250 unstructured 31926 32087 wikiClimate_change [201] when net emissions stabilise surface air temperatures will also stabilise, but oceans and ice caps will continue to absorb excess heat from the atmosphere. False False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +251 unstructured 32088 32198 wikiClimate_change the result is an estimated total sea level rise of 2.3 metres per degree celsius (4.2 ft/°f) after 2000 years. False True False True False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +252 unstructured 32199 32318 wikiClimate_change [202] oceanic co2 uptake is slow enough that ocean acidification will also continue for hundreds to thousands of years. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +253 unstructured 32319 32479 wikiClimate_change [203] deep oceans (below 2,000 metres (6,600 ft)) are also already committed to losing over 10% of their dissolved oxygen by the warming which occurred to date. False False False False True False False False False False False False False False False 0.0258 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +254 unstructured 32480 32673 wikiClimate_change [204] further, west antarctic ice sheet appears committed to practically irreversible melting, which would increase the sea levels by at least 3.3 m (10 ft 10 in) over approximately 2000 years. False False False True True False False False False False False False True False False 0.3818 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +255 unstructured 32674 32831 wikiClimate_change [196][205][206][207][208][209][210][211] nature and wildlife further information: effects of climate change on oceans and effects of climate change on biomes False True False False True False False True False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +256 unstructured 32833 32937 wikiClimate_change recent warming has driven many terrestrial and freshwater species poleward and towards higher altitudes. False False False False True False False True False False False False True False False 0.1531 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +257 unstructured 32938 33038 wikiClimate_change [212] higher atmospheric co2 levels and an extended growing season have resulted in global greening. False True False False False False False False False False False False True False False 0.1779 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +258 unstructured 33039 33122 wikiClimate_change however, heatwaves and drought have reduced ecosystem productivity in some regions. True False False False True False False True False True False False False True False 0.0 False True True False True False False False False False True True False False False False False False False True True False False False False False False False False False +259 unstructured 33123 33179 wikiClimate_change the future balance of these opposing effects is unclear. False False False False False False False False False False False False False False False -0.25 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +260 unstructured 33180 33309 wikiClimate_change [213] climate change has contributed to the expansion of drier climate zones, such as the expansion of deserts in the subtropics. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +261 unstructured 33310 33404 wikiClimate_change [214] the size and speed of global warming is making abrupt changes in ecosystems more likely. False True False False True False False True False True False False False False False 0.1531 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +262 unstructured 33405 33501 wikiClimate_change [215] overall, it is expected that climate change will result in the extinction of many species. False True False False True False False True False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +263 unstructured 33509 33662 wikiClimate_change the oceans have heated more slowly than the land, but plants and animals in the ocean have migrated towards the colder poles faster than species on land. False False False False True False False True False True False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +264 unstructured 33663 33824 wikiClimate_change [217] just as on land, heat waves in the ocean occur more frequently due to climate change, harming a wide range of organisms such as corals, kelp, and seabirds. False True False False True False False False True True False False False False False -0.5574 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +265 unstructured 33825 34010 wikiClimate_change [218] ocean acidification makes it harder for marine calcifying organisms such as mussels, barnacles and corals to produce shells and skeletons; and heatwaves have bleached coral reefs. True True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +266 unstructured 34011 34159 wikiClimate_change [219] harmful algal blooms enhanced by climate change and eutrophication lower oxygen levels, disrupt food webs and cause great loss of marine life. False True False False True False False False True False False False True False False 0.1531 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +267 unstructured 34160 34213 wikiClimate_change [220] coastal ecosystems are under particular stress. False False False False False False False True False True False False False False False -0.4215 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +268 unstructured 34214 34308 wikiClimate_change almost half of global wetlands have disappeared due to climate change and other human impacts. False True False False True False False False False False False False False False False -0.2263 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +269 unstructured 34309 34356 wikiClimate_change [221] climate change impacts on the environment False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +270 unstructured 34358 34442 wikiClimate_change underwater photograph of branching coral that is bleached white ecological collapse. False False False False False False False False False False False False False False False -0.4939 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +271 unstructured 34443 34550 wikiClimate_change coral bleaching from thermal stress has damaged the great barrier reef and threatens coral reefs worldwide. False True False False False False False False True False False False False False False -0.5719 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +272 unstructured 34551 34602 wikiClimate_change [222] photograph of evening in a valley settlement. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +273 unstructured 34603 34664 wikiClimate_change the skyline in the hills beyond is lit up red from the fires. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +274 unstructured 34665 34681 wikiClimate_change extreme weather. True False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True True False False False False False False False True True False False False False False False False False False +275 unstructured 34682 34753 wikiClimate_change drought and high temperatures worsened the 2020 bushfires in australia. True False False False True False False False False False False False False False False -0.4404 False False False True True False False False False False True True False False False False False False False True True False False False False False False False False False +276 unstructured 34754 34846 wikiClimate_change [223] the green landscape is interrupted by a huge muddy scar where the ground has subsided. False False False False False False False False False False False False False False False 0.0258 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +277 unstructured 34847 34862 wikiClimate_change arctic warming. False False False False True False False False False False False False False False False 0.1531 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +278 unstructured 34863 34943 wikiClimate_change permafrost thaws undermine infrastructure and release methane, a greenhouse gas. False True False False False False False False False False False False False True False -0.296 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +279 unstructured 34944 35020 wikiClimate_change [155] an emaciated polar bear stands atop the remains of a melting ice floe. False False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +280 unstructured 35021 35041 wikiClimate_change habitat destruction. False True False False False False False True False False False False False False False -0.5719 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +281 unstructured 35042 35127 wikiClimate_change many arctic animals rely on sea ice, which has been disappearing in a warming arctic. False False False False True False False True False False False False False False False 0.1531 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +282 unstructured 35128 35171 wikiClimate_change [224] photograph of a large area of forest. False False False False False False False False False True False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +283 unstructured 35172 35284 wikiClimate_change the green trees are interspersed with large patches of damaged or dead trees turning purple-brown and light red. False False False False False False False False False False False False False False False -0.802 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +284 unstructured 35285 35302 wikiClimate_change pest propagation. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +285 unstructured 35303 35382 wikiClimate_change mild winters allow more pine beetles to survive to kill large swaths of forest. False False False False False False False False False True False False False False False -0.5859 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +286 unstructured 35390 35506 wikiClimate_change humans main article: effects of climate change extreme weather will be progressively more common as the earth warms. True True False False True False False False False False False False False False False 0.2732 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +287 unstructured 35514 35589 wikiClimate_change the effects of climate change are impacting humans everywhere in the world. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +288 unstructured 35590 35727 wikiClimate_change [227] impacts can be observed on all continents and ocean regions,[228] with low-latitude, less developed areas facing the greatest risk. False False False False False False False False False False False False False False False 0.4767 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +289 unstructured 35728 35839 wikiClimate_change "[229] continued warming has potentially ""severe, pervasive and irreversible impacts"" for people and ecosystems." False False False False True False False True False True False False False False False -0.4215 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +290 unstructured 35840 35971 wikiClimate_change [230] the risks are unevenly distributed, but are generally greater for disadvantaged people in developing and developed countries. False False False False False False False False False False False False False False False -0.2144 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +291 unstructured 35972 36138 wikiClimate_change [231] food and health main articles: effects of climate change on agriculture § global food security and undernutrition, and effects of climate change on human health False True False False True False False False False False False False False False False 0.34 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +292 unstructured 36140 36254 wikiClimate_change the world health organization (who) calls climate change the greatest threat to global health in the 21st century. False True False False True False False False True False False False False False False 0.2023 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +293 unstructured 36255 36310 wikiClimate_change [232] extreme weather leads to injury and loss of life. True False False False True False False False False False False False False False False -0.6249 False False False True True False False False False False True True False False False False False False False True True False False False False False False False False False +294 unstructured 36311 36427 wikiClimate_change [233] various infectious diseases are more easily transmitted in a warmer climate, such as dengue fever and malaria. False True False False True False False False False False False False False False False 0.5984 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +295 unstructured 36428 36525 wikiClimate_change [234] crop failures can lead to food shortages and malnutrition, particularly effecting children. False False False False False False False False False False False False False False False -0.5574 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +296 unstructured 36526 36594 wikiClimate_change [235] both children and older people are vulnerable to extreme heat. True False False False True False False False False False False False False False False -0.2263 False False False True True False False False False False True True False False False False False False False True True False False False False False False False False False +297 unstructured 36595 36720 wikiClimate_change [236] the who has estimated that between 2030 and 2050, climate change would cause around 250,000 additional deaths per year. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +298 unstructured 36721 36865 wikiClimate_change they assessed deaths from heat exposure in elderly people, increases in diarrhea, malaria, dengue, coastal flooding, and childhood malnutrition. True False False True True False False False False False False False True False False 0.0 False True True False True True False False False False True False True False False False False False False True False True False False False False False False False False +299 unstructured 36866 37024 wikiClimate_change [237] by 2100, 50% to 75% of the global population may face climate conditions that are life-threatening due to combined effects of extreme heat and humidity. True True False False True False False False True False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +300 unstructured 37032 37074 wikiClimate_change climate change is affecting food security. False True False False True False False False False False False False False False False 0.34 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +301 unstructured 37075 37168 wikiClimate_change it has caused reduction in global yields of maize, wheat, and soybeans between 1981 and 2010. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +302 unstructured 37169 37240 wikiClimate_change [239] future warming could further reduce global yields of major crops. False False False False True False False False False False False False False True False 0.1531 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +303 unstructured 37241 37391 wikiClimate_change [240] crop production will probably be negatively affected in low-latitude countries, while effects at northern latitudes may be positive or negative. False False False False False False False False False False False False False False False -0.1779 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +304 unstructured 37392 37543 wikiClimate_change [241] up to an additional 183 million people worldwide, particularly those with lower incomes, are at risk of hunger as a consequence of these impacts. False False False False False False False False False False False False False False False -0.6771 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +305 unstructured 37544 37595 wikiClimate_change [242] climate change also impacts fish populations. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +306 unstructured 37596 37642 wikiClimate_change globally, less will be available to be fished. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +307 unstructured 37643 37790 wikiClimate_change [243] regions dependent on glacier water, regions that are already dry, and small islands have a higher risk of water stress due to climate change. False True False False True False False False False True False False True False False -0.5994 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +308 unstructured 37791 37901 wikiClimate_change [244] livelihoods and inequality further information: economic analysis of climate change and climate security False True False False True False False False False False False False False False False 0.5106 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +309 unstructured 37903 38005 wikiClimate_change economic damages due to climate change may be severe and there is a chance of disastrous consequences. False True False False True False False False False False False False False False False -0.8126 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +310 unstructured 38006 38173 wikiClimate_change [245] severe impacts are expected in south-east asia and sub-saharan africa, where most of the local inhabitants are dependent upon natural and agricultural resources. False False False False False False False False False True False False False False False -0.0258 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +311 unstructured 38174 38240 wikiClimate_change [246][247] heat stress can prevent outdoor labourers from working. False False False False True False False False False False False False False False False -0.4019 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +312 unstructured 38241 38333 wikiClimate_change if warming reaches 4 °c then labour capacity in those regions could be reduced by 30 to 50%. False False False False True False False False False False False False False True False 0.2023 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +313 unstructured 38334 38484 wikiClimate_change [248] the world bank estimates that between 2016 and 2030, climate change could drive over 120 million people into extreme poverty without adaptation. False True False False True False False False False False False False False False True -0.5563 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +314 unstructured 38492 38575 wikiClimate_change inequalities based on wealth and social status have worsened due to climate change. False True False False True False False False False False False False False False False 0.0772 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +315 unstructured 38576 38729 wikiClimate_change [250] major difficulties in mitigating, adapting, and recovering to climate shocks are faced by marginalised people who have less control over resources. False True False False True False False False False True False False False False False -0.624 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +316 unstructured 38730 38887 wikiClimate_change [251][246] indigenous people, who are subsistent on their land and ecosystems, will face endangerment to their wellness and lifestyles due to climate change. False True False False True False False True False True False False False False False 0.4404 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +317 unstructured 38888 39068 wikiClimate_change [252] an expert elicitation concluded that the role of climate change in armed conflict has been small compared to factors such as socio-economic inequality and state capabilities. False True False False True False True False True False True False False False False -0.3182 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +318 unstructured 39076 39261 wikiClimate_change while women are not inherently more at risk from climate change and shocks, limits on women's resources and discriminatory gender norms constrain their adaptive capacity and resilience. False True False False True False False False False True False False False False True -0.6096 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +319 unstructured 39262 39416 wikiClimate_change [254] for example, women's work burdens, including hours worked in agriculture, tend to decline less than men's during climate shocks such as heat stress. False True False False True False False False False False False False False False False -0.7845 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +320 unstructured 39417 39472 wikiClimate_change [254] climate migration main article: climate migration False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +321 unstructured 39474 39589 wikiClimate_change low-lying islands and coastal communities are threatened by sea level rise, which makes urban flooding more common. True True False True True False False False True False False False False False False -0.4588 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +322 unstructured 39590 39637 wikiClimate_change sometimes, land is permanently lost to the sea. False False False False False False False False False True False False False False False -0.3182 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +323 unstructured 39638 39739 wikiClimate_change [255] this could lead to statelessness for people in island nations, such as the maldives and tuvalu. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +324 unstructured 39740 39841 wikiClimate_change [256] in some regions, the rise in temperature and humidity may be too severe for humans to adapt to. False True False True True False False False False False False False False False False -0.3818 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +325 unstructured 39842 39996 wikiClimate_change [257] with worst-case climate change, models project that almost one-third of humanity might live in sahara-like uninhabitable and extremely hot climates. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +326 unstructured 40004 40093 wikiClimate_change these factors can drive climate or environmental migration, within and between countries. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +327 unstructured 40094 40246 wikiClimate_change [11] more people are expected to be displaced because of sea level rise, extreme weather and conflict from increased competition over natural resources. True True False True True False False False True True True False True False False 0.2585 True False True False False True False True False False True False True False False False False False False True False True False False False False False False False False +328 unstructured 40247 40380 wikiClimate_change "climate change may also increase vulnerability, leading to ""trapped populations"" who are not able to move due to a lack of resources." False True False True True False False False False True False False True False False -0.6486 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +329 unstructured 40381 40419 wikiClimate_change [259] climate change impacts on people False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +330 unstructured 40421 40445 wikiClimate_change environmental migration. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +331 unstructured 40446 40540 wikiClimate_change sparser rainfall leads to desertification that harms agriculture and can displace populations. True True False False True False False False True False False False False False False -0.4939 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +332 unstructured 40541 40567 wikiClimate_change shown: telly, mali (2008). False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +333 unstructured 40568 40598 wikiClimate_change [260] environmental migration. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +334 unstructured 40599 40693 wikiClimate_change sparser rainfall leads to desertification that harms agriculture and can displace populations. True True False False True False False False True False False False False False False -0.4939 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +335 unstructured 40694 40720 wikiClimate_change shown: telly, mali (2008). False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +336 unstructured 40721 40748 wikiClimate_change [260] agricultural changes. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +337 unstructured 40749 40830 wikiClimate_change droughts, rising temperatures, and extreme weather negatively impact agriculture. True False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True True False False False False False False False True True False False False False False False False False False +338 unstructured 40831 40855 wikiClimate_change shown: texas, us (2013). False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +339 unstructured 40856 40883 wikiClimate_change [261] agricultural changes. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +340 unstructured 40884 40965 wikiClimate_change droughts, rising temperatures, and extreme weather negatively impact agriculture. True False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True True False False False False False False False True True False False False False False False False False False +341 unstructured 40966 40990 wikiClimate_change shown: texas, us (2013). False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +342 unstructured 40991 41012 wikiClimate_change [261] tidal flooding. True False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True True False False False False False False False True True False False False False False False False False False +343 unstructured 41013 41076 wikiClimate_change sea-level rise increases flooding in low-lying coastal regions. True True False True True False False False False False False False True False False 0.0 False True True False True True False False False False True False True False False False False False False True False True False False False False False False False False +344 unstructured 41077 41105 wikiClimate_change shown: venice, italy (2004). False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +345 unstructured 41106 41127 wikiClimate_change [262] tidal flooding. True False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True True False False False False False False False True True False False False False False False False False False +346 unstructured 41128 41191 wikiClimate_change sea-level rise increases flooding in low-lying coastal regions. True True False True True False False False False False False False True False False 0.0 False True True False True True False False False False True False True False False False False False False True False True False False False False False False False False +347 unstructured 41192 41220 wikiClimate_change shown: venice, italy (2004). False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +348 unstructured 41221 41249 wikiClimate_change [262] storm intensification. True False False True True False False False False False False False False False False 0.0 False True True False True True False False False False True False True False False False False False False True False True False False False False False False False False +349 unstructured 41250 41350 wikiClimate_change bangladesh after cyclone sidr (2007) is an example of catastrophic flooding from increased rainfall. True False False True True False False False False False False False True False False -0.2732 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +350 unstructured 41351 41379 wikiClimate_change [263] storm intensification. True False False True True False False False False False False False False False False 0.0 False True True False True True False False False False True False True False False False False False False True False True False False False False False False False False +351 unstructured 41380 41480 wikiClimate_change bangladesh after cyclone sidr (2007) is an example of catastrophic flooding from increased rainfall. True False False True True False False False False False False False True False False -0.2732 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +352 unstructured 41481 41513 wikiClimate_change [263] heat wave intensification. False False False True True False False False False False False False False False False 0.0 False True True False True True False False False False True False True False False False False False False True False True False False False False False False False False +353 unstructured 41514 41584 wikiClimate_change events like the 2022 southern cone heat wave are becoming more common. False False False False True False False False False False False False False False False 0.3612 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +354 unstructured 41585 41617 wikiClimate_change [264] heat wave intensification. False False False True True False False False False False False False False False False 0.0 False True True False True True False False False False True False True False False False False False False True False True False False False False False False False False +355 unstructured 41618 41688 wikiClimate_change events like the 2022 southern cone heat wave are becoming more common. False False False False True False False False False False False False False False False 0.3612 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +356 unstructured 41696 41862 wikiClimate_change reducing and recapturing emissions main article: climate change mitigation global greenhouse gas emission scenarios, based on policies and pledges as of november 2021 False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +357 unstructured 41864 42058 wikiClimate_change climate change can be mitigated by reducing the rate at which greenhouse gases are emitted into the atmosphere, and by increasing the rate at which carbon dioxide is removed from the atmosphere. False True False True True False False False False False False False True False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +358 unstructured 42059 42210 wikiClimate_change [265] in order to limit global warming to less than 1.5 °c global greenhouse gas emissions needs to be net-zero by 2050, or by 2070 with a 2 °c target. False True False False True False False False False False False False False False False 0.1531 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +359 unstructured 42211 42347 wikiClimate_change [85] this requires far-reaching, systemic changes on an unprecedented scale in energy, land, cities, transport, buildings, and industry. False False False False False False False False False True False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +360 unstructured 42355 42531 wikiClimate_change the united nations environment programme estimates that countries need to triple their pledges under the paris agreement within the next decade to limit global warming to 2 °c. False True False False True False False False False False False False False False False 0.765 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +361 unstructured 42532 42603 wikiClimate_change an even greater level of reduction is required to meet the 1.5 °c goal. False False False False False False False False False False False False False False False 0.3612 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +362 unstructured 42604 42790 wikiClimate_change [267] with pledges made under the paris agreement as of october 2021, global warming would still have a 66% chance of reaching about 2.7 °c (range: 2.2-3.2 °c) by the end of the century. False True False False True False False False False False False False False False False 0.765 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +363 unstructured 42791 42890 wikiClimate_change [20] globally, limiting warming to 2 °c may result in higher economic benefits than economic costs. False False False False True False False False False False False False True False False 0.4939 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +364 unstructured 42898 43167 wikiClimate_change although there is no single pathway to limit global warming to 1.5 or 2 °c,[269] most scenarios and strategies see a major increase in the use of renewable energy in combination with increased energy efficiency measures to generate the needed greenhouse gas reductions. False True False True True False False False False True False False True False False 0.8313 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +365 unstructured 43168 43410 wikiClimate_change [270] to reduce pressures on ecosystems and enhance their carbon sequestration capabilities, changes would also be necessary in agriculture and forestry,[271] such as preventing deforestation and restoring natural ecosystems by reforestation. False True False False False True False True False True False False True True False 0.3182 True False True False False False False False False False True False True False False True False False False True False True True False False False True False False False +366 unstructured 43418 43492 wikiClimate_change other approaches to mitigating climate change have a higher level of risk. False True False False True False False False False False False False True False False -0.2732 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +367 unstructured 43493 43633 wikiClimate_change scenarios that limit global warming to 1.5 °c typically project the large-scale use of carbon dioxide removal methods over the 21st century. False True False False True False False False False False False False False False False 0.1531 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +368 unstructured 43634 43737 wikiClimate_change [273] there are concerns, though, about over-reliance on these technologies, and environmental impacts. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +369 unstructured 43738 43841 wikiClimate_change [274] solar radiation modification (srm) is also a possible supplement to deep reductions in emissions. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +370 unstructured 43842 43943 wikiClimate_change however, srm raises significant ethical and legal concerns, and the risks are imperfectly understood. False False False False False False False False False False False False False False False 0.5423 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +371 unstructured 43944 44140 wikiClimate_change [275] clean energy main articles: sustainable energy and sustainable transport coal, oil, and natural gas remain the primary global energy sources even as renewables have begun rapidly increasing. False False False True False False False False False True False False True False False 0.8591 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +372 unstructured 44141 44176 wikiClimate_change [276] wind and solar power, germany True False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True True False False False False False False False True True False False False False False False False False False +373 unstructured 44178 44229 wikiClimate_change renewable energy is key to limiting climate change. False True False False True False False False False True False False False False False 0.2732 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +374 unstructured 44230 44319 wikiClimate_change [277] for decades, fossil fuels have accounted for roughly 80% of the world's energy use. False False False False False False False False False True False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +375 unstructured 44320 44476 wikiClimate_change [278] the remaining share has been split between nuclear power and renewables (including hydropower, bioenergy, wind and solar power and geothermal energy). True False False False True False False False False True False False False False False 0.5106 True False True False False False False False False False True True False False False False False False False True True False False False False False False False False False +376 unstructured 44477 44623 wikiClimate_change [279] fossil fuel use is expected to peak in absolute terms prior to 2030 and then to decline, with coal use experiencing the sharpest reductions. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +377 unstructured 44624 44736 wikiClimate_change [280] renewables represented 75% of all new electricity generation installed in 2019, nearly all solar and wind. True False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True True False False False False False False False True True False False False False False False False False False +378 unstructured 44737 44855 wikiClimate_change [281] other forms of clean energy, such as nuclear and hydropower, currently have a larger share of the energy supply. False False False False False False False False False True False False False False False 0.7964 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +379 unstructured 44856 44924 wikiClimate_change however, their future growth forecasts appear limited in comparison. False False False False False False False False False False False False True False False 0.1779 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +380 unstructured 44932 45160 wikiClimate_change while solar panels and onshore wind are now among the cheapest forms of adding new power generation capacity in many locations,[283] green energy policies are needed to achieve a rapid transition from fossil fuels to renewables. True False False False True False False False False True False False False False False 0.2732 True False True False False False False False False False True True False False False False False False False True True False False False False False False False False False +381 unstructured 45161 45328 wikiClimate_change [284] to achieve carbon neutrality by 2050, renewable energy would become the dominant form of electricity generation, rising to 85% or more by 2050 in some scenarios. False False False False False False False False False True False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +382 unstructured 45329 45407 wikiClimate_change investment in coal would be eliminated and coal use nearly phased out by 2050. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +383 unstructured 45420 45540 wikiClimate_change electricity generated from renewable sources would also need to become the main energy source for heating and transport. False False False False True False False False False True False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +384 unstructured 45541 45704 wikiClimate_change [287] transport can switch away from internal combustion engine vehicles and towards electric vehicles, public transit, and active transport (cycling and walking). False False False False False False False False False False False False False False False 0.4019 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +385 unstructured 45705 45781 wikiClimate_change [288][289] for shipping and flying, low-carbon fuels would reduce emissions. False False False False False False False False False False False False False True False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +386 unstructured 45782 45865 wikiClimate_change [288] heating could be increasingly decarbonised with technologies like heat pumps. False False False True True False False False False False False False True False False 0.3612 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +387 unstructured 45873 45961 wikiClimate_change there are obstacles to the continued rapid growth of clean energy, including renewables. False False False False False False False False False True False False True False False 0.5859 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +388 unstructured 45962 46045 wikiClimate_change for wind and solar, there are environmental and land use concerns for new projects. True False False False True False False False False True False False False False False 0.0 False True True False True False False False False False True True False False False False False False False True True False False False False False False False False False +389 unstructured 46046 46132 wikiClimate_change [291] wind and solar also produce energy intermittently and with seasonal variability. True False False False True False False False False True False False False False False 0.2732 True False True False False False False False False False True True False False False False False False False True True False False False False False False False False False +390 unstructured 46133 46259 wikiClimate_change traditionally, hydro dams with reservoirs and conventional power plants have been used when variable energy production is low. False False False False False False False False False True False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +391 unstructured 46260 46424 wikiClimate_change going forward, battery storage can be expanded, energy demand and supply can be matched, and long-distance transmission can smooth variability of renewable outputs. False False False False False False False False False True False False False False False 0.1531 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +392 unstructured 46425 46522 wikiClimate_change [277] bioenergy is often not carbon-neutral and may have negative consequences for food security. False False False False False False False False False False False False False False False -0.3182 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +393 unstructured 46523 46657 wikiClimate_change [292] the growth of nuclear power is constrained by controversy around radioactive waste, nuclear weapon proliferation, and accidents. False False False False False False True False False False False False True False False -0.6249 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +394 unstructured 46658 46828 wikiClimate_change [293][294] hydropower growth is limited by the fact that the best sites have been developed, and new projects are confronting increased social and environmental concerns. False False False True False False False False True False False False True False False 0.7506 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +395 unstructured 46836 47003 wikiClimate_change low-carbon energy improves human health by minimising climate change as well as reducing air pollution deaths,[296] which were estimated at 7 million annually in 2016. False True False False True False False False False True False False True False False 0.7184 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +396 unstructured 47004 47267 wikiClimate_change [297] meeting the paris agreement goals that limit warming to a 2 °c increase could save about a million of those lives per year by 2050, whereas limiting global warming to 1.5 °c could save millions and simultaneously increase energy security and reduce poverty. False True False True True False False False False True False False True True False 0.9393 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +397 unstructured 47268 47365 wikiClimate_change [298] improving air quality also has economic benefits which may be larger than mitigation costs. False False False False False False False False False False False False True False False 0.6597 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +398 unstructured 47366 47451 wikiClimate_change [299] energy conservation main articles: efficient energy use and energy conservation False False False False False False False True False True False False False False False 0.7964 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +399 unstructured 47453 47522 wikiClimate_change reducing energy demand is another major aspect of reducing emissions. False False False False False False False False False True False False False False False 0.1531 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +400 unstructured 47523 47610 wikiClimate_change [300] if less energy is needed, there is more flexibility for clean energy development. False False False False False False False False False True False False False False False 0.8207 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +401 unstructured 47611 47725 wikiClimate_change it also makes it easier to manage the electricity grid, and minimises carbon-intensive infrastructure development. False False False False False False False False False False False False False False False 0.4215 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +402 unstructured 47726 47881 wikiClimate_change [301] major increases in energy efficiency investment will be required to achieve climate goals, comparable to the level of investment in renewable energy. False True False True True False False False False True False False True False False 0.7152 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +403 unstructured 47882 48053 wikiClimate_change [302] several covid-19 related changes in energy use patterns, energy efficiency investments, and funding have made forecasts for this decade more difficult and uncertain. False False False False False False False False False True False False False False False 0.1137 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +404 unstructured 48061 48111 wikiClimate_change strategies to reduce energy demand vary by sector. False False False False False False False False False True False False False True False 0.1531 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +405 unstructured 48112 48254 wikiClimate_change in the transport sector, passengers and freight can switch to more efficient travel modes, such as buses and trains, or use electric vehicles. False False False False False False False False False False False False False False False 0.4754 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +406 unstructured 48255 48428 wikiClimate_change [304] industrial strategies to reduce energy demand include improving heating systems and motors, designing less energy-intensive products, and increasing product lifetimes. False False False True True False False False False True False False True True False 0.5267 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +407 unstructured 48429 48561 wikiClimate_change [305] in the building sector the focus is on better design of new buildings, and higher levels of energy efficiency in retrofitting. False False False False False False False False False True False False True False False 0.7579 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +408 unstructured 48562 48653 wikiClimate_change [306] the use of technologies like heat pumps can also increase building energy efficiency. False False False True True False False False False True False False True False False 0.8126 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +409 unstructured 48654 48864 wikiClimate_change [307] agriculture and industry see also: sustainable agriculture and green industrial policy taking into account direct and indirect emissions, industry is the sector with the highest share of global emissions. False False False False False False False False False False False False False False False 0.296 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +410 unstructured 48865 48895 wikiClimate_change data as of 2019 from the ipcc. False False False False False False False False False False False True False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +411 unstructured 48897 49094 wikiClimate_change agriculture and forestry face a triple challenge of limiting greenhouse gas emissions, preventing the further conversion of forests to agricultural land, and meeting increases in world food demand. False True False True False False False False False True False False True False False -0.0772 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +412 unstructured 49095 49203 wikiClimate_change [308] a set of actions could reduce agriculture and forestry-based emissions by two thirds from 2010 levels. False False False False False False False False False True False False False True False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +413 unstructured 49204 49417 wikiClimate_change these include reducing growth in demand for food and other agricultural products, increasing land productivity, protecting and restoring forests, and reducing greenhouse gas emissions from agricultural production. False True False True False False False False False True False False True False False 0.5106 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +414 unstructured 49425 49528 wikiClimate_change on the demand side, a key component of reducing emissions is shifting people towards plant-based diets. False False False False False False False False False False False False False False False -0.128 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +415 unstructured 49529 49676 wikiClimate_change [310] eliminating the production of livestock for meat and dairy would eliminate about 3/4ths of all emissions from agriculture and other land use. False False False False False False False False False True False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +416 unstructured 49677 49842 wikiClimate_change [311] livestock also occupy 37% of ice-free land area on earth and consume feed from the 12% of land area used for crops, driving deforestation and land degradation. False True False False True True False True False True False False False False False -0.5267 False False False True True False False False False False True False True False False True False False False True False True True False False False True False False False +417 unstructured 49850 49936 wikiClimate_change steel and cement production are responsible for about 13% of industrial co2 emissions. False True False False False False False False False False False False False False False 0.3182 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +418 unstructured 49937 50126 wikiClimate_change in these industries, carbon-intensive materials such as coke and lime play an integral role in the production, so that reducing co2 emissions requires research into alternative chemistries. False True False False False False False False False False False False False False False 0.34 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +419 unstructured 50127 50354 wikiClimate_change [313] carbon sequestration main articles: carbon dioxide removal and carbon sequestration most co2 emissions have been absorbed by carbon sinks, including plant growth, soil uptake, and ocean uptake (2020 global carbon budget). False True False False False False False False False False False False True False False 0.3818 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +420 unstructured 50356 50476 wikiClimate_change natural carbon sinks can be enhanced to sequester significantly larger amounts of co2 beyond naturally occurring levels. False True False False False False False False False False False False True False False 0.3612 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +421 unstructured 50477 50665 wikiClimate_change [314] reforestation and afforestation (planting forests where there were none before) are among the most mature sequestration techniques, although the latter raises food security concerns. False False False False False False False False False True False False False False False 0.6697 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +422 unstructured 50666 50875 wikiClimate_change [315] farmers can promote sequestration of carbon in soils through practices such as use of winter cover crops, reducing the intensity and frequency of tillage, and using compost and manure as soil amendments. False False False False False False False False False False False False False False False 0.3818 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +423 unstructured 50876 51016 wikiClimate_change [316] forest and landscape restoration yields many benefits for the climate, including greenhouse gas emissions sequestration and reduction. False True False False True False False False False True False False False False False 0.3818 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +424 unstructured 51017 51153 wikiClimate_change [119] restoration/recreation of coastal wetlands, prairie plots and seagrass meadows increases the uptake of carbon into organic matter. False False False True False False False False False False False False True False False 0.0258 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +425 unstructured 51154 51378 wikiClimate_change [317][318] when carbon is sequestered in soils and in organic matter such as trees, there is a risk of the carbon being re-released into the atmosphere later through changes in land use, fire, or other changes in ecosystems. False False False False False False False True False True False False False False False -0.5267 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +426 unstructured 51386 51548 wikiClimate_change where energy production or co2-intensive heavy industries continue to produce waste co2, the gas can be captured and stored instead of released to the atmosphere. False True False False False False False False False True False False False False False -0.1779 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +427 unstructured 51549 51727 wikiClimate_change although its current use is limited in scale and expensive,[320] carbon capture and storage (ccs) may be able to play a significant role in limiting co2 emissions by mid-century. False True False False False False False False False False False False False False False 0.3182 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +428 unstructured 51728 51861 wikiClimate_change [321] this technique, in combination with bioenergy (beccs) can result in net negative emissions as co2 is drawn from the atmosphere. False True False False False False False False False False False False False False False -0.5719 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +429 unstructured 51862 52002 wikiClimate_change [322] it remains highly uncertain whether carbon dioxide removal techniques will be able to play a large role in limiting warming to 1.5 °c. False True False False True False False False False False False False False False False 0.1298 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +430 unstructured 52003 52126 wikiClimate_change policy decisions that rely on carbon dioxide removal increase the risk of global warming rising beyond international goals. False True False True True False False False False False False False True False False 0.2023 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +431 unstructured 52127 52183 wikiClimate_change [323] adaptation main article: climate change adaptation False True False False True False False False False False False False False False True 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +432 unstructured 52185 52285 wikiClimate_change "adaptation is ""the process of adjustment to current or expected changes in climate and its effects""." False True False False True False False False False False False False False False True 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +433 unstructured 52286 52408 wikiClimate_change "[324]: 5 without additional mitigation, adaptation cannot avert the risk of ""severe, widespread and irreversible"" impacts." False False False False False False False False False False False False False False True -0.2658 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +434 unstructured 52409 52520 wikiClimate_change [325] more severe climate change requires more transformative adaptation, which can be prohibitively expensive. False True False False True False False False False False False False False False True -0.4391 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +435 unstructured 52521 52685 wikiClimate_change [326] the capacity and potential for humans to adapt is unevenly distributed across different regions and populations, and developing countries generally have less. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +436 unstructured 52686 52891 wikiClimate_change [327] the first two decades of the 21st century saw an increase in adaptive capacity in most low- and middle-income countries with improved access to basic sanitation and electricity, but progress is slow. False False False True False False False False False False False False True False True 0.6911 True False True False False False False False True False True False False False False False False True False False False False False False False False False False True False +437 unstructured 52892 52944 wikiClimate_change many countries have implemented adaptation policies. False False False False False False False False False False False False False False True 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +438 unstructured 52945 53022 wikiClimate_change however, there is a considerable gap between necessary and available finance. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +439 unstructured 53030 53165 wikiClimate_change adaptation to sea level rise consists of avoiding at-risk areas, learning to live with increased flooding, and building flood controls. True True False True True False False False False False False False True False True -0.0772 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +440 unstructured 53166 53211 wikiClimate_change if that fails, managed retreat may be needed. False False False False False False False False False False False False False False False -0.25 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +441 unstructured 53212 53281 wikiClimate_change [329] there are economic barriers for tackling dangerous heat impact. False False False False True False False False False False False False False False False -0.4767 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +442 unstructured 53282 53363 wikiClimate_change avoiding strenuous work or having air conditioning is not possible for everybody. False False False False False False False False False False False False False False False -0.34 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +443 unstructured 53364 53554 wikiClimate_change [330] in agriculture, adaptation options include a switch to more sustainable diets, diversification, erosion control, and genetic improvements for increased tolerance to a changing climate. False True False True True False False False False False False False True False True 0.6808 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +444 unstructured 53555 53654 wikiClimate_change [331] insurance allows for risk-sharing, but is often difficult to get for people on lower incomes. False False False False False False False False False False False False False False False -0.7227 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +445 unstructured 53655 53741 wikiClimate_change [332] education, migration and early warning systems can reduce climate vulnerability. False True False False True False False False False False False False False True False -0.5106 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +446 unstructured 53742 53825 wikiClimate_change [333] planting mangroves or encouraging other coastal vegetation can buffer storms. True False False False True False False False False False False False False False False 0.5267 True False True False False False False False False False True True False False False False False False False True True False False False False False False False False False +447 unstructured 53838 53928 wikiClimate_change ecosystems adapt to climate change, a process that can be supported by human intervention. False True False False True False False True False True False False False False False 0.3182 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +448 unstructured 53929 54034 wikiClimate_change by increasing connectivity between ecosystems, species can migrate to more favourable climate conditions. False True False True True False False True False True False False True False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +449 unstructured 54035 54105 wikiClimate_change species can also be introduced to areas acquiring a favorable climate. False True False False True False False True False False False False False False False 0.4767 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +450 unstructured 54106 54232 wikiClimate_change protection and restoration of natural and semi-natural areas helps build resilience, making it easier for ecosystems to adapt. False False False False False False False True False True False False False False True 0.7845 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +451 unstructured 54233 54346 wikiClimate_change many of the actions that promote adaptation in ecosystems, also help humans adapt via ecosystem-based adaptation. False False False False False False False True False True False False False False True 0.6486 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +452 unstructured 54347 54462 wikiClimate_change for instance, restoration of natural fire regimes makes catastrophic fires less likely, and reduces human exposure. False False False False False False False False False False False False False True False -0.4767 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +453 unstructured 54463 54561 wikiClimate_change giving rivers more space allows for more water storage in the natural system, reducing flood risk. True False False False True False False False False True False False False False False 0.4215 True False True False False False False False False False True True False False False False False False False True True False False False False False False False False False +454 unstructured 54562 54673 wikiClimate_change restored forest acts as a carbon sink, but planting trees in unsuitable regions can exacerbate climate impacts. False True False True True False False False False True False False False False False 0.1779 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +455 unstructured 54681 54755 wikiClimate_change there are synergies but also trade-offs between adaptation and mitigation. False False False False False False False False False False False False False False True 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +456 unstructured 54756 54877 wikiClimate_change [337] an example for synergy is increased food productivity, which has large benefits for both adaptation and mitigation. False False False True False False False False False False False False True False True 0.5719 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +457 unstructured 54878 55020 wikiClimate_change [338] an example of a trade-off is that increased use of air conditioning allows people to better cope with heat, but increases energy demand. False False False True True False False False False True False False True False False 0.5267 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +458 unstructured 55021 55237 wikiClimate_change another trade-off example is that more compact urban development may reduce emissions from transport and construction, but may also increase the urban heat island effect, exposing people to heat-related health risks. False False False True True False False False False False False False True True False -0.3291 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +459 unstructured 55238 55274 wikiClimate_change [339] examples of adaptation methods False False False False False False False False False False False False False False True 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +460 unstructured 55276 55353 wikiClimate_change mangrove planting and other habitat conservation can reduce coastal flooding. True False False False True False False True False False False False False True False 0.0 False True True False True False False False False False True True False False False False False False False True True False False False False False False False False False +461 unstructured 55354 55431 wikiClimate_change mangrove planting and other habitat conservation can reduce coastal flooding. True False False False True False False True False False False False False True False 0.0 False True True False True False False False False False True True False False False False False False False True True False False False False False False False False False +462 unstructured 55432 55741 wikiClimate_change seawalls to protect against storm surge worsened by sea level rise seawalls to protect against storm surge worsened by sea level rise green roofs to provide cooling in cities green roofs to provide cooling in cities selective breeding for drought-resistant crops selective breeding for drought-resistant crops True True False True True False False False False False False False False False False -0.1531 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +463 unstructured 55743 56007 wikiClimate_change policies and politics see also: politics of climate change and climate change mitigation § policies the climate change performance index ranks countries by greenhouse gas emissions (40% of score), renewable energy (20%), energy use (20%), and climate policy (20%). False True False False True False False False False True False False False False False 0.4939 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +464 unstructured 56008 56012 wikiClimate_change high False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +465 unstructured 56014 56020 wikiClimate_change medium False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +466 unstructured 56022 56025 wikiClimate_change low False False False False False False False False False False False False False False False -0.2732 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +467 unstructured 56027 56035 wikiClimate_change very low False False False False False False False False False False False False False False False -0.3384 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +468 unstructured 56037 56160 wikiClimate_change countries that are most vulnerable to climate change have typically been responsible for a small share of global emissions. False True False False True False False False False False False False False False False 0.3197 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +469 unstructured 56161 56210 wikiClimate_change this raises questions about justice and fairness. False False False False False False False False False False False False False False False 0.5267 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +470 unstructured 56211 56367 wikiClimate_change [340] limiting global warming makes it much easier to achieve the un's sustainable development goals, such as eradicating poverty and reducing inequalities. False True False False True False False False False False False False False False False 0.0258 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +471 unstructured 56368 56506 wikiClimate_change "the connection is recognised in sustainable development goal 13 which is to ""take urgent action to combat climate change and its impacts""." False True False False True False False False False False False False False False False -0.1531 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +472 unstructured 56507 56608 wikiClimate_change [341] the goals on food, clean water and ecosystem protection have synergies with climate mitigation. False True False False True False False True False True False False False False False 0.4019 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +473 unstructured 56616 56661 wikiClimate_change the geopolitics of climate change is complex. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +474 unstructured 56662 56870 wikiClimate_change it has often been framed as a free-rider problem, in which all countries benefit from mitigation done by other countries, but individual countries would lose from switching to a low-carbon economy themselves. False False False False False False False False False False False False False False False -0.5267 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +475 unstructured 56871 56927 wikiClimate_change sometimes mitigation also has localised benefits though. False False False False False False False False False False False False False False False 0.3818 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +476 unstructured 56928 57054 wikiClimate_change for instance, the benefits of a coal phase-out to public health and local environments exceed the costs in almost all regions. False False False False False False False False False False False False False False False 0.3818 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +477 unstructured 57055 57230 wikiClimate_change [343] furthermore, net importers of fossil fuels win economically from switching to clean energy, causing net exporters to face stranded assets: fossil fuels they cannot sell. False False False False False False False False False True False False False False False 0.8519 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +478 unstructured 57231 57251 wikiClimate_change [344] policy options False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +479 unstructured 57253 57336 wikiClimate_change a wide range of policies, regulations, and laws are being used to reduce emissions. False False False False False False False False False False False False False True False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +480 unstructured 57337 57416 wikiClimate_change as of 2019, carbon pricing covers about 20% of global greenhouse gas emissions. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +481 unstructured 57417 57492 wikiClimate_change [345] carbon can be priced with carbon taxes and emissions trading systems. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +482 unstructured 57493 57639 wikiClimate_change [346] direct global fossil fuel subsidies reached $319 billion in 2017, and $5.2 trillion when indirect costs such as air pollution are priced in. False False False False False False False False False False False False False False False 0.1027 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +483 unstructured 57640 57756 wikiClimate_change [347] ending these can cause a 28% reduction in global carbon emissions and a 46% reduction in air pollution deaths. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +484 unstructured 57757 57859 wikiClimate_change [348] money saved on fossil subsidies could be used to support the transition to clean energy instead. False False False False False False False False False True False False False False False 0.8519 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +485 unstructured 57860 58025 wikiClimate_change [349] more direct methods to reduce greenhouse gases include vehicle efficiency standards, renewable fuel standards, and air pollution regulations on heavy industry. False True False False False False False False False False False False False True False 0.3612 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +486 unstructured 58026 58124 wikiClimate_change [350] several countries require utilities to increase the share of renewables in power production. False False False True False False False False False False False False True False False 0.5423 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +487 unstructured 58125 58146 wikiClimate_change [351] climate justice False True False False True False False False False False False False False False False 0.5267 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +488 unstructured 58148 58259 wikiClimate_change policy designed through the lens of climate justice tries to address human rights issues and social inequality. False True False False True False False False False False False False False False False 0.5267 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +489 unstructured 58260 58467 wikiClimate_change according to proponents of climate justice, the costs of climate adaptation should be paid by those most responsible for climate change, while the beneficiaries of payments should be those suffering impacts. False True False False True False False False False False False False False False True 0.6901 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +490 unstructured 58468 58567 wikiClimate_change one way this can be addressed in practice is to have wealthy nations pay poorer countries to adapt. False False False False False False False False False False False False False False False -0.1027 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +491 unstructured 58575 58725 wikiClimate_change oxfam found that in 2023 the wealthiest 10% of people were responsible for 50% of global emissions, while the bottom 50% were responsible for just 8%. False False False False False False False False False False False False False False False 0.7783 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +492 unstructured 58726 58936 wikiClimate_change [353] production of emissions is another way to look at responsibility: under that approach, the top 21 fossil fuel companies would owe cumulative climate reparations of $5.4 trillion over the period 2025-2050. False True False False True False False False False False False False False False False 0.2023 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +493 unstructured 58937 59087 wikiClimate_change [354] to achieve a just transition, people working in the fossil fuel sector would also need other jobs, and their communities would need investments. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +494 unstructured 59088 59324 wikiClimate_change [355] international climate agreements further information: united nations framework convention on climate change since 2000, rising co2 emissions in china and the rest of world have surpassed the output of the united states and europe. False True False False True False False False False False False False False False False 0.7717 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +495 unstructured 59325 59423 wikiClimate_change [356] per person, the united states generates co2 at a far faster rate than other primary regions. False True False False False False False False False False False False False False False 0.4215 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +496 unstructured 59431 59552 wikiClimate_change nearly all countries in the world are parties to the 1994 united nations framework convention on climate change (unfccc). False True False False True False False False False False False False False False False 0.6705 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +497 unstructured 59553 59649 wikiClimate_change [357] the goal of the unfccc is to prevent dangerous human interference with the climate system. False True False False True False False False False False False False False False False -0.4588 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +498 unstructured 59650 59911 wikiClimate_change [358] as stated in the convention, this requires that greenhouse gas concentrations are stabilised in the atmosphere at a level where ecosystems can adapt naturally to climate change, food production is not threatened, and economic development can be sustained. False True False False True False False True True True False False False False False 0.357 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +499 unstructured 59912 60018 wikiClimate_change [359] the unfccc does not itself restrict emissions but rather provides a framework for protocols that do. False False False False False False False False False False False False False False False 0.1511 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +500 unstructured 60019 60075 wikiClimate_change global emissions have risen since the unfccc was signed. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +501 unstructured 60076 60142 wikiClimate_change [360] its yearly conferences are the stage of global negotiations. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +502 unstructured 60150 60289 wikiClimate_change the 1997 kyoto protocol extended the unfccc and included legally binding commitments for most developed countries to limit their emissions. False False False False False False False False False False False False False False False 0.2263 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +503 unstructured 60290 60572 wikiClimate_change "[362] during the negotiations, the g77 (representing developing countries) pushed for a mandate requiring developed countries to ""[take] the lead"" in reducing their emissions,[363] since developed countries contributed most to the accumulation of greenhouse gases in the atmosphere." False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +504 unstructured 60573 60730 wikiClimate_change per-capita emissions were also still relatively low in developing countries and developing countries would need to emit more to meet their development needs. False False False False False False False False False False False False False False False -0.2732 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +505 unstructured 60738 60887 wikiClimate_change the 2009 copenhagen accord has been widely portrayed as disappointing because of its low goals, and was rejected by poorer nations including the g77. False False False False False False False False False False False False False False False -0.8779 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +506 unstructured 60888 60970 wikiClimate_change [365] associated parties aimed to limit the global temperature rise to below 2 °c. False True False True True False False False False False False False False False False 0.4019 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +507 unstructured 60971 61149 wikiClimate_change [366] the accord set the goal of sending $100 billion per year to developing countries for mitigation and adaptation by 2020, and proposed the founding of the green climate fund. False True False False True False False False False False False False False False True 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +508 unstructured 61150 61201 wikiClimate_change [367] as of 2020, only 83.3 billion were delivered. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +509 unstructured 61202 61253 wikiClimate_change only in 2023 the target is expected to be achieved. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +510 unstructured 61261 61436 wikiClimate_change in 2015 all un countries negotiated the paris agreement, which aims to keep global warming well below 2.0 °c and contains an aspirational goal of keeping warming under 1.5 °c. False True False False True False False False False False False False False False False 0.7579 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +511 unstructured 61437 61485 wikiClimate_change [369] the agreement replaced the kyoto protocol. False False False False False False False False False False False False False False False 0.4939 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +512 unstructured 61486 61560 wikiClimate_change unlike kyoto, no binding emission targets were set in the paris agreement. False False False False False False False False False False False False False False False 0.25 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +513 unstructured 61561 61607 wikiClimate_change instead, a set of procedures was made binding. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +514 unstructured 61608 61710 wikiClimate_change countries have to regularly set ever more ambitious goals and reevaluate these goals every five years. False False False False False False False False False False False False False False False 0.5256 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +515 unstructured 61711 61802 wikiClimate_change [370] the paris agreement restated that developing countries must be financially supported. False False False False False False False False False False False False False False False 0.6705 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +516 unstructured 61803 61954 wikiClimate_change [371] as of october 2021, 194 states and the european union have signed the treaty and 191 states and the eu have ratified or acceded to the agreement. False False False False False False False False False False False False False False False 0.5859 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +517 unstructured 61962 62177 wikiClimate_change the 1987 montreal protocol, an international agreement to stop emitting ozone-depleting gases, may have been more effective at curbing greenhouse gas emissions than the kyoto protocol specifically designed to do so. False True False False False False False False False False False False False False False 0.659 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +518 unstructured 62178 62387 wikiClimate_change [373] the 2016 kigali amendment to the montreal protocol aims to reduce the emissions of hydrofluorocarbons, a group of powerful greenhouse gases which served as a replacement for banned ozone-depleting gases. False True False False False False False False False False False False False True False -0.0516 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +519 unstructured 62388 62464 wikiClimate_change this made the montreal protocol a stronger agreement against climate change. False True False True True False False False False False False False False False False 0.7003 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +520 unstructured 62465 62489 wikiClimate_change [374] national responses False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +521 unstructured 62491 62598 wikiClimate_change in 2019, the united kingdom parliament became the first national government to declare a climate emergency. False True False False True False False False False False False False False False False 0.0516 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +522 unstructured 62599 62653 wikiClimate_change [375] other countries and jurisdictions followed suit. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +523 unstructured 62654 62749 wikiClimate_change "[376] that same year, the european parliament declared a ""climate and environmental emergency""." False True False False True False False False False False False False False False False -0.3818 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +524 unstructured 62750 62868 wikiClimate_change [377] the european commission presented its european green deal with the goal of making the eu carbon-neutral by 2050. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +525 unstructured 62869 63080 wikiClimate_change "[378] in 2021, the european commission released its ""fit for 55"" legislation package, which contains guidelines for the car industry; all new cars on the european market must be zero-emission vehicles from 2035." False False False False False False False False False False False False False False False 0.3612 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +526 unstructured 63088 63228 wikiClimate_change major countries in asia have made similar pledges: south korea and japan have committed to become carbon-neutral by 2050, and china by 2060. False False False False False False False False False False False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +527 unstructured 63229 63346 wikiClimate_change [380] while india has strong incentives for renewables, it also plans a significant expansion of coal in the country. False False False False False False False False False False False False False False False 0.7506 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +528 unstructured 63347 63514 wikiClimate_change [381] vietnam is among very few coal-dependent, fast-developing countries that pledged to phase out unabated coal power by the 2040s or as soon as possible thereafter. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +529 unstructured 63522 63824 wikiClimate_change as of 2021, based on information from 48 national climate plans, which represent 40% of the parties to the paris agreement, estimated total greenhouse gas emissions will be 0.5% lower compared to 2010 levels, below the 45% or 25% reduction goals to limit global warming to 1.5 °c or 2 °c, respectively. False True False False True False False False False False False False False False False 0.7717 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +530 unstructured 63825 64036 wikiClimate_change [383] society denial and misinformation further information: climate change denial and fossil fuels lobby data has been cherry picked from short periods to falsely assert that global temperatures are not rising. False True False False True False False False False False False False False False False -0.296 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +531 unstructured 64037 64126 wikiClimate_change blue trendlines show short periods that mask longer-term warming trends (red trendlines). False False False False True False False False False False False False False False False 0.1531 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +532 unstructured 64127 64199 wikiClimate_change blue rectangle with blue dots shows the so-called global warming hiatus. False True False False True False False False False False False False False False False 0.1531 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +533 unstructured 64207 64427 wikiClimate_change public debate about climate change has been strongly affected by climate change denial and misinformation, which originated in the united states and has since spread to other countries, particularly canada and australia. False True False False True False False False False False False False False False False 0.25 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +534 unstructured 64428 64562 wikiClimate_change climate change denial has originated from fossil fuel companies, industry groups, conservative think tanks, and contrarian scientists. False True False False True False False True False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +535 unstructured 64563 64717 wikiClimate_change [385] like the tobacco industry, the main strategy of these groups has been to manufacture doubt about climate-change related scientific data and results. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +536 unstructured 64718 64880 wikiClimate_change "[386] people who hold unwarranted doubt about climate change are called climate change ""skeptics"", although ""contrarians"" or ""deniers"" are more appropriate terms." False True False False True False False False False False False False False False False -0.6124 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +537 unstructured 64888 65101 wikiClimate_change there are different variants of climate denial: some deny that warming takes place at all, some acknowledge warming but attribute it to natural influences, and some minimise the negative impacts of climate change. False True False False True False False False False False False False False False False -0.4404 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +538 unstructured 65102 65345 wikiClimate_change [388] manufacturing uncertainty about the science later developed into a manufactured controversy: creating the belief that there is significant uncertainty about climate change within the scientific community in order to delay policy changes. False True False False True False False False False False False False False False False -0.4767 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +539 unstructured 65346 65490 wikiClimate_change [389] strategies to promote these ideas include criticism of scientific institutions,[390] and questioning the motives of individual scientists. False False False False False False False False False False False False False False False -0.1779 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +540 unstructured 65491 65604 wikiClimate_change [388] an echo chamber of climate-denying blogs and media has further fomented misunderstanding of climate change. False True False False True False False False False False False False False False False -0.4215 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +541 unstructured 65605 65869 wikiClimate_change [391] public awareness and opinion further information: climate communication, media coverage of climate change, and public opinion on climate change the public substantially underestimates the degree of scientific consensus that humans are causing climate change. False True False False True False False False False False False False False False False -0.3384 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +542 unstructured 65870 65969 wikiClimate_change [392] studies from 2019 to 2021[393][4][394] found scientific consensus to range from 98.7 to 100%. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +543 unstructured 65971 66043 wikiClimate_change climate change came to international public attention in the late 1980s. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +544 unstructured 66044 66182 wikiClimate_change [395] due to media coverage in the early 1990s, people often confused climate change with other environmental issues like ozone depletion. False True False False True False False False False False False False False False False 0.0516 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +545 unstructured 66183 66348 wikiClimate_change [396] in popular culture, the climate fiction movie the day after tomorrow (2004) and the al gore documentary an inconvenient truth (2006) focused on climate change. False True False False True False False False False False False False False False False 0.6486 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +546 unstructured 66356 66487 wikiClimate_change significant regional, gender, age and political differences exist in both public concern for, and understanding of, climate change. False True False False True False False False False False False False False False False 0.2023 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +547 unstructured 66488 66625 wikiClimate_change more highly educated people, and in some countries, women and younger people, were more likely to see climate change as a serious threat. False True False False True False False False True False False False False False False -0.5719 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +548 unstructured 66626 66745 wikiClimate_change [397] partisan gaps also exist in many countries,[398] and countries with high co2 emissions tend to be less concerned. False True False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +549 unstructured 66746 66816 wikiClimate_change [399] views on causes of climate change vary widely between countries. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +550 unstructured 66817 67014 wikiClimate_change [400] concern has increased over time,[398] to the point where in 2021 a majority of citizens in many countries express a high level of worry about climate change, or view it as a global emergency. False True False True True False False False False False False False True False False -0.5267 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +551 unstructured 67015 67129 wikiClimate_change [401] higher levels of worry are associated with stronger public support for policies that address climate change. False True False True True False False False False False False False True False False 0.34 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +552 unstructured 67130 67214 wikiClimate_change [402] climate movement main articles: climate movement and climate change litigation False True False False True False False False False False False False False False False -0.2023 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +553 unstructured 67216 67301 wikiClimate_change climate protests demand that political leaders take action to prevent climate change. False True False False True False False False False False False False False False False -0.3182 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +554 unstructured 67302 67405 wikiClimate_change they can take the form of public demonstrations, fossil fuel divestment, lawsuits and other activities. False False False False False False False False False False False False False False False -0.1531 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +555 unstructured 67406 67475 wikiClimate_change [403] prominent demonstrations include the school strike for climate. False True False False True False False False False False False False False False False 0.2023 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +556 unstructured 67476 67633 wikiClimate_change in this initiative, young people across the globe have been protesting since 2018 by skipping school on fridays, inspired by swedish teenager greta thunberg. False False False False False False False False False False False False False False False 0.1027 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +557 unstructured 67634 67764 wikiClimate_change [404] mass civil disobedience actions by groups like extinction rebellion have protested by disrupting roads and public transport. False False False False False False False False False False False False False False False 0.128 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +558 unstructured 67772 67882 wikiClimate_change litigation is increasingly used as a tool to strengthen climate action from public institutions and companies. False True False True True False False False False False False False True False False 0.128 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +559 unstructured 67883 68027 wikiClimate_change activists also initiate lawsuits which target governments and demand that they take ambitious action or enforce existing laws on climate change. False True False False True False False False False False False False False False False 0.25 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +560 unstructured 68028 68121 wikiClimate_change [406] lawsuits against fossil-fuel companies generally seek compensation for loss and damage. False False False False False False False False False False False False False False False -0.7269 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +561 unstructured 68122 68210 wikiClimate_change [407] history for broader coverage of this topic, see history of climate change science. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +562 unstructured 68211 68374 wikiClimate_change early discoveries this 1912 article succinctly describes the greenhouse effect, how burning coal creates carbon dioxide to cause global warming and climate change. False True False False True False False False False False False False False False False 0.4019 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +563 unstructured 68382 68491 wikiClimate_change scientists in the 19th century such as alexander von humboldt began to foresee the effects of climate change. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +564 unstructured 68492 68661 wikiClimate_change [409][410][411][412] in the 1820s, joseph fourier proposed the greenhouse effect to explain why earth's temperature was higher than the sun's energy alone could explain. False False False False True False False False False True False False True False False 0.0258 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +565 unstructured 68662 68771 wikiClimate_change earth's atmosphere is transparent to sunlight, so sunlight reaches the surface where it is converted to heat. False False False False True False False False False False False False False False False 0.1226 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +566 unstructured 68772 68914 wikiClimate_change however, the atmosphere is not transparent to heat radiating from the surface, and captures some of that heat, which in turn warms the planet. False False False False True False False False False False False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +567 unstructured 68922 69115 wikiClimate_change in 1856 eunice newton foote demonstrated that the warming effect of the sun is greater for air with water vapour than for dry air, and that the effect is even greater with carbon dioxide (co2). False True False False True False False False False True False False False False False 0.6808 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +568 unstructured 69116 69428 wikiClimate_change "she concluded that ""an atmosphere of that gas would give to our earth a high temperature.""[414][415] studying what would become known as the greenhouse effect, tyndall's pre-1861 ratio spectrophotometer measured how much various gases in a tube absorb and emit infrared radiation-which humans experience as heat." False False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +569 unstructured 69430 69569 wikiClimate_change starting in 1859,[416] john tyndall established that nitrogen and oxygen-together totaling 99% of dry air-are transparent to radiated heat. False False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +570 unstructured 69570 69703 wikiClimate_change however, water vapour and gases such as methane and carbon dioxide absorb radiated heat and re-radiate that heat into the atmosphere. False True False False True False False False False True False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +571 unstructured 69704 69836 wikiClimate_change tyndall proposed that changes in the concentrations of these gases may have caused climatic changes in the past, including ice ages. False False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +572 unstructured 69844 69995 wikiClimate_change svante arrhenius noted that water vapour in air continuously varied, but the co2 concentration in air was influenced by long-term geological processes. False True False False False False False False False True False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +573 unstructured 69996 70120 wikiClimate_change warming from increased co2 levels would increase the amount of water vapour, amplifying warming in a positive feedback loop. False True False True True False False False False True False False True False False 0.8481 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +574 unstructured 70121 70279 wikiClimate_change in 1896, he published the first climate model of its kind, projecting that halving co2 levels could have produced a drop in temperature initiating an ice age. False True False False True False False False False False False False False False False 0.3182 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +575 unstructured 70280 70373 wikiClimate_change arrhenius calculated the temperature increase expected from doubling co2 to be around 5-6 °c. False True False True True False False False False False False False True False False 0.3182 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +576 unstructured 70374 70572 wikiClimate_change [418] other scientists were initially skeptical and believed that the greenhouse effect was saturated so that adding more co2 would make no difference, and that the climate would be self-regulating. False True False False True False False False False False False False False False False -0.5423 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +577 unstructured 70573 70742 wikiClimate_change [419] beginning in 1938, guy stewart callendar published evidence that climate was warming and co2 levels were rising,[420] but his calculations met the same objections. False True False False True False False False False False False False False False False 0.0772 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +578 unstructured 70743 71058 wikiClimate_change [419] development of a scientific consensus see also: scientific consensus on climate change scientific consensus on causation: academic studies of scientific agreement on human-caused global warming among climate experts (2010-2015) reflect that the level of consensus correlates with expertise in climate science. False True False False True False False False False False False False False False False 0.5859 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +579 unstructured 71059 71180 wikiClimate_change [421] a 2019 study found scientific consensus to be at 100%,[422] and a 2021 study concluded that consensus exceeded 99%. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +580 unstructured 71181 71318 wikiClimate_change [423] another 2021 study found that 98.7% of climate experts indicated that the earth is getting warmer mostly because of human activity. False True False False True False False False False False False False False False False 0.296 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +581 unstructured 71326 71457 wikiClimate_change in the 1950s, gilbert plass created a detailed computer model that included different atmospheric layers and the infrared spectrum. False False False False False False False False False False False False False False False 0.25 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +582 unstructured 71458 71526 wikiClimate_change this model predicted that increasing co2 levels would cause warming. False True False True True False False False False False False False True False False 0.1531 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +583 unstructured 71527 71679 wikiClimate_change around the same time, hans suess found evidence that co2 levels had been rising, and roger revelle showed that the oceans would not absorb the increase. False True False True False False False False False False False False True False False -0.2411 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +584 unstructured 71680 71818 wikiClimate_change "the two scientists subsequently helped charles keeling to begin a record of continued increase, which has been termed the ""keeling curve""." False False False True False False False False False False False False True False False 0.3182 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +585 unstructured 71819 71941 wikiClimate_change [419] scientists alerted the public,[425] and the dangers were highlighted at james hansen's 1988 congressional testimony. False False False False False False False False False False False False False False False -0.4939 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +586 unstructured 71942 72104 wikiClimate_change [31] the intergovernmental panel on climate change (ipcc), set up in 1988 to provide formal advice to the world's governments, spurred interdisciplinary research. False True False False True False False False False False False True False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +587 unstructured 72105 72235 wikiClimate_change [426] as part of the ipcc reports, scientists assess the scientific discussion that takes place in peer-reviewed journal articles. False False False False False False False False False False False True False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +588 unstructured 72243 72361 wikiClimate_change there is a near-complete scientific consensus that the climate is warming and that this is caused by human activities. False True False False True False False False False False False False False False False 0.1531 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +589 unstructured 72362 72422 wikiClimate_change as of 2019, agreement in recent literature reached over 99%. False False False False False False False False False False False False False False False 0.5574 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +590 unstructured 72423 72516 wikiClimate_change [422][423] no scientific body of national or international standing disagrees with this view. False False False False False False False False False False False False False False False -0.5423 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +591 unstructured 72517 72652 wikiClimate_change [428] consensus has further developed that some form of action should be taken to protect people against the impacts of climate change. False True False False True False False False False False False False False False False 0.3818 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +592 unstructured 72653 72733 wikiClimate_change national science academies have called on world leaders to cut global emissions. False False False False False False False False False False False False False False False -0.2732 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +593 unstructured 72734 72844 wikiClimate_change "[429] the 2021 ipcc assessment report stated that it is ""unequivocal"" that climate change is caused by humans." False True False False True False False False False False False True False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +594 unstructured 72845 72859 wikiClimate_change [423] see also False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +595 unstructured 72861 72908 wikiClimate_change iconclimate change portaliconenvironment portal False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +596 unstructured 72910 73046 wikiClimate_change anthropocene - proposed new geological time interval in which humans are having significant geological impact list of climate scientists False True False False True False False False False False False False False False False 0.2023 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +1 unstructured 0 15 wikiClimate_justice climate justice False True False False True False False False False False False False False False False 0.5267 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +2 unstructured 17 29 wikiClimate_justice article talk False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +3 unstructured 31 53 wikiClimate_justice read edit view history False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +4 unstructured 55 60 wikiClimate_justice tools False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +5 unstructured 62 99 wikiClimate_justice from wikipedia, the free encyclopedia False False False False False False False False False False False False False False False 0.5106 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +6 unstructured 101 172 wikiClimate_justice this article may require cleanup to meet wikipedia's quality standards. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +7 unstructured 173 237 wikiClimate_justice the specific problem is: unclear because sentences are too long. False False False False False False False False False False False False False False False -0.5423 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +8 unstructured 238 282 wikiClimate_justice please help improve this article if you can. False False False False False False False False False False False False True False False 0.7845 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +9 unstructured 283 458 wikiClimate_justice "(september 2023) (learn how and when to remove this template message) fridays for future demonstration in berlin in september 2021 with the slogan ""fight for climate justice""." False True False False True False False False False False False False False False False 0.296 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +10 unstructured 460 615 wikiClimate_justice climate justice is an approach to climate action that focuses on the unequal impacts of climate change on marginalized or otherwise vulnerable populations. False True False False True False False False False False False False False False False 0.0258 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +11 unstructured 616 760 wikiClimate_justice [1] climate justice wants to achieve an equitable distribution of both the burdens of climate change and the efforts to mitigate climate change. False True False False True False False False False False False False False False False 0.2263 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +12 unstructured 761 816 wikiClimate_justice [2] climate justice is a type of environmental justice. False True False False True False False False False False False False False False False 0.7783 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +13 unstructured 822 962 wikiClimate_justice climate justice examines concepts such as equality, human rights, collective rights, and the historical responsibilities for climate change. False True False False True False False False False False False False False False False 0.5267 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +14 unstructured 963 1107 wikiClimate_justice this is done by relating the causes and effects of climate change to concepts of justice, particularly environmental justice and social justice. False True False False True False False False False False False False False False False 0.888 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +15 unstructured 1108 1198 wikiClimate_justice historically marginalized communities often face the worst consequences of climate change. False True False False True False False False False False False False False False False -0.6249 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +16 unstructured 1199 1326 wikiClimate_justice depending on the country and context, this may include people with low-incomes, indigenous communities or communities of color. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +17 unstructured 1327 1441 wikiClimate_justice generally speaking, those who are the least responsible for climate change often suffer the greatest consequences. False True False False True False False False False False False False False False False -0.0675 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +18 unstructured 1442 1569 wikiClimate_justice [4][5][6] they might also be further disadvantaged by responses to climate change which might exacerbate existing inequalities. False True False True True False False False False False False False False False False -0.4019 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +19 unstructured 1570 1639 wikiClimate_justice this situation is known as the 'triple injustices' of climate change. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +20 unstructured 1651 1760 wikiClimate_justice conceptions of climate justice can be grouped along the lines of procedural justice and distributive justice. False True False False True False False False False False False False False False False 0.8807 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +21 unstructured 1761 1829 wikiClimate_justice the former stresses fair, transparent and inclusive decision making. False False False False False False False False False False False False False False False -0.1779 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +22 unstructured 1830 1946 wikiClimate_justice the latter investigates a fair distribution of the costs of both climate change and the actions taken to address it. False True False False True False False False False False False False False False False 0.3182 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +23 unstructured 1947 2037 wikiClimate_justice [7] other approaches focus on addressing social implications of climate change mitigation. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +24 unstructured 2038 2134 wikiClimate_justice if these are not addressed properly, this could result in profound economic and social tensions. False False False False False False True False False False False False False False False -0.4019 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +25 unstructured 2135 2185 wikiClimate_justice it could even lead to delays in necessary changes. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +26 unstructured 2192 2274 wikiClimate_justice climate justice actions can include the growing global body of climate litigation. False True False False True False False False False False False False True False False 0.5106 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +27 unstructured 2275 2389 wikiClimate_justice [11] in 2017, a report of the united nations environment programme identified 894 ongoing legal actions worldwide. False False False False False False False False False False False False False False False 0.5106 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +28 unstructured 2390 2446 wikiClimate_justice [12] definition and objectives see also: just transition False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +29 unstructured 2448 2638 wikiClimate_justice use and popularity of climate justice language has increased dramatically in recent years, yet climate justice is understood in many ways, and the different meanings are sometimes contested. False True False True True False False False False False False False True False False 0.9001 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +30 unstructured 2639 2735 wikiClimate_justice at its simplest, conceptions of climate justice can be grouped along the following two lines:[7] False True False False True False False False False False False False False False False 0.5267 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +31 unstructured 2737 2957 wikiClimate_justice procedural justice, which emphasizes fair, transparent and inclusive decision making, and distributive justice, which places the emphasis on who bears the costs of both climate change and the actions taken to address it. False True False False True False False False False False False False False False False 0.8442 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +32 unstructured 2959 3234 wikiClimate_justice "the objectives of climate justice can be described as: ""to encompasses a set of rights and obligations, which corporations, individuals and governments have towards those vulnerable people who will be in a way significantly disproportionately affected by climate change.""[13]" False True False False True False False False False False False False False False False 0.2263 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +33 unstructured 3236 3376 wikiClimate_justice climate justice examines concepts such as equality, human rights, collective rights, and the historical responsibilities for climate change. False True False False True False False False False False False False False False False 0.5267 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +34 unstructured 3377 3506 wikiClimate_justice climate justice is mainly concerned with the procedural and distributive ethical dimensions of and for climate change mitigation. False True False False True False False False False False False False False False False 0.7717 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +35 unstructured 3508 3731 wikiClimate_justice "the ipcc sixth assessment report underlines another principle of climate justice which is the ""recognition which entails basic respect and robust engagement with and fair consideration of diverse cultures and perspectives""." False True False False True False False False False False False True False False False 0.9217 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +36 unstructured 3732 3858 wikiClimate_justice [14] alternatively, recognition and respect can be understood as the underlying basis for distributive and procedural justice. False False False False False False False False False False False False False False False 0.7579 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +37 unstructured 3860 3920 wikiClimate_justice related fields are environmental justice and social justice. False False False False False False False False False False False False False False False 0.7783 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +38 unstructured 3921 4078 wikiClimate_justice causes of injustice economic systems among major emitters, the u.s. has higher annual per capita emissions than china, which has more total annual emissions. False False False False False False False False False False False False True False False -0.5719 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +39 unstructured 4079 4183 wikiClimate_justice [15] cumulatively, u.s. and china emissions have caused the most greenhouse gas-related economic damage. False True False False False False False False False False False False False False False -0.4939 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +40 unstructured 4190 4350 wikiClimate_justice whether fundamental differences in economic systems, such as capitalism versus socialism, are the, or a, root cause of climate injustice is a contentious issue. False True False False True False False False False False False False False False False -0.7096 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +41 unstructured 4351 4517 wikiClimate_justice in this context, fundamental disagreements arise between liberal and conservative environmental groups on one side and leftist and radical organizations on the other. False False False False False False False True False False False False False False False -0.4215 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +42 unstructured 4518 4753 wikiClimate_justice while the former often tend to blame the excesses of neoliberalism for climate change and argue in favor of market-based reform within capitalism, the latter view capitalism with its exploitative traits as the underlying central issue. False True False False True False False False False False False False False False False -0.2732 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +43 unstructured 4754 4897 wikiClimate_justice [17][18] other possible causal explanations include hierarchies based on the group differences and the nature of the fossil fuel regime itself. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +44 unstructured 4898 5016 wikiClimate_justice [19] systemic causes many participants of grassroots movements that demand climate justice also ask for system change. False True False False True False False False False False False False False False False 0.4404 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +45 unstructured 5018 5146 wikiClimate_justice it has been argued that the unwarranted rate of climate change, along with its inequality of burdens, is a structural injustice. False True False False True False False False False False False False False False False -0.8271 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +46 unstructured 5147 5266 wikiClimate_justice there is political responsibility for the maintenance and support of historically constituted[20] structural processes. False False False False False False False False False False False False False False False 0.4019 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +47 unstructured 5267 5365 wikiClimate_justice this is despite assumed viable potential alternative models based on novel technologies and means. False False False False False False False False False False False False False False False 0.3182 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +48 unstructured 5366 5598 wikiClimate_justice as a criterion for determining responsibility for climate change, individual causal contribution or capacity does not matter as much as responsibility for the perpetuation of carbon-intensive structures, practices, and institutions. False True False False True False False False False False False False False False False -0.0191 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +49 unstructured 5599 5804 wikiClimate_justice these structures constitute the global politico-economic system, rather than enabling structural changes towards a system that does not naturally facilitate unsustainable exploitation of people and nature. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +50 unstructured 5815 5941 wikiClimate_justice for others, climate justice could be pursued through existing economic frameworks, global organizations and policy mechanisms. False True False False True False False False False False False False False False False 0.5267 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +51 unstructured 5942 6148 wikiClimate_justice therefore the root-causes could be found in the causes that so far inhibited global implementation of measures like emissions trading schemes, specifically forms that deliver the assumed mitigation results. False False False False False False False False False False False False False False False 0.2075 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +52 unstructured 6149 6274 wikiClimate_justice [23] disproportionality between causality and burden emissions of the richest 1% are more than twice that of the poorest 50%. False False False False False False False False False False False False False False False -0.4588 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +53 unstructured 6275 6473 wikiClimate_justice [24] compliance with the paris agreement's 1.5°c goal would require the richest 1% to reduce emissions by at least 30 times, while per-person emissions of the poorest 50% could approximately triple. False False False False False False False False False False False False False True False -0.0258 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +54 unstructured 6474 6683 wikiClimate_justice [24] though total co2 emissions (size of pie charts) differ substantially among high-emitting regions, the pattern of higher income classes emitting more than lower income classes is consistent across regions. False True False False False False False False False False False False True False False -0.3566 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +55 unstructured 6684 6765 wikiClimate_justice [25] the world's top 1% of emitters emit over 1000 times more than the bottom 1%. False False False False False False False False False False False False False False False 0.2023 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +56 unstructured 6766 6912 wikiClimate_justice [25] scaling the effect of wealth to the national level: richer (developed) countries emit more co2 per person than poorer (developing) countries. False True False False False False False False False False False False False False False 0.6249 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +57 unstructured 6913 7048 wikiClimate_justice [26] emissions are roughly proportional to gdp per person, though the rate of increase diminishes with average gdp/pp of about $10,000. False False False True False False False False False False False False True False False 0.3182 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +58 unstructured 7050 7153 wikiClimate_justice the responsibility for anthropogenic climate change differs substantially among individuals and groups. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +59 unstructured 7154 7256 wikiClimate_justice many of the people and nations most affected by climate change are among the least responsible for it. False True False False True False False False False False False False False False False -0.432 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +60 unstructured 7257 7455 wikiClimate_justice [27][28] the most affluent citizens of the world are responsible for most environmental impacts and robust action by them is necessary for prospects of moving towards safer environmental conditions. False False False False False False False False False False False False False False False 0.8271 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +61 unstructured 7466 7689 wikiClimate_justice according to a 2020 report by oxfam and the stockholm environment institute,[31][32] the richest 1% of the global population have caused twice as much carbon emissions as the poorest 50% over the 25 years from 1990 to 2015. False True False False False False False False False False False False False False False -0.0258 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +62 unstructured 7690 7790 wikiClimate_justice [33][34][24] this was, respectively, during that period, 15% of cumulative emissions compared to 7%. False False False False False False False False False False False False False False False 0.34 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +63 unstructured 7791 7983 wikiClimate_justice [35] a second 2023 report found the richest 1% of humans produce more carbon emissions than poorest 66%, while the top 10% richest people account for more than half of global carbon emissions. False True False False False False False False False False False False False False False 0.6249 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +64 unstructured 7984 8104 wikiClimate_justice [36][37] a visualisation by country of the impact of climate change (country size) and its contribution (country colour) False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +65 unstructured 8106 8271 wikiClimate_justice the bottom half of the population is directly-responsible for less than 20% of energy footprints and consume less than the top 5% in terms of trade-corrected energy. False False False False False False False False False True False False False False False 0.577 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +66 unstructured 8272 8424 wikiClimate_justice high-income individuals usually have higher energy footprints as they disproportionally use their larger financial resources for energy-intensive goods. False False False False False False False False False True False False True False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +67 unstructured 8425 8603 wikiClimate_justice in particular, the largest disproportionality was identified to be in the domain of transport, where the top 10% consume 56% of vehicle fuel and conduct 70% of vehicle purchases. False False False False False False False False False False False False False False False 0.2023 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +68 unstructured 8610 8806 wikiClimate_justice a 2023 review article found that if there were a 2oc temperature rise by 2100, roughly 1 billion primarily poor people would die as a result of primarily wealthy people's greenhouse gas emissions. False True False True True False False False False False False False False False False -0.6705 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +69 unstructured 8807 8886 wikiClimate_justice [39][40] intergenerational equity further information: intergenerational equity False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +70 unstructured 8888 9133 wikiClimate_justice the nations and the world population need to make changes, including sacrifices (like uncomfortable lifestyle-changes,[41][42] alterations to public spending and changes to choice of work), today to enable climate justice for future generations. False True False False True False False False False False False False False False False 0.5106 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +71 unstructured 9140 9257 wikiClimate_justice preventable severe effects of climate change are likely to occur during the lifetime of the present adult population. False True False False True False False False False False False False False False False -0.3818 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +72 unstructured 9258 9492 wikiClimate_justice "under current climate policy pledges, children born in 2020 (e.g. ""generation alpha"") will experience over their lifetimes, 2-7 times as many heat waves, as well as more of other extreme weather events compared to people born in 1960." True True False False True False False False False False False False False False False 0.2732 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +73 unstructured 9493 9751 wikiClimate_justice this, along with other projections, raises issues of intergenerational equity as it was these generations (specific groups and individuals and their collective governance and perpetuated economics) who are mainly responsible for the burden of climate change. False True False False True False False False False False False False False False False -0.1531 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +74 unstructured 9762 10016 wikiClimate_justice this illustrates that emissions produced by any given generation can lock-in damage for one or more future generations, making climate change progressively more threatening for the generations affected than for the generation responsible for the threats. False True False False True False False False True False False False False False False -0.8399 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +75 unstructured 10017 10162 wikiClimate_justice the climate system contains tipping points, such as the amount of deforestation of the amazon that will launch the forest's irreversible decline. False True False False True True False True False True False False False False False -0.0258 False True True False True False False False False False True False True False False True False False False True False True True False False False True False False False +76 unstructured 10163 10330 wikiClimate_justice [46][47] a generation whose continued emissions drive the climate system past such significant tipping points inflicts severe injustice on multiple future generations. False True False False True False False False False False False False False False False -0.6705 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +77 unstructured 10331 10434 wikiClimate_justice [48] disproportionate impacts on disadvantaged groups further information: climate change vulnerability False True False False True False False False False False False False False False False -0.5574 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +78 unstructured 10436 10532 wikiClimate_justice disadvantaged groups will continue to be disproportionately impacted as climate change persists. False True False False True False False False False False False False False False False -0.4019 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +79 unstructured 10533 10684 wikiClimate_justice these groups will be affected due to inequalities based on demographic characteristics such as differences in gender, race, ethnicity, age, and income. False False False False False False False False False False False False False False False -0.1531 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +80 unstructured 10685 10868 wikiClimate_justice [49] inequality increases the exposure of disadvantaged groups to harmful effects of climate change while also increasing their susceptibility to destruction caused by climate change. False True False True True False False False True False False False True False False -0.7506 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +81 unstructured 10869 11101 wikiClimate_justice [49] the damage is worsened because disadvantaged groups are last to receive emergency relief and are rarely included in the planning process at local, national and international levels for coping with the impacts of climate change. False True False False True False False False False False False False False False False -0.8074 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +82 unstructured 11108 11229 wikiClimate_justice communities of color, women, indigenous groups, and people of low-income all face higher vulnerability to climate change. False True False False True False False False False False False False True False False -0.2263 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +83 unstructured 11230 11343 wikiClimate_justice [51][14] these groups will be disproportionately impacted by heat waves, air quality, and extreme weather events. True False False False True False False False False False False False False False False 0.0 False True True False True False False False False False True True False False False False False False False True True False False False False False False False False False +84 unstructured 11344 11438 wikiClimate_justice [52] women are also disadvantaged and will be affected by climate change differently than men. False True False False True False False False False False False False False False False -0.5106 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +85 unstructured 11439 11588 wikiClimate_justice [53] this may impact the ability of minority groups to adapt, unless steps are taken to provide these groups with more access to universal resources. False False False False False False False False False True False False False False False 0.3182 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +86 unstructured 11589 11734 wikiClimate_justice [54] indigenous groups are affected by the consequences of climate change even though they historically have contributed the least to causing it. False True False False True False False False False False False False False False False -0.1531 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +87 unstructured 11735 11876 wikiClimate_justice [52] indigenous peoples are unjustifiably impacted due to their low income, and continue to have fewer resources to cope with climate change. False True False False True False False False False True False False False False False -0.2732 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +88 unstructured 11877 11947 wikiClimate_justice [55] responses to improve climate justice burden on future generations False True False False True False False False False False False False True False False 0.5267 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +89 unstructured 11949 12234 wikiClimate_justice one generation must not be allowed to consume large portions of the co2 budget while bearing a relatively minor share of the reduction effort if this would involve leaving subsequent generations with a drastic reduction burden and expose their lives to comprehensive losses of freedom. False True False False False False False False False False False False False False False 0.296 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +90 unstructured 12236 12323 wikiClimate_justice - german federal constitutional court april 2021[56] conclusion on the rights of nature False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +91 unstructured 12325 12510 wikiClimate_justice the rights of nature protect ecosystems and natural processes for their intrinsic value, thus complementing them with the human right to a healthy and ecologically balanced environment. False False False False False False False True False True False False False False False 0.8481 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +92 unstructured 12511 12637 wikiClimate_justice the rights of nature, like all constitutional rights, are justiciable and, consequently, judges are obliged to guarantee them. False False False False False False False False False False False False False False False 0.5423 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +93 unstructured 12639 12744 wikiClimate_justice - constitutional court of ecuador[57][58] 10 november 2021 common principles of justice in burden-sharing False False False False False False False False False False False False False False False 0.5267 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +94 unstructured 12746 12922 wikiClimate_justice there are three common principles of justice in burden-sharing that can be used in making decisions on who bears the larger burdens of climate change globally and domestically: False True False False True False False False False False False False False False False 0.2263 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +95 unstructured 12923 12960 wikiClimate_justice a) those who most caused the problem, False False False False False False False False False False False False False False False -0.4522 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +96 unstructured 12961 13015 wikiClimate_justice b) those who have the most burden-carrying ability and False False False False False False False False False False False False False False False 0.3774 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +97 unstructured 13016 13095 wikiClimate_justice c) those who have benefited most from the activities that cause climate change. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +98 unstructured 13096 13359 wikiClimate_justice [59] observing calls for climate reparations in scientific literature, among climate movements and in policy debates, a 2023 study estimated that the top 21 fossil fuel companies would owe cumulative climate reparations of $5.4 trillion over the period 2025-2050. False True False False True False False False False False False False False False False 0.2023 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +99 unstructured 13366 13530 wikiClimate_justice another method of deciding-making starts from the objective of preventing climate change e.g. beyond 1.5 °c, and from there reasons backwards to who should do what. False True False False True False False False False False False False False False False -0.0258 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +100 unstructured 13531 13619 wikiClimate_justice [61] this makes use of the principles of justice in burden-sharing to maintain fairness. False False False False False False False False False False False False False False False 0.5267 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +101 unstructured 13620 13817 wikiClimate_justice court cases and litigation in 2019, the supreme court of the netherlands confirmed that the government must cut carbon dioxide emissions further, as climate change threatens citizens' human rights. False True False False True False False False True False False False False False False -0.2263 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +102 unstructured 13824 13951 wikiClimate_justice by december 2022, the number of climate change-related lawsuits had grown to 2,180, more than half in the u.s (1,522 lawsuits). False True False False True False False False False False False False True False False -0.2263 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +103 unstructured 13952 14164 wikiClimate_justice [63] based on existing laws, some relevant parties can already be forced into action (to the degree of accountability, monitoring and law enforcement capacities and assessments of feasibility) by means of courts. False False False False False False False False False False False False False False False -0.0772 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +104 unstructured 14165 14228 wikiClimate_justice [62] this section is an excerpt from climate change litigation. False True False False True False False False False False False False False False False -0.2023 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +105 unstructured 14228 14489 wikiClimate_justice [edit] climate change litigation, also known as climate litigation, is an emerging body of environmental law using legal practice to set case law precedent to further climate change mitigation efforts from public institutions, such as governments and companies. False True False False True False False False False False False False False False False -0.2732 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +106 unstructured 14490 14689 wikiClimate_justice in the face of slow climate change politics delaying climate change mitigation, activists and lawyers have increased efforts to use national and international judiciary systems to advance the effort. False True False True True False False False False False False False True False False 0.2732 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +107 unstructured 14690 15235 wikiClimate_justice climate litigation typically engages in one of five types of legal claims:[64] constitutional law (focused on breaches of constitutional rights by the state),[65] administrative law (challenging the merits of administrative decision making), private law (challenging corporations or other organizations for negligence, nuisance, etc., fraud or consumer protection (challenging companies for misrepresenting information about climate impacts), or human rights (claiming that failure to act on climate change is a failure to protect human rights). False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +108 unstructured 15236 15466 wikiClimate_justice [66] rally for climate justice: mass mobilization at the chevron oil refinery in richmond, california (2009) tens of thousands marching in copenhagen for climate justice (2009)[67] human rights the cop26 climate change conference. False True False False True False False False False False False False False False False 0.7783 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +109 unstructured 15470 15942 wikiClimate_justice acknowledging that climate change is a common concern of humankind, parties should, when taking action to address climate change, respect, promote and consider their respective obligations on human rights, the right to health, the rights of indigenous peoples, local communities, migrants, children, persons with disabilities and people in vulnerable situations and the right to development, as well as gender equality, empowerment of women and intergenerational equity, . False True False False True False False False False False False False False False False 0.886 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +110 unstructured 15944 16056 wikiClimate_justice - the glasgow climate pact[68] 13 november 2021 this section is an excerpt from human rights and climate change. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +111 unstructured 16056 16248 wikiClimate_justice [edit] human rights and climate change is a conceptual and legal framework under which international human rights and their relationship to global warming are studied, analyzed, and addressed. False True False False True False False False False False False False False False False 0.2732 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +112 unstructured 16249 16628 wikiClimate_justice [69] the framework has been employed by governments, united nations organizations, intergovernmental and non-governmental organizations, human rights and environmental advocates, and academics to guide national and international policy on climate change under the united nations framework convention on climate change (unfccc) and the core international human rights instruments. False True False False True False False False False False False False False False False 0.6808 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +113 unstructured 16629 16834 wikiClimate_justice "[70][71][72] in 2022 working group ii of the ipcc suggested that ""climate justice comprises justice that links development and human rights to achieve a rights-based approach to addressing climate change""." False True False False True False False False False False False True False False False 0.7783 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +114 unstructured 16835 16935 wikiClimate_justice [73] challenges societal disruption and policy support see also: yellow vests protests § fuel prices False False False False False False False False False False False False False False False -0.1027 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +115 unstructured 16937 16994 wikiClimate_justice climate justice may often conflict with social stability. False True False False True False False False True False True False False False False 0.2732 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +116 unstructured 16995 17097 wikiClimate_justice for example, interventions that establish more just product pricing could result in social unrest[74]. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +117 unstructured 17098 17353 wikiClimate_justice [75] socioeconomic decarbonization interventions could lead to decreased material possessions, number of freely choosable options, comfort, maintained habits and salaries,[29][additional citation(s) needed] and also temporary increased unemployment rates. False False False True False False False False False False False False True True False 0.5994 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +118 unstructured 17354 17416 wikiClimate_justice this may be problematic with current socioeconomic structures. False False False False False False False False False False False False False False False -0.4404 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +119 unstructured 17423 17760 wikiClimate_justice however, multiple studies estimate that if a rapid transition were to be implemented in certain ways the number of full-time jobs formally recognized in the economy could increase overall at least temporarily due to increased demand for labor to e.g. build public infrastructure and other green jobs to build the renewable energy system. False False False True False False False False False True False False True False False 0.7506 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +120 unstructured 17775 18006 wikiClimate_justice the urgent need for and extent of policies, especially when seeking to facilitate lifestyle-changes and shifts on an industry scale, could lead to social tension and decrease levels of public support for political parties in power. False False False False False False False False False False True False False True False 0.5994 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +121 unstructured 18007 18110 wikiClimate_justice "[80][81] for instance, keeping gas prices low is often ""really good for the poor and the middle class""." False False False False False False False False False False False False False False False -0.2516 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +122 unstructured 18111 18186 wikiClimate_justice [82] loss and damage discussions see also: loss and damage (climate change) False True False False True False False False False False False False False False False -0.875 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +123 unstructured 18188 18690 wikiClimate_justice "some may see climate justice arguments for compensation by rich countries for disasters and similar problems in developing countries (as well as for domestic disasters) as a way for ""limitless liability"" by which at least high levels of compensations could drain a society's resources, efforts, focus and financial funds away from efficient preventive climate change mitigation towards e.g. immediate climate change relief compensations or climate-unrelated expenses of the receiving country or people." False True False False True False False False False True False False False False False 0.1531 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +124 unstructured 18691 18941 wikiClimate_justice [83][84] fossil-fuels dependent states further information: fossil fuel phase-out § assessment of extraction prevention responsibilities, and carbon budget the us, china and russia have cumulatively contributed the greatest amounts of co2 since 1850. False True False False False False False False False False False False False False False 0.6369 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +125 unstructured 18942 19046 wikiClimate_justice [85] many of the heaviest users of fossil fuels rely on them for a high percentage of their electricity. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +126 unstructured 19053 19241 wikiClimate_justice fossil fuel phase out is projected to affect states and their citizens with large or central industries of fossil-fuels extraction - including opec states - differently than other nations. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +127 unstructured 19242 19499 wikiClimate_justice these states have obstructed climate negotiations and it has been argued that, due to their wealth, they should not need to receive financial support from other countries but could implement adequate transitions on their own in terms of financial resources. False True False False True False False False False True False False False False False 0.5499 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +128 unstructured 19514 19748 wikiClimate_justice a study suggested governments of nations that have historically benefited from extraction should take the lead, with countries that have a high dependency on fossil fuels but low capacity for transition needing some support to follow. False False False False False False False False False False False False False False False 0.2263 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +129 unstructured 19749 20073 wikiClimate_justice "[90] in particular, transitional impacts of a rapid extraction phase out is thought to be better absorbed in diversified, wealthier economies that should bear discrepancies in costs needed for a ""just transition"" as they are most able to bear it and may have better capacities for enacting absorptive socioeconomic policies." False False False False False False False False False False False False False False False 0.8402 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +130 unstructured 20074 20234 wikiClimate_justice [91] conflicting interest-driven interpretations as barriers to agreements net income of the global oil and gas industry reached a record us$4 trillion in 2022. False False False False False False False False True False True False False False False -0.0516 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +131 unstructured 20235 20288 wikiClimate_justice [92] see also: group decision-making § formal systems False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +132 unstructured 20290 20485 wikiClimate_justice "different interpretations and perspectives, arising from different interests, needs, circumstances, expectations, considerations and histories, can lead to highly varying ideas of what is ""fair""." False False False False False False False False False False False False False False False 0.5106 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +133 unstructured 20486 20589 wikiClimate_justice this may make it more difficult for countries to reach an agreement, similar to the prisoner's dilemma. False False False False False False False False False False False False False False False -0.0498 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +134 unstructured 20590 20999 wikiClimate_justice [93] developing effective, legitimate and enforceable agreements could thereby be substantially complicated, especially if traditional methods or tools of policy-making are used, in-sum trusted third party expert authorities are absent and the scientific research base relevant for the decision-making - such as studies and data about the problem, potential mitigation measures and capacities - is not robust. False False False False False True False False False False False False False False False 0.7402 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +135 unstructured 21000 21359 wikiClimate_justice fundamental fairness principles could include: after recovering from the covid-19 pandemic, energy company profits increased with greater revenues from higher fuel prices resulting from the russian invasion of ukraine, falling debt levels, tax write-downs of projects shut down in russia, and backing off from earlier plans to reduce greenhouse gas emissions. False True False True False False False False False True False False True True False 0.6808 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +136 unstructured 21360 21420 wikiClimate_justice [94] record profits sparked public calls for windfall taxes. False False False False True False False False False False False False False False False 0.4404 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +137 unstructured 21427 21471 wikiClimate_justice responsibility capability and rights (needs) False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +138 unstructured 21473 21536 wikiClimate_justice for which country characteristics can predict relative support. False False False False False False False False False False False False False False False 0.4019 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +139 unstructured 21537 21823 wikiClimate_justice "[95][additional citation(s) needed] the shared problem-characteristics of climate change could incentivize developing countries to act in concert to deter developed countries from ""passing their climate costs onto them"" and thereby improve the global mitigation effectiveness to 1.5 °c." False True False False True False False False False False False False True False False 0.6486 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +140 unstructured 21824 21836 wikiClimate_justice [96] history False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +141 unstructured 21838 22088 wikiClimate_justice developed countries, as the main cause of climate change, in assuming their historical responsibility, must recognize and honor their climate debt in all of its dimensions as the basis for a just, effective, and scientific solution to climate change. False True False False True False False False False False False False False False False 0.7269 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +142 unstructured 22089 22262 wikiClimate_justice (.) the focus must not be only on financial compensation, but also on restorative justice, understood as the restitution of integrity to our mother earth and all its beings. False False False False False False False False False False False False False False False 0.8402 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +143 unstructured 22264 22540 wikiClimate_justice world people's conference on climate change and the rights of mother earth, people's agreement, april 2010, cochabamba, bolivia[97] though the u.s.'s per capita and per gdp emissions have declined significantly, the raw numerical decline in emissions is much less substantial. False True False False True False False False False False False False False False False 0.5729 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +144 unstructured 22541 22631 wikiClimate_justice [98] growing populations and increased economic activity work against mitigation attempts. False False False True False False False False False False False False True False False 0.4215 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +145 unstructured 22633 22788 wikiClimate_justice "the concept of climate justice was deeply influential on climate negotiations years before the term ""climate justice"" was regularly applied to the concept." False True False False True False False False False False False False False False False 0.8748 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +146 unstructured 22789 23059 wikiClimate_justice in december 1990 the united nations appointed an intergovernmental negotiating committee (inc) to draft what became the framework convention on climate change (fccc), adopted at the un conference on the environment and development (unced) in rio de janeiro in june 1992. False True False False True False False False False False False False False False False 0.4215 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +147 unstructured 23060 23221 wikiClimate_justice "[99] as the name ""environment and development"" indicated, the fundamental goal was to coordinate action on climate change with action on sustainable development." False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +148 unstructured 23222 23454 wikiClimate_justice it was impossible to draft the text of the fccc without confronting central questions of climate justice concerning how to share the responsibilities of slowing climate change fairly between developed nations and developing nations. False True False False True False False False True False False False False False False 0.7222 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +149 unstructured 23456 23607 wikiClimate_justice the issue of the fair terms for sharing responsibility was raised forcefully for the inc by statements about climate justice from developing countries. False True False False True False False False False False False False False False False 0.8176 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +150 unstructured 23608 24103 wikiClimate_justice "[100] in response, the fccc adopted the now-famous (and still-contentious) principles of climate justice embodied in article 3.1:[101] ""the parties should protect the climate system for the benefit of present and future generations of humankind, on the basis of equity and in accordance with their common but differentiated responsibilities and respective capabilities. accordingly, the developed country parties should take the lead in combating climate change and the adverse effects thereof.""" False True False False True False False False False False False False False False False 0.8705 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +151 unstructured 24104 24307 wikiClimate_justice the first principle of climate justice embedded in article 3.1 is that calculations of benefits (and burdens) must include not only those for the present generation but also those for future generations. False True False False True False False False False False False False False False False 0.3071 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +152 unstructured 24308 24505 wikiClimate_justice "the second is that responsibilities are ""common but differentiated"", that is, every country has some responsibilities, but equitable responsibilities are different for different types of countries." False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +153 unstructured 24506 24647 wikiClimate_justice the third is that a crucial instance of different responsibilities is that in fairness developed countries' responsibilities must be greater. False False False False False False False False False False False False False False False 0.3612 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +154 unstructured 24648 24701 wikiClimate_justice how much greater continues to be debated politically. False False False False False False False False False False False False False False False 0.3612 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +155 unstructured 24714 24845 wikiClimate_justice in 2000, at the same time as the sixth conference of the parties (cop 6), the first climate justice summit took place in the hague. False True False False True False False False False False False False False False False 0.7269 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +156 unstructured 24846 25028 wikiClimate_justice "this summit aimed to ""affirm that climate change is a rights issue"" and to ""build alliances across states and borders"" against climate change and in favor of sustainable development." False True False False True False False False False False False False False False False 0.4019 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +157 unstructured 25036 25152 wikiClimate_justice subsequently, in august-september 2002, international environmental groups met in johannesburg for the earth summit. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +158 unstructured 25153 25310 wikiClimate_justice [105] at this summit, also known as rio+10, as it took place ten years after the 1992 earth summit, the bali principles of climate justice[106] were adopted. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +159 unstructured 25312 25545 wikiClimate_justice climate justice affirms the rights of communities dependent on natural resources for their livelihood and cultures to own and manage the same in a sustainable manner, and is opposed to the commodification of nature and its resources. False True False False True False False False False True False False False False False 0.7717 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +160 unstructured 25547 25615 wikiClimate_justice bali principles of climate justice, article 18, august 29, 2002[106] False True False False True False False False False False False False False False False 0.5267 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +161 unstructured 25617 25726 wikiClimate_justice in 2004, the durban group for climate justice was formed at an international meeting in durban, south africa. False True False False True False False False False False False False False False False 0.5267 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +162 unstructured 25727 25840 wikiClimate_justice here representatives from ngos and peoples' movements discussed realistic policies for addressing climate change. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +163 unstructured 25848 26073 wikiClimate_justice in 2007 at the 13th conference of the parties (cop 13) in bali, the global coalition climate justice now! was founded, and, in 2008, the global humanitarian forum focused on climate justice at its inaugural meeting in geneva. False True False False True False False False False False False False False False False 0.908 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +164 unstructured 26081 26179 wikiClimate_justice in 2009, the climate justice action network was formed during the run-up to the copenhagen summit. False True False False True False False False False False False False False False False 0.5267 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +165 unstructured 26180 26332 wikiClimate_justice [109] it proposed civil disobedience and direct action during the summit, and many climate activists used the slogan 'system change not climate change'. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +166 unstructured 26340 26467 wikiClimate_justice in april 2010, the world people's conference on climate change and the rights of mother earth took place in tiquipaya, bolivia. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +167 unstructured 26468 26566 wikiClimate_justice it was hosted by the government of bolivia as a global gathering of civil society and governments. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +168 unstructured 26567 26672 wikiClimate_justice "the conference published a ""people's agreement"" calling, among other things, for greater climate justice." False True False False True False False False False False False False False False False 0.8442 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +169 unstructured 26679 26945 wikiClimate_justice in september 2013 the climate justice dialogue convened by the mary robinson foundation and the world resources institute released their declaration on climate justice in an appeal to those drafting the proposed agreement to be negotiated at cop-21 in paris in 2015. False True False False True False False False False True False False False False False 0.875 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +170 unstructured 26953 27161 wikiClimate_justice in december 2018, the people's demands for climate justice, signed by 292,000 individuals and 366 organizations, called upon government delegates at cop24 to comply with a list of six climate justice demands. False True False False True False False False False False False False False False False 0.7783 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +171 unstructured 27162 27281 wikiClimate_justice "[112] one of the demands was to ""ensure developed countries honor their ""fair shares"" for largely fueling this crisis.""" False False False False False False False False False False False False False False False 0.5307 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +172 unstructured 27283 27358 wikiClimate_justice some advance was achieved at the paris climate finance summit at june 2023. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +173 unstructured 27359 27472 wikiClimate_justice the world bank allowed to low income countries temporarily stop paying debts if they are hit by climate disaster. False True False False True False False False False False False False False False False -0.8126 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +174 unstructured 27473 27640 wikiClimate_justice most of financial help to climate vulnerable countries is coming in the form of debts, what often worsens the situation as those countries are overburdened with debts. False True False False True False False False False False False False False False False -0.2585 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +175 unstructured 27641 27770 wikiClimate_justice around 300 billion dollars was pledged as financial help in the next years, but trillions are needed to really solve the problem. False False False False False False False False False False False False False False False -0.1169 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +176 unstructured 27771 27921 wikiClimate_justice [113][114] more than 100 leading economists signed a letter calling for an extreme wealth tax as a solution (2% tax can generate around 2.5 trillion). False False False False False False False False False False False False False False False 0.6997 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +177 unstructured 27922 28056 wikiClimate_justice it can serve as a loss and damage mechanism as the 1% of richest people is responsible for twice as many emissions as the poorest 50%. False False False False False False False False False False False False False False False -0.5106 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +178 unstructured 28057 28108 wikiClimate_justice [115] examples subsistence farmers in latin america False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +179 unstructured 28110 28370 wikiClimate_justice several studies that investigated the impacts of climate change on agriculture in latin america suggest that in the poorer countries of latin america, agriculture composes the most important economic sector and the primary form of sustenance for small farmers. False True False False True False False False False False False False False False False -0.1045 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +180 unstructured 28371 28483 wikiClimate_justice [116][117] maize is the only grain still produced as a sustenance crop on small farms in latin american nations. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +181 unstructured 28484 28641 wikiClimate_justice [116] the projected decrease of this grain and other crops can threaten the welfare and the economic development of subsistence communities in latin america. False False False False False False False False True False False False False True False -0.3818 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +182 unstructured 28642 28789 wikiClimate_justice [118][116] food security is of particular concern to rural areas that have weak or non-existent food markets to rely on in the case food shortages. False False False False False False False False False False False False False False False -0.2732 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +183 unstructured 28790 28951 wikiClimate_justice [119] in august 2019, honduras declared a state of emergency when a drought caused the southern part of the country to lose 72% of its corn and 75% of its beans. True False False False True False False False False False False False False False False -0.6486 False False False True True False False False False False True True False False False False False False False True True False False False False False False False False False +184 unstructured 28952 29041 wikiClimate_justice food security issues are expected to worsen across central america due to climate change. False True False False True False False False False False False False False False False -0.2263 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +185 unstructured 29042 29150 wikiClimate_justice it is predicted that by 2070, corn yields in central america may fall by 10%, beans by 29%, and rice by 14%. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +186 unstructured 29151 29319 wikiClimate_justice with central american crop consumption dominated by corn (70%), beans (25%), and rice (6%), the expected drop in staple crop yields could have devastating consequences. False False False False False False False False False False False False False False False -0.7506 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +187 unstructured 29327 29462 wikiClimate_justice the expected impacts of climate change on subsistence farmers in latin america and other developing regions are unjust for two reasons. False True False False True False False False False False False False False False False -0.5106 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +188 unstructured 29463 29722 wikiClimate_justice [117][121] first, subsistence farmers in developing countries, including those in latin america are disproportionately vulnerable to climate change[121] second, these nations were the least responsible for causing the problem of anthropogenic induced climate. False True False False True False False False False False False False False False False -0.6769 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +189 unstructured 29723 29750 wikiClimate_justice [121][better source needed] False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +190 unstructured 29752 29827 wikiClimate_justice disproportionate vulnerability to climate disasters is socially determined. False True False False True False False False False False False False False False False -0.4767 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +191 unstructured 29828 29970 wikiClimate_justice [117][121] for example, socioeconomic and policy trends affecting smallholder and subsistence farmers limit their capacity to adapt to change. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +192 unstructured 29971 30059 wikiClimate_justice [117] a history of policies and economic dynamics has negatively impacted rural farmers. False False False False False False False False False False False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +193 unstructured 30060 30199 wikiClimate_justice [116] during the 1950s and through the 1980s, high inflation and appreciated real exchange rates reduced the value of agricultural exports. False False False False False False False False False False False False False True False 0.6908 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +194 unstructured 30200 30317 wikiClimate_justice [116] as a result, farmers in latin america received lower prices for their products compared to world market prices. False False False False False False False False False False False False False False False -0.296 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +195 unstructured 30318 30449 wikiClimate_justice [116] following these outcomes, latin american policies and national crop programs aimed to stimulate agricultural intensification. False False False True False False False False False False False False False False False 0.2263 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +196 unstructured 30450 30527 wikiClimate_justice [116] these national crop programs benefitted larger commercial farmers more. False False False False False False False False False False False False False False False 0.4019 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +197 unstructured 30528 30671 wikiClimate_justice in the 1980s and 1990s low world market prices for cereals and livestock resulted in decreased agricultural growth and increased rural poverty. False False False True False False False False False False False False True True False -0.1779 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +198 unstructured 30679 30815 wikiClimate_justice perceived vulnerability to climate change differs even within communities, as in the example of subsistence farmers in calakmul, mexico. False True False False True False False False False False False False False False False -0.2263 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +199 unstructured 30823 30922 wikiClimate_justice adaptive planning is challenged by the difficulty of predicting local scale climate change impacts. False True False False True False False False False False False False False False True -0.4215 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +200 unstructured 30923 31049 wikiClimate_justice [117] a crucial component to adaptation should include government efforts to lessen the effects of food shortages and famines. False False False False False False False False False False False False False False True -0.1531 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +201 unstructured 31050 31190 wikiClimate_justice [123] planning for equitable adaptation and agricultural sustainability will require the engagement of farmers in decision making processes. False False False False False False False False False False False False False False True 0.4588 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +202 unstructured 31191 31385 wikiClimate_justice [123] hurricane katrina a house is crushed and swept off its foundations by flooding from a breached levee in the ninth ward, new orleans, louisiana, due to a storm surge from hurricane katrina. True False False True True False False False False False False False False False False -0.4215 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +203 unstructured 31386 31437 wikiClimate_justice around 90% of the ninth ward's population is black. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +204 unstructured 31438 31495 wikiClimate_justice further information: tropical cyclones and climate change True True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +205 unstructured 31497 31677 wikiClimate_justice due to climate change, tropical cyclones are expected to increase in intensity and have increased rainfall, and have larger storm surges, but there might be fewer of them globally. True True False True True False False False False False False False True False False 0.296 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +206 unstructured 31678 31811 wikiClimate_justice these changes are driven by rising sea temperatures and increased maximum water vapour content of the atmosphere as the air heats up. False False False True True False False False False True False False True False False 0.2732 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +207 unstructured 31812 31993 wikiClimate_justice [124] hurricane katrina in 2005 showed how climate change disasters affect different people individually,[125] as it had a disproportionate effect on low-income and minority groups. True True False False True False False False False False False False False False False -0.5574 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +208 unstructured 31994 32161 wikiClimate_justice [125] a study on the race and class dimensions of hurricane katrina suggests that those most vulnerable include poor, black, brown, elderly, sick, and homeless people. True False False False True False False False False False False False False False False -0.8341 False False False True True False False False False False True True False False False False False False False True True False False False False False False False False False +209 unstructured 32162 32272 wikiClimate_justice [126] low-income and black communities had little resources and limited mobility to evacuate before the storm. True False False False True False False False False True False False False False False -0.1621 False False False True True False False False False False True True False False False False False False False True True False False False False False False False False False +210 unstructured 32273 32485 wikiClimate_justice [127][128] after the hurricane, low-income communities were most affected by contamination,[125] and this was made worse by the fact that government relief measures failed to adequately assist those most at risk. True False False False True False False False False False False False False False False -0.763 False False False True True False False False False False True True False False False False False False False True True False False False False False False False False False +1 unstructured 0 16 wikiClimate_security climate security False True False False True False False False False False False False False False False 0.34 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +2 unstructured 18 30 wikiClimate_security article talk False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +3 unstructured 32 54 wikiClimate_security read edit view history False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +4 unstructured 56 61 wikiClimate_security tools False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +5 unstructured 63 130 wikiClimate_security "from wikipedia, the free encyclopedia ""climate war"" redirects here." False True False False True False True False False False False False False False False -0.1531 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +6 unstructured 131 219 wikiClimate_security "for public debate or ""culture war"" about climate change, see global warming controversy." False True False False True False True False False False False False False False False -0.5106 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +7 unstructured 220 355 wikiClimate_security map of the earth with a six-meter sea level rise represented in red (uniform distribution, actual sea level rise will vary regionally). False True False True False False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +8 unstructured 356 497 wikiClimate_security hotspots of slr can divert 3-4 times in the rate of rise, compared to the global average, such as projected for parts of the u.s. east coast. False False False True False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +9 unstructured 503 605 wikiClimate_security climate security is a political and policy framework that looks at the impacts of climate on security. False True False False True False False False False False False False False False False 0.5859 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +10 unstructured 606 749 wikiClimate_security climate security often refers to the national and international security risks induced, directly or indirectly, by changes in climate patterns. False True False False True False False False False False False False False False False 0.4019 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +11 unstructured 750 941 wikiClimate_security it is a concept that summons the idea that climate-related change amplifies existing risks in society that endangers the security of humans, ecosystems, economy, infrastructure and societies. False True False False True False False True False True False False False False False 0.0772 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +12 unstructured 942 1053 wikiClimate_security climate-related security risks have far-reaching implications for the way the world manages peace and security. False True False False True False False False False False False False False False False 0.7351 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +13 unstructured 1054 1164 wikiClimate_security climate actions to adapt and mitigate impacts can also have a negative effect on human security if mishandled. False True False False True False False False False False False False False False False -0.3182 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +14 unstructured 1170 1433 wikiClimate_security the term climate security was initially promoted by national security analysts in the us and later europe, but has since been adopted by a wide variety of actors including the united nations, low and middle income states, civil society organizations and academia. False True False False True False False False False False False False False False False 0.6542 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +15 unstructured 1434 1541 wikiClimate_security the term is used in fields such as politics, diplomacy, environment and security with increasing frequency. False False False True False False False False False False False False True False False 0.34 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +16 unstructured 1543 1781 wikiClimate_security there are also critics of the term who argue that the term encourages a militarized response to the climate crisis, and ignores issues of maldistribution and inequity that underpin both the climate crisis and vulnerability to its impacts. False True False False True False False False False False False False False False False -0.9169 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +17 unstructured 1787 1979 wikiClimate_security those who look at the national and international security risks argue that climate change has the potential to exacerbate existing tensions or create new ones - serving as a threat multiplier. False True False True True False False False True False False False False False False -0.7269 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +18 unstructured 1980 2192 wikiClimate_security [4] for example, climate change is seen as a threat to military operations and national security, as the rise in sea level can affect military bases or extreme heat events can undermine the operability of armies. True True False True True False True False True False False False False True False -0.4939 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +19 unstructured 2193 2359 wikiClimate_security climate change is also seen as a catalyst for violent conflict and a threat to international security, although the causality of climate and conflict is also debated. False True False False True False False False True False True False False False False -0.8591 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +20 unstructured 2360 2594 wikiClimate_security [5][6] due to the growing importance of climate security on the agendas of many governments, international organizations, and other bodies some now run programs which are designed to mitigate the effects of climate change on conflict. False True False False True False False False True False True False True False False 0.5106 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +21 unstructured 2595 2651 wikiClimate_security these practices are known as climate security practices. False True False False True False False False False False False False False False False 0.34 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +22 unstructured 2652 2848 wikiClimate_security these practices stem from a variety of actors with different motivations in the sphere of development, diplomacy and defense; both nato and the un security council are involved in these practices. False False False False False False False False False False False False False False False 0.4404 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +23 unstructured 2849 2859 wikiClimate_security definition False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +24 unstructured 2861 2922 wikiClimate_security climate security looks at the impacts of climate on security. False True False False True False False False False False False False False False False 0.5859 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +25 unstructured 2923 3066 wikiClimate_security climate security often refers to the national and international security risks induced, directly or indirectly, by changes in climate patterns. False True False False True False False False False False False False False False False 0.4019 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +26 unstructured 3067 3265 wikiClimate_security [7][8] it is a concept that summons the idea that climate-related change amplifies existing risks in society that endangers the security of humans, ecosystems, economy, infrastructure and societies. False True False False True False False True False True False False False False False 0.0772 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +27 unstructured 3266 3356 wikiClimate_security [7][9] background see also: effects of climate change on humans and environmental security False True False False True False False False False False False False False False False 0.34 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +28 unstructured 3358 3468 wikiClimate_security climate security refers to the security risks induced, directly or indirectly, by changes in climate patterns. False True False False True False False False False False False False False False False 0.4019 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +29 unstructured 3469 3623 wikiClimate_security climate change has been identified as a severe-to-catastrophic threat to international security in the 21st century by multiple risk and security reports. False True False False True False False False True False False False False False False -0.1779 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +30 unstructured 3624 3792 wikiClimate_security [10][11] the 2020 global catastrophic risks report, issued by the global challenges foundation, concluded that climate change has a high likelihood to end civilization. False True False False True False False False False False False False False False False -0.6124 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +31 unstructured 3793 3887 wikiClimate_security [12] 70% of international governments consider climate change to be a national security issue. False True False False True False False False False False False False False False False 0.34 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +32 unstructured 3888 4065 wikiClimate_security [13] policy interest in climate security risks has grown rapidly and affects the policy agenda in relation to food and energy security, migration policy, and diplomatic efforts. False True False False True False False False False True False False True False False 0.7783 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +33 unstructured 4066 4172 wikiClimate_security usda, projected changes in food prices, depending on emissions scenarios, also factoring trade development False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +34 unstructured 4174 4437 wikiClimate_security the term climate security was initially promoted by national security analysts in the us and later europe, but has since been adopted by a wide variety of actors including the united nations, low and middle income states, civil society organizations and academia. False True False False True False False False False False False False False False False 0.6542 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +35 unstructured 4437 4562 wikiClimate_security [citation needed] the term is used in fields such as politics, diplomacy, environment and security with increasing frequency. False False False True False False False False False False False False True False False 0.34 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +36 unstructured 4564 4706 wikiClimate_security within academia, climate security emerged from a discourse of environmental security and was first mentioned in the brundtland report in 1987. False True False False True False False False False False False False False False False 0.5859 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +37 unstructured 4707 4825 wikiClimate_security [14] during the 70's and 80's the jason advisory group, concerned with security, conducted research on climate change. False True False False True False False False False False False False False False False 0.34 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +38 unstructured 4826 4982 wikiClimate_security [15] global climate change became an international issue with the broadening of the concept of security which emerged in the 1980s in the post-cold war era. False True False False True False True False False False False False False False False -0.3612 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +39 unstructured 4983 5158 wikiClimate_security the broadening of the concept of security sought to look beyond the military domain, and include political, economic, societal, and environmental areas in the security agenda. False False False False False False False False False False False False False False False 0.5859 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +40 unstructured 5159 5288 wikiClimate_security [16] the term security can refer to a broad range of securities including national, international, ecological and human security. False False False False False False False False False False False False False False False 0.7184 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +41 unstructured 5290 5533 wikiClimate_security to map the different ways in which climate change is conceptualized, scholar matt mcdonald identifies four discourses of climate security advanced by policymakers, lobbyists, environmental advocates, civil society groups and academic analysts. False True False False True False False False False False False False False False False 0.5267 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +42 unstructured 5534 5720 wikiClimate_security he divides them into national, human, international and ecological types of security which respectively concern nation-states, 'people', the international community, and the 'ecosystem'. False False False False False False False True False True False False False False False 0.5859 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +43 unstructured 5721 5953 wikiClimate_security [17] national climate security is the most dominant of the four discourses as it focuses on the threat climate change poses to nation-states and the maintenance of its sovereignty and 'territorial integrity' from an external threat. False True False False True False False False True False False False False False False -0.4215 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +44 unstructured 5954 6125 wikiClimate_security [17] this discourse is advanced by national security institutions where the nation-state is viewed as the most capable provider of security through the military apparatus. False False False False False False False False False False False False False False False 0.8268 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +45 unstructured 6126 6231 wikiClimate_security this discourse has also been advanced by policy think tanks embracing the concept of 'threat multiplier'. False False False False False False False False True False False False False False False -0.34 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +46 unstructured 6238 6396 wikiClimate_security the international security discourse focuses on internationalism and global cooperation where international organizations are viewed as providers of security. False False False False False False False False False False False False False False False 0.5859 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +47 unstructured 6397 6591 wikiClimate_security mitigation and adaptation strategies are central to this discourse, such as the transition to low carbon economies and the transfer of technology, sources, and expertise to developing countries. False False False False False False False False False False False False False False True -0.2732 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +48 unstructured 6592 6773 wikiClimate_security international organizations such as the un environment program are involved in these processes and the more recent sustainable development goals are an embodiment of such discourse. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +49 unstructured 6774 7095 wikiClimate_security however, the un security council plays a central role as the securitizing agent, which has been often criticized by developing countries, such as group 77 and the non-aligned movement, as they are concerned climate change will be used to justify military intervention and increased military budgets by powerful countries. False True False True True False False False False False False False True False False 0.7579 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +50 unstructured 7096 7264 wikiClimate_security [17] the human security discourse emerged as a counteracting alternative to national security, and was embraced first by the united nations development program in 1994. False False False False False False False False False False False False False False False 0.765 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +51 unstructured 7265 7327 wikiClimate_security it seeks to center the wellbeing of people rather than states. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +52 unstructured 7328 7464 wikiClimate_security for the un agencies, mitigation strategies and the redistribution of resources are seen as central to providing security to populations. False False False False False False False False False True False False False False False 0.34 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +53 unstructured 7465 7557 wikiClimate_security the ecological security discourse is seldom included in dominant policy or academic debates. False False False False False False False False False False False False False False False 0.34 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +54 unstructured 7564 7838 wikiClimate_security while many international relations (ir) scholars link climate change with security and conflict through a traditional military approach, there is an ongoing debate on whether climate change and environmental issues should be securitized and who and what is really protected. False True False False True False False False True False True False False False False 0.6901 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +55 unstructured 7839 8032 wikiClimate_security the scholars who theorized the concept of securitization allowed to deepen and broaden concepts of security beyond traditional military security through discourse methodology and 'speech acts.' False False False False False False False False False False False False False False False 0.6124 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +56 unstructured 8033 8324 wikiClimate_security for example, copenhagen school scholars, such as barry buzan and ole wæver, argue that security justifies urgency and exceptionalism, focusing on defense, the military and the state and that climate change should instead be placed into 'normal politics' and removed from the security agenda. False True False False True False False False False False False False False False False 0.4404 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +57 unstructured 8325 8603 wikiClimate_security furthermore, some scholars note how securitization theory, stemming as a response to traditional realism theory in the post-cold war era, is mostly a eurocentric field and does not include the legacies of colonialism and racial hierarchies inform global politics and governance. False False False False False False True False False False False False False False False -0.5719 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +58 unstructured 8610 8832 wikiClimate_security with the first ipcc assessment report in 1990 and the 1992 united nations framework convention on climate change, the scale of climate change impacts pushed international actors to view climate change as a security threat. False True False False True False False False True False False True False False False 0.2023 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +59 unstructured 8832 9237 wikiClimate_security "[citation needed] a report in 2003 by peter schwartz and doug randall looked at potential implications from climate-related scenarios for the national security of the united states, and concluded, ""we have created a climate change scenario that although not the most likely, is plausible, and would challenge united states national security in ways that should be considered immediately.""[citation needed]" False True False False True False False False False False False False False False False 0.8934 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +60 unstructured 9239 9409 wikiClimate_security in 2008, the eu published a report on climate change and international security, defining climate change as a 'threat multiplier' affecting eu own security and interests. False True False False True False False False True False False False False False False 0.34 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +61 unstructured 9410 9441 wikiClimate_security [19] critiques and alternatives False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +62 unstructured 9443 9649 wikiClimate_security the climate security approach has become prominent among political and policy spheres and has been called inevitable by some countries, inviting the un security council to adopt more militarized approaches. False True False False True False False False False False False False False False False 0.8442 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +63 unstructured 9649 9939 wikiClimate_security [citation needed] however, some scholars and activists criticize climate security, arguing that framing climate change as a security issue can be problematic as it could increase solutions that rely on militaries which can worsen the injustices of those most affected by the climate crisis. False True False True True False False False False False False False True False False -0.8748 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +64 unstructured 9940 10106 wikiClimate_security this can also mean that security solutions end up benefiting the status quo, ignoring the well-being of the rest, such as refugees and other marginalized communities. False False False False False False False False False False False False False False False 0.1027 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +65 unstructured 10106 10123 wikiClimate_security [citation needed] False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +66 unstructured 10125 10286 wikiClimate_security the climate security approach has also a significant impact on borders and migration, as its narrative emphasizes the 'threat' of climate-induced mass migration. False True False False True False False False True False False False False False False -0.0516 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +67 unstructured 10287 10369 wikiClimate_security indeed, the border industrial complex is expected to grow globally by 7% annually. False False False False False False False False False False False False True False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +68 unstructured 10370 10672 wikiClimate_security "[20] as the transnational institute report ""global climate wall"" shows, the seven biggest ghg emitters the united states, germany, japan, the united kingdom, canada, france, and australia spent collectively at least twice on border and immigration control than on climate finance between 2013 and 2018." False True False False True False False False False False False False False False False 0.6808 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +69 unstructured 10673 10774 wikiClimate_security [21] the eu's budget for frontex has increased by 2763% since its establishment in 2016 through 2021. False False False True False False False False False False False False True False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +70 unstructured 10781 11041 wikiClimate_security social movements and organizations, such as climate justice alliance, we are dissenters, grassroots global justice alliance, indigenous environmental network, call for a bigger emphasis on climate justice and environmental justice rather than climate security. False True False False True False False False False False False False False False False 0.9432 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +71 unstructured 11042 11264 wikiClimate_security climate justice puts the emphasis on the root causes of climate change, like colonialism and neocolonialism, global inequality, globalization and exploitative economic systems such as the exploitation of natural resources. False True False False True False False False False True False False False False False 0.8126 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +72 unstructured 11265 11322 wikiClimate_security many call this addressing the era of climate colonialism. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +73 unstructured 11323 11537 wikiClimate_security indeed, many proponents of climate justice call for bigger support for indigenous people and other frontline communities that are fighting for climate change and also already protecting 80% of earth's biodiversity. False True False False True False False False False False False False False False False 0.5574 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +74 unstructured 11544 11799 wikiClimate_security many civil society actors also call for climate reparations on top of more climate finance, and also the establishment of loss and damage finance facility (ldff),[23] which has been proposed by low-income countries, as well as sovereign debt cancellation. False True False False True False False False False False False False False False False -0.6249 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +75 unstructured 11800 11923 wikiClimate_security this way, low-income countries could tackle the impacts of climate change for which they are bear the least responsibility. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +76 unstructured 11925 12180 wikiClimate_security from an academic standpoint, the concept of ecological security, allows for a more systemic approach to climate change that examines the structural roots of the climate crisis as the overlapping economic, political, and social issues of the global system. False True False False True False False False False False False False False False False -0.4019 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +77 unstructured 12181 12309 wikiClimate_security [24] effects of climate change further information: effects of climate change, climate change vulnerability, and climate justice False True False False True False False False False False False False False False False 0.3612 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +78 unstructured 12311 12473 wikiClimate_security climate change is a global challenge which will affect all countries in the long-term as the impact of climate change is spread unevenly across different regions. False True False False True False False False False False False False False False False 0.0772 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +79 unstructured 12474 12639 wikiClimate_security however, there may be a disproportionately harsher effect in fragile contexts and/or socially vulnerable and marginalized groups due to climate change vulnerability. False True False False True False False False False False False False False False False -0.7184 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +80 unstructured 12640 12799 wikiClimate_security for example, the bay of bengal, which includes bangladesh, myanmar, india, indonesia, and sri lanka, is one of the world's most climate-vulnerable regions and. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +81 unstructured 12800 13046 wikiClimate_security marginalized groups and minority communities, both in the global north and global south, are most affected by the effects of climate change, for which they are least responsible, which many call environmental injustice (see also climate justice). False True False False True False False False False False False False False False False -0.4862 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +82 unstructured 13053 13323 wikiClimate_security indeed, the richest 10% (circa 630 million people), of the world's population, mostly from eu and north america, are in fact responsible for 52% of carbon emissions, whereas the poorest 50% (circa 3.1 billion people) were responsible for only 7% of cumulative emissions. False True False False False False False False False False False False False False False 0.5423 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +83 unstructured 13324 13455 wikiClimate_security the global north is responsible for 92 percent of ghg emissions and climate change is devastating the global south (external link). False True False False True False False False False False False False False False False -0.4588 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +84 unstructured 13456 13670 wikiClimate_security "to account for this, the unfccc embodies the notion of ""common but differentiated responsibilities (cbdr)"" which addresses developed countries' responsibility to transfer aid and technology to developing countries." False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +85 unstructured 13671 13734 wikiClimate_security [26] this section is an excerpt from effects of climate change. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +86 unstructured 13734 13821 wikiClimate_security [edit] climate change affects the physical environment, ecosystems and human societies. False True False False True False False True False True False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +87 unstructured 13822 13929 wikiClimate_security changes in the climate system include an overall warming trend, more extreme weather and rising sea levels. True True False False True False False False False False False False False False False 0.1531 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +88 unstructured 13930 14015 wikiClimate_security these in turn impact nature and wildlife, as well as human settlements and societies. False False False False False False False True False False False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +89 unstructured 14016 14091 wikiClimate_security [27] the effects of human-caused climate change are broad and far-reaching. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +90 unstructured 14092 14156 wikiClimate_security this is especially so if there is no significant climate action. False True False False True False False False False False False False False False False -0.1511 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +91 unstructured 14157 14268 wikiClimate_security experts sometimes describe the projected and observed negative impacts of climate change as the climate crisis. False True False False True False False False False False False False False False False -0.8316 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +92 unstructured 14269 14317 wikiClimate_security this section is an excerpt from climate justice. False True False False True False False False False False False False False False False 0.5267 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +93 unstructured 14317 14479 wikiClimate_security [edit] climate justice is an approach to climate action that focuses on the unequal impacts of climate change on marginalized or otherwise vulnerable populations. False True False False True False False False False False False False False False False 0.0258 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +94 unstructured 14480 14625 wikiClimate_security [28] climate justice wants to achieve an equitable distribution of both the burdens of climate change and the efforts to mitigate climate change. False True False False True False False False False False False False False False False 0.2263 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +95 unstructured 14626 14682 wikiClimate_security [29] climate justice is a type of environmental justice. False True False False True False False False False False False False False False False 0.7783 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +96 unstructured 14683 14697 wikiClimate_security [30] conflicts False False False False False False False False True False True False False False False -0.3818 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +97 unstructured 14699 14882 wikiClimate_security studies have shown that extreme weather events can damage economies, lower food production and raise inequality, which can increase risks of violence when combined with other factors. True False False True True False False False False False True False True False False -0.8519 False False False True True False False False False False True True False False False False False False False True True False False False False False False False False False +98 unstructured 14883 15208 wikiClimate_security an article by leading experts found that climate change has influenced between 3% and 20% of armed conflict in the last century, that an increase of 2°c above pre-industrial levels more than doubles the current risk of conflict, increasing it to 13%, and that an increase of 4°c multiplies the risk by five, up to a 26% risk. False True False True True False True False True False True False True False False -0.6486 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +99 unstructured 15209 15333 wikiClimate_security [31] other researchers have specified the conditions under which climate change increases the risk of conflict and violence. False True False True True False False False True False True False True False False -0.8176 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +100 unstructured 15334 15544 wikiClimate_security these include a history of political instability, agricultural dependence, low levels of development, the political of ethnic groups from the political system, and insufficient conflict resolution institutions. False False False False False False True False True False True False False False False -0.5267 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +101 unstructured 15559 15783 wikiClimate_security a report by the global peace index found that 971 million people lived in areas with either a high or very high climate change exposure and that 400 million of those people lived in countries with low levels of peacefulness. False True False False True False False False False False False False False False False 0.6705 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +102 unstructured 15784 15949 wikiClimate_security it warned that climate change can increase the likelihood of violent conflict by impacting upon resource availability, job security, and by causing forced migration. False True False True True False True False True True True False True False False -0.765 False False False True True True False True False False True False True False False False False False False True False True False False False False False False False False +103 unstructured 15950 16094 wikiClimate_security [35][36] predicting future risks of climate change and conflict remains difficult, despite the existence of several predictive models and tools. False True False False True False False False True False True False False False False -0.7096 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +104 unstructured 16095 16252 wikiClimate_security [37][38] future climate change is likely to be very different from what humanity has experienced previously and the ability of societies to adapt is unclear. False True False False True False False False False False False False False False False 0.0772 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +105 unstructured 16253 16287 wikiClimate_security [31][39] climate threats and risks False True False False True False False False True False False False False False False -0.5994 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +106 unstructured 16289 16359 wikiClimate_security the secondary impacts of climate hazards could be even more dangerous. False True False False True False False False False False False False False False False -0.5256 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +107 unstructured 16360 16488 wikiClimate_security chief among them is an increased risk of armed conflict in places where established social orders and populations are disrupted. False False False True False False True False True False True False True False False -0.3182 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +108 unstructured 16489 16713 wikiClimate_security the risk will rise even more where climate effects compound social instability, reduce access to basic necessities, undermine fragile governments and economies, damage vital infrastructure, and lower agricultural production. False True False True True False False False False False False False False True False -0.7579 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +109 unstructured 16715 16759 wikiClimate_security u.s. army climate strategy[40] february 2022 False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +110 unstructured 16761 16892 wikiClimate_security a 2016 article suggested that conflict over climate-related water issues could lead to nuclear conflict between india and pakistan. False True False False True False False False True True True False False False False -0.5574 False False False True True False False True False False True False True False False False False False False True False True False False False False False False False False +111 unstructured 16893 17102 wikiClimate_security [41] however, other scholars believe that climate change are unlikely to have major impacts on the nature of interstate wars, but have expressed concerns about its impacts on civil wars and communal conflicts. False True False False True False True False True False True False False False False -0.891 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +112 unstructured 17103 17345 wikiClimate_security [32][42][43] based on a meta-analysis of 60 studies, hsiang, burke and miguel concluded in 2013 that warmer temperatures and more extreme rainfall could increase interpersonal violence by 4%, and intergroup conflict by 14% (median estimates). True False False True True False False False True False True False True False False -0.3892 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +113 unstructured 17346 17483 wikiClimate_security [44] however, their results have been disputed by other researchers as being not sufficiently robust to alternative model specifications. False False False False False False False False False False False False False False False -0.5324 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +114 unstructured 17494 17894 wikiClimate_security recent studies have been more careful and agree that climate-related disasters (including heatwaves, droughts, storms and floods) modestly increase armed conflict risks, but only in the presence of contextual factors like agricultural dependence,[47] insufficient infrastructure,[48] the political exclusion of ethnic groups,[49] insufficient conflict management,[42] and high disaster vulnerability. True True False True True False True False True False True False True False False -0.8665 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +115 unstructured 17895 17997 wikiClimate_security "[50] climate change is therefore rather a ""risk multiplier"" that amplifies existing risks of conflict." False True False False True False False False True False True False False False False -0.6705 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +116 unstructured 17998 18301 wikiClimate_security [51] in line with this and other reviews of the topic,[32][52] an expert assessment published 2019 in nature concludes that between 3% and 20% of intrastate, armed conflict risks in the previous century were affected by climate-related factors, but that other drivers of conflict are far more important. False True False False True False True False True False True False False False False -0.4235 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +117 unstructured 18302 18504 wikiClimate_security [43] the expert assessment itself notes that major knowledge gaps and uncertainties continue to exist in the research field, especially regarding the pathways connecting climate change to conflict risk. False True False False True False False False True False True False False False False -0.7003 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +118 unstructured 18511 18677 wikiClimate_security recently, researchers have paid increased attention to the impacts of climate change on low-intensity and even non-violent conflicts, such as riots or demonstrations. False True False True True False True False True False True False True False False -0.5859 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +119 unstructured 18678 18852 wikiClimate_security even if people do not have the means or motivation to use violence, they can engage in such forms of conflict, for instance in the face of high food prices or water scarcity. False False False False False False False False True True True False False False False -0.3818 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +120 unstructured 18853 19020 wikiClimate_security studies indeed show that in vulnerable societies, the anticipated consequences of climate change such as reduced food and water security increase the risk of protests. False True False True True False False False False True False False True True False -0.0516 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +121 unstructured 19021 19131 wikiClimate_security [53][54][55] these conflicts often add to and trigger the escalation of deeper social and political struggles. False False False False False False False False True False True False False False False -0.6249 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +122 unstructured 19133 19382 wikiClimate_security on a country by country basis, several case studies have linked climate change to increased violent conflicts between farmers and herders in kenya[56] and sudan,[57] but have found mixed results for ghana,[58] mali,[59] nigeria[60][61] and tanzania. False True False True True False False False True False True False True False False -0.4019 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +123 unstructured 19383 19484 wikiClimate_security [62] evidence is also ambiguous and highly contested for high-intensity conflicts such as civil wars. False False False False False False True False True False True False False False False -0.7351 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +124 unstructured 19485 19601 wikiClimate_security some experts suggest a contribution of climate change to civil wars in mali,[63] nigeria,[64] somalia[65] and sudan. False True False False True False True False False False False False False False False -0.5574 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +125 unstructured 19602 19757 wikiClimate_security [66] other studies suggest that there is very little evidence for these causal claims,[61] including for the cases of darfur,[67] egypt,[68] and lake chad. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +126 unstructured 19764 19832 wikiClimate_security the most prominent example of these debates is the syrian civil war. False False False False False False True False False False False False False False False -0.3197 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +127 unstructured 19833 20020 wikiClimate_security several studies claim that a climate-induced drought between 2006 and 2009 led to mass migration into urban areas, contributing to grievances and unrest that erupted in the 2011 protests. True True False False True False False False False False False False False False False -0.5267 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +128 unstructured 20021 20084 wikiClimate_security the repression of the latter marked the start of the civil war. False False False False False False True False False False False False False False False -0.7579 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +129 unstructured 20085 20281 wikiClimate_security [70][71] a team around jan selby argues that these claims are overstated and that political decisions and mismanagement, rather than climate change and migration, have caused the onset of the war. False True False False True False True False False False False False False False False -0.7579 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +130 unstructured 20282 20443 wikiClimate_security [72][73] several recent studies find that the debate is not yet settled because there is evidence for both positions, yet a lack of comprehensive empirical data. False False False False False False False False False False False False False False False -0.0772 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +131 unstructured 20454 20765 wikiClimate_security there are a number of studies that criticize how climate-conflict research is based on a deterministic and conflict-oriented worldview, [76][77] that findings of statistical studies on the topic are based on problematic models and biased datasets,[78][79] and that constructivist approaches are largely ignored. False True False False True False False False True False True False False False False -0.8225 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +132 unstructured 20766 20921 wikiClimate_security [80] existing research also predominantly focuses on a few, well-known and already conflict-ridden regions, such as sub-saharan africa and the middle east. False False False False False False False False True False True False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +133 unstructured 20922 21144 wikiClimate_security this raises questions about sampling biases as well as implications for less-considered regions like latin america and the pacific, with topics such as peaceful adaptation and environmental peacebuilding also understudied. False False False False False False False False False False False False False False True 0.7783 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +134 unstructured 21145 21672 wikiClimate_security "[81] the ipcc's sixth assessment report concluded in 2022: ""climate hazards have affected armed conflict within countries (medium confidence), but the influence of climate is small compared to socio-economic, political, and cultural factors (high confidence). climate increases conflict risk by undermining food and water security, income and livelihoods, in situations where there are large populations, weather-sensitive economic activities, weak institutions and high levels of poverty and inequality (high confidence).""[82]" True True False True True False True False True True True True True False False -0.7935 False False False True True True False True False False True False True False False False False False False True False True False False False False False False False False +135 unstructured 21674 21785 wikiClimate_security many politicians, decision makers, and journalists have drawn a connection between climate change and conflict. False True False False True False False False True False True False False False False -0.3182 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +136 unstructured 21786 22105 wikiClimate_security already in a 2007 study on the topic, the german advisory council on global change identified four pathways potentially connecting climate change to conflict: degradation of freshwater resources, food insecurity, an increasing frequency and intensity of natural disasters, and increasing or changing migration patterns. True True False True True False False False True True True False True False False -0.8625 False False False True True False False True False False True False True False False False False False False True False True False False False False False False False False +137 unstructured 22106 22594 wikiClimate_security "a more recent 2021 report from the us office of the director of national intelligence predicts intensifying physical effects of climate change ""will exacerbate geopolitical flashpoints, particularly after 2030, and key countries and regions will face increasing risks of instability and need for humanitarian assistance.""[83] the united nations security council has discussed the links between climate change and security various times, even though the position of its member states vary." False True False True True False False False False False False False True False False 0.8225 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +138 unstructured 22595 22715 wikiClimate_security [84] other key decision makers in the usa, the european union, and nato are also concerned about climate conflict risks. False True False False True False False False True False True False False False False -0.5267 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +139 unstructured 22716 22803 wikiClimate_security [85] adaptation setup of a solar shade canopy for humanitarian aid and disaster relief. False False False False False False False False False False False False False False True -0.25 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +140 unstructured 22804 22890 wikiClimate_security the solar shade has the potential to provide enough energy for continuous 24-hour use. False False False False False False False False False True False False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +141 unstructured 22891 22950 wikiClimate_security energy see also: energy usage of the united states military False False False False False False False False False True False False False False False 0.7184 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +142 unstructured 22952 23114 wikiClimate_security at least since 2010, the u.s. military begun to push aggressively to develop, evaluate and deploy renewable energy to decrease its need to transport fossil fuels. False False False False False False False False False True False False False True False -0.0516 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +143 unstructured 23115 23325 wikiClimate_security [86] based on the 2015 annual report from nato, the alliance plans investments in renewables and energy efficiency to reduce risks to soldiers, and cites the impacts from climate change on security as a reason. False True False False True False True False False True False False False True False 0.5994 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +144 unstructured 23326 23371 wikiClimate_security [87] military operations and climate security False True False False True False True False False False False False False False False 0.34 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +145 unstructured 23373 23538 wikiClimate_security the main concern for military strategists - and central to many military and national security strategies - is the operability of armed forces during climate change. False True False False True False True False False False False False False False False 0.34 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +146 unstructured 23539 23700 wikiClimate_security there are concerns over the impact of climate change on infrastructure, such as military bases, as well as on the capacity to fight, for example in extreme heat. True True False False True False False False False False False False False False False -0.128 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +147 unstructured 23701 23887 wikiClimate_security a 2018 pentagon report revealed that half of 3,500 military sites were suffering the effects of six key categories of extreme weather events, such as storm surge, wildfires and droughts. True False False True True False False False False False False False False False False -0.4767 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +148 unstructured 23889 24115 wikiClimate_security there have consequently been efforts to 'green' the military and prepare it for a climate changed world, through the installation of solar panels at military bases, alternative fuels in shipping and renewable energy equipment. False True False False True False False False False True False False False False False 0.2732 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +149 unstructured 24122 24230 wikiClimate_security there are also concerns about the reliance on fossil fuels, which can pose vulnerabilities for armed forces. False False False False False False True False False False False False False False False -0.1531 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +150 unstructured 24231 24297 wikiClimate_security the pentagon alone is the world's largest consumer of fossil fuel. False False False False False False False False False False False False False False False -0.25 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +151 unstructured 24298 24432 wikiClimate_security [89] for example, during the us invasion of iraq, one in nearly 40 fuel convoys in iraq in 2007 resulted in a death or serious injury. False False False False False False False False False False False False False False False -0.7906 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +152 unstructured 24433 24587 wikiClimate_security [90] at least since 2010, the u.s. military begun to push to develop, evaluate and deploy renewable energy to decrease its need to transport fossil fuels. False False False False False False False False False True False False False True False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +153 unstructured 24588 24742 wikiClimate_security the nato's 2021 climate change and security action plan proposes strategies to protect its assets along with a promise of ghg emissions reduction by 2050. False True False False True False False False False False False False False False False 0.7906 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +154 unstructured 24749 24883 wikiClimate_security however, because military emissions reporting is only voluntary,[92] there is a lack of transparent data on militaries' ghg emissions. False True False False False False False False False False False False False False False -0.3182 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +155 unstructured 24884 25053 wikiClimate_security a 2019 study by brown university estimated 1.2 billion metric tons of ghg have been consumed by the u.s. military alone since the beginning of the war on terror in 2001. False True False False False False True False False False False False False False False -0.8519 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +156 unstructured 25054 25265 wikiClimate_security [93] additionally, scientists for global responsibility have calculated uk military emissions to be 11 million tonnes, and eu emissions to be 24.8 million tonnes with france contributing to a third of the total. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +157 unstructured 25272 25347 wikiClimate_security the military's sustainability plans have been criticized as 'greenwashing.' False False False False False False False False False False False False False False False -0.4404 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +158 unstructured 25348 25427 wikiClimate_security additionally, militarism and war have caused devastating environmental damages. False False False False False False True False False False False False False False False -0.9022 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +159 unstructured 25428 25606 wikiClimate_security [95] the chemical contamination left in afghanistan and the nuclear contamination in the marshalls islands are some examples of american imperialism and its environmental legacy. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +160 unstructured 25607 25642 wikiClimate_security [96][97] climate security practices False True False False True False False False False False False False False False False 0.34 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +161 unstructured 25644 25871 wikiClimate_security due to the growing importance of climate security on the agendas of many governments, international organizations, and other bodies some now run programs which are designed to mitigate the effects of climate change on conflict. False True False False True False False False True False True False True False False 0.5106 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +162 unstructured 25872 26245 wikiClimate_security "these practices are known as climate security practices [98] which are defined by von lossow et al. as ""tangible actions implemented by a (local or central) government, organization, community, private actor or individual to help prevent, reduce, mitigate or adapt (to) security risks and threats related to impacts of climate change and related environmental degradation""." False True False False True False False False True False False False False True False -0.1779 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +163 unstructured 26246 26369 wikiClimate_security [98] the planetary security initiative at the clingendael institute maintain an updated list of climate security practices. False True False False True False False False False False False False False False False 0.5859 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +164 unstructured 26376 26501 wikiClimate_security these practices stem from a variety of actors with different motivations in the sphere of development, diplomacy and defence. False False False False False False False False False False False False False False False 0.1027 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +165 unstructured 26502 26585 wikiClimate_security an example is the arms to farms project in kauswagan municipality, the philippines. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +166 unstructured 26586 26773 wikiClimate_security [100] an insurgency in the area was aggravated by food insecurity because irregular rainfall that caused poor harvests led to an uptick in insurgent recruitment sparking further violence. True False False False True False True False False False True False False False False -0.9169 False False False True True False False False False False True True False False False False False False False True True False False False False False False False False False +167 unstructured 26774 26994 wikiClimate_security [101] the project successfully integrated former insurgents into the community by training them in agricultural methods and fostering trust between communities, increasing food security, peace and human security overall. False False False True False False True False False False False False True False False 0.93 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +168 unstructured 26995 27176 wikiClimate_security another example is a division of the un peacekeeping mission in mali (minusma) that seeks to solve community conflicts, which can stem from climate change caused resource shortages. False True False False True False False False True True True False False False False 0.1531 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +169 unstructured 27177 27371 wikiClimate_security one project in kidal built a new and more effective water pump in order to solve the issue of conflict between different stakeholders in the area over water which risked a violent confrontation. False False False False False False False False True True True False False False False -0.6378 False False False True True False False True False False True False False False False False False False False False False False False False False False False False False False +170 unstructured 27379 27840 wikiClimate_security a growing number of non-military and civil society organizations are advocating for a national security approach including brookings institution and the council on foreign relations (us), the international institute for strategic studies and chatham house (uk), stockholm international peace research institute, clingendael (netherlands), french institute for international and strategic affairs, adelphi (germany) and the australian strategic policy institute. False False False False False False False False False False False False True False False 0.7845 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +171 unstructured 27841 28025 wikiClimate_security [88] environmental groups have also embraced a national security approach such as the world wildlife fund, the environmental defense fund and nature conservancy (us) and e3g in europe. False False False False False False False True False False False False False False False 0.4404 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +172 unstructured 28026 28158 wikiClimate_security the grassroots group extinction rebellion netherlands even invited a dutch military general to contribute to their 'rebel' handbook. False False False False False False False False False False False False False False False -0.2732 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +173 unstructured 28159 28359 wikiClimate_security even though these groups are often more concerned with human security they seek to involve the military as allies, driven by the belief that it can help achieve broader political and economic support. False False False False False False False False False False False False False False False 0.7783 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +174 unstructured 28366 28570 wikiClimate_security the field of climate security practices is still young and even though the issue is growing in importance, some actors are still reluctant to get involved due to the uncertainty inherent in the new field. False True False False True False False False False False False False True False False 0.296 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +175 unstructured 28571 28838 wikiClimate_security "[98] because climatic change will only increase in the near future von lossow et al. conclude that expanding the number of climate security practices in vulnerable areas of the world has ""huge potential to catalyse more sustainable and long-term peace and stability""." False True False True True False False False False False False False True False False 0.836 True False True False False True False False False False True False True False False False False False False True False True False False False False False False False False +176 unstructured 28839 28864 wikiClimate_security [98] political approaches False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +177 unstructured 28866 29005 wikiClimate_security the transnational character of climate-related security risks often goes beyond the capacity of national governments to respond adequately. False True False False True False False False False False False False False False False 0.0772 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +178 unstructured 29006 29144 wikiClimate_security [103] many parts of governments or state leaders acknowledge climate change as an issue for human security, national or regional security. False True False False True False False False False False False False False False False 0.5859 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +179 unstructured 29146 29404 wikiClimate_security despite ongoing concerns about the securitization of climate change, it has had little effect on the policies and activities of national governments, which have tended to take 'business as usual' approaches to managing and containing international migration. False True False False True False False False False False False False False False False 0.0516 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +180 unstructured 29405 29415 wikiClimate_security [104] nato False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +181 unstructured 29417 29532 wikiClimate_security "nato stated in 2015 that climate change is a significant security threat and that ""its bite is already being felt""." False True False False True False False False True False False False False False False -0.0516 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +182 unstructured 29533 29632 wikiClimate_security [105] in 2021, nato agreed a climate change and security action plan that committed the alliance to False True False False True False False False False False False False False False False 0.6808 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +183 unstructured 29633 29762 wikiClimate_security 1) analyze the impact of climate change on nato's strategic environment and nato's assets, installations, missions and operations False True False False True False False False False False False False False False False 0.1779 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +184 unstructured 29763 29874 wikiClimate_security 2) incorporate climate change considerations into its work3) contribute to the mitigation of climate change and False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +185 unstructured 29875 30015 wikiClimate_security 4) exchange with partner countries, as well as with international and regional organizations that are active on climate change and security. False True False False True False False False False False False False False False False 0.7351 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +186 unstructured 30023 30166 wikiClimate_security it is important to note that the deployment of security forces can sometimes lead to insecurity, rather than security, for certain populations. False False False False False False False False False False False False False False False 0.5994 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +187 unstructured 30167 30517 wikiClimate_security for example the 20-year us-led and nato-supported military invasion and occupation of afghanistan was launched in order to obtain security from terrorism and fight the war on terror , but it ended up fueling more war, conflict, and the return to power of the taliban as a result of the withdrawal of united states troops from afghanistan (2020-2021). False False False False False False True False True False True False False False False -0.9119 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +188 unstructured 30518 30532 wikiClimate_security united nations False False False False False False False False False False False False False False False 0.4215 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +189 unstructured 30534 30938 wikiClimate_security although climate change is first and foremost dealt within the united nations framework convention on climate change (unfccc) and now also under the paris agreement, the security implications of climate change do not have an institutional home within the united nations system, and hence remain largely unaddressed, in spite of the urgency of the threat it poses to peace and security in several regions. False True False False True False False False True False False False False False False 0.8519 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +190 unstructured 30939 31083 wikiClimate_security [107] the un, through its cop - the conference of the parties - is the supreme body to negotiate climate frameworks under the unfccc convention. False True False False True False False False False False False False False False False 0.743 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +191 unstructured 31084 31363 wikiClimate_security it consists of the representatives of the parties to the convention and holds its sessions every year, and takes decisions which are necessary to ensure the effective implementation of the provisions of the convention and regularly reviews the implementation of these provisions. False False False False False False False False False False False False False False False 0.8126 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +192 unstructured 31364 31464 wikiClimate_security "preventing ""dangerous"" human interference with the climate system is the ultimate aim of the unfccc." False True False False True False False False False False False False False False False -0.4939 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +193 unstructured 31465 31560 wikiClimate_security "[108] the unfccc is a ""rio convention"", one of three adopted at the ""rio earth summit"" in 1992." False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +194 unstructured 31561 31609 wikiClimate_security the unfccc entered into force on march 21, 1994. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +195 unstructured 31610 31650 wikiClimate_security today, it has near-universal membership. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +196 unstructured 31651 31765 wikiClimate_security [109] the cop has discussed climate security during panels, workshops as session, but not as a programmatic track. False True False False True False False False False False False False False False False 0.1779 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +197 unstructured 31766 31941 wikiClimate_security the greater focus on this topic by the un has led to the launch in october 2018 of the inter-agency dppa-undp-un environment cooperation called the climate security mechanism. False True False False True False False False False False False False False False False 0.5994 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +198 unstructured 31942 31979 wikiClimate_security [110] united nations security council False False False False False False False False False False False False False False False 0.6369 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +199 unstructured 31981 32177 wikiClimate_security the un security council first debated climate security and energy in 2007 and in 2011 issued a presidential statement expressing concern at the possible adverse security effects of climate change. False True False False True False False False False True False False False False False 0.7003 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +200 unstructured 32178 32273 wikiClimate_security there has been a series of informal arria-formula meetings on issues related to climate change. False True False False True False False False False False False False False False False 0.0 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +201 unstructured 32274 32377 wikiClimate_security in july 2018, sweden initiated a debate on climate and security in the united nations security council. False True False False True False False False False False False False False False False 0.765 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +202 unstructured 32378 32468 wikiClimate_security in 2021 the un security council convened for a high-level open debate on climate security. False True False False True False False False False False False False False False False 0.5859 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +203 unstructured 32469 32685 wikiClimate_security climate change grew beyond its categorization as a hypothetical, existential risk and became an operational concern of relevance to other peace and security practitioners beyond the diplomats in the security council. False True False False True False False False False False False False False False False 0.7351 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +204 unstructured 32687 32827 wikiClimate_security however, some countries, especially low and middle income countries (lmics), do not think climate change should be seen as a security issue. False True False False True False False False False False False False False False False 0.0018 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +205 unstructured 32828 32943 wikiClimate_security when the topic of climate and security first emerged in the un, lmics opposed the securitization of climate change. False True False False True False False False False False False False False False False 0.3818 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +206 unstructured 32944 33557 wikiClimate_security [111] in 2006 the group of 77 (g77) argued that 'the united nations framework convention on climate change (unfccc) is the primary international, intergovernmental forum for negotiating the global response to climate change' and that the richest countries should not only address the 'consequences [of climate change] but mainly the roots of the problem.'[112] the g77 also stated that it is inappropriate to consider the issue of energy in the unsc, 'reaffirming the key role of energy in achieving the goals of sustainable development, poverty eradication and achieving the mdgs [millennium development goals]'. False True False False True False False False False True False False False False False 0.296 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +207 unstructured 33558 33776 wikiClimate_security [113] in 2013, the g77 and china argued that the unsc was 'not the appropriate forum for this discussion' and that such issues should be assigned to the economic and social council (ecosoc) and the un general assembly. False False False False False False False False False False False False False False False -0.3612 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +208 unstructured 33777 33834 wikiClimate_security [114] the g77 has not issued public positions since then. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +209 unstructured 33835 33872 wikiClimate_security country and continent examples africa False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +210 unstructured 33874 33977 wikiClimate_security climate change has had devastating effects on the african continent, affecting the poorest communities. False True False False True False False False False False False False False False False -0.8316 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +211 unstructured 33978 34113 wikiClimate_security it has escalated food insecurity, led to the displacement of populations and exerted extreme pressure on the available water resources. False False False False False False False False False True False False False False False -0.6478 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +212 unstructured 34114 34340 wikiClimate_security [115] africa's exposure to climate change is high due to the legacy of colonialism, inequitable global trade arrangements, its low adaptive capacity and limited government capabilities, making it the most vulnerable continent. False True False False True False False False False False False False False False True -0.6361 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +213 unstructured 34341 34519 wikiClimate_security [116] a 2007 report by the un secretary-general, ban ki-moon points out that, climate change and environmental degradation were partly responsible for the darfur, sudan conflict. False True False False True False False False True False True False False False False -0.807 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +214 unstructured 34520 34667 wikiClimate_security [117] between 1967 and 2007, the total rainfall in the area had reduced by 30 percent and the expansion of the sahara was beyond a mile every year. True False False False True False False False False False False False False True False 0.0 False True True False True False False False False False True True False False False False False False False True True False False False False False False False False False +215 unstructured 34668 34830 wikiClimate_security the ensuing friction between farmers and pastoralists over the reducing grazing land and the few water sources available was at the heart of the darfur civil war. False False False False False False True False False True False False False False False 0.0772 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +216 unstructured 34831 35102 wikiClimate_security [118] more recently us and european security analysts refer to the sahel and the lake chad basin as a 'hotspot' because of its severe climate-related vulnerabilities, communal violence, jihadist insurgencies, political instability, and internal and regional displacement. False True False False True False True False False False True False False False False -0.7096 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +217 unstructured 35103 35238 wikiClimate_security climate change has caused rainfall variations and desertification threatening the well-being of people whose lives depend on lake chad. True True False False True False False False True False False False False False False -0.5267 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +218 unstructured 35239 35352 wikiClimate_security [119] reports suggest that lake chad is shrinking at a fast speed, which is creating sharp competition for water. False False False False False False False False True True True False False False False 0.296 True False True False False False False True False False True False False False False False False False False False False False False False False False False False False False +219 unstructured 35353 35459 wikiClimate_security [120][121] in 2017, the unsc adopted resolution 2349 connecting conflict and water scarcity in the region. False False False False False False False False True True True False False False False -0.3182 False False False True True False False True False False True False False False False False False False False False False False False False False False False False False False +220 unstructured 35460 35711 wikiClimate_security in 2020, niger co-organized a un security council meeting on climate security, following a 2018 unsc statement on the lake chad basin which identified climate change and the shrinking of lake chad as one of the root causes of the boko haram uprisings. False True False False True False False False False False False False False False False 0.5859 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +221 unstructured 35713 36033 wikiClimate_security however scholars disagree on whether the desertification of lake chad has indeed led to conflicts, because there are other factors such as pre-existing socioeconomic and political conditions, the influx of arms into the region, unfair terms of trade, religious issues, and the marginalization of pastoralist communities. False True False False False False False False True False True False False False False -0.8074 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +222 unstructured 36034 36043 wikiClimate_security australia False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +223 unstructured 36045 36378 wikiClimate_security "a 2018 published report by the australian senate noted how ""climate change as a current and existential national security risk. defined as one that threatens the premature extinction of earth-originating intelligent life or the permanent and drastic destruction of its potential for desirable future development.""[122] european union" False True False False True False False False True False False False False False False -0.1779 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +224 unstructured 36380 36796 wikiClimate_security "the european council's conclusions on climate diplomacy state that ""climate change is a decisive global challenge which, if not urgently managed, will put at risk . peace, stability and security.""[123] the intelligence on european pensions and institutional investment think-tank published a 2018 report with the key point, ""climate change is an existential risk whose elimination must become a corporate objective""." False True False False True False False False False False False False False False False 0.6808 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +225 unstructured 36797 36947 wikiClimate_security "[124] in june 2018 european external action service (eeas) high-level event hosted an event themed ""climate, peace and security: the time for action""." False True False False True False False False False False False False False False False 0.7096 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +226 unstructured 36948 37084 wikiClimate_security [125] the eu's comprehensive approach to security would suggest that the eu is well placed to respond to climate-related security risks. False True False False True False False False False False False False False False False 0.7003 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +227 unstructured 37085 37197 wikiClimate_security however, recent scientific research shows that the european union has not yet developed a fully coherent policy. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False True False False False False False False False False False False False False False False False False False False False +228 unstructured 37198 37218 wikiClimate_security [126] united kingdom False False False False False False False False False False False False False False False 0.4215 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +229 unstructured 37220 37471 wikiClimate_security the uk's former foreign and commonwealth office (fco) (now the foreign, commonwealth and development office, fcdo) was the first to push the un to hold a meeting in the un security council on climate change and urged the unfccc to take action in 2007. False True False False True False False False False False False False False False False 0.34 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +230 unstructured 37472 37629 wikiClimate_security in 2011, the then uk department for international development (dfid) committed to spending 30% of its aid in fragile and conflict-affected states by 2014-15. False False False False False False False False True False True False False False False 0.2732 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +231 unstructured 37630 37757 wikiClimate_security between 2013 and 2015, the fco had discussions on the climate, security and conflict in 'fragile' states at g8 and g7 meetings. False True False False True False False False True False True False False False False 0.0258 False True True False True False False False False False True False True False False False False False False True False True False False False False False False False False +232 unstructured 37758 37861 wikiClimate_security "in 2014, david cameron noted that ""climate change is one of the most serious threats facing our world""." False True False False True False False False True False False False False False False -0.5678 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +233 unstructured 37862 37933 wikiClimate_security [127] a 2018 article in uk's the independent also argued that the u.s.' False False False False False False False False False False False False False False False -0.3612 False False False True True False False False False False True False False False False False False False False False False False False False False False False False False False +234 unstructured 37934 38044 wikiClimate_security "trump administration is ""putting british national security at risk"", according to over 100 climate scientists." False True False False True False False False False False False False False False False 0.0772 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +235 unstructured 38045 38499 wikiClimate_security [128] in the same year, the former uk special representative for climate change, rear admiral neil morisetti, claimed that 'climate change will require more deployment of the british military in conflict prevention, conflict resolution or responding to increased humanitarian requirements due to extreme weather impacts.'[129] the uk-established international climate fund (icf) identifies climate change as a 'threat multiplier' within 'fragile' states. True True False True True False False False True False True False True False False -0.2263 False False False True True True False False False False True False True False False False False False False True False True False False False False False False False False +236 unstructured 38500 38697 wikiClimate_security in 2015 a report by the joint committee on the national security strategy (nss) confirmed that climate change presents a risk to national security and that spending on security would need to adapt. False True False False True False False False False False False False False False False 0.6249 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +237 unstructured 38699 38861 wikiClimate_security on september 23, 2021, lord ahmad of wimbledon, uk minister for the united nations stated that climate change threatened the safety of the country and all people. False True False False True False False False True False False False False False False 0.3818 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +238 unstructured 38862 39006 wikiClimate_security recently the united kingdom hosted the 26th un climate change conference of the parties (cop26) in glasgow on october 31 - 12 november 12, 2021. False True False False True False False False False False False False False False False 0.6705 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +239 unstructured 39007 39377 wikiClimate_security [130] the uk's 'integrated review of defence, security, development, and foreign policy', published in march 2021, argues that dealing with climate change and biodiversity loss was its 'number one international priority' and identified african countries as vulnerable countries to climate change, which can amplify insecurity, migration and instability on the continent. False True False False True False False False False False False False False False False -0.6705 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +240 unstructured 39378 39397 wikiClimate_security [131] united states False False False False False False False False False False False False False False False 0.4215 True False True False False False False False False False True False False False False False False False False False False False False False False False False False False False +241 unstructured 39399 39673 wikiClimate_security in the united states, analysis of climate security and the development of policy ideas for addressing it has been led by the center for climate and security, founded by francesco femia and caitlin werrell in 2011, which is now an institute of the council on strategic risks. False True False False True False False False False False False False False False False 0.6705 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +242 unstructured 39675 39792 wikiClimate_security "us intelligence analysts have expressed concern about the ""serious security risks"" of climate change since the 1980s." False True False False True False False False False False False False False False False 0.4767 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +243 unstructured 39793 40298 wikiClimate_security "[132] in 2007, the council on foreign relations released a report titled, climate change and national security: an agenda for action, stating that ""climate change presents a serious threat to the security and prosperity of the united states and other countries.""[133] a 2012 report published by the joint global change research institute indicated that second and third order impacts of climate change, such as migration and state stability, are of concern for the us defense and intelligence communities." False True False False True False False False True False False False False False False 0.7579 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +244 unstructured 40299 40537 wikiClimate_security [134] a 2015 report published by the white house found that climate change puts coastal areas at risk, that a changing arctic poses risks to other parts of the country, risk for infrastructure, and increases demands on military resources. False True False True True False False False False True False False True False False -0.6486 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +245 unstructured 40538 40903 wikiClimate_security "[135] in 2016, director of national intelligence james clapper noted: ""unpredictable instability has become the 'new normal,' and this trend will continue for the foreseeable future. extreme weather, climate change, environmental degradation, rising demand for food and water, poor policy decisions and inadequate infrastructure will magnify this instability.""[136]" True True False False True False False False False True False False False False False -0.765 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +246 unstructured 40905 40987 wikiClimate_security a 2015 pentagon report pointed out how climate denial threatens national security. False True False False True False False False True False False False False False False -0.0516 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +247 unstructured 40988 41087 wikiClimate_security [137] in 2017, the trump administration removed climate change from its national security strategy. False True False False True False False False False False False False False False False 0.34 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +248 unstructured 41088 41210 wikiClimate_security [138] but in january 2019 the pentagon released a report stating that climate change is a national security threat to usa. False True False False True False False False True False False False False False False -0.3612 False False False True True False False False False False True False True False False False False False False True False True False False False False False False False False +249 unstructured 41211 41536 wikiClimate_security [139][140][141] in june 2019, in the course of house select committee on intelligence hearings on the national security implications of climate change, the white house blocked the submission of a statement by the state department's bureau of intelligence and research office, and the analyst who wrote the statement resigned. False True False False True False False False False False False False False False False 0.6705 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +250 unstructured 41537 41632 wikiClimate_security [142][143] the idea of creating a presidential committee on climate security has been proposed. False True False False True False False False False False False False False False False 0.5574 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +251 unstructured 41633 41790 wikiClimate_security [144][132] as part of the united states national defense authorization act the u.s. congress asked the department of defense for a report on climate matters. False True False False True False False False False False False False False False False 0.5994 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +252 unstructured 41791 42119 wikiClimate_security "the report was published in 2019, and notes, ""the effects of a changing climate are a national security issue with potential impacts to department of defense (dod or the department) missions, operational plans, and installations.""[145] in 2021, united states president biden declared climate change a national security priority." False True False False True False False False False False False False False False False 0.7964 True False True False False False False False False False True False True False False False False False False True False True False False False False False False False False +1 unstructured 0 23 wikiHuman-wildlife_conflict human-wildlife conflict False False False False False True False True True False True False False False False -0.3182 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +2 unstructured 25 37 wikiHuman-wildlife_conflict article talk False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +3 unstructured 39 61 wikiHuman-wildlife_conflict read edit view history False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +4 unstructured 63 68 wikiHuman-wildlife_conflict tools False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +5 unstructured 70 223 wikiHuman-wildlife_conflict from wikipedia, the free encyclopedia grand canyon national park hosts millions of visitors every year and is home to a population of rocky mountain elk. False False False False False False False False False False False False False False False 0.743 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +6 unstructured 224 293 wikiHuman-wildlife_conflict interactions between humans and the elk sometimes result in injuries. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +7 unstructured 299 526 wikiHuman-wildlife_conflict human-wildlife conflict (hwc) refers to the negative interactions between humans and wild animals, with undesirable consequences both for people and their resources on the one hand, and wildlife and their habitats on the other. False False False False False True False True True True True False False False False -0.6908 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +8 unstructured 527 687 wikiHuman-wildlife_conflict hwc, caused by competition for natural resources between human and wildlife, influences human food security and the well-being of both humans and other animals. False False False False False True False True True True True False False False False 0.5994 True False True False False False False True False False False False False False False False True False False False False False False False False False False True False False +9 unstructured 688 848 wikiHuman-wildlife_conflict in many regions, the number of these conflicts has increased in recent[when?] decades as a result of human population growth and the transformation of land use. False False False True False False False False True True True False True False False 0.34 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +10 unstructured 850 976 wikiHuman-wildlife_conflict hwc is a serious global threat to sustainable development, food security and conservation in urban and rural landscapes alike. False False False False False True False True True False False False False False False -0.3182 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +11 unstructured 977 1269 wikiHuman-wildlife_conflict in general, the consequences of hwc include: crop destruction, reduced agricultural productivity, competition for grazing lands and water supply, livestock predation, injury and death to human, damage to infrastructure, and increased risk of disease transmission among wildlife and livestock. False False False True False True False True True True True False True True False -0.9274 False False False True True False False True False False False False False False False False True False False False False False False False False False False True False False +12 unstructured 1275 1421 wikiHuman-wildlife_conflict as of 2020, conflict mitigation strategies utilized lethal control, translocation, population size regulation and endangered species preservation. False True False False False False False True True False True False False False False -0.3182 False False False True True False False False False False False False True False False False False False False True False True False False False False False False False False +13 unstructured 1422 1509 wikiHuman-wildlife_conflict recent management now uses an interdisciplinary set of approaches to solving conflicts. False False False False False False False False True False True False False False False -0.0516 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +14 unstructured 1510 1610 wikiHuman-wildlife_conflict these include applying scientific research, sociological studies and the arts to reducing conflicts. False False False False False False False False True False True False False False False -0.3818 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +15 unstructured 1611 1798 wikiHuman-wildlife_conflict as human-wildlife conflict inflicts direct and indirect consequences on people and animals, its mitigation is an important priority for the management of biodiversity and protected areas. False False False False False True False True True False True False False False False 0.34 True False True False False False True False False False False False False False False False False False False False False False False False False False False False False False +16 unstructured 1799 1994 wikiHuman-wildlife_conflict resolving human-wildlife conflicts and fostering coexistence requires well-informed, holistic and collaborative processes that take into account underlying social, cultural and economic contexts. False False False False False True False True True False True False False False False 0.0 False True True False True False True False False False False False False False False False False False False False False False False False False False False False False False +17 unstructured 2000 2323 wikiHuman-wildlife_conflict in 2023, the iucn ssc human-wildlife conflict & coexistence specialist group published the iucn ssc guidelines on human-wildlife conflict and coexistence[4] that aim to provide foundations and principles for good practice, with clear, practical guidance on how best to tackle conflicts and enable coexistence with wildlife. False False False False False True False True True False True False False False False 0.5423 True False True False False False True False False False False False False False False False False False False False False False False False False False False False False False +18 unstructured 2325 2508 wikiHuman-wildlife_conflict as of 2013, many countries have started to explicitly include human-wildlife conflict in national policies and strategies for wildlife management, development and poverty alleviation. False False False False False True False True True False True False False False False -0.6808 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +19 unstructured 2509 2631 wikiHuman-wildlife_conflict at the national level, collaboration between forestry, wildlife, agriculture, livestock and other relevant sectors is key. False False False False False False False True False True False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +20 unstructured 2632 2643 wikiHuman-wildlife_conflict [5] meaning False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +21 unstructured 2645 2927 wikiHuman-wildlife_conflict "human-wildlife conflict has been defined by the world wide fund for nature (wwf) in 2004 as ""any interaction between humans and wildlife that results in negative impacts of human social, economic or cultural life, on the conservation of wildlife populations, or on the environment""." False False False False False True False True True False True False False False False -0.7184 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +22 unstructured 2928 3314 wikiHuman-wildlife_conflict "[6] the creating co-existence workshop at the 5th annual world parks congress (8-17 september 2003, montreal) defined human-wildlife conflict in the context of human goals and animal needs as follows: ""human-wildlife conflict occurs when the needs and behavior of wildlife impact negatively on the goals of humans or when the goals of humans negatively impact the needs of wildlife.""[7]" False False False False False True False True True False True False False False False -0.34 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +23 unstructured 3316 3592 wikiHuman-wildlife_conflict a 2007 review by the united states geological survey defined human-wildlife conflict in two contexts; firstly, actions by wildlife conflict with human goals i.e. life, livelihood and life-style, and secondly, human activities that threaten the safety and survival of wildlife. False False False False False True False True True False True False False False False 0.0516 True False True False False False True False False False False False False False False False False False False False False False False False False False False False False False +24 unstructured 3593 3676 wikiHuman-wildlife_conflict however, in both cases outcomes are decided by human responses to the interactions. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +25 unstructured 3682 4041 wikiHuman-wildlife_conflict "the government of yukon defined human-wildlife conflict simply, but through the lens of damage to property, i.e. ""any interaction between wildlife and humans which causes harm, whether it's to the human, the wild animal, or property.""[9] here, property includes buildings, equipment and camps, livestock and pets, but does not include crops, fields or fences." False False False False False True False True True False True False False False False -0.8934 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +26 unstructured 4043 4385 wikiHuman-wildlife_conflict "in 2020, the iucn ssc human-wildlife conflict task force described human-wildlife conflict as ""struggles that emerge when the presence or behaviour of wildlife poses actual or perceived, direct and recurring threat to human interests or needs, leading to disagreements between groups of people and negative impacts on people and/or wildlife""." False False False False False True False True True False True False False False False -0.9325 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +27 unstructured 4386 4398 wikiHuman-wildlife_conflict [10] history False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +28 unstructured 4400 4491 wikiHuman-wildlife_conflict human-wildlife interactions have occurred throughout man's prehistory and recorded history. False False False False False True False True False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +29 unstructured 4492 4685 wikiHuman-wildlife_conflict an early form of human-wildlife conflict is the depredation of the ancestors of prehistoric man by a number of predators of the miocene such as saber-toothed cats, leopards, and spotted hyenas. False False False False False True False True True False True False False False False -0.25 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +30 unstructured 4692 4991 wikiHuman-wildlife_conflict fossil remains of early hominids show evidence of depredation; the taung child, the fossilized skull of a young australopithecus africanus, is thought to have been killed by an eagle from the distinct marks on its skull and the fossil having been found among egg shells and remains of small animals. False False False False False False False True False False False False False False False -0.6705 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +31 unstructured 4998 5262 wikiHuman-wildlife_conflict a plio-pleistocene horned crocodile, crocodylus anthropophagus, whose fossil remains have been recorded from olduvai gorge, was the largest predator encountered by prehistoric man, as indicated by hominid specimens preserving crocodile bite marks from these sites. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +32 unstructured 5269 5368 wikiHuman-wildlife_conflict another 12,000 year old example is the buffalo jump cliff sites found in the western united states. False False False False False False False False False False False False False False False 0.4215 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +33 unstructured 5369 5482 wikiHuman-wildlife_conflict these sites occurred as a result of humans exploiting an animal's herding behavior and predator-flight instincts. False False False False False False False True False False False False False False False -0.4404 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +34 unstructured 5482 5499 wikiHuman-wildlife_conflict [citation needed] False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +35 unstructured 5501 5559 wikiHuman-wildlife_conflict the extinction of the passenger pigeon is another example. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +36 unstructured 5559 5576 wikiHuman-wildlife_conflict [citation needed] False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +37 unstructured 5578 5717 wikiHuman-wildlife_conflict in 2023 alone, over 1.8 million distinct human-wildlife conflicts occurred as animal involved auto accidents on roadways, seen as roadkill. False False False False False True False True True False True False False False False -0.7096 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +38 unstructured 5718 5796 wikiHuman-wildlife_conflict understanding bird strike frequency is important to aircraft safety engineers. False False False False False False False False False False False False False False False 0.4767 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +39 unstructured 5797 5960 wikiHuman-wildlife_conflict reducing the frequent animal collisions (strikes) from automobiles on roadways are shared concerns of biologists, civil engineers, and automobile safety designers. False False False False False False False True False False False False False False False 0.1531 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +40 unstructured 5962 6219 wikiHuman-wildlife_conflict as of 2020, with specific reference to forests, a high density of large ungulates such as deer, can cause severe damage to the vegetation and can threaten regeneration by trampling or browsing small trees, rubbing themselves on trees or stripping tree bark. False False False False False False False False True True False False False False False -0.8126 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +41 unstructured 6220 6341 wikiHuman-wildlife_conflict this behavior can have important economic implications and can lead to polarization between forest and wildlife managers. False False False False False False False True False True False False False False False 0.2023 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +42 unstructured 6342 6500 wikiHuman-wildlife_conflict [5] examples simultaneous use of water resources by humans and crocodiles sets up occasions for human-wildlife conflict asian elephant damage to houses africa False False False False False True False True True True True False False False False -0.6705 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +43 unstructured 6502 6654 wikiHuman-wildlife_conflict as a tropical continent with substantial anthropogenic development, africa is a hotspot for biodiversity[14] and therefore, for human-wildlife conflict. False False False False False True False True True False True False False False False -0.128 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +44 unstructured 6655 6786 wikiHuman-wildlife_conflict two of the primary examples of conflict in africa are human-predator (lions, leopards, cheetahs, etc.) and human-elephant conflict. False False False False False False False False True False True False False False False -0.5574 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +45 unstructured 6787 6902 wikiHuman-wildlife_conflict depredation of livestock by african predators is well documented in kenya,[15] namibia,[16] botswana,[17] and more. False False False False False False False False False False False False False False False 0.2732 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +46 unstructured 6903 7012 wikiHuman-wildlife_conflict african elephants frequently clash with humans, as their long-distance migrations often intersect with farms. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +47 unstructured 7013 7141 wikiHuman-wildlife_conflict the resulting damage to crops, infrastructure, and at times, people, can lead to the retaliatory killing of elephants by locals. False False False False False True False False False False False False False False False -0.8225 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +48 unstructured 7148 7257 wikiHuman-wildlife_conflict in 2017, more than 8 000 human-wildlife conflict incidents were reported in namibia alone (world bank, 2019). False False False False False True False True True False True False False False False -0.5106 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +49 unstructured 7258 7473 wikiHuman-wildlife_conflict hyenas killed more than 600 cattle in the zambezi region of namibia between 2011 and 2016 and there were more than 4 000 incidents of crop damage, mostly caused by elephants moving through the region (nacso, 2017a). False False False False False False False False False False False False False False False -0.8271 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +50 unstructured 7474 7482 wikiHuman-wildlife_conflict [5] asia False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +51 unstructured 7484 7636 wikiHuman-wildlife_conflict with a rapidly increasing human population and high biodiversity,[14] interactions between people and wild animals are becoming more and more prevalent. False False False True False False False True False False False False True False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +52 unstructured 7637 7767 wikiHuman-wildlife_conflict like human-predator in africa, encounters between tigers, people, and their livestock is a prominent issue on the asian continent. False False False False False False False False False False False False False False False 0.5859 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +53 unstructured 7768 7962 wikiHuman-wildlife_conflict attacks on humans and livestock have exacerbated major threats to tiger conservation such as mortality, removal of individuals from the wild, and negative perceptions of the animals from locals. False False False True False False False True True False False False False False False -0.8655 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +54 unstructured 7963 8122 wikiHuman-wildlife_conflict [19] even non-predator conflicts are common, with crop-raiding by elephants[20] and macaques[21] persisting in both rural and urban environments, respectively. False False False False False False False False True False True False False False False -0.0516 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +55 unstructured 8123 8301 wikiHuman-wildlife_conflict poor disposal of hotel waste in tourism-dominated towns have altered behaviours of carnivores such as sloth bears that usually avoid human habitation and human-generated garbage. False False False False False False False True False False False False False False False -0.7964 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +56 unstructured 8302 8482 wikiHuman-wildlife_conflict [22] for example, as a rersult of the human-elephant conflict in sri lanka each year as many as 80 people are killed by elephants and more than 230 elephants are killed by farmers. False False False False False False False False True False True False False False False -0.9062 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +57 unstructured 8483 8584 wikiHuman-wildlife_conflict the sri lankan elephant is listed as endangered, and only 2.500-4.000 individuals remain in the wild. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +58 unstructured 8585 8693 wikiHuman-wildlife_conflict [5] as of 2021, in india the conflict is exceedingly acute because of the country's wildlife protection act. False False False False False False False True True False True False False False False -0.3182 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +59 unstructured 8700 8870 wikiHuman-wildlife_conflict moreover, in asia, wildlife is considered sacred as a messenger of god, and in some cases, religious and political protections are implemented, which can cause conflicts. False False False False False False False True True False True False False False False -0.128 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +60 unstructured 8871 9039 wikiHuman-wildlife_conflict for example, in nara city, japan, the sacred japanese sika deer (cervus nippon), protected for over a millennium, has recently seen a population surge around nara park. False False False True False False False False False False False False False False False 0.4404 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +61 unstructured 9040 9192 wikiHuman-wildlife_conflict [24] genetic analysis reveals mixing between sacred deer from the sanctuary and common lineage deer, posing a risk to the sacred deer's unique genetics. False False False False False False False False False False False False False False False -0.2732 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +62 unstructured 9193 9406 wikiHuman-wildlife_conflict [25] this situation presents a complex challenge where excluding surrounding deer populations is necessary to maintain the genetic uniqueness of a sacred deer population that humans have protected for a long time. False False False False False False False False False False False False False False False 0.4939 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +63 unstructured 9407 9417 wikiHuman-wildlife_conflict antarctica False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +64 unstructured 9419 9611 wikiHuman-wildlife_conflict in antarctica the first known instance of death due to human-wildlife conflict occurred in 2003 when a leopard seal dragged a snorkelling british marine biologist underwater where she drowned. False False False False False True False True True False True False False False False -0.8834 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +65 unstructured 9612 9623 wikiHuman-wildlife_conflict [26] europe False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +66 unstructured 9625 9731 wikiHuman-wildlife_conflict human-wildlife conflict in europe includes interactions between people and both carnivores and herbivores. False False False False False True False True True False True False False False False -0.3182 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +67 unstructured 9732 9851 wikiHuman-wildlife_conflict a variety of non-predators such as deer, wild boar, rodents, and starlings have been shown to damage crops and forests. False False False False False False False False False True False False False False False -0.4939 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +68 unstructured 9852 10016 wikiHuman-wildlife_conflict [27] carnivores like raptors and bears create conflict with humans by eating both farmed and wild fish,[27] while others like lynxes and wolves prey upon livestock. False False False False False False False False True False True False False False False 0.5859 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +69 unstructured 10017 10221 wikiHuman-wildlife_conflict [28] even less apparent cases of human-wildlife conflict can cause substantial losses; 500,000 deer-vehicle collisions in europe (and 1-1.5 million in north america) led to 30,000 injuries and 200 deaths. False False False False False True False True True False True False False False False -0.6486 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +70 unstructured 10222 10240 wikiHuman-wildlife_conflict [29] north america False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +71 unstructured 10242 10311 wikiHuman-wildlife_conflict instances of human-wildlife conflict are widespread in north america. False False False False False True False True True False True False False False False -0.3182 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +72 unstructured 10312 10472 wikiHuman-wildlife_conflict in wisconsin, united states wolf depredation of livestock is a prominent issue that resulted in the injury or death of 377 domestic animals over a 24-year span. False False False False False False False True False False False False False False False -0.3818 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +73 unstructured 10473 10598 wikiHuman-wildlife_conflict [30] similar incidents were reported in the greater yellowstone ecosystem, with reports of wolves killing pets and livestock. False False False False False False False True False True False False False False False -0.4404 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +74 unstructured 10599 10817 wikiHuman-wildlife_conflict [31] expanding urban centers have created increasing human-wildlife conflicts, with interactions between human and coyotes and mountain lions documented in cities in colorado and california, respectively, among others. False False False True False True False True True False True False True False False 0.2023 True False True False False False True False False False False False False False False False False False False False False False False False False False False False False False +75 unstructured 10818 11034 wikiHuman-wildlife_conflict [32][33] big cats are a similar source of conflict in central mexico, where reports of livestock depredation are widespread,[34] while interactions between humans and coyotes were observed in canadian cities as well. False False False False False True False False True False True False False False False -0.0516 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +76 unstructured 11035 11111 wikiHuman-wildlife_conflict [35] diagram of human wildlife conflict in expanding american cities oceania False False False False False True False True True False True False False False False -0.3182 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +77 unstructured 11113 11314 wikiHuman-wildlife_conflict on k'gari-fraser island in australia, attacks by wild dingoes on humans (including the well-publicized death of a child) created a human-wildlife crisis that required scientific intervention to manage. False False False False False True False True False False False False False False False -0.872 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +78 unstructured 11315 11504 wikiHuman-wildlife_conflict [36] in new zealand, distrust and dislike of introducing predatory birds (such as the new zealand falcon) to vineyard landscapes led to tensions between people and the surrounding wildlife. False False False False False False False True False False False False False False False -0.7964 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +79 unstructured 11505 11666 wikiHuman-wildlife_conflict [37][38] in extreme cases large birds have been reported to attack people who approach their nests, with human-magpie conflict in australia a well-known example. False False False False False False False False True False True False False False False -0.6597 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +80 unstructured 11667 11810 wikiHuman-wildlife_conflict [39] even conflict in urban environments has been documented, with development increasing the frequency of human-possum interactions in sydney. False False False True False False False False True False True False True False False -0.3182 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +81 unstructured 11811 11829 wikiHuman-wildlife_conflict [40] south america False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +82 unstructured 11831 11965 wikiHuman-wildlife_conflict as with most continents, the depredation of livestock by wild animals is a primary source of human-wildlife conflict in south america. False False False False False True False True True False True False False False False -0.3182 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +83 unstructured 11966 12141 wikiHuman-wildlife_conflict the killings of guanacos by predators in patagonia, chile - which possess both economic and cultural value in the region - have created tensions between ranchers and wildlife. False False False False False False False True False False False False False False False -0.5859 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +84 unstructured 12142 12300 wikiHuman-wildlife_conflict [41] south america's only species of bear, the andean bear, faces population declines due to similar conflict with livestock owners in countries like ecuador. False False False False False False False True True False True False False False False 0.0516 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +85 unstructured 12301 12323 wikiHuman-wildlife_conflict [42] marine ecosystems False False False False False False False True False True False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +86 unstructured 12325 12573 wikiHuman-wildlife_conflict while many of the causes of human-wildlife conflict are the same between terrestrial and marine ecosystems (depredation, competition, human injury, etc.), as of 2019, ocean environments have been less studied and management approaches often differ. False False False False False True False True True True True False False False False -0.6249 False False False True True False True True False False False False False False False False True False False False False False False False False False False True False False +87 unstructured 12574 12697 wikiHuman-wildlife_conflict [43] as with terrestrial conflict, human-wildlife conflict in aquatic environments is diverse and extends across the globe. False False False False False True False True True False True False False False False -0.4767 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +88 unstructured 12698 12858 wikiHuman-wildlife_conflict "in hawaii, for example, an increase in monk seals around the islands has created a conflict between locals who believe that seals ""belong"" and those who do not." False False False True False False False False True False True False True False False 0.25 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +89 unstructured 12859 13043 wikiHuman-wildlife_conflict [44] marine predators such as killer whales[45] and fur seals[46] compete with fisheries for food and resources, while others like great white sharks have a history of injuring humans. False False False False False False False False True True False False False False False 0.3182 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +90 unstructured 13050 13154 wikiHuman-wildlife_conflict in the summer of 2022, a 1,300-pound walrus appeared in oslo harbor and moved in highly populated areas. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +91 unstructured 13155 13291 wikiHuman-wildlife_conflict norwegian authorities declared her a threat to human safety as she had moved onto boats, threatened to sink them and she was euthanized. False False False False False False False False True False False False False False False -0.5574 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +92 unstructured 13292 13422 wikiHuman-wildlife_conflict "in april 2023, a life sized bronze sculpture of her was installed at kongen marina to ""create a historic document about the case""." False False False False False False False False False False False False False False False 0.2732 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +93 unstructured 13429 13615 wikiHuman-wildlife_conflict mitigation strategies a traditional livestock corral surrounded by a predator-proof corral in south gobi desert, mongolia, to protect livestock from predators like snow leopard and wolf. True False False False False False False False False False False False False False False 0.6249 True False True False False False False False False False False True False False False False False False False True True False False False False False False False False False +94 unstructured 13617 13738 wikiHuman-wildlife_conflict mitigation strategies for managing human-wildlife conflict vary significantly depending on location and type of conflict. False False False False False True False True True False True False False False False -0.5574 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +95 unstructured 13739 13886 wikiHuman-wildlife_conflict the preference is always for passive, non-intrusive prevention measures but often active intervention is required to be carried out in conjunction. False False False False False False False False False False False False False False False 0.6059 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +96 unstructured 13887 14035 wikiHuman-wildlife_conflict [9] regardless of approach, the most successful solutions are those that include local communities in the planning, implementation, and maintenance. False False False False False False False False False False False False False False False 0.7245 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +97 unstructured 14036 14162 wikiHuman-wildlife_conflict [48] resolving conflicts, therefore, often requires a regional plan of attack with a response tailored to the specific crisis. False False False False False False False False True False True False False False False -0.802 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +98 unstructured 14163 14271 wikiHuman-wildlife_conflict [49] still, there are a variety of management techniques that are frequently employed to mitigate conflicts. False False False False False False False False True False True False False False False -0.3818 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +99 unstructured 14272 14289 wikiHuman-wildlife_conflict examples include: False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +100 unstructured 14291 14572 wikiHuman-wildlife_conflict "translocation of problematic animals: relocating so-called ""problem"" animals from a site of conflict to a new place is a mitigation technique used in the past, although recent research has shown that this approach can have detrimental impacts on species and is largely ineffective." False False False False False False False True True False True False False False False -0.8126 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +101 unstructured 14573 14756 wikiHuman-wildlife_conflict "[50] translocation can decrease survival rates and lead to extreme dispersal movements for a species, and often ""problem"" animals will resume conflict behaviors in their new location." False False False False False False False True True False True False False True False -0.6124 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +102 unstructured 14757 15069 wikiHuman-wildlife_conflict [51] erection of fences or other barriers: building barriers around cattle bomas,[15] creating distinct wildlife corridors,[52] and erecting beehive fences around farms to deter elephants[53] have all demonstrated the ability to be successful and cost-effective strategies for mitigating human-wildlife conflict. False False False False False True False True True False True False False False False 0.7184 True False True False False False True False False False False False False False False False False False False False False False False False False False False False False False +103 unstructured 15070 15314 wikiHuman-wildlife_conflict [54] improving community education and perception of animals: various cultures have myriad views and values associated with the natural world, and how wildlife is perceived can play a role in exacerbating or alleviating human-wildlife conflict. False False False True False True False True True False True False True False False 0.7964 True False True False False False True False False False False False False False False False False False False False False False False False False False False False False False +104 unstructured 15315 15550 wikiHuman-wildlife_conflict in one masaai community where young men once obtained status by killing lions, conservationists worked with community leaders to shift perceptions and allow those young men to achieve the same social status by protecting lions instead. False False False False False False False True False False False False False False False -0.5423 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +105 unstructured 15551 15680 wikiHuman-wildlife_conflict [48] effective land use planning: altering land use practices can help mitigate conflict between humans and crop-raiding animals. False False False False False False False True True True True False False False False 0.5423 True False True False False False True True False False False False False False False False True False False False False False False False False False False True False False +106 unstructured 15681 15850 wikiHuman-wildlife_conflict for example, in mozambique, communities started to grow more chili pepper plants after making the discovery that elephants dislike and avoid plants containing capsaicin. False False False False False False False False False False False False True False False -0.5859 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +107 unstructured 15851 15981 wikiHuman-wildlife_conflict this creative and effective method discourages elephants from trampling community farmers' fields as well as protects the species. False False False False False False False True False False False False False False False 0.7579 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +108 unstructured 15982 16142 wikiHuman-wildlife_conflict [55] compensation: in some cases, governmental systems have been established to offer monetary compensation for losses sustained due to human-wildlife conflict. False False False False False True False True True False True False False False False -0.6124 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +109 unstructured 16143 16287 wikiHuman-wildlife_conflict these systems hope to deter the need for retaliatory killings of animals, and to financially incentivize the co-existing of humans and wildlife. False False False False False True False True False False False False False False False -0.3818 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +110 unstructured 16288 16397 wikiHuman-wildlife_conflict [56] compensation strategies have been employed in india,[57] italy,[56] and south africa,[58] to name a few. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +111 unstructured 16398 16597 wikiHuman-wildlife_conflict the success of compensation in managing human-wildlife conflict has varied greatly due to under-compensation, a lack of local participation, or a failure by the government to provide timely payments. False False False False False True False True True False True False False False False -0.4939 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +112 unstructured 16598 16813 wikiHuman-wildlife_conflict [56][57] spatial analyses and mapping conflict hotspots: mapping interactions and creating spatial models has been successful in mitigating human-carnivore conflict[59] and human-elephant conflict,[60] among others. False False False False False False False False True False True False False False False 0.5719 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +113 unstructured 16814 17026 wikiHuman-wildlife_conflict in kenya, for example, using grid-based geographical information systems in collaboration with simple statistical analyses allowed conservationists to establish an effective predictor for human-elephant conflict. False False False False False False False True True False True False False False False 0.2023 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +114 unstructured 17027 17198 wikiHuman-wildlife_conflict [60] predator-deterring guard dogs: the use of guard dogs to protect livestock from depredation has been effective in mitigating human-carnivore conflict around the globe. False False False False False False False False True False True False False False False 0.5267 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +115 unstructured 17199 17399 wikiHuman-wildlife_conflict a recent review found that 15.4% of study cases researching human-carnivore conflict used livestock-guarding dogs as a management technique, with animal losses on average 60 times lower than the norm. False False False False False False False True True False True False False False False -0.7351 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +116 unstructured 17400 17607 wikiHuman-wildlife_conflict [61] managing garbage and artificial feeding to prevent attraction of wildlife: many wildlife species are attracted to garbage, especially including food wastes, leading to negative interactions with people. False False False False False False False True False False False False False False False 0.296 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +117 unstructured 17608 17769 wikiHuman-wildlife_conflict [62] poor disposal of garbage such as hotel waste is rapidly emerging as an important aspect that heightens human-carnivore conflicts in countries such as india. False False False False False False False False True False True False False False False -0.7717 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +118 unstructured 17770 17948 wikiHuman-wildlife_conflict [22] urgent research to increase knowledge of the impact of easily available garbage is needed, and improving management of garbage in areas where carnivores reside is essential. False False False True False False False False False False False False True False False 0.8074 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +119 unstructured 17949 18079 wikiHuman-wildlife_conflict managing garbage disposal and artificial feeding of primates can also reduce conflicts and opportunities for disease transmission. False False False False False False False False True False True False False True False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +120 unstructured 18080 18213 wikiHuman-wildlife_conflict one study found that prohibiting tourists from feeding japanese macaques reduced aggressive interactions between macaques and people. False False False False False False False False False False False False False True False -0.1531 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +121 unstructured 18214 18374 wikiHuman-wildlife_conflict [63][64] use of technology: rapid technology development (especially information technology) can play a vital role in the prevention of human-wildlife conflict. False False False False False True False True True False True False False False False 0.3182 True False True False False False True False False False False False False False False False False False False False False False False False False False False False False False +122 unstructured 18375 18554 wikiHuman-wildlife_conflict drones and mobile applications can be used to detect the movements of animals and warn highways and railways authorities to prevent collisions of animals with vehicles and trains. False False False False False False False True False False False False False False False -0.34 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +123 unstructured 18555 18671 wikiHuman-wildlife_conflict sms or whatsapp messaging systems have also been used to alert people about the presence of animals in nearby areas. False False False False False False False True False False False False False False False 0.296 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +124 unstructured 18672 18819 wikiHuman-wildlife_conflict early warning wireless systems have been successfully used in undulating and flat terrain to mitigate human-elephant conflict in tamil nadu, india. False False False False False False False False True False True False False False False -0.128 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +125 unstructured 18830 18952 wikiHuman-wildlife_conflict livestock guardian dogs can be an effective and popular way of deterring predators and reducing human-carnivore conflicts. False False False False False False False False True False True False False False False 0.5106 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +126 unstructured 18953 18991 wikiHuman-wildlife_conflict [61] hidden dimensions of the conflict False False False False False False False False True False True False False False False -0.3182 False False False True True False False False False False False False False False False False False False False False False False False False False False False False False False +127 unstructured 18993 19131 wikiHuman-wildlife_conflict human wildlife conflict also has a range of hidden dimensions that are not typically considered when the focus is on visible consequences. False False False False False True False True True False True False False False False -0.3182 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False +128 unstructured 19132 19207 wikiHuman-wildlife_conflict these can include health impacts, opportunity costs, and transaction costs. False False False False False False False False False False False False False False False 0.4215 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +129 unstructured 19208 19453 wikiHuman-wildlife_conflict [67] as of 2013, case studies have included work on elephants in uttarakhand, northeast india, where human-elephant interactions are correlated with increased imbibing of alcohol by crop guardians resulting in enhanced mortality in interactions. False False False True False False False False False False False False True False False 0.2732 True False True False False False False False False False False False False False False False False False False False False False False False False False False False False False +130 unstructured 19454 19506 wikiHuman-wildlife_conflict [68] and issues related to gender in northern india. False False False False False False False False False False False False False False False 0.0 False True True False True False False False False False False False False False False False False False False False False False False False False False False False False False +131 unstructured 19507 19683 wikiHuman-wildlife_conflict [69] in addition, research has shown that the fear caused by the presence of predators can aggravate human-wildlife conflict more than the actual damage produced by encounters. False False False False False True False True True False True False False False False -0.9042 False False False True True False True False False False False False False False False False False False False False False False False False False False False False False False diff --git a/tests/fixtures/v21_sentences_expected_changes.json b/tests/fixtures/v21_sentences_expected_changes.json new file mode 100644 index 0000000..3292ca8 --- /dev/null +++ b/tests/fixtures/v21_sentences_expected_changes.json @@ -0,0 +1,146 @@ +{ + "hwc": { + "rows": [ + 1483, + 1486, + 1487, + 1490, + 1493, + 1495, + 1501, + 1506, + 1512, + 1526, + 1530, + 1531, + 1534, + 1540, + 1543, + 1545, + 1552, + 1560, + 1565, + 1568, + 1587 + ], + "cause": "dash folding: taxonomy term \"human\u2013wildlife\" (en dash) now matches \"human-wildlife\" (hyphen)" + }, + "co_slow": { + "rows": [ + 11, + 503, + 672, + 700, + 702, + 1251, + 1341, + 1449 + ], + "cause": "distance = words strictly between the two matches (v2.1 also counted the first keyword)" + }, + "co_hwc": { + "rows": [ + 1488 + ], + "cause": "distance = words strictly between the two matches (v2.1 also counted the first keyword)" + }, + "co_resourceandcompetition": { + "rows": [ + 1358, + 1562 + ], + "cause": "distance = words strictly between the two matches (v2.1 also counted the first keyword)" + }, + "co_increasecapacity": { + "rows": [ + 132 + ], + "cause": "distance = words strictly between the two matches (v2.1 also counted the first keyword)" + }, + "a1": { + "rows": [ + 11, + 700, + 702 + ], + "cause": "follows from the changes above" + }, + "a2": { + "rows": [ + 11, + 700, + 702 + ], + "cause": "follows from the changes above" + }, + "b2": { + "rows": [ + 11 + ], + "cause": "follows from the changes above" + }, + "b3": { + "rows": [ + 11 + ], + "cause": "follows from the changes above" + }, + "c1": { + "rows": [ + 1562 + ], + "cause": "follows from the changes above" + }, + "d1": { + "rows": [ + 132 + ], + "cause": "follows from the changes above" + }, + "1.1 fast climate onset": { + "rows": [ + 11, + 700, + 702 + ], + "cause": "follows from the changes above" + }, + "1.2 slow climate onset": { + "rows": [ + 11, + 700, + 702 + ], + "cause": "follows from the changes above" + }, + "2.1 human-wildlife conflict and fast climate onset": { + "rows": [ + 11 + ], + "cause": "follows from the changes above" + }, + "2.1.2 human-wildlife conflict and fast climate onsets in articles that mention ipcc": { + "rows": [ + 11 + ], + "cause": "follows from the changes above" + }, + "2.2 human-wildlife conflict and slow climate onsets": { + "rows": [ + 11 + ], + "cause": "follows from the changes above" + }, + "3. resource competition and human-wild life conflict": { + "rows": [ + 1562 + ], + "cause": "follows from the changes above" + }, + "4. increase of adaptive capacity": { + "rows": [ + 132 + ], + "cause": "follows from the changes above" + } +} \ No newline at end of file diff --git a/tests/fwutil.py b/tests/fwutil.py new file mode 100644 index 0000000..317cb52 --- /dev/null +++ b/tests/fwutil.py @@ -0,0 +1,35 @@ +"""Build small format-1 frameworks for tests.""" + +from __future__ import annotations + +from pathlib import Path + +from ctrlfindr.spreadsheet import SheetSpec, write_workbook + + +def make_framework( + path: Path, + taxonomy: list[tuple] = (), + queries: list[tuple] = (), + variables: list[tuple] = (), + settings: dict | None = None, + tests: list[tuple] = (), + meta: dict | None = None, + drop_sheets: tuple[str, ...] = (), +) -> Path: + """taxonomy: (list, term[, note]); queries: (name, query[, note]); + variables: (number, name, definition[, parent]); tests: (sentence, expect_true, expect_false[, group]).""" + meta = {"format_version": "1", **(meta or {})} + settings = {"sentiment": True, **(settings or {})} # most tests use sentiment; the default is off + sheets = { + "meta": [["key", "value"]] + [[k, v] for k, v in meta.items()], + "settings": [["key", "value", "note"]] + [[k, str(v).lower() if isinstance(v, bool) else str(v), ""] + for k, v in settings.items()], + "taxonomy": [["list", "term", "note"]] + [list(r) + [""] * (3 - len(r)) for r in taxonomy], + "queries": [["name", "query", "note"]] + [list(r) + [""] * (3 - len(r)) for r in queries], + "variables": [["number", "name", "definition", "parent", "note"]] + [list(r) + [""] * (5 - len(r)) for r in variables], + "tests": [["sentence", "expect_true", "expect_false", "group", "note"]] + + [list(r) + [""] * (5 - len(r)) for r in tests], + } + specs = [SheetSpec(name, rows) for name, rows in sheets.items() if name not in drop_sheets] + return write_workbook(path, specs) diff --git a/tests/test_analyze.py b/tests/test_analyze.py new file mode 100644 index 0000000..adf3420 --- /dev/null +++ b/tests/test_analyze.py @@ -0,0 +1,182 @@ +import json +import logging +import sqlite3 + +import numpy as np +import pandas as pd +import pytest + +from ctrlfindr import analyze, config_from_dict, ingest +from ctrlfindr.common import CtrlFindrError, ValidationError +from ctrlfindr.engine import analyze_sentences +from ctrlfindr.framework import load_framework +from fwutil import make_framework + +TAX = [("risk", "risk*"), ("flood", "flood*"), ("ipcc", "ipcc")] +QUERIES = [("q1", "NEAR(risk, flood, 2)"), ("q2", "q1 AND DOC(ipcc)"), ("q3", "flood AND sentiment_negative")] +VARIABLES = [("1", "Risk", "q1"), ("1.1", "IPCC risk", "q2"), ("2", "Bad floods", "q3")] + + +@pytest.fixture +def small_fw(tmp_path): + return load_framework(make_framework(tmp_path / "fw.ods", TAX, QUERIES, VARIABLES)) + + +def project(tmp_path, fw_path, text="Flood risk rises. The IPCC agrees.", **cfg): + (tmp_path / "Text_files").mkdir(exist_ok=True) + (tmp_path / "Text_files" / "d1.txt").write_text(text, encoding="utf-8") + return config_from_dict({"paths": {"framework": str(fw_path)}, **cfg}, base=tmp_path) + + +def test_document_tables(small_fw): + sents = pd.DataFrame({ + "document": ["d1", "d1", "d2", "d2"], + "text": ["Flood risk rises.", "The IPCC agrees.", "Risk of terrible flooding.", "Calm day."], + "vader_compound": [0.2, 0.0, -0.6, 0.3], + }) + res = analyze_sentences(sents, small_fw) + docs = analyze.document_outputs(res, small_fw) + rd = docs["results_document"].set_index("document") + assert rd.loc["d1"].tolist() == [1, 1, 0] and rd.loc["d2"].tolist() == [1, 0, 1] + assert docs["results_document_percentage_of_sentences"].set_index("document").loc["d1", "1. Risk"] == 50.0 + sent = docs["results_document_sentiment"].set_index("document") + assert sent.loc["d2", "2. Bad floods"] == -0.6 + assert np.isnan(sent.loc["d1", "2. Bad floods"]) # never fired in d1 -> empty cell + + +def test_run_writes_outputs_and_keeps_source_tables(tmp_path, small_fw): + cfg = project(tmp_path, small_fw.path) + ingest.run(cfg, show_progress=False) + analyze.run(cfg, show_progress=False) + analyze.run(cfg, show_progress=False) # a second run still finds its input + with sqlite3.connect(cfg.paths.database) as conn: + tables = {r[0] for r in conn.execute("SELECT name FROM sqlite_master WHERE type='table'")} + assert {"documents", "sentences", "results_sentences"} <= tables + out = cfg.paths.output + assert b"\r\n" not in (out / "results_document.tsv").read_bytes() + assert "## Variables" in (out / "codebook.md").read_text(encoding="utf-8") + meta = json.loads((out / "run_metadata.json").read_text(encoding="utf-8")) + assert meta["framework"]["settings"]["sentiment_positive_threshold"] == 0.05 + assert meta["framework"]["meta"]["format_version"] == "1" + + +def test_run_requires_ingest(tmp_path, small_fw): + cfg = config_from_dict({"paths": {"framework": str(small_fw.path)}}, base=tmp_path) + with pytest.raises(CtrlFindrError, match="run the ingest step first"): + analyze.run(cfg, show_progress=False) + + +def test_outputs_of_earlier_runs_are_removed(tmp_path, small_fw): + cfg = project(tmp_path, small_fw.path) + ingest.run(cfg, show_progress=False) + analyze.run(cfg, show_progress=False) + cfg.export.sentence_level = False + analyze.run(cfg, show_progress=False) + assert not (cfg.paths.output / "results_sentences.tsv").exists() + assert (cfg.paths.output / "results_document.tsv").is_file() + + +def test_failing_framework_tests_stop_or_warn(tmp_path, caplog): + tests = [("Floods are wonderful, great and lovely.", "q3", "")] # q3 needs negative sentiment: fails + path = make_framework(tmp_path / "fw.ods", TAX, QUERIES, VARIABLES, tests=tests) + cfg = project(tmp_path, path) + ingest.run(cfg, show_progress=False) + with pytest.raises(ValidationError, match="framework test"): + analyze.run(cfg, show_progress=False) + path = make_framework(tmp_path / "fw2.ods", TAX, QUERIES, VARIABLES, tests=tests, settings={"tests_must_pass": False}) + cfg.paths.framework = path + with caplog.at_level(logging.WARNING, logger="ctrlfindr"): + analyze.run(cfg, show_progress=False) + assert "expected q3 to be true" in caplog.text + + +def test_long_segments_are_reported(small_fw, caplog): + sents = pd.DataFrame({"document": ["d"], "text": ["word " * 600], "vader_compound": [0.0]}) + with caplog.at_level(logging.WARNING, logger="ctrlfindr"): + analyze_sentences(sents, small_fw, long_segment_words=500) + assert "longer than 500 words" in caplog.text + + +def test_config_has_no_sentiment_section(tmp_path): + with pytest.raises(ValidationError, match="framework's settings sheet"): + config_from_dict({"sentiment": {"enabled": True}}, base=tmp_path) + + +def test_exclusion_changes_denominators(tmp_path): + path = make_framework(tmp_path / "fw.ods", TAX, QUERIES, VARIABLES, settings={"exclude_sentence_types": "header"}) + (tmp_path / "Text_files").mkdir() + (tmp_path / "Text_files" / "d1.md").write_text("# Flood risk\n\nFlood risk rises. Calm.", encoding="utf-8") + cfg = config_from_dict({"paths": {"framework": str(path)}}, base=tmp_path) + ingest.run(cfg, show_progress=False) + analyze.run(cfg, show_progress=False) + res = pd.read_csv(cfg.paths.output / "results_sentences.tsv", sep="\t") + assert res["structure"].tolist() == ["body", "body"] # the heading is gone + pct = pd.read_csv(cfg.paths.output / "results_document_percentage_of_sentences.tsv", sep="\t") + assert pct.loc[0, "1. Risk"] == 50.0 # 1 of 2 remaining sentences + + +def test_sentiment_off_needs_no_vader(tmp_path, monkeypatch): + import sys + + monkeypatch.setitem(sys.modules, "vaderSentiment", None) # importing it would fail + path = make_framework(tmp_path / "fw.ods", TAX, [("q1", "NEAR(risk, flood, 2)")], [("1", "Risk", "q1")], + settings={"sentiment": False}) + cfg = project(tmp_path, path) + ingest.run(cfg, show_progress=False) + analyze.run(cfg, show_progress=False) + assert not (cfg.paths.output / "results_document_sentiment.tsv").exists() + assert "vader_compound" not in pd.read_csv(cfg.paths.output / "results_sentences.tsv", sep="\t").columns + + +def test_sentiment_scores_are_cached(tmp_path, small_fw, monkeypatch): + from ctrlfindr import sentiment + + calls = [] + real = sentiment.score_texts + monkeypatch.setattr(sentiment, "score_texts", lambda texts: calls.append(len(texts)) or real(texts)) + cfg = project(tmp_path, small_fw.path) + ingest.run(cfg, show_progress=False) + analyze.run(cfg, show_progress=False) + analyze.run(cfg, show_progress=False) + (tmp_path / "Text_files" / "d1.txt").write_text("Flood risk rises. The IPCC agrees. New one.", encoding="utf-8") + ingest.run(cfg, show_progress=False) + analyze.run(cfg, show_progress=False) + assert calls == [2, 3] # second run used the cache; the changed document was rescored + + +def test_batches_never_split_documents(): + from ctrlfindr.analyze import _batches + + counts = [("a", 3), ("b", 4), ("c", 10), ("d", 1), ("e", 1)] + assert _batches(counts, 5) == [("a", "a", 3), ("b", "b", 4), ("c", "c", 10), ("d", "e", 2)] + assert _batches(counts, 100) == [("a", "e", 19)] + assert _batches(counts, 1) == [(d, d, c) for d, c in counts] + + +def test_stopped_analysis_keeps_previous_outputs(tmp_path, small_fw, monkeypatch): + from ctrlfindr import engine + + (tmp_path / "Text_files").mkdir() + for i in range(3): + (tmp_path / "Text_files" / f"d{i}.txt").write_text("Flood risk rises. Calm.", encoding="utf-8") + cfg = config_from_dict({"paths": {"framework": str(small_fw.path)}, "analysis": {"batch_sentences": 2}}, + base=tmp_path) + ingest.run(cfg, show_progress=False) + analyze.run(cfg, show_progress=False) + before = {p.name: p.read_bytes() for p in cfg.paths.output.glob("results_*.tsv")} + real, calls = engine.analyze_sentences, {"n": 0} + + def flaky(*a, **k): + calls["n"] += 1 + if calls["n"] == 2: + raise KeyboardInterrupt + return real(*a, **k) + + monkeypatch.setattr(analyze, "analyze_sentences", flaky) + with pytest.raises(KeyboardInterrupt): + analyze.run(cfg, show_progress=False) + assert {p.name: p.read_bytes() for p in cfg.paths.output.glob("results_*.tsv")} == before + assert not list(cfg.paths.output.glob("*.partial")) + with sqlite3.connect(cfg.paths.database) as conn: + assert conn.execute("SELECT COUNT(*) FROM results_sentences").fetchone()[0] == 6 + assert not conn.execute("SELECT 1 FROM sqlite_master WHERE name='results_sentences_new'").fetchone() diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..5f3e7ea --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,30 @@ +from ctrlfindr.__main__ import main +from fwutil import make_framework + + +def test_template_command(tmp_path, capsys): + assert main(["template", str(tmp_path / "f.xlsx")]) == 0 + assert (tmp_path / "f.xlsx").is_file() + assert main(["template", str(tmp_path / "f.xlsx")]) == 2 # never overwrites + assert main(["template", str(tmp_path / "f.csv")]) == 2 + + +def test_check_command(tmp_path): + (tmp_path / "Input").mkdir() + fw = make_framework(tmp_path / "Input" / "fw.ods", [("haz", "flood*")], [("h", "haz")], [("1", "Hazard", "h")], + tests=[("Floods came.", "h", "")]) + (tmp_path / "ctrlfindr.toml").write_text(f'[paths]\nframework = "{fw.as_posix()}"\n', encoding="utf-8") + assert main(["check", "-c", str(tmp_path / "ctrlfindr.toml")]) == 0 + bad = make_framework(tmp_path / "Input" / "bad.ods", [("haz", "flood*")], [("h", "missing")], [("1", "H", "h")]) + (tmp_path / "bad.toml").write_text(f'[paths]\nframework = "{bad.as_posix()}"\n', encoding="utf-8") + assert main(["check", "-c", str(tmp_path / "bad.toml")]) == 2 + assert main(["check", "-c", str(tmp_path / "nope.toml")]) == 2 + + +def test_stop_returns_130(tmp_path, monkeypatch, capsys): + from ctrlfindr import ingest + + (tmp_path / "ctrlfindr.toml").write_text("", encoding="utf-8") + monkeypatch.setattr(ingest, "run", lambda *a, **k: (_ for _ in ()).throw(KeyboardInterrupt)) + assert main(["ingest", "-c", str(tmp_path / "ctrlfindr.toml"), "--no-progress"]) == 130 + assert "run the same command again" in capsys.readouterr().err.lower() diff --git a/tests/test_engine.py b/tests/test_engine.py new file mode 100644 index 0000000..748c170 --- /dev/null +++ b/tests/test_engine.py @@ -0,0 +1,155 @@ +import logging + +import numpy as np +import pandas as pd +import pytest + +from ctrlfindr.engine import analyze_sentences, run_tests, sentiment_flags +from ctrlfindr.framework import Settings, load_framework +from fwutil import make_framework + +TAX = [("haz", "flood*"), ("haz", "drought*"), ("haz", "storm*"), ("conflict", "conflict*"), ("conflict", "clash*"), + ("ipcc", "ipcc"), ("clim", "climat*"), ("clim", "-business climate"), ("wild", "wildlife")] + +# document d1: 3 sentences; document d2: 2 sentences +SENTENCES = pd.DataFrame({ + "document": ["d1", "d1", "d1", "d2", "d2"], + "text": [ + "Floods and droughts caused conflict.", # 0 + "The IPCC warned about the business climate.", # 1 + "Calm years followed.", # 2 + "Storms, more storms, then a long drought and clashes.", # 3 + "Conflict over wildlife followed the climate shift.", # 4 + ], + "vader_compound": [-0.6, 0.0, 0.3, -0.2, 0.04], +}) + + +def run(tmp_path, queries, variables=None, settings=None, taxonomy=TAX, sentences=SENTENCES): + names = [qq[0] for qq in queries] + variables = variables or [("1", "All", " OR ".join(names))] + fw = load_framework(make_framework(tmp_path / "fw.ods", taxonomy, queries, variables, settings)) + return analyze_sentences(sentences, fw), fw + + +def col(res, name): + return res[name].tolist() + + +def test_lists_inline_terms_and_exclusions(tmp_path): + res, _ = run(tmp_path, [("q1", "haz AND {caus*}"), ("q2", "clim")]) + assert col(res, "q1") == [True, False, False, False, False] + assert col(res, "q2") == [False, False, False, False, True] # "business climate" is excluded + + +@pytest.mark.parametrize("query,expected", [ + ("NEAR(haz, conflict, 0)", [False, False, False, False, False]), + ("NEAR(haz, conflict, 1)", [True, False, False, True, False]), # "droughts caused conflict", "drought and clashes" + ("BEFORE(conflict, haz, 5)", [False, False, False, False, False]), + ("BEFORE(haz, conflict, 5)", [True, False, False, True, False]), + ("NOT_NEAR(haz, conflict, 2)", [True, False, False, True, False]), # "floods" is 3 words from "conflict" + ("NOT_NEAR(haz, conflict, 3)", [False, False, False, True, False]), # "storms" far from "clashes" + ("NEAR(haz OR {calm}, wild OR conflict, 10)", [True, False, False, True, False]), +]) +def test_proximity(tmp_path, query, expected): + res, _ = run(tmp_path, [("q", query)]) + assert col(res, "q") == expected + + +@pytest.mark.parametrize("query,expected", [ + ("DOC(ipcc)", [True, True, True, False, False]), + ("DOC(haz) >= 2", [False, False, False, False, False]), + ("DOC(conflict) >= 2", [False, False, False, True, True]), + ("DOC_SHARE(haz) >= 0.5", [False, False, False, True, True]), + ("DOC_SHARE(haz) > 0.3", [True, True, True, True, True]), + ("WINDOW(ipcc, 1)", [True, True, True, False, False]), + ("WINDOW(haz, 1) AND NOT haz", [False, True, False, False, True]), # windows stop at document boundaries + ("COUNT(haz) >= 3", [False, False, False, True, False]), + ("COUNT_DISTINCT(haz) >= 2", [True, False, False, True, False]), + ("COUNT_DISTINCT(haz OR conflict) = 3", [True, False, False, True, False]), + ("COUNT(haz) < 1", [False, True, True, False, True]), +]) +def test_scope_and_count_functions(tmp_path, query, expected): + res, _ = run(tmp_path, [("q", query)]) + assert col(res, "q") == expected + + +def test_sentiment_settings_come_from_the_framework(tmp_path): + res, _ = run(tmp_path, [("q", "sentiment_positive")], settings={"sentiment_positive_threshold": 0.3}) + assert col(res, "q") == [False, False, True, False, False] + res, _ = run(tmp_path, [("q", "haz")], settings={"sentiment": False}) + assert "vader_compound" not in res.columns and "sentiment_positive" not in res.columns + + +def test_sentiment_flags(): + c = np.array([-0.5, -0.05, -0.01, 0.0, 0.01, 0.05, 0.9]) + f = sentiment_flags(c, Settings()) + assert f["sentiment_positive"].tolist() == [0, 0, 0, 0, 0, 1, 1] + assert f["sentiment_negative"].tolist() == [1, 1, 0, 0, 0, 0, 0] + zero = sentiment_flags(c, Settings(sentiment_positive_threshold=0, sentiment_negative_threshold=0)) + assert zero["sentiment_positive"].tolist() == (c > 0).tolist() and zero["sentiment_neutral"].tolist() == (c == 0).tolist() + + +def test_variables_with_references_and_aggregation(tmp_path): + queries = [("h", "haz"), ("c", "conflict")] + res, fw = run(tmp_path, queries, variables=[("1", "Both", "V(1.1) AND V(1.2)"), ("1.1", "Hazard", "h"), + ("1.2", "Conflict", "c")]) + assert col(res, "1. Both") == [True, False, False, True, False] + res, _ = run(tmp_path, queries, settings={"aggregate_children": True}, + variables=[("1", "Any", ""), ("1.1", "Hazard", "h"), ("1.2", "Conflict", "c")]) + assert col(res, "1. Any") == [True, False, False, True, True] + + +def test_hierarchy_warning(tmp_path, caplog): + with caplog.at_level(logging.WARNING, logger="ctrlfindr"): + run(tmp_path, [("h", "haz"), ("c", "conflict")], variables=[("1", "Parent", "h"), ("1.1", "Child", "c")]) + assert "true in sentences where their parent is false" in caplog.text and "1.1 vs parent 1: 1 sentences" in caplog.text + + +def test_output_columns(tmp_path): + res, fw = run(tmp_path, [("q", "haz")]) + assert list(res.columns)[:2] == ["document", "sentences"] + assert "{caus*}" not in res.columns # inline terms are not exported + assert list(res.columns)[-1] == "1. All" + + +def test_framework_tests_pass_and_fail(tmp_path): + tests = [("Floods hit the coast.", "haz, q", "conflict"), + ("The IPCC report.", "", "", "g"), ("Clashes erupted.", "d", "", "g"), + ("Floods hit the coast.", "conflict", "haz")] + fw = load_framework(make_framework(tmp_path / "fw.ods", TAX, [("q", "haz"), ("d", "conflict AND DOC(ipcc)")], + [("1", "V", "q")], tests=tests)) + failures = run_tests(fw) + assert len(failures) == 2 and all("tests row 5" in f for f in failures) + assert "expected conflict to be true, got false" in failures[0] + assert "Matched: haz: floods" in failures[0] + + +def test_exclude_sentences_by_structure(): + from ctrlfindr.engine import exclude_sentences + + df = pd.DataFrame({"document": ["a", "a", "b"], "structure": ["header", "body", "table"], "text": ["H", "B", "T"]}) + out = exclude_sentences(df, Settings(exclude_sentence_types=("header", "table"))) + assert out["text"].tolist() == ["B"] + assert exclude_sentences(df, Settings()) is df + + +def test_window_counts_sentence_numbers_not_rows(tmp_path): + from ctrlfindr.engine import exclude_sentences + + fw = load_framework(make_framework(tmp_path / "f.ods", [("c", "conflict")], [("w", "WINDOW(c, 1)")], + [("1", "W", "w")], settings={"sentiment": False, + "exclude_sentence_types": "header"})) + s = pd.DataFrame({"document": ["a", "a", "a", "b", "b"], "sentence_id": [1, 2, 3, 1, 2], + "structure": ["body", "header", "body", "body", "body"], + "text": ["Conflict rose.", "Section two", "Calm.", "Quiet.", "Conflict here."]}) + res = analyze_sentences(exclude_sentences(s, fw.settings), fw) + assert res["w"].tolist() == [True, False, True, True] # "Calm." is 2 sentences away; b is its own document + + +def test_missing_sentiment_scores_are_explained(tmp_path): + from ctrlfindr.common import CtrlFindrError + + fw = load_framework(make_framework(tmp_path / "f.ods", [("c", "conflict")], [("q", "c")], [("1", "Q", "q")])) + with pytest.raises(CtrlFindrError, match="no vader_compound scores"): + analyze_sentences(pd.DataFrame({"document": ["a"], "text": ["Conflict."]}), fw) diff --git a/tests/test_framework.py b/tests/test_framework.py new file mode 100644 index 0000000..64e10ee --- /dev/null +++ b/tests/test_framework.py @@ -0,0 +1,175 @@ +import pytest + +from ctrlfindr.common import ValidationError +from ctrlfindr.framework import load_framework +from fwutil import make_framework + +TAX = [("fast", "flood*"), ("fast", "storm"), ("slow", "drought*"), ("hwc", "human~wildlife"), + ("idioms", "business climate"), ("clim", "climat*"), ("clim", "-@idioms"), ("clim", "-climate of fear")] +QUERIES = [("a1", "fast AND NOT slow"), ("co_x", "NEAR(fast, slow, 5)"), ("a2", '"a1" OR co_x'), + ("s", "slow AND sentiment_negative"), ("haz", "fast OR slow")] +VARIABLES = [("1", "Fast onset", "a1"), ("1.10", "Chained", "a2 OR s"), ("1,1", "Other", "V(1.10)")] + + +def build(tmp_path, ext="ods", **kw): + args = dict(taxonomy=TAX, queries=QUERIES, variables=VARIABLES) + args.update(kw) + return make_framework(tmp_path / f"fw.{ext}", **args) + + +def problems(tmp_path, **kw): + with pytest.raises(ValidationError) as exc: + load_framework(build(tmp_path, **kw)) + return "\n".join(exc.value.problems) + + +@pytest.mark.parametrize("ext", ["ods", "xlsx"]) +def test_valid_framework(tmp_path, ext): + fw = load_framework(build(tmp_path, ext, meta={"title": "T", "my_key": "kept"})) + assert list(fw.lists) == ["fast", "slow", "hwc", "idioms", "clim"] + assert fw.lists["clim"].exclude_lists == ["idioms"] and fw.lists["clim"].exclude_terms == ["climate of fear"] + assert fw.query_order.index("a1") < fw.query_order.index("a2") + assert [v.column for v in fw.variables] == ["1. Fast onset", "1.10 Chained", "1.1 Other"] # names keep case + assert fw.variable("1.1").parent == "1" and fw.variable("1.10").parent == "1" + assert fw.variable_order.index("1.10") < fw.variable_order.index("1.1") # V(1.10) evaluated first + assert fw.meta == {"format_version": "1", "title": "T", "my_key": "kept"} + + +def test_format_version_is_required_and_checked(tmp_path): + assert "format_version' is missing" in problems(tmp_path, meta={"format_version": ""}) + assert "reads framework format 1" in problems(tmp_path, meta={"format_version": "2"}) + path = make_framework(tmp_path / "x.ods", TAX, QUERIES, VARIABLES, drop_sheets=("meta",)) + with pytest.raises(ValidationError, match="not a CtrlFindr framework"): + load_framework(path) + + +def test_language(tmp_path): + assert "not supported" in problems(tmp_path, meta={"language": "it"}) + + +def test_settings(tmp_path): + fw = load_framework(build(tmp_path, settings={"aggregate_children": "yes", "sentiment_positive_threshold": "0,1"})) + assert fw.settings.aggregate_children and fw.settings.sentiment_positive_threshold == 0.1 + text = problems(tmp_path, settings={"sentiment": "maybe", "colour": "red", "sentiment_negative_threshold": "0.5"}) + assert "sentiment must be true or false" in text and "unknown setting 'colour'" in text and "thresholds must satisfy" in text + + +def test_sentiment_flags_need_sentiment(tmp_path): + assert "needs sentiment" in problems(tmp_path, settings={"sentiment": False}) + + +def test_name_rules(tmp_path): + text = problems(tmp_path, taxonomy=TAX + [("fast onset", "x"), ("near", "y"), ("2x", "z"), ("sentiment_positive", "w")]) + assert "'fast onset' must start with a letter" in text and "'fast_onset'" in text + assert "'near' is a reserved word" in text + assert "'2x' must start with a letter" in text and "'x_2x'" in text + assert "'sentiment_positive' is reserved by CtrlFindr" in text + + +def test_names_unique_across_lists_and_queries(tmp_path): + assert "name 'fast' is already used in sheet 'taxonomy'" in problems(tmp_path, queries=QUERIES + [("fast", "slow")]) + + +def test_taxonomy_problems(tmp_path): + text = problems(tmp_path, taxonomy=TAX + [("bad", "~x"), ("bad2", "_ risk"), ("only_excl", "-foo"), + ("x", "-@missing"), ("self", "a"), ("self", "-@self")]) + assert "'~' must join two parts" in text + assert "word gap '_' needs words on both sides" in text + assert "list 'only_excl'" in text and "only exclusions" in text + assert "'-@missing' refers to a list that does not exist" in text + assert "cannot exclude itself" in text + + +def test_exclusion_loop(tmp_path): + text = problems(tmp_path, taxonomy=TAX + [("p", "a"), ("p", "-@q"), ("q", "b"), ("q", "-@p")]) + assert "exclude each other in a loop" in text + + +def test_query_problems(tmp_path): + text = problems(tmp_path, queries=QUERIES + [ + ("q1", "fast AND missing"), ("q2", "NEAR(a1 AND slow, fast, 3)"), ("q3", "fast AND"), + ("q4", "V(1)"), ("q5", "{-foo} OR fast"), ("q6", "COUNT(s) >= 2"), + ]) + assert "'missing' is not a list, query or sentiment flag" in text + assert "NEAR needs word positions" in text + assert "query 'q3': the expression ends" in text + assert "V(1) can only be used in the variables sheet" in text + assert "exclusions belong in the taxonomy sheet" in text + assert "COUNT needs word positions, but 's'" in text + + +def test_query_defined_as_or_of_lists_is_a_set(tmp_path): + fw = load_framework(build(tmp_path, queries=QUERIES + [("q", "COUNT_DISTINCT(haz) >= 2"), ("n", "NEAR(haz, {rain}, 2)")])) + assert sorted(set(x for v in fw.set_lists.values() for x in v)) == ["fast", "slow", "{rain}"] + assert "{rain}" in fw.inline + + +def test_query_cycles(tmp_path): + assert "loop: p -> r -> p" in problems(tmp_path, queries=QUERIES + [("p", "r"), ("r", "p")]) + + +def test_variable_problems(tmp_path): + text = problems(tmp_path, variables=[("1", "A", "a1"), ("1.", "Dup", "a1"), ("x.1", "Bad", "a1"), ("2", "Empty", ""), + ("3", "Loop", "V(4)"), ("4", "Loop2", "V(3)"), ("5", "Ref", "V(9)"), + ("6", "Parent", "a1", "8")]) + assert "number '1' is already used" in text + assert "number 'x.1' must look like" in text + assert "variable 2 has no definition" in text + assert "variables refer to each other in a loop" in text + assert "V(9) refers to a variable number that does not exist" in text + assert "parent '8' is not a variable number" in text + + +def test_aggregate_children_allows_empty_parent(tmp_path): + fw = load_framework(build(tmp_path, settings={"aggregate_children": True}, + variables=[("1", "Parent", ""), ("1.1", "A", "a1"), ("1.2", "B", "s")])) + assert fw.variable("1").children == ["1.1", "1.2"] + assert fw.variable_order.index("1") > fw.variable_order.index("1.2") + + +def test_explicit_parent_overrides_numbering(tmp_path): + fw = load_framework(build(tmp_path, variables=[("1", "A", "a1"), ("2", "B", "s"), ("2.1", "C", "a2", "1")])) + assert fw.variable("2.1").parent == "1" and fw.variable("1").children == ["2.1"] + + +def test_tests_sheet_parsing(tmp_path): + fw = load_framework(build(tmp_path, tests=[("Floods came.", "fast, a1, 1", "slow", "g1")])) + t = fw.tests[0] + assert (t.expect_true, t.expect_false, t.group) == (("fast", "a1", "1"), ("slow",), "g1") + text = problems(tmp_path, tests=[("Floods.", "fast, nothing", ""), ("No expectations.", "", ""), + ("Context only.", "", "", "g1")]) + assert "names 'nothing'" in text and "row 3: the test has no expect_true" in text and "row 4" not in text + + +def test_all_problems_reported_together(tmp_path): + text = problems(tmp_path, taxonomy=TAX + [("bad", "~x")], queries=QUERIES + [("q", "nope")], + variables=VARIABLES + [("9", "X", "")]) + assert text.count("\n") >= 2 + + +def test_sentiment_is_off_by_default(tmp_path): + from ctrlfindr.spreadsheet import SheetSpec, write_workbook + + path = make_framework(tmp_path / "fw.ods", TAX, [("q", "fast")], [("1", "V", "q")]) + sheets = {"meta": [["key", "value"], ["format_version", "1"]], "taxonomy": [["list", "term"], ["a", "x"]], + "queries": [["name", "query"], ["q", "a"]], "variables": [["number", "name", "definition"], ["1", "V", "q"]]} + path = write_workbook(tmp_path / "nosettings.ods", [SheetSpec(k, v) for k, v in sheets.items()]) + assert load_framework(path).settings.sentiment is False + + +def test_exclude_sentence_types(tmp_path): + fw = load_framework(build(tmp_path, settings={"exclude_sentence_types": "Header; table"})) + assert fw.settings.exclude_sentence_types == ("header", "table") + assert "unknown sentence type(s) 'footer'" in problems(tmp_path, settings={"exclude_sentence_types": "header, footer"}) + + +def test_empty_setting_value_keeps_default(tmp_path): + fw = load_framework(build(tmp_path, settings={"sentiment_positive_threshold": ""})) + assert fw.settings.sentiment_positive_threshold == 0.05 + + +def test_sentiment_without_vader_is_reported(tmp_path, monkeypatch): + from ctrlfindr import sentiment + + monkeypatch.setattr(sentiment, "available", lambda: False) + assert "needs the vaderSentiment package" in problems(tmp_path) diff --git a/tests/test_golden.py b/tests/test_golden.py new file mode 100644 index 0000000..70d040c --- /dev/null +++ b/tests/test_golden.py @@ -0,0 +1,145 @@ +"""Regression tests against committed outputs. + +* v2.1: the 2.1 code (legacy/) must reproduce its own committed outputs, and the 3.0 engine + with the example framework (format 1, the same rules rewritten) must, on the sentences + committed with 2.1 and with 2.1's sentiment scores and thresholds of 0, change exactly the + rows listed and explained in fixtures/v21_sentences_expected_changes.json. +* v3: the full pipeline on examples/example_project must reproduce fixtures/golden_v3. +""" + +import importlib.util +import json +import shutil +import warnings + +import numpy as np +import pandas as pd +import pytest + +from conftest import EXAMPLE, FIXTURES, ROOT +from ctrlfindr import analyze, config_from_dict, ingest +from ctrlfindr.analyze import OUTPUT_FILES, document_outputs +from ctrlfindr.common import normalize_text +from ctrlfindr.engine import analyze_sentences +from ctrlfindr.framework import load_framework + +V21 = FIXTURES / "golden_v2.1" +V3 = FIXTURES / "golden_v3" + + +def assert_frames_equal(new: pd.DataFrame, old: pd.DataFrame, name: str, skip=()): + assert list(new.columns) == list(old.columns), name + assert len(new) == len(old), name + for col in old.columns: + if col in skip: + continue + a, b = new[col], old[col] + if pd.api.types.is_float_dtype(a) or pd.api.types.is_float_dtype(b): + ok = np.isclose(a.astype(float), b.astype(float), rtol=1e-9, atol=1e-12, equal_nan=True) + else: + ok = a.astype(str).to_numpy() == b.astype(str).to_numpy() + assert ok.all(), f"{name}: column {col!r} differs in {int((~ok).sum())} rows, first at {int(np.flatnonzero(~ok)[0])}" + + +def test_v21_fixtures_are_reproducible(golden_v21_sentences): + """The extracted 2.1 code reproduces its own committed outputs (fixture sanity check).""" + pytest.importorskip("odf", reason="the 2.1 code reads its framework with odfpy") + spec = importlib.util.spec_from_file_location("ctrlfindr_v21", ROOT / "legacy" / "v2.1" / "ctrlfindr_v21.py") + v21 = importlib.util.module_from_spec(spec) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", SyntaxWarning) + spec.loader.exec_module(v21) + out = v21.run_analysis_from_sentences(golden_v21_sentences, str(V21 / "Assessment_framework_v2.1.ods"), + compound=golden_v21_sentences["vadar_compound"]) + for name in OUTPUT_FILES: + assert_frames_equal(out[name].reset_index(drop=True), pd.read_csv(V21 / f"{name}.tsv", sep="\t"), name) + + +@pytest.fixture(scope="module") +def v3_on_v21_sentences(golden_v21_sentences): + fw = load_framework(EXAMPLE / "Input" / "Assessment_framework.ods") + fw.settings.sentiment_positive_threshold = fw.settings.sentiment_negative_threshold = 0 # as in 2.1 + g = golden_v21_sentences + inp = pd.DataFrame({"document": g["document"], "text": g["sentences"], "vader_compound": g["vadar_compound"]}) + res = analyze_sentences(inp, fw) + return fw, res + + +def lower_columns(df): + return df.rename(columns=str.lower) # 2.1 lowercased variable names + + +def test_v3_engine_on_v21_sentences_changes_only_reviewed_rows(v3_on_v21_sentences, golden_v21_sentences): + _, res = v3_on_v21_sentences + res = lower_columns(res) + old = golden_v21_sentences.rename(columns={"vadar_compound": "vader_compound"}) + expected = json.loads((FIXTURES / "v21_sentences_expected_changes.json").read_text(encoding="utf-8")) + assert set(res.columns) == set(old.columns) + assert (res["sentences"] == old["sentences"].map(lambda s: normalize_text(s).lower())).all() + for col in old.columns: + if old[col].dtype != bool: + continue + diff = np.flatnonzero(res[col].to_numpy() != old[col].to_numpy()).tolist() + assert diff == expected.get(col, {}).get("rows", []), col + + +def test_v3_document_tables_on_v21_sentences(v3_on_v21_sentences): + fw, res = v3_on_v21_sentences + expected = json.loads((FIXTURES / "v21_sentences_expected_changes.json").read_text(encoding="utf-8")) + changed = {c for c in expected} + for name, df in document_outputs(res, fw).items(): + old = pd.read_csv(V21 / f"{name}.tsv", sep="\t") + assert_frames_equal(lower_columns(df), old, name, skip=changed) + + +def test_example_framework_tests_pass(): + from ctrlfindr.engine import run_tests + + fw = load_framework(EXAMPLE / "Input" / "Assessment_framework.ods") + assert len(fw.tests) >= 5 and run_tests(fw) == [] + + +@pytest.mark.parametrize("batch", [50000, 1]) # one document per batch must give the same results +def test_end_to_end_example_matches_golden_v3(tmp_path, batch): + shutil.copytree(EXAMPLE / "Text_files", tmp_path / "Text_files") + shutil.copytree(EXAMPLE / "Input", tmp_path / "Input") + cfg = config_from_dict({"analysis": {"batch_sentences": batch}}, base=tmp_path) + ingest.run(cfg, show_progress=False) + analyze.run(cfg, show_progress=False) + for name in OUTPUT_FILES: + new = pd.read_csv(cfg.paths.output / f"{name}.tsv", sep="\t") + assert_frames_equal(new, pd.read_csv(V3 / f"{name}.tsv", sep="\t"), name) + + +def test_markdown_example(tmp_path): + from conftest import ROOT as root + + src = root / "examples" / "markdown_project" + shutil.copytree(src / "Text_files", tmp_path / "Text_files") + shutil.copytree(src / "Input", tmp_path / "Input") + cfg = config_from_dict({"paths": {"framework": "Input/Assessment_framework.xlsx"}}, base=tmp_path) + ingest.run(cfg, show_progress=False) + analyze.run(cfg, show_progress=False) + res = pd.read_csv(cfg.paths.output / "results_sentences.tsv", sep="\t") + assert set(res["structure"]) == {"body", "list"} # headers and table rows excluded by the framework + assert len(res) == 14 and res["1. Drought-related conflict"].sum() >= 2 + + +def test_food_security_sample_project(tmp_path): + """The sample project in examples/food_security_project reproduces its expected outputs.""" + from conftest import ROOT as root + from ctrlfindr.engine import run_tests + + src = root / "examples" / "food_security_project" + shutil.copytree(src / "Text_files", tmp_path / "Text_files") + shutil.copytree(src / "Input", tmp_path / "Input") + for name in ("Assessment_framework.xlsx", "Assessment_framework.ods"): + fw = load_framework(tmp_path / "Input" / name) + assert len(fw.tests) == 29 and run_tests(fw) == [] + cfg = config_from_dict({"paths": {"framework": "Input/Assessment_framework.xlsx"}}, base=tmp_path) + summary = ingest.run(cfg, show_progress=False) + assert (summary["files"], summary["sentences"]) == (6, 44) # ._notes.txt skipped, empty.txt has none + analyze.run(cfg, show_progress=False) + for name in OUTPUT_FILES: + new = pd.read_csv(cfg.paths.output / f"{name}.tsv", sep="\t") + assert_frames_equal(new, pd.read_csv(src / "expected" / f"{name}.tsv", sep="\t"), name) diff --git a/tests/test_ingest.py b/tests/test_ingest.py new file mode 100644 index 0000000..0f1797a --- /dev/null +++ b/tests/test_ingest.py @@ -0,0 +1,219 @@ +import sqlite3 +import sys + +import pytest + +from ctrlfindr import config_from_dict, ingest +from ctrlfindr.common import CtrlFindrError, ValidationError + + +def project(tmp_path, files: dict[str, bytes], **ingest_settings): + text_dir = tmp_path / "Text_files" + text_dir.mkdir() + for name, data in files.items(): + (text_dir / name).write_bytes(data) + return config_from_dict({"ingest": ingest_settings}, base=tmp_path) + + +def sentences(cfg): + with sqlite3.connect(cfg.paths.database) as conn: + return conn.execute('SELECT document, sentence_id, start, "end", text FROM sentences ORDER BY document, sentence_id').fetchall() + + +def test_clean_document_rules(): + raw = "Heading\nThe security\nclimate is here.\n\n\nNext para with spaces… see https://x.org now.[4] The end" + clean = ingest.clean_document(raw) + assert clean == "Heading The security climate is here.\n\nNext para with spaces. see now. [4] The end" + assert "securityclimate" not in ingest.clean_document("Climate security\nClimate change") + + +def test_config_rejects_fractional_whole_numbers(tmp_path): + with pytest.raises(ValidationError, match="must be a whole number"): + config_from_dict({"ingest": {"long_segment_words": 499.9}}, base=tmp_path) + assert config_from_dict({"ingest": {"long_segment_words": 400.0}}, base=tmp_path).ingest.long_segment_words == 400 + + +def test_crlf_and_bom(tmp_path): + cfg = project(tmp_path, {"a.txt": "First line.\r\nSecond line.\r\n\r\nNew paragraph.".encode("utf-8")}) + ingest.run(cfg, show_progress=False) + texts = [r[4] for r in sentences(cfg)] + assert texts == ["First line.", "Second line.", "New paragraph."] + + +def test_offsets_point_into_clean_text(tmp_path): + cfg = project(tmp_path, {"a.txt": b"One sentence here. Another one, Dr. Smith said. [12]\n\nThird."}) + ingest.run(cfg, show_progress=False) + with sqlite3.connect(cfg.paths.database) as conn: + clean = conn.execute("SELECT clean_text FROM documents").fetchone()[0] + rows = sentences(cfg) + assert [r[4] for r in rows] == ["One sentence here.", "Another one, Dr. Smith said.", "Third."] # "[12]" dropped + assert all(clean[s:e] == t for _, _, s, e, t in rows) + + +def test_discovery_rules(tmp_path): + cfg = project(tmp_path, {"b.TXT": b"Upper case extension.", "a.report.txt": b"Dotted name.", + "c.MD": b"# Title", "notes.docx": b"skip"}) + ingest.run(cfg, show_progress=False) + assert sorted({r[0] for r in sentences(cfg)}) == ["a.report", "b", "c"] + + +def test_duplicate_document_names(tmp_path): + text_dir = tmp_path / "Text_files" + text_dir.mkdir() + (text_dir / "a.txt").write_bytes(b"x.") + (text_dir / "a.Txt").write_bytes(b"y.") + if len(list(text_dir.iterdir())) < 2: + pytest.skip("case-insensitive file system") + with pytest.raises(ValidationError, match="clashing names"): + ingest.run(config_from_dict({}, base=tmp_path), show_progress=False) + + +def test_missing_or_empty_folder(tmp_path): + with pytest.raises(CtrlFindrError, match="not found"): + ingest.run(config_from_dict({}, base=tmp_path), show_progress=False) + (tmp_path / "Text_files").mkdir() + with pytest.raises(CtrlFindrError, match="no .txt or .md files"): + ingest.run(config_from_dict({}, base=tmp_path), show_progress=False) + + +def test_non_utf8(tmp_path): + cfg = project(tmp_path, {"good.txt": b"Fine.", "bad.txt": "Café crème.".encode("cp1252")}) + with pytest.raises(ValidationError, match="not UTF-8") as exc: + ingest.run(cfg, show_progress=False) + assert "bad.txt" in str(exc.value) + cfg.ingest.encoding_errors = "replace" + ingest.run(cfg, show_progress=False) + assert any("Caf�" in r[4] for r in sentences(cfg)) + + +def test_empty_file_is_kept_out_of_sentences(tmp_path): + cfg = project(tmp_path, {"empty.txt": b"", "a.txt": b"Something."}) + ingest.run(cfg, show_progress=False) + assert {r[0] for r in sentences(cfg)} == {"a"} + + +def test_incremental(tmp_path): + cfg = project(tmp_path, {"a.txt": b"Alpha one.", "b.txt": b"Beta one."}) + assert ingest.run(cfg, show_progress=False)["processed"] == 2 + assert ingest.run(cfg, show_progress=False)["processed"] == 0 + (cfg.paths.text_files / "b.txt").write_bytes(b"Beta two. Beta three.") + (cfg.paths.text_files / "a.txt").unlink() + (cfg.paths.text_files / "c.txt").write_bytes(b"Gamma.") + summary = ingest.run(cfg, show_progress=False) + assert (summary["processed"], summary["removed"], summary["unchanged"]) == (2, 1, 0) + assert [r[4] for r in sentences(cfg)] == ["Beta two.", "Beta three.", "Gamma."] + cfg.ingest.text_cleaning = False # a setting change rebuilds everything + assert ingest.run(cfg, show_progress=False)["processed"] == 2 + + +def structures(cfg): + with sqlite3.connect(cfg.paths.database) as conn: + return conn.execute("SELECT document, structure, text FROM sentences ORDER BY document, sentence_id").fetchall() + + +def test_txt_is_unstructured_and_md_is_labelled(tmp_path): + md = ("# Climate *risks*\n\nFloods rose. Droughts too.\n\n- Item one\n- Item two\n\n" + "| Region | Clashes |\n|---|---|\n| North | 14 |\n\nEnd.").encode() + cfg = project(tmp_path, {"plain.txt": b"HEADING\nText here. More text.", "brief.md": md}) + ingest.run(cfg, show_progress=False) + rows = structures(cfg) + assert [r[1:] for r in rows if r[0] == "brief"] == [ + ("header", "Climate risks"), ("body", "Floods rose."), ("body", "Droughts too."), + ("list", "Item one"), ("list", "Item two"), ("table", "Region | Clashes"), ("table", "North | 14"), + ("body", "End.")] + assert {r[1] for r in rows if r[0] == "plain"} == {"unstructured"} + with sqlite3.connect(cfg.paths.database) as conn: + clean = dict(conn.execute("SELECT document, clean_text FROM documents")) + formats = dict(conn.execute("SELECT document, format FROM documents")) + spans = conn.execute('SELECT document, start, "end", text FROM sentences').fetchall() + assert formats == {"brief": "md", "plain": "txt"} + assert all(clean[d][s:e] == t for d, s, e, t in spans) + + +def test_txt_and_md_with_the_same_name_clash(tmp_path): + cfg = project(tmp_path, {"a.txt": b"x.", "a.md": b"y."}) + with pytest.raises(ValidationError, match="clashing names"): + ingest.run(cfg, show_progress=False) + + +def test_ingest_does_not_need_vader(tmp_path, monkeypatch): + import importlib.util + real = importlib.util.find_spec + monkeypatch.setattr(importlib.util, "find_spec", lambda name, *a: None if name.startswith("vaderSentiment") else real(name, *a)) + monkeypatch.setitem(sys.modules, "vaderSentiment", None) + cfg = project(tmp_path, {"a.txt": b"Fine text."}) + assert ingest.run(cfg, show_progress=False)["sentences"] == 1 + + +@pytest.mark.parametrize("raw,clean", [ + ("The report was published.The minister disagreed.", "The report was published. The minister disagreed."), + ("It was late?The minister resigned!Then calm.", "It was late? The minister resigned! Then calm."), + ("COP28 opened in 2023.The talks ended.", "COP28 opened in 2023. The talks ended."), + ("The goal (net zero).Next steps follow.", "The goal (net zero). Next steps follow."), + ("The U.S.Army and the U.K.Government", "The U.S.Army and the U.K.Government"), + ("She has a Ph.D.Later she moved.", "She has a Ph.D.Later she moved."), + ("He lives in St.Louis now.", "He lives in St.Louis now."), + ("Section 4.2.Results show", "Section 4.2.Results show"), + ("See e.g.The report", "See e.g.The report"), +]) +def test_missing_space_after_full_stop_is_repaired_in_txt(raw, clean): + assert ingest.clean_document(raw, repair_spacing=True) == clean + assert ingest.clean_document(raw) == raw # off unless asked (Markdown keeps its own text) + + +def test_missing_space_repair_applies_to_txt_files(tmp_path): + cfg = project(tmp_path, {"a.txt": b"The report was published.The minister disagreed."}) + ingest.run(cfg, show_progress=False) + assert [r[4] for r in sentences(cfg)] == ["The report was published.", "The minister disagreed."] + + +def test_ingest_resumes_after_an_interruption(tmp_path, monkeypatch): + cfg = project(tmp_path, {f"d{i}.txt": f"Document {i} text.".encode() for i in range(5)}) + real = ingest.Segmenter.split + calls = {"n": 0} + + def flaky(self, text, labels=None): + calls["n"] += 1 + if calls["n"] == 3: + raise KeyboardInterrupt # stopped while the third file is processed + return real(self, text, labels) + + monkeypatch.setattr(ingest.Segmenter, "split", flaky) + with pytest.raises(KeyboardInterrupt): + ingest.run(cfg, show_progress=False) + with sqlite3.connect(cfg.paths.database) as conn: + assert conn.execute("SELECT COUNT(*) FROM documents").fetchone()[0] == 2 # two finished files kept + assert conn.execute("SELECT COUNT(*) FROM sentences").fetchone()[0] == 2 + monkeypatch.setattr(ingest.Segmenter, "split", real) + summary = ingest.run(cfg, show_progress=False) + assert (summary["processed"], summary["unchanged"], summary["sentences"]) == (3, 2, 5) + + +def test_changed_file_keeps_old_version_until_reprocessed(tmp_path, monkeypatch): + cfg = project(tmp_path, {"a.txt": b"Old text."}) + ingest.run(cfg, show_progress=False) + (cfg.paths.text_files / "a.txt").write_bytes(b"New text.") + monkeypatch.setattr(ingest.Segmenter, "split", lambda *a: (_ for _ in ()).throw(KeyboardInterrupt)) + with pytest.raises(KeyboardInterrupt): + ingest.run(cfg, show_progress=False) + assert [r[4] for r in sentences(cfg)] == ["Old text."] # never half-replaced + + +def test_hidden_and_temporary_files_are_skipped(tmp_path): + cfg = project(tmp_path, {"a.txt": b"Real text.", "._a.txt": b"\x00\x05\xff binary", "~$draft.md": b"lock"}) + ingest.run(cfg, show_progress=False) + assert {r[0] for r in sentences(cfg)} == {"a"} + + +def test_ingest_discards_results_that_no_longer_match(tmp_path): + cfg = project(tmp_path, {"a.txt": b"One. Two."}) + ingest.run(cfg, show_progress=False) + with sqlite3.connect(cfg.paths.database) as conn: + conn.execute("CREATE TABLE results_sentences (x)") + ingest.run(cfg, show_progress=False) # nothing changed: results kept + with sqlite3.connect(cfg.paths.database) as conn: + assert conn.execute("SELECT 1 FROM sqlite_master WHERE name='results_sentences'").fetchone() + (cfg.paths.text_files / "a.txt").write_bytes(b"One. Two. Three.") + ingest.run(cfg, show_progress=False) + with sqlite3.connect(cfg.paths.database) as conn: + assert not conn.execute("SELECT 1 FROM sqlite_master WHERE name='results_sentences'").fetchone() diff --git a/tests/test_markdown.py b/tests/test_markdown.py new file mode 100644 index 0000000..d5f443b --- /dev/null +++ b/tests/test_markdown.py @@ -0,0 +1,60 @@ +from ctrlfindr.markdown import blocks, inline_text + + +def test_headings_atx_and_setext(): + md = "# Title #\n\n## Sub\n\nSetext one\n==========\n\nSetext two\n----------\n\nBody." + assert blocks(md) == [("header", "Title"), ("header", "Sub"), ("header", "Setext one"), + ("header", "Setext two"), ("body", "Body.")] + + +def test_lists_with_continuation_and_numbers(): + md = "- one\n continued\n* two\n+ three\n1. four\n2) five\n• six" + assert blocks(md) == [("list", "one continued"), ("list", "two"), ("list", "three"), + ("list", "four"), ("list", "five"), ("list", "six")] + + +def test_tables_drop_the_separator_row(): + md = "| A | B |\n|:--|--:|\n| 1 | x |\n| 2 | y |\n\nAfter." + assert blocks(md) == [("table", "A | B"), ("table", "1 | x"), ("table", "2 | y"), ("body", "After.")] + + +def test_a_pipe_without_separator_row_is_not_a_table(): + assert blocks("This | that.") == [("body", "This | that.")] + + +def test_body_paragraphs_quotes_code_and_rules(): + md = "First line\nsecond line.\n\n> Quoted text.\n\n---\n\n```\ncode here\n```\n\n[ref]: http://x.org" + assert blocks(md) == [("body", "First line second line."), ("body", "Quoted text."), ("body", "code here")] + + +def test_footnotes_are_labelled_with_their_continuation(): + md = ("Text.[^1]\n\n[^1]: First line\n continued.\n\n Second paragraph.\n[^2]: Other note.\n- item\n\n" + "Body again.\n\n indented code is body, not a footnote") + assert blocks(md) == [("body", "Text."), ("footnote", "First line continued."), ("footnote", "Second paragraph."), + ("footnote", "Other note."), ("list", "item"), ("body", "Body again."), + ("body", "indented code is body, not a footnote")] + + +def test_footnote_markers_are_removed(): + assert inline_text("Emissions fell.[^1] The IPCC agreed.12 Others disagreed.^3^ a, b") == \ + "Emissions fell. The IPCC agreed. Others disagreed." + + +def test_inline_markup_is_removed(): + assert inline_text("**Bold** and *it* and _em_ but snake_case_name, `code`, ~~gone~~") == \ + "Bold and it and em but snake_case_name, code, gone" + assert inline_text("See [the IPCC](https://ipcc.ch)[^2] ![fig](f.png) now & \\*literal\\*") == \ + "See the IPCC now & *literal*" + + +def test_a_year_at_the_start_of_a_wrapped_line_stays_in_the_paragraph(): + assert blocks("Temperatures rose between 2015 and\n2020. Floods followed.") == [ + ("body", "Temperatures rose between 2015 and 2020. Floods followed.")] + assert blocks("Steps:\n1. Plan\n2. Act") == [("body", "Steps:"), ("list", "Plan"), ("list", "Act")] + + +def test_superscripts(): + assert inline_text("the 20th century and the 21^st^") == "the 20th century and the 21st" + assert inline_text("106 people, 10^9^ tonnes") == "10^6 people, 10^9 tonnes" + assert inline_text("agreed.12 Notesa, b and a Registered mark") == \ + "agreed. Notes and a Registered mark" diff --git a/tests/test_matching.py b/tests/test_matching.py new file mode 100644 index 0000000..cb8bd2e --- /dev/null +++ b/tests/test_matching.py @@ -0,0 +1,114 @@ +import re +import time + +import pytest + +from ctrlfindr.matching import (ListSpec, Matcher, TermError, count, count_distinct, near, not_near, parse_term, + word_tokens) + + +def found(lists, sentence): + m = Matcher({k: v if isinstance(v, ListSpec) else ListSpec(v) for k, v in lists.items()}) + return {name: [sentence[s:e] for s, e, _ in ms] for name, ms in zip(m.lists, m.matches(sentence))} + + +@pytest.mark.parametrize("term,sentence,hit", [ + ("flood", "the flood came", True), ("flood", "the floods came", False), ("flood*", "the floods came", True), + ("flood*", "flash-flooding", True), ("flash flood*", "a flash flooding event", True), + ("climat*", "anticlimatic", False), ("u.s.", "the u.s. policy", True), ("biodivers* loss", "biodiversity loss", True), + ("*fire", "a bushfire", True), ("co2", "co2 emissions", True), ("niño", "el niño year", True), + ("niño", "el niños year", False), ("ipcc", "the ipcc_report", False), + ("human~wildlife", "human-wildlife conflict", True), ("human~wildlife", "human wildlife conflict", True), + ("human~wildlife", "humanwildlife conflict", True), ("human~wildlife", "human/wildlife", False), + ("climate _ risk", "climate change risk", True), ("climate _ risk", "climate risk", False), + ("climate _ risk", "climate and other risk", False), ("adapt* _ _ change", "adapting to climate change", True), +]) +def test_term_semantics(term, sentence, hit): + assert bool(found({"l": [term]}, sentence)["l"]) is hit + assert bool(parse_term(term).regex.search(sentence)) is hit # automaton == regex definition + + +def test_automaton_equals_regex_on_mixed_terms(): + terms = ["climate change", "climat*", "warm*", "heat", "heat wave*", "sea level*", "el niño", "human~wildlife", + "climate _ risk", "*fire"] + rx = re.compile("|".join(parse_term(t).regex.pattern for t in terms)) + for s in ["the climate is changing", "heatwaves and heat waves", "sea levels", "el niño and la niña", "nothing", + "climatechange", "human wildlife", "climate change risk", "wildfire", "heat-wave"]: + assert bool(found({"l": terms}, s)["l"]) == bool(rx.search(s)), s + + +@pytest.mark.parametrize("bad", ["*", "_", "_ risk", "climate _", "~x", "x~", "a ~ b", "a~~b"]) +def test_invalid_terms(bad): + with pytest.raises(TermError): + parse_term(bad) + + +def test_exclusion_phrase_and_list_by_containment(): + lists = {"clim": ListSpec(["climat*"], ["business climate"], ["idioms"]), "idioms": ListSpec(["climate of fear"])} + s = "the business climate and the climate of fear grew while climate change worsened" + assert found(lists, s)["clim"] == ["climate"] # only the last mention counts + assert found(lists, "the business climate improved")["clim"] == [] + + +def test_exclusion_only_removes_contained_matches(): + lists = {"clim": ListSpec(["climate change"], ["climate"])} + assert found(lists, "climate change")["clim"] == ["climate change"] # longer than the exclusion + + +def test_exclusion_loop_is_rejected(): + with pytest.raises(TermError, match="loop"): + Matcher({"a": ListSpec(["x"], [], ["b"]), "b": ListSpec(["y"], [], ["a"])}) + + +def test_entry_ids_are_unique_across_lists(): + m = Matcher({"a": ListSpec(["flood*"]), "b": ListSpec(["drought*"])}) + fa, fb = m.matches("floods and droughts") + assert count_distinct(fa + fb) == 2 + + +def test_count_and_count_distinct(): + m = Matcher({"h": ListSpec(["flood*", "drought*", "climat*", "climate change"])}) + (h,) = m.matches("floods, more floods and drought") + assert count(h) == 3 and count_distinct(h) == 2 + (c,) = m.matches("climate change") # climat* inside climate change counts once + assert count(c) == 1 and count_distinct(c) == 1 + + +def span(s, word, entry=0): + i = s.index(word) + return (i, i + len(word), entry) + + +def test_near_distance_is_words_between(): + s = "increase the adaptive capacity" + t = word_tokens(s) + a, b = [span(s, "increase")], [span(s, "capacity")] + assert near(t, a, b, 2) and not near(t, a, b, 1) + assert near(t, b, a, 2) # any order + assert not near(t, b, a, 2, ordered=True) and near(t, a, b, 2, ordered=True) + + +def test_near_ignores_punctuation_and_overlap(): + s = "capacity - which we increase" + t = word_tokens(s) + assert near(t, [span(s, "increase")], [span(s, "capacity")], 2) + same = [span(s, "capacity")] + assert not near(t, same, same, 5) # a word never co-occurs with itself + + +def test_not_near(): + s = "conflict here, then many many many many words, then conflict near wildlife" + t = word_tokens(s) + first, second = (s.index("conflict"), s.index("conflict") + 8, 0), (s.rindex("conflict"), s.rindex("conflict") + 8, 0) + w = [span(s, "wildlife")] + assert not_near(t, [first, second], w, 2) # the first conflict has no wildlife nearby + assert not not_near(t, [second], w, 2) + assert not_near(t, [second], [], 2) and not not_near(t, [], w, 2) + + +def test_word_tokens_linear_on_punctuation_runs(): + s = "increase - the, (capacity) " + "=" * 200_000 + " end" + t0 = time.perf_counter() + starts, ends = word_tokens(s) + assert time.perf_counter() - t0 < 1 + assert [s[a:b] for a, b in zip(starts, ends)] == ["increase", "the,", "(capacity)", "end"] diff --git a/tests/test_perf.py b/tests/test_perf.py new file mode 100644 index 0000000..23fdf84 --- /dev/null +++ b/tests/test_perf.py @@ -0,0 +1,36 @@ +"""Loose time budgets that catch accidental slowdowns.""" + +import time + +import numpy as np +import pandas as pd + +from conftest import EXAMPLE +from ctrlfindr.engine import analyze_sentences +from ctrlfindr.framework import load_framework + + +def test_analysis_of_32k_sentences(golden_v21_sentences): + fw = load_framework(EXAMPLE / "Input" / "Assessment_framework.ods") + g = pd.concat([golden_v21_sentences] * 20, ignore_index=True) + inp = pd.DataFrame({ + "document": g["document"] + "_" + (np.arange(len(g)) // len(golden_v21_sentences)).astype(str), + "text": g["sentences"] + " " + np.arange(len(g)).astype(str), + "vader_compound": g["vadar_compound"], + }) + t = time.perf_counter() + res = analyze_sentences(inp, fw) + elapsed = time.perf_counter() - t + assert len(res) == 31780 + assert elapsed < 20, f"analysis took {elapsed:.1f}s" # ~1 s on a laptop; v2.1 needed ~20 s + + +def test_long_segments(golden_v21_sentences): + fw = load_framework(EXAMPLE / "Input" / "Assessment_framework.ods") + g = golden_v21_sentences + joined = g.groupby(g.index // 200).agg({"document": "first", "sentences": " ".join, "vadar_compound": "first"}) + inp = pd.DataFrame({"document": joined["document"] + joined.index.astype(str), "text": joined["sentences"], + "vader_compound": joined["vadar_compound"]}) + t = time.perf_counter() + analyze_sentences(inp, fw, long_segment_words=0) + assert time.perf_counter() - t < 10 # v2.1 needed ~10 s for this diff --git a/tests/test_query.py b/tests/test_query.py new file mode 100644 index 0000000..e5af331 --- /dev/null +++ b/tests/test_query.py @@ -0,0 +1,71 @@ +import pytest + +from ctrlfindr import query as q + + +@pytest.mark.parametrize("text,canonical", [ + ('fast AND NOT (slow OR co_slow)', "fast AND NOT (slow OR co_slow)"), + ('"fast" and not("slow")', "fast AND NOT slow"), # quotes optional, case-insensitive + ('(("a")AND("b"))OR"c"', "(a AND b) OR c"), + ('NEAR(fast OR {storm*}, hwc, 5)', "NEAR(fast OR {storm*}, hwc, 5)"), + ('doc(ipcc) >= 3 and doc_share(a1)>=0.05', "DOC(ipcc) >= 3 AND DOC_SHARE(a1) >= 0.05"), + ('doc(x)', "DOC(x)"), ('DOC(x) > 2', "DOC(x) > 2"), ('WINDOW(c, 2) AND d', "WINDOW(c, 2) AND d"), + ('count_distinct(h) >= 2', "COUNT_DISTINCT(h) >= 2"), ('V(2.1) or V(2.2)', "V(2.1) OR V(2.2)"), + ('not_near(a, b, 3)', "NOT_NEAR(a, b, 3)"), ('before(a, b, 0)', "BEFORE(a, b, 0)"), + ('"command-and-control"', "command-and-control"), # operator words inside names stay +]) +def test_parse_and_canonical_text(text, canonical): + assert q.to_text(q.parse(text)) == canonical + + +def test_precedence_not_and_or(): + node = q.parse("a or b and not c") + assert isinstance(node, q.Or) and isinstance(node.operands[1], q.And) + assert isinstance(node.operands[1].operands[1], q.Not) + + +def test_function_name_without_parenthesis_is_a_name(): + assert q.parse("near") == q.Name("near") + + +def test_paragraph_is_not_a_function(): + assert q.parse("paragraph") == q.Name("paragraph") # an ordinary name now + with pytest.raises(q.QuerySyntaxError): + q.parse("PARAGRAPH(x)") + + +def test_bare_count_means_at_least_one(): + c = q.parse("COUNT(x)") + assert (c.op, c.value) == (">=", 1.0) + assert (q.parse("DOC_SHARE(x)").op, q.parse("DOC_SHARE(x)").value) == (">", 0.0) + + +@pytest.mark.parametrize("bad,msg", [ + ("NEAR(a, b)", "takes 3 arguments"), ("NEAR(a, b, 2.5)", "whole number"), ("WINDOW(a, 2) > 1", "cannot be compared"), + ("doc_share(a) > 2", "between 0 and 1"), ("fast AND", "ends where"), ("{storm", "unmatched '{'"), + ("a & b", "unexpected character"), ("V(x)", "variable number"), ("a b", "unexpected 'b'"), ('""', "empty name"), + ("{}", "empty term"), ('"a', "unclosed"), ("", "empty"), ("(a", r"missing '\)'"), ("COUNT(a) >=", "a number after"), +]) +def test_syntax_errors(bad, msg): + with pytest.raises(q.QuerySyntaxError, match=msg): + q.parse(bad) + + +def test_helpers(): + node = q.parse("NEAR(a OR {x y}, b, 2) AND DOC(c) AND V(1.2)") + assert q.names(node) == {"a", "b", "c"} + assert q.inline_terms(node) == {"x y"} + assert q.variable_refs(node) == {"1.2"} + assert [(f, q.to_text(a)) for f, a in q.set_arguments(node)] == [("near", "a OR {x y}"), ("near", "b")] + + +def test_name_rules(): + assert q.check_name("co_slow") is None and q.check_name("doc-ipcc") is None + assert "reserved word" in q.check_name("window") + assert "'fast_onset'" in q.check_name("fast onset") + + +def test_dependency_order(): + order, cycles = q.dependency_order({"q1": {"q2"}, "q2": set(), "q3": {"q1"}}) + assert order == ["q2", "q1", "q3"] and cycles == [] + assert q.dependency_order({"a": {"b"}, "b": {"a"}})[1] == [["a", "b", "a"]] diff --git a/tests/test_spreadsheet.py b/tests/test_spreadsheet.py new file mode 100644 index 0000000..e1e6b00 --- /dev/null +++ b/tests/test_spreadsheet.py @@ -0,0 +1,55 @@ +import zipfile + +import pytest + +from ctrlfindr.framework import load_framework +from ctrlfindr.common import ValidationError +from ctrlfindr.spreadsheet import Dropdown, SheetSpec, read_workbook, write_workbook +from ctrlfindr.template import write_template + +ROWS = [["key", "value"], ["number", "1.10"], ["spaces", "a b"], ["lines", "one\ntwo"], [], ["after gap", "x"]] + + +@pytest.mark.parametrize("ext", ["ods", "xlsx"]) +def test_round_trip_keeps_text_and_row_numbers(tmp_path, ext): + path = write_workbook(tmp_path / f"t.{ext}", [SheetSpec("s", ROWS, dropdowns=[Dropdown(1, ("true", "false"))])]) + rows = read_workbook(path)["s"] + assert rows[:4] == ROWS[:4] + assert rows[4] == [] and rows[5] == ["after gap", "x"] # row 6 stays row 6 + + +def test_xlsx_numbers_and_booleans(tmp_path): + from openpyxl import Workbook + + wb = Workbook() + ws = wb.active + ws.title = "s" + ws.append(["a", 1, 2.5, True, None, 3.0]) + wb.save(tmp_path / "n.xlsx") + assert read_workbook(tmp_path / "n.xlsx")["s"] == [["a", "1", "2.5", "true", "", "3"]] + + +def test_ods_displayed_text_is_read(tmp_path): + # A numeric cell typed as 1.10 is stored as 1.1 but displayed as "1.10". + xml = ('' + '' + '1.10' + '') + with zipfile.ZipFile(tmp_path / "d.ods", "w") as zf: + zf.writestr("content.xml", xml) + assert read_workbook(tmp_path / "d.ods")["s"] == [["1.10"]] + + +@pytest.mark.parametrize("ext", ["ods", "xlsx"]) +def test_blank_template_only_lacks_content(tmp_path, ext): + path = write_template(tmp_path / f"t.{ext}") + with pytest.raises(ValidationError) as exc: + load_framework(path) + assert exc.value.problems == ["sheet 'taxonomy': no lists defined"] + + +def test_unsupported_file_type(tmp_path): + (tmp_path / "f.csv").write_text("x") + with pytest.raises(ValidationError, match="unsupported file type"): + load_framework(tmp_path / "f.csv") diff --git a/tests/test_validation.py b/tests/test_validation.py new file mode 100644 index 0000000..54c6a25 --- /dev/null +++ b/tests/test_validation.py @@ -0,0 +1,91 @@ +import pandas as pd +import pytest + +from ctrlfindr import analyze, config_from_dict, ingest +from ctrlfindr.common import CtrlFindrError, ValidationError +from ctrlfindr.validation import sample +from fwutil import make_framework + + +@pytest.fixture +def analyzed(tmp_path): + texts = ["Floods hit the town.", "Drought hurt farmers.", "Calm weather.", "The market rose.", + "Floods and storms again.", "Nothing to report.", "Storm damage everywhere.", "A quiet day."] + (tmp_path / "Text_files").mkdir() + (tmp_path / "Text_files" / "d.txt").write_text(" ".join(texts), encoding="utf-8") + fw = make_framework(tmp_path / "fw.ods", [("haz", "flood*"), ("haz", "storm*")], [("h", "haz")], [("1", "Hazard", "h")]) + cfg = config_from_dict({"paths": {"framework": str(fw)}}, base=tmp_path) + ingest.run(cfg, show_progress=False) + analyze.run(cfg, show_progress=False) + return cfg + + +def test_sample_is_stratified_and_blind(analyzed): + path = sample(analyzed, n=2, seed=3) + s = pd.read_csv(path, sep="\t", dtype=str, keep_default_na=False) + assert list(s.columns) == ["document", "sentence_id", "variable", "sentence", "human_code", "note"] + assert len(s) == 4 and set(s["variable"]) == {"1"} + assert sorted(s["sentence"].str.contains("Flood|Storm").tolist()) == [False, False, True, True] + assert (s["human_code"] == "").all() + + +def test_report_precision_and_estimated_recall(analyzed): + path = sample(analyzed, n=3, seed=1) + s = pd.read_csv(path, sep="\t", dtype=str, keep_default_na=False) + # the coder also counts drought as a hazard; CtrlFindr's list does not + s["human_code"] = s["sentence"].str.contains("Flood|Storm|Drought").astype(int).astype(str) + s.loc[s["sentence"].str.startswith("Storm damage"), "human_code"] = "0" # one false positive + analyzed.paths.validation.parent.mkdir(parents=True, exist_ok=True) + s.to_csv(analyzed.paths.validation, sep="\t", index=False) + analyze.run(analyzed, show_progress=False) + rep = pd.read_csv(analyzed.paths.output / "validation_report.tsv", sep="\t").iloc[0] + assert (rep.tp, rep.fp, rep.coded_flagged) == (2, 1, 3) + assert rep.precision == pytest.approx(2 / 3) + # FN rate in the unflagged sample (1 of 3) scaled to the 5 unflagged sentences in the corpus + tp_est, fn_est = (2 / 3) * 3, (1 / 3) * 5 + assert rep.recall_estimated == pytest.approx(tp_est / (tp_est + fn_est)) + dis = pd.read_csv(analyzed.paths.output / "validation_disagreements.tsv", sep="\t") + assert len(dis) == 2 + + +def test_invalid_codes_are_reported(analyzed): + analyzed.paths.validation.parent.mkdir(parents=True, exist_ok=True) + pd.DataFrame({"document": ["d"], "sentence_id": [1], "variable": ["1"], "human_code": ["maybe"]}).to_csv( + analyzed.paths.validation, sep="\t", index=False) + with pytest.raises(ValidationError, match="invalid codes"): + analyze.run(analyzed, show_progress=False) + + +def test_codes_follow_their_sentence_after_the_document_changes(tmp_path, caplog): + import logging + + (tmp_path / "Text_files").mkdir() + f = tmp_path / "Text_files" / "a.txt" + f.write_text("Floods hit. Calm day. Storms came. Quiet. Floods again.", encoding="utf-8") + fw = make_framework(tmp_path / "fw.ods", [("h", "flood*"), ("h", "storm*")], [("q", "h")], [("1", "H", "q")], + settings={"sentiment": False}) + cfg = config_from_dict({"paths": {"framework": str(fw)}}, base=tmp_path) + ingest.run(cfg, show_progress=False) + analyze.run(cfg, show_progress=False) + s = pd.read_csv(sample(cfg, n=2, seed=1), sep="\t", dtype=str, keep_default_na=False) + s["human_code"] = s["sentence"].str.contains("Flood|Storm").astype(int).astype(str) # agrees with CtrlFindr + cfg.paths.validation.parent.mkdir(parents=True, exist_ok=True) + s.to_csv(cfg.paths.validation, sep="\t", index=False) + # one sentence inserted at the start and one coded sentence deleted: all numbers shift + f.write_text("Introduction. Floods hit. Calm day. Storms came. Floods again.", encoding="utf-8") + ingest.run(cfg, show_progress=False) + with caplog.at_level(logging.INFO, logger="ctrlfindr"): + analyze.run(cfg, show_progress=False) + rep = pd.read_csv(cfg.paths.output / "validation_report.tsv", sep="\t").iloc[0] + assert rep.fp == 0 and rep.fn == 0 # moved sentences are found again, nothing is misaligned + assert rep.coded == len(s) - int(s["sentence"].eq("Quiet.").sum()) + assert "found under a new sentence number" in caplog.text + if s["sentence"].eq("Quiet.").any(): + assert "left out because the coded sentence is no longer in its document" in caplog.text + + +def test_sample_refuses_results_older_than_the_sentences(analyzed): + (analyzed.paths.text_files / "d.txt").write_text("Floods. Changed text.", encoding="utf-8") + ingest.run(analyzed, show_progress=False) + with pytest.raises(CtrlFindrError, match="run the analysis first"): + sample(analyzed, n=2) diff --git a/tools/compare_outputs.py b/tools/compare_outputs.py new file mode 100644 index 0000000..a7795a2 --- /dev/null +++ b/tools/compare_outputs.py @@ -0,0 +1,113 @@ +"""Compare two CtrlFindr output folders and print a Markdown report. + + python tools/compare_outputs.py OLD_OUTPUT_DIR NEW_OUTPUT_DIR [--rename old=new ...] + +Column names are compared case-insensitively. Document-level tables are compared cell by cell (numbers with a small tolerance). +Sentence-level tables usually have different rows when sentence splitting changed, so +they are compared through per-document sentence counts and per-column totals. +""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +import numpy as np +import pandas as pd + +DOC_TABLES = ( + "results_document", + "results_document_clipped", + "results_document_percentage_of_sentences", + "results_document_sentiment", +) + + +def read(path: Path) -> pd.DataFrame | None: + """Read a TSV; column names are lowercased, since 2.1 lowercased variable names.""" + return pd.read_csv(path, sep="\t").rename(columns=str.lower) if path.is_file() else None + + +def _equal(a: pd.Series, b: pd.Series) -> np.ndarray: + if pd.api.types.is_numeric_dtype(a) and pd.api.types.is_numeric_dtype(b): + return np.isclose(a.astype(float), b.astype(float), rtol=1e-9, atol=1e-12, equal_nan=True) + return (a.astype(str) == b.astype(str)).to_numpy() + + +def compare_document_table(old: pd.DataFrame, new: pd.DataFrame) -> list[str]: + lines = [] + added = [c for c in new.columns if c not in old.columns] + removed = [c for c in old.columns if c not in new.columns] + if added: + lines.append(f"- columns only in new: {', '.join(f'`{c}`' for c in added)}") + if removed: + lines.append(f"- columns only in old: {', '.join(f'`{c}`' for c in removed)}") + o = old.set_index("document") + n = new.set_index("document") + docs = [d for d in o.index if d in n.index] + if set(o.index) != set(n.index): + lines.append(f"- documents differ: only old {sorted(set(o.index) - set(n.index))}, only new {sorted(set(n.index) - set(o.index))}") + changed = [] + for col in [c for c in o.columns if c in n.columns]: + eq = _equal(o.loc[docs, col], n.loc[docs, col]) + if not eq.all(): + cells = [f"{d}: {o.at[d, col]:.4g} → {n.at[d, col]:.4g}" if isinstance(o.at[d, col], float) or isinstance(n.at[d, col], float) + else f"{d}: {o.at[d, col]} → {n.at[d, col]}" for d, e in zip(docs, eq) if not e] + changed.append((col, cells)) + if not changed: + lines.append("- all shared cells identical") + for col, cells in changed: + lines.append(f"- `{col}` ({len(cells)} of {len(docs)} documents): " + "; ".join(cells)) + return lines + + +def compare_sentence_table(old: pd.DataFrame, new: pd.DataFrame) -> list[str]: + lines = [f"- sentences: {len(old)} → {len(new)}"] + co, cn = old.groupby("document").size(), new.groupby("document").size() + lines.append("") + lines.append("| document | old sentences | new sentences |") + lines.append("|---|---:|---:|") + for d in sorted(set(co.index) | set(cn.index)): + lines.append(f"| {d} | {co.get(d, 0)} | {cn.get(d, 0)} |") + lines.append("") + bools = [c for c in old.columns if c in new.columns and old[c].dtype == bool] + lines.append("| column | old: sentences True | new: sentences True | change |") + lines.append("|---|---:|---:|---:|") + for c in bools: + a, b = int(old[c].sum()), int(new[c].sum()) + lines.append(f"| `{c}` | {a} | {b} | {b - a:+d} |") + return lines + + +def compare_dirs(old_dir: Path, new_dir: Path, renames: dict[str, str] | None = None) -> str: + renames = renames or {} + out = [f"Comparing `{old_dir}` (old) with `{new_dir}` (new).", ""] + for name in DOC_TABLES: + old, new = read(old_dir / f"{name}.tsv"), read(new_dir / f"{name}.tsv") + out.append(f"### {name}.tsv") + if old is None or new is None: + out.append(f"- missing in {'old' if old is None else 'new'}") + else: + out += compare_document_table(old.rename(columns=renames), new) + out.append("") + old, new = read(old_dir / "results_sentences.tsv"), read(new_dir / "results_sentences.tsv") + out.append("### results_sentences.tsv") + if old is None or new is None: + out.append(f"- missing in {'old' if old is None else 'new'}") + else: + out += compare_sentence_table(old.rename(columns=renames), new) + return "\n".join(out) + "\n" + + +def main() -> None: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("old", type=Path) + ap.add_argument("new", type=Path) + ap.add_argument("--rename", action="append", default=[], metavar="OLD=NEW", help="treat column OLD in the old output as NEW") + args = ap.parse_args() + renames = dict(r.split("=", 1) for r in args.rename) + print(compare_dirs(args.old, args.new, renames), end="") + + +if __name__ == "__main__": + main()