From 48ce3d23a1a30562e35ff93d53c9b0a261b678f7 Mon Sep 17 00:00:00 2001 From: he_sk Date: Sun, 27 Sep 2026 03:08:01 +0800 Subject: [PATCH] Add real DM8 failover regression procedure --- docs/README_zh.md | 2 +- docs/api-reference.md | 2 +- docs/test-results/2026-09-27-ha-routing.md | 48 ++++++++- scripts/verify_dm_failover.py | 116 +++++++++++++++++++++ 4 files changed, 161 insertions(+), 7 deletions(-) create mode 100644 scripts/verify_dm_failover.py diff --git a/docs/README_zh.md b/docs/README_zh.md index 9d2d44d..26edc9b 100644 --- a/docs/README_zh.md +++ b/docs/README_zh.md @@ -23,7 +23,7 @@ dmPython 是达梦数据库（DM8）的原生 Python 驱动程序，遵循 [Pyth - **Best-effort（尽力支持）**：尚未纳入 CI 覆盖的扩展使用场景。 - **Not guaranteed（不保证）**：生产 SLA 承诺、厂商认证兼容性与闭源组件支持协议。 - **连接安全**：`ssl_path` 支持使用客户端证书与私钥连接启用加密的 DM8。目录需包含 `ca-cert.pem`、`client-cert.pem`、`client-key.pem`；服务端证书没有 SAN 的旧版本还需提供与服务端完全一致的 `server-cert.pem`。指定 `ssl_path` 时，未协商加密的连接会报错。非空的 `ssl_pwd`、`ukey_name`、`ukey_pin` 仍不支持。 -- **主备路由**：读写分离模式 1 和 4 已在本机 DM8 主备环境、自动提交模式下验证。MPP 集群路由仍需对应环境回归。 +- **主备路由**：读写分离模式 1 和 4 已在本机 DM8 主备环境、自动提交模式下验证；通过双端点服务名建立的新连接也通过了自动接管后的回归。MPP 集群路由仍需对应环境回归。 ## 路线图与状态 diff --git a/docs/api-reference.md b/docs/api-reference.md index 6905f9e..9ce1a02 100644 --- a/docs/api-reference.md +++ b/docs/api-reference.md @@ -60,7 +60,7 @@ dmPython.connect( - `host` 与 `server` 互斥（只允许设置一个）。 - IPv6 地址使用方括号，例如 `server="[::1]"`；`dsn` 可写为 `"[::1]:5236"`。 -- `dmsvc_path` 指向包含 `dm_svc.conf` 的目录；连接时可把 `server` 设为配置文件中的服务名。 +- `dmsvc_path` 指向包含 `dm_svc.conf` 的目录；连接时可把 `server` 设为配置文件中的服务名。双端点服务名配置 `LOGIN_MODE=1` 后，已在本机 DM8 主备环境验证故障自动接管后的**新连接**会选择晋升的新主库；已有连接的自动恢复尚未验证。 - `mpp_login` 接受 `DSQL_MPP_LOGIN_GLOBAL` 或 `DSQL_MPP_LOGIN_LOCAL`；`rwseparate` 接受 `DSQL_RWSEPARATE_OFF`、`DSQL_RWSEPARATE_ON` 或 `DSQL_RWSEPARATE_ON2`，`rwseparate_percent` 范围为 0–100。这些选项在建连时传给底层驱动。读写分离模式 1 和 4 已在本机 DM8 主备环境中验证自动提交模式下的查询路由；MPP 只验证了单机 LOCAL 握手，集群路由尚待验证。 - `ssl_path` 指向含 `ca-cert.pem`、`client-cert.pem`、`client-key.pem` 的目录。服务端证书没有 SAN 时还需提供准确的 `server-cert.pem`，用于证书固定校验；有 SAN 的证书按 CA 链和主机名校验。设置后若服务端未协商加密，连接失败。非空的 `ssl_pwd`、`ukey_name`、`ukey_pin` 暂不支持。 - `user` 支持 `user/password@server:port[/schema][?catalog=...]` 形式。 diff --git a/docs/test-results/2026-09-27-ha-routing.md b/docs/test-results/2026-09-27-ha-routing.md index e014f63..5689999 100644 --- a/docs/test-results/2026-09-27-ha-routing.md +++ b/docs/test-results/2026-09-27-ha-routing.md @@ -39,8 +39,46 @@ the user's Orb network. The regular GitHub ARM real-database matrix still checks single-node behavior and build compatibility. MPP routing and UKey authentication remain unverified. -An additional stop-primary trial did not prove automatic takeover. The -monitor's confirmation mode remained active; after roughly 35 seconds, the -second node still reported `STANDBY, OPEN`. The replicated high-precision -value remained readable there. Automatic promotion and reconnect after -promotion need a separately configured failover regression. +## Automatic takeover and new connections + +An initial trial stopped the entire primary container. After roughly 35 +seconds, the second node still reported `STANDBY, OPEN`, and the monitor could +no longer resolve the stopped container's hostname. That trial did not prove +takeover. Confirmation mode was correctly enabled: [the DM monitor manual](https://eco.dameng.com/document/dm/zh-cn/pm/data-watch-monitor.html) +requires a confirmation monitor for automatic takeover, while [the watcher +configuration manual](https://eco.dameng.com/document/dm/zh-cn/pm/configuration-description) +defines `DW_MODE=AUTO` as automatic fault switching. + +The follow-up kept the primary container and its network identity running, +but stopped both `dmserver` and `dmwatcher` inside it. Before the failure, a +`DECIMAL(30,8)` value of `12345678901234567890.12345678` had reached the +standby. The monitor log recorded `start to auto takeover` and a successful +`ALTER DATABASE PRIMARY`; within the 20-second observation, the standby +reported `PRIMARY, OPEN`. A new ARM macOS Python 3.10 connection through a +two-endpoint `dm_svc.conf` service with `LOGIN_MODE=1` reached the promoted +node. It read the original value exactly and wrote and read +`98765432109876543210.87654321` exactly. + +The former primary then rejoined as `STANDBY, OPEN`. The same procedure was +repeated in the opposite direction with `scripts/verify_dm_failover.py`: +`prepare` checked the service route and standby replication; after stopping +the current primary's database and watcher, `verify` connected to the newly +promoted primary, checked both exact decimal values, and removed its test +table. Both phases passed. The script requires a unique `--table` name, +`DM_HA_USER`, `DM_HA_PASSWORD`, `DM_HA_SERVICE_NAME`, and +`DM_HA_SERVICE_PATH`; `prepare` also needs `DM_HA_STANDBY_HOST` and optional +`DM_HA_STANDBY_PORT`. Pass the standby instance name printed by `prepare` as +`--expected-new-primary` to `verify`. The operator triggers the fault between +the two phases. Existing open connections were not tested for transparent +reconnection; the verified behavior is a **new connection** after takeover. + +The service configuration used two reachable endpoints: + +```ini +DMPY_HA=(primary-host:5236,standby-host:5236) +[DMPY_HA] +LOGIN_MODE=1 +EP_SELECTION=1 +SWITCH_TIMES=1 +SWITCH_INTERVAL=0 +``` diff --git a/scripts/verify_dm_failover.py b/scripts/verify_dm_failover.py new file mode 100644 index 0000000..6a1d97f --- /dev/null +++ b/scripts/verify_dm_failover.py @@ -0,0 +1,116 @@ +"""Verify a DM primary/standby takeover with two separate invocations. + +Run ``prepare`` while both nodes are healthy, stop the primary database and +watcher, then run ``verify`` after the standby has been promoted. The service +name in dm_svc.conf must list both endpoints with LOGIN_MODE=1 (primary only). +""" + +from __future__ import annotations + +import argparse +import os +import re +import time +from decimal import Decimal + +import dmPython + + +BEFORE_AMOUNT = Decimal("12345678901234567890.12345678") +AFTER_AMOUNT = Decimal("98765432109876543210.87654321") + + +def connect(server: str, *, port: int | None = None, service: bool = False): + options = { + "user": os.environ["DM_HA_USER"], + "password": os.environ["DM_HA_PASSWORD"], + "server": server, + "autoCommit": dmPython.DSQL_AUTOCOMMIT_ON, + "login_timeout": 3000, + } + if service: + options["dmsvc_path"] = os.environ["DM_HA_SERVICE_PATH"] + else: + options["port"] = port + return dmPython.connect(**options) + + +def instance(conn): + with conn.cursor() as cur: + cur.execute("SELECT INSTANCE_NAME, MODE$, STATUS$ FROM V$INSTANCE") + return cur.fetchone() + + +def amount_at(conn, table: str, row_id: int): + with conn.cursor() as cur: + cur.execute(f"SELECT CAST(AMOUNT AS VARCHAR(80)) FROM {table} WHERE ID=?", (row_id,)) + row = cur.fetchone() + return None if row is None else Decimal(row[0]) + + +def prepare(table: str): + with connect(os.environ["DM_HA_SERVICE_NAME"], service=True) as primary, connect( + os.environ["DM_HA_STANDBY_HOST"], + port=int(os.environ.get("DM_HA_STANDBY_PORT", "5236")), + ) as standby: + primary_name, primary_mode, primary_status = instance(primary) + standby_name, standby_mode, standby_status = instance(standby) + assert (primary_mode, primary_status) == ("PRIMARY", "OPEN") + assert (standby_mode, standby_status) == ("STANDBY", "OPEN") + + with primary.cursor() as cur: + cur.execute(f"CREATE TABLE {table} (ID INT PRIMARY KEY, AMOUNT DECIMAL(30,8))") + cur.execute(f"INSERT INTO {table} VALUES (?, ?)", (1, BEFORE_AMOUNT)) + + deadline = time.monotonic() + 15 + while time.monotonic() < deadline: + try: + if amount_at(standby, table, 1) == BEFORE_AMOUNT: + print(f"prepared: {primary_name} PRIMARY, {standby_name} STANDBY") + print(f"table: {table}; standby has exact pre-failover decimal") + return + except dmPython.Error: + # The replicated table definition may not be visible yet. + pass + time.sleep(0.2) + raise AssertionError("standby did not receive exact value within 15 seconds") + + +def verify(table: str, expected_new_primary: str): + with connect(os.environ["DM_HA_SERVICE_NAME"], service=True) as primary: + actual = instance(primary) + assert actual == (expected_new_primary, "PRIMARY", "OPEN"), actual + assert amount_at(primary, table, 1) == BEFORE_AMOUNT + + if amount_at(primary, table, 2) is None: + with primary.cursor() as cur: + cur.execute(f"INSERT INTO {table} VALUES (?, ?)", (2, AFTER_AMOUNT)) + assert amount_at(primary, table, 2) == AFTER_AMOUNT + + with primary.cursor() as cur: + cur.execute(f"DROP TABLE {table}") + print(f"verified: connected to promoted primary {expected_new_primary}") + print("pre- and post-failover DECIMAL(30,8) values were exact") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("phase", choices=("prepare", "verify")) + parser.add_argument("--table", required=True, help="unique test table name") + parser.add_argument("--expected-new-primary", help="standby instance name from prepare") + args = parser.parse_args() + + table = args.table.upper() + if not re.fullmatch(r"[A-Z][A-Z0-9_]{0,63}", table): + parser.error("--table must be a simple unquoted SQL identifier") + if args.phase == "verify" and not args.expected_new_primary: + parser.error("--expected-new-primary is required for verify") + + if args.phase == "prepare": + prepare(table) + else: + verify(table, args.expected_new_primary) + + +if __name__ == "__main__": + main()