From 468ca7e976c4291f17e0e5cf87217be3cdd9a75c Mon Sep 17 00:00:00 2001 From: MobileMage Date: Thu, 1 Oct 2026 21:54:40 +0100 Subject: [PATCH 01/45] chore: create the spec repo with the standards and templates Signed-off-by: MobileMage --- .gitignore | 6 + CONTEXT.md | 137 ++++++++++ LICENSE | 202 ++++++++++++++ LICENSES/Apache-2.0.txt | 202 ++++++++++++++ NOTICE | 5 + README.md | 25 ++ REUSE.toml | 7 + VERSION | 1 + standards/CODING_STANDARDS.md | 268 +++++++++++++++++++ standards/languages/csharp.md | 105 ++++++++ standards/languages/dart.md | 115 ++++++++ standards/languages/go.md | 155 +++++++++++ standards/languages/kotlin-java.md | 116 ++++++++ standards/languages/php.md | 123 +++++++++ standards/languages/python.md | 133 +++++++++ standards/languages/swift.md | 109 ++++++++ standards/languages/typescript.md | 149 +++++++++++ templates/AGENTS.md | 30 +++ templates/CLAUDE.md | 1 + templates/CODING_STANDARDS.md | 8 + templates/CONTRIBUTING.md | 55 ++++ templates/ISSUE_TEMPLATE/bug_report.yml | 44 +++ templates/ISSUE_TEMPLATE/config.yml | 11 + templates/ISSUE_TEMPLATE/feature_request.yml | 29 ++ templates/NOTICE | 5 + templates/PULL_REQUEST_TEMPLATE.md | 17 ++ templates/README.md | 59 ++++ templates/SECURITY.md | 26 ++ 28 files changed, 2143 insertions(+) create mode 100644 .gitignore create mode 100644 CONTEXT.md create mode 100644 LICENSE create mode 100644 LICENSES/Apache-2.0.txt create mode 100644 NOTICE create mode 100644 README.md create mode 100644 REUSE.toml create mode 100644 VERSION create mode 100644 standards/CODING_STANDARDS.md create mode 100644 standards/languages/csharp.md create mode 100644 standards/languages/dart.md create mode 100644 standards/languages/go.md create mode 100644 standards/languages/kotlin-java.md create mode 100644 standards/languages/php.md create mode 100644 standards/languages/python.md create mode 100644 standards/languages/swift.md create mode 100644 standards/languages/typescript.md create mode 100644 templates/AGENTS.md create mode 100644 templates/CLAUDE.md create mode 100644 templates/CODING_STANDARDS.md create mode 100644 templates/CONTRIBUTING.md create mode 100644 templates/ISSUE_TEMPLATE/bug_report.yml create mode 100644 templates/ISSUE_TEMPLATE/config.yml create mode 100644 templates/ISSUE_TEMPLATE/feature_request.yml create mode 100644 templates/NOTICE create mode 100644 templates/PULL_REQUEST_TEMPLATE.md create mode 100644 templates/README.md create mode 100644 templates/SECURITY.md diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0eb32b5 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +observed-raw/ +__pycache__/ +.mypy_cache/ +.ruff_cache/ diff --git a/CONTEXT.md b/CONTEXT.md new file mode 100644 index 0000000..bb5bd5b --- /dev/null +++ b/CONTEXT.md @@ -0,0 +1,137 @@ +# Gatepost + +Unofficial, open-source developer tools for Nigeria's National Digital Postcode. This glossary fixes one name for each concept, in every language and every repo. + +## Language + +### The postcode + +**Postcode**: +The 11-character code that NIPOST gives to one addressable building or location. +_Avoid_: digital address, address code, zip code, postal code + +**Legacy postcode**: +An old 6-digit NIPOST postcode. It names an area, not a building. +_Avoid_: old code, zip + +**Segment**: +One of the five parts of a postcode: state, LGA, district, area and unit. +_Avoid_: part, field, component, chunk + +**State**: +The first segment. Two letters that name one of the 36 states or the Federal Capital Territory. +_Avoid_: region, province + +**LGA**: +The second segment. Two digits that name a local government area within a state. +_Avoid_: council, local government, municipality + +**District**: +The third segment. Three characters that name a district within an LGA. NIPOST uses districts for postal sorting. +_Avoid_: sorting code, sector + +**Area**: +The fourth segment. Two letters that name the smallest polygon in the system. An area holds at most 99 units. +_Avoid_: zone, block, neighbourhood + +**Unit**: +The fifth segment. Two digits that name one building or delivery point within an area. +_Avoid_: building number, house number, delivery unit + +**Partial postcode**: +A postcode that stops after the state, LGA, district or area segment. +_Avoid_: prefix, incomplete code + +**Precision**: +The most specific segment that a postcode or a result contains. +_Avoid_: level, granularity, resolution + +### Forms of a postcode + +**Compact form**: +The postcode with no separators, for example `EK01A03FK01`. +_Avoid_: raw, plain + +**Canonical form**: +The postcode with hyphens between segments, for example `EK-01-A03-FK-01`. +_Avoid_: formatted, standard form + +**Display form**: +The postcode with spaces between segments, for example `EK 01 A03 FK 01`. +_Avoid_: pretty, human form + +### NIPOST's API + +**Gateway**: +NIPOST's public API at `api.postcode.gov.ng`. +_Avoid_: server, backend, endpoint (for the whole API) + +**Lookup**: +A request to the gateway for the facts about one postcode. +_Avoid_: query, fetch, resolve + +**Lookup level**: +How much data a lookup returns, from 1 for validity only to 5 for point geometry. +_Avoid_: tier, access level, depth + +**Reverse geocode**: +A request for the nearest unit to a coordinate. +_Avoid_: reverse lookup, locate + +**Accuracy**: +The radius in metres within which a GPS fix is likely correct. +_Avoid_: precision, error margin + +**Secret key**: +An API key for servers. It starts with `nipost_test_` or `nipost_live_`. +_Avoid_: private key, API secret + +**Publishable key**: +An API key that can ship inside an app or a web page. It starts with `nipost_pk_`. +_Avoid_: public key, client key + +**Widget**: +NIPOST's own postcode picker. Gatepost does not build widgets. +_Avoid_: using this word for a Gatepost component + +### Gatepost + +**Spec**: +The `spec` repo, which holds the grammar, the vectors, the fixtures, the completed OpenAPI file and these standards. +_Avoid_: schema, contract (for the repo) + +**Vector**: +One shared test case in `spec/vectors/` that every SDK must pass. +_Avoid_: fixture, example, golden file + +**Fixture**: +Synthetic data that the mock server returns. +_Avoid_: vector, stub, sample + +**Mock server**: +The local server that imitates the gateway with fixtures. +_Avoid_: fake API, stub server, sandbox + +**SDK**: +The core and the client for one language. +_Avoid_: library (for the pair), wrapper, binding + +**Core**: +The part of an SDK that parses and formats postcodes without network access. +_Avoid_: utils, base, common + +**Client**: +The part of an SDK that calls the gateway. +_Avoid_: API wrapper, service, fetcher + +**Field**: +A Gatepost UI component in which a user enters or picks a postcode. +_Avoid_: input, widget, picker + +**Confirm**: +To show the user what a postcode points to before the app stores it. +_Avoid_: verify, validate (for this step) + +**Tell**: +A habit that makes code look machine-written, such as very long lines. +_Avoid_: smell (smells are design problems), anti-pattern diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSES/Apache-2.0.txt b/LICENSES/Apache-2.0.txt new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSES/Apache-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/NOTICE b/NOTICE new file mode 100644 index 0000000..bc94510 --- /dev/null +++ b/NOTICE @@ -0,0 +1,5 @@ +Gatepost +Copyright 2026 The Gatepost authors + +This product is unofficial. The Nigerian Postal Service (NIPOST) did not make it and does not endorse it. +Gatepost uses the name NIPOST only to describe compatibility with NIPOST's National Digital Postcode. diff --git a/README.md b/README.md new file mode 100644 index 0000000..fb23914 --- /dev/null +++ b/README.md @@ -0,0 +1,25 @@ +# Gatepost spec + +The shared contract for every Gatepost SDK: the postcode grammar, the data files, the test vectors, the coding standards and the repo templates. + +> Unofficial. Not made or endorsed by NIPOST. + +## Contents + +| Path | Holds | +|---|---| +| `grammar.md` | how every SDK reads and writes postcodes | +| `data/` | state codes, GPS accuracy limits and the postcode format | +| `vectors/` | shared test cases that every SDK must pass | +| `standards/` | the coding standards and one file for each language | +| `CONTEXT.md` | the glossary | +| `templates/` | files for new repos | +| `scripts/` | `check-tells` and the vector builder | + +## Check + +Run `make check`. It needs Python 3.11 or later and `uv`. + +## Licence + +Apache-2.0. See `LICENSE` and `NOTICE`. diff --git a/REUSE.toml b/REUSE.toml new file mode 100644 index 0000000..a074a60 --- /dev/null +++ b/REUSE.toml @@ -0,0 +1,7 @@ +version = 1 + +[[annotations]] +path = ["**/*.md", "**/*.json", "**/*.yml", "VERSION", "NOTICE", "templates/NOTICE"] +precedence = "aggregate" +SPDX-FileCopyrightText = "2026 The Gatepost authors" +SPDX-License-Identifier = "Apache-2.0" diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..6e8bf73 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.1.0 diff --git a/standards/CODING_STANDARDS.md b/standards/CODING_STANDARDS.md new file mode 100644 index 0000000..f6211ef --- /dev/null +++ b/standards/CODING_STANDARDS.md @@ -0,0 +1,268 @@ +# Gatepost coding standards + +Version 1.0, 1 Oct 2026. + +These standards apply to every Gatepost repo, in every language. Each repo also follows its language file in `standards/languages/`. Together, the two files are the standard for that repo. + +## How to use this document + +- Each rule has an ID, for example `API-3`. Reviews cite the ID. +- A **MUST** rule blocks a merge. +- A **SHOULD** rule can be skipped only with a written reason in the pull request. +- A rule marked **(tool)** is enforced by CI. Reviewers do not check it by hand. +- A language file can change a SHOULD rule for its language. It cannot change a MUST rule. +- The spec's interface section, vectors and contract scenarios rank above this document. This document ranks above the language files. If two of them disagree, the higher one wins. Then fix the lower one. +- To change a rule, open a pull request on the `spec` repo. Increase the version at the top of this file. + +## Principles + +The rules come from these seven principles. If no rule covers a case, apply the principles. + +1. **The spec is the contract.** The spec's interfaces and test vectors define correct behaviour in every language. +2. **One meaning, one name.** Use the terms in `CONTEXT.md`. Use each term the same way in code, tests, docs and errors. +3. **Deep modules.** Put a lot of behaviour behind a small interface. Delete layers that only pass calls through. +4. **Same behaviour, native feel.** Every SDK behaves the same. Every SDK reads like a well-made library in its own language. +5. **Boring is good.** Prefer the standard library, plain data and well-known tools. +6. **Private by default.** Send nothing the caller did not ask for. Keep nothing longer than necessary. +7. **Respect NIPOST's rules.** Follow the terms and the acceptable use policy that the spec lists. + +## Public interface + +- **API-1 MUST.** Every public symbol maps to a concept in the spec's interface section. A language file lists the idiomatic additions that its language needs, such as options types, async variants and callback types. To add, rename or remove a concept, change the spec first. To add an idiomatic addition, change the language file first. +- **API-2 MUST.** Names and call shapes follow the cross-language map below. +- **API-3 MUST.** Parse, don't validate. A function that reads a postcode returns a typed result, not a boolean. A boolean helper can exist only as a thin wrapper over `parse`. +- **API-4 MUST.** Expected failures are values. `parse` returns its errors. Only I/O failures, API failures and programmer errors raise or throw. +- **API-5 MUST.** Error codes are stable `snake_case` strings, identical in every language. A new code is a minor change. A removed or renamed code is a breaking change. +- **API-6 MUST.** Public value types are immutable. +- **API-7 MUST.** Options use named parameters or an options object. Do not add a positional boolean parameter, unless the language lets the caller name the argument. +- **API-8 MUST.** Defaults are safe. The cache is off. A call retries at most 2 times. The timeout is 8 seconds. A client sends at most 4 requests at a time. +- **API-9 MUST.** There is no global state. Two clients with different options can run in the same process. +- **API-10 MUST.** The caller can inject the HTTP transport. Tests replace the clock with the language's standard test tools, such as fake timers, or with an internal seam. The clock is not a public option. +- **API-11 MUST.** Each async operation supports cancellation in the language's standard way. A cancelled call ends with the platform's own cancellation error, not with a Gatepost error. +- **API-12 SHOULD.** Keep the interface small. Before you add a public symbol, apply the deletion test from the `codebase-design` skill. +- **API-13 MUST.** Releases follow semantic versioning. Mark a symbol deprecated for at least one minor release before you remove it. +- **API-14 MUST.** A client ignores unknown fields in a response. It maps an unknown value of a known field to a defined fallback. NIPOST's API is at version 0.1, so new fields will appear. A contract scenario tests this. + +### Cross-language map + +The behaviour is the same in every SDK. The shape follows each language. + +| Language | Parse | Look up a postcode | Error type and code | +|---|---|---|---| +| TypeScript | `parse(input, { allowPartial: true })` returns `ParseResult` | `await client.lookup(code, { level: 2 })` | `PostcodeError`, `code === 'rate_limited'` | +| PHP | `Postcode::parse($input, allowPartial: true)` | `$client->lookup($code, level: 2)` | `PostcodeException`, `errorCode() === ErrorCode::RateLimited` | +| Python | `parse(text, allow_partial=True)` | `client.lookup(code, level=2)`, also on `AsyncPostcodeClient` | `PostcodeError`, `code is ErrorCode.RATE_LIMITED` | +| Go | `postcode.Parse(s)` and `postcode.ParsePartial(s)` return `(Postcode, error)` | `client.Lookup(ctx, code, postcode.WithLevel(2))` | `*postcode.Error`, `Code == postcode.CodeRateLimited` | +| Kotlin and Java | `Postcode.parse(input, allowPartial = true)` returns `ParseResult` | `client.lookup(code, level = 2)`, also `lookupAsync` for Java | `PostcodeException`, `code == ErrorCode.RATE_LIMITED` | +| C# | `Postcode.Parse(input, allowPartial: true)` returns `ParseResult` | `await client.LookupAsync(code, new LookupOptions { Level = 2 }, ct)` | `PostcodeException`, `Code == ErrorCode.RateLimited` | +| Dart | `Postcode.parse(input, allowPartial: true)` returns `ParseResult` | `await client.lookup(code, level: 2)` | `PostcodeException`, `code == ErrorCode.rateLimited` | +| Swift | `Postcode.parse(_:allowPartial:)` returns `Result` | `try await client.lookup(code, level: 2)` | `PostcodeError`, `code == .rateLimited` | + +In every language, the error code's wire value is the same string, for example `rate_limited`. + +## Code structure + +- **CS-1 MUST.** A module has one reason to change. +- **CS-2 MUST.** The core has no I/O, no clock and no randomness. +- **CS-3 SHOULD.** A function fits on one screen, about 40 lines. A file stays under about 300 lines. +- **CS-4 MUST.** Values from the spec, such as the GPS accuracy limits and the state codes, come from the spec's data files. Each language keeps them in one place. The vectors test each value, so a wrong copy fails CI. +- **CS-5 MUST.** There is no dead code and no commented-out code. +- **CS-6 MUST.** Each TODO comment links an issue: `TODO(#123): ...` **(tool)** +- **CS-7 MUST.** Comments explain why. The code shows what. +- **CS-8 MUST.** Add no option, hook or abstraction that the spec does not need. +- **CS-9 SHOULD.** Prefer plain data and functions. Use a class when the language expects one, or when it holds configuration, like the client. + +## Agent tells + +Coding agents repeat some habits so often that readers spot them at once. Each rule below states the target. The "Tell" column names the habit that the rule prevents. These rules apply to people and to agents. + +| ID | Rule | Tell | Check | +|---|---|---|---| +| TELL-1 MUST | Keep each line of code at 100 characters or fewer. Lines with a URL, generated files and data files are exempt. | Long one-line chains, conditions, signatures and strings | tool | +| TELL-2 MUST | Keep nesting at 3 levels or fewer, and cyclomatic complexity at 10 or lower. Return early. | Arrow-shaped code, an `if` inside an `if` inside a loop | tool where available | +| TELL-3 MUST | Give a function 4 positional parameters or fewer. Named, keyword-only and defaulted parameters do not count. Group the rest in an options object or named arguments. | Long lists of positional parameters | tool where available | +| TELL-4 MUST | Write a comment only to give a reason, a constraint or a reference. | Comments that repeat the next line, "Step 1" comments, banner comments | review | +| TELL-5 MUST | Make each doc comment add facts that the name and the types do not give. | "Gets the name. Returns the name." | review | +| TELL-6 MUST | Name each value after its meaning in the domain. | `data`, `result`, `obj`, `temp`, `value2`, and type words inside names, such as `postcodeString` | review | +| TELL-7 MUST | Put each helper in the module that owns its concept. | Files named `utils`, `helpers`, `common` or `misc`, and classes named `Manager`, `Helper` or `Processor` | tool for file names, review for classes | +| TELL-8 MUST | Search the repo before you write a helper. Reuse what exists. | The same helper written twice | review | +| TELL-9 MUST | Catch an error only where the code can handle it. Handle it fully there. | `catch` blocks that log and rethrow, catch-all blocks, empty `catch` blocks | tool where available | +| TELL-10 MUST | Trust the type system. Check only for states that can happen. | Null checks on values that cannot be null, checks for impossible states | tool where available | +| TELL-11 MUST | Let errors reach the caller. Use a default value only where the spec defines one. | Fallbacks such as `?? ''` or `catch { return [] }` that hide failures | review | +| TELL-12 MUST | Commit only finished code. | `TODO: implement`, "not implemented" errors, stub functions, placeholder data | tool for TODOs, review for the rest | +| TELL-13 MUST | Remove debug output before you commit. | `console.log`, `print`, `var_dump`, `dd()` | tool | +| TELL-14 MUST | Use ASCII in code, comments, logs and commit messages. Write other characters as escape sequences, such as `\u00A0`. Message catalogues and docs prose can use any script, without emojis. | Emojis, invisible spaces and curly quotes in code | tool | +| TELL-15 MUST | Change only the lines that the task needs. | Diffs that reformat, rename or move unrelated code | review | +| TELL-16 MUST | Mark a function `async` only when it awaits. Send every gateway call through the client, which limits how many run at once. | `async` without `await`, hundreds of gateway calls started at once | tool for `async`, review for calls | +| TELL-17 MUST | Give each ternary one condition. Use `if` or `switch` for more. | Nested ternaries | tool where available | +| TELL-18 MUST | Write a commit subject of 72 characters or fewer, in the imperative mood. Explain why in the body, in prose. | "Update files", bullet lists of every change, emoji commits | tool | +| TELL-19 MUST | Make each test assert a behaviour that a user would notice. | Tests without assertions, snapshot tests of logic, tests of mocks, names like "should work" | review | +| TELL-20 MUST | Add a dependency only when it removes real work. | A package for one small function | review | + +### The `check-tells` script + +`spec/scripts/check-tells` runs in the CI of every repo. It covers the tool rules that no language tool covers: + +- **TELL-1.** It checks line length in source and config files, and it counts a tab as 4 columns. It skips lines with a URL, files marked as generated, lockfiles, `spec/vectors/` and Markdown prose. +- **TELL-7.** It rejects files named `utils`, `helpers`, `common` or `misc`, with any extension. +- **TELL-12 and CS-6.** It rejects each `TODO` without the form `TODO(#123)`. +- **TELL-13.** It rejects the debug calls that the language file lists. +- **TELL-14.** It rejects emojis everywhere, including Markdown. It rejects other non-ASCII characters in source files, config files and commit messages. Message catalogues, docs prose and the sign-off lines of commit messages are exempt from the non-ASCII check, because contributors' names can need any script. +- **TELL-18.** With `--commit-msg `, it checks the length of a commit subject. Repos without commitlint use this mode. + +A line can skip one check with a comment that names the rule and gives a reason, for example `check-tells: allow TELL-1 because the regex cannot be split`. Reviewers check each skip. + +## Dependencies + +- **DEP-1 MUST.** A `core` package has no runtime dependencies. +- **DEP-2 MUST.** A client uses the platform's standard HTTP and JSON libraries, or the ones that its language file names. +- **DEP-3 MUST.** A new runtime dependency needs an ADR in `docs/adr/`. +- **DEP-4 MUST.** Each dependency uses a permissive licence, such as MIT, BSD, ISC or Apache-2.0. **(tool)** +- **DEP-5 MUST.** Lockfiles are in the repo. Renovate opens update pull requests each week. **(tool)** + +## Runtime versions + +- **VER-1 MUST.** Each language file states its runtime floors and the end-of-life date of each floor. A floor that is past its end of life needs an ADR that explains the trade-off. +- **VER-2 MUST.** Raise a floor only in a minor release. Announce the change in the changelog of the release before it. + +## Errors and logging + +- **ERR-1 MUST.** Each HTTP status maps to exactly one error code. The client sub-plan holds the map. +- **ERR-2 MUST.** A library does not log. It returns or raises errors. A caller can attach a debug hook. +- **ERR-3 MUST.** No error, log or exception message contains an API key, a NIN, an email address, or a full postcode next to personal data. Use `redact()` for postcodes. +- **ERR-4 MUST.** An error message says what happened and what to do, in plain English. Example: "The postcode has 10 characters. A postcode has 11 characters." + +## Security and privacy + +- **SEC-1 MUST.** Client-side code refuses secret keys, which start with `nipost_test_` or `nipost_live_`. +- **SEC-2 MUST.** There is no telemetry. An SDK sends only the requests that the caller makes. +- **SEC-3 MUST.** The cache is off by default. When on, it has a time limit and a clear command. +- **SEC-4 MUST.** Call only the documented gateway endpoints. +- **SEC-5 MUST.** CI scans each push for secrets. **(tool)** +- **SEC-6 MUST.** Each GitHub Action is pinned to a full commit SHA. **(tool)** +- **SEC-7 MUST.** Releases come only from CI. They carry provenance or a signature where the registry supports it. +- **SEC-8 MUST.** Each repo has `SECURITY.md`. A reporter gets a reply within 3 working days. + +## Tests + +- **T-1 MUST.** Write the failing test first for each change in behaviour. +- **T-2 MUST.** Every SDK runs every vector in `spec/vectors/` for the spec version that it implements. **(tool)** +- **T-3 MUST.** Every client passes every contract scenario in `spec/contract/` against the mock server in CI. The scenarios fix the shared client behaviour: error codes, retries, waits, caching and the limit on parallel requests. **(tool)** +- **T-4 MUST.** A test can fail. It asserts a specific value, and it does not repeat the logic under test. +- **T-5 MUST.** Unit tests use no network, no real clock and no sleep. +- **T-6 MUST.** A test name states the behaviour in domain words, for example "rejects a unit of 00". +- **T-7 MUST.** Coverage floors are 95 % of branches for core, 90 % for clients and 80 % for UI. UI coverage includes browser tests. **(tool)** +- **T-8 SHOULD.** Core runs mutation tests each week, with a target score of 85 %. **(tool)** +- **T-9 SHOULD.** The parser has property-based or fuzz tests, where the language has a standard tool for them. +- **T-10 MUST.** UI tests include keyboard-only use and an automated accessibility check. + +## Documentation + +- **DOC-1 MUST.** Each public symbol has a doc comment. It gives a one-line summary, each parameter, each error and one example. +- **DOC-2 MUST.** Each README follows `templates/README.md`. +- **DOC-3 MUST.** Each code example in a README or on the docs site runs in CI. +- **DOC-4 MUST.** Prose uses plain English: short sentences, active voice, one idea per sentence and British spelling. Run the `asd-ste100` and `unslop` skills on new prose. +- **DOC-5 MUST.** Each package has a changelog in the Keep a Changelog format. Change files generate it. +- **DOC-6 SHOULD.** Record a decision as an ADR when it is hard to reverse, surprising, and the result of a real trade-off. The `domain-modeling` skill gives the format. +- **DOC-7 MUST.** A new domain term goes into `CONTEXT.md` before it goes into code. + +## Git, pull requests and releases + +- **GIT-1 MUST.** Commits follow Conventional Commits. The scope is the package name. **(tool)** +- **GIT-2 MUST.** Each commit has a DCO sign-off: `git commit -s`. **(tool)** +- **GIT-3 MUST.** Commits do not carry co-author lines for AI tools. The person who opens the pull request owns the change. +- **GIT-4 MUST.** Nobody force-pushes to `main` or to a shared branch. +- **GIT-5 SHOULD.** A pull request changes at most about 400 lines. Tests, vectors and generated files do not count. +- **GIT-6 MUST.** A pull request needs one approval and green CI. Merges use squash. +- **GIT-7 MUST.** A pull request that changes the public interface also updates the API report, the docs and a change file. **(tool)** +- **GIT-8 MUST.** Each package exposes a `SPEC_VERSION` constant. It names the spec version that the package implements. + +## UI + +These rules apply to the field, the docs site and every app screen. + +- **UI-1 MUST.** Meet WCAG 2.2 AA. +- **UI-2 MUST.** Each string comes from a message catalogue. Do not build sentences by joining strings. A UI shows the catalogue string for an error code, not the library's error message. +- **UI-3 MUST.** Colours, radii and fonts come from theme tokens. Components contain no fixed colours. +- **UI-4 MUST.** Respect the user's reduced-motion setting and text size. +- **UI-5 SHOULD.** Use the `frontend-design` skill for each new screen. + +## Performance + +- **PERF-1 MUST.** Each package stays inside its size limit. **(tool)** +- **PERF-2 MUST.** Importing a package does no work: no network calls, no timers and no global registration. The field's custom element is the one exception. +- **PERF-3 SHOULD.** Web packages are tree-shakeable. + +## Licence and notices + + +- **LIC-1 MUST.** Each file has licence information that `reuse lint` accepts. Where the file format allows comments, the file starts with `SPDX-License-Identifier: Apache-2.0`. Other files, such as JSON, get an entry in `REUSE.toml`. **(tool)** + +- **LIC-2 MUST.** Each repo has `LICENSE` and `NOTICE`. +- **LIC-3 MUST.** Each README and each package description says "Unofficial. Not made or endorsed by NIPOST." + +## Cross-language parity + +- **PAR-1 MUST.** An SDK that claims a spec version passes every vector and every contract scenario of that version. +- **PAR-2 MUST.** Error codes, defaults and option meanings are identical in every SDK. +- **PAR-3 MUST.** The docs site has a parity table. It shows each SDK, its version, its spec version and its status. + +## Gates + +### New package gate + +A new package can publish its first release only when all of these are true: + +- **NPG-1.** The repo has the community files from `templates/`. +- **NPG-2.** The language file's tools run in CI, and they pass. +- **NPG-3.** All vectors pass. +- **NPG-4.** The client passes every contract scenario against the mock server. +- **NPG-5.** The repo holds the API report or API dump. +- **NPG-6.** The README quickstart runs in CI. +- **NPG-7.** The coverage floors pass. +- **NPG-8.** A second person reviewed the whole package with the `code-review` skill. Both axes, Standards and Spec, are clean. + +### Release gate + +Each release meets all of these: + +- **REL-1.** CI is green on the release commit. +- **REL-2.** The change files describe each user-visible change. +- **REL-3.** Each breaking change has a migration note. +- **REL-4.** The release runs from CI, with provenance or a signature. + +### 1.0 gate + +A package can release 1.0 only when all of these are true: + +- **V1-1.** The spec is at 1.0. +- **V1-2.** The public interface did not change for 4 weeks. +- **V1-3.** At least two production users run the package. +- **V1-4.** The docs cover every public symbol and every error code. + +## Review checklist + +Tools check the **(tool)** rules. Reviewers check these by hand: + +1. The interface: API-1, API-3, API-7 and API-12. +2. The structure: CS-1, CS-7 and CS-8. +3. The errors: ERR-3 and ERR-4. +4. The tests: T-4 and T-6. +5. The docs: DOC-1, DOC-4 and DOC-7. +6. The tells that need a person: TELL-4, TELL-5, TELL-6, TELL-8, TELL-11, TELL-15, TELL-19 and TELL-20. +7. The smell baseline in the `code-review` skill. Treat each smell as a judgement call. + +## Skills for each stage + +| Stage | Skill | +|---|---| +| Design an interface | `codebase-design`, then `critic` on the result | +| Name a concept or record a decision | `domain-modeling` | +| Write code | `test-driven-development` | +| Write tests | `authoring-test-cases` | +| Run a plan | `subagent-driven-development` | +| Claim that work is done | `verification-before-completion` | +| Review a branch | `code-review` for both axes, `simplify` for cleanups, `security-review` for keys and input handling | +| Write prose | `asd-ste100`, then `unslop` | +| Design a screen | `frontend-design` | +| Write agent docs or skills | `writing-for-agents` | diff --git a/standards/languages/csharp.md b/standards/languages/csharp.md new file mode 100644 index 0000000..8f70de7 --- /dev/null +++ b/standards/languages/csharp.md @@ -0,0 +1,105 @@ +# C# standards + +Version 1.0, 1 Oct 2026. + +This file and `standards/CODING_STANDARDS.md` together are the standard for C# code. They apply to the `dotnet` repo. The repo publishes one package, `Gatepost.Postcode`, with the core types and the client. + +## Versions + +- The package targets `netstandard2.0` and `net10.0`. +- CI tests `net10.0` on Linux. CI also tests .NET Framework 4.8 on a Windows runner. +- .NET 10 has support until 14 Nov 2028. .NET Framework 4.8 has support for as long as the Windows version that it runs on. `netstandard2.0` is a specification, not a runtime, so it has no end-of-life date. +- On `netstandard2.0`, the package depends on `System.Text.Json`. An ADR records this under DEP-3. On `net10.0`, the package has no dependencies. +- The C# language version is `latest`. On `netstandard2.0`, `init` accessors need an internal `IsExternalInit` polyfill. + +## Tools + +`make check` runs every tool below. CI runs `make check` on each push and each pull request. + +| Tool | Job | Settings | +|---|---|---| +| `dotnet format --verify-no-changes` | format | `.editorconfig` | +| Roslyn analyzers | lint | `AnalysisLevel` `latest-recommended`, `TreatWarningsAsErrors`, `Nullable` `enable` | +| `GenerateDocumentationFile` | doc check | a missing XML doc comment fails the build | +| Microsoft.CodeAnalysis.PublicApiAnalyzers | API tracking | `PublicAPI.Shipped.txt` and `PublicAPI.Unshipped.txt` | +| Microsoft.CodeAnalysis.BannedApiAnalyzers | banned calls | `BannedSymbols.txt` | +| Microsoft.VisualStudio.Threading.Analyzers | async rules | the `Async` suffix and no `async void` | +| xUnit | unit, vector and contract tests | | +| coverlet | coverage | floors from T-7 | +| CsCheck | property tests | `Parse`, `Normalize` and the hierarchy methods | +| Stryker.NET | mutation tests | core types, weekly scheduled job | +| SourceLink, deterministic builds and symbol packages | debugging support | `ContinuousIntegrationBuild` in CI, `.snupkg` files | +| changie | change files and changelog | one change file per user-visible change | +| `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in source, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | +| REUSE | licence headers | `reuse lint` | +| gitleaks | secret scan | every push | + +### Agent-tell checks + +| Rule | Check in C# | +|---|---| +| TELL-1 | `max_line_length = 100` in `.editorconfig`. `dotnet format` does not enforce it, so `check-tells` does. | +| TELL-2 | CA1502 at error severity, with `CA1502: 10` in `CodeMetricsConfig.txt`. Nesting depth: review. | +| TELL-3 | review. Parameters with default values do not count. Options classes carry the rest, for example `LookupOptions`. | +| TELL-13 | BannedApiAnalyzers and `check-tells`. They ban `Console.Write`, `Console.WriteLine`, `Debug.WriteLine` and `Trace.WriteLine`. | +| TELL-14 | `check-tells` | + +## Rules + +- **NET-1 MUST.** Nullable reference types are on. The build treats warnings as errors. **(tool)** +- **NET-2 MUST.** Each public type and member has an XML doc comment with ``, ``, `` and ``. The build checks that the comment exists. Reviewers check its content. +- **NET-3 MUST.** Async methods end in `Async`. They take a `CancellationToken` as the last parameter. They call `ConfigureAwait(false)`. **(tool)** +- **NET-4 MUST.** When the caller's token is cancelled, the method throws `OperationCanceledException`. When the client's own timeout ends the call, the method throws `PostcodeException` with `ErrorCode.Timeout`. The client checks the caller's token to tell the two apart. +- **NET-5 MUST.** The caller can pass an `HttpClient`. The client never disposes an `HttpClient` that it did not create. +- **NET-6 MUST.** Validated value types are sealed classes with get-only properties. Do not use records for them, because `with` can bypass validation. +- **NET-7 MUST.** Options classes, such as `LookupOptions`, use `init` accessors. +- **NET-8 MUST.** `ErrorCode` is an enum. One internal table maps each member to its wire value, for example `RateLimited` to `rate_limited`. +- **NET-9 MUST.** Do not add idiom aliases, such as `TryParse`, unless the spec adds them first. This follows API-1. +- **NET-10 MUST.** Durations use `TimeSpan`. +- **NET-11 MUST.** A change to the public API updates `PublicAPI.Unshipped.txt`. A release moves the entries to `PublicAPI.Shipped.txt`. **(tool)** +- **NET-12 MUST.** Each `#pragma warning disable` and each `SuppressMessage` attribute gives a justification. +- **NET-13 MUST.** Code writes each non-ASCII or invisible character as an escape sequence, for example `"\u00A0"` and `"\u200B"`. **(tool)** +- **NET-14 SHOULD.** Prefer sealed classes and static methods. Do not add an interface that has one implementation. +- **NET-15 MUST.** Tests replace the clock through an internal seam, with `InternalsVisibleTo`, until the spec defines a clock option for API-10. +- **NET-16 MUST.** The client decodes JSON with `System.Text.Json`, which ignores unknown members by default. Do not set `UnmappedMemberHandling.Disallow`. A custom converter maps an unknown enum string to the fallback that the spec defines. This follows API-14. +- **NET-17 MUST.** A `SemaphoreSlim` with 4 slots limits each client to 4 requests at a time, as API-8 requires. A waiting call still ends when its `CancellationToken` is cancelled. + +## Names + +| Element | Style | Example | +|---|---|---| +| type, method, property | PascalCase | `PostcodeClient`, `LookupAsync` | +| parameter, local variable | camelCase | `allowPartial` | +| private field | underscore and camelCase | `_httpClient` | +| enum member | PascalCase | `ErrorCode.RateLimited` | +| constant | PascalCase | `Postcode.SpecVersion` | +| error code value | snake_case string | `"rate_limited"` | +| namespace | PascalCase | `Gatepost.Postcode` | +| file | one type per file, named after the type | `PostcodeClient.cs` | + +## Package layout + +``` +src/Gatepost.Postcode/ + Gatepost.Postcode.csproj + Postcode.cs + ParseResult.cs + PostcodeClient.cs + PublicAPI.Shipped.txt + PublicAPI.Unshipped.txt + BannedSymbols.txt + CodeMetricsConfig.txt +tests/Gatepost.Postcode.Tests/ unit, vector and contract tests +Directory.Build.props shared build and analyzer settings +.editorconfig +.changes/ changie fragments +Makefile +README.md from templates/README.md +CHANGELOG.md generated by changie +``` + +## Publishing + +- CI packs the package with `dotnet pack` and publishes it to NuGet.org with trusted publishing. +- Each package carries SourceLink data and a symbol package. +- A signed tag `vX.Y.Z` starts each release. diff --git a/standards/languages/dart.md b/standards/languages/dart.md new file mode 100644 index 0000000..7e6c44c --- /dev/null +++ b/standards/languages/dart.md @@ -0,0 +1,115 @@ +# Dart standards + +Version 1.0, 1 Oct 2026. + +This file and `standards/CODING_STANDARDS.md` together are the standard for Dart and Flutter code. They apply to the `dart` repo. The repo publishes two packages: `gatepost_postcode` is pure Dart and holds the core and the client. `gatepost_postcode_field` is the Flutter widget. + +## Versions + +- Dart 3.7 or later. `pubspec.yaml` sets `sdk: ^3.7.0`, because the formatter reads `page_width` only from language version 3.7. +- The widget package needs a Flutter release that ships Dart 3.7 or later. +- Dart and Flutter support only their newest stable release. No floor has a published end-of-life date. +- package:http is the one runtime dependency of `gatepost_postcode`. The core code does not import it. + +## Tools + +`make check` runs every tool below. CI runs `make check` on each push and each pull request. + +| Tool | Job | Settings | +|---|---|---| +| `dart format --set-exit-if-changed` | format | `page_width: 100` in `analysis_options.yaml` | +| `dart analyze --fatal-infos` | lint | very_good_analysis and the settings below | +| `flutter analyze --fatal-infos` | lint for the widget package | the same settings | +| `dart test` with the coverage package | unit, vector and contract tests | coverage floors from T-7 | +| `flutter test --coverage` | widget tests | semantics, text scaling and reduced motion | +| dart_apitool | API check against the last release | `diff`, with `--old` set to the last release and `--new` set to the working copy | +| pana | pub.dev score | 160 of 160 points | +| changie | change files and changelog | one change file per user-visible change | +| `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in source, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | +| REUSE | licence headers | `reuse lint` | +| gitleaks | secret scan | every push | + +### `analysis_options.yaml` + +```yaml +include: package:very_good_analysis/analysis_options.yaml + +formatter: + page_width: 100 + +linter: + rules: + lines_longer_than_80_chars: false + document_ignores: true +``` + +very_good_analysis turns on `lines_longer_than_80_chars`. This file turns it off, because TELL-1 sets the limit at 100. + +### Agent-tell checks + +| Rule | Check in Dart | +|---|---| +| TELL-1 | the formatter's `page_width: 100`, and `check-tells` for lines that the formatter cannot split, such as long strings | +| TELL-2 | review. The Dart analyzer has no complexity lint and no nesting lint. | +| TELL-3 | review. Named parameters do not count. The `avoid_positional_boolean_parameters` lint is on. | +| TELL-13 | the `avoid_print` lint, and `check-tells`. They ban `print` and `debugPrint`. | +| TELL-14 | `check-tells` | + +## Rules + +T-8 and T-9 do not apply to Dart. The shared vectors cover the parser. + +- **DART-1 MUST.** Results are sealed classes. `ParseResult` has one subclass for success and one for failure. Callers use an exhaustive `switch`. +- **DART-2 MUST.** Value types are `final class` types with private constructors and `final` fields. +- **DART-3 MUST.** `ErrorCode` is an enhanced enum with a `wireValue` field, for example `ErrorCode.rateLimited` has the value `'rate_limited'`. +- **DART-4 MUST.** The caller can pass an `http.Client`. The client closes only an `http.Client` that it created. +- **DART-5 MUST.** Each client call takes an optional `abortTrigger` future. The client sends an `AbortableRequest`. When the future completes, the call ends with the `RequestAbortedException` from package:http. `IOClient`, `BrowserClient` and `RetryClient` support this. If the caller's `http.Client` cannot abort, the timeout still ends the call. +- **DART-6 MUST.** Durations use `Duration`. +- **DART-7 MUST.** Each public member has a doc comment with an example. The analyzer checks that the comment exists. **(tool)** +- **DART-8 MUST.** Code lives in `lib/src/`. The file `lib/gatepost_postcode.dart` exports only the public API. +- **DART-9 MUST.** Each `// ignore:` comment has a comment that gives the reason. **(tool)** +- **DART-10 MUST.** Code writes each non-ASCII or invisible character as an escape sequence, for example `'\u00A0'` and `'\u200B'`. **(tool)** +- **DART-11 MUST.** The client parses JSON with `dart:convert` and hand-written `fromJson` functions. They read only the keys that they need. An unknown value of a known field maps to the fallback that the spec defines. This follows API-14. +- **DART-12 MUST.** An internal queue limits each client to 4 requests at a time, as API-8 requires. A fifth call waits until one of the first four ends. +- **DART-13 MUST.** Tests replace the clock through an internal seam until the spec defines a clock option for API-10. +- **DART-14 MUST.** Widgets use `const` constructors where possible. **(tool)** +- **DART-15 MUST.** Widgets read colours, radii and fonts from a `ThemeExtension`. They contain no fixed colours. +- **DART-16 MUST.** Each control has a semantics label. Widgets follow `MediaQuery.textScalerOf` and `MediaQuery.disableAnimationsOf`. + +## Names + +| Element | Style | Example | +|---|---|---| +| class, enum, typedef, extension | UpperCamelCase | `PostcodeClient` | +| function, method, variable, parameter | lowerCamelCase | `precisionForAccuracy` | +| constant | lowerCamelCase | `specVersion` | +| enum value | lowerCamelCase | `ErrorCode.rateLimited` | +| error code value | snake_case string | `'rate_limited'` | +| file and library | lowercase_with_underscores | `postcode_client.dart` | +| package | lowercase_with_underscores | `gatepost_postcode` | + +## Package layout + +``` +packages/gatepost_postcode/ + lib/gatepost_postcode.dart public exports only + lib/src/ one concept per file + test/ unit, vector and contract tests + analysis_options.yaml + pubspec.yaml + README.md from templates/README.md + CHANGELOG.md generated by changie +packages/gatepost_postcode_field/ + lib/gatepost_postcode_field.dart + lib/src/ + test/ widget tests + example/ a runnable example app +.changes/ changie fragments +Makefile +``` + +## Publishing + +- A GitHub Actions job publishes to pub.dev with automated publishing when a version tag reaches the repo. +- A verified publisher owns both packages. The publisher uses the team's domain. +- `dart pub publish --dry-run` runs on each pull request. diff --git a/standards/languages/go.md b/standards/languages/go.md new file mode 100644 index 0000000..01e40b6 --- /dev/null +++ b/standards/languages/go.md @@ -0,0 +1,155 @@ +# Go standards + +Version 1.0, 1 Oct 2026. + +This file and `standards/CODING_STANDARDS.md` together are the standard for Go code. They apply to the `go` repo. The repo publishes one module, `github.com/gatepost-dev/postcode-go`, with the package `postcode`. The package holds the core functions and the client. + +## Versions + +- The two newest Go releases. CI tests both. The `go` line in `go.mod` names the older one. +- Go supports each release until two newer releases exist. So the older floor loses support when the next Go release ships, in February or August. +- The module has no dependencies outside the standard library. +- Tools are not dependencies. The Makefile pins each tool version and runs it with `go run`. + +## Tools + +`make check` runs every tool below. CI runs `make check` on each push and each pull request. + +| Tool | Job | Settings | +|---|---|---| +| gofmt and goimports | format | `goimports -local github.com/gatepost-dev/postcode-go` | +| go vet | static checks | default analyzers | +| staticcheck | static analysis | default checks | +| golangci-lint | lint | the settings below | +| `go test` | unit, vector, contract and example tests | `-race -cover`, coverage floors from T-7 | +| `go test -fuzz` | fuzz tests | `FuzzParse` for 30 seconds on each pull request | +| govulncheck | vulnerability scan | every push | +| gorelease | API check against the last tag | enforces GIT-7 and API-13 | +| changie | change files and changelog | one change file per user-visible change | +| `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in source, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | +| REUSE | licence headers | `reuse lint` | +| gitleaks | secret scan | every push | + +### golangci-lint settings + +```yaml +version: "2" +linters: + enable: + - errcheck + - errorlint + - exhaustive + - forbidigo + - gochecknoglobals + - gochecknoinits + - godot + - gosec + - lll + - misspell + - nolintlint + - revive + settings: + lll: + line-length: 100 + revive: + rules: + - name: argument-limit + arguments: [4] + - name: max-control-nesting + arguments: [3] + - name: cyclomatic + arguments: [10] + - name: exported + - name: var-naming + forbidigo: + forbid: + - pattern: ^fmt\.Print.*$ + - pattern: ^log\.(Print|Fatal|Panic).*$ + - pattern: ^print(ln)?$ + nolintlint: + require-explanation: true + require-specific: true + exclusions: + rules: + - path: _test\.go + linters: + - forbidigo +``` + +Example functions print their output, so `forbidigo` skips test files. + +### Agent-tell checks + +| Rule | Check in Go | +|---|---| +| TELL-1 | golangci-lint `lll` with `line-length: 100`, and `check-tells`. gofmt has no line limit. | +| TELL-2 | revive `max-control-nesting` 3 and `cyclomatic` 10 | +| TELL-3 | revive `argument-limit` 4. Functional options carry the rest. | +| TELL-13 | `forbidigo` and `check-tells`. They ban `fmt.Print`, `fmt.Printf`, `fmt.Println`, `log.Print`, `log.Fatal`, `log.Panic`, `print` and `println` in library code. | +| TELL-14 | `check-tells` | + +## Rules + +T-8 does not apply to Go. Fuzz tests cover T-9. + +- **GO-1 MUST.** `Parse` and `ParsePartial` return `(Postcode, error)`. A parse failure is a `*ParseError` with `Code`, `Segment` and `Suggestion`. +- **GO-2 MUST.** A client failure is a `*Error` with `Code`, `Status`, `APICode` and `RetryAfter`. Callers read it with `errors.As`. +- **GO-3 MUST.** Each I/O method takes `context.Context` as its first parameter. +- **GO-4 MUST.** If the caller's context ends, the method returns an error that wraps `ctx.Err()`. Then `errors.Is(err, context.Canceled)` works. The client's own 8-second timeout returns a `*Error` with `CodeTimeout`. +- **GO-5 MUST.** `NewClient` takes functional options: `NewClient(opts ...Option)`. Per-call options use the type `LookupOption`, for example `WithLevel(2)`. +- **GO-6 MUST.** The caller can pass an `*http.Client` with `WithHTTPClient`. The client never changes or closes it. +- **GO-7 MUST.** Library code does not panic. The package has no `init` functions and no package-level variables. **(tool)** +- **GO-8 MUST.** Value types keep their fields unexported. Accessor methods expose them, for example `p.Canonical()`. +- **GO-9 MUST.** Each exported symbol has a doc comment that starts with its name and ends with a period. **(tool)** +- **GO-10 MUST.** Each exported function and method has an `Example` function with an `// Output:` comment. +- **GO-11 MUST.** Initialisms keep one case, for example `LGA`, `API`, `URL` and `HTTP`. **(tool)** +- **GO-12 MUST.** Durations use `time.Duration`. +- **GO-13 MUST.** Error strings use lower case and end without punctuation. Wrapped errors use `%w`. **(tool)** +- **GO-14 MUST.** Each `//nolint` comment names the linter and gives a reason. **(tool)** +- **GO-15 MUST.** Names do not repeat the package name. Use `postcode.Client`, not `postcode.PostcodeClient`. **(tool)** +- **GO-16 MUST.** Code writes each non-ASCII or invisible character as an escape sequence, for example `"\u00a0"` and `"\u200b"`. **(tool)** +- **GO-17 MUST.** Contract tests carry the build tag `contract`. CI runs them against the mock server. +- **GO-18 MUST.** Tests replace the clock through an unexported field until the spec defines a clock option for API-10. +- **GO-19 MUST.** Each error string starts with `postcode:`, so that a caller can see which package made it. +- **GO-20 MUST.** The client decodes JSON with `encoding/json`, which ignores unknown fields. Do not call `DisallowUnknownFields`. An unknown value of a known field maps to the fallback that the spec defines. This follows API-14. +- **GO-21 MUST.** A buffered channel of size 4 limits each client to 4 requests at a time, as API-8 requires. A waiting call still ends when its context ends. + +## Names + +| Element | Style | Example | +|---|---|---| +| exported type, function, method | PascalCase | `ParsePartial`, `Client` | +| unexported identifier | camelCase | `normalize` | +| initialism | one case | `LGA`, `apiKey` | +| client error code | `Code` prefix | `CodeRateLimited` | +| parse error code | `ParseCode` prefix | `ParseCodeBadLength` | +| error code value | snake_case string | `"rate_limited"` | +| option | `With` prefix | `WithLevel`, `WithHTTPClient` | +| spec version | exported constant | `SpecVersion` | +| file | lower case, underscores | `parse.go`, `parse_test.go` | + +## Package layout + +``` +go.mod module github.com/gatepost-dev/postcode-go +doc.go package documentation +postcode.go the Postcode type +parse.go +client.go +errors.go +parse_test.go +example_test.go Example functions +fuzz_test.go +vectors_test.go runs spec/vectors/ +contract_test.go build tag contract, runs against the mock server +.changes/ changie fragments +Makefile +README.md from templates/README.md +CHANGELOG.md generated by changie +``` + +## Publishing + +- CI runs `gorelease` and then creates a signed tag `vX.Y.Z`. +- The Go module proxy serves each tagged version. +- After 1.0, a breaking change needs a new major version path, for example `/v2`. diff --git a/standards/languages/kotlin-java.md b/standards/languages/kotlin-java.md new file mode 100644 index 0000000..f3a1c7e --- /dev/null +++ b/standards/languages/kotlin-java.md @@ -0,0 +1,116 @@ +# Kotlin and Java standards + +Version 1.0, 1 Oct 2026. + +This file and `standards/CODING_STANDARDS.md` together are the standard for Kotlin code that Kotlin and Java callers use. They apply to the `kotlin` repo. The repo publishes two artifacts: `postcode-core` and `postcode-client`. Later Android work follows them too, such as the Jetpack Compose field. + +## Versions + +- The newest stable Kotlin. The build emits JVM bytecode 11. +- Java callers need Java 11 or later. Eclipse Temurin 11 has support until at least 31 Oct 2027. +- Android code supports minSdk 21. No module depends on Google Play services. +- Android 5.0, which is API 21, gets no security updates. An ADR records why the floor stays at API 21: agent-banking POS terminals run old Android versions. +- `postcode-core` depends on the Kotlin standard library only. +- `postcode-client` depends on OkHttp, `kotlinx-coroutines-core` and `kotlinx-serialization-json`. An ADR records the two kotlinx dependencies under DEP-3. A `suspend` call can be cancelled only through `suspendCancellableCoroutine`. The JVM has no standard JSON library. + +## Tools + +`./gradlew check` runs every tool below. CI runs `./gradlew check` on each push and each pull request. + +| Tool | Job | Settings | +|---|---|---| +| Gradle Kotlin DSL with a version catalog | build | `gradle/libs.versions.toml` | +| Kotlin explicit API mode | public API discipline | `kotlin { explicitApi() }` | +| ktlint | format | `max_line_length = 100` in `.editorconfig` | +| detekt | lint | the settings below | +| binary-compatibility-validator | API dump | `apiCheck` runs in `check`. The `.api` files are in the repo. | +| JUnit 5 | unit, vector and contract tests | | +| kotest-property | property tests | `parse`, `normalize` and the hierarchy functions | +| Kover | coverage | floors from T-7 | +| PIT | mutation tests | `postcode-core`, weekly scheduled job | +| Dokka | API docs | every public symbol | +| Android Lint | Android checks | Android modules only, warnings as errors | +| changie | change files and changelog | one change file per user-visible change | +| `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in source, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | +| REUSE | licence headers | `reuse lint` | +| gitleaks | secret scan | every push | + +### detekt settings + +| Rule | Setting | Enforces | +|---|---|---| +| `style>MaxLineLength` | `maxLineLength: 100` | TELL-1 | +| `complexity>NestedBlockDepth` | `threshold: 4`, which allows 3 levels | TELL-2 | +| `complexity>CyclomaticComplexMethod` | the limit of 10 from TELL-2 | TELL-2 | +| `complexity>LongParameterList` | `functionThreshold: 5`, `constructorThreshold: 5`, `ignoreDefaultParameters: true` | TELL-3 | +| `style>ForbiddenMethodCall` | the debug calls listed under Agent-tell checks | TELL-13 | +| `potential-bugs>UnsafeCallOnNullableType` | active | KT-9 | + +### Agent-tell checks + +| Rule | Check in Kotlin | +|---|---| +| TELL-1 | ktlint `max_line_length = 100`, detekt `MaxLineLength`, and `check-tells` | +| TELL-2 | detekt `NestedBlockDepth` and `CyclomaticComplexMethod` | +| TELL-3 | detekt `LongParameterList`. Parameters with default values do not count, because Kotlin callers name them. | +| TELL-13 | detekt `ForbiddenMethodCall` and `check-tells`. They ban `println`, `print`, `System.out`, `System.err`, `printStackTrace` and the Android `Log` methods. | +| TELL-14 | `check-tells` | + +## Rules + +- **KT-1 MUST.** Explicit API mode is on. Each public declaration states its visibility and its return type. **(tool)** +- **KT-2 MUST.** Factory functions live in the companion object of their type, with `@JvmStatic`, for example `Postcode.parse`. Functions with default arguments carry `@JvmOverloads`. Blocking methods carry `@Throws(PostcodeException::class)`. +- **KT-3 MUST.** The public API uses no Kotlin-only types. Do not expose `kotlin.Result`, value classes or `kotlin.time.Duration`. Java callers can call each operation. +- **KT-4 MUST.** Each client operation has three forms. Kotlin callers use `suspend fun lookup()`. Java callers use `lookupBlocking()` or `lookupAsync(..., callback)`. +- **KT-5 MUST.** `lookupAsync` returns a `Cancellable` handle. After a call to `cancel()`, the callback never runs. +- **KT-6 MUST.** The client does not use `CompletableFuture`, because it needs Android API 24. +- **KT-7 MUST.** Callbacks run on a background thread. The docs tell Android callers to move to the main thread. +- **KT-8 MUST.** The client rethrows each `CancellationException`. It never wraps one in `PostcodeException`. +- **KT-9 MUST.** Do not use `!!` outside tests. **(tool)** +- **KT-10 MUST.** Validated value types have a private constructor. A data class that is a value type carries `@ConsistentCopyVisibility`, so its `copy` is private too. +- **KT-11 MUST.** Error codes use `enum class ErrorCode(val wireValue: String)`. +- **KT-12 MUST.** Durations are `Long` milliseconds with the suffix `Millis`, for example `timeoutMillis`. `java.time` needs Android API 26 or desugaring. +- **KT-13 MUST.** The caller can pass an `OkHttpClient`. The client never shuts down an `OkHttpClient` that it did not create. +- **KT-14 MUST.** Each public symbol has a KDoc comment with an example. Dokka renders it. +- **KT-15 MUST.** Each `@Suppress` annotation names the rule and has a comment that gives the reason. +- **KT-16 MUST.** Code writes each non-ASCII or invisible character as an escape sequence, for example `"\u00A0"` and `"\u200B"`. **(tool)** +- **KT-17 MUST.** Tests replace the clock through an internal seam until the spec defines a clock option for API-10. +- **KT-18 MUST.** The client decodes JSON with `kotlinx-serialization-json`, configured with `ignoreUnknownKeys = true` and `coerceInputValues = true`. An unknown enum value then takes the default value of its property, which is the fallback that the spec defines. This follows API-14. +- **KT-19 MUST.** A kotlinx.coroutines `Semaphore` with 4 permits limits each client to 4 requests at a time, as API-8 requires. The blocking and callback forms go through the same semaphore. + +## Names + +| Element | Style | Example | +|---|---|---| +| class, interface, enum | PascalCase | `PostcodeClient` | +| function, property, parameter | camelCase | `precisionForAccuracy` | +| enum entry | UPPER_SNAKE_CASE | `ErrorCode.RATE_LIMITED` | +| constant | UPPER_SNAKE_CASE | `Postcode.SPEC_VERSION` | +| error code value | snake_case string | `"rate_limited"` | +| package | lower case, no hyphens | `dev.gatepost.postcode` | +| file | PascalCase, named after its main type | `PostcodeClient.kt` | + +## Package layout + +``` +postcode-core/ + src/main/kotlin/dev/gatepost/postcode/ + src/test/kotlin/dev/gatepost/postcode/ unit, vector and property tests + api/postcode-core.api API dump +postcode-client/ + src/main/kotlin/dev/gatepost/postcode/client/ + src/test/kotlin/dev/gatepost/postcode/client/ unit and contract tests + api/postcode-client.api +gradle/libs.versions.toml +build.gradle.kts +settings.gradle.kts +.changes/ changie fragments +README.md from templates/README.md +CHANGELOG.md generated by changie +``` + +## Publishing + +- CI publishes both artifacts to Maven Central through the Central Portal. CI signs each artifact. +- The Maven group is `dev.gatepost` if the team owns `gatepost.dev`. Otherwise the group is `io.github.gatepost-dev`, and the package root is `io.github.gatepostdev.postcode`. +- A signed tag `vX.Y.Z` starts each release. diff --git a/standards/languages/php.md b/standards/languages/php.md new file mode 100644 index 0000000..4321773 --- /dev/null +++ b/standards/languages/php.md @@ -0,0 +1,123 @@ +# PHP standards + +Version 1.0, 1 Oct 2026. + +This file and `standards/CODING_STANDARDS.md` together are the standard for PHP code. Part A applies to the SDK in the `php` repo. Part B applies to WordPress plugins, such as the WooCommerce plugin. Each part names its own code style, because the two ecosystems use different conventions. + +## Versions + +- PHP 8.1 or later. CI tests PHP 8.1 and the newest PHP release. +- PHP 8.1 reached end of life on 31 Dec 2025, and PHP 8.2 reaches it on 31 Dec 2026. The 8.1 floor serves WordPress stores on older hosting. Under VER-1, ADR 0001 must record this trade-off with current WordPress usage data before the first release. If the data does not support 8.1, raise the floor. +- PHPUnit 11 and later need PHP 8.2. CI runs PHPUnit 10 on PHP 8.1 and the newest PHPUnit on newer PHP versions. +- WordPress and WooCommerce: the plugin sub-plan sets the floors after it tests the Additional Checkout Fields API. + +## Part A: the SDK + +### Tools + +`composer check` runs every tool below. CI runs `composer check` on each push and each pull request. + +| Tool | Job | Settings | +|---|---|---| +| PHP-CS-Fixer | format and style | `@PER-CS`, `@PER-CS:risky`, `declare_strict_types`, `strict_comparison`, `strict_param`, `final_class`, `no_unused_imports`, `ordered_imports` | +| PHPStan | static analysis | level `max`, with `phpstan-strict-rules`, `phpstan-deprecation-rules` and `phpstan-phpunit` | +| PHPUnit 10 | unit and contract tests | coverage floors from T-7 | +| Infection | mutation tests | the core namespace, weekly scheduled job | +| Roave BackwardCompatibilityCheck | API check against the last tag | enforces GIT-7 and API-13 | +| `composer validate --strict` and `composer-normalize` | package checks | every push | +| PHPMD | size and complexity | `CyclomaticComplexity` 10, `ExcessiveParameterList` minimum 5, enforces TELL-2 and TELL-3 | +| changie | change files and changelog | one change file per user-visible change, enforces DOC-5 | +| REUSE | licence headers | `reuse lint` | +| gitleaks | secret scan | every push | +| `spec/scripts/check-tells` | agent tells | line length 100, file names, TODO form, debug calls, emojis and non-ASCII | + +PHP-CS-Fixer does not limit line length, so `check-tells` enforces TELL-1. PHPMD has no nesting rule, so reviewers check nesting for TELL-2. + +`check-tells` bans these debug calls in PHP: `var_dump`, `print_r`, `var_export`, `dd`, `dump`, `error_log` and `debug_print_backtrace`. + +### Rules + +- **PHP-1 MUST.** Each file declares `declare(strict_types=1);`. **(tool)** +- **PHP-2 MUST.** Namespaces follow PSR-4 under `Gatepost\Postcode\`. **(tool)** +- **PHP-3 MUST.** Classes are `final`, unless the spec names an extension point. **(tool)** +- **PHP-4 MUST.** Value objects have `readonly` properties. Their constructors are private, and named constructors create them, for example `Postcode::parse()`. +- **PHP-5 MUST.** Error codes are string-backed enums. Each case's value is the shared wire value, for example `ErrorCode::RateLimited` is `'rate_limited'`. +- **PHP-6 MUST.** The client depends only on the PSR-18, PSR-17 and PSR-16 interfaces. A concrete HTTP client, such as Guzzle, is a dev dependency only. +- **PHP-7 MUST.** There is no static mutable state. +- **PHP-8 MUST.** Each array has a PHPDoc shape or generic type, for example `list`. **(tool)** +- **PHP-9 MUST.** The package does not depend on `ext-intl` or `ext-mbstring`. Use PCRE patterns with the `u` flag, and static lookup tables with `strtr`. +- **PHP-10 MUST.** The package throws only `PostcodeException`. For programmer errors, it throws `InvalidArgumentException` or `LogicException`. +- **PHP-11 MUST.** Do not use the `@` operator or `eval`. **(tool)** +- **PHP-12 MUST.** Each method declares its parameter types and its return type. **(tool)** +- **PHP-13 SHOULD.** Examples and docs use named arguments, for example `$client->lookup($code, level: 2)`. +- **PHP-14 MUST.** The client decodes JSON into arrays and reads only the keys that it knows (API-14). +- **PHP-15 MUST.** The client is synchronous, so it sends one request at a time. This meets the limit in API-8. + +### Idiomatic additions (API-1) + +The PHP SDK follows the spec's interface section. Its client constructor takes PSR-18, PSR-17 and PSR-16 objects. These are parameters, not new public symbols. + +### Names + +| Element | Style | Example | +|---|---|---| +| class, enum, interface | PascalCase | `PostcodeClient` | +| method, variable | camelCase | `precisionForAccuracy` | +| enum case | PascalCase | `ErrorCode::BadLength` | +| constant | UPPER_SNAKE_CASE | `SPEC_VERSION` | +| error code value | snake_case string | `'bad_length'` | + +### Package layout + +``` +src/ PSR-4 root for Gatepost\Postcode\ +src/Client/ the client and its errors +tests/Unit/ +tests/Contract/ runs against the mock server +tests/Vectors/ runs spec/vectors/ +README.md from templates/README.md +CHANGELOG.md +``` + +### Publishing + +- A Git tag `vX.Y.Z` on `main` publishes the version on Packagist. +- CI creates the tag. Tags are signed. + +## Part B: WordPress plugins + +### Tools + +`composer check` runs every tool below. CI runs it on each push and each pull request. + +| Tool | Job | Settings | +|---|---|---| +| PHPCS with WordPress Coding Standards 3 | style and security sniffs | `WordPress-Extra` and `WordPress-Docs` | +| PHPCompatibilityWP | PHP version check | `testVersion` 8.1 and later | +| PHPStan with `szepeviktor/phpstan-wordpress` | static analysis | level 8 | +| Plugin Check | WordPress.org review rules | `wp plugin check` on the built zip | +| PHPUnit with the WordPress test suite | integration tests | WooCommerce installed | +| Playwright | checkout tests | classic checkout and block checkout | +| Strauss | namespace prefixing for bundled packages | one prefix per plugin, `Gatepost\\Vendor\`, for example `Gatepost\WooCommerce\Vendor\` | +| PHPCS `Generic.Metrics` and `Generic.Files.LineLength` | agent tells | nesting level 3, cyclomatic complexity 10, line length 100 | +| changie | change files and changelog | one change file per user-visible change, enforces DOC-5 | +| `spec/scripts/check-tells` | agent tells | file names, TODO form, debug calls, emojis and non-ASCII | + +### Rules + +- **WP-1 MUST.** Plugin code follows WordPress Coding Standards 3. PER-CS does not apply to plugin code. **(tool)** +- **WP-2 MUST.** Each global function, class, hook, option, meta key and script handle starts with `gatepost_`, `Gatepost_` or the namespace `Gatepost\`. **(tool)** +- **WP-3 MUST.** Escape output late. Sanitize input early. Check a nonce and a capability before each write. **(tool)** +- **WP-4 MUST.** Each plugin bundles the PHP SDK under its own prefixed namespace with Strauss, for example `Gatepost\WooCommerce\Vendor\`. Two plugins that bundle different SDK versions, including two Gatepost plugins, must not clash. +- **WP-5 MUST.** Plugin Check reports no errors. **(tool)** +- **WP-6 MUST.** The plugin declares compatibility with HPOS and with the cart and checkout blocks. +- **WP-7 MUST.** Each user-facing string uses the plugin's text domain. A string with a placeholder has a translator comment. **(tool)** +- **WP-8 MUST.** The plugin stores only the postcode, the check result and the time of the check. It stores no other data from NIPOST's responses. +- **WP-9 MUST.** Uninstalling the plugin deletes its options. Order meta stays, because it belongs to the store's order records. +- **WP-10 MUST.** The plugin logs only through `wc_get_logger()`, with the source `gatepost`. The logs follow ERR-3. + +### Publishing + +- CI builds the zip with Strauss and runs Plugin Check on it. +- A GitHub release carries the zip. +- After WordPress.org approves the plugin, CI deploys each release to the WordPress.org SVN repository. diff --git a/standards/languages/python.md b/standards/languages/python.md new file mode 100644 index 0000000..22ada52 --- /dev/null +++ b/standards/languages/python.md @@ -0,0 +1,133 @@ +# Python standards + +Version 1.0, 1 Oct 2026. + +This file and `standards/CODING_STANDARDS.md` together are the standard for Python code. They apply to the `python` repo. The repo publishes one distribution, `gatepost-postcode`, with the import name `gatepost_postcode`. It holds the core functions, `PostcodeClient` and `AsyncPostcodeClient`. Later Python work follows them too, such as the QGIS plugin. + +## Versions + +- Python 3.11 or later. CI tests Python 3.11 and the newest Python release. +- Python 3.11 reaches end of life on 31 Oct 2027. +- httpx is the one runtime dependency of the client. The core has none. + +## Tools + +`make check` runs every tool below. CI runs `make check` on each push and each pull request. + +| Tool | Job | Settings | +|---|---|---| +| uv | environments, lockfile and builds | `uv.lock` is in the repo | +| hatchling | build backend | `pyproject.toml` | +| Ruff format | format | `line-length = 100` | +| Ruff lint | lint | the settings below | +| mypy | type check | `--strict` on `src/` and `tests/` | +| pytest with pytest-cov | unit, vector and contract tests | coverage floors from T-7 | +| Hypothesis | property tests | `parse`, `normalize` and the hierarchy functions | +| mutmut | mutation tests | the core modules, weekly scheduled job | +| griffe | API check against the last tag | `griffe check gatepost_postcode --against ` | +| towncrier | change files and changelog | one fragment in `changes/` per user-visible change | +| `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in source, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | +| REUSE | licence headers | `reuse lint` | +| gitleaks | secret scan | every push | + +### Ruff settings + +```toml +[tool.ruff] +target-version = "py311" +line-length = 100 + +[tool.ruff.lint] +preview = true +explicit-preview-rules = true +select = [ + "E", "F", "W", "I", "B", "UP", "SIM", "RUF", "D", "ANN", "S", "PT", + "PL", "C4", "PIE", "RET", "ERA", "T20", "T10", "C90", +] +extend-select = ["PLR1702", "PLR0917"] +ignore = ["PLR0913"] + +[tool.ruff.lint.mccabe] +max-complexity = 10 + +[tool.ruff.lint.pylint] +max-nested-blocks = 3 +max-positional-args = 4 + +[tool.ruff.lint.pydocstyle] +convention = "google" + +[tool.ruff.lint.per-file-ignores] +"tests/**" = ["S101", "D", "ANN", "PLR2004"] +``` + +`PLR1702` is a preview rule. `explicit-preview-rules` turns on only the preview rules that the file names. `PLR0913` is off, because TELL-3 counts only positional parameters. `PLR0917` counts only positional parameters. + +### Agent-tell checks + +| Rule | Check in Python | +|---|---| +| TELL-1 | Ruff `E501` with `line-length = 100`, and `check-tells` | +| TELL-2 | Ruff `C901` with `max-complexity = 10`, and `PLR1702` with `max-nested-blocks = 3` | +| TELL-3 | Ruff `PLR0917` with `max-positional-args = 4`. Keyword-only parameters do not count. | +| TELL-13 | Ruff `T20` and `T10`, and `check-tells`. They ban `print`, `pprint`, `breakpoint` and `pdb.set_trace`. | +| TELL-14 | `check-tells` | + +## Rules + +- **PY-1 MUST.** The package ships a `py.typed` marker. Every public function and method has full type hints. **(tool)** +- **PY-2 MUST.** Value types are frozen dataclasses with slots: `@dataclass(frozen=True, slots=True)`. +- **PY-3 MUST.** Options are keyword-only. The text argument of `parse` is positional-only: `parse(text, /, *, allow_partial=False)`. +- **PY-4 MUST.** Error codes use `class ErrorCode(StrEnum)`. Each member's value is the shared wire value, for example `ErrorCode.RATE_LIMITED` is `"rate_limited"`. +- **PY-5 MUST.** `PostcodeClient` and `AsyncPostcodeClient` have the same methods, the same options and the same errors. Only `async` and `await` differ. +- **PY-6 MUST.** The caller can pass an `httpx.Client` or an `httpx.AsyncClient` through the keyword `http_client`. The client closes only an HTTP client that it created. +- **PY-7 MUST.** Each client is a context manager. `with` closes the sync client. `async with` closes the async client. +- **PY-8 MUST.** Durations are seconds as `float`, as in httpx. Names have no unit suffix: `timeout`, `cache_ttl` and `retry_after`. +- **PY-9 MUST.** The async client lets `asyncio.CancelledError` pass through. It never catches it and never wraps it. +- **PY-10 MUST.** Errors subclass `PostcodeError`, which subclasses `Exception`. A raised error keeps its cause with `raise ... from ...`. +- **PY-11 MUST.** Each public symbol has a Google-style docstring with an `Example:` section. **(tool)** +- **PY-12 MUST.** The package does not call `print`, `pprint`, `breakpoint` or `pdb.set_trace`. Importing a module has no side effects. **(tool)** +- **PY-13 MUST.** `__all__` in `gatepost_postcode/__init__.py` lists every public symbol. Other modules start with an underscore. +- **PY-14 MUST.** Each `noqa` and each `type: ignore` comment names the rule and gives a reason. +- **PY-15 SHOULD.** Prefer functions and dataclasses. The allowed classes with behaviour are the two clients and the errors. +- **PY-16 MUST.** Code writes each non-ASCII or invisible character as an escape sequence, for example `"\u00a0"` and `"\u200b"`. **(tool)** +- **PY-17 MUST.** Tests replace the clock through an internal seam until the spec defines a clock option for API-10. +- **PY-18 MUST.** The client reads JSON with the `json` module and reads each field that it needs by name. It ignores unknown fields. An unknown value of a known field maps to the fallback that the spec defines. This follows API-14. +- **PY-19 MUST.** A semaphore limits each client to 4 requests at a time, as API-8 requires. The async client uses `asyncio.Semaphore`. The sync client uses `threading.BoundedSemaphore`. + +## Names + +| Element | Style | Example | +|---|---|---| +| function, method, variable | snake_case | `precision_for_accuracy` | +| class, dataclass, enum | PascalCase | `ParseResult` | +| enum member | UPPER_SNAKE_CASE | `ErrorCode.BAD_LENGTH` | +| constant | UPPER_SNAKE_CASE | `SPEC_VERSION` | +| error code value | snake_case string | `"bad_length"` | +| private module | leading underscore | `_normalize.py` | +| keyword option | snake_case | `allow_partial`, `http_client` | + +## Package layout + +``` +src/gatepost_postcode/ + __init__.py public API, __all__ only + _parse.py one private module per concept + _client.py + _async_client.py + py.typed +tests/unit/ +tests/vectors/ runs spec/vectors/ +tests/contract/ runs against the mock server +changes/ towncrier fragments +pyproject.toml +uv.lock +README.md from templates/README.md +CHANGELOG.md generated by towncrier +``` + +## Publishing + +- A GitHub Actions job builds with `uv build`. +- The job publishes to PyPI with trusted publishing and attestations. +- Pre-releases use PEP 440 suffixes, for example `0.2.0a1`. diff --git a/standards/languages/swift.md b/standards/languages/swift.md new file mode 100644 index 0000000..9d9e4be --- /dev/null +++ b/standards/languages/swift.md @@ -0,0 +1,109 @@ +# Swift standards + +Version 1.0, 1 Oct 2026. + +This file and `standards/CODING_STANDARDS.md` together are the standard for Swift code. They apply to the `swift` repo. The repo publishes one Swift package. Its product `GatepostPostcode` holds the core and the client. A later product, `GatepostPostcodeUI`, holds the SwiftUI field. + +## Versions + +- Swift 6 language mode with complete strict concurrency checking. +- iOS 15 and macOS 12 or later. +- Apple publishes no end-of-life dates. iOS 15 and macOS 12 no longer get regular updates. An ADR records why the floors stay: NIPOST's own iOS widget supports iOS 15. +- The package has no dependencies outside Apple's swiftlang projects. `swift-docc-plugin` is the one package dependency, and only the docs build uses it. + +## Tools + +`make check` runs every tool below. CI runs `make check` on each push and each pull request. + +| Tool | Job | Settings | +|---|---|---| +| swift-format | format and lint | `lint --strict`, `lineLength` 100 in `.swift-format` | +| SwiftLint | lint | `--strict`, the settings below | +| `swift test` with Swift Testing | unit, vector and contract tests | `--enable-code-coverage` | +| llvm-cov | coverage | floors from T-7 | +| `swift package diagnose-api-breaking-changes` | API check against the last tag | runs on a macOS runner | +| DocC | API docs | `swift-docc-plugin` | +| changie | change files and changelog | one change file per user-visible change | +| `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in source, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | +| REUSE | licence headers | `reuse lint` | +| gitleaks | secret scan | every push | + +### SwiftLint settings + +```yaml +line_length: 100 +cyclomatic_complexity: 10 +function_parameter_count: + warning: 4 + ignores_default_parameters: true +opt_in_rules: + - force_unwrapping + - implicitly_unwrapped_optional +``` + +`--strict` turns each warning into a failure. + +### Agent-tell checks + +| Rule | Check in Swift | +|---|---| +| TELL-1 | swift-format `lineLength` 100, and SwiftLint `line_length` 100 | +| TELL-2 | SwiftLint `cyclomatic_complexity` 10. Nesting of control flow: review. | +| TELL-3 | SwiftLint `function_parameter_count` 4 with `ignores_default_parameters: true`. Parameters with default values do not count. | +| TELL-13 | `check-tells`. It bans `print`, `debugPrint`, `dump` and `NSLog`. | +| TELL-14 | `check-tells` | + +## Rules + +T-8 and T-9 do not apply to Swift. Swift has no standard tool for mutation tests or property tests, and the shared vectors cover the parser. + +- **SW-1 MUST.** Follow the Swift API Design Guidelines. Argument labels make each call read as a phrase, for example `client.lookup(code, level: 2)`. +- **SW-2 MUST.** Each public type conforms to `Sendable`. **(tool)** +- **SW-3 MUST.** Value types are structs with `let` properties. The static `Postcode.parse(_:allowPartial:)` creates them and returns `Result`. +- **SW-4 MUST.** Client methods are `async throws`. They throw `PostcodeError`, or `CancellationError` when the task is cancelled. Do not use typed throws for them, because `throws(PostcodeError)` cannot pass on `CancellationError`. +- **SW-5 MUST.** Cancellation follows the task. A cancelled task ends its request, and the method throws `CancellationError`. +- **SW-6 MUST.** `URLSession` reaches the client through a small public protocol with one method. Tests pass a fake that conforms to it. The caller's session stays open after the client ends. +- **SW-7 MUST.** Durations use `TimeInterval` seconds. The Swift `Duration` type needs iOS 16. +- **SW-8 MUST.** Do not use force unwraps, `try!` or implicitly unwrapped optionals outside tests. **(tool)** +- **SW-9 MUST.** `ErrorCode` is a `String` enum. Each raw value is the shared wire value, for example `case rateLimited = "rate_limited"`. +- **SW-10 MUST.** Each public symbol has a DocC comment with a summary, `- Parameters:`, `- Throws:` and a code example. +- **SW-11 MUST.** Each `swiftlint:disable` comment names the rule and gives a reason. Prefer `swiftlint:disable:next` for one line. +- **SW-12 MUST.** Code writes each non-ASCII or invisible character as an escape sequence, for example `"\u{00A0}"` and `"\u{200B}"`. **(tool)** +- **SW-13 MUST.** Tests replace the clock through an internal seam until the spec defines a clock option for API-10. +- **SW-14 MUST.** The client decodes JSON with `Codable`, which ignores unknown keys. A custom `init(from:)` maps an unknown raw value of a known enum to the fallback that the spec defines. This follows API-14. +- **SW-15 MUST.** An internal actor limits each client to 4 requests at a time, as API-8 requires. A waiting call still ends when its task is cancelled. + +## Names + +| Element | Style | Example | +|---|---|---| +| type, protocol | UpperCamelCase | `PostcodeClient` | +| function, method, property, variable | lowerCamelCase | `precisionForAccuracy` | +| enum case | lowerCamelCase | `ErrorCode.rateLimited` | +| constant | lowerCamelCase static property | `Postcode.specVersion` | +| error code value | snake_case raw value | `"rate_limited"` | +| module and product | UpperCamelCase | `GatepostPostcode` | +| file | named after its main type | `PostcodeClient.swift` | + +## Package layout + +``` +Package.swift +Sources/GatepostPostcode/ + Postcode.swift + ParseError.swift + PostcodeClient.swift + GatepostPostcode.docc/ DocC catalogue +Tests/GatepostPostcodeTests/ unit, vector and contract tests +.swift-format +.swiftlint.yml +.changes/ changie fragments +Makefile +README.md from templates/README.md +CHANGELOG.md generated by changie +``` + +## Publishing + +- CI runs the API check and then creates a signed tag without a `v` prefix, for example `1.2.0`. Swift Package Manager reads the tags. +- Do not publish to CocoaPods. Swift Package Manager is the one channel. diff --git a/standards/languages/typescript.md b/standards/languages/typescript.md new file mode 100644 index 0000000..8a099a4 --- /dev/null +++ b/standards/languages/typescript.md @@ -0,0 +1,149 @@ +# TypeScript standards + +Version 1.0, 1 Oct 2026. + +This file and `standards/CODING_STANDARDS.md` together are the standard for TypeScript code. They apply to every package and app in the `js` repo: core, client, field, React wrapper, docs site and demos. Later TypeScript work follows them too, such as the MCP server, the n8n node and the spreadsheet add-ins. + +## Versions + +- Published packages run on Node 22 or later. Node 22 reaches end of life on 30 Apr 2027. Raise the floor to Node 24 before then (VER-2). +- The development tools need Node 22.22.2 or later, or Node 24.15 or later. CI runs the tests on Node 22 and Node 24, and the other tools on Node 24. +- TypeScript 6.0. typescript-eslint supports only TypeScript versions below 6.1, so TypeScript 7 waits until typescript-eslint supports it. This was checked on 1 Oct 2026. +- React 18 and React 19, for `@gatepost/react`. +- Current evergreen browsers and Safari 16.4 or later, for browser code. +- Packages ship ESM only. + +## Tools + +`pnpm check` runs every tool below. CI runs `pnpm check` on each push and each pull request. + +| Tool | Job | Settings | +|---|---|---| +| `tsc --noEmit` | type check | `tsconfig.base.json`, see below | +| ESLint with `typescript-eslint` | lint | `strictTypeChecked` and `stylisticTypeChecked`, plus the rules below | +| Prettier | format | `singleQuote: true`, `printWidth: 100`, other options at default | +| Vitest with V8 coverage | unit tests | coverage floors from T-7 | +| fast-check | property tests | `parse`, `normalize` and the hierarchy functions | +| Playwright with `@axe-core/playwright` | browser and accessibility tests | field and docs site | +| monocart-coverage-reports | merged coverage | merges Vitest and Playwright V8 coverage for the field, so T-7 counts browser tests | +| StrykerJS | mutation tests | `@gatepost/core`, weekly scheduled job | +| size-limit | size limits | core 4 KB, client 5 KB, field 20 KB, compressed, as size-limit measures them | +| publint and `@arethetypeswrong/cli` | package checks | every published package | +| API Extractor | API report | `etc/.api.md`, checked in CI | +| Changesets | versions and changelogs | one change file per user-visible change | +| commitlint | commit messages | `@commitlint/config-conventional`, scopes from package names | +| REUSE | licence headers | `reuse lint` | +| gitleaks | secret scan | every push | +| `spec/scripts/check-tells` | agent tells | line length 100, file names, TODO form, debug calls, emojis and non-ASCII | +| TypeDoc | API pages | the docs site renders them | + +### `tsconfig.base.json` + +```json +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022"], + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "noUncheckedIndexedAccess": true, + "exactOptionalPropertyTypes": true, + "noImplicitOverride": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "noPropertyAccessFromIndexSignature": true, + "verbatimModuleSyntax": true, + "isolatedDeclarations": true, + "declaration": true, + "sourceMap": true, + "skipLibCheck": false, + "noEmit": true + } +} +``` + +tsdown builds the packages, so `tsc` only checks types. Each package's `tsconfig.json` includes `src` and `test`. Config files, such as `vitest.config.ts`, stay outside the TypeScript project, because `isolatedDeclarations` rejects their default exports. ESLint checks config files without type information. + +### Extra ESLint rules + +| Rule | Setting | Enforces | +|---|---|---| +| `no-restricted-syntax` for `TSEnumDeclaration` and `TSModuleDeclaration` | error | TS-5 | +| `no-restricted-exports` with `restrictDefaultExports` | error | TS-1 | +| `no-console` | error in packages | TS-12 | +| `@eslint-community/eslint-comments/require-description` | error | TS-13 | +| `jsdoc/require-jsdoc` with `publicOnly: true`, and `jsdoc/require-example` for exported functions, with `exemptedBy: ['internal']` | error in packages | TS-9 | +| `jsdoc/tag-lines` with `startLines: 1` | error in packages | one blank line between a description and its tags | +| `no-restricted-imports` for `node:*` | error in `packages/*/src` | CS-2 | +| `max-depth` | 3 | TELL-2 | +| `complexity` | 10 | TELL-2 | +| `max-params` | 4 | TELL-3 | +| `no-empty` with `allowEmptyCatch: false` | error | TELL-9 | +| `@typescript-eslint/no-unnecessary-condition` | error | TELL-10 | +| `@typescript-eslint/require-await` | error | TELL-16 | +| `no-nested-ternary` | error | TELL-17 | + +Prettier's `printWidth: 100` matches TELL-1. Prettier does not split long strings or comments, so `check-tells` also checks line length. + +`check-tells` bans these debug calls in TypeScript: `console.log`, `console.debug`, `console.dir` and `debugger`. + +## Rules + +- **TS-1 MUST.** Use named exports only. **(tool)** +- **TS-2 MUST.** Each package has one public entry, `src/index.ts`. The `exports` map in `package.json` has only `.`, plus the field's custom element entry. **(tool)** +- **TS-3 MUST.** Do not use `any`. Narrow `unknown` instead. **(tool)** +- **TS-4 MUST.** Do not use non-null assertions outside tests. **(tool)** +- **TS-5 MUST.** Do not use `enum` or `namespace`. Use string literal unions and modules. **(tool)** +- **TS-6 MUST.** Public types use `readonly` properties and `ReadonlyArray`. +- **TS-7 MUST.** A result type is a discriminated union with an `ok` field. +- **TS-8 MUST.** Each exported function and method declares its return type. **(tool)** +- **TS-9 MUST.** Each exported function has a TSDoc comment with `@example`. Each exported type and constant has a TSDoc comment. A helper that other modules use, but that `src/index.ts` does not export, carries the tag `@internal`. **(tool)** +- **TS-10 MUST.** An error class extends `Error`, sets `name` and passes `cause`. +- **TS-11 MUST.** A `catch` block treats its value as `unknown` and narrows it. **(tool)** +- **TS-12 MUST.** Packages do not call `console`. **(tool)** +- **TS-13 MUST.** Each `eslint-disable` comment gives a reason. **(tool)** +- **TS-14 MUST.** Each package ships ESM with an `exports` map and type declarations. **(tool)** +- **TS-15 MUST.** Browser code does not read `window`, `document` or `customElements` at import time. The field's guarded registration is the one exception. +- **TS-16 SHOULD.** Prefer functions and plain objects. The allowed classes are `PostcodeClient`, `PostcodeError` and the custom element. +- **TS-17 MUST.** File names use kebab-case. Tests sit in `test/` and end in `.test.ts`. +- **TS-18 MUST.** Type-only imports use `import type`. **(tool)** +- **TS-19 MUST.** `@gatepost/react` supports React 18 and React 19. It uses `forwardRef` while it supports React 18. Each component starts with `'use client'`. +- **TS-20 MUST.** The client reads only the response fields that it knows and ignores the rest (API-14). It parses each response with a narrowing function, not with a type assertion. +- **TS-21 MUST.** The client limits parallel requests with an internal queue, set by API-8. Batch tools, such as spreadsheet add-ins, call the client and add no parallelism of their own. + +### Idiomatic additions (API-1) + +The TypeScript SDK follows the spec's interface section exactly. It adds no idiomatic symbols. + +## Names + +| Element | Style | Example | +|---|---|---| +| function, variable, method | camelCase | `precisionForAccuracy` | +| type, class, interface | PascalCase | `ParseResult` | +| exported constant | UPPER_SNAKE_CASE | `SPEC_VERSION` | +| error code value | snake_case string | `'rate_limited'` | +| custom element | kebab-case with prefix | `gatepost-postcode-field` | +| DOM event | kebab-case with prefix | `gatepost-change` | +| CSS custom property | kebab-case with prefix | `--gatepost-accent` | +| file | kebab-case | `precision-for-accuracy.ts` | + +## Package layout + +``` +packages// + src/index.ts public entry, exports only + src/.ts one module per file + test/.test.ts + etc/.api.md API Extractor report + package.json + README.md from templates/README.md + CHANGELOG.md generated by Changesets +``` + +## Publishing + +- Changesets opens the release pull request. +- A GitHub Actions job publishes with npm trusted publishing and provenance. +- Pre-releases use the dist-tag `alpha` or `beta`. Stable releases use `latest`. diff --git a/templates/AGENTS.md b/templates/AGENTS.md new file mode 100644 index 0000000..15bbb6e --- /dev/null +++ b/templates/AGENTS.md @@ -0,0 +1,30 @@ +# AGENTS.md + +This repo is part of Gatepost, unofficial open-source developer tools for Nigeria's National Digital Postcode. {{REPO_PURPOSE}} + +## Read first + +- `CODING_STANDARDS.md` and the two files it names hold the rules. Cite rule IDs, such as `API-3`, in reviews and commit bodies. +- `CONTEXT.md` is the glossary. Name things with its terms in code, tests, docs and commits. +- `docs/adr/` holds the decisions. When a change contradicts an ADR, say so in the pull request. +- `spec/` holds the interfaces and the vectors. The spec wins over code and over these docs. + +## Rules that no config file shows + +- Call only the documented NIPOST gateway endpoints. +- Commit only NIPOST's published test codes and synthetic values, such as `FC-01-Z99-ZZ-01`. +- Sign off each commit with `git commit -s`. The person who opens the pull request is the only author. Leave out co-author lines for AI tools. +- Change `main` only through pull requests. Keep the history of `main` and of other people's branches as it is: never force-push to them. +- If a commit lacks its sign-off, add a DCO remediation commit, as the DCO check explains. +- Send every call to the gateway through the client. The client limits how many calls run at once. +- Write prose in plain British English, with short sentences in the active voice. +- Some agent tools change a typed backslash-u escape, such as the one for an en dash, into the character itself. Build such characters with code, for example `chr(0x2013)` in Python or `String.fromCodePoint(0x2013)` in TypeScript. In data and prose, write `U+2013`. + +## Done means all of these + +1. A failing test came first, and it passes now. +2. `{{CHECK_COMMAND}}` passes. It runs the formatter, linters, tests, vectors, size limits and `check-tells`. +3. Each user-visible change has a change file. +4. A change to the public interface also updates the API report and the docs. +5. Each new domain term is in `CONTEXT.md`. +6. The diff touches only the lines that the task needs. diff --git a/templates/CLAUDE.md b/templates/CLAUDE.md new file mode 100644 index 0000000..43c994c --- /dev/null +++ b/templates/CLAUDE.md @@ -0,0 +1 @@ +@AGENTS.md diff --git a/templates/CODING_STANDARDS.md b/templates/CODING_STANDARDS.md new file mode 100644 index 0000000..3baa029 --- /dev/null +++ b/templates/CODING_STANDARDS.md @@ -0,0 +1,8 @@ +# Coding standards + +This repo follows two files. Both are part of its standard: + +1. `spec/standards/CODING_STANDARDS.md` holds the rules for every Gatepost repo. +2. `spec/standards/languages/{{LANGUAGE_FILE}}` holds the rules for {{LANGUAGE_NAME}}. + +Rules for this repo only: {{REPO_RULES}} diff --git a/templates/CONTRIBUTING.md b/templates/CONTRIBUTING.md new file mode 100644 index 0000000..a141065 --- /dev/null +++ b/templates/CONTRIBUTING.md @@ -0,0 +1,55 @@ +# Contributing to Gatepost + +Thank you for your interest in Gatepost. This guide explains how to make a change that we can merge. + +Gatepost is unofficial. For problems with NIPOST's own app, data or API, contact NIPOST at postcode@nipost.gov.ng. + +## Before you start + +1. Read `CODING_STANDARDS.md` and the files it names. Reviews cite their rule IDs. +2. Read `CONTEXT.md`. Use its terms in code, tests and docs. +3. For a change bigger than a small fix, open an issue first. Agree on the approach before you write code. +4. A change to the public interface starts in the `spec` repo (API-1). + +## Make a change + +1. Create a branch named `/`, for example `fix/unit-zero-message`. +2. Write a failing test first (T-1). +3. Make the test pass. +4. Run the repo's check command. The README of the repo names it. +5. Add a change file for each user-visible change (DOC-5). +6. If you changed the public interface, update the API report and the docs (GIT-7). + +## Commit + +- Use Conventional Commits, for example `fix(core): reject a unit of 00` (GIT-1). +- Keep the subject at 72 characters or fewer. Explain why in the body (TELL-18). +- Sign off each commit with `git commit -s` (GIT-2). The sign-off states that you accept the Developer Certificate of Origin at https://developercertificate.org. +- If you forget a sign-off, add a remediation commit, as the DCO check explains. You do not need to rewrite history. +- You are the only author of your commits. Do not add co-author lines for AI tools (GIT-3). + +## AI-assisted contributions + +You can use AI tools. You are responsible for each line that you submit. If an AI tool wrote a large part of the change, say which parts in the pull request. The same standards apply to every line, whoever wrote it. + +## Open a pull request + +1. Keep it to one topic. Aim for 400 changed lines or fewer, not counting tests, vectors and generated files (GIT-5). +2. Fill in the pull request template. +3. Wait for green CI and one approval. We merge with squash (GIT-6). + +## Rules from NIPOST + +Gatepost follows NIPOST's terms and acceptable use policy: + +- Call only the documented gateway endpoints. +- Commit only NIPOST's published test codes and synthetic values. Do not commit real API responses, NINs, email addresses or the postcodes of private homes. +- Do not use NIPOST's name or logo as branding. + +## Conduct + +Everyone in this project follows the Code of Conduct. Report a problem to {{CONDUCT_EMAIL}}. + +## Licence + +You submit each contribution under the Apache-2.0 licence. diff --git a/templates/ISSUE_TEMPLATE/bug_report.yml b/templates/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 0000000..bf4e1f8 --- /dev/null +++ b/templates/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,44 @@ +name: Bug report +description: A Gatepost package does not work as its docs say. +labels: [bug, needs-triage] +body: + - type: markdown + attributes: + value: | + Gatepost is unofficial. For problems with NIPOST's own app, data or API, contact NIPOST at postcode@nipost.gov.ng. + Do not paste API keys, NINs, email addresses or the postcodes of private homes. Use synthetic postcodes, such as FC-01-Z99-ZZ-01. + - type: input + id: package + attributes: + label: Package and version + placeholder: "@gatepost/client 0.2.1" + validations: + required: true + - type: input + id: runtime + attributes: + label: Runtime and version + placeholder: "Node 22.11, Chrome 131, PHP 8.3 or iOS 18" + validations: + required: true + - type: textarea + id: steps + attributes: + label: Steps to reproduce + description: The smallest code that shows the problem. + render: text + validations: + required: true + - type: textarea + id: expected + attributes: + label: What you expected + validations: + required: true + - type: textarea + id: actual + attributes: + label: What happened + description: Include the error code, if there is one. + validations: + required: true diff --git a/templates/ISSUE_TEMPLATE/config.yml b/templates/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000..4a72d6f --- /dev/null +++ b/templates/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,11 @@ +blank_issues_enabled: false +contact_links: + - name: Question + url: https://github.com/orgs/gatepost-dev/discussions + about: Ask how to use a Gatepost package. + - name: Security problem + url: https://github.com/gatepost-dev/.github/blob/main/SECURITY.md + about: Report a vulnerability privately. + - name: NIPOST's own app or data + url: https://postcode.gov.ng/faq + about: Gatepost is unofficial. Contact NIPOST about NIPOST's services. diff --git a/templates/ISSUE_TEMPLATE/feature_request.yml b/templates/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 0000000..6b3e7fa --- /dev/null +++ b/templates/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,29 @@ +name: Feature request +description: Suggest a change to a Gatepost package. +labels: [enhancement, needs-triage] +body: + - type: textarea + id: problem + attributes: + label: The problem + description: What do you want to do? What stops you now? + validations: + required: true + - type: textarea + id: proposal + attributes: + label: Your proposal + description: A change to the public interface starts in the spec repo (API-1). + - type: dropdown + id: packages + attributes: + label: Packages + multiple: true + options: + - core + - client + - field + - React wrapper + - PHP SDK + - WooCommerce plugin + - another package diff --git a/templates/NOTICE b/templates/NOTICE new file mode 100644 index 0000000..bc94510 --- /dev/null +++ b/templates/NOTICE @@ -0,0 +1,5 @@ +Gatepost +Copyright 2026 The Gatepost authors + +This product is unofficial. The Nigerian Postal Service (NIPOST) did not make it and does not endorse it. +Gatepost uses the name NIPOST only to describe compatibility with NIPOST's National Digital Postcode. diff --git a/templates/PULL_REQUEST_TEMPLATE.md b/templates/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 0000000..e7110a0 --- /dev/null +++ b/templates/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,17 @@ +## What and why + + + +## Checklist + +- [ ] I wrote a failing test first, and it passes now (T-1). +- [ ] The repo's check command passes on my machine. +- [ ] Each user-visible change has a change file (DOC-5). +- [ ] If the public interface changed, I updated the API report and the docs (GIT-7). +- [ ] Each new domain term is in `CONTEXT.md` (DOC-7). +- [ ] The diff touches only the lines that the task needs (TELL-15). +- [ ] Each commit has a sign-off from `git commit -s` (GIT-2). + +## AI assistance + + diff --git a/templates/README.md b/templates/README.md new file mode 100644 index 0000000..de22839 --- /dev/null +++ b/templates/README.md @@ -0,0 +1,59 @@ +# {{PACKAGE_NAME}} + +{{ONE_SENTENCE_SUMMARY}} + +> Unofficial. Not made or endorsed by NIPOST. + +[![CI]({{CI_BADGE_URL}})]({{CI_URL}}) +[![Version]({{VERSION_BADGE_URL}})]({{REGISTRY_URL}}) +[![Licence](https://img.shields.io/badge/licence-Apache--2.0-blue)](LICENSE) + +## Install + +```{{LANGUAGE_ID}} +{{INSTALL_COMMAND}} +``` + +## Quickstart + +```{{LANGUAGE_ID}} +{{QUICKSTART_CODE}} +``` + +{{ONE_SENTENCE_ABOUT_THE_OUTPUT}} + +## What it does + +- {{CAPABILITY}} + +## Requirements + +| Requirement | Version | +|---|---| +| {{RUNTIME}} | {{MINIMUM_VERSION}} or later | +| Gatepost spec | {{SPEC_VERSION}} | + +## Docs + +The guide and the API reference are at {{DOCS_URL}}. + +## Support + +Ask questions in GitHub Discussions. Report bugs in GitHub Issues. Report security problems privately, as `SECURITY.md` describes. + +## Contributing + +Read `CONTRIBUTING.md` before you open a pull request. + +## Licence + +Apache-2.0. See `LICENSE` and `NOTICE`. + + diff --git a/templates/SECURITY.md b/templates/SECURITY.md new file mode 100644 index 0000000..a92d30b --- /dev/null +++ b/templates/SECURITY.md @@ -0,0 +1,26 @@ +# Security policy + +## Supported versions + +We fix security problems in the newest minor version of each package. + +## Report a problem + +Report each problem privately. Do not open a public issue. + +1. Open the repo's "Security" tab on GitHub. +2. Select "Report a vulnerability". +3. Describe the problem, the affected package and version, and the steps to reproduce it. + +You can also email {{SECURITY_EMAIL}}. + +## What happens next + +- We reply within 3 working days (SEC-8). +- We confirm the problem and agree on a disclosure date with you. +- We aim to release a fix within 30 days for a high or critical problem. +- We credit you in the release notes, unless you ask us not to. + +## Problems in NIPOST's systems + +This policy covers Gatepost code only. Report a problem in NIPOST's systems to security@nipost.gov.ng. Do not test NIPOST's systems. NIPOST's acceptable use policy forbids probing them. From 4306d1874270fdbf51e0c500b871fa2cd7091a6b Mon Sep 17 00:00:00 2001 From: MobileMage Date: Thu, 1 Oct 2026 22:10:54 +0100 Subject: [PATCH 02/45] feat: add the postcode grammar and data files Signed-off-by: MobileMage --- Makefile | 18 +++++++++ REUSE.toml | 6 +++ data/format.json | 22 +++++++++++ data/precision.json | 10 +++++ data/states.json | 43 ++++++++++++++++++++++ grammar.md | 75 ++++++++++++++++++++++++++++++++++++++ mypy.ini | 6 +++ ruff.toml | 18 +++++++++ scripts/tests/__init__.py | 2 + scripts/tests/test_data.py | 56 ++++++++++++++++++++++++++++ 10 files changed, 256 insertions(+) create mode 100644 Makefile create mode 100644 data/format.json create mode 100644 data/precision.json create mode 100644 data/states.json create mode 100644 grammar.md create mode 100644 mypy.ini create mode 100644 ruff.toml create mode 100644 scripts/tests/__init__.py create mode 100644 scripts/tests/test_data.py diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c51148a --- /dev/null +++ b/Makefile @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +.PHONY: check test lint types licences + +check: test lint types licences + +test: + python3 -m unittest discover -s scripts/tests -t scripts + +lint: + uvx ruff check scripts + uvx ruff format --check scripts + +types: + uvx mypy + +licences: + uvx --with charset-normalizer reuse lint diff --git a/REUSE.toml b/REUSE.toml index a074a60..c9b73b6 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -5,3 +5,9 @@ path = ["**/*.md", "**/*.json", "**/*.yml", "VERSION", "NOTICE", "templates/NOTI precedence = "aggregate" SPDX-FileCopyrightText = "2026 The Gatepost authors" SPDX-License-Identifier = "Apache-2.0" + +[[annotations]] +path = ["**/__pycache__/**"] +precedence = "aggregate" +SPDX-FileCopyrightText = "2026 The Gatepost authors" +SPDX-License-Identifier = "Apache-2.0" diff --git a/data/format.json b/data/format.json new file mode 100644 index 0000000..a4fa779 --- /dev/null +++ b/data/format.json @@ -0,0 +1,22 @@ +{ + "version": 1, + "segments": [ + { "name": "state", "length": 2, "characters": "letters" }, + { "name": "lga", "length": 2, "characters": "digits", "minimum": 1 }, + { "name": "district", "length": 3, "characters": "letters-or-digits" }, + { "name": "area", "length": 2, "characters": "letters" }, + { "name": "unit", "length": 2, "characters": "digits", "minimum": 1 } + ], + "legacyPattern": "^[0-9]{6}$", + "separators": [ + "U+0009", "U+000A", "U+000B", "U+000C", "U+000D", "U+0020", "U+002D", "U+002E", + "U+0085", "U+00A0", "U+1680", "U+2000", "U+2001", "U+2002", "U+2003", "U+2004", + "U+2005", "U+2006", "U+2007", "U+2008", "U+2009", "U+200A", "U+200B", "U+200C", + "U+200D", "U+2010", "U+2011", "U+2012", "U+2013", "U+2014", "U+2015", "U+2028", + "U+2029", "U+202F", "U+205F", "U+2060", "U+2212", "U+3000", "U+FEFF" + ], + "suggestions": { + "digits": { "O": "0", "I": "1", "L": "1" }, + "letters": { "0": "O", "1": "I" } + } +} diff --git a/data/precision.json b/data/precision.json new file mode 100644 index 0000000..7411fbc --- /dev/null +++ b/data/precision.json @@ -0,0 +1,10 @@ +{ + "version": 1, + "source": "The GPS rule in NIPOST's postcode widget docs.", + "thresholds": [ + { "maxAccuracyM": 8, "precision": "unit" }, + { "maxAccuracyM": 20, "precision": "area" }, + { "maxAccuracyM": 50, "precision": "district" } + ], + "fallback": "lga" +} diff --git a/data/states.json b/data/states.json new file mode 100644 index 0000000..e7f419c --- /dev/null +++ b/data/states.json @@ -0,0 +1,43 @@ +{ + "version": 1, + "source": "ISO 3166-2:NG. All 11 state codes in NIPOST's published examples match it. A keyed autocomplete call must confirm the full list.", + "states": [ + { "code": "AB", "name": "Abia" }, + { "code": "AD", "name": "Adamawa" }, + { "code": "AK", "name": "Akwa Ibom" }, + { "code": "AN", "name": "Anambra" }, + { "code": "BA", "name": "Bauchi" }, + { "code": "BE", "name": "Benue" }, + { "code": "BO", "name": "Borno" }, + { "code": "BY", "name": "Bayelsa" }, + { "code": "CR", "name": "Cross River" }, + { "code": "DE", "name": "Delta" }, + { "code": "EB", "name": "Ebonyi" }, + { "code": "ED", "name": "Edo" }, + { "code": "EK", "name": "Ekiti" }, + { "code": "EN", "name": "Enugu" }, + { "code": "FC", "name": "Federal Capital Territory" }, + { "code": "GO", "name": "Gombe" }, + { "code": "IM", "name": "Imo" }, + { "code": "JI", "name": "Jigawa" }, + { "code": "KD", "name": "Kaduna" }, + { "code": "KE", "name": "Kebbi" }, + { "code": "KN", "name": "Kano" }, + { "code": "KO", "name": "Kogi" }, + { "code": "KT", "name": "Katsina" }, + { "code": "KW", "name": "Kwara" }, + { "code": "LA", "name": "Lagos" }, + { "code": "NA", "name": "Nasarawa" }, + { "code": "NI", "name": "Niger" }, + { "code": "OG", "name": "Ogun" }, + { "code": "ON", "name": "Ondo" }, + { "code": "OS", "name": "Osun" }, + { "code": "OY", "name": "Oyo" }, + { "code": "PL", "name": "Plateau" }, + { "code": "RI", "name": "Rivers" }, + { "code": "SO", "name": "Sokoto" }, + { "code": "TA", "name": "Taraba" }, + { "code": "YO", "name": "Yobe" }, + { "code": "ZA", "name": "Zamfara" } + ] +} diff --git a/grammar.md b/grammar.md new file mode 100644 index 0000000..5e84f2a --- /dev/null +++ b/grammar.md @@ -0,0 +1,75 @@ +# Postcode grammar + +Spec version 0.1.0. This file defines how every Gatepost SDK reads and writes Nigeria's digital postcodes. The values live in `data/`. The vectors in `vectors/` test every rule. + +## Forms + +| Form | Example | Use | +|---|---|---| +| Compact | `EK01A03FK01` | storage and comparison | +| Canonical | `EK-01-A03-FK-01` | APIs and logs | +| Display | `EK 01 A03 FK 01` | screens and print | + +## Segments + +| Segment | Length | Characters | Rule | +|---|---|---|---| +| state | 2 | letters | one of the 37 codes in `data/states.json` | +| lga | 2 | digits | 01 to 99 | +| district | 3 | letters or digits | any combination | +| area | 2 | letters | | +| unit | 2 | digits | 01 to 99 | + +A partial postcode stops after the state, LGA, district or area segment. Its length is 2, 4, 7 or 9 characters. Its precision is the last segment that it contains. + +## Normalise + +1. Apply Unicode NFKC. This changes full-width letters and digits to ASCII. +2. Remove each separator in `data/format.json`: white space, hyphens and dashes, the full stop, and zero-width characters. +3. Change the ASCII letters a to z to upper case. Keep every other character as it is. + +## Parse + +Normalise the input. Then apply these checks in order. The first check that fails gives the error code. + +1. `empty`: no character is left. +2. `legacy_code`: exactly 6 digits are left. These are old NIPOST postcodes. +3. `bad_character`: a character other than A to Z and 0 to 9 is left. +4. `bad_length`: the length is not 11. With `allowPartial`, the lengths 2, 4, 7 and 9 also pass. +5. `unknown_state`: the first two characters are not a code in `data/states.json`. +6. `bad_segment`: a segment breaks its rule in the table above. Check the segments in order, and report the first one that fails. + +For `unknown_state` and `bad_segment`, the error names the segment. For the other codes, the segment is null. + +## Suggestions + +For `unknown_state` and `bad_segment`, build one corrected code: + +- In the letter segments, state and area, change 0 to O and 1 to I. +- In the digit segments, LGA and unit, change O to 0, and change I and L to 1. +- Never change the district. NIPOST has not confirmed which characters it uses there. + +Parse the corrected code with the same options. If it parses, the suggestion is its canonical form. If it does not parse, or if no character changed, the suggestion is null. A suggestion is a hint only. `parse` never returns a corrected code as a success. + +## Legacy codes + +`isLegacy` is true when the normalised input is exactly 6 digits. + +## Hierarchy + +- `truncate(code, to)` keeps the segments up to `to`. It fails when `to` is more precise than the code. Each language uses its standard error for a programmer mistake, for example `RangeError` in TypeScript. +- `parent(code)` returns the code with one segment fewer. A state code has no parent. +- `contains(prefix, code)` is true when the code has every segment of the prefix, in the same places. +- `redact(code)` replaces the unit with `**` in the canonical form. A code without a unit stays the same. + +## State names + +`stateName(code)` returns the name in `data/states.json`, and it ignores letter case. An unknown code gives null. The codes follow ISO 3166-2:NG. All 11 state codes in NIPOST's published examples match it. A keyed autocomplete call must confirm the full list. + +## GPS precision + +`precisionForAccuracy(metres)` returns the most precise segment that a GPS fix of that accuracy supports, from `data/precision.json`. An unknown, negative, infinite or NaN accuracy gives `lga`. + +## Versions + +`VERSION` holds the spec version. Each SDK exposes it as `SPEC_VERSION`. Each vector file carries a format version, which is 1. diff --git a/mypy.ini b/mypy.ini new file mode 100644 index 0000000..d9ed64d --- /dev/null +++ b/mypy.ini @@ -0,0 +1,6 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +[mypy] +strict = True +mypy_path = scripts +files = scripts diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..b9ec627 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +line-length = 100 +target-version = "py311" + +[lint] +select = [ + "E", "F", "W", "I", "B", "UP", "SIM", "RUF", "D", "ANN", "PL", "C4", "PIE", "RET", "ERA", "T20", "S", +] +# S603 and S607: check-tells runs git with a fixed argument list. +# ANN401: JSON values in the vector builder are Any. +ignore = ["S603", "S607", "ANN401"] + +[lint.pydocstyle] +convention = "google" + +[lint.per-file-ignores] +"scripts/tests/*" = ["D", "PLR2004", "S101"] diff --git a/scripts/tests/__init__.py b/scripts/tests/__init__.py new file mode 100644 index 0000000..7cb53b7 --- /dev/null +++ b/scripts/tests/__init__.py @@ -0,0 +1,2 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 diff --git a/scripts/tests/test_data.py b/scripts/tests/test_data.py new file mode 100644 index 0000000..02b71ab --- /dev/null +++ b/scripts/tests/test_data.py @@ -0,0 +1,56 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +"""Tests for the shared data files.""" + +import json +import re +import unittest +from pathlib import Path +from typing import Any + +DATA = Path(__file__).resolve().parents[2] / "data" +PRECISIONS = ["state", "lga", "district", "area", "unit"] + + +def load(name: str) -> dict[str, Any]: + value: object = json.loads((DATA / name).read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError(f"{name} must hold a JSON object.") + return value + + +class StatesTest(unittest.TestCase): + def test_lists_37_states_in_code_order(self) -> None: + codes = [state["code"] for state in load("states.json")["states"]] + self.assertEqual(len(codes), 37) + self.assertEqual(codes, sorted(set(codes))) + + def test_uses_two_capital_letters_for_each_code(self) -> None: + for state in load("states.json")["states"]: + with self.subTest(code=state["code"]): + self.assertRegex(state["code"], r"^[A-Z]{2}$") + self.assertTrue(state["name"].isascii() and state["name"].strip()) + + +class PrecisionTest(unittest.TestCase): + def test_raises_the_limit_as_precision_falls(self) -> None: + data = load("precision.json") + limits = [row["maxAccuracyM"] for row in data["thresholds"]] + self.assertEqual(limits, sorted(limits)) + names = [row["precision"] for row in data["thresholds"]] + [data["fallback"]] + self.assertEqual(names, ["unit", "area", "district", "lga"]) + + +class FormatTest(unittest.TestCase): + def test_names_five_segments_of_11_characters(self) -> None: + segments = load("format.json")["segments"] + self.assertEqual([segment["name"] for segment in segments], PRECISIONS) + self.assertEqual(sum(segment["length"] for segment in segments), 11) + + def test_writes_each_separator_as_a_code_point(self) -> None: + for label in load("format.json")["separators"]: + with self.subTest(label=label): + self.assertRegex(label, r"^U\+[0-9A-F]{4}$") + + def test_compiles_the_legacy_pattern(self) -> None: + self.assertTrue(re.fullmatch(load("format.json")["legacyPattern"], "900108")) From c5ee5f8210ecb58e023b0f5ec6fe32a0cb0d1ed8 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Thu, 1 Oct 2026 22:30:22 +0100 Subject: [PATCH 03/45] fix: apply Task 2 review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Remove REUSE.toml __pycache__ annotation (ignored by .gitignore) - Split ruff.toml select list to 100 column limit - Add Python.md Ruff settings: preview rules, extend-select, C90, T10 - Add mccabe max-complexity and pylint max-nested-blocks settings - Update per-file-ignores to scripts/tests/** with PT rule excluded - Rename generic test variables for clarity: data → precision_rules, value → document, PRECISIONS → SEGMENT_NAMES - Fix line-too-long in test by extracting intermediate variable - Fix collection-literal-concatenation using unpacking syntax Signed-off-by: MobileMage --- Makefile | 2 +- REUSE.toml | 6 ------ ruff.toml | 20 ++++++++++++++++---- scripts/tests/test_data.py | 17 +++++++++-------- 4 files changed, 26 insertions(+), 19 deletions(-) diff --git a/Makefile b/Makefile index c51148a..bc88b73 100644 --- a/Makefile +++ b/Makefile @@ -15,4 +15,4 @@ types: uvx mypy licences: - uvx --with charset-normalizer reuse lint + uvx reuse lint diff --git a/REUSE.toml b/REUSE.toml index c9b73b6..a074a60 100644 --- a/REUSE.toml +++ b/REUSE.toml @@ -5,9 +5,3 @@ path = ["**/*.md", "**/*.json", "**/*.yml", "VERSION", "NOTICE", "templates/NOTI precedence = "aggregate" SPDX-FileCopyrightText = "2026 The Gatepost authors" SPDX-License-Identifier = "Apache-2.0" - -[[annotations]] -path = ["**/__pycache__/**"] -precedence = "aggregate" -SPDX-FileCopyrightText = "2026 The Gatepost authors" -SPDX-License-Identifier = "Apache-2.0" diff --git a/ruff.toml b/ruff.toml index b9ec627..f39bbe8 100644 --- a/ruff.toml +++ b/ruff.toml @@ -1,18 +1,30 @@ # SPDX-FileCopyrightText: 2026 The Gatepost authors # SPDX-License-Identifier: Apache-2.0 -line-length = 100 target-version = "py311" +line-length = 100 [lint] +preview = true +explicit-preview-rules = true select = [ - "E", "F", "W", "I", "B", "UP", "SIM", "RUF", "D", "ANN", "PL", "C4", "PIE", "RET", "ERA", "T20", "S", + "E", "F", "W", "I", "B", "UP", "SIM", "RUF", "D", "ANN", "S", "PT", + "PL", "C4", "PIE", "RET", "ERA", "T20", "T10", "C90", ] +extend-select = ["PLR1702", "PLR0917"] # S603 and S607: check-tells runs git with a fixed argument list. # ANN401: JSON values in the vector builder are Any. -ignore = ["S603", "S607", "ANN401"] +# PLR0913: TELL-3 counts only positional parameters. +ignore = ["S603", "S607", "ANN401", "PLR0913"] + +[lint.mccabe] +max-complexity = 10 + +[lint.pylint] +max-nested-blocks = 3 +max-positional-args = 4 [lint.pydocstyle] convention = "google" [lint.per-file-ignores] -"scripts/tests/*" = ["D", "PLR2004", "S101"] +"scripts/tests/**" = ["S101", "D", "ANN", "PLR2004", "PT"] diff --git a/scripts/tests/test_data.py b/scripts/tests/test_data.py index 02b71ab..a8eaff3 100644 --- a/scripts/tests/test_data.py +++ b/scripts/tests/test_data.py @@ -9,14 +9,14 @@ from typing import Any DATA = Path(__file__).resolve().parents[2] / "data" -PRECISIONS = ["state", "lga", "district", "area", "unit"] +SEGMENT_NAMES = ["state", "lga", "district", "area", "unit"] def load(name: str) -> dict[str, Any]: - value: object = json.loads((DATA / name).read_text(encoding="utf-8")) - if not isinstance(value, dict): + document: object = json.loads((DATA / name).read_text(encoding="utf-8")) + if not isinstance(document, dict): raise TypeError(f"{name} must hold a JSON object.") - return value + return document class StatesTest(unittest.TestCase): @@ -34,17 +34,18 @@ def test_uses_two_capital_letters_for_each_code(self) -> None: class PrecisionTest(unittest.TestCase): def test_raises_the_limit_as_precision_falls(self) -> None: - data = load("precision.json") - limits = [row["maxAccuracyM"] for row in data["thresholds"]] + precision_rules = load("precision.json") + limits = [row["maxAccuracyM"] for row in precision_rules["thresholds"]] self.assertEqual(limits, sorted(limits)) - names = [row["precision"] for row in data["thresholds"]] + [data["fallback"]] + thresholds = [row["precision"] for row in precision_rules["thresholds"]] + names = [*thresholds, precision_rules["fallback"]] self.assertEqual(names, ["unit", "area", "district", "lga"]) class FormatTest(unittest.TestCase): def test_names_five_segments_of_11_characters(self) -> None: segments = load("format.json")["segments"] - self.assertEqual([segment["name"] for segment in segments], PRECISIONS) + self.assertEqual([segment["name"] for segment in segments], SEGMENT_NAMES) self.assertEqual(sum(segment["length"] for segment in segments), 11) def test_writes_each_separator_as_a_code_point(self) -> None: From 24b25dc460eb83339b3dc565d8acdbfc255d760e Mon Sep 17 00:00:00 2001 From: MobileMage Date: Thu, 1 Oct 2026 22:33:56 +0100 Subject: [PATCH 04/45] fix: restore the encoding detector for the licence check Signed-off-by: MobileMage --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index bc88b73..2eb6f5b 100644 --- a/Makefile +++ b/Makefile @@ -14,5 +14,6 @@ lint: types: uvx mypy +# reuse needs an encoding detector, and uvx installs reuse without one. licences: - uvx reuse lint + uvx --with charset-normalizer reuse lint From 6a97d50336683d0d6690dc183cca10640f96ce45 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Thu, 1 Oct 2026 22:47:58 +0100 Subject: [PATCH 05/45] test: name the data folder and the threshold precisions exactly TELL-6 asks for names that say what a value holds. DATA held the path of the data folder, and thresholds held precision names, not limits. ruff.toml now also says why the tests turn off the pytest-style rules. Signed-off-by: MobileMage --- ruff.toml | 1 + scripts/tests/test_data.py | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ruff.toml b/ruff.toml index f39bbe8..fe1955b 100644 --- a/ruff.toml +++ b/ruff.toml @@ -27,4 +27,5 @@ max-positional-args = 4 convention = "google" [lint.per-file-ignores] +# PT: these tests use unittest, so the pytest-style rules do not apply. "scripts/tests/**" = ["S101", "D", "ANN", "PLR2004", "PT"] diff --git a/scripts/tests/test_data.py b/scripts/tests/test_data.py index a8eaff3..215cdb4 100644 --- a/scripts/tests/test_data.py +++ b/scripts/tests/test_data.py @@ -8,12 +8,12 @@ from pathlib import Path from typing import Any -DATA = Path(__file__).resolve().parents[2] / "data" +DATA_DIR = Path(__file__).resolve().parents[2] / "data" SEGMENT_NAMES = ["state", "lga", "district", "area", "unit"] def load(name: str) -> dict[str, Any]: - document: object = json.loads((DATA / name).read_text(encoding="utf-8")) + document: object = json.loads((DATA_DIR / name).read_text(encoding="utf-8")) if not isinstance(document, dict): raise TypeError(f"{name} must hold a JSON object.") return document @@ -37,8 +37,8 @@ def test_raises_the_limit_as_precision_falls(self) -> None: precision_rules = load("precision.json") limits = [row["maxAccuracyM"] for row in precision_rules["thresholds"]] self.assertEqual(limits, sorted(limits)) - thresholds = [row["precision"] for row in precision_rules["thresholds"]] - names = [*thresholds, precision_rules["fallback"]] + threshold_precisions = [row["precision"] for row in precision_rules["thresholds"]] + names = [*threshold_precisions, precision_rules["fallback"]] self.assertEqual(names, ["unit", "area", "district", "lga"]) From 66d880666ed86a81cc9f04a895e79a8d60926029 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Thu, 1 Oct 2026 23:02:09 +0100 Subject: [PATCH 06/45] feat: add check-tells for agent tells that linters miss Linters miss some of the agent tells in the coding standards, such as long lines in config files, debug calls in shipped code, emoji and files named utils. check-tells finds them in every tracked file, and make check now runs it. The issue template wraps two long lines, and the TELL-1 text now names the exemption for JSON data files, so the repo passes its own check. Signed-off-by: MobileMage --- Makefile | 7 +- scripts/check-tells | 13 + scripts/check_tells.py | 388 ++++++++++++++++++++++++ scripts/tests/test_check_tells.py | 185 +++++++++++ standards/CODING_STANDARDS.md | 2 +- templates/ISSUE_TEMPLATE/bug_report.yml | 9 +- 6 files changed, 598 insertions(+), 6 deletions(-) create mode 100755 scripts/check-tells create mode 100644 scripts/check_tells.py create mode 100644 scripts/tests/test_check_tells.py diff --git a/Makefile b/Makefile index 2eb6f5b..a0afd8a 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,15 @@ # SPDX-FileCopyrightText: 2026 The Gatepost authors # SPDX-License-Identifier: Apache-2.0 -.PHONY: check test lint types licences +.PHONY: check test tells lint types licences -check: test lint types licences +check: test tells lint types licences test: python3 -m unittest discover -s scripts/tests -t scripts +tells: + python3 scripts/check-tells + lint: uvx ruff check scripts uvx ruff format --check scripts diff --git a/scripts/check-tells b/scripts/check-tells new file mode 100755 index 0000000..a7dc92a --- /dev/null +++ b/scripts/check-tells @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +"""Run check_tells.py. The rules are in that file.""" + +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parent)) + +from check_tells import main + +raise SystemExit(main()) diff --git a/scripts/check_tells.py b/scripts/check_tells.py new file mode 100644 index 0000000..5e913b4 --- /dev/null +++ b/scripts/check_tells.py @@ -0,0 +1,388 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +"""Find the agent tells that language tools miss. + +The rules come from standards/CODING_STANDARDS.md: TELL-1, TELL-7, TELL-13, TELL-14 +and TELL-18, and CS-6 for the form of to-do comments. +""" + +from __future__ import annotations + +import argparse +import re +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path, PurePosixPath +from typing import Literal + +Kind = Literal["code", "config", "prose", "catalogue", "exempt", "other"] + +MAX_LINE = 100 +TAB_WIDTH = 4 +MAX_SUBJECT = 72 +LAST_ASCII = 0x7F +GENERATED_MARKERS = ("@generated", "DO NOT EDIT") +SIGN_OFF_PREFIXES = ("Signed-off-by:", "Co-authored-by:") + +CODE_EXTENSIONS = frozenset( + { + ".ts", + ".tsx", + ".mts", + ".cts", + ".js", + ".jsx", + ".mjs", + ".cjs", + ".php", + ".py", + ".go", + ".kt", + ".kts", + ".java", + ".cs", + ".dart", + ".swift", + ".sql", + ".sh", + ".bash", + } +) +CONFIG_EXTENSIONS = frozenset( + {".json", ".yml", ".yaml", ".toml", ".xml", ".ini", ".cfg", ".properties", ".gradle"} +) +CONFIG_NAMES = frozenset( + {"Makefile", "Dockerfile", ".editorconfig", ".gitignore", ".gitattributes", ".npmrc"} +) +PROSE_EXTENSIONS = frozenset({".md", ".mdx"}) +EXEMPT_EXTENSIONS = frozenset( + {".svg", ".lock", ".csv", ".txt", ".api", ".snap", ".png", ".jpg", ".ico"} +) +LOCKFILES = frozenset( + { + "pnpm-lock.yaml", + "package-lock.json", + "yarn.lock", + "composer.lock", + "go.sum", + "uv.lock", + "poetry.lock", + "Cargo.lock", + "Package.resolved", + "pubspec.lock", + "gradle.lockfile", + "packages.lock.json", + } +) +CATALOGUE_FOLDERS = frozenset({"locales", "i18n", "l10n", "messages"}) +DATA_FOLDERS = frozenset({"data", "vectors"}) +CATALOGUE_EXTENSIONS = frozenset({".arb", ".po", ".xlf", ".xliff", ".strings", ".stringsdict"}) +FORBIDDEN_STEMS = frozenset({"utils", "helpers", "common", "misc"}) +SKIPPABLE_RULES = frozenset({"TELL-1", "TELL-13", "TELL-14", "CS-6"}) + +LANGUAGE_BY_EXTENSION = { + ".ts": "javascript", + ".tsx": "javascript", + ".mts": "javascript", + ".cts": "javascript", + ".js": "javascript", + ".jsx": "javascript", + ".mjs": "javascript", + ".cjs": "javascript", + ".php": "php", + ".py": "python", + ".go": "go", + ".kt": "jvm", + ".kts": "jvm", + ".java": "jvm", + ".cs": "dotnet", + ".dart": "dart", + ".swift": "swift", +} +LANGUAGE_BY_INTERPRETER = {"python3": "python", "python": "python", "node": "javascript"} + +DEBUG_CALLS: dict[str, tuple[re.Pattern[str], ...]] = { + "javascript": (re.compile(r"\bconsole\.(log|debug|dir)\s*\("), re.compile(r"\bdebugger\b")), + "php": ( + re.compile( + r"(?$:])(var_dump|print_r|var_export|dd|dump|error_log|debug_print_backtrace)" + r"\s*\(" + ), + ), + "python": ( + re.compile(r"(? str: + """Format the violation like a compiler message.""" + return f"{self.path}:{self.line}:{self.column}: {self.rule} {self.message}" + + +@dataclass(frozen=True) +class FileInfo: + """What a file holds, which decides the rules that apply to it.""" + + path: str + kind: Kind + language: str | None + is_test: bool + is_generated: bool + + +def classify(path: str, text: str) -> FileInfo: + """Decide the kind and the language of one file.""" + pure = PurePosixPath(path) + head = "\n".join(text.splitlines()[:5]) + return FileInfo( + path=path, + kind=_kind(pure, text), + language=_language(pure, text), + is_test=bool(TEST_PATH.search(path)), + is_generated=any(marker in head for marker in GENERATED_MARKERS), + ) + + +def _kind(pure: PurePosixPath, text: str) -> Kind: + suffix = pure.suffix.lower() + if pure.name in LOCKFILES or suffix in EXEMPT_EXTENSIONS: + return "exempt" + if _is_catalogue(pure): + return "catalogue" + if suffix in PROSE_EXTENSIONS: + return "prose" + if suffix in CODE_EXTENSIONS or (not suffix and text.startswith("#!")): + return "code" + if suffix in CONFIG_EXTENSIONS or pure.name in CONFIG_NAMES: + return "config" + return "other" + + +def _is_catalogue(pure: PurePosixPath) -> bool: + in_folder = any(part in CATALOGUE_FOLDERS for part in pure.parts[:-1]) + android = pure.name == "strings.xml" and pure.parent.name.startswith("values") + return in_folder or android or pure.suffix.lower() in CATALOGUE_EXTENSIONS + + +def _language(pure: PurePosixPath, text: str) -> str | None: + if pure.suffix: + return LANGUAGE_BY_EXTENSION.get(pure.suffix.lower()) + first_line = text.splitlines()[0] if text else "" + if not first_line.startswith("#!"): + return None + interpreter = first_line.split()[-1].rsplit("/", 1)[-1] + return LANGUAGE_BY_INTERPRETER.get(interpreter) + + +def _is_emoji(character: str) -> bool: + return any(low <= ord(character) <= high for low, high in EMOJI_RANGES) + + +def check_text(path: str, text: str) -> list[Violation]: + """Check the name and every line of one file.""" + info = classify(path, text) + violations = _check_name(info) + for number, line in enumerate(text.splitlines(), start=1): + violations.extend(_check_line(info, number, line)) + return violations + + +def _check_name(info: FileInfo) -> list[Violation]: + stem = PurePosixPath(info.path).name.split(".")[0].lower() + if stem not in FORBIDDEN_STEMS: + return [] + message = "Name the file after the concept that it holds, not utils, helpers, common or misc." + return [Violation(info.path, 1, 1, "TELL-7", message)] + + +def _check_line(info: FileInfo, number: int, line: str) -> list[Violation]: + skipped, problems = _skips(info.path, number, line) + found = [ + violation + for check in (_line_length, _todo_form, _debug_calls, _characters) + for violation in check(info, number, line) + if violation.rule not in skipped + ] + return problems + found + + +def _skips(path: str, number: int, line: str) -> tuple[set[str], list[Violation]]: + skipped: set[str] = set() + problems: list[Violation] = [] + for match in SKIP.finditer(line): + rule, reason = match.group(1), match.group(2) + if rule in SKIPPABLE_RULES and reason: + skipped.add(rule) + continue + message = ( + f"Skip only {', '.join(sorted(SKIPPABLE_RULES))}, and give a reason after because." + ) + problems.append(Violation(path, number, match.start() + 1, "SKIP", message)) + return skipped, problems + + +def _line_length(info: FileInfo, number: int, line: str) -> list[Violation]: + if info.kind not in ("code", "config") or info.is_generated or _is_data_json(info.path): + return [] + width = len(line.expandtabs(TAB_WIDTH)) + if width <= MAX_LINE or URL.search(line): + return [] + message = f"The line is {width} characters wide. Keep lines at {MAX_LINE} or fewer." + return [Violation(info.path, number, MAX_LINE + 1, "TELL-1", message)] + + +def _is_data_json(path: str) -> bool: + pure = PurePosixPath(path) + in_folder = any(part in DATA_FOLDERS for part in pure.parts[:-1]) + return in_folder and pure.suffix.lower() == ".json" + + +def _todo_form(info: FileInfo, number: int, line: str) -> list[Violation]: + if info.kind != "code": + return [] + message = "Link an issue in the form TODO(#123)." + return [ + Violation(info.path, number, match.start() + 1, "CS-6", message) + for match in TODO_FORM.finditer(line) + ] + + +def _debug_calls(info: FileInfo, number: int, line: str) -> list[Violation]: + if info.kind != "code" or info.is_test or info.language is None or COMMENT_LINE.match(line): + return [] + patterns = DEBUG_CALLS.get(info.language, ()) + return [ + Violation(info.path, number, match.start() + 1, "TELL-13", f"Remove {match.group(0)!r}.") + for pattern in patterns + for match in pattern.finditer(line) + ] + + +def _characters(info: FileInfo, number: int, line: str) -> list[Violation]: + if info.kind == "exempt": + return [] + found = [ + Violation(info.path, number, index + 1, "TELL-14", f"Remove the emoji U+{ord(char):04X}.") + for index, char in enumerate(line) + if _is_emoji(char) + ] + if found or info.kind not in ("code", "config"): + return found + for index, char in enumerate(line): + if ord(char) > LAST_ASCII: + message = f"Write U+{ord(char):04X} as an escape sequence. Code and config use ASCII." + return [Violation(info.path, number, index + 1, "TELL-14", message)] + return [] + + +def check_commit_message(text: str) -> list[Violation]: + """Check the subject length and the characters of a commit message.""" + lines = [line for line in text.splitlines() if not line.startswith("#")] + violations: list[Violation] = [] + subject = next((line for line in lines if line.strip()), "") + if len(subject) > MAX_SUBJECT: + message = f"The subject has {len(subject)} characters. Use {MAX_SUBJECT} or fewer." + violations.append(Violation("commit message", 1, MAX_SUBJECT + 1, "TELL-18", message)) + for number, line in enumerate(lines, start=1): + for index, char in enumerate(line): + if _is_emoji(char) or ( + ord(char) > LAST_ASCII and not line.startswith(SIGN_OFF_PREFIXES) + ): + message = f"Remove U+{ord(char):04X}. Commit messages use ASCII." + violations.append( + Violation("commit message", number, index + 1, "TELL-14", message) + ) + break + return violations + + +def tracked_files(root: Path) -> list[Path]: + """List the files that git tracks, or would track, under root.""" + command = [ + "git", + "-C", + str(root), + "ls-files", + "-z", + "--cached", + "--others", + "--exclude-standard", + ] + names = subprocess.run(command, check=True, capture_output=True).stdout.decode().split("\0") + return [root / name for name in names if name and (root / name).is_file()] + + +def check_files(root: Path, files: list[Path]) -> list[Violation]: + """Check each file. Skip files that are not UTF-8 text, because they have no lines.""" + violations: list[Violation] = [] + for file in files: + try: + text = file.read_text(encoding="utf-8") + except UnicodeDecodeError: + continue + violations.extend(check_text(file.relative_to(root).as_posix(), text)) + return violations + + +def main(argv: list[str] | None = None) -> int: + """Run the checks, print each violation and return 1 if any rule broke.""" + parser = argparse.ArgumentParser( + prog="check-tells", description="Find the agent tells that language tools miss." + ) + parser.add_argument("--root", type=Path, default=Path.cwd(), help="The repo root.") + parser.add_argument("--commit-msg", type=Path, help="Check this commit message file.") + parser.add_argument("paths", nargs="*", type=Path, help="Files to check. Default: all.") + args = parser.parse_args(argv) + if args.commit_msg is not None: + violations = check_commit_message(args.commit_msg.read_text(encoding="utf-8")) + else: + root = args.root.resolve() + files = [(root / path).resolve() for path in args.paths] or tracked_files(root) + violations = check_files(root, files) + for violation in sorted(violations): + sys.stdout.write(f"{violation}\n") + return 1 if violations else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/test_check_tells.py b/scripts/tests/test_check_tells.py new file mode 100644 index 0000000..186cb65 --- /dev/null +++ b/scripts/tests/test_check_tells.py @@ -0,0 +1,185 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +"""Tests for check_tells.""" + +import contextlib +import io +import subprocess +import tempfile +import unittest +from pathlib import Path + +import check_tells +from check_tells import Violation, check_commit_message, check_text + +EMOJI = chr(0x1F600) +NO_BREAK_SPACE = chr(0x00A0) +E_ACUTE = chr(0x00E9) +# Built from two parts, so that this file passes its own to-do check. +TO_DO = "TO" + "DO" +# Built from two parts, so that this file passes its own skip check. +SKIP_MARKER = "check-tells: " + "allow" + + +def rules(violations: list[Violation]) -> list[str]: + return [violation.rule for violation in violations] + + +class LineLengthTest(unittest.TestCase): + def test_accepts_a_line_of_100_characters(self) -> None: + self.assertEqual(check_text("src/a.ts", "x" * 100), []) + + def test_rejects_a_line_of_101_characters(self) -> None: + found = check_text("src/a.ts", "x" * 101) + self.assertEqual(rules(found), ["TELL-1"]) + self.assertEqual(found[0].column, 101) + + def test_counts_a_tab_as_4_columns(self) -> None: + self.assertEqual(rules(check_text("Makefile", "\t" + "x" * 97)), ["TELL-1"]) + + def test_skips_a_line_with_a_url(self) -> None: + self.assertEqual(check_text("src/a.ts", "// https://example.com/" + "a" * 100), []) + + def test_skips_markdown_lockfiles_and_vectors(self) -> None: + self.assertEqual(check_text("README.md", "word " * 40), []) + self.assertEqual(check_text("pnpm-lock.yaml", "x" * 150), []) + self.assertEqual(check_text("vectors/parse.json", "x" * 150), []) + + def test_skips_json_in_a_data_folder(self) -> None: + self.assertEqual(check_text("data/states.json", "x" * 150), []) + self.assertEqual(check_text("spec/vectors/parse.json", "x" * 150), []) + + def test_checks_other_files_in_a_data_folder(self) -> None: + self.assertEqual(rules(check_text("data/rules.yml", "x" * 150)), ["TELL-1"]) + self.assertEqual(rules(check_text("vectors/build.py", "x" * 150)), ["TELL-1"]) + self.assertEqual(rules(check_text("package.json", "x" * 150)), ["TELL-1"]) + + def test_skips_a_generated_file(self) -> None: + text = "// @generated by a script\n" + "x" * 150 + self.assertEqual(check_text("src/spec-data.ts", text), []) + + def test_honours_a_skip_with_a_reason(self) -> None: + line = "x" * 120 + " // check-tells: allow TELL-1 because the regex cannot be split" + self.assertEqual(check_text("src/a.ts", line), []) + + +class FileNameTest(unittest.TestCase): + def test_rejects_utils_helpers_common_and_misc(self) -> None: + for name in ("src/utils.ts", "lib/helpers.py", "common.go", "pkg/Misc.kt"): + with self.subTest(name=name): + self.assertEqual(rules(check_text(name, "")), ["TELL-7"]) + + def test_accepts_a_folder_named_common(self) -> None: + self.assertEqual(check_text("common/parse.ts", ""), []) + + +class TodoFormTest(unittest.TestCase): + def test_rejects_a_to_do_without_an_issue(self) -> None: + self.assertEqual(rules(check_text("src/a.ts", f"// {TO_DO}: handle 00")), ["CS-6"]) + + def test_accepts_a_to_do_with_an_issue(self) -> None: + self.assertEqual(check_text("src/a.ts", f"// {TO_DO}(#12): handle 00"), []) + + def test_ignores_prose(self) -> None: + self.assertEqual(check_text("docs/notes.md", f"{TO_DO}: write this"), []) + + +class DebugCallTest(unittest.TestCase): + def test_rejects_a_debug_call_in_each_language(self) -> None: + cases = { + "src/a.ts": "console.log(code);", + "src/a.php": "var_dump($code);", + "pkg/a.py": "print(code)", + "a.go": "fmt.Println(code)", + "src/A.kt": "println(code)", + "src/A.cs": "Console.WriteLine(code);", + "lib/a.dart": "debugPrint(code);", + "Sources/A.swift": "dump(code)", + } + for path, line in cases.items(): + with self.subTest(path=path): + self.assertEqual(rules(check_text(path, line)), ["TELL-13"]) + + def test_accepts_methods_with_similar_names(self) -> None: + self.assertEqual(check_text("pkg/a.py", "report.print(code)"), []) + self.assertEqual(check_text("pkg/a.py", "pprint_table(code)"), []) + + def test_ignores_test_files_and_comment_lines(self) -> None: + self.assertEqual(check_text("example_test.go", 'fmt.Println("EK-01")'), []) + self.assertEqual(check_text("src/a.ts", " * console.log(parse(code));"), []) + + def test_reads_the_language_from_a_shebang(self) -> None: + self.assertEqual( + rules(check_text("scripts/run", "#!/usr/bin/env python3\nprint(1)")), ["TELL-13"] + ) + + +class CharacterTest(unittest.TestCase): + def test_rejects_an_emoji_in_markdown(self) -> None: + self.assertEqual(rules(check_text("README.md", f"Done {EMOJI}")), ["TELL-14"]) + + def test_rejects_non_ascii_in_code(self) -> None: + found = check_text("src/a.ts", f"const space = '{NO_BREAK_SPACE}';") + self.assertEqual(rules(found), ["TELL-14"]) + self.assertIn("U+00A0", found[0].message) + + def test_accepts_non_ascii_in_prose_and_catalogues(self) -> None: + self.assertEqual(check_text("docs/guide.md", f"Caf{E_ACUTE}"), []) + self.assertEqual(check_text("src/locales/yo.json", f'{{"title": "Caf{E_ACUTE}"}}'), []) + + def test_rejects_an_emoji_in_a_catalogue(self) -> None: + found = check_text("src/locales/en.json", f'{{"ok": "{EMOJI}"}}') + self.assertEqual(rules(found), ["TELL-14"]) + + +class SkipTest(unittest.TestCase): + def test_rejects_a_skip_without_a_reason(self) -> None: + line = "x" * 120 + f" // {SKIP_MARKER} TELL-1" + self.assertEqual(sorted(rules(check_text("src/a.ts", line))), ["SKIP", "TELL-1"]) + + def test_rejects_a_skip_of_a_rule_that_cannot_be_skipped(self) -> None: + line = f"const a = 1; // {SKIP_MARKER} TELL-7 because it is short" + self.assertEqual(rules(check_text("src/a.ts", line)), ["SKIP"]) + + +class CommitMessageTest(unittest.TestCase): + def test_accepts_a_good_message(self) -> None: + message = "fix(core): reject a unit of 00\n\nThe spec forbids 00.\n" + self.assertEqual(check_commit_message(message), []) + + def test_rejects_a_long_subject(self) -> None: + self.assertEqual(rules(check_commit_message("fix: " + "a" * 70)), ["TELL-18"]) + + def test_rejects_an_emoji(self) -> None: + self.assertEqual(rules(check_commit_message(f"fix: parse codes {EMOJI}")), ["TELL-14"]) + + def test_accepts_an_accented_name_in_a_sign_off(self) -> None: + message = f"fix: parse codes\n\nSigned-off-by: Ad{E_ACUTE} Bello \n" + self.assertEqual(check_commit_message(message), []) + + def test_ignores_git_comment_lines(self) -> None: + self.assertEqual(check_commit_message(f"fix: parse codes\n# {EMOJI} from git\n"), []) + + +class CommandTest(unittest.TestCase): + def test_checks_new_files_and_skips_ignored_ones(self) -> None: + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + subprocess.run(["git", "init", "-q", str(root)], check=True) + (root / ".gitignore").write_text("ignored.ts\n", encoding="utf-8") + (root / "ignored.ts").write_text("console.log(1);\n", encoding="utf-8") + (root / "new.ts").write_text("console.log(1);\n", encoding="utf-8") + output = io.StringIO() + with contextlib.redirect_stdout(output): + status = check_tells.main(["--root", str(root)]) + self.assertEqual(status, 1) + self.assertIn("new.ts:1:1: TELL-13", output.getvalue()) + self.assertNotIn("ignored.ts", output.getvalue()) + + def test_exits_with_0_for_a_clean_repo(self) -> None: + with tempfile.TemporaryDirectory() as folder: + root = Path(folder) + subprocess.run(["git", "init", "-q", str(root)], check=True) + (root / "clean.ts").write_text("export const a = 1;\n", encoding="utf-8") + command = [str(Path(__file__).resolve().parents[1] / "check-tells"), "--root", folder] + self.assertEqual(subprocess.run(command, check=False).returncode, 0) diff --git a/standards/CODING_STANDARDS.md b/standards/CODING_STANDARDS.md index f6211ef..1bdcffc 100644 --- a/standards/CODING_STANDARDS.md +++ b/standards/CODING_STANDARDS.md @@ -103,7 +103,7 @@ Coding agents repeat some habits so often that readers spot them at once. Each r `spec/scripts/check-tells` runs in the CI of every repo. It covers the tool rules that no language tool covers: -- **TELL-1.** It checks line length in source and config files, and it counts a tab as 4 columns. It skips lines with a URL, files marked as generated, lockfiles, `spec/vectors/` and Markdown prose. +- **TELL-1.** It checks line length in source and config files, and it counts a tab as 4 columns. It skips lines with a URL, files marked as generated, lockfiles, JSON files in a folder named `data` or `vectors`, and Markdown prose. - **TELL-7.** It rejects files named `utils`, `helpers`, `common` or `misc`, with any extension. - **TELL-12 and CS-6.** It rejects each `TODO` without the form `TODO(#123)`. - **TELL-13.** It rejects the debug calls that the language file lists. diff --git a/templates/ISSUE_TEMPLATE/bug_report.yml b/templates/ISSUE_TEMPLATE/bug_report.yml index bf4e1f8..f5ce424 100644 --- a/templates/ISSUE_TEMPLATE/bug_report.yml +++ b/templates/ISSUE_TEMPLATE/bug_report.yml @@ -4,9 +4,12 @@ labels: [bug, needs-triage] body: - type: markdown attributes: - value: | - Gatepost is unofficial. For problems with NIPOST's own app, data or API, contact NIPOST at postcode@nipost.gov.ng. - Do not paste API keys, NINs, email addresses or the postcodes of private homes. Use synthetic postcodes, such as FC-01-Z99-ZZ-01. + value: > + Gatepost is unofficial. For problems with NIPOST's own app, data or API, + contact NIPOST at postcode@nipost.gov.ng. + + Do not paste API keys, NINs, email addresses or the postcodes of private homes. + Use synthetic postcodes, such as FC-01-Z99-ZZ-01. - type: input id: package attributes: From 5f933797366f3434635298da6dda27e4e7b35a97 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Thu, 1 Oct 2026 23:07:43 +0100 Subject: [PATCH 07/45] fix: skip doctest examples in the TELL-13 check PY-11 asks for an Example section in each public docstring, and such examples often call print. TELL-13 now skips doctest lines, which start with >>> or ..., in the same way as it skips comment lines. The standards now say so, and PY-11 asks for doctest prompts. Signed-off-by: MobileMage --- scripts/check_tells.py | 3 ++- scripts/tests/test_check_tells.py | 4 ++++ standards/CODING_STANDARDS.md | 2 +- standards/languages/python.md | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/scripts/check_tells.py b/scripts/check_tells.py index 5e913b4..4a0a416 100644 --- a/scripts/check_tells.py +++ b/scripts/check_tells.py @@ -137,7 +137,8 @@ r"|(^|/)test_[^/]*\.py$" r"|Tests?\.(kt|java|cs|swift)$" ) -COMMENT_LINE = re.compile(r"^\s*(//|/\*|\*|#|--)") +# Doctest prompts (>>> and ...) start the examples in Python docstrings. +COMMENT_LINE = re.compile(r"^\s*(//|/\*|\*|#|--|>>>|\.\.\.)") URL = re.compile(r"https?://") TODO_FORM = re.compile(r"\bTODO\b(?!\(#\d+\))") SKIP = re.compile(r"check-tells: allow ([A-Z]+-\d+)(?: because (\S.*))?") diff --git a/scripts/tests/test_check_tells.py b/scripts/tests/test_check_tells.py index 186cb65..e0b5598 100644 --- a/scripts/tests/test_check_tells.py +++ b/scripts/tests/test_check_tells.py @@ -108,6 +108,10 @@ def test_ignores_test_files_and_comment_lines(self) -> None: self.assertEqual(check_text("example_test.go", 'fmt.Println("EK-01")'), []) self.assertEqual(check_text("src/a.ts", " * console.log(parse(code));"), []) + def test_ignores_doctest_examples(self) -> None: + self.assertEqual(check_text("pkg/a.py", ' >>> print(parse("EK01A03FK01"))'), []) + self.assertEqual(check_text("pkg/a.py", " ... print(code)"), []) + def test_reads_the_language_from_a_shebang(self) -> None: self.assertEqual( rules(check_text("scripts/run", "#!/usr/bin/env python3\nprint(1)")), ["TELL-13"] diff --git a/standards/CODING_STANDARDS.md b/standards/CODING_STANDARDS.md index 1bdcffc..44549f5 100644 --- a/standards/CODING_STANDARDS.md +++ b/standards/CODING_STANDARDS.md @@ -106,7 +106,7 @@ Coding agents repeat some habits so often that readers spot them at once. Each r - **TELL-1.** It checks line length in source and config files, and it counts a tab as 4 columns. It skips lines with a URL, files marked as generated, lockfiles, JSON files in a folder named `data` or `vectors`, and Markdown prose. - **TELL-7.** It rejects files named `utils`, `helpers`, `common` or `misc`, with any extension. - **TELL-12 and CS-6.** It rejects each `TODO` without the form `TODO(#123)`. -- **TELL-13.** It rejects the debug calls that the language file lists. +- **TELL-13.** It rejects the debug calls that the language file lists. It skips test files, comment lines and doctest lines that start with `>>>` or `...`. - **TELL-14.** It rejects emojis everywhere, including Markdown. It rejects other non-ASCII characters in source files, config files and commit messages. Message catalogues, docs prose and the sign-off lines of commit messages are exempt from the non-ASCII check, because contributors' names can need any script. - **TELL-18.** With `--commit-msg `, it checks the length of a commit subject. Repos without commitlint use this mode. diff --git a/standards/languages/python.md b/standards/languages/python.md index 22ada52..98e00dd 100644 --- a/standards/languages/python.md +++ b/standards/languages/python.md @@ -85,7 +85,7 @@ convention = "google" - **PY-8 MUST.** Durations are seconds as `float`, as in httpx. Names have no unit suffix: `timeout`, `cache_ttl` and `retry_after`. - **PY-9 MUST.** The async client lets `asyncio.CancelledError` pass through. It never catches it and never wraps it. - **PY-10 MUST.** Errors subclass `PostcodeError`, which subclasses `Exception`. A raised error keeps its cause with `raise ... from ...`. -- **PY-11 MUST.** Each public symbol has a Google-style docstring with an `Example:` section. **(tool)** +- **PY-11 MUST.** Each public symbol has a Google-style docstring with an `Example:` section. Write the example with doctest prompts (`>>>`). **(tool)** - **PY-12 MUST.** The package does not call `print`, `pprint`, `breakpoint` or `pdb.set_trace`. Importing a module has no side effects. **(tool)** - **PY-13 MUST.** `__all__` in `gatepost_postcode/__init__.py` lists every public symbol. Other modules start with an underscore. - **PY-14 MUST.** Each `noqa` and each `type: ignore` comment names the rule and gives a reason. From 4452d5c094217c7099bb01e33309a63026ba3220 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Thu, 1 Oct 2026 23:42:04 +0100 Subject: [PATCH 08/45] fix: close gaps in the check-tells rules check-tells missed Swift NSLog calls. It also skipped the debug, to-do and line-length checks on code files in a folder named messages or locales. Line numbers were wrong after comment lines in commit messages, and splitlines hid U+2028 and U+2029. Doctest prompts now count only in Python, the to-do check covers config files, skip markers count only in code and config, and two more markers flag generated files. New tests pin each path rule and each debug pattern. Signed-off-by: MobileMage --- scripts/check_tells.py | 109 +++++++++++++++++++-------- scripts/tests/test_check_tells.py | 118 ++++++++++++++++++++++++++++++ standards/CODING_STANDARDS.md | 6 +- 3 files changed, 201 insertions(+), 32 deletions(-) diff --git a/scripts/check_tells.py b/scripts/check_tells.py index 4a0a416..a5fec3f 100644 --- a/scripts/check_tells.py +++ b/scripts/check_tells.py @@ -17,13 +17,15 @@ from typing import Literal Kind = Literal["code", "config", "prose", "catalogue", "exempt", "other"] +NumberedLine = tuple[int, str] MAX_LINE = 100 TAB_WIDTH = 4 MAX_SUBJECT = 72 LAST_ASCII = 0x7F -GENERATED_MARKERS = ("@generated", "DO NOT EDIT") -SIGN_OFF_PREFIXES = ("Signed-off-by:", "Co-authored-by:") +GENERATED_MARKERS = ("@generated", "DO NOT EDIT", ">> and ...) start the examples in Python docstrings. -COMMENT_LINE = re.compile(r"^\s*(//|/\*|\*|#|--|>>>|\.\.\.)") +DOCTEST_LINE = re.compile(r"^\s*(>>>|\.\.\.)") URL = re.compile(r"https?://") TODO_FORM = re.compile(r"\bTODO\b(?!\(#\d+\))") SKIP = re.compile(r"check-tells: allow ([A-Z]+-\d+)(?: because (\S.*))?") -EMOJI_RANGES = ((0x1F000, 0x1FAFF), (0x2600, 0x27BF), (0x2B00, 0x2BFF), (0xFE0F, 0xFE0F)) +EMOJI_RANGES = ( + (0x1F000, 0x1FAFF), + (0x231A, 0x231B), + (0x23E9, 0x23F3), + (0x2600, 0x27BF), + (0x2B00, 0x2BFF), + (0xFE0F, 0xFE0F), +) @dataclass(frozen=True, order=True) @@ -171,10 +181,19 @@ class FileInfo: is_generated: bool +def _split_lines(text: str) -> list[str]: + """Split text at line feeds only, and remove one carriage return from each line. + + str.splitlines also splits at U+2028, U+2029, U+0085 and the form feed. That hides the + characters that TELL-14 looks for, and it moves the line numbers. + """ + return [line.removesuffix("\r") for line in text.split("\n")] + + def classify(path: str, text: str) -> FileInfo: """Decide the kind and the language of one file.""" pure = PurePosixPath(path) - head = "\n".join(text.splitlines()[:5]) + head = "\n".join(_split_lines(text)[:5]) return FileInfo( path=path, kind=_kind(pure, text), @@ -188,12 +207,12 @@ def _kind(pure: PurePosixPath, text: str) -> Kind: suffix = pure.suffix.lower() if pure.name in LOCKFILES or suffix in EXEMPT_EXTENSIONS: return "exempt" + if suffix in CODE_EXTENSIONS or (not suffix and text.startswith("#!")): + return "code" if _is_catalogue(pure): return "catalogue" if suffix in PROSE_EXTENSIONS: return "prose" - if suffix in CODE_EXTENSIONS or (not suffix and text.startswith("#!")): - return "code" if suffix in CONFIG_EXTENSIONS or pure.name in CONFIG_NAMES: return "config" return "other" @@ -208,7 +227,7 @@ def _is_catalogue(pure: PurePosixPath) -> bool: def _language(pure: PurePosixPath, text: str) -> str | None: if pure.suffix: return LANGUAGE_BY_EXTENSION.get(pure.suffix.lower()) - first_line = text.splitlines()[0] if text else "" + first_line = _split_lines(text)[0] if not first_line.startswith("#!"): return None interpreter = first_line.split()[-1].rsplit("/", 1)[-1] @@ -223,7 +242,7 @@ def check_text(path: str, text: str) -> list[Violation]: """Check the name and every line of one file.""" info = classify(path, text) violations = _check_name(info) - for number, line in enumerate(text.splitlines(), start=1): + for number, line in enumerate(_split_lines(text), start=1): violations.extend(_check_line(info, number, line)) return violations @@ -237,7 +256,10 @@ def _check_name(info: FileInfo) -> list[Violation]: def _check_line(info: FileInfo, number: int, line: str) -> list[Violation]: - skipped, problems = _skips(info.path, number, line) + skipped: set[str] = set() + problems: list[Violation] = [] + if info.kind in ("code", "config"): + skipped, problems = _skips(info.path, number, line) found = [ violation for check in (_line_length, _todo_form, _debug_calls, _characters) @@ -279,7 +301,7 @@ def _is_data_json(path: str) -> bool: def _todo_form(info: FileInfo, number: int, line: str) -> list[Violation]: - if info.kind != "code": + if info.kind not in ("code", "config"): return [] message = "Link an issue in the form TODO(#123)." return [ @@ -288,8 +310,14 @@ def _todo_form(info: FileInfo, number: int, line: str) -> list[Violation]: ] +def _is_comment(info: FileInfo, line: str) -> bool: + if COMMENT_LINE.match(line): + return True + return info.language == "python" and bool(DOCTEST_LINE.match(line)) + + def _debug_calls(info: FileInfo, number: int, line: str) -> list[Violation]: - if info.kind != "code" or info.is_test or info.language is None or COMMENT_LINE.match(line): + if info.kind != "code" or info.is_test or info.language is None or _is_comment(info, line): return [] patterns = DEBUG_CALLS.get(info.language, ()) return [ @@ -316,24 +344,47 @@ def _characters(info: FileInfo, number: int, line: str) -> list[Violation]: return [] +def _message_lines(text: str) -> list[NumberedLine]: + """Pair each line with its number. Drop git's comment lines, and stop at its scissors line.""" + numbered: list[NumberedLine] = [] + for number, line in enumerate(_split_lines(text), start=1): + if line.startswith("# ") and ">8" in line: + break + if not line.startswith("#"): + numbered.append((number, line)) + return numbered + + +def _check_subject(lines: list[NumberedLine]) -> list[Violation]: + """Check the subject, which is the first line that has text.""" + for number, subject in lines: + if not subject.strip(): + continue + if len(subject) <= MAX_SUBJECT: + return [] + message = f"The subject has {len(subject)} characters. Use {MAX_SUBJECT} or fewer." + return [Violation("commit message", number, MAX_SUBJECT + 1, "TELL-18", message)] + return [] + + +def _first_bad_character(number: int, line: str) -> Violation | None: + """Find the first emoji, or the first non-ASCII character outside a sign-off line.""" + allow_non_ascii = line.lower().startswith(SIGN_OFF_PREFIXES) + for index, char in enumerate(line): + if _is_emoji(char) or (ord(char) > LAST_ASCII and not allow_non_ascii): + message = f"Remove U+{ord(char):04X}. Commit messages use ASCII." + return Violation("commit message", number, index + 1, "TELL-14", message) + return None + + def check_commit_message(text: str) -> list[Violation]: """Check the subject length and the characters of a commit message.""" - lines = [line for line in text.splitlines() if not line.startswith("#")] - violations: list[Violation] = [] - subject = next((line for line in lines if line.strip()), "") - if len(subject) > MAX_SUBJECT: - message = f"The subject has {len(subject)} characters. Use {MAX_SUBJECT} or fewer." - violations.append(Violation("commit message", 1, MAX_SUBJECT + 1, "TELL-18", message)) - for number, line in enumerate(lines, start=1): - for index, char in enumerate(line): - if _is_emoji(char) or ( - ord(char) > LAST_ASCII and not line.startswith(SIGN_OFF_PREFIXES) - ): - message = f"Remove U+{ord(char):04X}. Commit messages use ASCII." - violations.append( - Violation("commit message", number, index + 1, "TELL-14", message) - ) - break + lines = _message_lines(text) + violations = _check_subject(lines) + for number, line in lines: + violation = _first_bad_character(number, line) + if violation is not None: + violations.append(violation) return violations diff --git a/scripts/tests/test_check_tells.py b/scripts/tests/test_check_tells.py index e0b5598..9a8ca5e 100644 --- a/scripts/tests/test_check_tells.py +++ b/scripts/tests/test_check_tells.py @@ -15,6 +15,8 @@ EMOJI = chr(0x1F600) NO_BREAK_SPACE = chr(0x00A0) E_ACUTE = chr(0x00E9) +EM_DASH = chr(0x2014) +LINE_SEPARATOR = chr(0x2028) # Built from two parts, so that this file passes its own to-do check. TO_DO = "TO" + "DO" # Built from two parts, so that this file passes its own skip check. @@ -54,10 +56,23 @@ def test_checks_other_files_in_a_data_folder(self) -> None: self.assertEqual(rules(check_text("vectors/build.py", "x" * 150)), ["TELL-1"]) self.assertEqual(rules(check_text("package.json", "x" * 150)), ["TELL-1"]) + def test_skips_a_message_catalogue(self) -> None: + line = '{"title": "' + "x" * 150 + '"}' + self.assertEqual(check_text("src/locales/en.json", line), []) + def test_skips_a_generated_file(self) -> None: text = "// @generated by a script\n" + "x" * 150 self.assertEqual(check_text("src/spec-data.ts", text), []) + def test_skips_files_that_other_tools_generate(self) -> None: + cases = { + "src/A.cs": "// \n", + "lib/a.g.dart": "// GENERATED CODE - DO NOT MODIFY BY HAND\n", + } + for path, header in cases.items(): + with self.subTest(path=path): + self.assertEqual(check_text(path, header + "x" * 150), []) + def test_honours_a_skip_with_a_reason(self) -> None: line = "x" * 120 + " // check-tells: allow TELL-1 because the regex cannot be split" self.assertEqual(check_text("src/a.ts", line), []) @@ -77,6 +92,10 @@ class TodoFormTest(unittest.TestCase): def test_rejects_a_to_do_without_an_issue(self) -> None: self.assertEqual(rules(check_text("src/a.ts", f"// {TO_DO}: handle 00")), ["CS-6"]) + def test_rejects_a_to_do_in_a_config_file(self) -> None: + found = check_text("Makefile", "# " + TO_DO + ": pin the version") + self.assertEqual(rules(found), ["CS-6"]) + def test_accepts_a_to_do_with_an_issue(self) -> None: self.assertEqual(check_text("src/a.ts", f"// {TO_DO}(#12): handle 00"), []) @@ -88,18 +107,30 @@ class DebugCallTest(unittest.TestCase): def test_rejects_a_debug_call_in_each_language(self) -> None: cases = { "src/a.ts": "console.log(code);", + "src/b.ts": "debugger;", "src/a.php": "var_dump($code);", "pkg/a.py": "print(code)", + "pkg/b.py": "pdb.set_trace()", + "pkg/c.py": "breakpoint()", "a.go": "fmt.Println(code)", + "b.go": 'log.Printf("x")', "src/A.kt": "println(code)", + "src/B.kt": 'Log.d(TAG, "x")', + "src/A.java": "System.out.println(x);", + "src/B.java": "e.printStackTrace();", "src/A.cs": "Console.WriteLine(code);", "lib/a.dart": "debugPrint(code);", "Sources/A.swift": "dump(code)", + "Sources/B.swift": 'NSLog("x")', } for path, line in cases.items(): with self.subTest(path=path): self.assertEqual(rules(check_text(path, line)), ["TELL-13"]) + def test_checks_a_code_file_in_a_catalogue_folder(self) -> None: + found = check_text("src/messages/format.ts", "console.log(1);") + self.assertEqual(rules(found), ["TELL-13"]) + def test_accepts_methods_with_similar_names(self) -> None: self.assertEqual(check_text("pkg/a.py", "report.print(code)"), []) self.assertEqual(check_text("pkg/a.py", "pprint_table(code)"), []) @@ -108,10 +139,40 @@ def test_ignores_test_files_and_comment_lines(self) -> None: self.assertEqual(check_text("example_test.go", 'fmt.Println("EK-01")'), []) self.assertEqual(check_text("src/a.ts", " * console.log(parse(code));"), []) + def test_ignores_each_kind_of_test_path(self) -> None: + cases = { + "test/parse.ts": "console.log(code);", + "src/__tests__/parse.ts": "console.log(code);", + "lib/parse_test.dart": "debugPrint(code);", + "src/parse.test.ts": "console.log(code);", + "src/parse.spec.mjs": "console.log(code);", + "test_parse.py": "print(code)", + "src/ParseTest.kt": "println(code)", + "Sources/ParseTests.swift": "dump(code)", + } + for path, line in cases.items(): + with self.subTest(path=path): + self.assertEqual(check_text(path, line), []) + + def test_ignores_each_kind_of_comment_start(self) -> None: + cases = [ + ("src/a.ts", "// console.log(x)"), + ("src/a.ts", "/* console.log(x)"), + ("src/a.ts", " * console.log(x)"), + ("pkg/a.py", "# print(x)"), + ] + for path, line in cases: + with self.subTest(line=line): + self.assertEqual(check_text(path, line), []) + def test_ignores_doctest_examples(self) -> None: self.assertEqual(check_text("pkg/a.py", ' >>> print(parse("EK01A03FK01"))'), []) self.assertEqual(check_text("pkg/a.py", " ... print(code)"), []) + def test_checks_a_spread_line_outside_python(self) -> None: + line = " ...items.map((item) => console.log(item))," + self.assertEqual(rules(check_text("src/a.ts", line)), ["TELL-13"]) + def test_reads_the_language_from_a_shebang(self) -> None: self.assertEqual( rules(check_text("scripts/run", "#!/usr/bin/env python3\nprint(1)")), ["TELL-13"] @@ -122,6 +183,10 @@ class CharacterTest(unittest.TestCase): def test_rejects_an_emoji_in_markdown(self) -> None: self.assertEqual(rules(check_text("README.md", f"Done {EMOJI}")), ["TELL-14"]) + def test_rejects_an_hourglass_in_markdown(self) -> None: + self.assertEqual(rules(check_text("README.md", "Waiting " + chr(0x23F3))), ["TELL-14"]) + self.assertEqual(rules(check_text("README.md", "Done " + chr(0x231B))), ["TELL-14"]) + def test_rejects_non_ascii_in_code(self) -> None: found = check_text("src/a.ts", f"const space = '{NO_BREAK_SPACE}';") self.assertEqual(rules(found), ["TELL-14"]) @@ -136,6 +201,23 @@ def test_rejects_an_emoji_in_a_catalogue(self) -> None: self.assertEqual(rules(found), ["TELL-14"]) +class LineSplitTest(unittest.TestCase): + def test_keeps_a_line_separator_inside_the_line(self) -> None: + text = "const a = 1;" + LINE_SEPARATOR + "const b = 2;" + self.assertEqual(rules(check_text("src/a.ts", text)), ["TELL-14"]) + + def test_keeps_the_line_numbers_after_a_form_feed(self) -> None: + found = check_text("src/a.ts", "const a = 1;" + chr(0x0C) + "\nconsole.log(1);") + self.assertEqual(rules(found), ["TELL-13"]) + self.assertEqual(found[0].line, 2) + + def test_treats_crlf_like_lf(self) -> None: + text = "x" * 100 + "\nconsole.log(1);\n" + found = check_text("src/a.ts", text.replace("\n", "\r\n")) + self.assertEqual(rules(found), ["TELL-13"]) + self.assertEqual(found, check_text("src/a.ts", text)) + + class SkipTest(unittest.TestCase): def test_rejects_a_skip_without_a_reason(self) -> None: line = "x" * 120 + f" // {SKIP_MARKER} TELL-1" @@ -145,6 +227,10 @@ def test_rejects_a_skip_of_a_rule_that_cannot_be_skipped(self) -> None: line = f"const a = 1; // {SKIP_MARKER} TELL-7 because it is short" self.assertEqual(rules(check_text("src/a.ts", line)), ["SKIP"]) + def test_ignores_a_skip_marker_in_prose(self) -> None: + line = "A line such as `" + SKIP_MARKER + " TELL-1` has no reason." + self.assertEqual(check_text("README.md", line), []) + class CommitMessageTest(unittest.TestCase): def test_accepts_a_good_message(self) -> None: @@ -164,6 +250,38 @@ def test_accepts_an_accented_name_in_a_sign_off(self) -> None: def test_ignores_git_comment_lines(self) -> None: self.assertEqual(check_commit_message(f"fix: parse codes\n# {EMOJI} from git\n"), []) + def test_rejects_non_ascii_in_a_subject(self) -> None: + self.assertEqual(rules(check_commit_message("fix: parse caf" + E_ACUTE)), ["TELL-14"]) + + def test_rejects_non_ascii_in_a_body_but_not_in_a_sign_off(self) -> None: + message = ( + "fix: parse codes\n\n" + f"Use a dash {EM_DASH} here.\n\n" + f"Signed-off-by: Ad{E_ACUTE} Bello \n" + ) + found = check_commit_message(message) + self.assertEqual(rules(found), ["TELL-14"]) + self.assertEqual(found[0].line, 3) + + def test_reads_a_sign_off_without_regard_to_case(self) -> None: + message = f"fix: parse codes\n\nCo-Authored-By: Ad{E_ACUTE} Bello " + self.assertEqual(check_commit_message(message), []) + + def test_stops_at_the_scissors_line(self) -> None: + scissors = "# ------------------------ >8 ------------------------" + message = f"fix: parse codes\n{scissors}\n+ caf{E_ACUTE}" + self.assertEqual(check_commit_message(message), []) + + def test_reports_the_real_line_of_a_character(self) -> None: + found = check_commit_message(f"# c1\n# c2\nfix: caf{E_ACUTE}") + self.assertEqual(rules(found), ["TELL-14"]) + self.assertEqual(found[0].line, 3) + + def test_reports_the_real_line_of_a_long_subject(self) -> None: + found = check_commit_message("# note\nfix: " + "a" * 70) + self.assertEqual(rules(found), ["TELL-18"]) + self.assertEqual(found[0].line, 2) + class CommandTest(unittest.TestCase): def test_checks_new_files_and_skips_ignored_ones(self) -> None: diff --git a/standards/CODING_STANDARDS.md b/standards/CODING_STANDARDS.md index 44549f5..3f978a8 100644 --- a/standards/CODING_STANDARDS.md +++ b/standards/CODING_STANDARDS.md @@ -103,10 +103,10 @@ Coding agents repeat some habits so often that readers spot them at once. Each r `spec/scripts/check-tells` runs in the CI of every repo. It covers the tool rules that no language tool covers: -- **TELL-1.** It checks line length in source and config files, and it counts a tab as 4 columns. It skips lines with a URL, files marked as generated, lockfiles, JSON files in a folder named `data` or `vectors`, and Markdown prose. +- **TELL-1.** It checks line length in source and config files, and it counts a tab as 4 columns. It skips lines with a URL, files marked as generated, lockfiles, message catalogues, JSON files in a folder named `data` or `vectors`, and Markdown prose. - **TELL-7.** It rejects files named `utils`, `helpers`, `common` or `misc`, with any extension. -- **TELL-12 and CS-6.** It rejects each `TODO` without the form `TODO(#123)`. -- **TELL-13.** It rejects the debug calls that the language file lists. It skips test files, comment lines and doctest lines that start with `>>>` or `...`. +- **TELL-12 and CS-6.** It rejects each `TODO` in source and config files that does not have the form `TODO(#123)`. +- **TELL-13.** It rejects the debug calls that the language file lists. It skips test files, comment lines and, in Python, doctest lines that start with `>>>` or `...`. - **TELL-14.** It rejects emojis everywhere, including Markdown. It rejects other non-ASCII characters in source files, config files and commit messages. Message catalogues, docs prose and the sign-off lines of commit messages are exempt from the non-ASCII check, because contributors' names can need any script. - **TELL-18.** With `--commit-msg `, it checks the length of a commit subject. Repos without commitlint use this mode. From 776f093d1c9071d2c4dbdbc913110bf8dc345326 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 00:05:32 +0100 Subject: [PATCH 09/45] docs: define message catalogues for check-tells A code file in a locales or i18n folder is code, so check-tells holds it to the ASCII rule, and translations live in data files. The standards now say so, and say that skip comments work in code and config files only. Two tests pin the catalogue rule and the line split in commit messages. Signed-off-by: MobileMage --- scripts/tests/test_check_tells.py | 9 +++++++++ standards/CODING_STANDARDS.md | 4 ++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/scripts/tests/test_check_tells.py b/scripts/tests/test_check_tells.py index 9a8ca5e..798f705 100644 --- a/scripts/tests/test_check_tells.py +++ b/scripts/tests/test_check_tells.py @@ -17,6 +17,7 @@ E_ACUTE = chr(0x00E9) EM_DASH = chr(0x2014) LINE_SEPARATOR = chr(0x2028) +O_DOT_BELOW = chr(0x1ECD) # Built from two parts, so that this file passes its own to-do check. TO_DO = "TO" + "DO" # Built from two parts, so that this file passes its own skip check. @@ -196,6 +197,10 @@ def test_accepts_non_ascii_in_prose_and_catalogues(self) -> None: self.assertEqual(check_text("docs/guide.md", f"Caf{E_ACUTE}"), []) self.assertEqual(check_text("src/locales/yo.json", f'{{"title": "Caf{E_ACUTE}"}}'), []) + def test_holds_a_code_file_in_a_catalogue_folder_to_ascii(self) -> None: + line = "export const title = '" + O_DOT_BELOW + "';" + self.assertEqual(rules(check_text("src/i18n/yo.ts", line)), ["TELL-14"]) + def test_rejects_an_emoji_in_a_catalogue(self) -> None: found = check_text("src/locales/en.json", f'{{"ok": "{EMOJI}"}}') self.assertEqual(rules(found), ["TELL-14"]) @@ -282,6 +287,10 @@ def test_reports_the_real_line_of_a_long_subject(self) -> None: self.assertEqual(rules(found), ["TELL-18"]) self.assertEqual(found[0].line, 2) + def test_keeps_a_line_separator_in_a_commit_message(self) -> None: + found = check_commit_message("fix: a\n\nb" + LINE_SEPARATOR + "c") + self.assertEqual([(v.rule, v.line) for v in found], [("TELL-14", 3)]) + class CommandTest(unittest.TestCase): def test_checks_new_files_and_skips_ignored_ones(self) -> None: diff --git a/standards/CODING_STANDARDS.md b/standards/CODING_STANDARDS.md index 3f978a8..743c1a9 100644 --- a/standards/CODING_STANDARDS.md +++ b/standards/CODING_STANDARDS.md @@ -107,10 +107,10 @@ Coding agents repeat some habits so often that readers spot them at once. Each r - **TELL-7.** It rejects files named `utils`, `helpers`, `common` or `misc`, with any extension. - **TELL-12 and CS-6.** It rejects each `TODO` in source and config files that does not have the form `TODO(#123)`. - **TELL-13.** It rejects the debug calls that the language file lists. It skips test files, comment lines and, in Python, doctest lines that start with `>>>` or `...`. -- **TELL-14.** It rejects emojis everywhere, including Markdown. It rejects other non-ASCII characters in source files, config files and commit messages. Message catalogues, docs prose and the sign-off lines of commit messages are exempt from the non-ASCII check, because contributors' names can need any script. +- **TELL-14.** It rejects emojis everywhere, including Markdown. It rejects other non-ASCII characters in source files, config files and commit messages. Message catalogues, docs prose and the sign-off lines of commit messages are exempt from the non-ASCII check, because translations and contributors' names can need any script. A message catalogue is a data file: a file that is not code in a folder named `locales`, `i18n`, `l10n` or `messages`, a `.po`, `.arb`, `.xlf`, `.xliff`, `.strings` or `.stringsdict` file, or an Android `strings.xml`. A code file in such a folder is code, so translations live in data files. - **TELL-18.** With `--commit-msg `, it checks the length of a commit subject. Repos without commitlint use this mode. -A line can skip one check with a comment that names the rule and gives a reason, for example `check-tells: allow TELL-1 because the regex cannot be split`. Reviewers check each skip. +A line in a code or config file can skip one check with a comment that names the rule and gives a reason, for example `check-tells: allow TELL-1 because the regex cannot be split`. Reviewers check each skip. ## Dependencies From b73198bf92f0ee0718cec6fb278154a50b7d8a7d Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 00:25:59 +0100 Subject: [PATCH 10/45] feat: add the shared vectors and their builder Every SDK must give the same result for the same input, so the cases live in one place. build_vectors.py writes them from the data files, and make check fails when a vector file is out of date. The separators now include the soft hyphen and the two direction marks, because the grammar removes hyphens and zero-width characters. Signed-off-by: MobileMage --- Makefile | 7 +- data/format.json | 9 +- grammar.md | 2 + scripts/build_vectors.py | 564 +++++++++++++++++++++ scripts/tests/test_build_vectors.py | 46 ++ vectors/README.md | 34 ++ vectors/contains.json | 103 ++++ vectors/is-legacy.json | 79 +++ vectors/normalize.json | 214 ++++++++ vectors/parent.json | 62 +++ vectors/parse-segments.json | 389 +++++++++++++++ vectors/parse-states.json | 747 ++++++++++++++++++++++++++++ vectors/parse.json | 623 +++++++++++++++++++++++ vectors/precision-for-accuracy.json | 133 +++++ vectors/redact.json | 40 ++ vectors/state-name.json | 385 ++++++++++++++ vectors/truncate.json | 103 ++++ 17 files changed, 3534 insertions(+), 6 deletions(-) create mode 100755 scripts/build_vectors.py create mode 100644 scripts/tests/test_build_vectors.py create mode 100644 vectors/README.md create mode 100644 vectors/contains.json create mode 100644 vectors/is-legacy.json create mode 100644 vectors/normalize.json create mode 100644 vectors/parent.json create mode 100644 vectors/parse-segments.json create mode 100644 vectors/parse-states.json create mode 100644 vectors/parse.json create mode 100644 vectors/precision-for-accuracy.json create mode 100644 vectors/redact.json create mode 100644 vectors/state-name.json create mode 100644 vectors/truncate.json diff --git a/Makefile b/Makefile index a0afd8a..f37f214 100644 --- a/Makefile +++ b/Makefile @@ -1,12 +1,15 @@ # SPDX-FileCopyrightText: 2026 The Gatepost authors # SPDX-License-Identifier: Apache-2.0 -.PHONY: check test tells lint types licences +.PHONY: check test vectors tells lint types licences -check: test tells lint types licences +check: test vectors tells lint types licences test: python3 -m unittest discover -s scripts/tests -t scripts +vectors: + python3 scripts/build_vectors.py --check + tells: python3 scripts/check-tells diff --git a/data/format.json b/data/format.json index a4fa779..b60e3a7 100644 --- a/data/format.json +++ b/data/format.json @@ -10,10 +10,11 @@ "legacyPattern": "^[0-9]{6}$", "separators": [ "U+0009", "U+000A", "U+000B", "U+000C", "U+000D", "U+0020", "U+002D", "U+002E", - "U+0085", "U+00A0", "U+1680", "U+2000", "U+2001", "U+2002", "U+2003", "U+2004", - "U+2005", "U+2006", "U+2007", "U+2008", "U+2009", "U+200A", "U+200B", "U+200C", - "U+200D", "U+2010", "U+2011", "U+2012", "U+2013", "U+2014", "U+2015", "U+2028", - "U+2029", "U+202F", "U+205F", "U+2060", "U+2212", "U+3000", "U+FEFF" + "U+0085", "U+00A0", "U+00AD", "U+1680", "U+2000", "U+2001", "U+2002", "U+2003", + "U+2004", "U+2005", "U+2006", "U+2007", "U+2008", "U+2009", "U+200A", "U+200B", + "U+200C", "U+200D", "U+200E", "U+200F", "U+2010", "U+2011", "U+2012", "U+2013", + "U+2014", "U+2015", "U+2028", "U+2029", "U+202F", "U+205F", "U+2060", "U+2212", + "U+3000", "U+FEFF" ], "suggestions": { "digits": { "O": "0", "I": "1", "L": "1" }, diff --git a/grammar.md b/grammar.md index 5e84f2a..8d98d62 100644 --- a/grammar.md +++ b/grammar.md @@ -28,6 +28,8 @@ A partial postcode stops after the state, LGA, district or area segment. Its len 2. Remove each separator in `data/format.json`: white space, hyphens and dashes, the full stop, and zero-width characters. 3. Change the ASCII letters a to z to upper case. Keep every other character as it is. +A format character that is not in the list stays. For example, the right-to-left override U+202E can change the order in which a code shows on screen, so `parse` gives `bad_character` for it. + ## Parse Normalise the input. Then apply these checks in order. The first check that fails gives the error code. diff --git a/scripts/build_vectors.py b/scripts/build_vectors.py new file mode 100755 index 0000000..5652644 --- /dev/null +++ b/scripts/build_vectors.py @@ -0,0 +1,564 @@ +#!/usr/bin/env python3 +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +"""Build the shared test vectors in vectors/. + +This file is the source of truth for the vectors. Edit the cases here, then run +python3 scripts/build_vectors.py. CI runs it with --check, which fails when a vector +file differs from what this script builds. +""" + +from __future__ import annotations + +import argparse +import json +import sys +from collections.abc import Callable +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parent.parent +VECTORS = ROOT / "vectors" +FORMAT_VERSION = 1 +STATE_COUNT = 37 + +EN_DASH = chr(0x2013) +EM_DASH = chr(0x2014) +MINUS_SIGN = chr(0x2212) +NO_BREAK_SPACE = chr(0x00A0) +NARROW_NO_BREAK_SPACE = chr(0x202F) +ZERO_WIDTH_SPACE = chr(0x200B) +BYTE_ORDER_MARK = chr(0xFEFF) +SMALL_E_ACUTE = chr(0x00E9) +FI_LIGATURE = chr(0xFB01) +TAB = chr(0x09) +LINE_FEED = chr(0x0A) +SOFT_HYPHEN = chr(0x00AD) +LEFT_TO_RIGHT_MARK = chr(0x200E) +RIGHT_TO_LEFT_MARK = chr(0x200F) +RIGHT_TO_LEFT_OVERRIDE = chr(0x202E) + +WIDTHS = (("state", 2), ("lga", 2), ("district", 3), ("area", 2), ("unit", 2)) +PRECISION_BY_LENGTH = {2: "state", 4: "lga", 7: "district", 9: "area", 11: "unit"} +PARTIAL = {"allowPartial": True} +CODE = "EK01A03FK01" + +Row = tuple[str, Any, dict[str, Any], dict[str, Any]] + + +def row( + description: str, input_: Any, expect: dict[str, Any], options: dict[str, Any] | None = None +) -> Row: + """Make one case. Options default to none.""" + return (description, input_, expect, options or {}) + + +def full_width(text: str) -> str: + """Change the ASCII letters and digits in text to their full-width forms.""" + return "".join(chr(ord(c) + 0xFEE0) if c.isascii() and c.isalnum() else c for c in text) + + +def joined(separator: str) -> str: + """Return the code EK 01 A03 FK 01 with this separator between segments.""" + return separator.join(("EK", "01", "A03", "FK", "01")) + + +def segments_of(compact: str) -> dict[str, str | None]: + """Split a compact code into named segments. A missing segment is None.""" + segments: dict[str, str | None] = {} + start = 0 + for name, width in WIDTHS: + segments[name] = compact[start : start + width] or None + start += width + return segments + + +def ok(compact: str) -> dict[str, Any]: + """The expected result of a successful parse.""" + parts = [part for part in segments_of(compact).values() if part] + return { + "ok": True, + "compact": compact, + "canonical": "-".join(parts), + "display": " ".join(parts), + "precision": PRECISION_BY_LENGTH[len(compact)], + "segments": segments_of(compact), + } + + +def failed(code: str, segment: str | None = None, suggestion: str | None = None) -> dict[str, Any]: + """The expected result of a failed parse.""" + return {"ok": False, "error": {"code": code, "segment": segment, "suggestion": suggestion}} + + +def value(result: Any) -> dict[str, Any]: + """The expected result of a function that returns one value.""" + return {"value": result} + + +def load_states() -> list[dict[str, str]]: + """Read data/states.json and check that it lists 37 different states.""" + states: list[dict[str, str]] = json.loads( + (ROOT / "data" / "states.json").read_text(encoding="utf-8") + )["states"] + if len({state["code"] for state in states}) != STATE_COUNT: + raise ValueError(f"data/states.json must list {STATE_COUNT} different state codes.") + return states + + +def normalize_rows() -> list[Row]: + """Cases for normalize.""" + want = value(CODE) + return [ + row("removes hyphens", "EK-01-A03-FK-01", want), + row("removes spaces and makes letters upper case", "ek 01 a03 fk 01", want), + row("removes leading and trailing spaces", " EK01A03FK01 ", want), + row("removes dots", "EK.01.A03.FK.01", want), + row("removes en dashes", joined(EN_DASH), want), + row("removes em dashes", joined(EM_DASH), want), + row("removes minus signs", joined(MINUS_SIGN), want), + row("removes no-break spaces", joined(NO_BREAK_SPACE), want), + row("removes narrow no-break spaces", joined(NARROW_NO_BREAK_SPACE), want), + row("removes a zero-width space", CODE + ZERO_WIDTH_SPACE, want), + row("removes a byte order mark", BYTE_ORDER_MARK + CODE, want), + row("removes soft hyphens", joined(SOFT_HYPHEN), want), + row("removes direction marks", LEFT_TO_RIGHT_MARK + CODE + RIGHT_TO_LEFT_MARK, want), + row( + "keeps a right-to-left override", + RIGHT_TO_LEFT_OVERRIDE + CODE, + value(RIGHT_TO_LEFT_OVERRIDE + CODE), + ), + row("changes full-width letters and digits to ASCII", full_width(CODE), want), + row("removes tabs and line feeds", "EK01" + TAB + "A03" + LINE_FEED + "FK01", want), + row( + "changes a ligature to its letters", + "EK01A03" + FI_LIGATURE + "01", + value("EK01A03FI01"), + ), + row("makes mixed-case letters upper case", "eK01a03Fk01", want), + row("keeps a legacy code", "900 108", value("900108")), + row("keeps characters that are not separators", "EK01A03FK0!", value("EK01A03FK0!")), + row( + "makes only ASCII letters upper case", + "ek01a03f" + SMALL_E_ACUTE + "01", + value("EK01A03F" + SMALL_E_ACUTE + "01"), + ), + row("returns an empty string for empty input", "", value("")), + row("returns an empty string for separators only", " - . ", value("")), + ] + + +def parse_rows() -> list[Row]: + """Cases for forms, lengths, characters and legacy codes.""" + return [ + row("parses the canonical form", "EK-01-A03-FK-01", ok(CODE)), + row("parses the compact form", CODE, ok(CODE)), + row("parses the display form in lower case", "ek 01 a03 fk 01", ok(CODE)), + row("parses a code with en dashes", joined(EN_DASH), ok(CODE)), + row("parses a code with no-break spaces", joined(NO_BREAK_SPACE), ok(CODE)), + row("parses a code with a zero-width space", CODE + ZERO_WIDTH_SPACE, ok(CODE)), + row("parses full-width letters and digits", full_width("EK-01-A03-FK-01"), ok(CODE)), + row("parses the example on NIPOST's homepage", "FC-02-A09-DB-09", ok("FC02A09DB09")), + row("parses the example in NIPOST's blog", "LA 12 K23 IV 95", ok("LA12K23IV95")), + row("parses a NIPOST test code", "AK-11-I61-ZF-12", ok("AK11I61ZF12")), + row("accepts a letter O in the district", "JI-24-O18-JP-23", ok("JI24O18JP23")), + row("rejects empty input", "", failed("empty")), + row("rejects spaces only", " ", failed("empty")), + row("rejects separators only", " - . ", failed("empty")), + row("names a legacy code", "900108", failed("legacy_code")), + row("names a legacy code with a space", "900 108", failed("legacy_code")), + row( + "names a legacy code in full-width digits", full_width("900108"), failed("legacy_code") + ), + row( + "names a legacy code when partial codes are allowed", + "123456", + failed("legacy_code"), + PARTIAL, + ), + row("rejects five digits", "90010", failed("bad_length")), + row("rejects an exclamation mark", "EK01A03FK0!", failed("bad_character")), + row("rejects an underscore", "EK01A03FK0_", failed("bad_character")), + row("rejects slashes", "EK01/A03/FK01", failed("bad_character")), + row( + "rejects a letter outside ASCII", + "EK01A03F" + SMALL_E_ACUTE + "01", + failed("bad_character"), + ), + row( + "rejects a right-to-left override", + RIGHT_TO_LEFT_OVERRIDE + CODE, + failed("bad_character"), + ), + row("rejects 10 characters", "EK01A03FK1", failed("bad_length")), + row("rejects 12 characters", "EK01A03FK01X", failed("bad_length")), + row("rejects a unit that lost its zero", "EK-01-A03-FK-1", failed("bad_length")), + row("rejects a state alone by default", "EK", failed("bad_length")), + row("rejects a district code by default", "EK01A03", failed("bad_length")), + row("parses a state when partial codes are allowed", "EK", ok("EK"), PARTIAL), + row("parses an LGA code when partial codes are allowed", "EK01", ok("EK01"), PARTIAL), + row( + "parses a district code when partial codes are allowed", + "ek-01-a03", + ok("EK01A03"), + PARTIAL, + ), + row( + "parses an area code when partial codes are allowed", + "EK01A03FK", + ok("EK01A03FK"), + PARTIAL, + ), + row("parses a full code when partial codes are allowed", CODE, ok(CODE), PARTIAL), + row( + "rejects 3 characters when partial codes are allowed", + "EK0", + failed("bad_length"), + PARTIAL, + ), + row( + "rejects 6 characters when partial codes are allowed", + "EK01A0", + failed("bad_length"), + PARTIAL, + ), + ] + + +def segment_rows() -> list[Row]: + """Cases for segment rules and typo suggestions.""" + return [ + row("rejects an unknown state", "XX01A03FK01", failed("unknown_state", "state")), + row("rejects the synthetic state ZZ", "ZZ01Z99ZZ01", failed("unknown_state", "state")), + row( + "suggests O for a zero in the state", + "0G01A03FK01", + failed("unknown_state", "state", "OG-01-A03-FK-01"), + ), + row( + "suggests I for a one in the state", + "1M01A03FK01", + failed("unknown_state", "state", "IM-01-A03-FK-01"), + ), + row( + "gives no suggestion when the fixed state is unknown", + "E101A03FK01", + failed("unknown_state", "state"), + ), + row( + "gives no suggestion when the state cannot be fixed", + "XXO1A03FK01", + failed("unknown_state", "state"), + ), + row( + "fixes later segments in a state suggestion", + "0GO1A03FK01", + failed("unknown_state", "state", "OG-01-A03-FK-01"), + ), + row("rejects an LGA of 00", "EK00A03FK01", failed("bad_segment", "lga")), + row("rejects a unit of 00", "EK01A03FK00", failed("bad_segment", "unit")), + row( + "suggests 0 for a letter O in the LGA", + "EKO1A03FK01", + failed("bad_segment", "lga", "EK-01-A03-FK-01"), + ), + row( + "suggests 1 for a letter I in the LGA", + "EKI1A03FK01", + failed("bad_segment", "lga", "EK-11-A03-FK-01"), + ), + row( + "suggests 1 for a letter L in the LGA", + "EKL1A03FK01", + failed("bad_segment", "lga", "EK-11-A03-FK-01"), + ), + row( + "suggests I for a one in the area", + "EK01A03F101", + failed("bad_segment", "area", "EK-01-A03-FI-01"), + ), + row( + "suggests O for a zero in the area", + "EK01A03F001", + failed("bad_segment", "area", "EK-01-A03-FO-01"), + ), + row( + "suggests 0 for a letter O in the unit", + "EK01A03FKO1", + failed("bad_segment", "unit", "EK-01-A03-FK-01"), + ), + row( + "suggests 1 for a letter I in the unit", + "EK01A03FKI2", + failed("bad_segment", "unit", "EK-01-A03-FK-12"), + ), + row( + "suggests 1 for a letter I after a zero in the unit", + "EK01A03FK0I", + failed("bad_segment", "unit", "EK-01-A03-FK-01"), + ), + row( + "gives no suggestion when the fix makes a unit of 00", + "EK01A03FK0O", + failed("bad_segment", "unit"), + ), + row( + "gives no suggestion when a segment cannot be fixed", + "EK01A03F2O1", + failed("bad_segment", "area"), + ), + row( + "reports the first bad segment and fixes them all", + "EKO1A03F1O1", + failed("bad_segment", "lga", "EK-01-A03-FI-01"), + ), + row( + "suggests a fix after making letters upper case", + "eko1a03fk01", + failed("bad_segment", "lga", "EK-01-A03-FK-01"), + ), + row( + "rejects an LGA of 00 in a partial code", "EK00", failed("bad_segment", "lga"), PARTIAL + ), + row("suggests a partial code", "EKO1", failed("bad_segment", "lga", "EK-01"), PARTIAL), + row( + "suggests a partial area code", + "EK01A03F1", + failed("bad_segment", "area", "EK-01-A03-FI"), + PARTIAL, + ), + row("never changes the district", "EK01AO3FK01", ok("EK01AO3FK01")), + row( + "accepts letters and digits in any district position", "EK01A0OFK01", ok("EK01A0OFK01") + ), + ] + + +def state_parse_rows() -> list[Row]: + """One valid synthetic code for each state.""" + return [ + row( + f"parses a code in {state['name']}", + f"{state['code']}-01-Z99-ZZ-01", + ok(f"{state['code']}01Z99ZZ01"), + ) + for state in load_states() + ] + + +def legacy_rows() -> list[Row]: + """Cases for isLegacy.""" + return [ + row("accepts six digits", "900108", value(True)), + row("accepts six digits with a space", "900 108", value(True)), + row("accepts six full-width digits", full_width("900108"), value(True)), + row("rejects five digits", "90010", value(False)), + row("rejects seven digits", "9001080", value(False)), + row("rejects a new postcode", CODE, value(False)), + row("rejects empty input", "", value(False)), + row("rejects a letter O among digits", "90O108", value(False)), + ] + + +def truncate_rows() -> list[Row]: + """Cases for truncate. Inputs are canonical codes.""" + full = "EK-01-A03-FK-01" + return [ + row( + "keeps a full code at unit precision", {"code": full, "to": "unit"}, {"canonical": full} + ), + row( + "cuts a full code to its area", + {"code": full, "to": "area"}, + {"canonical": "EK-01-A03-FK"}, + ), + row( + "cuts a full code to its district", + {"code": full, "to": "district"}, + {"canonical": "EK-01-A03"}, + ), + row("cuts a full code to its LGA", {"code": full, "to": "lga"}, {"canonical": "EK-01"}), + row("cuts a full code to its state", {"code": full, "to": "state"}, {"canonical": "EK"}), + row( + "keeps a district code at district precision", + {"code": "EK-01-A03", "to": "district"}, + {"canonical": "EK-01-A03"}, + ), + row( + "refuses to add precision to a district code", + {"code": "EK-01-A03", "to": "area"}, + {"rejects": True}, + ), + row( + "refuses to add precision to a state code", + {"code": "EK", "to": "lga"}, + {"rejects": True}, + ), + ] + + +def parent_rows() -> list[Row]: + """Cases for parent.""" + return [ + row( + "gives the area of a full code", + {"code": "EK-01-A03-FK-01"}, + {"canonical": "EK-01-A03-FK"}, + ), + row( + "gives the district of an area code", + {"code": "EK-01-A03-FK"}, + {"canonical": "EK-01-A03"}, + ), + row("gives the LGA of a district code", {"code": "EK-01-A03"}, {"canonical": "EK-01"}), + row("gives the state of an LGA code", {"code": "EK-01"}, {"canonical": "EK"}), + row("gives nothing for a state code", {"code": "EK"}, {"canonical": None}), + ] + + +def contains_rows() -> list[Row]: + """Cases for contains.""" + full = "EK-01-A03-FK-01" + return [ + row( + "finds a full code in its district", {"prefix": "EK-01-A03", "code": full}, value(True) + ), + row("finds a full code in its state", {"prefix": "EK", "code": full}, value(True)), + row("finds a code in itself", {"prefix": full, "code": full}, value(True)), + row( + "finds a unit in its area", + {"prefix": "EK-01-A03-FK", "code": "EK-01-A03-FK-02"}, + value(True), + ), + row( + "rejects a code from another district", + {"prefix": "EK-01-A04", "code": full}, + value(False), + ), + row("rejects a code from another state", {"prefix": "FC", "code": full}, value(False)), + row( + "rejects a less precise code inside a more precise one", + {"prefix": full, "code": "EK-01-A03"}, + value(False), + ), + row("rejects a different unit", {"prefix": full, "code": "EK-01-A03-FK-02"}, value(False)), + ] + + +def redact_rows() -> list[Row]: + """Cases for redact.""" + return [ + row("hides the unit of a full code", {"code": "EK-01-A03-FK-01"}, value("EK-01-A03-FK-**")), + row("keeps an area code as it is", {"code": "EK-01-A03-FK"}, value("EK-01-A03-FK")), + row("keeps a state code as it is", {"code": "EK"}, value("EK")), + ] + + +def state_name_rows() -> list[Row]: + """Cases for stateName: one for each state, then the edge cases.""" + named = [ + row(f"names {state['code']}", state["code"], value(state["name"])) + for state in load_states() + ] + return [ + *named, + row("accepts lower case", "ek", value("Ekiti")), + row("accepts mixed case", "Fc", value("Federal Capital Territory")), + row("returns nothing for an unknown code", "XX", value(None)), + row("returns nothing for empty input", "", value(None)), + row("returns nothing for three letters", "EKI", value(None)), + ] + + +def accuracy_rows() -> list[Row]: + """Cases for precisionForAccuracy. + + Runners turn the strings NaN, Infinity and -Infinity into numbers. + """ + return [ + row("gives the unit at 0 m", 0, value("unit")), + row("gives the unit at 5 m", 5, value("unit")), + row("gives the unit at exactly 8 m", 8, value("unit")), + row("gives the area just above 8 m", 8.1, value("area")), + row("gives the area at exactly 20 m", 20, value("area")), + row("gives the district above 20 m", 20.5, value("district")), + row("gives the district at exactly 50 m", 50, value("district")), + row("gives the LGA just above 50 m", 50.1, value("lga")), + row("gives the LGA at 1000 m", 1000, value("lga")), + row("gives the LGA when the accuracy is unknown", None, value("lga")), + row("gives the LGA for a negative accuracy", -1, value("lga")), + row("gives the LGA for NaN", "NaN", value("lga")), + row("gives the LGA for infinity", "Infinity", value("lga")), + row("gives the LGA for negative infinity", "-Infinity", value("lga")), + ] + + +FILES: dict[str, tuple[str, str, Callable[[], list[Row]]]] = { + "normalize": ("normalize", "Cleans input before parsing.", normalize_rows), + "parse": ("parse", "Forms, lengths, characters and legacy codes.", parse_rows), + "parse-segments": ("parse", "Segment rules and typo suggestions.", segment_rows), + "parse-states": ("parse", "One valid code for each state.", state_parse_rows), + "is-legacy": ("isLegacy", "Finds old 6-digit postcodes.", legacy_rows), + "truncate": ("truncate", "Cuts a code to a less precise segment.", truncate_rows), + "parent": ("parent", "Gives the code one segment shorter.", parent_rows), + "contains": ("contains", "Checks that one code lies inside another.", contains_rows), + "redact": ("redact", "Hides the unit for logs.", redact_rows), + "state-name": ("stateName", "Names the state for a state code.", state_name_rows), + "precision-for-accuracy": ( + "precisionForAccuracy", + "Maps GPS accuracy to precision.", + accuracy_rows, + ), +} + + +def render(stem: str) -> str: + """Build the JSON text of one vector file.""" + function, description, rows = FILES[stem] + cases = [ + { + "id": f"{stem}-{index:03d}", + "description": text, + "input": input_, + "options": options, + "expect": expect, + } + for index, (text, input_, expect, options) in enumerate(rows(), start=1) + ] + document = { + "version": FORMAT_VERSION, + "function": function, + "description": description, + "cases": cases, + } + return json.dumps(document, ensure_ascii=True, indent=2) + "\n" + + +def main(argv: list[str] | None = None) -> int: + """Write the vector files, or with --check, report the ones that are out of date.""" + parser = argparse.ArgumentParser(description="Build the shared test vectors.") + parser.add_argument( + "--check", action="store_true", help="Fail if a vector file is out of date." + ) + args = parser.parse_args(argv) + VECTORS.mkdir(exist_ok=True) + stale: list[str] = [] + for stem in FILES: + path = VECTORS / f"{stem}.json" + text = render(stem) + if not args.check: + path.write_text(text, encoding="utf-8") + elif not path.exists() or path.read_text(encoding="utf-8") != text: + stale.append(path.name) + extra = sorted({path.stem for path in VECTORS.glob("*.json")} - set(FILES)) + stale.extend(f"{stem}.json (not built by this script)" for stem in extra) + if stale: + sys.stderr.write( + f"Out of date: {', '.join(stale)}. Run python3 scripts/build_vectors.py.\n" + ) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/tests/test_build_vectors.py b/scripts/tests/test_build_vectors.py new file mode 100644 index 0000000..03cfe8a --- /dev/null +++ b/scripts/tests/test_build_vectors.py @@ -0,0 +1,46 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +"""Tests for build_vectors.""" + +import json +import unittest +from pathlib import Path +from typing import Any + +import build_vectors + +ROOT = Path(__file__).resolve().parents[2] + + +def load(stem: str) -> dict[str, Any]: + value: object = json.loads((ROOT / "vectors" / f"{stem}.json").read_text(encoding="utf-8")) + if not isinstance(value, dict): + raise TypeError(f"{stem}.json must hold a JSON object.") + return value + + +class BuildVectorsTest(unittest.TestCase): + def test_vector_files_match_the_builder(self) -> None: + self.assertEqual(build_vectors.main(["--check"]), 0) + + def test_vector_files_are_ascii(self) -> None: + for path in sorted((ROOT / "vectors").glob("*.json")): + with self.subTest(file=path.name): + self.assertTrue(path.read_text(encoding="utf-8").isascii()) + + def test_each_case_has_a_unique_id_and_a_description(self) -> None: + ids: list[str] = [] + for stem in build_vectors.FILES: + for case in load(stem)["cases"]: + ids.append(case["id"]) + self.assertTrue(case["description"]) + self.assertEqual(len(ids), len(set(ids))) + self.assertGreaterEqual(len(ids), 200) + + def test_each_state_has_a_parse_case_and_a_name_case(self) -> None: + states = json.loads((ROOT / "data" / "states.json").read_text(encoding="utf-8")) + codes = {state["code"] for state in states["states"]} + parsed = {case["input"][:2] for case in load("parse-states")["cases"]} + named = {case["input"] for case in load("state-name")["cases"]} + self.assertEqual(parsed, codes) + self.assertLessEqual(codes, named) diff --git a/vectors/README.md b/vectors/README.md new file mode 100644 index 0000000..e4b0d61 --- /dev/null +++ b/vectors/README.md @@ -0,0 +1,34 @@ +# Vectors + +Each JSON file here holds shared test cases for one core function. Every SDK runs every case. + +`scripts/build_vectors.py` builds these files. Do not edit them by hand. Edit the builder, then run `python3 scripts/build_vectors.py`. + +## Format + + { "version": 1, "function": "parse", "description": "...", "cases": [ ... ] } + +Each case has these fields: + +| Field | Meaning | +|---|---| +| `id` | a stable id, such as `parse-001` | +| `description` | the behaviour in domain words. Use it in the test name. | +| `input` | the argument, or an object of named arguments | +| `options` | options for the function, such as `{ "allowPartial": true }` | +| `expect` | the expected result | + +## Expected results + +| Function | `expect` | +|---|---| +| `normalize`, `isLegacy`, `contains`, `redact`, `stateName`, `precisionForAccuracy` | `{ "value": ... }` | +| `parse` | `{ "ok": true, "compact", "canonical", "display", "precision", "segments" }`, or `{ "ok": false, "error": { "code", "segment", "suggestion" } }` | +| `truncate` | `{ "canonical": ... }`, or `{ "rejects": true }` when the call must fail with the language's programmer error | +| `parent` | `{ "canonical": ... }`, where null means no parent | + +## Runner rules + +- For `truncate`, `parent`, `contains` and `redact`, the inputs are canonical codes. Parse them with `allowPartial` before the call. +- For `precisionForAccuracy`, change the strings `NaN`, `Infinity` and `-Infinity` into the language's numbers. Change null into the language's "no value". +- Non-ASCII inputs are JSON escapes. Every file is ASCII. diff --git a/vectors/contains.json b/vectors/contains.json new file mode 100644 index 0000000..c97abb0 --- /dev/null +++ b/vectors/contains.json @@ -0,0 +1,103 @@ +{ + "version": 1, + "function": "contains", + "description": "Checks that one code lies inside another.", + "cases": [ + { + "id": "contains-001", + "description": "finds a full code in its district", + "input": { + "prefix": "EK-01-A03", + "code": "EK-01-A03-FK-01" + }, + "options": {}, + "expect": { + "value": true + } + }, + { + "id": "contains-002", + "description": "finds a full code in its state", + "input": { + "prefix": "EK", + "code": "EK-01-A03-FK-01" + }, + "options": {}, + "expect": { + "value": true + } + }, + { + "id": "contains-003", + "description": "finds a code in itself", + "input": { + "prefix": "EK-01-A03-FK-01", + "code": "EK-01-A03-FK-01" + }, + "options": {}, + "expect": { + "value": true + } + }, + { + "id": "contains-004", + "description": "finds a unit in its area", + "input": { + "prefix": "EK-01-A03-FK", + "code": "EK-01-A03-FK-02" + }, + "options": {}, + "expect": { + "value": true + } + }, + { + "id": "contains-005", + "description": "rejects a code from another district", + "input": { + "prefix": "EK-01-A04", + "code": "EK-01-A03-FK-01" + }, + "options": {}, + "expect": { + "value": false + } + }, + { + "id": "contains-006", + "description": "rejects a code from another state", + "input": { + "prefix": "FC", + "code": "EK-01-A03-FK-01" + }, + "options": {}, + "expect": { + "value": false + } + }, + { + "id": "contains-007", + "description": "rejects a less precise code inside a more precise one", + "input": { + "prefix": "EK-01-A03-FK-01", + "code": "EK-01-A03" + }, + "options": {}, + "expect": { + "value": false + } + }, + { + "id": "contains-008", + "description": "rejects a different unit", + "input": { + "prefix": "EK-01-A03-FK-01", + "code": "EK-01-A03-FK-02" + }, + "options": {}, + "expect": { + "value": false + } + } + ] +} diff --git a/vectors/is-legacy.json b/vectors/is-legacy.json new file mode 100644 index 0000000..9d5fb1e --- /dev/null +++ b/vectors/is-legacy.json @@ -0,0 +1,79 @@ +{ + "version": 1, + "function": "isLegacy", + "description": "Finds old 6-digit postcodes.", + "cases": [ + { + "id": "is-legacy-001", + "description": "accepts six digits", + "input": "900108", + "options": {}, + "expect": { + "value": true + } + }, + { + "id": "is-legacy-002", + "description": "accepts six digits with a space", + "input": "900 108", + "options": {}, + "expect": { + "value": true + } + }, + { + "id": "is-legacy-003", + "description": "accepts six full-width digits", + "input": "\uff19\uff10\uff10\uff11\uff10\uff18", + "options": {}, + "expect": { + "value": true + } + }, + { + "id": "is-legacy-004", + "description": "rejects five digits", + "input": "90010", + "options": {}, + "expect": { + "value": false + } + }, + { + "id": "is-legacy-005", + "description": "rejects seven digits", + "input": "9001080", + "options": {}, + "expect": { + "value": false + } + }, + { + "id": "is-legacy-006", + "description": "rejects a new postcode", + "input": "EK01A03FK01", + "options": {}, + "expect": { + "value": false + } + }, + { + "id": "is-legacy-007", + "description": "rejects empty input", + "input": "", + "options": {}, + "expect": { + "value": false + } + }, + { + "id": "is-legacy-008", + "description": "rejects a letter O among digits", + "input": "90O108", + "options": {}, + "expect": { + "value": false + } + } + ] +} diff --git a/vectors/normalize.json b/vectors/normalize.json new file mode 100644 index 0000000..981794e --- /dev/null +++ b/vectors/normalize.json @@ -0,0 +1,214 @@ +{ + "version": 1, + "function": "normalize", + "description": "Cleans input before parsing.", + "cases": [ + { + "id": "normalize-001", + "description": "removes hyphens", + "input": "EK-01-A03-FK-01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-002", + "description": "removes spaces and makes letters upper case", + "input": "ek 01 a03 fk 01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-003", + "description": "removes leading and trailing spaces", + "input": " EK01A03FK01 ", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-004", + "description": "removes dots", + "input": "EK.01.A03.FK.01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-005", + "description": "removes en dashes", + "input": "EK\u201301\u2013A03\u2013FK\u201301", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-006", + "description": "removes em dashes", + "input": "EK\u201401\u2014A03\u2014FK\u201401", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-007", + "description": "removes minus signs", + "input": "EK\u221201\u2212A03\u2212FK\u221201", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-008", + "description": "removes no-break spaces", + "input": "EK\u00a001\u00a0A03\u00a0FK\u00a001", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-009", + "description": "removes narrow no-break spaces", + "input": "EK\u202f01\u202fA03\u202fFK\u202f01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-010", + "description": "removes a zero-width space", + "input": "EK01A03FK01\u200b", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-011", + "description": "removes a byte order mark", + "input": "\ufeffEK01A03FK01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-012", + "description": "removes soft hyphens", + "input": "EK\u00ad01\u00adA03\u00adFK\u00ad01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-013", + "description": "removes direction marks", + "input": "\u200eEK01A03FK01\u200f", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-014", + "description": "keeps a right-to-left override", + "input": "\u202eEK01A03FK01", + "options": {}, + "expect": { + "value": "\u202eEK01A03FK01" + } + }, + { + "id": "normalize-015", + "description": "changes full-width letters and digits to ASCII", + "input": "\uff25\uff2b\uff10\uff11\uff21\uff10\uff13\uff26\uff2b\uff10\uff11", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-016", + "description": "removes tabs and line feeds", + "input": "EK01\tA03\nFK01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-017", + "description": "changes a ligature to its letters", + "input": "EK01A03\ufb0101", + "options": {}, + "expect": { + "value": "EK01A03FI01" + } + }, + { + "id": "normalize-018", + "description": "makes mixed-case letters upper case", + "input": "eK01a03Fk01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-019", + "description": "keeps a legacy code", + "input": "900 108", + "options": {}, + "expect": { + "value": "900108" + } + }, + { + "id": "normalize-020", + "description": "keeps characters that are not separators", + "input": "EK01A03FK0!", + "options": {}, + "expect": { + "value": "EK01A03FK0!" + } + }, + { + "id": "normalize-021", + "description": "makes only ASCII letters upper case", + "input": "ek01a03f\u00e901", + "options": {}, + "expect": { + "value": "EK01A03F\u00e901" + } + }, + { + "id": "normalize-022", + "description": "returns an empty string for empty input", + "input": "", + "options": {}, + "expect": { + "value": "" + } + }, + { + "id": "normalize-023", + "description": "returns an empty string for separators only", + "input": " - . ", + "options": {}, + "expect": { + "value": "" + } + } + ] +} diff --git a/vectors/parent.json b/vectors/parent.json new file mode 100644 index 0000000..5858306 --- /dev/null +++ b/vectors/parent.json @@ -0,0 +1,62 @@ +{ + "version": 1, + "function": "parent", + "description": "Gives the code one segment shorter.", + "cases": [ + { + "id": "parent-001", + "description": "gives the area of a full code", + "input": { + "code": "EK-01-A03-FK-01" + }, + "options": {}, + "expect": { + "canonical": "EK-01-A03-FK" + } + }, + { + "id": "parent-002", + "description": "gives the district of an area code", + "input": { + "code": "EK-01-A03-FK" + }, + "options": {}, + "expect": { + "canonical": "EK-01-A03" + } + }, + { + "id": "parent-003", + "description": "gives the LGA of a district code", + "input": { + "code": "EK-01-A03" + }, + "options": {}, + "expect": { + "canonical": "EK-01" + } + }, + { + "id": "parent-004", + "description": "gives the state of an LGA code", + "input": { + "code": "EK-01" + }, + "options": {}, + "expect": { + "canonical": "EK" + } + }, + { + "id": "parent-005", + "description": "gives nothing for a state code", + "input": { + "code": "EK" + }, + "options": {}, + "expect": { + "canonical": null + } + } + ] +} diff --git a/vectors/parse-segments.json b/vectors/parse-segments.json new file mode 100644 index 0000000..e84450f --- /dev/null +++ b/vectors/parse-segments.json @@ -0,0 +1,389 @@ +{ + "version": 1, + "function": "parse", + "description": "Segment rules and typo suggestions.", + "cases": [ + { + "id": "parse-segments-001", + "description": "rejects an unknown state", + "input": "XX01A03FK01", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "unknown_state", + "segment": "state", + "suggestion": null + } + } + }, + { + "id": "parse-segments-002", + "description": "rejects the synthetic state ZZ", + "input": "ZZ01Z99ZZ01", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "unknown_state", + "segment": "state", + "suggestion": null + } + } + }, + { + "id": "parse-segments-003", + "description": "suggests O for a zero in the state", + "input": "0G01A03FK01", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "unknown_state", + "segment": "state", + "suggestion": "OG-01-A03-FK-01" + } + } + }, + { + "id": "parse-segments-004", + "description": "suggests I for a one in the state", + "input": "1M01A03FK01", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "unknown_state", + "segment": "state", + "suggestion": "IM-01-A03-FK-01" + } + } + }, + { + "id": "parse-segments-005", + "description": "gives no suggestion when the fixed state is unknown", + "input": "E101A03FK01", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "unknown_state", + "segment": "state", + "suggestion": null + } + } + }, + { + "id": "parse-segments-006", + "description": "gives no suggestion when the state cannot be fixed", + "input": "XXO1A03FK01", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "unknown_state", + "segment": "state", + "suggestion": null + } + } + }, + { + "id": "parse-segments-007", + "description": "fixes later segments in a state suggestion", + "input": "0GO1A03FK01", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "unknown_state", + "segment": "state", + "suggestion": "OG-01-A03-FK-01" + } + } + }, + { + "id": "parse-segments-008", + "description": "rejects an LGA of 00", + "input": "EK00A03FK01", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "lga", + "suggestion": null + } + } + }, + { + "id": "parse-segments-009", + "description": "rejects a unit of 00", + "input": "EK01A03FK00", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "unit", + "suggestion": null + } + } + }, + { + "id": "parse-segments-010", + "description": "suggests 0 for a letter O in the LGA", + "input": "EKO1A03FK01", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "lga", + "suggestion": "EK-01-A03-FK-01" + } + } + }, + { + "id": "parse-segments-011", + "description": "suggests 1 for a letter I in the LGA", + "input": "EKI1A03FK01", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "lga", + "suggestion": "EK-11-A03-FK-01" + } + } + }, + { + "id": "parse-segments-012", + "description": "suggests 1 for a letter L in the LGA", + "input": "EKL1A03FK01", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "lga", + "suggestion": "EK-11-A03-FK-01" + } + } + }, + { + "id": "parse-segments-013", + "description": "suggests I for a one in the area", + "input": "EK01A03F101", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "area", + "suggestion": "EK-01-A03-FI-01" + } + } + }, + { + "id": "parse-segments-014", + "description": "suggests O for a zero in the area", + "input": "EK01A03F001", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "area", + "suggestion": "EK-01-A03-FO-01" + } + } + }, + { + "id": "parse-segments-015", + "description": "suggests 0 for a letter O in the unit", + "input": "EK01A03FKO1", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "unit", + "suggestion": "EK-01-A03-FK-01" + } + } + }, + { + "id": "parse-segments-016", + "description": "suggests 1 for a letter I in the unit", + "input": "EK01A03FKI2", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "unit", + "suggestion": "EK-01-A03-FK-12" + } + } + }, + { + "id": "parse-segments-017", + "description": "suggests 1 for a letter I after a zero in the unit", + "input": "EK01A03FK0I", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "unit", + "suggestion": "EK-01-A03-FK-01" + } + } + }, + { + "id": "parse-segments-018", + "description": "gives no suggestion when the fix makes a unit of 00", + "input": "EK01A03FK0O", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "unit", + "suggestion": null + } + } + }, + { + "id": "parse-segments-019", + "description": "gives no suggestion when a segment cannot be fixed", + "input": "EK01A03F2O1", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "area", + "suggestion": null + } + } + }, + { + "id": "parse-segments-020", + "description": "reports the first bad segment and fixes them all", + "input": "EKO1A03F1O1", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "lga", + "suggestion": "EK-01-A03-FI-01" + } + } + }, + { + "id": "parse-segments-021", + "description": "suggests a fix after making letters upper case", + "input": "eko1a03fk01", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "lga", + "suggestion": "EK-01-A03-FK-01" + } + } + }, + { + "id": "parse-segments-022", + "description": "rejects an LGA of 00 in a partial code", + "input": "EK00", + "options": { + "allowPartial": true + }, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "lga", + "suggestion": null + } + } + }, + { + "id": "parse-segments-023", + "description": "suggests a partial code", + "input": "EKO1", + "options": { + "allowPartial": true + }, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "lga", + "suggestion": "EK-01" + } + } + }, + { + "id": "parse-segments-024", + "description": "suggests a partial area code", + "input": "EK01A03F1", + "options": { + "allowPartial": true + }, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "area", + "suggestion": "EK-01-A03-FI" + } + } + }, + { + "id": "parse-segments-025", + "description": "never changes the district", + "input": "EK01AO3FK01", + "options": {}, + "expect": { + "ok": true, + "compact": "EK01AO3FK01", + "canonical": "EK-01-AO3-FK-01", + "display": "EK 01 AO3 FK 01", + "precision": "unit", + "segments": { + "state": "EK", + "lga": "01", + "district": "AO3", + "area": "FK", + "unit": "01" + } + } + }, + { + "id": "parse-segments-026", + "description": "accepts letters and digits in any district position", + "input": "EK01A0OFK01", + "options": {}, + "expect": { + "ok": true, + "compact": "EK01A0OFK01", + "canonical": "EK-01-A0O-FK-01", + "display": "EK 01 A0O FK 01", + "precision": "unit", + "segments": { + "state": "EK", + "lga": "01", + "district": "A0O", + "area": "FK", + "unit": "01" + } + } + } + ] +} diff --git a/vectors/parse-states.json b/vectors/parse-states.json new file mode 100644 index 0000000..05fa55a --- /dev/null +++ b/vectors/parse-states.json @@ -0,0 +1,747 @@ +{ + "version": 1, + "function": "parse", + "description": "One valid code for each state.", + "cases": [ + { + "id": "parse-states-001", + "description": "parses a code in Abia", + "input": "AB-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "AB01Z99ZZ01", + "canonical": "AB-01-Z99-ZZ-01", + "display": "AB 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "AB", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-002", + "description": "parses a code in Adamawa", + "input": "AD-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "AD01Z99ZZ01", + "canonical": "AD-01-Z99-ZZ-01", + "display": "AD 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "AD", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-003", + "description": "parses a code in Akwa Ibom", + "input": "AK-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "AK01Z99ZZ01", + "canonical": "AK-01-Z99-ZZ-01", + "display": "AK 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "AK", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-004", + "description": "parses a code in Anambra", + "input": "AN-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "AN01Z99ZZ01", + "canonical": "AN-01-Z99-ZZ-01", + "display": "AN 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "AN", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-005", + "description": "parses a code in Bauchi", + "input": "BA-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "BA01Z99ZZ01", + "canonical": "BA-01-Z99-ZZ-01", + "display": "BA 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "BA", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-006", + "description": "parses a code in Benue", + "input": "BE-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "BE01Z99ZZ01", + "canonical": "BE-01-Z99-ZZ-01", + "display": "BE 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "BE", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-007", + "description": "parses a code in Borno", + "input": "BO-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "BO01Z99ZZ01", + "canonical": "BO-01-Z99-ZZ-01", + "display": "BO 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "BO", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-008", + "description": "parses a code in Bayelsa", + "input": "BY-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "BY01Z99ZZ01", + "canonical": "BY-01-Z99-ZZ-01", + "display": "BY 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "BY", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-009", + "description": "parses a code in Cross River", + "input": "CR-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "CR01Z99ZZ01", + "canonical": "CR-01-Z99-ZZ-01", + "display": "CR 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "CR", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-010", + "description": "parses a code in Delta", + "input": "DE-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "DE01Z99ZZ01", + "canonical": "DE-01-Z99-ZZ-01", + "display": "DE 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "DE", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-011", + "description": "parses a code in Ebonyi", + "input": "EB-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "EB01Z99ZZ01", + "canonical": "EB-01-Z99-ZZ-01", + "display": "EB 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "EB", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-012", + "description": "parses a code in Edo", + "input": "ED-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "ED01Z99ZZ01", + "canonical": "ED-01-Z99-ZZ-01", + "display": "ED 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "ED", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-013", + "description": "parses a code in Ekiti", + "input": "EK-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "EK01Z99ZZ01", + "canonical": "EK-01-Z99-ZZ-01", + "display": "EK 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "EK", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-014", + "description": "parses a code in Enugu", + "input": "EN-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "EN01Z99ZZ01", + "canonical": "EN-01-Z99-ZZ-01", + "display": "EN 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "EN", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-015", + "description": "parses a code in Federal Capital Territory", + "input": "FC-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "FC01Z99ZZ01", + "canonical": "FC-01-Z99-ZZ-01", + "display": "FC 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "FC", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-016", + "description": "parses a code in Gombe", + "input": "GO-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "GO01Z99ZZ01", + "canonical": "GO-01-Z99-ZZ-01", + "display": "GO 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "GO", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-017", + "description": "parses a code in Imo", + "input": "IM-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "IM01Z99ZZ01", + "canonical": "IM-01-Z99-ZZ-01", + "display": "IM 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "IM", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-018", + "description": "parses a code in Jigawa", + "input": "JI-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "JI01Z99ZZ01", + "canonical": "JI-01-Z99-ZZ-01", + "display": "JI 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "JI", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-019", + "description": "parses a code in Kaduna", + "input": "KD-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "KD01Z99ZZ01", + "canonical": "KD-01-Z99-ZZ-01", + "display": "KD 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "KD", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-020", + "description": "parses a code in Kebbi", + "input": "KE-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "KE01Z99ZZ01", + "canonical": "KE-01-Z99-ZZ-01", + "display": "KE 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "KE", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-021", + "description": "parses a code in Kano", + "input": "KN-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "KN01Z99ZZ01", + "canonical": "KN-01-Z99-ZZ-01", + "display": "KN 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "KN", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-022", + "description": "parses a code in Kogi", + "input": "KO-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "KO01Z99ZZ01", + "canonical": "KO-01-Z99-ZZ-01", + "display": "KO 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "KO", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-023", + "description": "parses a code in Katsina", + "input": "KT-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "KT01Z99ZZ01", + "canonical": "KT-01-Z99-ZZ-01", + "display": "KT 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "KT", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-024", + "description": "parses a code in Kwara", + "input": "KW-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "KW01Z99ZZ01", + "canonical": "KW-01-Z99-ZZ-01", + "display": "KW 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "KW", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-025", + "description": "parses a code in Lagos", + "input": "LA-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "LA01Z99ZZ01", + "canonical": "LA-01-Z99-ZZ-01", + "display": "LA 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "LA", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-026", + "description": "parses a code in Nasarawa", + "input": "NA-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "NA01Z99ZZ01", + "canonical": "NA-01-Z99-ZZ-01", + "display": "NA 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "NA", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-027", + "description": "parses a code in Niger", + "input": "NI-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "NI01Z99ZZ01", + "canonical": "NI-01-Z99-ZZ-01", + "display": "NI 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "NI", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-028", + "description": "parses a code in Ogun", + "input": "OG-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "OG01Z99ZZ01", + "canonical": "OG-01-Z99-ZZ-01", + "display": "OG 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "OG", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-029", + "description": "parses a code in Ondo", + "input": "ON-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "ON01Z99ZZ01", + "canonical": "ON-01-Z99-ZZ-01", + "display": "ON 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "ON", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-030", + "description": "parses a code in Osun", + "input": "OS-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "OS01Z99ZZ01", + "canonical": "OS-01-Z99-ZZ-01", + "display": "OS 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "OS", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-031", + "description": "parses a code in Oyo", + "input": "OY-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "OY01Z99ZZ01", + "canonical": "OY-01-Z99-ZZ-01", + "display": "OY 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "OY", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-032", + "description": "parses a code in Plateau", + "input": "PL-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "PL01Z99ZZ01", + "canonical": "PL-01-Z99-ZZ-01", + "display": "PL 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "PL", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-033", + "description": "parses a code in Rivers", + "input": "RI-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "RI01Z99ZZ01", + "canonical": "RI-01-Z99-ZZ-01", + "display": "RI 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "RI", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-034", + "description": "parses a code in Sokoto", + "input": "SO-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "SO01Z99ZZ01", + "canonical": "SO-01-Z99-ZZ-01", + "display": "SO 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "SO", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-035", + "description": "parses a code in Taraba", + "input": "TA-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "TA01Z99ZZ01", + "canonical": "TA-01-Z99-ZZ-01", + "display": "TA 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "TA", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-036", + "description": "parses a code in Yobe", + "input": "YO-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "YO01Z99ZZ01", + "canonical": "YO-01-Z99-ZZ-01", + "display": "YO 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "YO", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + }, + { + "id": "parse-states-037", + "description": "parses a code in Zamfara", + "input": "ZA-01-Z99-ZZ-01", + "options": {}, + "expect": { + "ok": true, + "compact": "ZA01Z99ZZ01", + "canonical": "ZA-01-Z99-ZZ-01", + "display": "ZA 01 Z99 ZZ 01", + "precision": "unit", + "segments": { + "state": "ZA", + "lga": "01", + "district": "Z99", + "area": "ZZ", + "unit": "01" + } + } + } + ] +} diff --git a/vectors/parse.json b/vectors/parse.json new file mode 100644 index 0000000..1fba2e9 --- /dev/null +++ b/vectors/parse.json @@ -0,0 +1,623 @@ +{ + "version": 1, + "function": "parse", + "description": "Forms, lengths, characters and legacy codes.", + "cases": [ + { + "id": "parse-001", + "description": "parses the canonical form", + "input": "EK-01-A03-FK-01", + "options": {}, + "expect": { + "ok": true, + "compact": "EK01A03FK01", + "canonical": "EK-01-A03-FK-01", + "display": "EK 01 A03 FK 01", + "precision": "unit", + "segments": { + "state": "EK", + "lga": "01", + "district": "A03", + "area": "FK", + "unit": "01" + } + } + }, + { + "id": "parse-002", + "description": "parses the compact form", + "input": "EK01A03FK01", + "options": {}, + "expect": { + "ok": true, + "compact": "EK01A03FK01", + "canonical": "EK-01-A03-FK-01", + "display": "EK 01 A03 FK 01", + "precision": "unit", + "segments": { + "state": "EK", + "lga": "01", + "district": "A03", + "area": "FK", + "unit": "01" + } + } + }, + { + "id": "parse-003", + "description": "parses the display form in lower case", + "input": "ek 01 a03 fk 01", + "options": {}, + "expect": { + "ok": true, + "compact": "EK01A03FK01", + "canonical": "EK-01-A03-FK-01", + "display": "EK 01 A03 FK 01", + "precision": "unit", + "segments": { + "state": "EK", + "lga": "01", + "district": "A03", + "area": "FK", + "unit": "01" + } + } + }, + { + "id": "parse-004", + "description": "parses a code with en dashes", + "input": "EK\u201301\u2013A03\u2013FK\u201301", + "options": {}, + "expect": { + "ok": true, + "compact": "EK01A03FK01", + "canonical": "EK-01-A03-FK-01", + "display": "EK 01 A03 FK 01", + "precision": "unit", + "segments": { + "state": "EK", + "lga": "01", + "district": "A03", + "area": "FK", + "unit": "01" + } + } + }, + { + "id": "parse-005", + "description": "parses a code with no-break spaces", + "input": "EK\u00a001\u00a0A03\u00a0FK\u00a001", + "options": {}, + "expect": { + "ok": true, + "compact": "EK01A03FK01", + "canonical": "EK-01-A03-FK-01", + "display": "EK 01 A03 FK 01", + "precision": "unit", + "segments": { + "state": "EK", + "lga": "01", + "district": "A03", + "area": "FK", + "unit": "01" + } + } + }, + { + "id": "parse-006", + "description": "parses a code with a zero-width space", + "input": "EK01A03FK01\u200b", + "options": {}, + "expect": { + "ok": true, + "compact": "EK01A03FK01", + "canonical": "EK-01-A03-FK-01", + "display": "EK 01 A03 FK 01", + "precision": "unit", + "segments": { + "state": "EK", + "lga": "01", + "district": "A03", + "area": "FK", + "unit": "01" + } + } + }, + { + "id": "parse-007", + "description": "parses full-width letters and digits", + "input": "\uff25\uff2b-\uff10\uff11-\uff21\uff10\uff13-\uff26\uff2b-\uff10\uff11", + "options": {}, + "expect": { + "ok": true, + "compact": "EK01A03FK01", + "canonical": "EK-01-A03-FK-01", + "display": "EK 01 A03 FK 01", + "precision": "unit", + "segments": { + "state": "EK", + "lga": "01", + "district": "A03", + "area": "FK", + "unit": "01" + } + } + }, + { + "id": "parse-008", + "description": "parses the example on NIPOST's homepage", + "input": "FC-02-A09-DB-09", + "options": {}, + "expect": { + "ok": true, + "compact": "FC02A09DB09", + "canonical": "FC-02-A09-DB-09", + "display": "FC 02 A09 DB 09", + "precision": "unit", + "segments": { + "state": "FC", + "lga": "02", + "district": "A09", + "area": "DB", + "unit": "09" + } + } + }, + { + "id": "parse-009", + "description": "parses the example in NIPOST's blog", + "input": "LA 12 K23 IV 95", + "options": {}, + "expect": { + "ok": true, + "compact": "LA12K23IV95", + "canonical": "LA-12-K23-IV-95", + "display": "LA 12 K23 IV 95", + "precision": "unit", + "segments": { + "state": "LA", + "lga": "12", + "district": "K23", + "area": "IV", + "unit": "95" + } + } + }, + { + "id": "parse-010", + "description": "parses a NIPOST test code", + "input": "AK-11-I61-ZF-12", + "options": {}, + "expect": { + "ok": true, + "compact": "AK11I61ZF12", + "canonical": "AK-11-I61-ZF-12", + "display": "AK 11 I61 ZF 12", + "precision": "unit", + "segments": { + "state": "AK", + "lga": "11", + "district": "I61", + "area": "ZF", + "unit": "12" + } + } + }, + { + "id": "parse-011", + "description": "accepts a letter O in the district", + "input": "JI-24-O18-JP-23", + "options": {}, + "expect": { + "ok": true, + "compact": "JI24O18JP23", + "canonical": "JI-24-O18-JP-23", + "display": "JI 24 O18 JP 23", + "precision": "unit", + "segments": { + "state": "JI", + "lga": "24", + "district": "O18", + "area": "JP", + "unit": "23" + } + } + }, + { + "id": "parse-012", + "description": "rejects empty input", + "input": "", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "empty", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-013", + "description": "rejects spaces only", + "input": " ", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "empty", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-014", + "description": "rejects separators only", + "input": " - . ", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "empty", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-015", + "description": "names a legacy code", + "input": "900108", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "legacy_code", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-016", + "description": "names a legacy code with a space", + "input": "900 108", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "legacy_code", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-017", + "description": "names a legacy code in full-width digits", + "input": "\uff19\uff10\uff10\uff11\uff10\uff18", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "legacy_code", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-018", + "description": "names a legacy code when partial codes are allowed", + "input": "123456", + "options": { + "allowPartial": true + }, + "expect": { + "ok": false, + "error": { + "code": "legacy_code", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-019", + "description": "rejects five digits", + "input": "90010", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_length", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-020", + "description": "rejects an exclamation mark", + "input": "EK01A03FK0!", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_character", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-021", + "description": "rejects an underscore", + "input": "EK01A03FK0_", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_character", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-022", + "description": "rejects slashes", + "input": "EK01/A03/FK01", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_character", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-023", + "description": "rejects a letter outside ASCII", + "input": "EK01A03F\u00e901", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_character", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-024", + "description": "rejects a right-to-left override", + "input": "\u202eEK01A03FK01", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_character", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-025", + "description": "rejects 10 characters", + "input": "EK01A03FK1", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_length", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-026", + "description": "rejects 12 characters", + "input": "EK01A03FK01X", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_length", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-027", + "description": "rejects a unit that lost its zero", + "input": "EK-01-A03-FK-1", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_length", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-028", + "description": "rejects a state alone by default", + "input": "EK", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_length", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-029", + "description": "rejects a district code by default", + "input": "EK01A03", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_length", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-030", + "description": "parses a state when partial codes are allowed", + "input": "EK", + "options": { + "allowPartial": true + }, + "expect": { + "ok": true, + "compact": "EK", + "canonical": "EK", + "display": "EK", + "precision": "state", + "segments": { + "state": "EK", + "lga": null, + "district": null, + "area": null, + "unit": null + } + } + }, + { + "id": "parse-031", + "description": "parses an LGA code when partial codes are allowed", + "input": "EK01", + "options": { + "allowPartial": true + }, + "expect": { + "ok": true, + "compact": "EK01", + "canonical": "EK-01", + "display": "EK 01", + "precision": "lga", + "segments": { + "state": "EK", + "lga": "01", + "district": null, + "area": null, + "unit": null + } + } + }, + { + "id": "parse-032", + "description": "parses a district code when partial codes are allowed", + "input": "ek-01-a03", + "options": { + "allowPartial": true + }, + "expect": { + "ok": true, + "compact": "EK01A03", + "canonical": "EK-01-A03", + "display": "EK 01 A03", + "precision": "district", + "segments": { + "state": "EK", + "lga": "01", + "district": "A03", + "area": null, + "unit": null + } + } + }, + { + "id": "parse-033", + "description": "parses an area code when partial codes are allowed", + "input": "EK01A03FK", + "options": { + "allowPartial": true + }, + "expect": { + "ok": true, + "compact": "EK01A03FK", + "canonical": "EK-01-A03-FK", + "display": "EK 01 A03 FK", + "precision": "area", + "segments": { + "state": "EK", + "lga": "01", + "district": "A03", + "area": "FK", + "unit": null + } + } + }, + { + "id": "parse-034", + "description": "parses a full code when partial codes are allowed", + "input": "EK01A03FK01", + "options": { + "allowPartial": true + }, + "expect": { + "ok": true, + "compact": "EK01A03FK01", + "canonical": "EK-01-A03-FK-01", + "display": "EK 01 A03 FK 01", + "precision": "unit", + "segments": { + "state": "EK", + "lga": "01", + "district": "A03", + "area": "FK", + "unit": "01" + } + } + }, + { + "id": "parse-035", + "description": "rejects 3 characters when partial codes are allowed", + "input": "EK0", + "options": { + "allowPartial": true + }, + "expect": { + "ok": false, + "error": { + "code": "bad_length", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-036", + "description": "rejects 6 characters when partial codes are allowed", + "input": "EK01A0", + "options": { + "allowPartial": true + }, + "expect": { + "ok": false, + "error": { + "code": "bad_length", + "segment": null, + "suggestion": null + } + } + } + ] +} diff --git a/vectors/precision-for-accuracy.json b/vectors/precision-for-accuracy.json new file mode 100644 index 0000000..49f845e --- /dev/null +++ b/vectors/precision-for-accuracy.json @@ -0,0 +1,133 @@ +{ + "version": 1, + "function": "precisionForAccuracy", + "description": "Maps GPS accuracy to precision.", + "cases": [ + { + "id": "precision-for-accuracy-001", + "description": "gives the unit at 0 m", + "input": 0, + "options": {}, + "expect": { + "value": "unit" + } + }, + { + "id": "precision-for-accuracy-002", + "description": "gives the unit at 5 m", + "input": 5, + "options": {}, + "expect": { + "value": "unit" + } + }, + { + "id": "precision-for-accuracy-003", + "description": "gives the unit at exactly 8 m", + "input": 8, + "options": {}, + "expect": { + "value": "unit" + } + }, + { + "id": "precision-for-accuracy-004", + "description": "gives the area just above 8 m", + "input": 8.1, + "options": {}, + "expect": { + "value": "area" + } + }, + { + "id": "precision-for-accuracy-005", + "description": "gives the area at exactly 20 m", + "input": 20, + "options": {}, + "expect": { + "value": "area" + } + }, + { + "id": "precision-for-accuracy-006", + "description": "gives the district above 20 m", + "input": 20.5, + "options": {}, + "expect": { + "value": "district" + } + }, + { + "id": "precision-for-accuracy-007", + "description": "gives the district at exactly 50 m", + "input": 50, + "options": {}, + "expect": { + "value": "district" + } + }, + { + "id": "precision-for-accuracy-008", + "description": "gives the LGA just above 50 m", + "input": 50.1, + "options": {}, + "expect": { + "value": "lga" + } + }, + { + "id": "precision-for-accuracy-009", + "description": "gives the LGA at 1000 m", + "input": 1000, + "options": {}, + "expect": { + "value": "lga" + } + }, + { + "id": "precision-for-accuracy-010", + "description": "gives the LGA when the accuracy is unknown", + "input": null, + "options": {}, + "expect": { + "value": "lga" + } + }, + { + "id": "precision-for-accuracy-011", + "description": "gives the LGA for a negative accuracy", + "input": -1, + "options": {}, + "expect": { + "value": "lga" + } + }, + { + "id": "precision-for-accuracy-012", + "description": "gives the LGA for NaN", + "input": "NaN", + "options": {}, + "expect": { + "value": "lga" + } + }, + { + "id": "precision-for-accuracy-013", + "description": "gives the LGA for infinity", + "input": "Infinity", + "options": {}, + "expect": { + "value": "lga" + } + }, + { + "id": "precision-for-accuracy-014", + "description": "gives the LGA for negative infinity", + "input": "-Infinity", + "options": {}, + "expect": { + "value": "lga" + } + } + ] +} diff --git a/vectors/redact.json b/vectors/redact.json new file mode 100644 index 0000000..3f06884 --- /dev/null +++ b/vectors/redact.json @@ -0,0 +1,40 @@ +{ + "version": 1, + "function": "redact", + "description": "Hides the unit for logs.", + "cases": [ + { + "id": "redact-001", + "description": "hides the unit of a full code", + "input": { + "code": "EK-01-A03-FK-01" + }, + "options": {}, + "expect": { + "value": "EK-01-A03-FK-**" + } + }, + { + "id": "redact-002", + "description": "keeps an area code as it is", + "input": { + "code": "EK-01-A03-FK" + }, + "options": {}, + "expect": { + "value": "EK-01-A03-FK" + } + }, + { + "id": "redact-003", + "description": "keeps a state code as it is", + "input": { + "code": "EK" + }, + "options": {}, + "expect": { + "value": "EK" + } + } + ] +} diff --git a/vectors/state-name.json b/vectors/state-name.json new file mode 100644 index 0000000..b95638d --- /dev/null +++ b/vectors/state-name.json @@ -0,0 +1,385 @@ +{ + "version": 1, + "function": "stateName", + "description": "Names the state for a state code.", + "cases": [ + { + "id": "state-name-001", + "description": "names AB", + "input": "AB", + "options": {}, + "expect": { + "value": "Abia" + } + }, + { + "id": "state-name-002", + "description": "names AD", + "input": "AD", + "options": {}, + "expect": { + "value": "Adamawa" + } + }, + { + "id": "state-name-003", + "description": "names AK", + "input": "AK", + "options": {}, + "expect": { + "value": "Akwa Ibom" + } + }, + { + "id": "state-name-004", + "description": "names AN", + "input": "AN", + "options": {}, + "expect": { + "value": "Anambra" + } + }, + { + "id": "state-name-005", + "description": "names BA", + "input": "BA", + "options": {}, + "expect": { + "value": "Bauchi" + } + }, + { + "id": "state-name-006", + "description": "names BE", + "input": "BE", + "options": {}, + "expect": { + "value": "Benue" + } + }, + { + "id": "state-name-007", + "description": "names BO", + "input": "BO", + "options": {}, + "expect": { + "value": "Borno" + } + }, + { + "id": "state-name-008", + "description": "names BY", + "input": "BY", + "options": {}, + "expect": { + "value": "Bayelsa" + } + }, + { + "id": "state-name-009", + "description": "names CR", + "input": "CR", + "options": {}, + "expect": { + "value": "Cross River" + } + }, + { + "id": "state-name-010", + "description": "names DE", + "input": "DE", + "options": {}, + "expect": { + "value": "Delta" + } + }, + { + "id": "state-name-011", + "description": "names EB", + "input": "EB", + "options": {}, + "expect": { + "value": "Ebonyi" + } + }, + { + "id": "state-name-012", + "description": "names ED", + "input": "ED", + "options": {}, + "expect": { + "value": "Edo" + } + }, + { + "id": "state-name-013", + "description": "names EK", + "input": "EK", + "options": {}, + "expect": { + "value": "Ekiti" + } + }, + { + "id": "state-name-014", + "description": "names EN", + "input": "EN", + "options": {}, + "expect": { + "value": "Enugu" + } + }, + { + "id": "state-name-015", + "description": "names FC", + "input": "FC", + "options": {}, + "expect": { + "value": "Federal Capital Territory" + } + }, + { + "id": "state-name-016", + "description": "names GO", + "input": "GO", + "options": {}, + "expect": { + "value": "Gombe" + } + }, + { + "id": "state-name-017", + "description": "names IM", + "input": "IM", + "options": {}, + "expect": { + "value": "Imo" + } + }, + { + "id": "state-name-018", + "description": "names JI", + "input": "JI", + "options": {}, + "expect": { + "value": "Jigawa" + } + }, + { + "id": "state-name-019", + "description": "names KD", + "input": "KD", + "options": {}, + "expect": { + "value": "Kaduna" + } + }, + { + "id": "state-name-020", + "description": "names KE", + "input": "KE", + "options": {}, + "expect": { + "value": "Kebbi" + } + }, + { + "id": "state-name-021", + "description": "names KN", + "input": "KN", + "options": {}, + "expect": { + "value": "Kano" + } + }, + { + "id": "state-name-022", + "description": "names KO", + "input": "KO", + "options": {}, + "expect": { + "value": "Kogi" + } + }, + { + "id": "state-name-023", + "description": "names KT", + "input": "KT", + "options": {}, + "expect": { + "value": "Katsina" + } + }, + { + "id": "state-name-024", + "description": "names KW", + "input": "KW", + "options": {}, + "expect": { + "value": "Kwara" + } + }, + { + "id": "state-name-025", + "description": "names LA", + "input": "LA", + "options": {}, + "expect": { + "value": "Lagos" + } + }, + { + "id": "state-name-026", + "description": "names NA", + "input": "NA", + "options": {}, + "expect": { + "value": "Nasarawa" + } + }, + { + "id": "state-name-027", + "description": "names NI", + "input": "NI", + "options": {}, + "expect": { + "value": "Niger" + } + }, + { + "id": "state-name-028", + "description": "names OG", + "input": "OG", + "options": {}, + "expect": { + "value": "Ogun" + } + }, + { + "id": "state-name-029", + "description": "names ON", + "input": "ON", + "options": {}, + "expect": { + "value": "Ondo" + } + }, + { + "id": "state-name-030", + "description": "names OS", + "input": "OS", + "options": {}, + "expect": { + "value": "Osun" + } + }, + { + "id": "state-name-031", + "description": "names OY", + "input": "OY", + "options": {}, + "expect": { + "value": "Oyo" + } + }, + { + "id": "state-name-032", + "description": "names PL", + "input": "PL", + "options": {}, + "expect": { + "value": "Plateau" + } + }, + { + "id": "state-name-033", + "description": "names RI", + "input": "RI", + "options": {}, + "expect": { + "value": "Rivers" + } + }, + { + "id": "state-name-034", + "description": "names SO", + "input": "SO", + "options": {}, + "expect": { + "value": "Sokoto" + } + }, + { + "id": "state-name-035", + "description": "names TA", + "input": "TA", + "options": {}, + "expect": { + "value": "Taraba" + } + }, + { + "id": "state-name-036", + "description": "names YO", + "input": "YO", + "options": {}, + "expect": { + "value": "Yobe" + } + }, + { + "id": "state-name-037", + "description": "names ZA", + "input": "ZA", + "options": {}, + "expect": { + "value": "Zamfara" + } + }, + { + "id": "state-name-038", + "description": "accepts lower case", + "input": "ek", + "options": {}, + "expect": { + "value": "Ekiti" + } + }, + { + "id": "state-name-039", + "description": "accepts mixed case", + "input": "Fc", + "options": {}, + "expect": { + "value": "Federal Capital Territory" + } + }, + { + "id": "state-name-040", + "description": "returns nothing for an unknown code", + "input": "XX", + "options": {}, + "expect": { + "value": null + } + }, + { + "id": "state-name-041", + "description": "returns nothing for empty input", + "input": "", + "options": {}, + "expect": { + "value": null + } + }, + { + "id": "state-name-042", + "description": "returns nothing for three letters", + "input": "EKI", + "options": {}, + "expect": { + "value": null + } + } + ] +} diff --git a/vectors/truncate.json b/vectors/truncate.json new file mode 100644 index 0000000..1161a4e --- /dev/null +++ b/vectors/truncate.json @@ -0,0 +1,103 @@ +{ + "version": 1, + "function": "truncate", + "description": "Cuts a code to a less precise segment.", + "cases": [ + { + "id": "truncate-001", + "description": "keeps a full code at unit precision", + "input": { + "code": "EK-01-A03-FK-01", + "to": "unit" + }, + "options": {}, + "expect": { + "canonical": "EK-01-A03-FK-01" + } + }, + { + "id": "truncate-002", + "description": "cuts a full code to its area", + "input": { + "code": "EK-01-A03-FK-01", + "to": "area" + }, + "options": {}, + "expect": { + "canonical": "EK-01-A03-FK" + } + }, + { + "id": "truncate-003", + "description": "cuts a full code to its district", + "input": { + "code": "EK-01-A03-FK-01", + "to": "district" + }, + "options": {}, + "expect": { + "canonical": "EK-01-A03" + } + }, + { + "id": "truncate-004", + "description": "cuts a full code to its LGA", + "input": { + "code": "EK-01-A03-FK-01", + "to": "lga" + }, + "options": {}, + "expect": { + "canonical": "EK-01" + } + }, + { + "id": "truncate-005", + "description": "cuts a full code to its state", + "input": { + "code": "EK-01-A03-FK-01", + "to": "state" + }, + "options": {}, + "expect": { + "canonical": "EK" + } + }, + { + "id": "truncate-006", + "description": "keeps a district code at district precision", + "input": { + "code": "EK-01-A03", + "to": "district" + }, + "options": {}, + "expect": { + "canonical": "EK-01-A03" + } + }, + { + "id": "truncate-007", + "description": "refuses to add precision to a district code", + "input": { + "code": "EK-01-A03", + "to": "area" + }, + "options": {}, + "expect": { + "rejects": true + } + }, + { + "id": "truncate-008", + "description": "refuses to add precision to a state code", + "input": { + "code": "EK", + "to": "lga" + }, + "options": {}, + "expect": { + "rejects": true + } + } + ] +} From ae50806530f8592eb920dcba6fe757f57a37fe9e Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 00:30:16 +0100 Subject: [PATCH 11/45] docs: say that vector ids hold within one spec version The builder numbers each case by its position, so a new case moves the ids after it. Each SDK runs the vectors of one spec version, so an id and that version name one case. Signed-off-by: MobileMage --- vectors/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vectors/README.md b/vectors/README.md index e4b0d61..6ac5c23 100644 --- a/vectors/README.md +++ b/vectors/README.md @@ -12,7 +12,7 @@ Each case has these fields: | Field | Meaning | |---|---| -| `id` | a stable id, such as `parse-001` | +| `id` | an id that is unique within one spec version, such as `parse-001`. A new case can change the ids after it. | | `description` | the behaviour in domain words. Use it in the test name. | | `input` | the argument, or an object of named arguments | | `options` | options for the function, such as `{ "allowPartial": true }` | From bf4eba327dc088f414e0b771fb5dfd375f4e1391 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 00:56:23 +0100 Subject: [PATCH 12/45] fix: close gaps in the shared vectors A wrong SDK could pass every vector. No parse case held a byte order mark, 13 separators had no case, no case showed that a suggestion keeps the district, and no case held non-ASCII digits or partial codes of 5, 8 or 10 characters. The builder now writes one case for each separator in data/format.json. A test ties the GPS limits in data/precision.json to the vectors, and descriptions are unique across all files. Signed-off-by: MobileMage --- grammar.md | 2 +- scripts/build_vectors.py | 81 +++++- scripts/tests/test_build_vectors.py | 28 ++ vectors/is-legacy.json | 13 +- vectors/normalize.json | 401 +++++++++++++++++++++++++++- vectors/parse-segments.json | 66 +++-- vectors/parse.json | 142 +++++++--- 7 files changed, 661 insertions(+), 72 deletions(-) diff --git a/grammar.md b/grammar.md index 8d98d62..a531dc0 100644 --- a/grammar.md +++ b/grammar.md @@ -70,7 +70,7 @@ Parse the corrected code with the same options. If it parses, the suggestion is ## GPS precision -`precisionForAccuracy(metres)` returns the most precise segment that a GPS fix of that accuracy supports, from `data/precision.json`. An unknown, negative, infinite or NaN accuracy gives `lga`. +`precisionForAccuracy(metres)` returns the most precise segment that a GPS fix of that accuracy supports, from `data/precision.json`. An unknown, negative, infinite or NaN accuracy gives `lga`. A fix with an accuracy equal to a limit gets the precision of that limit. ## Versions diff --git a/scripts/build_vectors.py b/scripts/build_vectors.py index 5652644..4b4254c 100755 --- a/scripts/build_vectors.py +++ b/scripts/build_vectors.py @@ -37,6 +37,10 @@ LEFT_TO_RIGHT_MARK = chr(0x200E) RIGHT_TO_LEFT_MARK = chr(0x200F) RIGHT_TO_LEFT_OVERRIDE = chr(0x202E) +FULL_WIDTH_HYPHEN = chr(0xFF0D) +ARABIC_INDIC_LEGACY = "".join( + chr(code_point) for code_point in (0x0669, 0x0660, 0x0660, 0x0661, 0x0660, 0x0668) +) WIDTHS = (("state", 2), ("lga", 2), ("district", 3), ("area", 2), ("unit", 2)) PRECISION_BY_LENGTH = {2: "state", 4: "lga", 7: "district", 9: "area", 11: "unit"} @@ -91,9 +95,9 @@ def failed(code: str, segment: str | None = None, suggestion: str | None = None) return {"ok": False, "error": {"code": code, "segment": segment, "suggestion": suggestion}} -def value(result: Any) -> dict[str, Any]: +def value(returned: Any) -> dict[str, Any]: """The expected result of a function that returns one value.""" - return {"value": result} + return {"value": returned} def load_states() -> list[dict[str, str]]: @@ -106,9 +110,19 @@ def load_states() -> list[dict[str, str]]: return states +def load_separators() -> list[int]: + """Read data/format.json and return its separators as code points.""" + postcode_format = json.loads((ROOT / "data" / "format.json").read_text(encoding="utf-8")) + return [int(label.removeprefix("U+"), 16) for label in postcode_format["separators"]] + + def normalize_rows() -> list[Row]: """Cases for normalize.""" want = value(CODE) + separator_rows = [ + row(f"removes the separator U+{code_point:04X}", joined(chr(code_point)), want) + for code_point in load_separators() + ] return [ row("removes hyphens", "EK-01-A03-FK-01", want), row("removes spaces and makes letters upper case", "ek 01 a03 fk 01", want), @@ -129,6 +143,7 @@ def normalize_rows() -> list[Row]: value(RIGHT_TO_LEFT_OVERRIDE + CODE), ), row("changes full-width letters and digits to ASCII", full_width(CODE), want), + row("removes full-width hyphens", joined(FULL_WIDTH_HYPHEN), want), row("removes tabs and line feeds", "EK01" + TAB + "A03" + LINE_FEED + "FK01", want), row( "changes a ligature to its letters", @@ -145,6 +160,7 @@ def normalize_rows() -> list[Row]: ), row("returns an empty string for empty input", "", value("")), row("returns an empty string for separators only", " - . ", value("")), + *separator_rows, ] @@ -157,11 +173,16 @@ def parse_rows() -> list[Row]: row("parses a code with en dashes", joined(EN_DASH), ok(CODE)), row("parses a code with no-break spaces", joined(NO_BREAK_SPACE), ok(CODE)), row("parses a code with a zero-width space", CODE + ZERO_WIDTH_SPACE, ok(CODE)), + row("parses a code with a byte order mark", BYTE_ORDER_MARK + CODE, ok(CODE)), row("parses full-width letters and digits", full_width("EK-01-A03-FK-01"), ok(CODE)), row("parses the example on NIPOST's homepage", "FC-02-A09-DB-09", ok("FC02A09DB09")), row("parses the example in NIPOST's blog", "LA 12 K23 IV 95", ok("LA12K23IV95")), row("parses a NIPOST test code", "AK-11-I61-ZF-12", ok("AK11I61ZF12")), - row("accepts a letter O in the district", "JI-24-O18-JP-23", ok("JI24O18JP23")), + row( + "accepts a letter O in the district of a NIPOST test code", + "JI-24-O18-JP-23", + ok("JI24O18JP23"), + ), row("rejects empty input", "", failed("empty")), row("rejects spaces only", " ", failed("empty")), row("rejects separators only", " - . ", failed("empty")), @@ -190,6 +211,7 @@ def parse_rows() -> list[Row]: RIGHT_TO_LEFT_OVERRIDE + CODE, failed("bad_character"), ), + row("rejects Arabic-Indic digits", ARABIC_INDIC_LEGACY, failed("bad_character")), row("rejects 10 characters", "EK01A03FK1", failed("bad_length")), row("rejects 12 characters", "EK01A03FK01X", failed("bad_length")), row("rejects a unit that lost its zero", "EK-01-A03-FK-1", failed("bad_length")), @@ -216,12 +238,30 @@ def parse_rows() -> list[Row]: failed("bad_length"), PARTIAL, ), + row( + "rejects 5 characters when partial codes are allowed", + "EK01A", + failed("bad_length"), + PARTIAL, + ), row( "rejects 6 characters when partial codes are allowed", "EK01A0", failed("bad_length"), PARTIAL, ), + row( + "rejects 8 characters when partial codes are allowed", + "EK01A03F", + failed("bad_length"), + PARTIAL, + ), + row( + "rejects 10 characters when partial codes are allowed", + "EK01A03FK0", + failed("bad_length"), + PARTIAL, + ), ] @@ -292,6 +332,11 @@ def segment_rows() -> list[Row]: "EK01A03FKI2", failed("bad_segment", "unit", "EK-01-A03-FK-12"), ), + row( + "suggests 1 for a letter L in the unit", + "EK01A03FKL1", + failed("bad_segment", "unit", "EK-01-A03-FK-11"), + ), row( "suggests 1 for a letter I after a zero in the unit", "EK01A03FK0I", @@ -327,7 +372,16 @@ def segment_rows() -> list[Row]: failed("bad_segment", "area", "EK-01-A03-FI"), PARTIAL, ), - row("never changes the district", "EK01AO3FK01", ok("EK01AO3FK01")), + row( + "keeps a letter O in the district when it fixes the LGA", + "EKO1AO3FK01", + failed("bad_segment", "lga", "EK-01-AO3-FK-01"), + ), + row( + "keeps letters I and L in the district when it fixes the LGA", + "EKO1AILFK01", + failed("bad_segment", "lga", "EK-01-AIL-FK-01"), + ), row( "accepts letters and digits in any district position", "EK01A0OFK01", ok("EK01A0OFK01") ), @@ -352,11 +406,12 @@ def legacy_rows() -> list[Row]: row("accepts six digits", "900108", value(True)), row("accepts six digits with a space", "900 108", value(True)), row("accepts six full-width digits", full_width("900108"), value(True)), - row("rejects five digits", "90010", value(False)), + row("rejects five digits as a legacy code", "90010", value(False)), row("rejects seven digits", "9001080", value(False)), row("rejects a new postcode", CODE, value(False)), - row("rejects empty input", "", value(False)), + row("rejects empty input as a legacy code", "", value(False)), row("rejects a letter O among digits", "90O108", value(False)), + row("rejects six Arabic-Indic digits as a legacy code", ARABIC_INDIC_LEGACY, value(False)), ] @@ -535,13 +590,17 @@ def render(stem: str) -> str: def main(argv: list[str] | None = None) -> int: - """Write the vector files, or with --check, report the ones that are out of date.""" + """Write the vector files, or with --check, report the ones that are out of date. + + In both modes, report each JSON file in vectors/ that the builder does not write. + """ parser = argparse.ArgumentParser(description="Build the shared test vectors.") parser.add_argument( "--check", action="store_true", help="Fail if a vector file is out of date." ) args = parser.parse_args(argv) - VECTORS.mkdir(exist_ok=True) + if not args.check: + VECTORS.mkdir(exist_ok=True) stale: list[str] = [] for stem in FILES: path = VECTORS / f"{stem}.json" @@ -550,12 +609,14 @@ def main(argv: list[str] | None = None) -> int: path.write_text(text, encoding="utf-8") elif not path.exists() or path.read_text(encoding="utf-8") != text: stale.append(path.name) - extra = sorted({path.stem for path in VECTORS.glob("*.json")} - set(FILES)) - stale.extend(f"{stem}.json (not built by this script)" for stem in extra) + strays = sorted(path.name for path in VECTORS.glob("*.json") if path.stem not in FILES) if stale: sys.stderr.write( f"Out of date: {', '.join(stale)}. Run python3 scripts/build_vectors.py.\n" ) + for name in strays: + sys.stderr.write(f"vectors/{name} is not a builder output. Remove it.\n") + if stale or strays: return 1 return 0 diff --git a/scripts/tests/test_build_vectors.py b/scripts/tests/test_build_vectors.py index 03cfe8a..090ca21 100644 --- a/scripts/tests/test_build_vectors.py +++ b/scripts/tests/test_build_vectors.py @@ -3,7 +3,9 @@ """Tests for build_vectors.""" import json +import math import unittest +from collections import Counter from pathlib import Path from typing import Any @@ -37,6 +39,13 @@ def test_each_case_has_a_unique_id_and_a_description(self) -> None: self.assertEqual(len(ids), len(set(ids))) self.assertGreaterEqual(len(ids), 200) + def test_each_description_is_unique_across_all_vector_files(self) -> None: + counts = Counter( + case["description"] for stem in build_vectors.FILES for case in load(stem)["cases"] + ) + repeated = sorted(text for text, count in counts.items() if count > 1) + self.assertEqual(repeated, []) + def test_each_state_has_a_parse_case_and_a_name_case(self) -> None: states = json.loads((ROOT / "data" / "states.json").read_text(encoding="utf-8")) codes = {state["code"] for state in states["states"]} @@ -44,3 +53,22 @@ def test_each_state_has_a_parse_case_and_a_name_case(self) -> None: named = {case["input"] for case in load("state-name")["cases"]} self.assertEqual(parsed, codes) self.assertLessEqual(codes, named) + + def test_each_gps_limit_has_a_case_at_the_limit_and_a_case_above_it(self) -> None: + rules = json.loads((ROOT / "data" / "precision.json").read_text(encoding="utf-8")) + limits = [threshold["maxAccuracyM"] for threshold in rules["thresholds"]] + at_limit = [threshold["precision"] for threshold in rules["thresholds"]] + above_limit = [*at_limit[1:], rules["fallback"]] + ceilings = [*limits[1:], math.inf] + measured = [ + (case["input"], case["expect"]["value"]) + for case in load("precision-for-accuracy")["cases"] + if isinstance(case["input"], int | float) + ] + for limit, ceiling, precision, next_precision in zip( + limits, ceilings, at_limit, above_limit, strict=True + ): + with self.subTest(limit=limit): + self.assertIn((limit, precision), measured) + above = [expected for metres, expected in measured if limit < metres <= ceiling] + self.assertIn(next_precision, above) diff --git a/vectors/is-legacy.json b/vectors/is-legacy.json index 9d5fb1e..fc4cb55 100644 --- a/vectors/is-legacy.json +++ b/vectors/is-legacy.json @@ -32,7 +32,7 @@ }, { "id": "is-legacy-004", - "description": "rejects five digits", + "description": "rejects five digits as a legacy code", "input": "90010", "options": {}, "expect": { @@ -59,7 +59,7 @@ }, { "id": "is-legacy-007", - "description": "rejects empty input", + "description": "rejects empty input as a legacy code", "input": "", "options": {}, "expect": { @@ -74,6 +74,15 @@ "expect": { "value": false } + }, + { + "id": "is-legacy-009", + "description": "rejects six Arabic-Indic digits as a legacy code", + "input": "\u0669\u0660\u0660\u0661\u0660\u0668", + "options": {}, + "expect": { + "value": false + } } ] } diff --git a/vectors/normalize.json b/vectors/normalize.json index 981794e..0c3d583 100644 --- a/vectors/normalize.json +++ b/vectors/normalize.json @@ -140,6 +140,15 @@ }, { "id": "normalize-016", + "description": "removes full-width hyphens", + "input": "EK\uff0d01\uff0dA03\uff0dFK\uff0d01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-017", "description": "removes tabs and line feeds", "input": "EK01\tA03\nFK01", "options": {}, @@ -148,7 +157,7 @@ } }, { - "id": "normalize-017", + "id": "normalize-018", "description": "changes a ligature to its letters", "input": "EK01A03\ufb0101", "options": {}, @@ -157,7 +166,7 @@ } }, { - "id": "normalize-018", + "id": "normalize-019", "description": "makes mixed-case letters upper case", "input": "eK01a03Fk01", "options": {}, @@ -166,7 +175,7 @@ } }, { - "id": "normalize-019", + "id": "normalize-020", "description": "keeps a legacy code", "input": "900 108", "options": {}, @@ -175,7 +184,7 @@ } }, { - "id": "normalize-020", + "id": "normalize-021", "description": "keeps characters that are not separators", "input": "EK01A03FK0!", "options": {}, @@ -184,7 +193,7 @@ } }, { - "id": "normalize-021", + "id": "normalize-022", "description": "makes only ASCII letters upper case", "input": "ek01a03f\u00e901", "options": {}, @@ -193,7 +202,7 @@ } }, { - "id": "normalize-022", + "id": "normalize-023", "description": "returns an empty string for empty input", "input": "", "options": {}, @@ -202,13 +211,391 @@ } }, { - "id": "normalize-023", + "id": "normalize-024", "description": "returns an empty string for separators only", "input": " - . ", "options": {}, "expect": { "value": "" } + }, + { + "id": "normalize-025", + "description": "removes the separator U+0009", + "input": "EK\t01\tA03\tFK\t01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-026", + "description": "removes the separator U+000A", + "input": "EK\n01\nA03\nFK\n01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-027", + "description": "removes the separator U+000B", + "input": "EK\u000b01\u000bA03\u000bFK\u000b01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-028", + "description": "removes the separator U+000C", + "input": "EK\f01\fA03\fFK\f01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-029", + "description": "removes the separator U+000D", + "input": "EK\r01\rA03\rFK\r01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-030", + "description": "removes the separator U+0020", + "input": "EK 01 A03 FK 01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-031", + "description": "removes the separator U+002D", + "input": "EK-01-A03-FK-01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-032", + "description": "removes the separator U+002E", + "input": "EK.01.A03.FK.01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-033", + "description": "removes the separator U+0085", + "input": "EK\u008501\u0085A03\u0085FK\u008501", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-034", + "description": "removes the separator U+00A0", + "input": "EK\u00a001\u00a0A03\u00a0FK\u00a001", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-035", + "description": "removes the separator U+00AD", + "input": "EK\u00ad01\u00adA03\u00adFK\u00ad01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-036", + "description": "removes the separator U+1680", + "input": "EK\u168001\u1680A03\u1680FK\u168001", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-037", + "description": "removes the separator U+2000", + "input": "EK\u200001\u2000A03\u2000FK\u200001", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-038", + "description": "removes the separator U+2001", + "input": "EK\u200101\u2001A03\u2001FK\u200101", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-039", + "description": "removes the separator U+2002", + "input": "EK\u200201\u2002A03\u2002FK\u200201", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-040", + "description": "removes the separator U+2003", + "input": "EK\u200301\u2003A03\u2003FK\u200301", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-041", + "description": "removes the separator U+2004", + "input": "EK\u200401\u2004A03\u2004FK\u200401", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-042", + "description": "removes the separator U+2005", + "input": "EK\u200501\u2005A03\u2005FK\u200501", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-043", + "description": "removes the separator U+2006", + "input": "EK\u200601\u2006A03\u2006FK\u200601", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-044", + "description": "removes the separator U+2007", + "input": "EK\u200701\u2007A03\u2007FK\u200701", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-045", + "description": "removes the separator U+2008", + "input": "EK\u200801\u2008A03\u2008FK\u200801", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-046", + "description": "removes the separator U+2009", + "input": "EK\u200901\u2009A03\u2009FK\u200901", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-047", + "description": "removes the separator U+200A", + "input": "EK\u200a01\u200aA03\u200aFK\u200a01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-048", + "description": "removes the separator U+200B", + "input": "EK\u200b01\u200bA03\u200bFK\u200b01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-049", + "description": "removes the separator U+200C", + "input": "EK\u200c01\u200cA03\u200cFK\u200c01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-050", + "description": "removes the separator U+200D", + "input": "EK\u200d01\u200dA03\u200dFK\u200d01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-051", + "description": "removes the separator U+200E", + "input": "EK\u200e01\u200eA03\u200eFK\u200e01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-052", + "description": "removes the separator U+200F", + "input": "EK\u200f01\u200fA03\u200fFK\u200f01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-053", + "description": "removes the separator U+2010", + "input": "EK\u201001\u2010A03\u2010FK\u201001", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-054", + "description": "removes the separator U+2011", + "input": "EK\u201101\u2011A03\u2011FK\u201101", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-055", + "description": "removes the separator U+2012", + "input": "EK\u201201\u2012A03\u2012FK\u201201", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-056", + "description": "removes the separator U+2013", + "input": "EK\u201301\u2013A03\u2013FK\u201301", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-057", + "description": "removes the separator U+2014", + "input": "EK\u201401\u2014A03\u2014FK\u201401", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-058", + "description": "removes the separator U+2015", + "input": "EK\u201501\u2015A03\u2015FK\u201501", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-059", + "description": "removes the separator U+2028", + "input": "EK\u202801\u2028A03\u2028FK\u202801", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-060", + "description": "removes the separator U+2029", + "input": "EK\u202901\u2029A03\u2029FK\u202901", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-061", + "description": "removes the separator U+202F", + "input": "EK\u202f01\u202fA03\u202fFK\u202f01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-062", + "description": "removes the separator U+205F", + "input": "EK\u205f01\u205fA03\u205fFK\u205f01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-063", + "description": "removes the separator U+2060", + "input": "EK\u206001\u2060A03\u2060FK\u206001", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-064", + "description": "removes the separator U+2212", + "input": "EK\u221201\u2212A03\u2212FK\u221201", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-065", + "description": "removes the separator U+3000", + "input": "EK\u300001\u3000A03\u3000FK\u300001", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-066", + "description": "removes the separator U+FEFF", + "input": "EK\ufeff01\ufeffA03\ufeffFK\ufeff01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } } ] } diff --git a/vectors/parse-segments.json b/vectors/parse-segments.json index e84450f..ab759b3 100644 --- a/vectors/parse-segments.json +++ b/vectors/parse-segments.json @@ -229,6 +229,20 @@ }, { "id": "parse-segments-017", + "description": "suggests 1 for a letter L in the unit", + "input": "EK01A03FKL1", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "unit", + "suggestion": "EK-01-A03-FK-11" + } + } + }, + { + "id": "parse-segments-018", "description": "suggests 1 for a letter I after a zero in the unit", "input": "EK01A03FK0I", "options": {}, @@ -242,7 +256,7 @@ } }, { - "id": "parse-segments-018", + "id": "parse-segments-019", "description": "gives no suggestion when the fix makes a unit of 00", "input": "EK01A03FK0O", "options": {}, @@ -256,7 +270,7 @@ } }, { - "id": "parse-segments-019", + "id": "parse-segments-020", "description": "gives no suggestion when a segment cannot be fixed", "input": "EK01A03F2O1", "options": {}, @@ -270,7 +284,7 @@ } }, { - "id": "parse-segments-020", + "id": "parse-segments-021", "description": "reports the first bad segment and fixes them all", "input": "EKO1A03F1O1", "options": {}, @@ -284,7 +298,7 @@ } }, { - "id": "parse-segments-021", + "id": "parse-segments-022", "description": "suggests a fix after making letters upper case", "input": "eko1a03fk01", "options": {}, @@ -298,7 +312,7 @@ } }, { - "id": "parse-segments-022", + "id": "parse-segments-023", "description": "rejects an LGA of 00 in a partial code", "input": "EK00", "options": { @@ -314,7 +328,7 @@ } }, { - "id": "parse-segments-023", + "id": "parse-segments-024", "description": "suggests a partial code", "input": "EKO1", "options": { @@ -330,7 +344,7 @@ } }, { - "id": "parse-segments-024", + "id": "parse-segments-025", "description": "suggests a partial area code", "input": "EK01A03F1", "options": { @@ -346,27 +360,35 @@ } }, { - "id": "parse-segments-025", - "description": "never changes the district", - "input": "EK01AO3FK01", + "id": "parse-segments-026", + "description": "keeps a letter O in the district when it fixes the LGA", + "input": "EKO1AO3FK01", "options": {}, "expect": { - "ok": true, - "compact": "EK01AO3FK01", - "canonical": "EK-01-AO3-FK-01", - "display": "EK 01 AO3 FK 01", - "precision": "unit", - "segments": { - "state": "EK", - "lga": "01", - "district": "AO3", - "area": "FK", - "unit": "01" + "ok": false, + "error": { + "code": "bad_segment", + "segment": "lga", + "suggestion": "EK-01-AO3-FK-01" } } }, { - "id": "parse-segments-026", + "id": "parse-segments-027", + "description": "keeps letters I and L in the district when it fixes the LGA", + "input": "EKO1AILFK01", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "lga", + "suggestion": "EK-01-AIL-FK-01" + } + } + }, + { + "id": "parse-segments-028", "description": "accepts letters and digits in any district position", "input": "EK01A0OFK01", "options": {}, diff --git a/vectors/parse.json b/vectors/parse.json index 1fba2e9..bf7d8bc 100644 --- a/vectors/parse.json +++ b/vectors/parse.json @@ -125,6 +125,26 @@ }, { "id": "parse-007", + "description": "parses a code with a byte order mark", + "input": "\ufeffEK01A03FK01", + "options": {}, + "expect": { + "ok": true, + "compact": "EK01A03FK01", + "canonical": "EK-01-A03-FK-01", + "display": "EK 01 A03 FK 01", + "precision": "unit", + "segments": { + "state": "EK", + "lga": "01", + "district": "A03", + "area": "FK", + "unit": "01" + } + } + }, + { + "id": "parse-008", "description": "parses full-width letters and digits", "input": "\uff25\uff2b-\uff10\uff11-\uff21\uff10\uff13-\uff26\uff2b-\uff10\uff11", "options": {}, @@ -144,7 +164,7 @@ } }, { - "id": "parse-008", + "id": "parse-009", "description": "parses the example on NIPOST's homepage", "input": "FC-02-A09-DB-09", "options": {}, @@ -164,7 +184,7 @@ } }, { - "id": "parse-009", + "id": "parse-010", "description": "parses the example in NIPOST's blog", "input": "LA 12 K23 IV 95", "options": {}, @@ -184,7 +204,7 @@ } }, { - "id": "parse-010", + "id": "parse-011", "description": "parses a NIPOST test code", "input": "AK-11-I61-ZF-12", "options": {}, @@ -204,8 +224,8 @@ } }, { - "id": "parse-011", - "description": "accepts a letter O in the district", + "id": "parse-012", + "description": "accepts a letter O in the district of a NIPOST test code", "input": "JI-24-O18-JP-23", "options": {}, "expect": { @@ -224,7 +244,7 @@ } }, { - "id": "parse-012", + "id": "parse-013", "description": "rejects empty input", "input": "", "options": {}, @@ -238,7 +258,7 @@ } }, { - "id": "parse-013", + "id": "parse-014", "description": "rejects spaces only", "input": " ", "options": {}, @@ -252,7 +272,7 @@ } }, { - "id": "parse-014", + "id": "parse-015", "description": "rejects separators only", "input": " - . ", "options": {}, @@ -266,7 +286,7 @@ } }, { - "id": "parse-015", + "id": "parse-016", "description": "names a legacy code", "input": "900108", "options": {}, @@ -280,7 +300,7 @@ } }, { - "id": "parse-016", + "id": "parse-017", "description": "names a legacy code with a space", "input": "900 108", "options": {}, @@ -294,7 +314,7 @@ } }, { - "id": "parse-017", + "id": "parse-018", "description": "names a legacy code in full-width digits", "input": "\uff19\uff10\uff10\uff11\uff10\uff18", "options": {}, @@ -308,7 +328,7 @@ } }, { - "id": "parse-018", + "id": "parse-019", "description": "names a legacy code when partial codes are allowed", "input": "123456", "options": { @@ -324,7 +344,7 @@ } }, { - "id": "parse-019", + "id": "parse-020", "description": "rejects five digits", "input": "90010", "options": {}, @@ -338,7 +358,7 @@ } }, { - "id": "parse-020", + "id": "parse-021", "description": "rejects an exclamation mark", "input": "EK01A03FK0!", "options": {}, @@ -352,7 +372,7 @@ } }, { - "id": "parse-021", + "id": "parse-022", "description": "rejects an underscore", "input": "EK01A03FK0_", "options": {}, @@ -366,7 +386,7 @@ } }, { - "id": "parse-022", + "id": "parse-023", "description": "rejects slashes", "input": "EK01/A03/FK01", "options": {}, @@ -380,7 +400,7 @@ } }, { - "id": "parse-023", + "id": "parse-024", "description": "rejects a letter outside ASCII", "input": "EK01A03F\u00e901", "options": {}, @@ -394,7 +414,7 @@ } }, { - "id": "parse-024", + "id": "parse-025", "description": "rejects a right-to-left override", "input": "\u202eEK01A03FK01", "options": {}, @@ -408,7 +428,21 @@ } }, { - "id": "parse-025", + "id": "parse-026", + "description": "rejects Arabic-Indic digits", + "input": "\u0669\u0660\u0660\u0661\u0660\u0668", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_character", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-027", "description": "rejects 10 characters", "input": "EK01A03FK1", "options": {}, @@ -422,7 +456,7 @@ } }, { - "id": "parse-026", + "id": "parse-028", "description": "rejects 12 characters", "input": "EK01A03FK01X", "options": {}, @@ -436,7 +470,7 @@ } }, { - "id": "parse-027", + "id": "parse-029", "description": "rejects a unit that lost its zero", "input": "EK-01-A03-FK-1", "options": {}, @@ -450,7 +484,7 @@ } }, { - "id": "parse-028", + "id": "parse-030", "description": "rejects a state alone by default", "input": "EK", "options": {}, @@ -464,7 +498,7 @@ } }, { - "id": "parse-029", + "id": "parse-031", "description": "rejects a district code by default", "input": "EK01A03", "options": {}, @@ -478,7 +512,7 @@ } }, { - "id": "parse-030", + "id": "parse-032", "description": "parses a state when partial codes are allowed", "input": "EK", "options": { @@ -500,7 +534,7 @@ } }, { - "id": "parse-031", + "id": "parse-033", "description": "parses an LGA code when partial codes are allowed", "input": "EK01", "options": { @@ -522,7 +556,7 @@ } }, { - "id": "parse-032", + "id": "parse-034", "description": "parses a district code when partial codes are allowed", "input": "ek-01-a03", "options": { @@ -544,7 +578,7 @@ } }, { - "id": "parse-033", + "id": "parse-035", "description": "parses an area code when partial codes are allowed", "input": "EK01A03FK", "options": { @@ -566,7 +600,7 @@ } }, { - "id": "parse-034", + "id": "parse-036", "description": "parses a full code when partial codes are allowed", "input": "EK01A03FK01", "options": { @@ -588,7 +622,7 @@ } }, { - "id": "parse-035", + "id": "parse-037", "description": "rejects 3 characters when partial codes are allowed", "input": "EK0", "options": { @@ -604,7 +638,23 @@ } }, { - "id": "parse-036", + "id": "parse-038", + "description": "rejects 5 characters when partial codes are allowed", + "input": "EK01A", + "options": { + "allowPartial": true + }, + "expect": { + "ok": false, + "error": { + "code": "bad_length", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-039", "description": "rejects 6 characters when partial codes are allowed", "input": "EK01A0", "options": { @@ -618,6 +668,38 @@ "suggestion": null } } + }, + { + "id": "parse-040", + "description": "rejects 8 characters when partial codes are allowed", + "input": "EK01A03F", + "options": { + "allowPartial": true + }, + "expect": { + "ok": false, + "error": { + "code": "bad_length", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-041", + "description": "rejects 10 characters when partial codes are allowed", + "input": "EK01A03FK0", + "options": { + "allowPartial": true + }, + "expect": { + "ok": false, + "error": { + "code": "bad_length", + "segment": null, + "suggestion": null + } + } } ] } From 13f96dc4bd30315824360adcff9c9510eba89d4d Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 01:19:30 +0100 Subject: [PATCH 13/45] test: tie the separators and GPS limits to the vectors The GPS test passed even without the cases just above each limit, and no test tied the separator list to its cases. Both tests now fail when a case goes missing. A new case shows that stateName does not trim spaces, and the grammar says that legacy codes use ASCII digits. Signed-off-by: MobileMage --- grammar.md | 6 +++--- scripts/build_vectors.py | 1 + scripts/tests/test_build_vectors.py | 15 ++++++++++++++- vectors/state-name.json | 9 +++++++++ 4 files changed, 27 insertions(+), 4 deletions(-) diff --git a/grammar.md b/grammar.md index a531dc0..8246c2b 100644 --- a/grammar.md +++ b/grammar.md @@ -35,7 +35,7 @@ A format character that is not in the list stays. For example, the right-to-left Normalise the input. Then apply these checks in order. The first check that fails gives the error code. 1. `empty`: no character is left. -2. `legacy_code`: exactly 6 digits are left. These are old NIPOST postcodes. +2. `legacy_code`: exactly 6 ASCII digits are left. These are old NIPOST postcodes. 3. `bad_character`: a character other than A to Z and 0 to 9 is left. 4. `bad_length`: the length is not 11. With `allowPartial`, the lengths 2, 4, 7 and 9 also pass. 5. `unknown_state`: the first two characters are not a code in `data/states.json`. @@ -55,7 +55,7 @@ Parse the corrected code with the same options. If it parses, the suggestion is ## Legacy codes -`isLegacy` is true when the normalised input is exactly 6 digits. +`isLegacy` is true when the normalised input is exactly 6 ASCII digits. ## Hierarchy @@ -66,7 +66,7 @@ Parse the corrected code with the same options. If it parses, the suggestion is ## State names -`stateName(code)` returns the name in `data/states.json`, and it ignores letter case. An unknown code gives null. The codes follow ISO 3166-2:NG. All 11 state codes in NIPOST's published examples match it. A keyed autocomplete call must confirm the full list. +`stateName(code)` returns the name in `data/states.json`, and it ignores letter case. It does not remove spaces or other characters. An unknown code gives null. The codes follow ISO 3166-2:NG. All 11 state codes in NIPOST's published examples match it. A keyed autocomplete call must confirm the full list. ## GPS precision diff --git a/scripts/build_vectors.py b/scripts/build_vectors.py index 4b4254c..cef6b05 100755 --- a/scripts/build_vectors.py +++ b/scripts/build_vectors.py @@ -522,6 +522,7 @@ def state_name_rows() -> list[Row]: row("returns nothing for an unknown code", "XX", value(None)), row("returns nothing for empty input", "", value(None)), row("returns nothing for three letters", "EKI", value(None)), + row("returns nothing for a code with spaces", " EK ", value(None)), ] diff --git a/scripts/tests/test_build_vectors.py b/scripts/tests/test_build_vectors.py index 090ca21..be297e5 100644 --- a/scripts/tests/test_build_vectors.py +++ b/scripts/tests/test_build_vectors.py @@ -54,6 +54,19 @@ def test_each_state_has_a_parse_case_and_a_name_case(self) -> None: self.assertEqual(parsed, codes) self.assertLessEqual(codes, named) + def test_each_separator_has_a_normalize_case(self) -> None: + postcode_format = json.loads((ROOT / "data" / "format.json").read_text(encoding="utf-8")) + expected = {"value": "EK01A03FK01"} + inputs = { + case["input"] for case in load("normalize")["cases"] if case["expect"] == expected + } + missing: list[str] = [] + for label in postcode_format["separators"]: + separator = chr(int(label.removeprefix("U+"), 16)) + if separator.join(("EK", "01", "A03", "FK", "01")) not in inputs: + missing.append(label) + self.assertEqual(missing, []) + def test_each_gps_limit_has_a_case_at_the_limit_and_a_case_above_it(self) -> None: rules = json.loads((ROOT / "data" / "precision.json").read_text(encoding="utf-8")) limits = [threshold["maxAccuracyM"] for threshold in rules["thresholds"]] @@ -70,5 +83,5 @@ def test_each_gps_limit_has_a_case_at_the_limit_and_a_case_above_it(self) -> Non ): with self.subTest(limit=limit): self.assertIn((limit, precision), measured) - above = [expected for metres, expected in measured if limit < metres <= ceiling] + above = [expected for metres, expected in measured if limit < metres < ceiling] self.assertIn(next_precision, above) diff --git a/vectors/state-name.json b/vectors/state-name.json index b95638d..8a00299 100644 --- a/vectors/state-name.json +++ b/vectors/state-name.json @@ -380,6 +380,15 @@ "expect": { "value": null } + }, + { + "id": "state-name-043", + "description": "returns nothing for a code with spaces", + "input": " EK ", + "options": {}, + "expect": { + "value": null + } } ] } From 2948a22929200c8936eab027a194982c711c1e55 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 01:34:09 +0100 Subject: [PATCH 14/45] ci: run make check and a secret scan The checks in make check help only if every pull request runs them. CI now runs them on Python 3.11, the oldest version that the scripts support, and gitleaks scans the full history for secrets. Each action is pinned to a commit SHA (SEC-6). Signed-off-by: MobileMage --- .github/workflows/ci.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..5e7df99 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +name: ci + +on: + push: + branches: [main] + pull_request: + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.11" + - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + - run: make check + + secrets: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + - run: docker run --rm -v "$PWD:/repo" ghcr.io/gitleaks/gitleaks:v8.30.1 git /repo From aa8ca5790bb0ad8ebc8f53413b4288d802acb43e Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 02:08:37 +0100 Subject: [PATCH 15/45] ci: pin the gitleaks image and test the newest Python A moved image tag could swap the scanner for one that always passes, so the secrets job now pins the image by digest. The check job also runs on the newest Python, because the README promises 3.11 or later. The uv cache is off, because the repo has no lock file to key it. SEC-5 now names the triggers that CI uses: each pull request and each push to main. Signed-off-by: MobileMage --- .github/workflows/ci.yml | 15 +++++++++++++-- standards/CODING_STANDARDS.md | 2 +- standards/languages/python.md | 4 ++-- 3 files changed, 16 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5e7df99..0b3cb3b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,14 +13,21 @@ permissions: jobs: check: runs-on: ubuntu-latest + strategy: + matrix: + # The oldest Python that the scripts support, and the newest release. + python-version: ["3.11", "3.x"] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: - python-version: "3.11" + python-version: ${{ matrix.python-version }} - uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0 + with: + # The repo has no lock file to key a cache, and the tools are small. + enable-cache: false - run: make check secrets: @@ -28,6 +35,10 @@ jobs: steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + # gitleaks must read the whole history. fetch-depth: 0 persist-credentials: false - - run: docker run --rm -v "$PWD:/repo" ghcr.io/gitleaks/gitleaks:v8.30.1 git /repo + - run: | + image=ghcr.io/gitleaks/gitleaks # v8.30.1 + digest=sha256:c00b6bd0aeb3071cbcb79009cb16a60dd9e0a7c60e2be9ab65d25e6bc8abbb7f + docker run --rm -v "$PWD:/repo" "$image@$digest" git /repo diff --git a/standards/CODING_STANDARDS.md b/standards/CODING_STANDARDS.md index 743c1a9..23f2a47 100644 --- a/standards/CODING_STANDARDS.md +++ b/standards/CODING_STANDARDS.md @@ -138,7 +138,7 @@ A line in a code or config file can skip one check with a comment that names the - **SEC-2 MUST.** There is no telemetry. An SDK sends only the requests that the caller makes. - **SEC-3 MUST.** The cache is off by default. When on, it has a time limit and a clear command. - **SEC-4 MUST.** Call only the documented gateway endpoints. -- **SEC-5 MUST.** CI scans each push for secrets. **(tool)** +- **SEC-5 MUST.** CI scans each pull request and each push to `main` for secrets. **(tool)** - **SEC-6 MUST.** Each GitHub Action is pinned to a full commit SHA. **(tool)** - **SEC-7 MUST.** Releases come only from CI. They carry provenance or a signature where the registry supports it. - **SEC-8 MUST.** Each repo has `SECURITY.md`. A reporter gets a reply within 3 working days. diff --git a/standards/languages/python.md b/standards/languages/python.md index 98e00dd..830bb77 100644 --- a/standards/languages/python.md +++ b/standards/languages/python.md @@ -12,7 +12,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for Pyth ## Tools -`make check` runs every tool below. CI runs `make check` on each push and each pull request. +`make check` runs every tool below. CI runs `make check` on each pull request and each push to `main`. | Tool | Job | Settings | |---|---|---| @@ -28,7 +28,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for Pyth | towncrier | change files and changelog | one fragment in `changes/` per user-visible change | | `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in source, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | | REUSE | licence headers | `reuse lint` | -| gitleaks | secret scan | every push | +| gitleaks | secret scan | each pull request and each push to `main` | ### Ruff settings From fc7f8089d91ca0eb515c9de81a79ea5f0d03b82c Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 02:13:04 +0100 Subject: [PATCH 16/45] docs: name the CI triggers in each language standard SEC-5 now says that CI runs on each pull request and each push to main. The language standards still said each push. They now use the same words as SEC-5. Signed-off-by: MobileMage --- standards/languages/csharp.md | 4 ++-- standards/languages/dart.md | 4 ++-- standards/languages/go.md | 6 +++--- standards/languages/kotlin-java.md | 4 ++-- standards/languages/php.md | 8 ++++---- standards/languages/swift.md | 4 ++-- standards/languages/typescript.md | 4 ++-- 7 files changed, 17 insertions(+), 17 deletions(-) diff --git a/standards/languages/csharp.md b/standards/languages/csharp.md index 8f70de7..fee5e70 100644 --- a/standards/languages/csharp.md +++ b/standards/languages/csharp.md @@ -14,7 +14,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for C# c ## Tools -`make check` runs every tool below. CI runs `make check` on each push and each pull request. +`make check` runs every tool below. CI runs `make check` on each pull request and each push to `main`. | Tool | Job | Settings | |---|---|---| @@ -32,7 +32,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for C# c | changie | change files and changelog | one change file per user-visible change | | `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in source, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | | REUSE | licence headers | `reuse lint` | -| gitleaks | secret scan | every push | +| gitleaks | secret scan | each pull request and each push to `main` | ### Agent-tell checks diff --git a/standards/languages/dart.md b/standards/languages/dart.md index 7e6c44c..69f1fda 100644 --- a/standards/languages/dart.md +++ b/standards/languages/dart.md @@ -13,7 +13,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for Dart ## Tools -`make check` runs every tool below. CI runs `make check` on each push and each pull request. +`make check` runs every tool below. CI runs `make check` on each pull request and each push to `main`. | Tool | Job | Settings | |---|---|---| @@ -27,7 +27,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for Dart | changie | change files and changelog | one change file per user-visible change | | `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in source, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | | REUSE | licence headers | `reuse lint` | -| gitleaks | secret scan | every push | +| gitleaks | secret scan | each pull request and each push to `main` | ### `analysis_options.yaml` diff --git a/standards/languages/go.md b/standards/languages/go.md index 01e40b6..951196f 100644 --- a/standards/languages/go.md +++ b/standards/languages/go.md @@ -13,7 +13,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for Go c ## Tools -`make check` runs every tool below. CI runs `make check` on each push and each pull request. +`make check` runs every tool below. CI runs `make check` on each pull request and each push to `main`. | Tool | Job | Settings | |---|---|---| @@ -23,12 +23,12 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for Go c | golangci-lint | lint | the settings below | | `go test` | unit, vector, contract and example tests | `-race -cover`, coverage floors from T-7 | | `go test -fuzz` | fuzz tests | `FuzzParse` for 30 seconds on each pull request | -| govulncheck | vulnerability scan | every push | +| govulncheck | vulnerability scan | each pull request and each push to `main` | | gorelease | API check against the last tag | enforces GIT-7 and API-13 | | changie | change files and changelog | one change file per user-visible change | | `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in source, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | | REUSE | licence headers | `reuse lint` | -| gitleaks | secret scan | every push | +| gitleaks | secret scan | each pull request and each push to `main` | ### golangci-lint settings diff --git a/standards/languages/kotlin-java.md b/standards/languages/kotlin-java.md index f3a1c7e..71d592b 100644 --- a/standards/languages/kotlin-java.md +++ b/standards/languages/kotlin-java.md @@ -15,7 +15,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for Kotl ## Tools -`./gradlew check` runs every tool below. CI runs `./gradlew check` on each push and each pull request. +`./gradlew check` runs every tool below. CI runs `./gradlew check` on each pull request and each push to `main`. | Tool | Job | Settings | |---|---|---| @@ -33,7 +33,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for Kotl | changie | change files and changelog | one change file per user-visible change | | `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in source, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | | REUSE | licence headers | `reuse lint` | -| gitleaks | secret scan | every push | +| gitleaks | secret scan | each pull request and each push to `main` | ### detekt settings diff --git a/standards/languages/php.md b/standards/languages/php.md index 4321773..affe314 100644 --- a/standards/languages/php.md +++ b/standards/languages/php.md @@ -15,7 +15,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for PHP ### Tools -`composer check` runs every tool below. CI runs `composer check` on each push and each pull request. +`composer check` runs every tool below. CI runs `composer check` on each pull request and each push to `main`. | Tool | Job | Settings | |---|---|---| @@ -24,11 +24,11 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for PHP | PHPUnit 10 | unit and contract tests | coverage floors from T-7 | | Infection | mutation tests | the core namespace, weekly scheduled job | | Roave BackwardCompatibilityCheck | API check against the last tag | enforces GIT-7 and API-13 | -| `composer validate --strict` and `composer-normalize` | package checks | every push | +| `composer validate --strict` and `composer-normalize` | package checks | each pull request and each push to `main` | | PHPMD | size and complexity | `CyclomaticComplexity` 10, `ExcessiveParameterList` minimum 5, enforces TELL-2 and TELL-3 | | changie | change files and changelog | one change file per user-visible change, enforces DOC-5 | | REUSE | licence headers | `reuse lint` | -| gitleaks | secret scan | every push | +| gitleaks | secret scan | each pull request and each push to `main` | | `spec/scripts/check-tells` | agent tells | line length 100, file names, TODO form, debug calls, emojis and non-ASCII | PHP-CS-Fixer does not limit line length, so `check-tells` enforces TELL-1. PHPMD has no nesting rule, so reviewers check nesting for TELL-2. @@ -88,7 +88,7 @@ CHANGELOG.md ### Tools -`composer check` runs every tool below. CI runs it on each push and each pull request. +`composer check` runs every tool below. CI runs it on each pull request and each push to `main`. | Tool | Job | Settings | |---|---|---| diff --git a/standards/languages/swift.md b/standards/languages/swift.md index 9d9e4be..dab4950 100644 --- a/standards/languages/swift.md +++ b/standards/languages/swift.md @@ -13,7 +13,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for Swif ## Tools -`make check` runs every tool below. CI runs `make check` on each push and each pull request. +`make check` runs every tool below. CI runs `make check` on each pull request and each push to `main`. | Tool | Job | Settings | |---|---|---| @@ -26,7 +26,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for Swif | changie | change files and changelog | one change file per user-visible change | | `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in source, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | | REUSE | licence headers | `reuse lint` | -| gitleaks | secret scan | every push | +| gitleaks | secret scan | each pull request and each push to `main` | ### SwiftLint settings diff --git a/standards/languages/typescript.md b/standards/languages/typescript.md index 8a099a4..da48693 100644 --- a/standards/languages/typescript.md +++ b/standards/languages/typescript.md @@ -15,7 +15,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for Type ## Tools -`pnpm check` runs every tool below. CI runs `pnpm check` on each push and each pull request. +`pnpm check` runs every tool below. CI runs `pnpm check` on each pull request and each push to `main`. | Tool | Job | Settings | |---|---|---| @@ -33,7 +33,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for Type | Changesets | versions and changelogs | one change file per user-visible change | | commitlint | commit messages | `@commitlint/config-conventional`, scopes from package names | | REUSE | licence headers | `reuse lint` | -| gitleaks | secret scan | every push | +| gitleaks | secret scan | each pull request and each push to `main` | | `spec/scripts/check-tells` | agent tells | line length 100, file names, TODO form, debug calls, emojis and non-ASCII | | TypeDoc | API pages | the docs site renders them | From eaa3760a75fc589f49447d482d4c3c622bf936ff Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 04:55:04 +0100 Subject: [PATCH 17/45] feat: limit the input of parse to 64 code points NFKC can take seconds on a long run of combining marks, and parse normalises its whole input before it checks the length. A server that parses untrusted text could stall. parse now rejects input with more than 64 code points as bad_length before it normalises, and isLegacy returns false for it. The limit lives in data/format.json, and vectors pin both sides of it. Signed-off-by: MobileMage --- data/format.json | 1 + grammar.md | 4 +-- scripts/build_vectors.py | 42 ++++++++++++++++++++-- scripts/tests/test_build_vectors.py | 14 ++++++++ scripts/tests/test_data.py | 3 ++ vectors/is-legacy.json | 18 ++++++++++ vectors/parse.json | 54 +++++++++++++++++++++++++++++ 7 files changed, 132 insertions(+), 4 deletions(-) diff --git a/data/format.json b/data/format.json index b60e3a7..5804e6a 100644 --- a/data/format.json +++ b/data/format.json @@ -1,5 +1,6 @@ { "version": 1, + "maxInputCodePoints": 64, "segments": [ { "name": "state", "length": 2, "characters": "letters" }, { "name": "lga", "length": 2, "characters": "digits", "minimum": 1 }, diff --git a/grammar.md b/grammar.md index 8246c2b..a7d1f28 100644 --- a/grammar.md +++ b/grammar.md @@ -32,7 +32,7 @@ A format character that is not in the list stays. For example, the right-to-left ## Parse -Normalise the input. Then apply these checks in order. The first check that fails gives the error code. +First count the Unicode code points of the input. If the input has more than `maxInputCodePoints` code points (64 in `data/format.json`), the error code is `bad_length`, and `parse` does not normalise the input. The limit bounds the cost of NFKC, which can take seconds on a long run of combining marks. Otherwise, normalise the input. Then apply these checks in order. The first check that fails gives the error code. 1. `empty`: no character is left. 2. `legacy_code`: exactly 6 ASCII digits are left. These are old NIPOST postcodes. @@ -55,7 +55,7 @@ Parse the corrected code with the same options. If it parses, the suggestion is ## Legacy codes -`isLegacy` is true when the normalised input is exactly 6 ASCII digits. +`isLegacy` is false for input with more than `maxInputCodePoints` code points, and it does not normalise such input. Otherwise, it is true when the normalised input is exactly 6 ASCII digits. ## Hierarchy diff --git a/scripts/build_vectors.py b/scripts/build_vectors.py index cef6b05..5bcfaa9 100755 --- a/scripts/build_vectors.py +++ b/scripts/build_vectors.py @@ -46,6 +46,11 @@ PRECISION_BY_LENGTH = {2: "state", 4: "lga", 7: "district", 9: "area", 11: "unit"} PARTIAL = {"allowPartial": True} CODE = "EK01A03FK01" +# Mathematical bold capitals start at U+1D400 and digits at U+1D7CE. Each one is 1 code point +# and 2 UTF-16 units, and NFKC maps it to its ASCII form. +MATH_BOLD_CODE = "".join( + chr(0x1D7CE + int(c)) if c.isdigit() else chr(0x1D400 + ord(c) - ord("A")) for c in CODE +) Row = tuple[str, Any, dict[str, Any], dict[str, Any]] @@ -110,10 +115,23 @@ def load_states() -> list[dict[str, str]]: return states +def load_format() -> dict[str, Any]: + """Read data/format.json.""" + postcode_format: dict[str, Any] = json.loads( + (ROOT / "data" / "format.json").read_text(encoding="utf-8") + ) + return postcode_format + + def load_separators() -> list[int]: """Read data/format.json and return its separators as code points.""" - postcode_format = json.loads((ROOT / "data" / "format.json").read_text(encoding="utf-8")) - return [int(label.removeprefix("U+"), 16) for label in postcode_format["separators"]] + return [int(label.removeprefix("U+"), 16) for label in load_format()["separators"]] + + +def load_input_limit() -> int: + """Read data/format.json and return the input limit in code points.""" + limit: int = load_format()["maxInputCodePoints"] + return limit def normalize_rows() -> list[Row]: @@ -166,6 +184,7 @@ def normalize_rows() -> list[Row]: def parse_rows() -> list[Row]: """Cases for forms, lengths, characters and legacy codes.""" + limit = load_input_limit() return [ row("parses the canonical form", "EK-01-A03-FK-01", ok(CODE)), row("parses the compact form", CODE, ok(CODE)), @@ -262,6 +281,14 @@ def parse_rows() -> list[Row]: failed("bad_length"), PARTIAL, ), + row("parses a code padded to the input limit", CODE + " " * (limit - len(CODE)), ok(CODE)), + row( + "rejects input over the input limit", + CODE + " " * (limit - len(CODE) + 1), + failed("bad_length"), + ), + # 54 code points and 65 UTF-16 units. A runner that counts UTF-16 units rejects it. + row("counts code points, not UTF-16 units", MATH_BOLD_CODE + " " * 43, ok(CODE)), ] @@ -402,6 +429,7 @@ def state_parse_rows() -> list[Row]: def legacy_rows() -> list[Row]: """Cases for isLegacy.""" + limit = load_input_limit() return [ row("accepts six digits", "900108", value(True)), row("accepts six digits with a space", "900 108", value(True)), @@ -412,6 +440,16 @@ def legacy_rows() -> list[Row]: row("rejects empty input as a legacy code", "", value(False)), row("rejects a letter O among digits", "90O108", value(False)), row("rejects six Arabic-Indic digits as a legacy code", ARABIC_INDIC_LEGACY, value(False)), + row( + "finds a legacy code padded to the input limit", + "900108" + " " * (limit - 6), + value(True), + ), + row( + "rejects a legacy code over the input limit", + "900108" + " " * (limit - 5), + value(False), + ), ] diff --git a/scripts/tests/test_build_vectors.py b/scripts/tests/test_build_vectors.py index be297e5..a2f1435 100644 --- a/scripts/tests/test_build_vectors.py +++ b/scripts/tests/test_build_vectors.py @@ -85,3 +85,17 @@ def test_each_gps_limit_has_a_case_at_the_limit_and_a_case_above_it(self) -> Non self.assertIn((limit, precision), measured) above = [expected for metres, expected in measured if limit < metres < ceiling] self.assertIn(next_precision, above) + + def test_the_input_limit_has_a_case_at_the_limit_and_a_case_above_it(self) -> None: + postcode_format = json.loads((ROOT / "data" / "format.json").read_text(encoding="utf-8")) + limit = postcode_format["maxInputCodePoints"] + parse_outcomes = [ + (len(case["input"]), "ok" if case["expect"]["ok"] else case["expect"]["error"]["code"]) + for case in load("parse")["cases"] + ] + legacy_outcomes = [ + (len(case["input"]), case["expect"]["value"]) for case in load("is-legacy")["cases"] + ] + self.assertIn((limit, "ok"), parse_outcomes) + self.assertIn((limit + 1, "bad_length"), parse_outcomes) + self.assertIn((limit + 1, False), legacy_outcomes) diff --git a/scripts/tests/test_data.py b/scripts/tests/test_data.py index 215cdb4..98468b2 100644 --- a/scripts/tests/test_data.py +++ b/scripts/tests/test_data.py @@ -55,3 +55,6 @@ def test_writes_each_separator_as_a_code_point(self) -> None: def test_compiles_the_legacy_pattern(self) -> None: self.assertTrue(re.fullmatch(load("format.json")["legacyPattern"], "900108")) + + def test_limits_the_input_to_64_code_points(self) -> None: + self.assertEqual(load("format.json")["maxInputCodePoints"], 64) diff --git a/vectors/is-legacy.json b/vectors/is-legacy.json index fc4cb55..48a3451 100644 --- a/vectors/is-legacy.json +++ b/vectors/is-legacy.json @@ -83,6 +83,24 @@ "expect": { "value": false } + }, + { + "id": "is-legacy-010", + "description": "finds a legacy code padded to the input limit", + "input": "900108 ", + "options": {}, + "expect": { + "value": true + } + }, + { + "id": "is-legacy-011", + "description": "rejects a legacy code over the input limit", + "input": "900108 ", + "options": {}, + "expect": { + "value": false + } } ] } diff --git a/vectors/parse.json b/vectors/parse.json index bf7d8bc..5f801ca 100644 --- a/vectors/parse.json +++ b/vectors/parse.json @@ -700,6 +700,60 @@ "suggestion": null } } + }, + { + "id": "parse-042", + "description": "parses a code padded to the input limit", + "input": "EK01A03FK01 ", + "options": {}, + "expect": { + "ok": true, + "compact": "EK01A03FK01", + "canonical": "EK-01-A03-FK-01", + "display": "EK 01 A03 FK 01", + "precision": "unit", + "segments": { + "state": "EK", + "lga": "01", + "district": "A03", + "area": "FK", + "unit": "01" + } + } + }, + { + "id": "parse-043", + "description": "rejects input over the input limit", + "input": "EK01A03FK01 ", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_length", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-044", + "description": "counts code points, not UTF-16 units", + "input": "\ud835\udc04\ud835\udc0a\ud835\udfce\ud835\udfcf\ud835\udc00\ud835\udfce\ud835\udfd1\ud835\udc05\ud835\udc0a\ud835\udfce\ud835\udfcf ", + "options": {}, + "expect": { + "ok": true, + "compact": "EK01A03FK01", + "canonical": "EK-01-A03-FK-01", + "display": "EK 01 A03 FK 01", + "precision": "unit", + "segments": { + "state": "EK", + "lga": "01", + "district": "A03", + "area": "FK", + "unit": "01" + } + } } ] } From 3966ed66e1ab3b8a109f2c45f61a7dcf71f863e1 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 05:23:24 +0100 Subject: [PATCH 18/45] test: pin how parse counts its input A grapheme counter or an SDK that runs NFKC before it counts passed every vector and still left the stall open. New cases count combining marks one by one, show that the count comes before NFKC, and catch a limit check that ignores a final line feed. The grammar now names the units that an SDK must not count. Signed-off-by: MobileMage --- grammar.md | 4 ++- scripts/build_vectors.py | 22 ++++++++++++- scripts/tests/test_build_vectors.py | 34 ++++++++++++++++++-- vectors/parse.json | 48 +++++++++++++++++++++++++++++ 4 files changed, 104 insertions(+), 4 deletions(-) diff --git a/grammar.md b/grammar.md index a7d1f28..2c13251 100644 --- a/grammar.md +++ b/grammar.md @@ -30,9 +30,11 @@ A partial postcode stops after the state, LGA, district or area segment. Its len A format character that is not in the list stays. For example, the right-to-left override U+202E can change the order in which a code shows on screen, so `parse` gives `bad_character` for it. +`normalize` has no length limit. Bound untrusted text before you call it, or call `parse`, which checks the limit first. + ## Parse -First count the Unicode code points of the input. If the input has more than `maxInputCodePoints` code points (64 in `data/format.json`), the error code is `bad_length`, and `parse` does not normalise the input. The limit bounds the cost of NFKC, which can take seconds on a long run of combining marks. Otherwise, normalise the input. Then apply these checks in order. The first check that fails gives the error code. +First count the Unicode code points of the input. Do not count UTF-16 units, UTF-8 bytes or grapheme clusters. A lone surrogate, in a language that can hold one, counts as one code point. An SDK can stop counting at 65. If the input has more than `maxInputCodePoints` code points (64 in `data/format.json`), the error code is `bad_length`, and `parse` does not normalise the input. The limit bounds the cost of NFKC, which can take seconds on a long run of combining marks. Otherwise, normalise the input. Then apply these checks in order. The first check that fails gives the error code. 1. `empty`: no character is left. 2. `legacy_code`: exactly 6 ASCII digits are left. These are old NIPOST postcodes. diff --git a/scripts/build_vectors.py b/scripts/build_vectors.py index 5bcfaa9..0face04 100755 --- a/scripts/build_vectors.py +++ b/scripts/build_vectors.py @@ -38,6 +38,8 @@ RIGHT_TO_LEFT_MARK = chr(0x200F) RIGHT_TO_LEFT_OVERRIDE = chr(0x202E) FULL_WIDTH_HYPHEN = chr(0xFF0D) +COMBINING_ACUTE = chr(0x0301) +HORIZONTAL_ELLIPSIS = chr(0x2026) ARABIC_INDIC_LEGACY = "".join( chr(code_point) for code_point in (0x0669, 0x0660, 0x0660, 0x0661, 0x0660, 0x0668) ) @@ -288,7 +290,25 @@ def parse_rows() -> list[Row]: failed("bad_length"), ), # 54 code points and 65 UTF-16 units. A runner that counts UTF-16 units rejects it. - row("counts code points, not UTF-16 units", MATH_BOLD_CODE + " " * 43, ok(CODE)), + row( + "counts code points, not UTF-16 units", + MATH_BOLD_CODE + " " * (limit - 2 * len(CODE) + 1), + ok(CODE), + ), + # 65 code points, but 1 grapheme cluster and 64 code points after NFKC. + row( + "counts combining marks as code points", + "E" + COMBINING_ACUTE * limit, + failed("bad_length"), + ), + # 33 code points, but 77 after NFKC, because NFKC changes U+2026 to 3 full stops. + row("counts the input before it normalises", CODE + HORIZONTAL_ELLIPSIS * 22, ok(CODE)), + # 65 code points. In many regex engines, $ also matches before a final line feed. + row( + "rejects input over the limit that ends in a line feed", + CODE + " " * (limit - len(CODE)) + LINE_FEED, + failed("bad_length"), + ), ] diff --git a/scripts/tests/test_build_vectors.py b/scripts/tests/test_build_vectors.py index a2f1435..a8009cb 100644 --- a/scripts/tests/test_build_vectors.py +++ b/scripts/tests/test_build_vectors.py @@ -4,6 +4,7 @@ import json import math +import unicodedata import unittest from collections import Counter from pathlib import Path @@ -21,6 +22,10 @@ def load(stem: str) -> dict[str, Any]: return value +def utf16_units(text: str) -> int: + return len(text.encode("utf-16-le")) // 2 + + class BuildVectorsTest(unittest.TestCase): def test_vector_files_match_the_builder(self) -> None: self.assertEqual(build_vectors.main(["--check"]), 0) @@ -89,13 +94,38 @@ def test_each_gps_limit_has_a_case_at_the_limit_and_a_case_above_it(self) -> Non def test_the_input_limit_has_a_case_at_the_limit_and_a_case_above_it(self) -> None: postcode_format = json.loads((ROOT / "data" / "format.json").read_text(encoding="utf-8")) limit = postcode_format["maxInputCodePoints"] - parse_outcomes = [ - (len(case["input"]), "ok" if case["expect"]["ok"] else case["expect"]["error"]["code"]) + parsed = [ + (case["input"], "ok" if case["expect"]["ok"] else case["expect"]["error"]["code"]) for case in load("parse")["cases"] ] + parse_outcomes = [(len(text), outcome) for text, outcome in parsed] legacy_outcomes = [ (len(case["input"]), case["expect"]["value"]) for case in load("is-legacy")["cases"] ] self.assertIn((limit, "ok"), parse_outcomes) self.assertIn((limit + 1, "bad_length"), parse_outcomes) self.assertIn((limit + 1, False), legacy_outcomes) + self.assertIn((limit, True), legacy_outcomes) + accepted = [text for text, outcome in parsed if outcome == "ok" and len(text) <= limit] + rejected = [ + text for text, outcome in parsed if outcome == "bad_length" and len(text) > limit + ] + pins = { + "a success with more UTF-16 units than the limit": [ + text for text in accepted if utf16_units(text) > limit + ], + "a success that grows past the limit under NFKC": [ + text for text in accepted if len(unicodedata.normalize("NFKC", text)) > limit + ], + "a rejection over the limit with a combining mark": [ + text + for text in rejected + if any(unicodedata.combining(character) for character in text) + ], + "a rejection over the limit that ends in a line feed": [ + text for text in rejected if text.endswith("\n") + ], + } + for requirement, texts in pins.items(): + with self.subTest(requirement): + self.assertTrue(texts) diff --git a/vectors/parse.json b/vectors/parse.json index 5f801ca..a5c022c 100644 --- a/vectors/parse.json +++ b/vectors/parse.json @@ -754,6 +754,54 @@ "unit": "01" } } + }, + { + "id": "parse-045", + "description": "counts combining marks as code points", + "input": "E\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_length", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-046", + "description": "counts the input before it normalises", + "input": "EK01A03FK01\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026", + "options": {}, + "expect": { + "ok": true, + "compact": "EK01A03FK01", + "canonical": "EK-01-A03-FK-01", + "display": "EK 01 A03 FK 01", + "precision": "unit", + "segments": { + "state": "EK", + "lga": "01", + "district": "A03", + "area": "FK", + "unit": "01" + } + } + }, + { + "id": "parse-047", + "description": "rejects input over the limit that ends in a line feed", + "input": "EK01A03FK01 \n", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_length", + "segment": null, + "suggestion": null + } + } } ] } From 8c3f4364cb82e5d765e5f40c76526b04b6468b85 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 07:01:50 +0100 Subject: [PATCH 19/45] fix: use NIPOST's codes for five states NIPOST's autocomplete API returns 37 state codes. For five states they differ from ISO 3166-2:NG. Borno is BR, Gombe GM, Kogi KG, Sokoto SK and Yobe YB. With the ISO codes, parse rejected every real postcode of these states. Each state now also keeps its ISO code in the iso field. Signed-off-by: MobileMage --- data/states.json | 76 +++++++++++++++++++------------------- grammar.md | 2 +- scripts/tests/test_data.py | 19 ++++++++++ vectors/parse-states.json | 64 ++++++++++++++++---------------- vectors/state-name.json | 28 +++++++------- 5 files changed, 104 insertions(+), 85 deletions(-) diff --git a/data/states.json b/data/states.json index e7f419c..7a21f72 100644 --- a/data/states.json +++ b/data/states.json @@ -1,43 +1,43 @@ { "version": 1, - "source": "ISO 3166-2:NG. All 11 state codes in NIPOST's published examples match it. A keyed autocomplete call must confirm the full list.", + "source": "NIPOST's autocomplete API returned these 37 codes on 2 Oct 2026. 32 equal ISO 3166-2:NG. NIPOST uses BR, GM, KG, SK and YB for Borno, Gombe, Kogi, Sokoto and Yobe, where ISO uses BO, GO, KO, SO and YO. The API returns no names. The names of these five come from the first-letter groups and the Nigerian vehicle plate codes.", "states": [ - { "code": "AB", "name": "Abia" }, - { "code": "AD", "name": "Adamawa" }, - { "code": "AK", "name": "Akwa Ibom" }, - { "code": "AN", "name": "Anambra" }, - { "code": "BA", "name": "Bauchi" }, - { "code": "BE", "name": "Benue" }, - { "code": "BO", "name": "Borno" }, - { "code": "BY", "name": "Bayelsa" }, - { "code": "CR", "name": "Cross River" }, - { "code": "DE", "name": "Delta" }, - { "code": "EB", "name": "Ebonyi" }, - { "code": "ED", "name": "Edo" }, - { "code": "EK", "name": "Ekiti" }, - { "code": "EN", "name": "Enugu" }, - { "code": "FC", "name": "Federal Capital Territory" }, - { "code": "GO", "name": "Gombe" }, - { "code": "IM", "name": "Imo" }, - { "code": "JI", "name": "Jigawa" }, - { "code": "KD", "name": "Kaduna" }, - { "code": "KE", "name": "Kebbi" }, - { "code": "KN", "name": "Kano" }, - { "code": "KO", "name": "Kogi" }, - { "code": "KT", "name": "Katsina" }, - { "code": "KW", "name": "Kwara" }, - { "code": "LA", "name": "Lagos" }, - { "code": "NA", "name": "Nasarawa" }, - { "code": "NI", "name": "Niger" }, - { "code": "OG", "name": "Ogun" }, - { "code": "ON", "name": "Ondo" }, - { "code": "OS", "name": "Osun" }, - { "code": "OY", "name": "Oyo" }, - { "code": "PL", "name": "Plateau" }, - { "code": "RI", "name": "Rivers" }, - { "code": "SO", "name": "Sokoto" }, - { "code": "TA", "name": "Taraba" }, - { "code": "YO", "name": "Yobe" }, - { "code": "ZA", "name": "Zamfara" } + { "code": "AB", "name": "Abia", "iso": "NG-AB" }, + { "code": "AD", "name": "Adamawa", "iso": "NG-AD" }, + { "code": "AK", "name": "Akwa Ibom", "iso": "NG-AK" }, + { "code": "AN", "name": "Anambra", "iso": "NG-AN" }, + { "code": "BA", "name": "Bauchi", "iso": "NG-BA" }, + { "code": "BE", "name": "Benue", "iso": "NG-BE" }, + { "code": "BR", "name": "Borno", "iso": "NG-BO" }, + { "code": "BY", "name": "Bayelsa", "iso": "NG-BY" }, + { "code": "CR", "name": "Cross River", "iso": "NG-CR" }, + { "code": "DE", "name": "Delta", "iso": "NG-DE" }, + { "code": "EB", "name": "Ebonyi", "iso": "NG-EB" }, + { "code": "ED", "name": "Edo", "iso": "NG-ED" }, + { "code": "EK", "name": "Ekiti", "iso": "NG-EK" }, + { "code": "EN", "name": "Enugu", "iso": "NG-EN" }, + { "code": "FC", "name": "Federal Capital Territory", "iso": "NG-FC" }, + { "code": "GM", "name": "Gombe", "iso": "NG-GO" }, + { "code": "IM", "name": "Imo", "iso": "NG-IM" }, + { "code": "JI", "name": "Jigawa", "iso": "NG-JI" }, + { "code": "KD", "name": "Kaduna", "iso": "NG-KD" }, + { "code": "KE", "name": "Kebbi", "iso": "NG-KE" }, + { "code": "KG", "name": "Kogi", "iso": "NG-KO" }, + { "code": "KN", "name": "Kano", "iso": "NG-KN" }, + { "code": "KT", "name": "Katsina", "iso": "NG-KT" }, + { "code": "KW", "name": "Kwara", "iso": "NG-KW" }, + { "code": "LA", "name": "Lagos", "iso": "NG-LA" }, + { "code": "NA", "name": "Nasarawa", "iso": "NG-NA" }, + { "code": "NI", "name": "Niger", "iso": "NG-NI" }, + { "code": "OG", "name": "Ogun", "iso": "NG-OG" }, + { "code": "ON", "name": "Ondo", "iso": "NG-ON" }, + { "code": "OS", "name": "Osun", "iso": "NG-OS" }, + { "code": "OY", "name": "Oyo", "iso": "NG-OY" }, + { "code": "PL", "name": "Plateau", "iso": "NG-PL" }, + { "code": "RI", "name": "Rivers", "iso": "NG-RI" }, + { "code": "SK", "name": "Sokoto", "iso": "NG-SO" }, + { "code": "TA", "name": "Taraba", "iso": "NG-TA" }, + { "code": "YB", "name": "Yobe", "iso": "NG-YO" }, + { "code": "ZA", "name": "Zamfara", "iso": "NG-ZA" } ] } diff --git a/grammar.md b/grammar.md index 2c13251..ba62e13 100644 --- a/grammar.md +++ b/grammar.md @@ -68,7 +68,7 @@ Parse the corrected code with the same options. If it parses, the suggestion is ## State names -`stateName(code)` returns the name in `data/states.json`, and it ignores letter case. It does not remove spaces or other characters. An unknown code gives null. The codes follow ISO 3166-2:NG. All 11 state codes in NIPOST's published examples match it. A keyed autocomplete call must confirm the full list. +`stateName(code)` returns the name in `data/states.json`, and it ignores letter case. It does not remove spaces or other characters. An unknown code gives null. The codes are NIPOST's. They equal ISO 3166-2:NG, except BR, GM, KG, SK and YB, where ISO uses BO, GO, KO, SO and YO. Each state in `data/states.json` also keeps its ISO code in the `iso` field. ## GPS precision diff --git a/scripts/tests/test_data.py b/scripts/tests/test_data.py index 98468b2..e294d83 100644 --- a/scripts/tests/test_data.py +++ b/scripts/tests/test_data.py @@ -10,6 +10,14 @@ DATA_DIR = Path(__file__).resolve().parents[2] / "data" SEGMENT_NAMES = ["state", "lga", "district", "area", "unit"] +# The states where NIPOST's code is not the ISO 3166-2:NG code. Name: (code, iso). +NIPOST_CODES_OFF_ISO = { + "Borno": ("BR", "NG-BO"), + "Gombe": ("GM", "NG-GO"), + "Kogi": ("KG", "NG-KO"), + "Sokoto": ("SK", "NG-SO"), + "Yobe": ("YB", "NG-YO"), +} def load(name: str) -> dict[str, Any]: @@ -31,6 +39,17 @@ def test_uses_two_capital_letters_for_each_code(self) -> None: self.assertRegex(state["code"], r"^[A-Z]{2}$") self.assertTrue(state["name"].isascii() and state["name"].strip()) + def test_keeps_the_iso_code_of_each_state_beside_the_nipost_code(self) -> None: + for state in load("states.json")["states"]: + with self.subTest(name=state["name"]): + self.assertRegex(state["iso"], r"^NG-[A-Z]{2}$") + if state["name"] in NIPOST_CODES_OFF_ISO: + self.assertEqual( + (state["code"], state["iso"]), NIPOST_CODES_OFF_ISO[state["name"]] + ) + else: + self.assertEqual(state["iso"], "NG-" + state["code"]) + class PrecisionTest(unittest.TestCase): def test_raises_the_limit_as_precision_falls(self) -> None: diff --git a/vectors/parse-states.json b/vectors/parse-states.json index 05fa55a..e109cbc 100644 --- a/vectors/parse-states.json +++ b/vectors/parse-states.json @@ -126,16 +126,16 @@ { "id": "parse-states-007", "description": "parses a code in Borno", - "input": "BO-01-Z99-ZZ-01", + "input": "BR-01-Z99-ZZ-01", "options": {}, "expect": { "ok": true, - "compact": "BO01Z99ZZ01", - "canonical": "BO-01-Z99-ZZ-01", - "display": "BO 01 Z99 ZZ 01", + "compact": "BR01Z99ZZ01", + "canonical": "BR-01-Z99-ZZ-01", + "display": "BR 01 Z99 ZZ 01", "precision": "unit", "segments": { - "state": "BO", + "state": "BR", "lga": "01", "district": "Z99", "area": "ZZ", @@ -306,16 +306,16 @@ { "id": "parse-states-016", "description": "parses a code in Gombe", - "input": "GO-01-Z99-ZZ-01", + "input": "GM-01-Z99-ZZ-01", "options": {}, "expect": { "ok": true, - "compact": "GO01Z99ZZ01", - "canonical": "GO-01-Z99-ZZ-01", - "display": "GO 01 Z99 ZZ 01", + "compact": "GM01Z99ZZ01", + "canonical": "GM-01-Z99-ZZ-01", + "display": "GM 01 Z99 ZZ 01", "precision": "unit", "segments": { - "state": "GO", + "state": "GM", "lga": "01", "district": "Z99", "area": "ZZ", @@ -405,17 +405,17 @@ }, { "id": "parse-states-021", - "description": "parses a code in Kano", - "input": "KN-01-Z99-ZZ-01", + "description": "parses a code in Kogi", + "input": "KG-01-Z99-ZZ-01", "options": {}, "expect": { "ok": true, - "compact": "KN01Z99ZZ01", - "canonical": "KN-01-Z99-ZZ-01", - "display": "KN 01 Z99 ZZ 01", + "compact": "KG01Z99ZZ01", + "canonical": "KG-01-Z99-ZZ-01", + "display": "KG 01 Z99 ZZ 01", "precision": "unit", "segments": { - "state": "KN", + "state": "KG", "lga": "01", "district": "Z99", "area": "ZZ", @@ -425,17 +425,17 @@ }, { "id": "parse-states-022", - "description": "parses a code in Kogi", - "input": "KO-01-Z99-ZZ-01", + "description": "parses a code in Kano", + "input": "KN-01-Z99-ZZ-01", "options": {}, "expect": { "ok": true, - "compact": "KO01Z99ZZ01", - "canonical": "KO-01-Z99-ZZ-01", - "display": "KO 01 Z99 ZZ 01", + "compact": "KN01Z99ZZ01", + "canonical": "KN-01-Z99-ZZ-01", + "display": "KN 01 Z99 ZZ 01", "precision": "unit", "segments": { - "state": "KO", + "state": "KN", "lga": "01", "district": "Z99", "area": "ZZ", @@ -666,16 +666,16 @@ { "id": "parse-states-034", "description": "parses a code in Sokoto", - "input": "SO-01-Z99-ZZ-01", + "input": "SK-01-Z99-ZZ-01", "options": {}, "expect": { "ok": true, - "compact": "SO01Z99ZZ01", - "canonical": "SO-01-Z99-ZZ-01", - "display": "SO 01 Z99 ZZ 01", + "compact": "SK01Z99ZZ01", + "canonical": "SK-01-Z99-ZZ-01", + "display": "SK 01 Z99 ZZ 01", "precision": "unit", "segments": { - "state": "SO", + "state": "SK", "lga": "01", "district": "Z99", "area": "ZZ", @@ -706,16 +706,16 @@ { "id": "parse-states-036", "description": "parses a code in Yobe", - "input": "YO-01-Z99-ZZ-01", + "input": "YB-01-Z99-ZZ-01", "options": {}, "expect": { "ok": true, - "compact": "YO01Z99ZZ01", - "canonical": "YO-01-Z99-ZZ-01", - "display": "YO 01 Z99 ZZ 01", + "compact": "YB01Z99ZZ01", + "canonical": "YB-01-Z99-ZZ-01", + "display": "YB 01 Z99 ZZ 01", "precision": "unit", "segments": { - "state": "YO", + "state": "YB", "lga": "01", "district": "Z99", "area": "ZZ", diff --git a/vectors/state-name.json b/vectors/state-name.json index 8a00299..1fc02de 100644 --- a/vectors/state-name.json +++ b/vectors/state-name.json @@ -59,8 +59,8 @@ }, { "id": "state-name-007", - "description": "names BO", - "input": "BO", + "description": "names BR", + "input": "BR", "options": {}, "expect": { "value": "Borno" @@ -140,8 +140,8 @@ }, { "id": "state-name-016", - "description": "names GO", - "input": "GO", + "description": "names GM", + "input": "GM", "options": {}, "expect": { "value": "Gombe" @@ -185,20 +185,20 @@ }, { "id": "state-name-021", - "description": "names KN", - "input": "KN", + "description": "names KG", + "input": "KG", "options": {}, "expect": { - "value": "Kano" + "value": "Kogi" } }, { "id": "state-name-022", - "description": "names KO", - "input": "KO", + "description": "names KN", + "input": "KN", "options": {}, "expect": { - "value": "Kogi" + "value": "Kano" } }, { @@ -302,8 +302,8 @@ }, { "id": "state-name-034", - "description": "names SO", - "input": "SO", + "description": "names SK", + "input": "SK", "options": {}, "expect": { "value": "Sokoto" @@ -320,8 +320,8 @@ }, { "id": "state-name-036", - "description": "names YO", - "input": "YO", + "description": "names YB", + "input": "YB", "options": {}, "expect": { "value": "Yobe" From 548d783212f5b4f15ded295942c3eea67ec7dfc9 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 07:08:15 +0100 Subject: [PATCH 20/45] test: add vectors for partial codes, state names and counting Reviews of the TypeScript core found wrong implementations that passed every vector. Partial codes could skip the state or segment checks, state names could trim or use Unicode upper case, normalize could upper case before NFKC, and legacy checks could count graphemes or count after NFKC. Each of these now fails at least one vector. Signed-off-by: MobileMage --- scripts/build_vectors.py | 75 ++++++++++++++++++ scripts/tests/test_build_vectors.py | 34 ++++++-- vectors/is-legacy.json | 27 +++++++ vectors/normalize.json | 118 ++++++++++++++++------------ vectors/parse-segments.json | 68 +++++++++++++++- vectors/parse.json | 92 ++++++++++++++++++---- vectors/state-name.json | 45 +++++++++++ 7 files changed, 387 insertions(+), 72 deletions(-) diff --git a/scripts/build_vectors.py b/scripts/build_vectors.py index 0face04..f717fa8 100755 --- a/scripts/build_vectors.py +++ b/scripts/build_vectors.py @@ -40,6 +40,9 @@ FULL_WIDTH_HYPHEN = chr(0xFF0D) COMBINING_ACUTE = chr(0x0301) HORIZONTAL_ELLIPSIS = chr(0x2026) +ZERO_WIDTH_JOINER = chr(0x200D) +LONG_S = chr(0x017F) +DOTLESS_I = chr(0x0131) ARABIC_INDIC_LEGACY = "".join( chr(code_point) for code_point in (0x0669, 0x0660, 0x0660, 0x0661, 0x0660, 0x0668) ) @@ -163,6 +166,12 @@ def normalize_rows() -> list[Row]: value(RIGHT_TO_LEFT_OVERRIDE + CODE), ), row("changes full-width letters and digits to ASCII", full_width(CODE), want), + # An SDK that makes ASCII letters upper case before NFKC leaves these in lower case. + row( + "changes full-width lower-case letters to ASCII upper case", + full_width("ek01a03fk01"), + want, + ), row("removes full-width hyphens", joined(FULL_WIDTH_HYPHEN), want), row("removes tabs and line feeds", "EK01" + TAB + "A03" + LINE_FEED + "FK01", want), row( @@ -178,6 +187,12 @@ def normalize_rows() -> list[Row]: "ek01a03f" + SMALL_E_ACUTE + "01", value("EK01A03F" + SMALL_E_ACUTE + "01"), ), + # Unicode upper case changes U+0131 to I, and NFKC leaves it alone. + row( + "keeps a dotless i", + "EK01A03F" + DOTLESS_I + "01", + value("EK01A03F" + DOTLESS_I + "01"), + ), row("returns an empty string for empty input", "", value("")), row("returns an empty string for separators only", " - . ", value("")), *separator_rows, @@ -238,6 +253,10 @@ def parse_rows() -> list[Row]: row("rejects a unit that lost its zero", "EK-01-A03-FK-1", failed("bad_length")), row("rejects a state alone by default", "EK", failed("bad_length")), row("rejects a district code by default", "EK01A03", failed("bad_length")), + # The length check comes before the state and segment checks and before suggestions. + row("checks the length before the state", "XX", failed("bad_length")), + row("checks the length before the segments", "EK00", failed("bad_length")), + row("checks the length before a fixable segment", "EKO1A03FK1", failed("bad_length")), row("parses a state when partial codes are allowed", "EK", ok("EK"), PARTIAL), row("parses an LGA code when partial codes are allowed", "EK01", ok("EK01"), PARTIAL), row( @@ -295,6 +314,12 @@ def parse_rows() -> list[Row]: MATH_BOLD_CODE + " " * (limit - 2 * len(CODE) + 1), ok(CODE), ), + # 64 code points and 75 UTF-16 units. A runner that counts UTF-16 units rejects it. + row( + "parses an astral code padded to the input limit", + MATH_BOLD_CODE + " " * (limit - len(CODE)), + ok(CODE), + ), # 65 code points, but 1 grapheme cluster and 64 code points after NFKC. row( "counts combining marks as code points", @@ -419,6 +444,30 @@ def segment_rows() -> list[Row]: failed("bad_segment", "area", "EK-01-A03-FI"), PARTIAL, ), + row( + "rejects an unknown state when partial codes are allowed", + "XX", + failed("unknown_state", "state"), + PARTIAL, + ), + row( + "suggests a state for a partial code", + "0G", + failed("unknown_state", "state", "OG"), + PARTIAL, + ), + row( + "suggests an LGA fix for a district code", + "EKO1A03", + failed("bad_segment", "lga", "EK-01-A03"), + PARTIAL, + ), + row( + "rejects an LGA of 00 in a district code", + "EK00A03", + failed("bad_segment", "lga"), + PARTIAL, + ), row( "keeps a letter O in the district when it fixes the LGA", "EKO1AO3FK01", @@ -470,6 +519,26 @@ def legacy_rows() -> list[Row]: "900108" + " " * (limit - 5), value(False), ), + # 65 code points, but 6 grapheme clusters. A grapheme counter gives true, because + # normalize removes the joiners. + row( + "counts zero-width joiners as code points for legacy codes", + "900108" + ZERO_WIDTH_JOINER * (limit - 5), + value(False), + ), + # 26 code points, but 66 after NFKC, which changes U+2026 to 3 full stops. A count + # after NFKC gives false. + row( + "counts the input before it normalises for legacy codes", + "900108" + HORIZONTAL_ELLIPSIS * 20, + value(True), + ), + # 65 code points. In many regex engines, $ also matches before a final line feed. + row( + "rejects a legacy code over the limit that ends in a line feed", + "900108" + " " * (limit - 6) + LINE_FEED, + value(False), + ), ] @@ -581,6 +650,12 @@ def state_name_rows() -> list[Row]: row("returns nothing for empty input", "", value(None)), row("returns nothing for three letters", "EKI", value(None)), row("returns nothing for a code with spaces", " EK ", value(None)), + row("returns nothing for a code with a leading space", " EK", value(None)), + row("returns nothing for a code with a trailing space", "EK ", value(None)), + row("returns nothing for full-width letters", full_width("EK"), value(None)), + # Python's upper() changes U+017F to S and U+0131 to I. These two would give Osun and Imo. + row("returns nothing for a long s", "O" + LONG_S, value(None)), + row("returns nothing for a dotless i", DOTLESS_I + "M", value(None)), ] diff --git a/scripts/tests/test_build_vectors.py b/scripts/tests/test_build_vectors.py index a8009cb..60bf2df 100644 --- a/scripts/tests/test_build_vectors.py +++ b/scripts/tests/test_build_vectors.py @@ -13,6 +13,7 @@ import build_vectors ROOT = Path(__file__).resolve().parents[2] +ZERO_WIDTH_JOINER = chr(0x200D) def load(stem: str) -> dict[str, Any]: @@ -23,7 +24,7 @@ def load(stem: str) -> dict[str, Any]: def utf16_units(text: str) -> int: - return len(text.encode("utf-16-le")) // 2 + return len(text.encode("utf-16-le", errors="surrogatepass")) // 2 class BuildVectorsTest(unittest.TestCase): @@ -91,17 +92,16 @@ def test_each_gps_limit_has_a_case_at_the_limit_and_a_case_above_it(self) -> Non above = [expected for metres, expected in measured if limit < metres < ceiling] self.assertIn(next_precision, above) - def test_the_input_limit_has_a_case_at_the_limit_and_a_case_above_it(self) -> None: + def test_the_input_limit_has_cases_on_both_sides_and_for_each_counting_mistake(self) -> None: postcode_format = json.loads((ROOT / "data" / "format.json").read_text(encoding="utf-8")) limit = postcode_format["maxInputCodePoints"] parsed = [ (case["input"], "ok" if case["expect"]["ok"] else case["expect"]["error"]["code"]) for case in load("parse")["cases"] ] + legacy = [(case["input"], case["expect"]["value"]) for case in load("is-legacy")["cases"]] parse_outcomes = [(len(text), outcome) for text, outcome in parsed] - legacy_outcomes = [ - (len(case["input"]), case["expect"]["value"]) for case in load("is-legacy")["cases"] - ] + legacy_outcomes = [(len(text), outcome) for text, outcome in legacy] self.assertIn((limit, "ok"), parse_outcomes) self.assertIn((limit + 1, "bad_length"), parse_outcomes) self.assertIn((limit + 1, False), legacy_outcomes) @@ -110,6 +110,12 @@ def test_the_input_limit_has_a_case_at_the_limit_and_a_case_above_it(self) -> No rejected = [ text for text, outcome in parsed if outcome == "bad_length" and len(text) > limit ] + legacy_accepted = [ + text for text, outcome in legacy if outcome is True and len(text) <= limit + ] + legacy_rejected = [ + text for text, outcome in legacy if outcome is False and len(text) > limit + ] pins = { "a success with more UTF-16 units than the limit": [ text for text in accepted if utf16_units(text) > limit @@ -125,7 +131,25 @@ def test_the_input_limit_has_a_case_at_the_limit_and_a_case_above_it(self) -> No "a rejection over the limit that ends in a line feed": [ text for text in rejected if text.endswith("\n") ], + "an accepted legacy code that grows past the limit under NFKC": [ + text for text in legacy_accepted if len(unicodedata.normalize("NFKC", text)) > limit + ], + "a rejected legacy code that is within the limit without its zero-width joiners": [ + text + for text in legacy_rejected + if len(text.replace(ZERO_WIDTH_JOINER, "")) <= limit + ], + "a rejected legacy code over the limit that ends in a line feed": [ + text for text in legacy_rejected if text.endswith("\n") + ], } for requirement, texts in pins.items(): with self.subTest(requirement): self.assertTrue(texts) + # A check that ends in $ instead of \z skips a final line feed. Only an input with + # exactly one code point over the limit shows it. + line_feed_lengths = { + len(text) for text in [*rejected, *legacy_rejected] if text.endswith("\n") + } + with self.subTest("each rejection that ends in a line feed is one code point over"): + self.assertEqual(line_feed_lengths, {limit + 1}) diff --git a/vectors/is-legacy.json b/vectors/is-legacy.json index 48a3451..c0e6b4d 100644 --- a/vectors/is-legacy.json +++ b/vectors/is-legacy.json @@ -101,6 +101,33 @@ "expect": { "value": false } + }, + { + "id": "is-legacy-012", + "description": "counts zero-width joiners as code points for legacy codes", + "input": "900108\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d\u200d", + "options": {}, + "expect": { + "value": false + } + }, + { + "id": "is-legacy-013", + "description": "counts the input before it normalises for legacy codes", + "input": "900108\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026", + "options": {}, + "expect": { + "value": true + } + }, + { + "id": "is-legacy-014", + "description": "rejects a legacy code over the limit that ends in a line feed", + "input": "900108 \n", + "options": {}, + "expect": { + "value": false + } } ] } diff --git a/vectors/normalize.json b/vectors/normalize.json index 0c3d583..c907860 100644 --- a/vectors/normalize.json +++ b/vectors/normalize.json @@ -140,6 +140,15 @@ }, { "id": "normalize-016", + "description": "changes full-width lower-case letters to ASCII upper case", + "input": "\uff45\uff4b\uff10\uff11\uff41\uff10\uff13\uff46\uff4b\uff10\uff11", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-017", "description": "removes full-width hyphens", "input": "EK\uff0d01\uff0dA03\uff0dFK\uff0d01", "options": {}, @@ -148,7 +157,7 @@ } }, { - "id": "normalize-017", + "id": "normalize-018", "description": "removes tabs and line feeds", "input": "EK01\tA03\nFK01", "options": {}, @@ -157,7 +166,7 @@ } }, { - "id": "normalize-018", + "id": "normalize-019", "description": "changes a ligature to its letters", "input": "EK01A03\ufb0101", "options": {}, @@ -166,7 +175,7 @@ } }, { - "id": "normalize-019", + "id": "normalize-020", "description": "makes mixed-case letters upper case", "input": "eK01a03Fk01", "options": {}, @@ -175,7 +184,7 @@ } }, { - "id": "normalize-020", + "id": "normalize-021", "description": "keeps a legacy code", "input": "900 108", "options": {}, @@ -184,7 +193,7 @@ } }, { - "id": "normalize-021", + "id": "normalize-022", "description": "keeps characters that are not separators", "input": "EK01A03FK0!", "options": {}, @@ -193,7 +202,7 @@ } }, { - "id": "normalize-022", + "id": "normalize-023", "description": "makes only ASCII letters upper case", "input": "ek01a03f\u00e901", "options": {}, @@ -202,7 +211,16 @@ } }, { - "id": "normalize-023", + "id": "normalize-024", + "description": "keeps a dotless i", + "input": "EK01A03F\u013101", + "options": {}, + "expect": { + "value": "EK01A03F\u013101" + } + }, + { + "id": "normalize-025", "description": "returns an empty string for empty input", "input": "", "options": {}, @@ -211,7 +229,7 @@ } }, { - "id": "normalize-024", + "id": "normalize-026", "description": "returns an empty string for separators only", "input": " - . ", "options": {}, @@ -220,7 +238,7 @@ } }, { - "id": "normalize-025", + "id": "normalize-027", "description": "removes the separator U+0009", "input": "EK\t01\tA03\tFK\t01", "options": {}, @@ -229,7 +247,7 @@ } }, { - "id": "normalize-026", + "id": "normalize-028", "description": "removes the separator U+000A", "input": "EK\n01\nA03\nFK\n01", "options": {}, @@ -238,7 +256,7 @@ } }, { - "id": "normalize-027", + "id": "normalize-029", "description": "removes the separator U+000B", "input": "EK\u000b01\u000bA03\u000bFK\u000b01", "options": {}, @@ -247,7 +265,7 @@ } }, { - "id": "normalize-028", + "id": "normalize-030", "description": "removes the separator U+000C", "input": "EK\f01\fA03\fFK\f01", "options": {}, @@ -256,7 +274,7 @@ } }, { - "id": "normalize-029", + "id": "normalize-031", "description": "removes the separator U+000D", "input": "EK\r01\rA03\rFK\r01", "options": {}, @@ -265,7 +283,7 @@ } }, { - "id": "normalize-030", + "id": "normalize-032", "description": "removes the separator U+0020", "input": "EK 01 A03 FK 01", "options": {}, @@ -274,7 +292,7 @@ } }, { - "id": "normalize-031", + "id": "normalize-033", "description": "removes the separator U+002D", "input": "EK-01-A03-FK-01", "options": {}, @@ -283,7 +301,7 @@ } }, { - "id": "normalize-032", + "id": "normalize-034", "description": "removes the separator U+002E", "input": "EK.01.A03.FK.01", "options": {}, @@ -292,7 +310,7 @@ } }, { - "id": "normalize-033", + "id": "normalize-035", "description": "removes the separator U+0085", "input": "EK\u008501\u0085A03\u0085FK\u008501", "options": {}, @@ -301,7 +319,7 @@ } }, { - "id": "normalize-034", + "id": "normalize-036", "description": "removes the separator U+00A0", "input": "EK\u00a001\u00a0A03\u00a0FK\u00a001", "options": {}, @@ -310,7 +328,7 @@ } }, { - "id": "normalize-035", + "id": "normalize-037", "description": "removes the separator U+00AD", "input": "EK\u00ad01\u00adA03\u00adFK\u00ad01", "options": {}, @@ -319,7 +337,7 @@ } }, { - "id": "normalize-036", + "id": "normalize-038", "description": "removes the separator U+1680", "input": "EK\u168001\u1680A03\u1680FK\u168001", "options": {}, @@ -328,7 +346,7 @@ } }, { - "id": "normalize-037", + "id": "normalize-039", "description": "removes the separator U+2000", "input": "EK\u200001\u2000A03\u2000FK\u200001", "options": {}, @@ -337,7 +355,7 @@ } }, { - "id": "normalize-038", + "id": "normalize-040", "description": "removes the separator U+2001", "input": "EK\u200101\u2001A03\u2001FK\u200101", "options": {}, @@ -346,7 +364,7 @@ } }, { - "id": "normalize-039", + "id": "normalize-041", "description": "removes the separator U+2002", "input": "EK\u200201\u2002A03\u2002FK\u200201", "options": {}, @@ -355,7 +373,7 @@ } }, { - "id": "normalize-040", + "id": "normalize-042", "description": "removes the separator U+2003", "input": "EK\u200301\u2003A03\u2003FK\u200301", "options": {}, @@ -364,7 +382,7 @@ } }, { - "id": "normalize-041", + "id": "normalize-043", "description": "removes the separator U+2004", "input": "EK\u200401\u2004A03\u2004FK\u200401", "options": {}, @@ -373,7 +391,7 @@ } }, { - "id": "normalize-042", + "id": "normalize-044", "description": "removes the separator U+2005", "input": "EK\u200501\u2005A03\u2005FK\u200501", "options": {}, @@ -382,7 +400,7 @@ } }, { - "id": "normalize-043", + "id": "normalize-045", "description": "removes the separator U+2006", "input": "EK\u200601\u2006A03\u2006FK\u200601", "options": {}, @@ -391,7 +409,7 @@ } }, { - "id": "normalize-044", + "id": "normalize-046", "description": "removes the separator U+2007", "input": "EK\u200701\u2007A03\u2007FK\u200701", "options": {}, @@ -400,7 +418,7 @@ } }, { - "id": "normalize-045", + "id": "normalize-047", "description": "removes the separator U+2008", "input": "EK\u200801\u2008A03\u2008FK\u200801", "options": {}, @@ -409,7 +427,7 @@ } }, { - "id": "normalize-046", + "id": "normalize-048", "description": "removes the separator U+2009", "input": "EK\u200901\u2009A03\u2009FK\u200901", "options": {}, @@ -418,7 +436,7 @@ } }, { - "id": "normalize-047", + "id": "normalize-049", "description": "removes the separator U+200A", "input": "EK\u200a01\u200aA03\u200aFK\u200a01", "options": {}, @@ -427,7 +445,7 @@ } }, { - "id": "normalize-048", + "id": "normalize-050", "description": "removes the separator U+200B", "input": "EK\u200b01\u200bA03\u200bFK\u200b01", "options": {}, @@ -436,7 +454,7 @@ } }, { - "id": "normalize-049", + "id": "normalize-051", "description": "removes the separator U+200C", "input": "EK\u200c01\u200cA03\u200cFK\u200c01", "options": {}, @@ -445,7 +463,7 @@ } }, { - "id": "normalize-050", + "id": "normalize-052", "description": "removes the separator U+200D", "input": "EK\u200d01\u200dA03\u200dFK\u200d01", "options": {}, @@ -454,7 +472,7 @@ } }, { - "id": "normalize-051", + "id": "normalize-053", "description": "removes the separator U+200E", "input": "EK\u200e01\u200eA03\u200eFK\u200e01", "options": {}, @@ -463,7 +481,7 @@ } }, { - "id": "normalize-052", + "id": "normalize-054", "description": "removes the separator U+200F", "input": "EK\u200f01\u200fA03\u200fFK\u200f01", "options": {}, @@ -472,7 +490,7 @@ } }, { - "id": "normalize-053", + "id": "normalize-055", "description": "removes the separator U+2010", "input": "EK\u201001\u2010A03\u2010FK\u201001", "options": {}, @@ -481,7 +499,7 @@ } }, { - "id": "normalize-054", + "id": "normalize-056", "description": "removes the separator U+2011", "input": "EK\u201101\u2011A03\u2011FK\u201101", "options": {}, @@ -490,7 +508,7 @@ } }, { - "id": "normalize-055", + "id": "normalize-057", "description": "removes the separator U+2012", "input": "EK\u201201\u2012A03\u2012FK\u201201", "options": {}, @@ -499,7 +517,7 @@ } }, { - "id": "normalize-056", + "id": "normalize-058", "description": "removes the separator U+2013", "input": "EK\u201301\u2013A03\u2013FK\u201301", "options": {}, @@ -508,7 +526,7 @@ } }, { - "id": "normalize-057", + "id": "normalize-059", "description": "removes the separator U+2014", "input": "EK\u201401\u2014A03\u2014FK\u201401", "options": {}, @@ -517,7 +535,7 @@ } }, { - "id": "normalize-058", + "id": "normalize-060", "description": "removes the separator U+2015", "input": "EK\u201501\u2015A03\u2015FK\u201501", "options": {}, @@ -526,7 +544,7 @@ } }, { - "id": "normalize-059", + "id": "normalize-061", "description": "removes the separator U+2028", "input": "EK\u202801\u2028A03\u2028FK\u202801", "options": {}, @@ -535,7 +553,7 @@ } }, { - "id": "normalize-060", + "id": "normalize-062", "description": "removes the separator U+2029", "input": "EK\u202901\u2029A03\u2029FK\u202901", "options": {}, @@ -544,7 +562,7 @@ } }, { - "id": "normalize-061", + "id": "normalize-063", "description": "removes the separator U+202F", "input": "EK\u202f01\u202fA03\u202fFK\u202f01", "options": {}, @@ -553,7 +571,7 @@ } }, { - "id": "normalize-062", + "id": "normalize-064", "description": "removes the separator U+205F", "input": "EK\u205f01\u205fA03\u205fFK\u205f01", "options": {}, @@ -562,7 +580,7 @@ } }, { - "id": "normalize-063", + "id": "normalize-065", "description": "removes the separator U+2060", "input": "EK\u206001\u2060A03\u2060FK\u206001", "options": {}, @@ -571,7 +589,7 @@ } }, { - "id": "normalize-064", + "id": "normalize-066", "description": "removes the separator U+2212", "input": "EK\u221201\u2212A03\u2212FK\u221201", "options": {}, @@ -580,7 +598,7 @@ } }, { - "id": "normalize-065", + "id": "normalize-067", "description": "removes the separator U+3000", "input": "EK\u300001\u3000A03\u3000FK\u300001", "options": {}, @@ -589,7 +607,7 @@ } }, { - "id": "normalize-066", + "id": "normalize-068", "description": "removes the separator U+FEFF", "input": "EK\ufeff01\ufeffA03\ufeffFK\ufeff01", "options": {}, diff --git a/vectors/parse-segments.json b/vectors/parse-segments.json index ab759b3..07bfa2a 100644 --- a/vectors/parse-segments.json +++ b/vectors/parse-segments.json @@ -361,6 +361,70 @@ }, { "id": "parse-segments-026", + "description": "rejects an unknown state when partial codes are allowed", + "input": "XX", + "options": { + "allowPartial": true + }, + "expect": { + "ok": false, + "error": { + "code": "unknown_state", + "segment": "state", + "suggestion": null + } + } + }, + { + "id": "parse-segments-027", + "description": "suggests a state for a partial code", + "input": "0G", + "options": { + "allowPartial": true + }, + "expect": { + "ok": false, + "error": { + "code": "unknown_state", + "segment": "state", + "suggestion": "OG" + } + } + }, + { + "id": "parse-segments-028", + "description": "suggests an LGA fix for a district code", + "input": "EKO1A03", + "options": { + "allowPartial": true + }, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "lga", + "suggestion": "EK-01-A03" + } + } + }, + { + "id": "parse-segments-029", + "description": "rejects an LGA of 00 in a district code", + "input": "EK00A03", + "options": { + "allowPartial": true + }, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "lga", + "suggestion": null + } + } + }, + { + "id": "parse-segments-030", "description": "keeps a letter O in the district when it fixes the LGA", "input": "EKO1AO3FK01", "options": {}, @@ -374,7 +438,7 @@ } }, { - "id": "parse-segments-027", + "id": "parse-segments-031", "description": "keeps letters I and L in the district when it fixes the LGA", "input": "EKO1AILFK01", "options": {}, @@ -388,7 +452,7 @@ } }, { - "id": "parse-segments-028", + "id": "parse-segments-032", "description": "accepts letters and digits in any district position", "input": "EK01A0OFK01", "options": {}, diff --git a/vectors/parse.json b/vectors/parse.json index a5c022c..9057115 100644 --- a/vectors/parse.json +++ b/vectors/parse.json @@ -513,6 +513,48 @@ }, { "id": "parse-032", + "description": "checks the length before the state", + "input": "XX", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_length", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-033", + "description": "checks the length before the segments", + "input": "EK00", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_length", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-034", + "description": "checks the length before a fixable segment", + "input": "EKO1A03FK1", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_length", + "segment": null, + "suggestion": null + } + } + }, + { + "id": "parse-035", "description": "parses a state when partial codes are allowed", "input": "EK", "options": { @@ -534,7 +576,7 @@ } }, { - "id": "parse-033", + "id": "parse-036", "description": "parses an LGA code when partial codes are allowed", "input": "EK01", "options": { @@ -556,7 +598,7 @@ } }, { - "id": "parse-034", + "id": "parse-037", "description": "parses a district code when partial codes are allowed", "input": "ek-01-a03", "options": { @@ -578,7 +620,7 @@ } }, { - "id": "parse-035", + "id": "parse-038", "description": "parses an area code when partial codes are allowed", "input": "EK01A03FK", "options": { @@ -600,7 +642,7 @@ } }, { - "id": "parse-036", + "id": "parse-039", "description": "parses a full code when partial codes are allowed", "input": "EK01A03FK01", "options": { @@ -622,7 +664,7 @@ } }, { - "id": "parse-037", + "id": "parse-040", "description": "rejects 3 characters when partial codes are allowed", "input": "EK0", "options": { @@ -638,7 +680,7 @@ } }, { - "id": "parse-038", + "id": "parse-041", "description": "rejects 5 characters when partial codes are allowed", "input": "EK01A", "options": { @@ -654,7 +696,7 @@ } }, { - "id": "parse-039", + "id": "parse-042", "description": "rejects 6 characters when partial codes are allowed", "input": "EK01A0", "options": { @@ -670,7 +712,7 @@ } }, { - "id": "parse-040", + "id": "parse-043", "description": "rejects 8 characters when partial codes are allowed", "input": "EK01A03F", "options": { @@ -686,7 +728,7 @@ } }, { - "id": "parse-041", + "id": "parse-044", "description": "rejects 10 characters when partial codes are allowed", "input": "EK01A03FK0", "options": { @@ -702,7 +744,7 @@ } }, { - "id": "parse-042", + "id": "parse-045", "description": "parses a code padded to the input limit", "input": "EK01A03FK01 ", "options": {}, @@ -722,7 +764,7 @@ } }, { - "id": "parse-043", + "id": "parse-046", "description": "rejects input over the input limit", "input": "EK01A03FK01 ", "options": {}, @@ -736,7 +778,7 @@ } }, { - "id": "parse-044", + "id": "parse-047", "description": "counts code points, not UTF-16 units", "input": "\ud835\udc04\ud835\udc0a\ud835\udfce\ud835\udfcf\ud835\udc00\ud835\udfce\ud835\udfd1\ud835\udc05\ud835\udc0a\ud835\udfce\ud835\udfcf ", "options": {}, @@ -756,7 +798,27 @@ } }, { - "id": "parse-045", + "id": "parse-048", + "description": "parses an astral code padded to the input limit", + "input": "\ud835\udc04\ud835\udc0a\ud835\udfce\ud835\udfcf\ud835\udc00\ud835\udfce\ud835\udfd1\ud835\udc05\ud835\udc0a\ud835\udfce\ud835\udfcf ", + "options": {}, + "expect": { + "ok": true, + "compact": "EK01A03FK01", + "canonical": "EK-01-A03-FK-01", + "display": "EK 01 A03 FK 01", + "precision": "unit", + "segments": { + "state": "EK", + "lga": "01", + "district": "A03", + "area": "FK", + "unit": "01" + } + } + }, + { + "id": "parse-049", "description": "counts combining marks as code points", "input": "E\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301\u0301", "options": {}, @@ -770,7 +832,7 @@ } }, { - "id": "parse-046", + "id": "parse-050", "description": "counts the input before it normalises", "input": "EK01A03FK01\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026\u2026", "options": {}, @@ -790,7 +852,7 @@ } }, { - "id": "parse-047", + "id": "parse-051", "description": "rejects input over the limit that ends in a line feed", "input": "EK01A03FK01 \n", "options": {}, diff --git a/vectors/state-name.json b/vectors/state-name.json index 1fc02de..b5c4e9f 100644 --- a/vectors/state-name.json +++ b/vectors/state-name.json @@ -389,6 +389,51 @@ "expect": { "value": null } + }, + { + "id": "state-name-044", + "description": "returns nothing for a code with a leading space", + "input": " EK", + "options": {}, + "expect": { + "value": null + } + }, + { + "id": "state-name-045", + "description": "returns nothing for a code with a trailing space", + "input": "EK ", + "options": {}, + "expect": { + "value": null + } + }, + { + "id": "state-name-046", + "description": "returns nothing for full-width letters", + "input": "\uff25\uff2b", + "options": {}, + "expect": { + "value": null + } + }, + { + "id": "state-name-047", + "description": "returns nothing for a long s", + "input": "O\u017f", + "options": {}, + "expect": { + "value": null + } + }, + { + "id": "state-name-048", + "description": "returns nothing for a dotless i", + "input": "\u0131M", + "options": {}, + "expect": { + "value": null + } } ] } From 80172d2b87cde66fc68dc619268fc08e415a2ed3 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 07:12:06 +0100 Subject: [PATCH 21/45] docs: add glossary terms and settle TypeScript rules The glossary gains input limit, code point, suggestion and normalise, and it says where the word prefix is allowed. typescript.md now matches the ESLint config: readonly arrays, the extra rules, the Node versions and one spelling for options types. Signed-off-by: MobileMage --- CONTEXT.md | 17 +++++++++++++++++ grammar.md | 4 ++-- standards/languages/typescript.md | 13 +++++++++---- 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index bb5bd5b..f256b04 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -41,6 +41,7 @@ _Avoid_: building number, house number, delivery unit **Partial postcode**: A postcode that stops after the state, LGA, district or area segment. _Avoid_: prefix, incomplete code +Exception: `prefix` is the name of the first parameter of `contains`. **Precision**: The most specific segment that a postcode or a result contains. @@ -60,6 +61,10 @@ _Avoid_: formatted, standard form The postcode with spaces between segments, for example `EK 01 A03 FK 01`. _Avoid_: pretty, human form +**Normalise**: +To change input in three steps, in this order: apply NFKC, remove the separators, and change the ASCII letters to upper case. `grammar.md` defines the steps. +_Avoid_: sanitise, canonicalise + ### NIPOST's API **Gateway**: @@ -135,3 +140,15 @@ _Avoid_: verify, validate (for this step) **Tell**: A habit that makes code look machine-written, such as very long lines. _Avoid_: smell (smells are design problems), anti-pattern + +**Code point**: +One Unicode code point. It is not a UTF-16 unit, a byte or a grapheme cluster. For example, U+1D404 is one code point and two UTF-16 units. +_Avoid_: character (when you count input), char + +**Input limit**: +The most code points that `parse` and `isLegacy` read. It is `maxInputCodePoints` in `data/format.json`. Over the limit, `parse` gives `bad_length` and `isLegacy` gives false. Both functions apply the limit before they normalise. +_Avoid_: max length, length limit, size limit + +**Suggestion**: +The canonical code that `parse` offers with `unknown_state` or `bad_segment`, when a fix of look-alike characters makes the code parse. It is a hint, never a success. +_Avoid_: correction, autocorrect diff --git a/grammar.md b/grammar.md index ba62e13..5c380e5 100644 --- a/grammar.md +++ b/grammar.md @@ -55,14 +55,14 @@ For `unknown_state` and `bad_segment`, build one corrected code: Parse the corrected code with the same options. If it parses, the suggestion is its canonical form. If it does not parse, or if no character changed, the suggestion is null. A suggestion is a hint only. `parse` never returns a corrected code as a success. -## Legacy codes +## Legacy postcodes `isLegacy` is false for input with more than `maxInputCodePoints` code points, and it does not normalise such input. Otherwise, it is true when the normalised input is exactly 6 ASCII digits. ## Hierarchy - `truncate(code, to)` keeps the segments up to `to`. It fails when `to` is more precise than the code. Each language uses its standard error for a programmer mistake, for example `RangeError` in TypeScript. -- `parent(code)` returns the code with one segment fewer. A state code has no parent. +- `parent(code)` returns the code with one segment fewer. A postcode with only the state segment has no parent, so `parent` returns null for it. - `contains(prefix, code)` is true when the code has every segment of the prefix, in the same places. - `redact(code)` replaces the unit with `**` in the canonical form. A code without a unit stays the same. diff --git a/standards/languages/typescript.md b/standards/languages/typescript.md index da48693..3dfe70e 100644 --- a/standards/languages/typescript.md +++ b/standards/languages/typescript.md @@ -7,7 +7,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for Type ## Versions - Published packages run on Node 22 or later. Node 22 reaches end of life on 30 Apr 2027. Raise the floor to Node 24 before then (VER-2). -- The development tools need Node 22.22.2 or later, or Node 24.15 or later. CI runs the tests on Node 22 and Node 24, and the other tools on Node 24. +- The development tools need Node 22.22.2 or a later 22.x release, or Node 24.15 or later. CI runs the tests on Node 22 and Node 24, and the other tools on Node 24. - TypeScript 6.0. typescript-eslint supports only TypeScript versions below 6.1, so TypeScript 7 waits until typescript-eslint supports it. This was checked on 1 Oct 2026. - React 18 and React 19, for `@gatepost/react`. - Current evergreen browsers and Safari 16.4 or later, for browser code. @@ -31,7 +31,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for Type | publint and `@arethetypeswrong/cli` | package checks | every published package | | API Extractor | API report | `etc/.api.md`, checked in CI | | Changesets | versions and changelogs | one change file per user-visible change | -| commitlint | commit messages | `@commitlint/config-conventional`, scopes from package names | +| commitlint | commit messages | `@commitlint/config-conventional`, scopes from package names, and `scope-empty` set to `never`, so each commit needs a scope (GIT-1) | | REUSE | licence headers | `reuse lint` | | gitleaks | secret scan | each pull request and each push to `main` | | `spec/scripts/check-tells` | agent tells | line length 100, file names, TODO form, debug calls, emojis and non-ASCII | @@ -73,9 +73,13 @@ tsdown builds the packages, so `tsc` only checks types. Each package's `tsconfig | `no-restricted-exports` with `restrictDefaultExports` | error | TS-1 | | `no-console` | error in packages | TS-12 | | `@eslint-community/eslint-comments/require-description` | error | TS-13 | +| `eslint --max-warnings 0` | a warning fails the check, such as an unused `eslint-disable` comment | TS-13 | | `jsdoc/require-jsdoc` with `publicOnly: true`, and `jsdoc/require-example` for exported functions, with `exemptedBy: ['internal']` | error in packages | TS-9 | | `jsdoc/tag-lines` with `startLines: 1` | error in packages | one blank line between a description and its tags | -| `no-restricted-imports` for `node:*` | error in `packages/*/src` | CS-2 | +| `@typescript-eslint/explicit-module-boundary-types` | error in `packages/*/src` | TS-8 | +| `@typescript-eslint/no-non-null-assertion` | off in `packages/*/test` | TS-4 | +| `no-restricted-imports` for every Node built-in module, by bare name and as `node:*` | error in `packages/*/src` | CS-2 | +| `no-restricted-globals` for `process`, `Buffer`, `__dirname`, `__filename` and `global` | error in `packages/*/src` | CS-2 | | `max-depth` | 3 | TELL-2 | | `complexity` | 10 | TELL-2 | | `max-params` | 4 | TELL-3 | @@ -95,10 +99,11 @@ Prettier's `printWidth: 100` matches TELL-1. Prettier does not split long string - **TS-3 MUST.** Do not use `any`. Narrow `unknown` instead. **(tool)** - **TS-4 MUST.** Do not use non-null assertions outside tests. **(tool)** - **TS-5 MUST.** Do not use `enum` or `namespace`. Use string literal unions and modules. **(tool)** -- **TS-6 MUST.** Public types use `readonly` properties and `ReadonlyArray`. +- **TS-6 MUST.** Public types use `readonly` properties and `readonly` arrays, written `readonly T[]`. - **TS-7 MUST.** A result type is a discriminated union with an `ok` field. - **TS-8 MUST.** Each exported function and method declares its return type. **(tool)** - **TS-9 MUST.** Each exported function has a TSDoc comment with `@example`. Each exported type and constant has a TSDoc comment. A helper that other modules use, but that `src/index.ts` does not export, carries the tag `@internal`. **(tool)** +- **TS-22 MUST.** Write an options object type inline as `Readonly<{ name?: Type }>`, and document it with one `@param options` line. The literal `readonly` form fails `eslint-plugin-jsdoc`, and a dotted `@param` name fails the TSDoc parser that API Extractor uses. - **TS-10 MUST.** An error class extends `Error`, sets `name` and passes `cause`. - **TS-11 MUST.** A `catch` block treats its value as `unknown` and narrows it. **(tool)** - **TS-12 MUST.** Packages do not call `console`. **(tool)** From 7e7756fc22bb701522feaf2d02ed8cb1643047d6 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 07:19:19 +0100 Subject: [PATCH 22/45] test: reject the ISO codes that NIPOST does not use NIPOST uses BR, GM, KG, SK and YB where ISO uses BO, GO, KO, SO and YO. New cases show that stateName and parse treat the five ISO codes as unknown, so an SDK cannot keep both lists. The grammar now says that stateName folds the ASCII letters a to z only. Signed-off-by: MobileMage --- grammar.md | 2 +- scripts/build_vectors.py | 11 +++++ scripts/tests/test_build_vectors.py | 9 ++++ vectors/parse-segments.json | 72 +++++++++++++++++------------ vectors/state-name.json | 45 ++++++++++++++++++ 5 files changed, 109 insertions(+), 30 deletions(-) diff --git a/grammar.md b/grammar.md index 5c380e5..68ac1e0 100644 --- a/grammar.md +++ b/grammar.md @@ -68,7 +68,7 @@ Parse the corrected code with the same options. If it parses, the suggestion is ## State names -`stateName(code)` returns the name in `data/states.json`, and it ignores letter case. It does not remove spaces or other characters. An unknown code gives null. The codes are NIPOST's. They equal ISO 3166-2:NG, except BR, GM, KG, SK and YB, where ISO uses BO, GO, KO, SO and YO. Each state in `data/states.json` also keeps its ISO code in the `iso` field. +`stateName(code)` returns the name in `data/states.json`, and it ignores the case of the ASCII letters a to z only. It does not remove spaces or other characters. An unknown code gives null. The codes are NIPOST's. They equal ISO 3166-2:NG, except BR, GM, KG, SK and YB, where ISO uses BO, GO, KO, SO and YO. Each state in `data/states.json` also keeps its ISO code in the `iso` field. ## GPS precision diff --git a/scripts/build_vectors.py b/scripts/build_vectors.py index f717fa8..b41f91f 100755 --- a/scripts/build_vectors.py +++ b/scripts/build_vectors.py @@ -342,6 +342,11 @@ def segment_rows() -> list[Row]: return [ row("rejects an unknown state", "XX01A03FK01", failed("unknown_state", "state")), row("rejects the synthetic state ZZ", "ZZ01Z99ZZ01", failed("unknown_state", "state")), + row( + "rejects the ISO code of a state that NIPOST does not use", + "BO01Z99ZZ01", + failed("unknown_state", "state"), + ), row( "suggests O for a zero in the state", "0G01A03FK01", @@ -656,6 +661,12 @@ def state_name_rows() -> list[Row]: # Python's upper() changes U+017F to S and U+0131 to I. These two would give Osun and Imo. row("returns nothing for a long s", "O" + LONG_S, value(None)), row("returns nothing for a dotless i", DOTLESS_I + "M", value(None)), + # NIPOST uses BR, GM, KG, SK and YB for these five states, so ISO's codes are unknown. + row("returns nothing for BO, the ISO code of Borno", "BO", value(None)), + row("returns nothing for GO, the ISO code of Gombe", "GO", value(None)), + row("returns nothing for KO, the ISO code of Kogi", "KO", value(None)), + row("returns nothing for SO, the ISO code of Sokoto", "SO", value(None)), + row("returns nothing for YO, the ISO code of Yobe", "YO", value(None)), ] diff --git a/scripts/tests/test_build_vectors.py b/scripts/tests/test_build_vectors.py index 60bf2df..359ce65 100644 --- a/scripts/tests/test_build_vectors.py +++ b/scripts/tests/test_build_vectors.py @@ -14,6 +14,9 @@ ROOT = Path(__file__).resolve().parents[2] ZERO_WIDTH_JOINER = chr(0x200D) +# The ISO 3166-2:NG codes that NIPOST does not use. Its codes for the same five states are +# BR, GM, KG, SK and YB. +UNUSED_ISO_CODES = ("BO", "GO", "KO", "SO", "YO") def load(stem: str) -> dict[str, Any]: @@ -60,6 +63,12 @@ def test_each_state_has_a_parse_case_and_a_name_case(self) -> None: self.assertEqual(parsed, codes) self.assertLessEqual(codes, named) + def test_each_unused_iso_code_has_a_state_name_case_that_returns_nothing(self) -> None: + names = [(case["input"], case["expect"]["value"]) for case in load("state-name")["cases"]] + for iso_code in UNUSED_ISO_CODES: + with self.subTest(iso_code): + self.assertIn((iso_code, None), names) + def test_each_separator_has_a_normalize_case(self) -> None: postcode_format = json.loads((ROOT / "data" / "format.json").read_text(encoding="utf-8")) expected = {"value": "EK01A03FK01"} diff --git a/vectors/parse-segments.json b/vectors/parse-segments.json index 07bfa2a..6f17a09 100644 --- a/vectors/parse-segments.json +++ b/vectors/parse-segments.json @@ -33,6 +33,20 @@ }, { "id": "parse-segments-003", + "description": "rejects the ISO code of a state that NIPOST does not use", + "input": "BO01Z99ZZ01", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "unknown_state", + "segment": "state", + "suggestion": null + } + } + }, + { + "id": "parse-segments-004", "description": "suggests O for a zero in the state", "input": "0G01A03FK01", "options": {}, @@ -46,7 +60,7 @@ } }, { - "id": "parse-segments-004", + "id": "parse-segments-005", "description": "suggests I for a one in the state", "input": "1M01A03FK01", "options": {}, @@ -60,7 +74,7 @@ } }, { - "id": "parse-segments-005", + "id": "parse-segments-006", "description": "gives no suggestion when the fixed state is unknown", "input": "E101A03FK01", "options": {}, @@ -74,7 +88,7 @@ } }, { - "id": "parse-segments-006", + "id": "parse-segments-007", "description": "gives no suggestion when the state cannot be fixed", "input": "XXO1A03FK01", "options": {}, @@ -88,7 +102,7 @@ } }, { - "id": "parse-segments-007", + "id": "parse-segments-008", "description": "fixes later segments in a state suggestion", "input": "0GO1A03FK01", "options": {}, @@ -102,7 +116,7 @@ } }, { - "id": "parse-segments-008", + "id": "parse-segments-009", "description": "rejects an LGA of 00", "input": "EK00A03FK01", "options": {}, @@ -116,7 +130,7 @@ } }, { - "id": "parse-segments-009", + "id": "parse-segments-010", "description": "rejects a unit of 00", "input": "EK01A03FK00", "options": {}, @@ -130,7 +144,7 @@ } }, { - "id": "parse-segments-010", + "id": "parse-segments-011", "description": "suggests 0 for a letter O in the LGA", "input": "EKO1A03FK01", "options": {}, @@ -144,7 +158,7 @@ } }, { - "id": "parse-segments-011", + "id": "parse-segments-012", "description": "suggests 1 for a letter I in the LGA", "input": "EKI1A03FK01", "options": {}, @@ -158,7 +172,7 @@ } }, { - "id": "parse-segments-012", + "id": "parse-segments-013", "description": "suggests 1 for a letter L in the LGA", "input": "EKL1A03FK01", "options": {}, @@ -172,7 +186,7 @@ } }, { - "id": "parse-segments-013", + "id": "parse-segments-014", "description": "suggests I for a one in the area", "input": "EK01A03F101", "options": {}, @@ -186,7 +200,7 @@ } }, { - "id": "parse-segments-014", + "id": "parse-segments-015", "description": "suggests O for a zero in the area", "input": "EK01A03F001", "options": {}, @@ -200,7 +214,7 @@ } }, { - "id": "parse-segments-015", + "id": "parse-segments-016", "description": "suggests 0 for a letter O in the unit", "input": "EK01A03FKO1", "options": {}, @@ -214,7 +228,7 @@ } }, { - "id": "parse-segments-016", + "id": "parse-segments-017", "description": "suggests 1 for a letter I in the unit", "input": "EK01A03FKI2", "options": {}, @@ -228,7 +242,7 @@ } }, { - "id": "parse-segments-017", + "id": "parse-segments-018", "description": "suggests 1 for a letter L in the unit", "input": "EK01A03FKL1", "options": {}, @@ -242,7 +256,7 @@ } }, { - "id": "parse-segments-018", + "id": "parse-segments-019", "description": "suggests 1 for a letter I after a zero in the unit", "input": "EK01A03FK0I", "options": {}, @@ -256,7 +270,7 @@ } }, { - "id": "parse-segments-019", + "id": "parse-segments-020", "description": "gives no suggestion when the fix makes a unit of 00", "input": "EK01A03FK0O", "options": {}, @@ -270,7 +284,7 @@ } }, { - "id": "parse-segments-020", + "id": "parse-segments-021", "description": "gives no suggestion when a segment cannot be fixed", "input": "EK01A03F2O1", "options": {}, @@ -284,7 +298,7 @@ } }, { - "id": "parse-segments-021", + "id": "parse-segments-022", "description": "reports the first bad segment and fixes them all", "input": "EKO1A03F1O1", "options": {}, @@ -298,7 +312,7 @@ } }, { - "id": "parse-segments-022", + "id": "parse-segments-023", "description": "suggests a fix after making letters upper case", "input": "eko1a03fk01", "options": {}, @@ -312,7 +326,7 @@ } }, { - "id": "parse-segments-023", + "id": "parse-segments-024", "description": "rejects an LGA of 00 in a partial code", "input": "EK00", "options": { @@ -328,7 +342,7 @@ } }, { - "id": "parse-segments-024", + "id": "parse-segments-025", "description": "suggests a partial code", "input": "EKO1", "options": { @@ -344,7 +358,7 @@ } }, { - "id": "parse-segments-025", + "id": "parse-segments-026", "description": "suggests a partial area code", "input": "EK01A03F1", "options": { @@ -360,7 +374,7 @@ } }, { - "id": "parse-segments-026", + "id": "parse-segments-027", "description": "rejects an unknown state when partial codes are allowed", "input": "XX", "options": { @@ -376,7 +390,7 @@ } }, { - "id": "parse-segments-027", + "id": "parse-segments-028", "description": "suggests a state for a partial code", "input": "0G", "options": { @@ -392,7 +406,7 @@ } }, { - "id": "parse-segments-028", + "id": "parse-segments-029", "description": "suggests an LGA fix for a district code", "input": "EKO1A03", "options": { @@ -408,7 +422,7 @@ } }, { - "id": "parse-segments-029", + "id": "parse-segments-030", "description": "rejects an LGA of 00 in a district code", "input": "EK00A03", "options": { @@ -424,7 +438,7 @@ } }, { - "id": "parse-segments-030", + "id": "parse-segments-031", "description": "keeps a letter O in the district when it fixes the LGA", "input": "EKO1AO3FK01", "options": {}, @@ -438,7 +452,7 @@ } }, { - "id": "parse-segments-031", + "id": "parse-segments-032", "description": "keeps letters I and L in the district when it fixes the LGA", "input": "EKO1AILFK01", "options": {}, @@ -452,7 +466,7 @@ } }, { - "id": "parse-segments-032", + "id": "parse-segments-033", "description": "accepts letters and digits in any district position", "input": "EK01A0OFK01", "options": {}, diff --git a/vectors/state-name.json b/vectors/state-name.json index b5c4e9f..f76b006 100644 --- a/vectors/state-name.json +++ b/vectors/state-name.json @@ -434,6 +434,51 @@ "expect": { "value": null } + }, + { + "id": "state-name-049", + "description": "returns nothing for BO, the ISO code of Borno", + "input": "BO", + "options": {}, + "expect": { + "value": null + } + }, + { + "id": "state-name-050", + "description": "returns nothing for GO, the ISO code of Gombe", + "input": "GO", + "options": {}, + "expect": { + "value": null + } + }, + { + "id": "state-name-051", + "description": "returns nothing for KO, the ISO code of Kogi", + "input": "KO", + "options": {}, + "expect": { + "value": null + } + }, + { + "id": "state-name-052", + "description": "returns nothing for SO, the ISO code of Sokoto", + "input": "SO", + "options": {}, + "expect": { + "value": null + } + }, + { + "id": "state-name-053", + "description": "returns nothing for YO, the ISO code of Yobe", + "input": "YO", + "options": {}, + "expect": { + "value": null + } } ] } From cd988235e3d7cb9669a3e81ee6394e71d9e84c2f Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 07:20:01 +0100 Subject: [PATCH 23/45] docs: point the templates to NIPOST's website The issue, contributing and security templates named two NIPOST email addresses that NIPOST's developer docs do not list. They now send readers to NIPOST's official website for its contact details. Signed-off-by: MobileMage --- templates/CONTRIBUTING.md | 2 +- templates/ISSUE_TEMPLATE/bug_report.yml | 2 +- templates/SECURITY.md | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/templates/CONTRIBUTING.md b/templates/CONTRIBUTING.md index a141065..c73adf4 100644 --- a/templates/CONTRIBUTING.md +++ b/templates/CONTRIBUTING.md @@ -2,7 +2,7 @@ Thank you for your interest in Gatepost. This guide explains how to make a change that we can merge. -Gatepost is unofficial. For problems with NIPOST's own app, data or API, contact NIPOST at postcode@nipost.gov.ng. +Gatepost is unofficial. For problems with NIPOST's own app, data or API, contact NIPOST through the contact details on its official website. ## Before you start diff --git a/templates/ISSUE_TEMPLATE/bug_report.yml b/templates/ISSUE_TEMPLATE/bug_report.yml index f5ce424..49504cb 100644 --- a/templates/ISSUE_TEMPLATE/bug_report.yml +++ b/templates/ISSUE_TEMPLATE/bug_report.yml @@ -6,7 +6,7 @@ body: attributes: value: > Gatepost is unofficial. For problems with NIPOST's own app, data or API, - contact NIPOST at postcode@nipost.gov.ng. + contact NIPOST through the contact details on its official website. Do not paste API keys, NINs, email addresses or the postcodes of private homes. Use synthetic postcodes, such as FC-01-Z99-ZZ-01. diff --git a/templates/SECURITY.md b/templates/SECURITY.md index a92d30b..4a06087 100644 --- a/templates/SECURITY.md +++ b/templates/SECURITY.md @@ -23,4 +23,4 @@ You can also email {{SECURITY_EMAIL}}. ## Problems in NIPOST's systems -This policy covers Gatepost code only. Report a problem in NIPOST's systems to security@nipost.gov.ng. Do not test NIPOST's systems. NIPOST's acceptable use policy forbids probing them. +This policy covers Gatepost code only. Report a problem in NIPOST's systems to NIPOST, through the contact details on its official website. Do not test NIPOST's systems. NIPOST's acceptable use policy forbids probing them. From 2d67dbcf6c921c63accd92d35e84c167fa19296d Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 07:48:46 +0100 Subject: [PATCH 24/45] fix: derive the ISO cases from the data and drop a guess The five ISO cases were typed by hand in three places, and only BO had a parse case. The builder and the test now read them from states.json, and each ISO code has a parse case. The source text no longer names vehicle plate codes, which nobody verified. The grammar, glossary and TypeScript standard now say what the cases and the config enforce. Signed-off-by: MobileMage --- CONTEXT.md | 24 +++--- data/states.json | 2 +- grammar.md | 2 +- scripts/build_vectors.py | 35 ++++++--- scripts/tests/test_build_vectors.py | 91 ++++++++++++++-------- scripts/tests/test_data.py | 3 + standards/languages/typescript.md | 2 +- vectors/parse-segments.json | 116 +++++++++++++++++++++------- 8 files changed, 187 insertions(+), 88 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index f256b04..840b4b9 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -65,6 +65,18 @@ _Avoid_: pretty, human form To change input in three steps, in this order: apply NFKC, remove the separators, and change the ASCII letters to upper case. `grammar.md` defines the steps. _Avoid_: sanitise, canonicalise +**Code point**: +One Unicode code point. It is not a UTF-16 unit, a byte or a grapheme cluster. For example, U+1D404 is one code point and two UTF-16 units. +_Avoid_: character (when you count input), char + +**Input limit**: +The most code points that `parse` and `isLegacy` read. It is `maxInputCodePoints` in `data/format.json`. Over the limit, `parse` gives `bad_length` and `isLegacy` gives false. Both functions apply the limit before they normalise. +_Avoid_: max length, length limit, size limit + +**Suggestion**: +The canonical form that `parse` offers with `unknown_state` or `bad_segment`, when a fix of look-alike characters makes the code parse. It is a hint, never a success. The word means this hint from `parse`. It does not mean the suggestions that NIPOST's autocomplete API returns. +_Avoid_: correction, autocorrect + ### NIPOST's API **Gateway**: @@ -140,15 +152,3 @@ _Avoid_: verify, validate (for this step) **Tell**: A habit that makes code look machine-written, such as very long lines. _Avoid_: smell (smells are design problems), anti-pattern - -**Code point**: -One Unicode code point. It is not a UTF-16 unit, a byte or a grapheme cluster. For example, U+1D404 is one code point and two UTF-16 units. -_Avoid_: character (when you count input), char - -**Input limit**: -The most code points that `parse` and `isLegacy` read. It is `maxInputCodePoints` in `data/format.json`. Over the limit, `parse` gives `bad_length` and `isLegacy` gives false. Both functions apply the limit before they normalise. -_Avoid_: max length, length limit, size limit - -**Suggestion**: -The canonical code that `parse` offers with `unknown_state` or `bad_segment`, when a fix of look-alike characters makes the code parse. It is a hint, never a success. -_Avoid_: correction, autocorrect diff --git a/data/states.json b/data/states.json index 7a21f72..16a49e3 100644 --- a/data/states.json +++ b/data/states.json @@ -1,6 +1,6 @@ { "version": 1, - "source": "NIPOST's autocomplete API returned these 37 codes on 2 Oct 2026. 32 equal ISO 3166-2:NG. NIPOST uses BR, GM, KG, SK and YB for Borno, Gombe, Kogi, Sokoto and Yobe, where ISO uses BO, GO, KO, SO and YO. The API returns no names. The names of these five come from the first-letter groups and the Nigerian vehicle plate codes.", + "source": "NIPOST's autocomplete API returned these 37 codes on 2 Oct 2026. 32 equal ISO 3166-2:NG. NIPOST uses BR, GM, KG, SK and YB for Borno, Gombe, Kogi, Sokoto and Yobe, where ISO uses BO, GO, KO, SO and YO. The API returns no names. The names of these five come from the first-letter groups: each group lacks one ISO code and has one NIPOST code.", "states": [ { "code": "AB", "name": "Abia", "iso": "NG-AB" }, { "code": "AD", "name": "Adamawa", "iso": "NG-AD" }, diff --git a/grammar.md b/grammar.md index 68ac1e0..72cfd23 100644 --- a/grammar.md +++ b/grammar.md @@ -68,7 +68,7 @@ Parse the corrected code with the same options. If it parses, the suggestion is ## State names -`stateName(code)` returns the name in `data/states.json`, and it ignores the case of the ASCII letters a to z only. It does not remove spaces or other characters. An unknown code gives null. The codes are NIPOST's. They equal ISO 3166-2:NG, except BR, GM, KG, SK and YB, where ISO uses BO, GO, KO, SO and YO. Each state in `data/states.json` also keeps its ISO code in the `iso` field. +`stateName(code)` returns the name in `data/states.json`, and it ignores the case of the ASCII letters a to z only. It does not remove spaces or other characters. It does not apply NFKC. An unknown code gives null. The codes are NIPOST's. They equal ISO 3166-2:NG, except BR, GM, KG, SK and YB, where ISO uses BO, GO, KO, SO and YO. Each state in `data/states.json` also keeps its ISO code in the `iso` field. ## GPS precision diff --git a/scripts/build_vectors.py b/scripts/build_vectors.py index b41f91f..77eee08 100755 --- a/scripts/build_vectors.py +++ b/scripts/build_vectors.py @@ -120,6 +120,15 @@ def load_states() -> list[dict[str, str]]: return states +def load_unused_iso_codes() -> list[tuple[str, str]]: + """Return the name and ISO code of each state where NIPOST's code is not the ISO code.""" + return [ + (state["name"], state["iso"].removeprefix("NG-")) + for state in load_states() + if state["iso"] != "NG-" + state["code"] + ] + + def load_format() -> dict[str, Any]: """Read data/format.json.""" postcode_format: dict[str, Any] = json.loads( @@ -339,14 +348,18 @@ def parse_rows() -> list[Row]: def segment_rows() -> list[Row]: """Cases for segment rules and typo suggestions.""" + unused_iso = [ + row( + f"rejects {iso_code}, the ISO code of {name}, as a state", + iso_code + "01Z99ZZ01", + failed("unknown_state", "state"), + ) + for name, iso_code in load_unused_iso_codes() + ] return [ row("rejects an unknown state", "XX01A03FK01", failed("unknown_state", "state")), row("rejects the synthetic state ZZ", "ZZ01Z99ZZ01", failed("unknown_state", "state")), - row( - "rejects the ISO code of a state that NIPOST does not use", - "BO01Z99ZZ01", - failed("unknown_state", "state"), - ), + *unused_iso, row( "suggests O for a zero in the state", "0G01A03FK01", @@ -647,6 +660,10 @@ def state_name_rows() -> list[Row]: row(f"names {state['code']}", state["code"], value(state["name"])) for state in load_states() ] + unused_iso = [ + row(f"returns nothing for {iso_code}, the ISO code of {name}", iso_code, value(None)) + for name, iso_code in load_unused_iso_codes() + ] return [ *named, row("accepts lower case", "ek", value("Ekiti")), @@ -661,12 +678,8 @@ def state_name_rows() -> list[Row]: # Python's upper() changes U+017F to S and U+0131 to I. These two would give Osun and Imo. row("returns nothing for a long s", "O" + LONG_S, value(None)), row("returns nothing for a dotless i", DOTLESS_I + "M", value(None)), - # NIPOST uses BR, GM, KG, SK and YB for these five states, so ISO's codes are unknown. - row("returns nothing for BO, the ISO code of Borno", "BO", value(None)), - row("returns nothing for GO, the ISO code of Gombe", "GO", value(None)), - row("returns nothing for KO, the ISO code of Kogi", "KO", value(None)), - row("returns nothing for SO, the ISO code of Sokoto", "SO", value(None)), - row("returns nothing for YO, the ISO code of Yobe", "YO", value(None)), + # Where NIPOST's code is not the ISO code, the ISO code is an unknown state. + *unused_iso, ] diff --git a/scripts/tests/test_build_vectors.py b/scripts/tests/test_build_vectors.py index 359ce65..be0049a 100644 --- a/scripts/tests/test_build_vectors.py +++ b/scripts/tests/test_build_vectors.py @@ -14,9 +14,6 @@ ROOT = Path(__file__).resolve().parents[2] ZERO_WIDTH_JOINER = chr(0x200D) -# The ISO 3166-2:NG codes that NIPOST does not use. Its codes for the same five states are -# BR, GM, KG, SK and YB. -UNUSED_ISO_CODES = ("BO", "GO", "KO", "SO", "YO") def load(stem: str) -> dict[str, Any]: @@ -63,11 +60,33 @@ def test_each_state_has_a_parse_case_and_a_name_case(self) -> None: self.assertEqual(parsed, codes) self.assertLessEqual(codes, named) - def test_each_unused_iso_code_has_a_state_name_case_that_returns_nothing(self) -> None: - names = [(case["input"], case["expect"]["value"]) for case in load("state-name")["cases"]] - for iso_code in UNUSED_ISO_CODES: - with self.subTest(iso_code): - self.assertIn((iso_code, None), names) + def test_each_unused_iso_code_has_a_state_name_case_and_a_parse_case(self) -> None: + states = json.loads((ROOT / "data" / "states.json").read_text(encoding="utf-8"))["states"] + iso_codes = [ + state["iso"].removeprefix("NG-") + for state in states + if state["iso"] != "NG-" + state["code"] + ] + self.assertTrue(iso_codes) + unknown_state = { + "ok": False, + "error": {"code": "unknown_state", "segment": "state", "suggestion": None}, + } + nameless = { + case["input"] for case in load("state-name")["cases"] if case["expect"]["value"] is None + } + rejected = { + case["input"][:2] + for case in load("parse-segments")["cases"] + if case["expect"] == unknown_state and not case["options"] + } + missing: list[str] = [] + for iso_code in iso_codes: + if iso_code not in nameless: + missing.append(f"state-name: {iso_code}") + if iso_code not in rejected: + missing.append(f"parse-segments: {iso_code}") + self.assertEqual(missing, []) def test_each_separator_has_a_normalize_case(self) -> None: postcode_format = json.loads((ROOT / "data" / "format.json").read_text(encoding="utf-8")) @@ -101,34 +120,27 @@ def test_each_gps_limit_has_a_case_at_the_limit_and_a_case_above_it(self) -> Non above = [expected for metres, expected in measured if limit < metres < ceiling] self.assertIn(next_precision, above) - def test_the_input_limit_has_cases_on_both_sides_and_for_each_counting_mistake(self) -> None: + def test_parse_has_cases_at_the_limit_above_it_and_for_each_counting_mistake(self) -> None: postcode_format = json.loads((ROOT / "data" / "format.json").read_text(encoding="utf-8")) limit = postcode_format["maxInputCodePoints"] parsed = [ (case["input"], "ok" if case["expect"]["ok"] else case["expect"]["error"]["code"]) for case in load("parse")["cases"] ] - legacy = [(case["input"], case["expect"]["value"]) for case in load("is-legacy")["cases"]] - parse_outcomes = [(len(text), outcome) for text, outcome in parsed] - legacy_outcomes = [(len(text), outcome) for text, outcome in legacy] - self.assertIn((limit, "ok"), parse_outcomes) - self.assertIn((limit + 1, "bad_length"), parse_outcomes) - self.assertIn((limit + 1, False), legacy_outcomes) - self.assertIn((limit, True), legacy_outcomes) + outcomes = [(len(text), outcome) for text, outcome in parsed] + self.assertIn((limit, "ok"), outcomes) + self.assertIn((limit + 1, "bad_length"), outcomes) accepted = [text for text, outcome in parsed if outcome == "ok" and len(text) <= limit] rejected = [ text for text, outcome in parsed if outcome == "bad_length" and len(text) > limit ] - legacy_accepted = [ - text for text, outcome in legacy if outcome is True and len(text) <= limit - ] - legacy_rejected = [ - text for text, outcome in legacy if outcome is False and len(text) > limit - ] pins = { "a success with more UTF-16 units than the limit": [ text for text in accepted if utf16_units(text) > limit ], + "a success with the limit in code points and more UTF-16 units": [ + text for text in accepted if len(text) == limit and utf16_units(text) > limit + ], "a success that grows past the limit under NFKC": [ text for text in accepted if len(unicodedata.normalize("NFKC", text)) > limit ], @@ -140,25 +152,40 @@ def test_the_input_limit_has_cases_on_both_sides_and_for_each_counting_mistake(s "a rejection over the limit that ends in a line feed": [ text for text in rejected if text.endswith("\n") ], + } + for requirement, texts in pins.items(): + with self.subTest(requirement): + self.assertTrue(texts) + # A check that ends in $ instead of \z skips a final line feed. Only an input with + # exactly one code point over the limit shows it. + line_feed_lengths = {len(text) for text in rejected if text.endswith("\n")} + with self.subTest("each rejection that ends in a line feed is one code point over"): + self.assertEqual(line_feed_lengths, {limit + 1}) + + def test_is_legacy_has_cases_at_the_limit_above_it_and_for_each_counting_mistake(self) -> None: + postcode_format = json.loads((ROOT / "data" / "format.json").read_text(encoding="utf-8")) + limit = postcode_format["maxInputCodePoints"] + legacy = [(case["input"], case["expect"]["value"]) for case in load("is-legacy")["cases"]] + outcomes = [(len(text), outcome) for text, outcome in legacy] + self.assertIn((limit, True), outcomes) + self.assertIn((limit + 1, False), outcomes) + accepted = [text for text, outcome in legacy if outcome is True and len(text) <= limit] + rejected = [text for text, outcome in legacy if outcome is False and len(text) > limit] + pins = { "an accepted legacy code that grows past the limit under NFKC": [ - text for text in legacy_accepted if len(unicodedata.normalize("NFKC", text)) > limit + text for text in accepted if len(unicodedata.normalize("NFKC", text)) > limit ], "a rejected legacy code that is within the limit without its zero-width joiners": [ - text - for text in legacy_rejected - if len(text.replace(ZERO_WIDTH_JOINER, "")) <= limit + text for text in rejected if len(text.replace(ZERO_WIDTH_JOINER, "")) <= limit ], "a rejected legacy code over the limit that ends in a line feed": [ - text for text in legacy_rejected if text.endswith("\n") + text for text in rejected if text.endswith("\n") ], } for requirement, texts in pins.items(): with self.subTest(requirement): self.assertTrue(texts) - # A check that ends in $ instead of \z skips a final line feed. Only an input with - # exactly one code point over the limit shows it. - line_feed_lengths = { - len(text) for text in [*rejected, *legacy_rejected] if text.endswith("\n") - } + # The same one-over rule as for parse: a check that ends in $ lets such an input through. + line_feed_lengths = {len(text) for text in rejected if text.endswith("\n")} with self.subTest("each rejection that ends in a line feed is one code point over"): self.assertEqual(line_feed_lengths, {limit + 1}) diff --git a/scripts/tests/test_data.py b/scripts/tests/test_data.py index e294d83..7e6bc61 100644 --- a/scripts/tests/test_data.py +++ b/scripts/tests/test_data.py @@ -40,7 +40,9 @@ def test_uses_two_capital_letters_for_each_code(self) -> None: self.assertTrue(state["name"].isascii() and state["name"].strip()) def test_keeps_the_iso_code_of_each_state_beside_the_nipost_code(self) -> None: + seen: list[str] = [] for state in load("states.json")["states"]: + seen.append(state["name"]) with self.subTest(name=state["name"]): self.assertRegex(state["iso"], r"^NG-[A-Z]{2}$") if state["name"] in NIPOST_CODES_OFF_ISO: @@ -49,6 +51,7 @@ def test_keeps_the_iso_code_of_each_state_beside_the_nipost_code(self) -> None: ) else: self.assertEqual(state["iso"], "NG-" + state["code"]) + self.assertEqual(sorted(set(NIPOST_CODES_OFF_ISO) - set(seen)), []) class PrecisionTest(unittest.TestCase): diff --git a/standards/languages/typescript.md b/standards/languages/typescript.md index 3dfe70e..1dad8a9 100644 --- a/standards/languages/typescript.md +++ b/standards/languages/typescript.md @@ -31,7 +31,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for Type | publint and `@arethetypeswrong/cli` | package checks | every published package | | API Extractor | API report | `etc/.api.md`, checked in CI | | Changesets | versions and changelogs | one change file per user-visible change | -| commitlint | commit messages | `@commitlint/config-conventional`, scopes from package names, and `scope-empty` set to `never`, so each commit needs a scope (GIT-1) | +| commitlint | commit messages | `@commitlint/config-conventional`, `scope-enum` with the scopes `core`, `client`, `field`, `react`, `repo`, `deps` and `release`, `scope-empty` set to `never` so each commit needs a scope (GIT-1), and `header-max-length` set to 72 characters (TELL-18) | | REUSE | licence headers | `reuse lint` | | gitleaks | secret scan | each pull request and each push to `main` | | `spec/scripts/check-tells` | agent tells | line length 100, file names, TODO form, debug calls, emojis and non-ASCII | diff --git a/vectors/parse-segments.json b/vectors/parse-segments.json index 6f17a09..d932393 100644 --- a/vectors/parse-segments.json +++ b/vectors/parse-segments.json @@ -33,7 +33,7 @@ }, { "id": "parse-segments-003", - "description": "rejects the ISO code of a state that NIPOST does not use", + "description": "rejects BO, the ISO code of Borno, as a state", "input": "BO01Z99ZZ01", "options": {}, "expect": { @@ -47,6 +47,62 @@ }, { "id": "parse-segments-004", + "description": "rejects GO, the ISO code of Gombe, as a state", + "input": "GO01Z99ZZ01", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "unknown_state", + "segment": "state", + "suggestion": null + } + } + }, + { + "id": "parse-segments-005", + "description": "rejects KO, the ISO code of Kogi, as a state", + "input": "KO01Z99ZZ01", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "unknown_state", + "segment": "state", + "suggestion": null + } + } + }, + { + "id": "parse-segments-006", + "description": "rejects SO, the ISO code of Sokoto, as a state", + "input": "SO01Z99ZZ01", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "unknown_state", + "segment": "state", + "suggestion": null + } + } + }, + { + "id": "parse-segments-007", + "description": "rejects YO, the ISO code of Yobe, as a state", + "input": "YO01Z99ZZ01", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "unknown_state", + "segment": "state", + "suggestion": null + } + } + }, + { + "id": "parse-segments-008", "description": "suggests O for a zero in the state", "input": "0G01A03FK01", "options": {}, @@ -60,7 +116,7 @@ } }, { - "id": "parse-segments-005", + "id": "parse-segments-009", "description": "suggests I for a one in the state", "input": "1M01A03FK01", "options": {}, @@ -74,7 +130,7 @@ } }, { - "id": "parse-segments-006", + "id": "parse-segments-010", "description": "gives no suggestion when the fixed state is unknown", "input": "E101A03FK01", "options": {}, @@ -88,7 +144,7 @@ } }, { - "id": "parse-segments-007", + "id": "parse-segments-011", "description": "gives no suggestion when the state cannot be fixed", "input": "XXO1A03FK01", "options": {}, @@ -102,7 +158,7 @@ } }, { - "id": "parse-segments-008", + "id": "parse-segments-012", "description": "fixes later segments in a state suggestion", "input": "0GO1A03FK01", "options": {}, @@ -116,7 +172,7 @@ } }, { - "id": "parse-segments-009", + "id": "parse-segments-013", "description": "rejects an LGA of 00", "input": "EK00A03FK01", "options": {}, @@ -130,7 +186,7 @@ } }, { - "id": "parse-segments-010", + "id": "parse-segments-014", "description": "rejects a unit of 00", "input": "EK01A03FK00", "options": {}, @@ -144,7 +200,7 @@ } }, { - "id": "parse-segments-011", + "id": "parse-segments-015", "description": "suggests 0 for a letter O in the LGA", "input": "EKO1A03FK01", "options": {}, @@ -158,7 +214,7 @@ } }, { - "id": "parse-segments-012", + "id": "parse-segments-016", "description": "suggests 1 for a letter I in the LGA", "input": "EKI1A03FK01", "options": {}, @@ -172,7 +228,7 @@ } }, { - "id": "parse-segments-013", + "id": "parse-segments-017", "description": "suggests 1 for a letter L in the LGA", "input": "EKL1A03FK01", "options": {}, @@ -186,7 +242,7 @@ } }, { - "id": "parse-segments-014", + "id": "parse-segments-018", "description": "suggests I for a one in the area", "input": "EK01A03F101", "options": {}, @@ -200,7 +256,7 @@ } }, { - "id": "parse-segments-015", + "id": "parse-segments-019", "description": "suggests O for a zero in the area", "input": "EK01A03F001", "options": {}, @@ -214,7 +270,7 @@ } }, { - "id": "parse-segments-016", + "id": "parse-segments-020", "description": "suggests 0 for a letter O in the unit", "input": "EK01A03FKO1", "options": {}, @@ -228,7 +284,7 @@ } }, { - "id": "parse-segments-017", + "id": "parse-segments-021", "description": "suggests 1 for a letter I in the unit", "input": "EK01A03FKI2", "options": {}, @@ -242,7 +298,7 @@ } }, { - "id": "parse-segments-018", + "id": "parse-segments-022", "description": "suggests 1 for a letter L in the unit", "input": "EK01A03FKL1", "options": {}, @@ -256,7 +312,7 @@ } }, { - "id": "parse-segments-019", + "id": "parse-segments-023", "description": "suggests 1 for a letter I after a zero in the unit", "input": "EK01A03FK0I", "options": {}, @@ -270,7 +326,7 @@ } }, { - "id": "parse-segments-020", + "id": "parse-segments-024", "description": "gives no suggestion when the fix makes a unit of 00", "input": "EK01A03FK0O", "options": {}, @@ -284,7 +340,7 @@ } }, { - "id": "parse-segments-021", + "id": "parse-segments-025", "description": "gives no suggestion when a segment cannot be fixed", "input": "EK01A03F2O1", "options": {}, @@ -298,7 +354,7 @@ } }, { - "id": "parse-segments-022", + "id": "parse-segments-026", "description": "reports the first bad segment and fixes them all", "input": "EKO1A03F1O1", "options": {}, @@ -312,7 +368,7 @@ } }, { - "id": "parse-segments-023", + "id": "parse-segments-027", "description": "suggests a fix after making letters upper case", "input": "eko1a03fk01", "options": {}, @@ -326,7 +382,7 @@ } }, { - "id": "parse-segments-024", + "id": "parse-segments-028", "description": "rejects an LGA of 00 in a partial code", "input": "EK00", "options": { @@ -342,7 +398,7 @@ } }, { - "id": "parse-segments-025", + "id": "parse-segments-029", "description": "suggests a partial code", "input": "EKO1", "options": { @@ -358,7 +414,7 @@ } }, { - "id": "parse-segments-026", + "id": "parse-segments-030", "description": "suggests a partial area code", "input": "EK01A03F1", "options": { @@ -374,7 +430,7 @@ } }, { - "id": "parse-segments-027", + "id": "parse-segments-031", "description": "rejects an unknown state when partial codes are allowed", "input": "XX", "options": { @@ -390,7 +446,7 @@ } }, { - "id": "parse-segments-028", + "id": "parse-segments-032", "description": "suggests a state for a partial code", "input": "0G", "options": { @@ -406,7 +462,7 @@ } }, { - "id": "parse-segments-029", + "id": "parse-segments-033", "description": "suggests an LGA fix for a district code", "input": "EKO1A03", "options": { @@ -422,7 +478,7 @@ } }, { - "id": "parse-segments-030", + "id": "parse-segments-034", "description": "rejects an LGA of 00 in a district code", "input": "EK00A03", "options": { @@ -438,7 +494,7 @@ } }, { - "id": "parse-segments-031", + "id": "parse-segments-035", "description": "keeps a letter O in the district when it fixes the LGA", "input": "EKO1AO3FK01", "options": {}, @@ -452,7 +508,7 @@ } }, { - "id": "parse-segments-032", + "id": "parse-segments-036", "description": "keeps letters I and L in the district when it fixes the LGA", "input": "EKO1AILFK01", "options": {}, @@ -466,7 +522,7 @@ } }, { - "id": "parse-segments-033", + "id": "parse-segments-037", "description": "accepts letters and digits in any district position", "input": "EK01A0OFK01", "options": {}, From 0e60119fcce310f396ef9eaf9c2e46e79d1f910e Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 08:10:38 +0100 Subject: [PATCH 25/45] docs: say how SDKs treat broken text and NFKC tables PHP strings hold bytes, and PHP has no NFKC without an extension. The grammar now says what parse, isLegacy and normalize do with text that is not well-formed Unicode, and when an SDK may apply NFKC through a table. A test keeps every normalize vector valid for such an SDK. GIT-1 now names the scopes that commitlint allows. Signed-off-by: MobileMage --- grammar.md | 4 ++++ scripts/tests/test_build_vectors.py | 31 +++++++++++++++++++++++++++++ standards/CODING_STANDARDS.md | 2 +- 3 files changed, 36 insertions(+), 1 deletion(-) diff --git a/grammar.md b/grammar.md index 72cfd23..2f2f0e2 100644 --- a/grammar.md +++ b/grammar.md @@ -28,6 +28,8 @@ A partial postcode stops after the state, LGA, district or area segment. Its len 2. Remove each separator in `data/format.json`: white space, hyphens and dashes, the full stop, and zero-width characters. 3. Change the ASCII letters a to z to upper case. Keep every other character as it is. +An SDK without a full NFKC can apply NFKC only to the characters whose NFKC form holds only ASCII characters and separators. Its `parse` and `isLegacy` must then give the same results as with full NFKC, for every input. Its `normalize` can keep a character that full NFKC changes, such as U+00B5, in text that holds no postcode. Each `normalize` vector gives the same result either way. + A format character that is not in the list stays. For example, the right-to-left override U+202E can change the order in which a code shows on screen, so `parse` gives `bad_character` for it. `normalize` has no length limit. Bound untrusted text before you call it, or call `parse`, which checks the limit first. @@ -36,6 +38,8 @@ A format character that is not in the list stays. For example, the right-to-left First count the Unicode code points of the input. Do not count UTF-16 units, UTF-8 bytes or grapheme clusters. A lone surrogate, in a language that can hold one, counts as one code point. An SDK can stop counting at 65. If the input has more than `maxInputCodePoints` code points (64 in `data/format.json`), the error code is `bad_length`, and `parse` does not normalise the input. The limit bounds the cost of NFKC, which can take seconds on a long run of combining marks. Otherwise, normalise the input. Then apply these checks in order. The first check that fails gives the error code. +Input that is not well-formed Unicode is never a postcode. This covers bytes that are not valid UTF-8, in a language whose strings hold bytes, and a lone surrogate, in a language whose strings can hold one. Such input gives `bad_length` when it is over the input limit, and `bad_character` otherwise. An SDK whose strings hold bytes can treat input of more than 4 x `maxInputCodePoints` bytes as over the limit without decoding it, because a code point takes at most 4 bytes in UTF-8. `isLegacy` gives false for such input. `normalize` keeps each unit that is not well-formed. No function throws for such input. + 1. `empty`: no character is left. 2. `legacy_code`: exactly 6 ASCII digits are left. These are old NIPOST postcodes. 3. `bad_character`: a character other than A to Z and 0 to 9 is left. diff --git a/scripts/tests/test_build_vectors.py b/scripts/tests/test_build_vectors.py index be0049a..32d9c20 100644 --- a/scripts/tests/test_build_vectors.py +++ b/scripts/tests/test_build_vectors.py @@ -4,6 +4,7 @@ import json import math +import string import unicodedata import unittest from collections import Counter @@ -14,6 +15,7 @@ ROOT = Path(__file__).resolve().parents[2] ZERO_WIDTH_JOINER = chr(0x200D) +ASCII_UPPER_CASE = str.maketrans(string.ascii_lowercase, string.ascii_uppercase) def load(stem: str) -> dict[str, Any]: @@ -27,6 +29,25 @@ def utf16_units(text: str) -> int: return len(text.encode("utf-16-le", errors="surrogatepass")) // 2 +def table_form(character: str, separators: frozenset[str]) -> str: + """Return the NFKC form of a character if the form holds only ASCII characters and separators. + + An SDK that applies NFKC through a table has entries for such characters only. It keeps + every other character as it is. + """ + form = unicodedata.normalize("NFKC", character) + if all(part.isascii() or part in separators for part in form): + return form + return character + + +def reduced_normalize(text: str, separators: frozenset[str]) -> str: + """Normalise as an SDK without a full NFKC does (grammar.md, section Normalise).""" + mapped = "".join(table_form(character, separators) for character in text) + kept = "".join(character for character in mapped if character not in separators) + return kept.translate(ASCII_UPPER_CASE) + + class BuildVectorsTest(unittest.TestCase): def test_vector_files_match_the_builder(self) -> None: self.assertEqual(build_vectors.main(["--check"]), 0) @@ -101,6 +122,16 @@ def test_each_separator_has_a_normalize_case(self) -> None: missing.append(label) self.assertEqual(missing, []) + def test_each_normalize_case_gives_the_same_result_with_nfkc_through_a_table(self) -> None: + separators = frozenset(chr(code_point) for code_point in build_vectors.load_separators()) + for case in load("normalize")["cases"]: + with self.subTest(case["id"]): + # ASCII forms keep a failure readable: U+00B5 and U+03BC look alike. + self.assertEqual( + ascii(reduced_normalize(case["input"], separators)), + ascii(case["expect"]["value"]), + ) + def test_each_gps_limit_has_a_case_at_the_limit_and_a_case_above_it(self) -> None: rules = json.loads((ROOT / "data" / "precision.json").read_text(encoding="utf-8")) limits = [threshold["maxAccuracyM"] for threshold in rules["thresholds"]] diff --git a/standards/CODING_STANDARDS.md b/standards/CODING_STANDARDS.md index 23f2a47..fa0b58a 100644 --- a/standards/CODING_STANDARDS.md +++ b/standards/CODING_STANDARDS.md @@ -168,7 +168,7 @@ A line in a code or config file can skip one check with a comment that names the ## Git, pull requests and releases -- **GIT-1 MUST.** Commits follow Conventional Commits. The scope is the package name. **(tool)** +- **GIT-1 MUST.** Commits follow Conventional Commits. The scope names the package, or is repo, deps or release for a change outside a package. **(tool)** - **GIT-2 MUST.** Each commit has a DCO sign-off: `git commit -s`. **(tool)** - **GIT-3 MUST.** Commits do not carry co-author lines for AI tools. The person who opens the pull request owns the change. - **GIT-4 MUST.** Nobody force-pushes to `main` or to a shared branch. From aa1b4c260e1bd7344b3a844b6041f6c6233e72a3 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 08:14:36 +0100 Subject: [PATCH 26/45] docs: put the broken-text rule after the parse checks The new paragraph sat between the lead-in of the parse checks and their list, and it named error codes before the list defined them. It now follows the list. GIT-1 also says that the spec repo, which holds no package, uses no scope. Signed-off-by: MobileMage --- grammar.md | 4 ++-- standards/CODING_STANDARDS.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/grammar.md b/grammar.md index 2f2f0e2..42df785 100644 --- a/grammar.md +++ b/grammar.md @@ -38,8 +38,6 @@ A format character that is not in the list stays. For example, the right-to-left First count the Unicode code points of the input. Do not count UTF-16 units, UTF-8 bytes or grapheme clusters. A lone surrogate, in a language that can hold one, counts as one code point. An SDK can stop counting at 65. If the input has more than `maxInputCodePoints` code points (64 in `data/format.json`), the error code is `bad_length`, and `parse` does not normalise the input. The limit bounds the cost of NFKC, which can take seconds on a long run of combining marks. Otherwise, normalise the input. Then apply these checks in order. The first check that fails gives the error code. -Input that is not well-formed Unicode is never a postcode. This covers bytes that are not valid UTF-8, in a language whose strings hold bytes, and a lone surrogate, in a language whose strings can hold one. Such input gives `bad_length` when it is over the input limit, and `bad_character` otherwise. An SDK whose strings hold bytes can treat input of more than 4 x `maxInputCodePoints` bytes as over the limit without decoding it, because a code point takes at most 4 bytes in UTF-8. `isLegacy` gives false for such input. `normalize` keeps each unit that is not well-formed. No function throws for such input. - 1. `empty`: no character is left. 2. `legacy_code`: exactly 6 ASCII digits are left. These are old NIPOST postcodes. 3. `bad_character`: a character other than A to Z and 0 to 9 is left. @@ -49,6 +47,8 @@ Input that is not well-formed Unicode is never a postcode. This covers bytes tha For `unknown_state` and `bad_segment`, the error names the segment. For the other codes, the segment is null. +Input that is not well-formed Unicode is never a postcode. This covers bytes that are not valid UTF-8, in a language whose strings hold bytes, and a lone surrogate, in a language whose strings can hold one. Such input gives `bad_length` when it is over the input limit, and `bad_character` otherwise. An SDK whose strings hold bytes can treat input of more than 4 x `maxInputCodePoints` bytes as over the limit without decoding it, because a code point takes at most 4 bytes in UTF-8. `isLegacy` gives false for such input. `normalize` keeps each unit that is not well-formed. No function throws for such input. + ## Suggestions For `unknown_state` and `bad_segment`, build one corrected code: diff --git a/standards/CODING_STANDARDS.md b/standards/CODING_STANDARDS.md index fa0b58a..a55be5b 100644 --- a/standards/CODING_STANDARDS.md +++ b/standards/CODING_STANDARDS.md @@ -168,7 +168,7 @@ A line in a code or config file can skip one check with a comment that names the ## Git, pull requests and releases -- **GIT-1 MUST.** Commits follow Conventional Commits. The scope names the package, or is repo, deps or release for a change outside a package. **(tool)** +- **GIT-1 MUST.** Commits follow Conventional Commits. The scope names the package, or is repo, deps or release for a change outside a package. The spec repo holds no package, so its commits use no scope. **(tool)** - **GIT-2 MUST.** Each commit has a DCO sign-off: `git commit -s`. **(tool)** - **GIT-3 MUST.** Commits do not carry co-author lines for AI tools. The person who opens the pull request owns the change. - **GIT-4 MUST.** Nobody force-pushes to `main` or to a shared branch. From a66d39dc80e458e7bc131aa70c0929c3775095e0 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 08:51:34 +0100 Subject: [PATCH 27/45] docs: name the Unicode baseline and count broken text exactly NFKC changes with the Unicode version, so a table SDK and a full NFKC on another version could disagree. The grammar now names Unicode 17.0 as the baseline. It also says how broken bytes and lone surrogates meet the input limit, so a Go and a PHP SDK give the same codes. A premise test and a new vector guard the separators. Signed-off-by: MobileMage --- grammar.md | 6 +- scripts/build_vectors.py | 3 + scripts/tests/test_build_vectors.py | 67 ++++++++++++++--- standards/CODING_STANDARDS.md | 2 +- vectors/normalize.json | 109 +++++++++++++++------------- 5 files changed, 123 insertions(+), 64 deletions(-) diff --git a/grammar.md b/grammar.md index 42df785..06e66e1 100644 --- a/grammar.md +++ b/grammar.md @@ -28,7 +28,9 @@ A partial postcode stops after the state, LGA, district or area segment. Its len 2. Remove each separator in `data/format.json`: white space, hyphens and dashes, the full stop, and zero-width characters. 3. Change the ASCII letters a to z to upper case. Keep every other character as it is. -An SDK without a full NFKC can apply NFKC only to the characters whose NFKC form holds only ASCII characters and separators. Its `parse` and `isLegacy` must then give the same results as with full NFKC, for every input. Its `normalize` can keep a character that full NFKC changes, such as U+00B5, in text that holds no postcode. Each `normalize` vector gives the same result either way. +An SDK without a full NFKC can apply NFKC through a table. The table lists only the characters whose NFKC form holds only ASCII characters and separators. The SDK's `parse` and `isLegacy` must then give the same results as with full NFKC, for every input. The SDK's `normalize` can keep a character that full NFKC changes, such as U+00B5, in text that `parse` rejects. Each `normalize` vector gives the same result either way. + +Unicode 17.0 is the baseline for NFKC. A table must come from that version. A platform can have a Unicode version older than 17.0. An SDK on that platform can give other results for a character that a later version added or changed. For example, NFKC in Unicode 17.0 changes U+1CCDA to E. No vector holds such a character. A format character that is not in the list stays. For example, the right-to-left override U+202E can change the order in which a code shows on screen, so `parse` gives `bad_character` for it. @@ -47,7 +49,7 @@ First count the Unicode code points of the input. Do not count UTF-16 units, UTF For `unknown_state` and `bad_segment`, the error names the segment. For the other codes, the segment is null. -Input that is not well-formed Unicode is never a postcode. This covers bytes that are not valid UTF-8, in a language whose strings hold bytes, and a lone surrogate, in a language whose strings can hold one. Such input gives `bad_length` when it is over the input limit, and `bad_character` otherwise. An SDK whose strings hold bytes can treat input of more than 4 x `maxInputCodePoints` bytes as over the limit without decoding it, because a code point takes at most 4 bytes in UTF-8. `isLegacy` gives false for such input. `normalize` keeps each unit that is not well-formed. No function throws for such input. +Input that is not well-formed Unicode is never a postcode. In a language whose strings hold bytes, the SDK does not count the code points of input that is not valid UTF-8. `parse` gives `bad_length` for such input when it has more than 4 x `maxInputCodePoints` bytes, and `bad_character` otherwise. In a language whose strings can hold a lone surrogate, the surrogate counts as one code point. The input limit applies as usual. Within the limit, `parse` gives `bad_character` for input that holds a lone surrogate. `isLegacy` gives false for input that is not well-formed. `normalize` keeps each byte that is not part of a valid UTF-8 sequence, and each lone surrogate, in place. It normalises the text on each side of such a byte or surrogate on its own. No function throws for input that is not well-formed. ## Suggestions diff --git a/scripts/build_vectors.py b/scripts/build_vectors.py index 77eee08..82db250 100755 --- a/scripts/build_vectors.py +++ b/scripts/build_vectors.py @@ -38,6 +38,7 @@ RIGHT_TO_LEFT_MARK = chr(0x200F) RIGHT_TO_LEFT_OVERRIDE = chr(0x202E) FULL_WIDTH_HYPHEN = chr(0xFF0D) +SUPERSCRIPT_MINUS = chr(0x207B) COMBINING_ACUTE = chr(0x0301) HORIZONTAL_ELLIPSIS = chr(0x2026) ZERO_WIDTH_JOINER = chr(0x200D) @@ -182,6 +183,8 @@ def normalize_rows() -> list[Row]: want, ), row("removes full-width hyphens", joined(FULL_WIDTH_HYPHEN), want), + # NFKC changes U+207B to the separator U+2212. A table of ASCII forms only would keep it. + row("removes a superscript minus", "EK" + SUPERSCRIPT_MINUS + "01A03FK01", want), row("removes tabs and line feeds", "EK01" + TAB + "A03" + LINE_FEED + "FK01", want), row( "changes a ligature to its letters", diff --git a/scripts/tests/test_build_vectors.py b/scripts/tests/test_build_vectors.py index 32d9c20..51a3b4c 100644 --- a/scripts/tests/test_build_vectors.py +++ b/scripts/tests/test_build_vectors.py @@ -29,6 +29,14 @@ def utf16_units(text: str) -> int: return len(text.encode("utf-16-le", errors="surrogatepass")) // 2 +def separator_characters() -> frozenset[str]: + return frozenset(chr(code_point) for code_point in build_vectors.load_separators()) + + +def holds_only_ascii_and_separators(text: str, separators: frozenset[str]) -> bool: + return all(character.isascii() or character in separators for character in text) + + def table_form(character: str, separators: frozenset[str]) -> str: """Return the NFKC form of a character if the form holds only ASCII characters and separators. @@ -36,19 +44,33 @@ def table_form(character: str, separators: frozenset[str]) -> str: every other character as it is. """ form = unicodedata.normalize("NFKC", character) - if all(part.isascii() or part in separators for part in form): + if holds_only_ascii_and_separators(form, separators): return form return character +def remove_separators_and_upper_case(text: str, separators: frozenset[str]) -> str: + """Apply steps 2 and 3 of the Normalise section in grammar.md.""" + kept = "".join(character for character in text if character not in separators) + return kept.translate(ASCII_UPPER_CASE) + + +def full_normalize(text: str, separators: frozenset[str]) -> str: + """Normalise as grammar.md says, with a full NFKC.""" + return remove_separators_and_upper_case(unicodedata.normalize("NFKC", text), separators) + + def reduced_normalize(text: str, separators: frozenset[str]) -> str: """Normalise as an SDK without a full NFKC does (grammar.md, section Normalise).""" mapped = "".join(table_form(character, separators) for character in text) - kept = "".join(character for character in mapped if character not in separators) - return kept.translate(ASCII_UPPER_CASE) + return remove_separators_and_upper_case(mapped, separators) class BuildVectorsTest(unittest.TestCase): + def assert_same_text(self, actual: str, expected: str) -> None: + # ASCII forms keep a failure readable: U+00B5 and U+03BC look alike. + self.assertEqual(ascii(actual), ascii(expected)) + def test_vector_files_match_the_builder(self) -> None: self.assertEqual(build_vectors.main(["--check"]), 0) @@ -122,15 +144,38 @@ def test_each_separator_has_a_normalize_case(self) -> None: missing.append(label) self.assertEqual(missing, []) - def test_each_normalize_case_gives_the_same_result_with_nfkc_through_a_table(self) -> None: - separators = frozenset(chr(code_point) for code_point in build_vectors.load_separators()) + def test_each_separator_has_an_nfkc_form_of_ascii_characters_and_separators(self) -> None: + separators = separator_characters() + # Otherwise full NFKC leaves a character where a table removes the separator. + outside = [ + f"U+{ord(separator):04X}" + for separator in sorted(separators) + if not holds_only_ascii_and_separators( + unicodedata.normalize("NFKC", separator), separators + ) + ] + self.assertEqual(outside, []) + + def test_each_normalize_case_holds_with_full_nfkc_and_with_a_table(self) -> None: + separators = separator_characters() + normalizers = {"full NFKC": full_normalize, "NFKC through a table": reduced_normalize} for case in load("normalize")["cases"]: - with self.subTest(case["id"]): - # ASCII forms keep a failure readable: U+00B5 and U+03BC look alike. - self.assertEqual( - ascii(reduced_normalize(case["input"], separators)), - ascii(case["expect"]["value"]), - ) + for name, normalizer in normalizers.items(): + with self.subTest(case["id"], normalizer=name): + self.assert_same_text( + normalizer(case["input"], separators), case["expect"]["value"] + ) + + def test_a_table_keeps_a_micro_sign_but_changes_a_full_width_letter(self) -> None: + separators = separator_characters() + micro_sign = chr(0x00B5) + full_width_a = chr(0xFF21) + with self.subTest("full NFKC changes the micro sign"): + self.assert_same_text(full_normalize(micro_sign, separators), chr(0x03BC)) + with self.subTest("a table keeps the micro sign"): + self.assert_same_text(reduced_normalize(micro_sign, separators), micro_sign) + with self.subTest("a table changes the full-width letter"): + self.assert_same_text(reduced_normalize(full_width_a, separators), "A") def test_each_gps_limit_has_a_case_at_the_limit_and_a_case_above_it(self) -> None: rules = json.loads((ROOT / "data" / "precision.json").read_text(encoding="utf-8")) diff --git a/standards/CODING_STANDARDS.md b/standards/CODING_STANDARDS.md index a55be5b..6b3e69e 100644 --- a/standards/CODING_STANDARDS.md +++ b/standards/CODING_STANDARDS.md @@ -168,7 +168,7 @@ A line in a code or config file can skip one check with a comment that names the ## Git, pull requests and releases -- **GIT-1 MUST.** Commits follow Conventional Commits. The scope names the package, or is repo, deps or release for a change outside a package. The spec repo holds no package, so its commits use no scope. **(tool)** +- **GIT-1 MUST.** Commits follow Conventional Commits. The scope names the package, or is repo, deps or release for a change outside a package. The spec repo has no packages. Its commits use no scope. **(tool)** - **GIT-2 MUST.** Each commit has a DCO sign-off: `git commit -s`. **(tool)** - **GIT-3 MUST.** Commits do not carry co-author lines for AI tools. The person who opens the pull request owns the change. - **GIT-4 MUST.** Nobody force-pushes to `main` or to a shared branch. diff --git a/vectors/normalize.json b/vectors/normalize.json index c907860..da1042c 100644 --- a/vectors/normalize.json +++ b/vectors/normalize.json @@ -158,6 +158,15 @@ }, { "id": "normalize-018", + "description": "removes a superscript minus", + "input": "EK\u207b01A03FK01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-019", "description": "removes tabs and line feeds", "input": "EK01\tA03\nFK01", "options": {}, @@ -166,7 +175,7 @@ } }, { - "id": "normalize-019", + "id": "normalize-020", "description": "changes a ligature to its letters", "input": "EK01A03\ufb0101", "options": {}, @@ -175,7 +184,7 @@ } }, { - "id": "normalize-020", + "id": "normalize-021", "description": "makes mixed-case letters upper case", "input": "eK01a03Fk01", "options": {}, @@ -184,7 +193,7 @@ } }, { - "id": "normalize-021", + "id": "normalize-022", "description": "keeps a legacy code", "input": "900 108", "options": {}, @@ -193,7 +202,7 @@ } }, { - "id": "normalize-022", + "id": "normalize-023", "description": "keeps characters that are not separators", "input": "EK01A03FK0!", "options": {}, @@ -202,7 +211,7 @@ } }, { - "id": "normalize-023", + "id": "normalize-024", "description": "makes only ASCII letters upper case", "input": "ek01a03f\u00e901", "options": {}, @@ -211,7 +220,7 @@ } }, { - "id": "normalize-024", + "id": "normalize-025", "description": "keeps a dotless i", "input": "EK01A03F\u013101", "options": {}, @@ -220,7 +229,7 @@ } }, { - "id": "normalize-025", + "id": "normalize-026", "description": "returns an empty string for empty input", "input": "", "options": {}, @@ -229,7 +238,7 @@ } }, { - "id": "normalize-026", + "id": "normalize-027", "description": "returns an empty string for separators only", "input": " - . ", "options": {}, @@ -238,7 +247,7 @@ } }, { - "id": "normalize-027", + "id": "normalize-028", "description": "removes the separator U+0009", "input": "EK\t01\tA03\tFK\t01", "options": {}, @@ -247,7 +256,7 @@ } }, { - "id": "normalize-028", + "id": "normalize-029", "description": "removes the separator U+000A", "input": "EK\n01\nA03\nFK\n01", "options": {}, @@ -256,7 +265,7 @@ } }, { - "id": "normalize-029", + "id": "normalize-030", "description": "removes the separator U+000B", "input": "EK\u000b01\u000bA03\u000bFK\u000b01", "options": {}, @@ -265,7 +274,7 @@ } }, { - "id": "normalize-030", + "id": "normalize-031", "description": "removes the separator U+000C", "input": "EK\f01\fA03\fFK\f01", "options": {}, @@ -274,7 +283,7 @@ } }, { - "id": "normalize-031", + "id": "normalize-032", "description": "removes the separator U+000D", "input": "EK\r01\rA03\rFK\r01", "options": {}, @@ -283,7 +292,7 @@ } }, { - "id": "normalize-032", + "id": "normalize-033", "description": "removes the separator U+0020", "input": "EK 01 A03 FK 01", "options": {}, @@ -292,7 +301,7 @@ } }, { - "id": "normalize-033", + "id": "normalize-034", "description": "removes the separator U+002D", "input": "EK-01-A03-FK-01", "options": {}, @@ -301,7 +310,7 @@ } }, { - "id": "normalize-034", + "id": "normalize-035", "description": "removes the separator U+002E", "input": "EK.01.A03.FK.01", "options": {}, @@ -310,7 +319,7 @@ } }, { - "id": "normalize-035", + "id": "normalize-036", "description": "removes the separator U+0085", "input": "EK\u008501\u0085A03\u0085FK\u008501", "options": {}, @@ -319,7 +328,7 @@ } }, { - "id": "normalize-036", + "id": "normalize-037", "description": "removes the separator U+00A0", "input": "EK\u00a001\u00a0A03\u00a0FK\u00a001", "options": {}, @@ -328,7 +337,7 @@ } }, { - "id": "normalize-037", + "id": "normalize-038", "description": "removes the separator U+00AD", "input": "EK\u00ad01\u00adA03\u00adFK\u00ad01", "options": {}, @@ -337,7 +346,7 @@ } }, { - "id": "normalize-038", + "id": "normalize-039", "description": "removes the separator U+1680", "input": "EK\u168001\u1680A03\u1680FK\u168001", "options": {}, @@ -346,7 +355,7 @@ } }, { - "id": "normalize-039", + "id": "normalize-040", "description": "removes the separator U+2000", "input": "EK\u200001\u2000A03\u2000FK\u200001", "options": {}, @@ -355,7 +364,7 @@ } }, { - "id": "normalize-040", + "id": "normalize-041", "description": "removes the separator U+2001", "input": "EK\u200101\u2001A03\u2001FK\u200101", "options": {}, @@ -364,7 +373,7 @@ } }, { - "id": "normalize-041", + "id": "normalize-042", "description": "removes the separator U+2002", "input": "EK\u200201\u2002A03\u2002FK\u200201", "options": {}, @@ -373,7 +382,7 @@ } }, { - "id": "normalize-042", + "id": "normalize-043", "description": "removes the separator U+2003", "input": "EK\u200301\u2003A03\u2003FK\u200301", "options": {}, @@ -382,7 +391,7 @@ } }, { - "id": "normalize-043", + "id": "normalize-044", "description": "removes the separator U+2004", "input": "EK\u200401\u2004A03\u2004FK\u200401", "options": {}, @@ -391,7 +400,7 @@ } }, { - "id": "normalize-044", + "id": "normalize-045", "description": "removes the separator U+2005", "input": "EK\u200501\u2005A03\u2005FK\u200501", "options": {}, @@ -400,7 +409,7 @@ } }, { - "id": "normalize-045", + "id": "normalize-046", "description": "removes the separator U+2006", "input": "EK\u200601\u2006A03\u2006FK\u200601", "options": {}, @@ -409,7 +418,7 @@ } }, { - "id": "normalize-046", + "id": "normalize-047", "description": "removes the separator U+2007", "input": "EK\u200701\u2007A03\u2007FK\u200701", "options": {}, @@ -418,7 +427,7 @@ } }, { - "id": "normalize-047", + "id": "normalize-048", "description": "removes the separator U+2008", "input": "EK\u200801\u2008A03\u2008FK\u200801", "options": {}, @@ -427,7 +436,7 @@ } }, { - "id": "normalize-048", + "id": "normalize-049", "description": "removes the separator U+2009", "input": "EK\u200901\u2009A03\u2009FK\u200901", "options": {}, @@ -436,7 +445,7 @@ } }, { - "id": "normalize-049", + "id": "normalize-050", "description": "removes the separator U+200A", "input": "EK\u200a01\u200aA03\u200aFK\u200a01", "options": {}, @@ -445,7 +454,7 @@ } }, { - "id": "normalize-050", + "id": "normalize-051", "description": "removes the separator U+200B", "input": "EK\u200b01\u200bA03\u200bFK\u200b01", "options": {}, @@ -454,7 +463,7 @@ } }, { - "id": "normalize-051", + "id": "normalize-052", "description": "removes the separator U+200C", "input": "EK\u200c01\u200cA03\u200cFK\u200c01", "options": {}, @@ -463,7 +472,7 @@ } }, { - "id": "normalize-052", + "id": "normalize-053", "description": "removes the separator U+200D", "input": "EK\u200d01\u200dA03\u200dFK\u200d01", "options": {}, @@ -472,7 +481,7 @@ } }, { - "id": "normalize-053", + "id": "normalize-054", "description": "removes the separator U+200E", "input": "EK\u200e01\u200eA03\u200eFK\u200e01", "options": {}, @@ -481,7 +490,7 @@ } }, { - "id": "normalize-054", + "id": "normalize-055", "description": "removes the separator U+200F", "input": "EK\u200f01\u200fA03\u200fFK\u200f01", "options": {}, @@ -490,7 +499,7 @@ } }, { - "id": "normalize-055", + "id": "normalize-056", "description": "removes the separator U+2010", "input": "EK\u201001\u2010A03\u2010FK\u201001", "options": {}, @@ -499,7 +508,7 @@ } }, { - "id": "normalize-056", + "id": "normalize-057", "description": "removes the separator U+2011", "input": "EK\u201101\u2011A03\u2011FK\u201101", "options": {}, @@ -508,7 +517,7 @@ } }, { - "id": "normalize-057", + "id": "normalize-058", "description": "removes the separator U+2012", "input": "EK\u201201\u2012A03\u2012FK\u201201", "options": {}, @@ -517,7 +526,7 @@ } }, { - "id": "normalize-058", + "id": "normalize-059", "description": "removes the separator U+2013", "input": "EK\u201301\u2013A03\u2013FK\u201301", "options": {}, @@ -526,7 +535,7 @@ } }, { - "id": "normalize-059", + "id": "normalize-060", "description": "removes the separator U+2014", "input": "EK\u201401\u2014A03\u2014FK\u201401", "options": {}, @@ -535,7 +544,7 @@ } }, { - "id": "normalize-060", + "id": "normalize-061", "description": "removes the separator U+2015", "input": "EK\u201501\u2015A03\u2015FK\u201501", "options": {}, @@ -544,7 +553,7 @@ } }, { - "id": "normalize-061", + "id": "normalize-062", "description": "removes the separator U+2028", "input": "EK\u202801\u2028A03\u2028FK\u202801", "options": {}, @@ -553,7 +562,7 @@ } }, { - "id": "normalize-062", + "id": "normalize-063", "description": "removes the separator U+2029", "input": "EK\u202901\u2029A03\u2029FK\u202901", "options": {}, @@ -562,7 +571,7 @@ } }, { - "id": "normalize-063", + "id": "normalize-064", "description": "removes the separator U+202F", "input": "EK\u202f01\u202fA03\u202fFK\u202f01", "options": {}, @@ -571,7 +580,7 @@ } }, { - "id": "normalize-064", + "id": "normalize-065", "description": "removes the separator U+205F", "input": "EK\u205f01\u205fA03\u205fFK\u205f01", "options": {}, @@ -580,7 +589,7 @@ } }, { - "id": "normalize-065", + "id": "normalize-066", "description": "removes the separator U+2060", "input": "EK\u206001\u2060A03\u2060FK\u206001", "options": {}, @@ -589,7 +598,7 @@ } }, { - "id": "normalize-066", + "id": "normalize-067", "description": "removes the separator U+2212", "input": "EK\u221201\u2212A03\u2212FK\u221201", "options": {}, @@ -598,7 +607,7 @@ } }, { - "id": "normalize-067", + "id": "normalize-068", "description": "removes the separator U+3000", "input": "EK\u300001\u3000A03\u3000FK\u300001", "options": {}, @@ -607,7 +616,7 @@ } }, { - "id": "normalize-068", + "id": "normalize-069", "description": "removes the separator U+FEFF", "input": "EK\ufeff01\ufeffA03\ufeffFK\ufeff01", "options": {}, From c8b85f8b42de8dbf9c5ec682a2b95cf6fe1b1a00 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 12:30:17 +0100 Subject: [PATCH 28/45] docs: drop a NIPOST claim with no source and fix the Spec entry No public NIPOST page says that NIPOST uses districts for postal sorting. The pages on postcode.gov.ng describe a district only as a district within the LGA. The glossary now states only what those pages state. The Spec entry named the fixtures and the OpenAPI file as contents of this repo, but the repo holds neither yet. The entry now lists what the repo holds today. It says that the API fixtures and the completed OpenAPI file come later. Signed-off-by: MobileMage --- CONTEXT.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index 840b4b9..a08e6e1 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -27,7 +27,7 @@ The second segment. Two digits that name a local government area within a state. _Avoid_: council, local government, municipality **District**: -The third segment. Three characters that name a district within an LGA. NIPOST uses districts for postal sorting. +The third segment. Three characters that name a district within an LGA. _Avoid_: sorting code, sector **Area**: @@ -114,7 +114,7 @@ _Avoid_: using this word for a Gatepost component ### Gatepost **Spec**: -The `spec` repo, which holds the grammar, the vectors, the fixtures, the completed OpenAPI file and these standards. +The `spec` repo. It holds the grammar, the data, the vectors and the standards. The API fixtures and the completed OpenAPI file come later. _Avoid_: schema, contract (for the repo) **Vector**: From a3925ed2a2b229702eeaf3a5124efa0ccc443e26 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 12:34:30 +0100 Subject: [PATCH 29/45] test: run every vector through a plain reading of the grammar The vectors are the contract for every SDK, but nothing in this repo tied them, or the data files, to the grammar. The builder holds its expected results as typed literals, and no test read the segment rules or the suggestion tables. A wrong suggestion in the builder, or a new LGA minimum or legacy pattern in data/, passed make check. scripts/reference.py now reads grammar.md with every value taken from data/, and a test runs each case of each vector file through it. A new vector file fails until the reference has a reader for its function. A second test changes each data value in turn and requires the reference to notice, so a value cannot hide inside the reference. grammar.md repeats some values of data/format.json so that a reader needs no other file. Tests now compare the segment table, the lengths, the input limit, the legacy rule and the fixes with the data. The grammar also says that it wins when a vector disagrees with it. The legacy pattern gets negative cases. The --check mode of the builder gets tests for a changed file, a stray file and a missing file. Signed-off-by: MobileMage --- grammar.md | 12 +- scripts/reference.py | 325 ++++++++++++++++++++++++++++ scripts/tests/test_build_vectors.py | 78 +++++++ scripts/tests/test_data.py | 118 +++++++++- scripts/tests/test_reference.py | 132 +++++++++++ vectors/README.md | 2 + 6 files changed, 655 insertions(+), 12 deletions(-) create mode 100644 scripts/reference.py create mode 100644 scripts/tests/test_reference.py diff --git a/grammar.md b/grammar.md index 06e66e1..9f2c178 100644 --- a/grammar.md +++ b/grammar.md @@ -1,6 +1,6 @@ # Postcode grammar -Spec version 0.1.0. This file defines how every Gatepost SDK reads and writes Nigeria's digital postcodes. The values live in `data/`. The vectors in `vectors/` test every rule. +Spec version 0.1.0. This file defines how every Gatepost SDK reads and writes Nigeria's digital postcodes. The values live in `data/`. The vectors in `vectors/` test every rule. If this file and a vector disagree, this file wins. Such a disagreement is a bug in the spec. ## Forms @@ -12,6 +12,8 @@ Spec version 0.1.0. This file defines how every Gatepost SDK reads and writes Ni ## Segments +`segments` in `data/format.json` holds the length, the characters and the minimum of each segment. + | Segment | Length | Characters | Rule | |---|---|---|---| | state | 2 | letters | one of the 37 codes in `data/states.json` | @@ -41,7 +43,7 @@ A format character that is not in the list stays. For example, the right-to-left First count the Unicode code points of the input. Do not count UTF-16 units, UTF-8 bytes or grapheme clusters. A lone surrogate, in a language that can hold one, counts as one code point. An SDK can stop counting at 65. If the input has more than `maxInputCodePoints` code points (64 in `data/format.json`), the error code is `bad_length`, and `parse` does not normalise the input. The limit bounds the cost of NFKC, which can take seconds on a long run of combining marks. Otherwise, normalise the input. Then apply these checks in order. The first check that fails gives the error code. 1. `empty`: no character is left. -2. `legacy_code`: exactly 6 ASCII digits are left. These are old NIPOST postcodes. +2. `legacy_code`: exactly 6 ASCII digits are left (`legacyPattern` in `data/format.json`). These are old NIPOST postcodes. 3. `bad_character`: a character other than A to Z and 0 to 9 is left. 4. `bad_length`: the length is not 11. With `allowPartial`, the lengths 2, 4, 7 and 9 also pass. 5. `unknown_state`: the first two characters are not a code in `data/states.json`. @@ -53,10 +55,10 @@ Input that is not well-formed Unicode is never a postcode. In a language whose s ## Suggestions -For `unknown_state` and `bad_segment`, build one corrected code: +For `unknown_state` and `bad_segment`, build one corrected code. `suggestions` in `data/format.json` holds the fixes: -- In the letter segments, state and area, change 0 to O and 1 to I. -- In the digit segments, LGA and unit, change O to 0, and change I and L to 1. +- In the letter segments, state and area, change `0` to `O` and `1` to `I`. +- In the digit segments, LGA and unit, change `O` to `0`, `I` to `1` and `L` to `1`. - Never change the district. NIPOST has not confirmed which characters it uses there. Parse the corrected code with the same options. If it parses, the suggestion is its canonical form. If it does not parse, or if no character changed, the suggestion is null. A suggestion is a hint only. `parse` never returns a corrected code as a success. diff --git a/scripts/reference.py b/scripts/reference.py new file mode 100644 index 0000000..2f21b3d --- /dev/null +++ b/scripts/reference.py @@ -0,0 +1,325 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +"""A plain reading of grammar.md, with every value taken from data/. + +The builder holds its expected results as typed literals, so nothing else ties the vectors to +the grammar. This module is a second reading. It does not import the builder, and it keeps no +number, code or table that data/ holds. The tests run every vector through it, so a vector or a +data file that contradicts grammar.md fails there. + +No vector can carry text that is not well-formed, so this module leaves out the rules for such +text (grammar.md, Parse). +""" + +from __future__ import annotations + +import json +import math +import re +import string +import unicodedata +from collections.abc import Callable, Mapping +from dataclasses import dataclass, replace +from itertools import accumulate, pairwise, zip_longest +from pathlib import Path +from typing import Any + +DATA_DIR = Path(__file__).resolve().parent.parent / "data" + +LETTERS = frozenset(string.ascii_uppercase) +DIGITS = frozenset(string.digits) +# The words that data/format.json uses for the characters of a segment. +CHARACTER_SETS = { + "letters": LETTERS, + "digits": DIGITS, + "letters-or-digits": LETTERS | DIGITS, +} +# str.upper() would also change letters such as U+0131, which grammar.md keeps. +ASCII_UPPER_CASE = str.maketrans(string.ascii_lowercase, string.ascii_uppercase) +# Only these two codes name a segment and offer a suggestion. +CODES_WITH_SUGGESTION = frozenset({"unknown_state", "bad_segment"}) +UNIT_MASK = "**" + +# The segments of an accepted postcode, from the state down to its precision. +Postcode = tuple[str, ...] + + +@dataclass(frozen=True, slots=True) +class SegmentRule: + """The rule for one segment, as data/format.json gives it. A minimum applies to digits.""" + + name: str + length: int + characters: str + minimum: int | None + + +@dataclass(frozen=True, slots=True) +class Failure: + """A parse error. The segment and the suggestion are None where grammar.md says null.""" + + code: str + segment: str | None = None + suggestion: str | None = None + + +@dataclass(frozen=True, slots=True) +class Grammar: + """The rules of grammar.md, filled with the values of data/.""" + + input_limit: int + segments: tuple[SegmentRule, ...] + legacy: re.Pattern[str] + separators: frozenset[str] + fixes: Mapping[str, Mapping[str, str]] + states: Mapping[str, str] + limits: tuple[tuple[float, str], ...] + fallback: str + + def normalize(self, text: str) -> str: + """Apply NFKC, remove the separators, then make the ASCII letters upper case.""" + folded = unicodedata.normalize("NFKC", text) + kept = "".join(character for character in folded if character not in self.separators) + return kept.translate(ASCII_UPPER_CASE) + + def parse(self, text: str, *, allow_partial: bool = False) -> Postcode | Failure: + """Read text as a postcode. Over the input limit, give bad_length before normalising.""" + if len(text) > self.input_limit: + return Failure("bad_length") + compact = self.normalize(text) + outcome = self._read_compact(compact, allow_partial) + if isinstance(outcome, Failure) and outcome.code in CODES_WITH_SUGGESTION: + return replace(outcome, suggestion=self._suggest(compact, allow_partial)) + return outcome + + def is_legacy(self, text: str) -> bool: + """Tell whether the text is an old postcode. Over the input limit, the answer is no.""" + if len(text) > self.input_limit: + return False + # fullmatch, because a pattern that ends in $ also matches before a final line feed. + return self.legacy.fullmatch(self.normalize(text)) is not None + + def precision(self, code: Postcode) -> str: + """Name the last segment of the code.""" + return self.segments[len(code) - 1].name + + def truncate(self, code: Postcode, to: str) -> Postcode: + """Keep the segments up to a precision. A precision that the code lacks is an error.""" + ranks = {rule.name: rank for rank, rule in enumerate(self.segments, start=1)} + if to not in ranks or ranks[to] > len(code): + raise ValueError(f"The code {'-'.join(code)} has no {to!r} precision.") + return code[: ranks[to]] + + def parent(self, code: Postcode) -> Postcode | None: + """Drop the last segment. A code with only the state has no parent.""" + return code[:-1] if len(code) > 1 else None + + def contains(self, prefix: Postcode, code: Postcode) -> bool: + """Tell whether the code has every segment of the prefix, in the same places.""" + return code[: len(prefix)] == prefix + + def redact(self, code: Postcode) -> str: + """Write the canonical form with the unit hidden. A code without a unit is unchanged.""" + if len(code) == len(self.segments): + code = (*code[:-1], UNIT_MASK) + return "-".join(code) + + def state_name(self, code: str) -> str | None: + """Look up a state code. Only the ASCII letters a to z change case, and NFKC is not used.""" + return self.states.get(code.translate(ASCII_UPPER_CASE)) + + def precision_for_accuracy(self, metres: float | None) -> str: + """Give the most precise segment that a GPS fix of this accuracy supports.""" + if metres is None or not math.isfinite(metres) or metres < 0: + return self.fallback + for limit, precision in self.limits: + if metres <= limit: + return precision + return self.fallback + + def _lengths(self, allow_partial: bool) -> frozenset[int]: + """Give the lengths that pass: the full length, and each segment end for a partial code.""" + ends = list(accumulate(rule.length for rule in self.segments)) + return frozenset(ends if allow_partial else ends[-1:]) + + def _split(self, compact: str) -> Postcode: + """Cut text of a valid length into its segments.""" + bounds = [0, *accumulate(rule.length for rule in self.segments)] + return tuple(compact[start:end] for start, end in pairwise(bounds) if start < len(compact)) + + def _read_compact(self, compact: str, allow_partial: bool) -> Postcode | Failure: + """Run the checks of grammar.md on normalised text. Offer no suggestion.""" + failure = self._reject_text(compact, allow_partial) + if failure is not None: + return failure + segments = self._split(compact) + failure = self._reject_segments(segments) + return segments if failure is None else failure + + def _reject_text(self, compact: str, allow_partial: bool) -> Failure | None: + """Run the checks on the whole text: empty, legacy_code, bad_character and bad_length.""" + if not compact: + return Failure("empty") + if self.legacy.fullmatch(compact): + return Failure("legacy_code") + if not set(compact) <= LETTERS | DIGITS: + return Failure("bad_character") + if len(compact) not in self._lengths(allow_partial): + return Failure("bad_length") + return None + + def _reject_segments(self, segments: Postcode) -> Failure | None: + """Run the checks on the segments, in order: unknown_state, then bad_segment.""" + if segments[0] not in self.states: + return Failure("unknown_state", self.segments[0].name) + # A partial code has fewer segments than rules, so zip stops at the last segment. + for rule, segment in zip(self.segments, segments, strict=False): + if not _fits(rule, segment): + return Failure("bad_segment", rule.name) + return None + + def _suggest(self, compact: str, allow_partial: bool) -> str | None: + """Parse the code with its look-alike characters fixed. Give its canonical form.""" + fixed = self._fix(compact) + # An unchanged code fails the same way, so it has no suggestion. + if fixed == compact: + return None + # The fixed code has only A to Z and 0 to 9, so it needs no normalising. + outcome = self._read_compact(fixed, allow_partial) + return None if isinstance(outcome, Failure) else "-".join(outcome) + + def _fix(self, compact: str) -> str: + """Change the characters that data/format.json lists, in the kinds of segment it names.""" + fixed: list[str] = [] + # A kind with no table, such as the district, keeps its characters. + for rule, segment in zip(self.segments, self._split(compact), strict=False): + table = self.fixes.get(rule.characters, {}) + fixed.append("".join(table.get(character, character) for character in segment)) + return "".join(fixed) + + +def _fits(rule: SegmentRule, segment: str) -> bool: + """Tell whether a segment has the characters of its rule and reaches its minimum.""" + allowed = set(segment) <= CHARACTER_SETS[rule.characters] + return allowed and (rule.minimum is None or int(segment) >= rule.minimum) + + +def load(data_dir: Path = DATA_DIR) -> Grammar: + """Read format.json, states.json and precision.json from a folder.""" + layout = _read_json(data_dir / "format.json") + states = _read_json(data_dir / "states.json")["states"] + precision = _read_json(data_dir / "precision.json") + return Grammar( + input_limit=layout["maxInputCodePoints"], + segments=tuple( + SegmentRule( + segment["name"], segment["length"], segment["characters"], segment.get("minimum") + ) + for segment in layout["segments"] + ), + legacy=re.compile(layout["legacyPattern"]), + separators=frozenset( + chr(int(label.removeprefix("U+"), 16)) for label in layout["separators"] + ), + fixes=layout["suggestions"], + states={state["code"]: state["name"] for state in states}, + limits=tuple((row["maxAccuracyM"], row["precision"]) for row in precision["thresholds"]), + fallback=precision["fallback"], + ) + + +def _read_json(path: Path) -> dict[str, Any]: + document: dict[str, Any] = json.loads(path.read_text(encoding="utf-8")) + return document + + +Case = Mapping[str, Any] +Expected = dict[str, Any] +Reader = Callable[[Grammar, Case], Expected] + + +def _postcode(grammar: Grammar, canonical: str) -> Postcode: + """Read an input of a hierarchy vector. The runner rules say to parse it with allowPartial.""" + outcome = grammar.parse(canonical, allow_partial=True) + if isinstance(outcome, Failure): + raise ValueError(f"The vector input {canonical!r} is not a postcode. Fix the vector.") + return outcome + + +def _success(grammar: Grammar, code: Postcode) -> Expected: + """Write an accepted code the way a vector does.""" + names = [rule.name for rule in grammar.segments] + return { + "ok": True, + "compact": "".join(code), + "canonical": "-".join(code), + "display": " ".join(code), + "precision": grammar.precision(code), + # zip_longest gives null for each segment after the precision. + "segments": dict(zip_longest(names, code)), + } + + +def _read_normalize(grammar: Grammar, case: Case) -> Expected: + return {"value": grammar.normalize(case["input"])} + + +def _read_parse(grammar: Grammar, case: Case) -> Expected: + outcome = grammar.parse(case["input"], allow_partial=bool(case["options"].get("allowPartial"))) + if isinstance(outcome, Failure): + error = {"code": outcome.code, "segment": outcome.segment, "suggestion": outcome.suggestion} + return {"ok": False, "error": error} + return _success(grammar, outcome) + + +def _read_is_legacy(grammar: Grammar, case: Case) -> Expected: + return {"value": grammar.is_legacy(case["input"])} + + +def _read_truncate(grammar: Grammar, case: Case) -> Expected: + code = _postcode(grammar, case["input"]["code"]) + try: + shorter = grammar.truncate(code, case["input"]["to"]) + except ValueError: + return {"rejects": True} + return {"canonical": "-".join(shorter)} + + +def _read_parent(grammar: Grammar, case: Case) -> Expected: + above = grammar.parent(_postcode(grammar, case["input"]["code"])) + return {"canonical": None if above is None else "-".join(above)} + + +def _read_contains(grammar: Grammar, case: Case) -> Expected: + prefix = _postcode(grammar, case["input"]["prefix"]) + return {"value": grammar.contains(prefix, _postcode(grammar, case["input"]["code"]))} + + +def _read_redact(grammar: Grammar, case: Case) -> Expected: + return {"value": grammar.redact(_postcode(grammar, case["input"]["code"]))} + + +def _read_state_name(grammar: Grammar, case: Case) -> Expected: + return {"value": grammar.state_name(case["input"])} + + +def _read_precision_for_accuracy(grammar: Grammar, case: Case) -> Expected: + metres = case["input"] + # The runner rules turn the strings NaN, Infinity and -Infinity into numbers. + if isinstance(metres, str): + metres = float(metres) + return {"value": grammar.precision_for_accuracy(metres)} + + +# One reader for each function that a vector file tests, by the name in its function field. +READERS: dict[str, Reader] = { + "normalize": _read_normalize, + "parse": _read_parse, + "isLegacy": _read_is_legacy, + "truncate": _read_truncate, + "parent": _read_parent, + "contains": _read_contains, + "redact": _read_redact, + "stateName": _read_state_name, + "precisionForAccuracy": _read_precision_for_accuracy, +} diff --git a/scripts/tests/test_build_vectors.py b/scripts/tests/test_build_vectors.py index 51a3b4c..804d7dd 100644 --- a/scripts/tests/test_build_vectors.py +++ b/scripts/tests/test_build_vectors.py @@ -2,14 +2,19 @@ # SPDX-License-Identifier: Apache-2.0 """Tests for build_vectors.""" +import contextlib +import io import json import math +import shutil import string +import tempfile import unicodedata import unittest from collections import Counter from pathlib import Path from typing import Any +from unittest import mock import build_vectors @@ -265,3 +270,76 @@ def test_is_legacy_has_cases_at_the_limit_above_it_and_for_each_counting_mistake line_feed_lengths = {len(text) for text in rejected if text.endswith("\n")} with self.subTest("each rejection that ends in a line feed is one code point over"): self.assertEqual(line_feed_lengths, {limit + 1}) + + +class BuilderModesTest(unittest.TestCase): + """Run the builder on a temporary copy of the vector folder.""" + + def setUp(self) -> None: + folder = tempfile.TemporaryDirectory() + self.addCleanup(folder.cleanup) + self.vectors = Path(folder.name) + for stem in build_vectors.FILES: + shutil.copy(ROOT / "vectors" / f"{stem}.json", self.vectors) + patch = mock.patch.object(build_vectors, "VECTORS", self.vectors) + patch.start() + self.addCleanup(patch.stop) + + def snapshot(self) -> dict[str, str]: + return { + path.name: path.read_text(encoding="utf-8") for path in sorted(self.vectors.glob("*")) + } + + def run_builder(self, *arguments: str) -> tuple[int, str]: + errors = io.StringIO() + with contextlib.redirect_stderr(errors): + exit_code = build_vectors.main(list(arguments)) + return exit_code, errors.getvalue() + + def test_check_passes_files_that_match_the_builder(self) -> None: + before = self.snapshot() + self.assertEqual(self.run_builder("--check"), (0, "")) + self.assertEqual(self.snapshot(), before) + + def test_check_reports_a_changed_file_and_a_stray_file_and_writes_nothing(self) -> None: + changed = self.vectors / "redact.json" + changed.write_text(changed.read_text(encoding="utf-8") + " ", encoding="utf-8") + (self.vectors / "extra.json").write_text("{}\n", encoding="utf-8") + before = self.snapshot() + exit_code, errors = self.run_builder("--check") + self.assertEqual(exit_code, 1) + self.assertIn("Out of date: redact.json.", errors) + self.assertIn("vectors/extra.json is not a builder output. Remove it.", errors) + self.assertEqual(self.snapshot(), before) + + def test_check_fails_for_a_stray_file_alone(self) -> None: + (self.vectors / "extra.json").write_text("{}\n", encoding="utf-8") + exit_code, errors = self.run_builder("--check") + self.assertEqual(exit_code, 1) + self.assertNotIn("Out of date", errors) + + def test_check_reports_a_missing_file_and_does_not_write_it(self) -> None: + (self.vectors / "parent.json").unlink() + before = self.snapshot() + exit_code, errors = self.run_builder("--check") + self.assertEqual(exit_code, 1) + self.assertIn("Out of date: parent.json.", errors) + self.assertEqual(self.snapshot(), before) + + def test_writing_makes_a_missing_folder_and_rewrites_a_changed_file(self) -> None: + expected = self.snapshot() + fresh = self.vectors / "fresh" + with mock.patch.object(build_vectors, "VECTORS", fresh): + self.assertEqual(self.run_builder(), (0, "")) + changed = fresh / "redact.json" + changed.write_text("{}\n", encoding="utf-8") + self.assertEqual(self.run_builder(), (0, "")) + self.assertEqual({path.name: path.read_text("utf-8") for path in fresh.glob("*")}, expected) + + def test_writing_reports_a_stray_file_and_keeps_it(self) -> None: + stray = self.vectors / "extra.json" + stray.write_text("{}\n", encoding="utf-8") + exit_code, errors = self.run_builder() + self.assertEqual(exit_code, 1) + self.assertIn("vectors/extra.json is not a builder output. Remove it.", errors) + self.assertEqual(stray.read_text(encoding="utf-8"), "{}\n") diff --git a/scripts/tests/test_data.py b/scripts/tests/test_data.py index 7e6bc61..a88ae33 100644 --- a/scripts/tests/test_data.py +++ b/scripts/tests/test_data.py @@ -1,14 +1,16 @@ # SPDX-FileCopyrightText: 2026 The Gatepost authors # SPDX-License-Identifier: Apache-2.0 -"""Tests for the shared data files.""" +"""Tests for the shared data files, and for the grammar text that repeats their values.""" import json import re import unittest +from itertools import accumulate from pathlib import Path from typing import Any -DATA_DIR = Path(__file__).resolve().parents[2] / "data" +ROOT = Path(__file__).resolve().parents[2] +DATA_DIR = ROOT / "data" SEGMENT_NAMES = ["state", "lga", "district", "area", "unit"] # The states where NIPOST's code is not the ISO 3166-2:NG code. Name: (code, iso). NIPOST_CODES_OFF_ISO = { @@ -27,6 +29,26 @@ def load(name: str) -> dict[str, Any]: return document +def grammar_text() -> str: + return (ROOT / "grammar.md").read_text(encoding="utf-8") + + +def grammar_section(heading: str) -> str: + """Return the text under a second-level heading of grammar.md, up to the next heading.""" + for section in re.split(r"^## ", grammar_text(), flags=re.MULTILINE): + if section.startswith(heading + "\n"): + return section + raise AssertionError(f"grammar.md has no section {heading!r}.") + + +def integers_after(anchor: str) -> list[int]: + """Return the integers in the sentence of grammar.md that follows the anchor text.""" + sentence = re.search(re.escape(anchor) + r"([^.]*)\.", grammar_text()) + if sentence is None: + raise AssertionError(f"grammar.md has no sentence that follows {anchor!r}.") + return [int(number) for number in re.findall(r"\d+", sentence.group(1))] + + class StatesTest(unittest.TestCase): def test_lists_37_states_in_code_order(self) -> None: codes = [state["code"] for state in load("states.json")["states"]] @@ -37,7 +59,8 @@ def test_uses_two_capital_letters_for_each_code(self) -> None: for state in load("states.json")["states"]: with self.subTest(code=state["code"]): self.assertRegex(state["code"], r"^[A-Z]{2}$") - self.assertTrue(state["name"].isascii() and state["name"].strip()) + self.assertTrue(state["name"].isascii()) + self.assertNotEqual(state["name"].strip(), "") def test_keeps_the_iso_code_of_each_state_beside_the_nipost_code(self) -> None: seen: list[str] = [] @@ -60,8 +83,8 @@ def test_raises_the_limit_as_precision_falls(self) -> None: limits = [row["maxAccuracyM"] for row in precision_rules["thresholds"]] self.assertEqual(limits, sorted(limits)) threshold_precisions = [row["precision"] for row in precision_rules["thresholds"]] - names = [*threshold_precisions, precision_rules["fallback"]] - self.assertEqual(names, ["unit", "area", "district", "lga"]) + precision_names = [*threshold_precisions, precision_rules["fallback"]] + self.assertEqual(precision_names, ["unit", "area", "district", "lga"]) class FormatTest(unittest.TestCase): @@ -70,13 +93,94 @@ def test_names_five_segments_of_11_characters(self) -> None: self.assertEqual([segment["name"] for segment in segments], SEGMENT_NAMES) self.assertEqual(sum(segment["length"] for segment in segments), 11) + def test_describes_each_segment_with_a_known_word_for_its_characters(self) -> None: + for segment in load("format.json")["segments"]: + with self.subTest(segment=segment["name"]): + self.assertIn(segment["characters"], {"letters", "digits", "letters-or-digits"}) + if "minimum" in segment: + self.assertEqual(segment["characters"], "digits") + def test_writes_each_separator_as_a_code_point(self) -> None: for label in load("format.json")["separators"]: with self.subTest(label=label): self.assertRegex(label, r"^U\+[0-9A-F]{4}$") - def test_compiles_the_legacy_pattern(self) -> None: - self.assertTrue(re.fullmatch(load("format.json")["legacyPattern"], "900108")) + def test_matches_a_legacy_code_of_exactly_six_ascii_digits(self) -> None: + pattern = load("format.json")["legacyPattern"] + arabic_indic_digits = "".join(chr(0x0660 + digit) for digit in (9, 0, 0, 1, 0, 8)) + cases = [ + ("900108", True), + ("90010", False), + ("9001081", False), + ("90010A", False), + ("", False), + (arabic_indic_digits, False), + ] + for text, matches in cases: + with self.subTest(text=ascii(text)): + self.assertEqual(re.fullmatch(pattern, text) is not None, matches) def test_limits_the_input_to_64_code_points(self) -> None: self.assertEqual(load("format.json")["maxInputCodePoints"], 64) + + +class GrammarTextTest(unittest.TestCase): + """grammar.md repeats some values of data/format.json, so that a reader needs no other file.""" + + def test_the_segment_table_matches_the_segments(self) -> None: + segments = load("format.json")["segments"] + state_count = len(load("states.json")["states"]) + lines = grammar_section("Segments").splitlines() + rows = [ + line for line in lines if line.startswith("| ") and not line.startswith("| Segment") + ] + cells = [[cell.strip() for cell in row.strip("|").split("|")] for row in rows] + self.assertEqual([row[0] for row in cells], [segment["name"] for segment in segments]) + for row, segment in zip(cells, segments, strict=True): + with self.subTest(segment=segment["name"]): + length = segment["length"] + self.assertEqual(int(row[1]), length) + self.assertEqual(row[2].replace(" or ", "-or-"), segment["characters"]) + if "minimum" in segment: + expected = [f"{segment['minimum']:0{length}d}", "9" * length] + elif segment["name"] == "state": + # The rule of the state segment counts the codes in states.json. + expected = [str(state_count)] + else: + expected = [] + self.assertEqual(re.findall(r"\d+", row[3]), expected) + + def test_the_partial_lengths_are_the_sums_of_the_segment_lengths(self) -> None: + lengths = [segment["length"] for segment in load("format.json")["segments"]] + ends = list(accumulate(lengths)) + self.assertEqual(integers_after("Its length is"), ends[:-1]) + self.assertEqual(integers_after("the length is not"), ends[-1:]) + self.assertEqual(integers_after("the lengths"), ends[:-1]) + + def test_the_input_limit_matches_max_input_code_points(self) -> None: + limit = load("format.json")["maxInputCodePoints"] + cited = re.findall(r"\((\d+) in `data/format.json`\)", grammar_text()) + self.assertEqual(cited, [str(limit)]) + self.assertEqual(integers_after("An SDK can stop counting at"), [limit + 1]) + + def test_the_legacy_rule_matches_the_legacy_pattern(self) -> None: + pattern = load("format.json")["legacyPattern"] + counts = [int(count) for count in re.findall(r"exactly (\d+) ASCII digits", grammar_text())] + # The legacy_code check and the isLegacy section both state the rule. + self.assertGreaterEqual(len(counts), 2) + for count in counts: + with self.subTest(count=count): + self.assertTrue(re.fullmatch(pattern, "0" * count)) + self.assertFalse(re.fullmatch(pattern, "0" * (count - 1))) + self.assertFalse(re.fullmatch(pattern, "0" * (count + 1))) + + def test_the_fixes_match_the_suggestions(self) -> None: + suggestions = load("format.json")["suggestions"] + lead_ins = {"letters": "- In the letter segments", "digits": "- In the digit segments"} + # A kind of segment with no table keeps its characters, so the district has no fixes. + self.assertEqual(sorted(suggestions), sorted(lead_ins)) + lines = grammar_section("Suggestions").splitlines() + for kind, lead_in in lead_ins.items(): + with self.subTest(kind=kind): + line = next((line for line in lines if line.startswith(lead_in)), "") + self.assertEqual(dict(re.findall(r"`(.)` to `(.)`", line)), suggestions[kind]) diff --git a/scripts/tests/test_reference.py b/scripts/tests/test_reference.py new file mode 100644 index 0000000..c43a6d3 --- /dev/null +++ b/scripts/tests/test_reference.py @@ -0,0 +1,132 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +"""Tests for reference: it agrees with every vector, and it reads each value from data/.""" + +import json +import tempfile +import unittest +from collections.abc import Callable +from pathlib import Path +from typing import Any + +import reference + +ROOT = Path(__file__).resolve().parents[2] +CODE = "EK01A03FK01" + +Edit = Callable[[dict[str, Any]], object] +Probe = Callable[[reference.Grammar], object] + + +def rename_ekiti(document: dict[str, Any]) -> None: + for state in document["states"]: + if state["code"] == "EK": + state["name"] = "Ekiti State" + + +def remove_ekiti(document: dict[str, Any]) -> None: + document["states"] = [state for state in document["states"] if state["code"] != "EK"] + + +# One row for each value in data/: the edit that changes it, and a call whose answer must change +# with it. A reference that keeps its own copy of the value gives the same answer both times. +DATA_CHANGES: dict[str, tuple[dict[str, Edit], Probe]] = { + "the input limit": ( + {"format": lambda document: document.update(maxInputCodePoints=12)}, + lambda grammar: grammar.parse(CODE + " "), + ), + "a separator": ( + {"format": lambda document: document["separators"].remove("U+002D")}, + lambda grammar: grammar.normalize("EK-01"), + ), + "the length of a segment": ( + {"format": lambda document: document["segments"][1].update(length=3)}, + lambda grammar: grammar.parse(CODE), + ), + "the characters of a segment": ( + {"format": lambda document: document["segments"][2].update(characters="letters")}, + lambda grammar: grammar.parse(CODE), + ), + "the minimum of a segment": ( + {"format": lambda document: document["segments"][1].update(minimum=2)}, + lambda grammar: grammar.parse(CODE), + ), + "a fix for a letter segment": ( + {"format": lambda document: document["suggestions"]["letters"].update({"5": "S"})}, + lambda grammar: grammar.parse("EK01A03F501"), + ), + "a fix for a digit segment": ( + {"format": lambda document: document["suggestions"]["digits"].update({"B": "8"})}, + lambda grammar: grammar.parse("EK0BA03FK01"), + ), + "the legacy pattern": ( + {"format": lambda document: document.update(legacyPattern="^[0-9]{6,7}$")}, + lambda grammar: grammar.is_legacy("9001080"), + ), + "a state code": ({"states": remove_ekiti}, lambda grammar: grammar.parse(CODE)), + "a state name": ({"states": rename_ekiti}, lambda grammar: grammar.state_name("EK")), + "a GPS limit": ( + {"precision": lambda document: document["thresholds"][1].update(maxAccuracyM=25)}, + lambda grammar: grammar.precision_for_accuracy(22), + ), + "the GPS fallback": ( + {"precision": lambda document: document.update(fallback="district")}, + lambda grammar: grammar.precision_for_accuracy(None), + ), +} + + +def vector_documents() -> list[tuple[str, dict[str, Any]]]: + paths = sorted((ROOT / "vectors").glob("*.json")) + return [(path.name, json.loads(path.read_text(encoding="utf-8"))) for path in paths] + + +def load_with(edits: dict[str, Edit]) -> reference.Grammar: + """Load the data files from a temporary folder, after the edits to the named files.""" + with tempfile.TemporaryDirectory() as folder: + for stem in ("format", "states", "precision"): + source = reference.DATA_DIR / f"{stem}.json" + document = json.loads(source.read_text(encoding="utf-8")) + if stem in edits: + edits[stem](document) + (Path(folder) / f"{stem}.json").write_text(json.dumps(document), encoding="utf-8") + return reference.load(Path(folder)) + + +class ReferenceVectorsTest(unittest.TestCase): + def test_each_vector_file_has_a_reader(self) -> None: + missing = [ + f"{name}: {document['function']}" + for name, document in vector_documents() + if document["function"] not in reference.READERS + ] + self.assertEqual(missing, []) + + def test_each_case_gives_its_expected_value(self) -> None: + grammar = reference.load() + for _, document in vector_documents(): + reader = reference.READERS.get(document["function"]) + if reader is None: + continue + for case in document["cases"]: + with self.subTest(case["id"], description=case["description"]): + self.assertEqual(reader(grammar, case), case["expect"]) + + +class ReferenceDataTest(unittest.TestCase): + def test_each_data_value_changes_what_the_reference_reads(self) -> None: + unchanged = reference.load() + for name, (edits, probe) in DATA_CHANGES.items(): + with self.subTest(name): + self.assertNotEqual(probe(load_with(edits)), probe(unchanged)) + + def test_truncate_rejects_a_value_that_is_not_a_precision(self) -> None: + grammar = reference.load() + code = ("EK", "01", "A03") + for precision in ("street", "", "UNIT"): + with self.subTest(precision=precision), self.assertRaises(ValueError): + grammar.truncate(code, precision) + + def test_truncate_rejects_a_precision_that_the_code_lacks(self) -> None: + with self.assertRaises(ValueError): + reference.load().truncate(("EK", "01", "A03"), "area") diff --git a/vectors/README.md b/vectors/README.md index 6ac5c23..d0d0ea9 100644 --- a/vectors/README.md +++ b/vectors/README.md @@ -4,6 +4,8 @@ Each JSON file here holds shared test cases for one core function. Every SDK run `scripts/build_vectors.py` builds these files. Do not edit them by hand. Edit the builder, then run `python3 scripts/build_vectors.py`. +`scripts/reference.py` is a plain reading of `grammar.md` and `data/`. `make check` runs every case through it. A case that the grammar contradicts fails the check, and the grammar wins. + ## Format { "version": 1, "function": "parse", "description": "...", "cases": [ ... ] } From 3c0b6779b0fb62a0b7b8112e4ddd925de4e4a1a0 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 12:38:07 +0100 Subject: [PATCH 30/45] test: add vectors for table gaps, an area digit and negative zero Four wrong readings of the grammar passed every vector. A letter check that looks only at the last character of the area accepted EK01A031K01. An NFKC table without U+208B, U+FE31, U+FE32 and U+FE58 kept four characters that NFKC turns into a separator that is not ASCII. A table without its 4-byte entries passed too, because no normalize case held a character above U+FFFF. Each of these now has a vector. A test computes the characters of the second kind over all code points, so a new one fails the check on a Python that knows it. Some languages can test the sign bit of a negative zero. An SDK that does so gives lga for an accuracy of -0.0, but a plain comparison gives unit. The grammar now says that -0 counts as 0, and a vector pins it. A test checks that the vector file keeps the sign of the zero. The ids after each new case move up by one, as the vector format allows. Signed-off-by: MobileMage --- grammar.md | 2 +- scripts/build_vectors.py | 28 +++++- scripts/reference.py | 5 +- scripts/tests/test_build_vectors.py | 56 +++++++++++ vectors/normalize.json | 151 ++++++++++++++++++---------- vectors/parse-segments.json | 48 +++++---- vectors/precision-for-accuracy.json | 13 ++- 7 files changed, 226 insertions(+), 77 deletions(-) diff --git a/grammar.md b/grammar.md index 9f2c178..4c12943 100644 --- a/grammar.md +++ b/grammar.md @@ -80,7 +80,7 @@ Parse the corrected code with the same options. If it parses, the suggestion is ## GPS precision -`precisionForAccuracy(metres)` returns the most precise segment that a GPS fix of that accuracy supports, from `data/precision.json`. An unknown, negative, infinite or NaN accuracy gives `lga`. A fix with an accuracy equal to a limit gets the precision of that limit. +`precisionForAccuracy(metres)` returns the most precise segment that a GPS fix of that accuracy supports, from `data/precision.json`. An unknown, negative, infinite or NaN accuracy gives `lga`. The value -0 counts as 0, so it is not negative. A fix with an accuracy equal to a limit gets the precision of that limit. ## Versions diff --git a/scripts/build_vectors.py b/scripts/build_vectors.py index 82db250..32ee9a3 100755 --- a/scripts/build_vectors.py +++ b/scripts/build_vectors.py @@ -38,7 +38,6 @@ RIGHT_TO_LEFT_MARK = chr(0x200F) RIGHT_TO_LEFT_OVERRIDE = chr(0x202E) FULL_WIDTH_HYPHEN = chr(0xFF0D) -SUPERSCRIPT_MINUS = chr(0x207B) COMBINING_ACUTE = chr(0x0301) HORIZONTAL_ELLIPSIS = chr(0x2026) ZERO_WIDTH_JOINER = chr(0x200D) @@ -48,6 +47,16 @@ chr(code_point) for code_point in (0x0669, 0x0660, 0x0660, 0x0661, 0x0660, 0x0668) ) +# The characters, other than the separators themselves, whose NFKC form holds a separator that +# is not ASCII. A table of ASCII forms only keeps them. A test computes the full set. +NFKC_SEPARATOR_SOURCES = ( + ("a superscript minus", chr(0x207B)), + ("a subscript minus", chr(0x208B)), + ("a vertical em dash", chr(0xFE31)), + ("a vertical en dash", chr(0xFE32)), + ("a small em dash", chr(0xFE58)), +) + WIDTHS = (("state", 2), ("lga", 2), ("district", 3), ("area", 2), ("unit", 2)) PRECISION_BY_LENGTH = {2: "state", 4: "lga", 7: "district", 9: "area", 11: "unit"} PARTIAL = {"allowPartial": True} @@ -156,6 +165,10 @@ def normalize_rows() -> list[Row]: row(f"removes the separator U+{code_point:04X}", joined(chr(code_point)), want) for code_point in load_separators() ] + nfkc_separator_rows = [ + row(f"removes {name}", "EK" + character + "01A03FK01", want) + for name, character in NFKC_SEPARATOR_SOURCES + ] return [ row("removes hyphens", "EK-01-A03-FK-01", want), row("removes spaces and makes letters upper case", "ek 01 a03 fk 01", want), @@ -176,6 +189,8 @@ def normalize_rows() -> list[Row]: value(RIGHT_TO_LEFT_OVERRIDE + CODE), ), row("changes full-width letters and digits to ASCII", full_width(CODE), want), + # Each character is above U+FFFF, so a table that skips the 4-byte entries keeps it. + row("changes mathematical bold letters and digits to ASCII", MATH_BOLD_CODE, want), # An SDK that makes ASCII letters upper case before NFKC leaves these in lower case. row( "changes full-width lower-case letters to ASCII upper case", @@ -183,8 +198,7 @@ def normalize_rows() -> list[Row]: want, ), row("removes full-width hyphens", joined(FULL_WIDTH_HYPHEN), want), - # NFKC changes U+207B to the separator U+2212. A table of ASCII forms only would keep it. - row("removes a superscript minus", "EK" + SUPERSCRIPT_MINUS + "01A03FK01", want), + *nfkc_separator_rows, row("removes tabs and line feeds", "EK01" + TAB + "A03" + LINE_FEED + "FK01", want), row( "changes a ligature to its letters", @@ -415,6 +429,12 @@ def segment_rows() -> list[Row]: "EK01A03F001", failed("bad_segment", "area", "EK-01-A03-FO-01"), ), + # A letter check that looks only at the last character of the area accepts this code. + row( + "rejects a digit before a letter in the area", + "EK01A031K01", + failed("bad_segment", "area", "EK-01-A03-IK-01"), + ), row( "suggests 0 for a letter O in the unit", "EK01A03FKO1", @@ -703,6 +723,8 @@ def accuracy_rows() -> list[Row]: row("gives the LGA at 1000 m", 1000, value("lga")), row("gives the LGA when the accuracy is unknown", None, value("lga")), row("gives the LGA for a negative accuracy", -1, value("lga")), + # An accuracy of -0.0 is not negative. A sign test such as double.IsNegative says it is. + row("gives the unit for negative zero", -0.0, value("unit")), row("gives the LGA for NaN", "NaN", value("lga")), row("gives the LGA for infinity", "Infinity", value("lga")), row("gives the LGA for negative infinity", "-Infinity", value("lga")), diff --git a/scripts/reference.py b/scripts/reference.py index 2f21b3d..05dab5e 100644 --- a/scripts/reference.py +++ b/scripts/reference.py @@ -130,6 +130,7 @@ def state_name(self, code: str) -> str | None: def precision_for_accuracy(self, metres: float | None) -> str: """Give the most precise segment that a GPS fix of this accuracy supports.""" + # -0.0 is not below 0, so it counts as 0 here. Testing the sign bit would give lga. if metres is None or not math.isfinite(metres) or metres < 0: return self.fallback for limit, precision in self.limits: @@ -242,7 +243,9 @@ def _postcode(grammar: Grammar, canonical: str) -> Postcode: """Read an input of a hierarchy vector. The runner rules say to parse it with allowPartial.""" outcome = grammar.parse(canonical, allow_partial=True) if isinstance(outcome, Failure): - raise ValueError(f"The vector input {canonical!r} is not a postcode. Fix the vector.") + raise ValueError( + f"The vector input {canonical!r} does not parse. Check the vector and data/." + ) return outcome diff --git a/scripts/tests/test_build_vectors.py b/scripts/tests/test_build_vectors.py index 804d7dd..d5b09bd 100644 --- a/scripts/tests/test_build_vectors.py +++ b/scripts/tests/test_build_vectors.py @@ -8,6 +8,7 @@ import math import shutil import string +import sys import tempfile import unicodedata import unittest @@ -161,6 +162,61 @@ def test_each_separator_has_an_nfkc_form_of_ascii_characters_and_separators(self ] self.assertEqual(outside, []) + def test_each_character_that_nfkc_changes_to_a_non_ascii_separator_has_a_normalize_case( + self, + ) -> None: + separators = separator_characters() + inputs = {character for case in load("normalize")["cases"] for character in case["input"]} + sources = { + character + for character in map(chr, range(sys.maxunicode + 1)) + if character not in separators + and any( + not form.isascii() and form in separators + for form in unicodedata.normalize("NFKC", character) + ) + } + # A table of ASCII forms only keeps these characters. Each separator has its own case. + self.assertTrue(sources) + self.assertEqual([f"U+{ord(c):04X}" for c in sorted(sources - inputs)], []) + + def test_normalize_has_cases_for_letters_and_digits_above_u_ffff(self) -> None: + astral = { + character + for case in load("normalize")["cases"] + for character in case["input"] + if ord(character) > 0xFFFF + } + forms = [unicodedata.normalize("NFKC", character) for character in sorted(astral)] + # A table that skips each 4-byte entry passes every other normalize case. + with self.subTest("a letter"): + self.assertTrue(any(form.isascii() and form.isalpha() for form in forms)) + with self.subTest("a digit"): + self.assertTrue(any(form.isascii() and form.isdigit() for form in forms)) + + def test_parse_segments_rejects_a_digit_before_a_letter_in_the_area(self) -> None: + rejected = [ + case + for case in load("parse-segments")["cases"] + if len(case["input"]) == 11 + and case["input"][7] in string.digits + and case["input"][8] in string.ascii_letters + and case["expect"].get("error", {}).get("segment") == "area" + ] + # A letter check that is anchored only at the end accepts such an area. + self.assertTrue(rejected) + + def test_precision_for_accuracy_has_a_case_for_negative_zero(self) -> None: + # The JSON literal -0 loads as the integer 0, so only -0.0 keeps the sign. + negative_zero = [ + case["expect"] + for case in load("precision-for-accuracy")["cases"] + if isinstance(case["input"], float) + and case["input"] == 0 + and math.copysign(1.0, case["input"]) < 0 + ] + self.assertEqual(negative_zero, [{"value": "unit"}]) + def test_each_normalize_case_holds_with_full_nfkc_and_with_a_table(self) -> None: separators = separator_characters() normalizers = {"full NFKC": full_normalize, "NFKC through a table": reduced_normalize} diff --git a/vectors/normalize.json b/vectors/normalize.json index da1042c..048d380 100644 --- a/vectors/normalize.json +++ b/vectors/normalize.json @@ -140,6 +140,15 @@ }, { "id": "normalize-016", + "description": "changes mathematical bold letters and digits to ASCII", + "input": "\ud835\udc04\ud835\udc0a\ud835\udfce\ud835\udfcf\ud835\udc00\ud835\udfce\ud835\udfd1\ud835\udc05\ud835\udc0a\ud835\udfce\ud835\udfcf", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-017", "description": "changes full-width lower-case letters to ASCII upper case", "input": "\uff45\uff4b\uff10\uff11\uff41\uff10\uff13\uff46\uff4b\uff10\uff11", "options": {}, @@ -148,7 +157,7 @@ } }, { - "id": "normalize-017", + "id": "normalize-018", "description": "removes full-width hyphens", "input": "EK\uff0d01\uff0dA03\uff0dFK\uff0d01", "options": {}, @@ -157,7 +166,7 @@ } }, { - "id": "normalize-018", + "id": "normalize-019", "description": "removes a superscript minus", "input": "EK\u207b01A03FK01", "options": {}, @@ -166,7 +175,43 @@ } }, { - "id": "normalize-019", + "id": "normalize-020", + "description": "removes a subscript minus", + "input": "EK\u208b01A03FK01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-021", + "description": "removes a vertical em dash", + "input": "EK\ufe3101A03FK01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-022", + "description": "removes a vertical en dash", + "input": "EK\ufe3201A03FK01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-023", + "description": "removes a small em dash", + "input": "EK\ufe5801A03FK01", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } + }, + { + "id": "normalize-024", "description": "removes tabs and line feeds", "input": "EK01\tA03\nFK01", "options": {}, @@ -175,7 +220,7 @@ } }, { - "id": "normalize-020", + "id": "normalize-025", "description": "changes a ligature to its letters", "input": "EK01A03\ufb0101", "options": {}, @@ -184,7 +229,7 @@ } }, { - "id": "normalize-021", + "id": "normalize-026", "description": "makes mixed-case letters upper case", "input": "eK01a03Fk01", "options": {}, @@ -193,7 +238,7 @@ } }, { - "id": "normalize-022", + "id": "normalize-027", "description": "keeps a legacy code", "input": "900 108", "options": {}, @@ -202,7 +247,7 @@ } }, { - "id": "normalize-023", + "id": "normalize-028", "description": "keeps characters that are not separators", "input": "EK01A03FK0!", "options": {}, @@ -211,7 +256,7 @@ } }, { - "id": "normalize-024", + "id": "normalize-029", "description": "makes only ASCII letters upper case", "input": "ek01a03f\u00e901", "options": {}, @@ -220,7 +265,7 @@ } }, { - "id": "normalize-025", + "id": "normalize-030", "description": "keeps a dotless i", "input": "EK01A03F\u013101", "options": {}, @@ -229,7 +274,7 @@ } }, { - "id": "normalize-026", + "id": "normalize-031", "description": "returns an empty string for empty input", "input": "", "options": {}, @@ -238,7 +283,7 @@ } }, { - "id": "normalize-027", + "id": "normalize-032", "description": "returns an empty string for separators only", "input": " - . ", "options": {}, @@ -247,7 +292,7 @@ } }, { - "id": "normalize-028", + "id": "normalize-033", "description": "removes the separator U+0009", "input": "EK\t01\tA03\tFK\t01", "options": {}, @@ -256,7 +301,7 @@ } }, { - "id": "normalize-029", + "id": "normalize-034", "description": "removes the separator U+000A", "input": "EK\n01\nA03\nFK\n01", "options": {}, @@ -265,7 +310,7 @@ } }, { - "id": "normalize-030", + "id": "normalize-035", "description": "removes the separator U+000B", "input": "EK\u000b01\u000bA03\u000bFK\u000b01", "options": {}, @@ -274,7 +319,7 @@ } }, { - "id": "normalize-031", + "id": "normalize-036", "description": "removes the separator U+000C", "input": "EK\f01\fA03\fFK\f01", "options": {}, @@ -283,7 +328,7 @@ } }, { - "id": "normalize-032", + "id": "normalize-037", "description": "removes the separator U+000D", "input": "EK\r01\rA03\rFK\r01", "options": {}, @@ -292,7 +337,7 @@ } }, { - "id": "normalize-033", + "id": "normalize-038", "description": "removes the separator U+0020", "input": "EK 01 A03 FK 01", "options": {}, @@ -301,7 +346,7 @@ } }, { - "id": "normalize-034", + "id": "normalize-039", "description": "removes the separator U+002D", "input": "EK-01-A03-FK-01", "options": {}, @@ -310,7 +355,7 @@ } }, { - "id": "normalize-035", + "id": "normalize-040", "description": "removes the separator U+002E", "input": "EK.01.A03.FK.01", "options": {}, @@ -319,7 +364,7 @@ } }, { - "id": "normalize-036", + "id": "normalize-041", "description": "removes the separator U+0085", "input": "EK\u008501\u0085A03\u0085FK\u008501", "options": {}, @@ -328,7 +373,7 @@ } }, { - "id": "normalize-037", + "id": "normalize-042", "description": "removes the separator U+00A0", "input": "EK\u00a001\u00a0A03\u00a0FK\u00a001", "options": {}, @@ -337,7 +382,7 @@ } }, { - "id": "normalize-038", + "id": "normalize-043", "description": "removes the separator U+00AD", "input": "EK\u00ad01\u00adA03\u00adFK\u00ad01", "options": {}, @@ -346,7 +391,7 @@ } }, { - "id": "normalize-039", + "id": "normalize-044", "description": "removes the separator U+1680", "input": "EK\u168001\u1680A03\u1680FK\u168001", "options": {}, @@ -355,7 +400,7 @@ } }, { - "id": "normalize-040", + "id": "normalize-045", "description": "removes the separator U+2000", "input": "EK\u200001\u2000A03\u2000FK\u200001", "options": {}, @@ -364,7 +409,7 @@ } }, { - "id": "normalize-041", + "id": "normalize-046", "description": "removes the separator U+2001", "input": "EK\u200101\u2001A03\u2001FK\u200101", "options": {}, @@ -373,7 +418,7 @@ } }, { - "id": "normalize-042", + "id": "normalize-047", "description": "removes the separator U+2002", "input": "EK\u200201\u2002A03\u2002FK\u200201", "options": {}, @@ -382,7 +427,7 @@ } }, { - "id": "normalize-043", + "id": "normalize-048", "description": "removes the separator U+2003", "input": "EK\u200301\u2003A03\u2003FK\u200301", "options": {}, @@ -391,7 +436,7 @@ } }, { - "id": "normalize-044", + "id": "normalize-049", "description": "removes the separator U+2004", "input": "EK\u200401\u2004A03\u2004FK\u200401", "options": {}, @@ -400,7 +445,7 @@ } }, { - "id": "normalize-045", + "id": "normalize-050", "description": "removes the separator U+2005", "input": "EK\u200501\u2005A03\u2005FK\u200501", "options": {}, @@ -409,7 +454,7 @@ } }, { - "id": "normalize-046", + "id": "normalize-051", "description": "removes the separator U+2006", "input": "EK\u200601\u2006A03\u2006FK\u200601", "options": {}, @@ -418,7 +463,7 @@ } }, { - "id": "normalize-047", + "id": "normalize-052", "description": "removes the separator U+2007", "input": "EK\u200701\u2007A03\u2007FK\u200701", "options": {}, @@ -427,7 +472,7 @@ } }, { - "id": "normalize-048", + "id": "normalize-053", "description": "removes the separator U+2008", "input": "EK\u200801\u2008A03\u2008FK\u200801", "options": {}, @@ -436,7 +481,7 @@ } }, { - "id": "normalize-049", + "id": "normalize-054", "description": "removes the separator U+2009", "input": "EK\u200901\u2009A03\u2009FK\u200901", "options": {}, @@ -445,7 +490,7 @@ } }, { - "id": "normalize-050", + "id": "normalize-055", "description": "removes the separator U+200A", "input": "EK\u200a01\u200aA03\u200aFK\u200a01", "options": {}, @@ -454,7 +499,7 @@ } }, { - "id": "normalize-051", + "id": "normalize-056", "description": "removes the separator U+200B", "input": "EK\u200b01\u200bA03\u200bFK\u200b01", "options": {}, @@ -463,7 +508,7 @@ } }, { - "id": "normalize-052", + "id": "normalize-057", "description": "removes the separator U+200C", "input": "EK\u200c01\u200cA03\u200cFK\u200c01", "options": {}, @@ -472,7 +517,7 @@ } }, { - "id": "normalize-053", + "id": "normalize-058", "description": "removes the separator U+200D", "input": "EK\u200d01\u200dA03\u200dFK\u200d01", "options": {}, @@ -481,7 +526,7 @@ } }, { - "id": "normalize-054", + "id": "normalize-059", "description": "removes the separator U+200E", "input": "EK\u200e01\u200eA03\u200eFK\u200e01", "options": {}, @@ -490,7 +535,7 @@ } }, { - "id": "normalize-055", + "id": "normalize-060", "description": "removes the separator U+200F", "input": "EK\u200f01\u200fA03\u200fFK\u200f01", "options": {}, @@ -499,7 +544,7 @@ } }, { - "id": "normalize-056", + "id": "normalize-061", "description": "removes the separator U+2010", "input": "EK\u201001\u2010A03\u2010FK\u201001", "options": {}, @@ -508,7 +553,7 @@ } }, { - "id": "normalize-057", + "id": "normalize-062", "description": "removes the separator U+2011", "input": "EK\u201101\u2011A03\u2011FK\u201101", "options": {}, @@ -517,7 +562,7 @@ } }, { - "id": "normalize-058", + "id": "normalize-063", "description": "removes the separator U+2012", "input": "EK\u201201\u2012A03\u2012FK\u201201", "options": {}, @@ -526,7 +571,7 @@ } }, { - "id": "normalize-059", + "id": "normalize-064", "description": "removes the separator U+2013", "input": "EK\u201301\u2013A03\u2013FK\u201301", "options": {}, @@ -535,7 +580,7 @@ } }, { - "id": "normalize-060", + "id": "normalize-065", "description": "removes the separator U+2014", "input": "EK\u201401\u2014A03\u2014FK\u201401", "options": {}, @@ -544,7 +589,7 @@ } }, { - "id": "normalize-061", + "id": "normalize-066", "description": "removes the separator U+2015", "input": "EK\u201501\u2015A03\u2015FK\u201501", "options": {}, @@ -553,7 +598,7 @@ } }, { - "id": "normalize-062", + "id": "normalize-067", "description": "removes the separator U+2028", "input": "EK\u202801\u2028A03\u2028FK\u202801", "options": {}, @@ -562,7 +607,7 @@ } }, { - "id": "normalize-063", + "id": "normalize-068", "description": "removes the separator U+2029", "input": "EK\u202901\u2029A03\u2029FK\u202901", "options": {}, @@ -571,7 +616,7 @@ } }, { - "id": "normalize-064", + "id": "normalize-069", "description": "removes the separator U+202F", "input": "EK\u202f01\u202fA03\u202fFK\u202f01", "options": {}, @@ -580,7 +625,7 @@ } }, { - "id": "normalize-065", + "id": "normalize-070", "description": "removes the separator U+205F", "input": "EK\u205f01\u205fA03\u205fFK\u205f01", "options": {}, @@ -589,7 +634,7 @@ } }, { - "id": "normalize-066", + "id": "normalize-071", "description": "removes the separator U+2060", "input": "EK\u206001\u2060A03\u2060FK\u206001", "options": {}, @@ -598,7 +643,7 @@ } }, { - "id": "normalize-067", + "id": "normalize-072", "description": "removes the separator U+2212", "input": "EK\u221201\u2212A03\u2212FK\u221201", "options": {}, @@ -607,7 +652,7 @@ } }, { - "id": "normalize-068", + "id": "normalize-073", "description": "removes the separator U+3000", "input": "EK\u300001\u3000A03\u3000FK\u300001", "options": {}, @@ -616,7 +661,7 @@ } }, { - "id": "normalize-069", + "id": "normalize-074", "description": "removes the separator U+FEFF", "input": "EK\ufeff01\ufeffA03\ufeffFK\ufeff01", "options": {}, diff --git a/vectors/parse-segments.json b/vectors/parse-segments.json index d932393..ea15662 100644 --- a/vectors/parse-segments.json +++ b/vectors/parse-segments.json @@ -271,6 +271,20 @@ }, { "id": "parse-segments-020", + "description": "rejects a digit before a letter in the area", + "input": "EK01A031K01", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "area", + "suggestion": "EK-01-A03-IK-01" + } + } + }, + { + "id": "parse-segments-021", "description": "suggests 0 for a letter O in the unit", "input": "EK01A03FKO1", "options": {}, @@ -284,7 +298,7 @@ } }, { - "id": "parse-segments-021", + "id": "parse-segments-022", "description": "suggests 1 for a letter I in the unit", "input": "EK01A03FKI2", "options": {}, @@ -298,7 +312,7 @@ } }, { - "id": "parse-segments-022", + "id": "parse-segments-023", "description": "suggests 1 for a letter L in the unit", "input": "EK01A03FKL1", "options": {}, @@ -312,7 +326,7 @@ } }, { - "id": "parse-segments-023", + "id": "parse-segments-024", "description": "suggests 1 for a letter I after a zero in the unit", "input": "EK01A03FK0I", "options": {}, @@ -326,7 +340,7 @@ } }, { - "id": "parse-segments-024", + "id": "parse-segments-025", "description": "gives no suggestion when the fix makes a unit of 00", "input": "EK01A03FK0O", "options": {}, @@ -340,7 +354,7 @@ } }, { - "id": "parse-segments-025", + "id": "parse-segments-026", "description": "gives no suggestion when a segment cannot be fixed", "input": "EK01A03F2O1", "options": {}, @@ -354,7 +368,7 @@ } }, { - "id": "parse-segments-026", + "id": "parse-segments-027", "description": "reports the first bad segment and fixes them all", "input": "EKO1A03F1O1", "options": {}, @@ -368,7 +382,7 @@ } }, { - "id": "parse-segments-027", + "id": "parse-segments-028", "description": "suggests a fix after making letters upper case", "input": "eko1a03fk01", "options": {}, @@ -382,7 +396,7 @@ } }, { - "id": "parse-segments-028", + "id": "parse-segments-029", "description": "rejects an LGA of 00 in a partial code", "input": "EK00", "options": { @@ -398,7 +412,7 @@ } }, { - "id": "parse-segments-029", + "id": "parse-segments-030", "description": "suggests a partial code", "input": "EKO1", "options": { @@ -414,7 +428,7 @@ } }, { - "id": "parse-segments-030", + "id": "parse-segments-031", "description": "suggests a partial area code", "input": "EK01A03F1", "options": { @@ -430,7 +444,7 @@ } }, { - "id": "parse-segments-031", + "id": "parse-segments-032", "description": "rejects an unknown state when partial codes are allowed", "input": "XX", "options": { @@ -446,7 +460,7 @@ } }, { - "id": "parse-segments-032", + "id": "parse-segments-033", "description": "suggests a state for a partial code", "input": "0G", "options": { @@ -462,7 +476,7 @@ } }, { - "id": "parse-segments-033", + "id": "parse-segments-034", "description": "suggests an LGA fix for a district code", "input": "EKO1A03", "options": { @@ -478,7 +492,7 @@ } }, { - "id": "parse-segments-034", + "id": "parse-segments-035", "description": "rejects an LGA of 00 in a district code", "input": "EK00A03", "options": { @@ -494,7 +508,7 @@ } }, { - "id": "parse-segments-035", + "id": "parse-segments-036", "description": "keeps a letter O in the district when it fixes the LGA", "input": "EKO1AO3FK01", "options": {}, @@ -508,7 +522,7 @@ } }, { - "id": "parse-segments-036", + "id": "parse-segments-037", "description": "keeps letters I and L in the district when it fixes the LGA", "input": "EKO1AILFK01", "options": {}, @@ -522,7 +536,7 @@ } }, { - "id": "parse-segments-037", + "id": "parse-segments-038", "description": "accepts letters and digits in any district position", "input": "EK01A0OFK01", "options": {}, diff --git a/vectors/precision-for-accuracy.json b/vectors/precision-for-accuracy.json index 49f845e..9e6a48f 100644 --- a/vectors/precision-for-accuracy.json +++ b/vectors/precision-for-accuracy.json @@ -104,6 +104,15 @@ }, { "id": "precision-for-accuracy-012", + "description": "gives the unit for negative zero", + "input": -0.0, + "options": {}, + "expect": { + "value": "unit" + } + }, + { + "id": "precision-for-accuracy-013", "description": "gives the LGA for NaN", "input": "NaN", "options": {}, @@ -112,7 +121,7 @@ } }, { - "id": "precision-for-accuracy-013", + "id": "precision-for-accuracy-014", "description": "gives the LGA for infinity", "input": "Infinity", "options": {}, @@ -121,7 +130,7 @@ } }, { - "id": "precision-for-accuracy-014", + "id": "precision-for-accuracy-015", "description": "gives the LGA for negative infinity", "input": "-Infinity", "options": {}, From 0119d83d3746f7a4348305ae94a61f7b2dfea7aa Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 12:43:18 +0100 Subject: [PATCH 31/45] docs: define the public interface and correct the grammar text The standards cited the spec's interface section, but the spec had none. The fields of a parse result appeared only in the vectors README. grammar.md now has an Interface section. It names each public function and constant, the fields of a parse result and of a parse error, and the rule that each segment after the precision is null. The standards and the PHP and TypeScript files point to it. The glossary gains the two terms that the section uses, parse result and parse error. Tests check that the section names each function and field that the vectors use. The grammar text also had faults. It said that the vectors test every rule, but no shared vector can test the Unicode version or text that is not well-formed. Each SDK tests these rules itself, and the text now says so. Its Unicode example, U+1CCDA, is already in Unicode 16.0, so it showed no 17.0 difference. U+A7F1 does. Unicode also freezes the NFKC form of an assigned character, so only an added character can differ. A test now rejects a vector character that the running Python leaves unassigned, so the oldest CI Python rejects a character that is newer than Unicode 14.0. The text also lacked the rule for a value in truncate that is not a precision, and a pointer from the count of code points to the rule for broken text. A test now ties the version in the text to the VERSION file. The standards pointed to terms that the spec never listed, and to two documents that the repo does not hold. They now give NIPOST's terms and acceptable use policy with their addresses. They say that the status map and the WordPress floors are not defined yet, and when they will be. Signed-off-by: MobileMage --- CONTEXT.md | 8 ++++++ grammar.md | 39 ++++++++++++++++++++++++++--- scripts/tests/test_build_vectors.py | 24 ++++++++++++++++++ scripts/tests/test_data.py | 5 ++++ scripts/tests/test_reference.py | 27 ++++++++++++++++++++ standards/CODING_STANDARDS.md | 8 +++--- standards/languages/php.md | 4 +-- standards/languages/typescript.md | 2 +- vectors/README.md | 2 ++ 9 files changed, 108 insertions(+), 11 deletions(-) diff --git a/CONTEXT.md b/CONTEXT.md index a08e6e1..64a0f59 100644 --- a/CONTEXT.md +++ b/CONTEXT.md @@ -77,6 +77,14 @@ _Avoid_: max length, length limit, size limit The canonical form that `parse` offers with `unknown_state` or `bad_segment`, when a fix of look-alike characters makes the code parse. It is a hint, never a success. The word means this hint from `parse`. It does not mean the suggestions that NIPOST's autocomplete API returns. _Avoid_: correction, autocorrect +**Parse result**: +What `parse` returns: a postcode, or a parse error. The Interface section of `grammar.md` defines its fields. +_Avoid_: outcome, response + +**Parse error**: +The reason that `parse` rejected its input. It has a code, the failing segment and a suggestion. It is a value that `parse` returns, not an exception. +_Avoid_: validation error, exception + ### NIPOST's API **Gateway**: diff --git a/grammar.md b/grammar.md index 4c12943..63d2927 100644 --- a/grammar.md +++ b/grammar.md @@ -1,6 +1,6 @@ # Postcode grammar -Spec version 0.1.0. This file defines how every Gatepost SDK reads and writes Nigeria's digital postcodes. The values live in `data/`. The vectors in `vectors/` test every rule. If this file and a vector disagree, this file wins. Such a disagreement is a bug in the spec. +Spec version 0.1.0. This file defines how every Gatepost SDK reads and writes Nigeria's digital postcodes. The values live in `data/`. The vectors in `vectors/` test every rule, with two exceptions. Two groups of rules have no shared vectors: the Unicode version (see Normalise) and text that is not well-formed (see Parse). JSON cannot carry invalid UTF-8, and PHP rejects an escape for a lone surrogate. Each SDK tests these rules itself. If this file and a vector disagree, this file wins. Such a disagreement is a bug in the spec. ## Forms @@ -32,7 +32,7 @@ A partial postcode stops after the state, LGA, district or area segment. Its len An SDK without a full NFKC can apply NFKC through a table. The table lists only the characters whose NFKC form holds only ASCII characters and separators. The SDK's `parse` and `isLegacy` must then give the same results as with full NFKC, for every input. The SDK's `normalize` can keep a character that full NFKC changes, such as U+00B5, in text that `parse` rejects. Each `normalize` vector gives the same result either way. -Unicode 17.0 is the baseline for NFKC. A table must come from that version. A platform can have a Unicode version older than 17.0. An SDK on that platform can give other results for a character that a later version added or changed. For example, NFKC in Unicode 17.0 changes U+1CCDA to E. No vector holds such a character. +Unicode 17.0 is the baseline for NFKC. A table must come from that version. A platform can have a Unicode version older than 17.0. An SDK on that platform can give other results for a character that a later version added. Unicode's stability policy freezes the NFKC form of an assigned character, so only an added character can differ. For example, NFKC in Unicode 17.0 changes U+A7F1 to S, but a platform with Unicode 16.0 or older keeps it. No vector holds such a character. A format character that is not in the list stays. For example, the right-to-left override U+202E can change the order in which a code shows on screen, so `parse` gives `bad_character` for it. @@ -40,7 +40,7 @@ A format character that is not in the list stays. For example, the right-to-left ## Parse -First count the Unicode code points of the input. Do not count UTF-16 units, UTF-8 bytes or grapheme clusters. A lone surrogate, in a language that can hold one, counts as one code point. An SDK can stop counting at 65. If the input has more than `maxInputCodePoints` code points (64 in `data/format.json`), the error code is `bad_length`, and `parse` does not normalise the input. The limit bounds the cost of NFKC, which can take seconds on a long run of combining marks. Otherwise, normalise the input. Then apply these checks in order. The first check that fails gives the error code. +First count the Unicode code points of the input. Do not count UTF-16 units, UTF-8 bytes or grapheme clusters. A lone surrogate, in a language that can hold one, counts as one code point. The last paragraph of this section gives the rule for input that is not well-formed. An SDK can stop counting at 65. If the input has more than `maxInputCodePoints` code points (64 in `data/format.json`), the error code is `bad_length`, and `parse` does not normalise the input. The limit bounds the cost of NFKC, which can take seconds on a long run of combining marks. Otherwise, normalise the input. Then apply these checks in order. The first check that fails gives the error code. 1. `empty`: no character is left. 2. `legacy_code`: exactly 6 ASCII digits are left (`legacyPattern` in `data/format.json`). These are old NIPOST postcodes. @@ -69,7 +69,7 @@ Parse the corrected code with the same options. If it parses, the suggestion is ## Hierarchy -- `truncate(code, to)` keeps the segments up to `to`. It fails when `to` is more precise than the code. Each language uses its standard error for a programmer mistake, for example `RangeError` in TypeScript. +- `truncate(code, to)` keeps the segments up to `to`. It fails when `to` is more precise than the code, and when `to` is not a precision. Both are programmer mistakes. Each language uses its standard error for them, for example `RangeError` in TypeScript. - `parent(code)` returns the code with one segment fewer. A postcode with only the state segment has no parent, so `parent` returns null for it. - `contains(prefix, code)` is true when the code has every segment of the prefix, in the same places. - `redact(code)` replaces the unit with `**` in the canonical form. A code without a unit stays the same. @@ -85,3 +85,34 @@ Parse the corrected code with the same options. If it parses, the suggestion is ## Versions `VERSION` holds the spec version. Each SDK exposes it as `SPEC_VERSION`. Each vector file carries a format version, which is 1. + +## Interface + +An SDK exposes the symbols in this table and the types that they need. A language file gives the idiomatic form of each name and lists the additions that its language needs, such as an options type. The function names are the names in the vector files. This section covers the core. The interface of the client comes later, with the completed OpenAPI file. + +| Symbol | Takes | Returns | Rules | +|---|---|---|---| +| `normalize(text)` | text | text | Normalise | +| `parse(text, allowPartial)` | text, and `allowPartial`, which is false by default | a parse result | Parse | +| `isLegacy(text)` | text | true or false | Legacy postcodes | +| `truncate(code, to)` | a postcode and a precision | a postcode | Hierarchy | +| `parent(code)` | a postcode | a postcode, or null | Hierarchy | +| `contains(prefix, code)` | two postcodes | true or false | Hierarchy | +| `redact(code)` | a postcode | text | Hierarchy | +| `stateName(code)` | text | the name of a state, or null | State names | +| `precisionForAccuracy(metres)` | a number, or no value | a precision | GPS precision | +| `SPEC_VERSION` (a constant) | none | the spec version, as text | Versions | + +A parse result is a success or a failure. A success holds a postcode. A failure holds a parse error. A language can shape the result in the way that it expects, if it keeps the fields below. In the vectors, a success has the field `ok` with the value true, beside the fields of the postcode. A failure has `ok` with the value false, and an `error` field that holds the parse error. + +A postcode that `parse` accepted has these fields: + +- `compact`, `canonical` and `display`: the three forms of the code. +- `precision`: the last segment that the code contains. It is one of `state`, `lga`, `district`, `area` and `unit`. +- `segments`: the text of each of `state`, `lga`, `district`, `area` and `unit`. Each segment after the precision is null. For example, the segments of `EK-01` are `EK` and `01`, and null for the district, the area and the unit. + +A parse error has these fields: + +- `code`: one of `empty`, `legacy_code`, `bad_character`, `bad_length`, `unknown_state` and `bad_segment`. +- `segment`: the name of the failing segment for `unknown_state` and `bad_segment`, and null for the other codes. +- `suggestion`: the canonical form of one corrected code, or null. diff --git a/scripts/tests/test_build_vectors.py b/scripts/tests/test_build_vectors.py index d5b09bd..d2c33c8 100644 --- a/scripts/tests/test_build_vectors.py +++ b/scripts/tests/test_build_vectors.py @@ -31,6 +31,15 @@ def load(stem: str) -> dict[str, Any]: return value +def input_strings(value: Any) -> list[str]: + """Return each string in a vector input: the input itself, or the strings in an object.""" + if isinstance(value, str): + return [value] + if isinstance(value, dict): + return [text for item in value.values() for text in input_strings(item)] + return [] + + def utf16_units(text: str) -> int: return len(text.encode("utf-16-le", errors="surrogatepass")) // 2 @@ -94,6 +103,21 @@ def test_each_case_has_a_unique_id_and_a_description(self) -> None: self.assertEqual(len(ids), len(set(ids))) self.assertGreaterEqual(len(ids), 200) + def test_each_input_character_is_assigned_in_the_running_unicode_version(self) -> None: + # NFKC differs between Unicode versions only for a character that a later version added. + # CI runs the oldest Python that the scripts support, so it rejects a newer character. + unassigned = sorted( + { + f"U+{ord(character):04X}" + for stem in build_vectors.FILES + for case in load(stem)["cases"] + for text in input_strings(case["input"]) + for character in text + if unicodedata.category(character) == "Cn" + } + ) + self.assertEqual(unassigned, []) + def test_each_description_is_unique_across_all_vector_files(self) -> None: counts = Counter( case["description"] for stem in build_vectors.FILES for case in load(stem)["cases"] diff --git a/scripts/tests/test_data.py b/scripts/tests/test_data.py index a88ae33..78cfbbc 100644 --- a/scripts/tests/test_data.py +++ b/scripts/tests/test_data.py @@ -184,3 +184,8 @@ def test_the_fixes_match_the_suggestions(self) -> None: with self.subTest(kind=kind): line = next((line for line in lines if line.startswith(lead_in)), "") self.assertEqual(dict(re.findall(r"`(.)` to `(.)`", line)), suggestions[kind]) + + def test_the_version_matches_the_version_file(self) -> None: + version = (ROOT / "VERSION").read_text(encoding="utf-8").strip() + stated = re.search(r"^Spec version (\S+?)\. ", grammar_text(), flags=re.MULTILINE) + self.assertEqual(stated and stated.group(1), version) diff --git a/scripts/tests/test_reference.py b/scripts/tests/test_reference.py index c43a6d3..dd2260f 100644 --- a/scripts/tests/test_reference.py +++ b/scripts/tests/test_reference.py @@ -11,6 +11,8 @@ import reference +from tests.test_data import grammar_section + ROOT = Path(__file__).resolve().parents[2] CODE = "EK01A03FK01" @@ -130,3 +132,28 @@ def test_truncate_rejects_a_value_that_is_not_a_precision(self) -> None: def test_truncate_rejects_a_precision_that_the_code_lacks(self) -> None: with self.assertRaises(ValueError): reference.load().truncate(("EK", "01", "A03"), "area") + + +class InterfaceTest(unittest.TestCase): + def test_the_interface_names_each_function_that_a_vector_file_tests(self) -> None: + interface = grammar_section("Interface") + functions = {document["function"] for _, document in vector_documents()} + missing = sorted(name for name in functions if f"`{name}(" not in interface) + self.assertEqual(missing, []) + self.assertIn("`SPEC_VERSION`", interface) + + def test_the_interface_names_each_field_of_a_parse_result(self) -> None: + interface = grammar_section("Interface") + expected = [ + case["expect"] + for _, document in vector_documents() + if document["function"] == "parse" + for case in document["cases"] + ] + fields = {field for outcome in expected for field in outcome} + fields |= { + field for outcome in expected if "error" in outcome for field in outcome["error"] + } + fields |= {field for outcome in expected if outcome["ok"] for field in outcome["segments"]} + self.assertEqual(sorted(field for field in fields if f"`{field}`" not in interface), []) + self.assertIn("after the precision is null", interface) diff --git a/standards/CODING_STANDARDS.md b/standards/CODING_STANDARDS.md index 6b3e69e..13a23f9 100644 --- a/standards/CODING_STANDARDS.md +++ b/standards/CODING_STANDARDS.md @@ -11,7 +11,7 @@ These standards apply to every Gatepost repo, in every language. Each repo also - A **SHOULD** rule can be skipped only with a written reason in the pull request. - A rule marked **(tool)** is enforced by CI. Reviewers do not check it by hand. - A language file can change a SHOULD rule for its language. It cannot change a MUST rule. -- The spec's interface section, vectors and contract scenarios rank above this document. This document ranks above the language files. If two of them disagree, the higher one wins. Then fix the lower one. +- `spec/grammar.md` (with its Interface section), `spec/data/`, `spec/vectors/` and the contract scenarios in `spec/contract/` rank above this document. `spec/grammar.md` wins over a vector. This document ranks above the language files. If two of them disagree, the higher one wins. Then fix the lower one. - To change a rule, open a pull request on the `spec` repo. Increase the version at the top of this file. ## Principles @@ -24,11 +24,11 @@ The rules come from these seven principles. If no rule covers a case, apply the 4. **Same behaviour, native feel.** Every SDK behaves the same. Every SDK reads like a well-made library in its own language. 5. **Boring is good.** Prefer the standard library, plain data and well-known tools. 6. **Private by default.** Send nothing the caller did not ask for. Keep nothing longer than necessary. -7. **Respect NIPOST's rules.** Follow the terms and the acceptable use policy that the spec lists. +7. **Respect NIPOST's rules.** Follow NIPOST's Terms and Conditions of Use (https://postcode.gov.ng/terms) and its Acceptable Use Policy (https://postcode.gov.ng/acceptable-use). ## Public interface -- **API-1 MUST.** Every public symbol maps to a concept in the spec's interface section. A language file lists the idiomatic additions that its language needs, such as options types, async variants and callback types. To add, rename or remove a concept, change the spec first. To add an idiomatic addition, change the language file first. +- **API-1 MUST.** Every public symbol maps to a concept in the Interface section of `spec/grammar.md`. A language file lists the idiomatic additions that its language needs, such as options types, async variants and callback types. To add, rename or remove a concept, change the spec first. To add an idiomatic addition, change the language file first. - **API-2 MUST.** Names and call shapes follow the cross-language map below. - **API-3 MUST.** Parse, don't validate. A function that reads a postcode returns a typed result, not a boolean. A boolean helper can exist only as a thin wrapper over `parse`. - **API-4 MUST.** Expected failures are values. `parse` returns its errors. Only I/O failures, API failures and programmer errors raise or throw. @@ -127,7 +127,7 @@ A line in a code or config file can skip one check with a comment that names the ## Errors and logging -- **ERR-1 MUST.** Each HTTP status maps to exactly one error code. The client sub-plan holds the map. +- **ERR-1 MUST.** Each HTTP status maps to exactly one error code. The spec does not define the map yet. It will define the map with the contract scenarios, before the first client release. - **ERR-2 MUST.** A library does not log. It returns or raises errors. A caller can attach a debug hook. - **ERR-3 MUST.** No error, log or exception message contains an API key, a NIN, an email address, or a full postcode next to personal data. Use `redact()` for postcodes. - **ERR-4 MUST.** An error message says what happened and what to do, in plain English. Example: "The postcode has 10 characters. A postcode has 11 characters." diff --git a/standards/languages/php.md b/standards/languages/php.md index affe314..1d13801 100644 --- a/standards/languages/php.md +++ b/standards/languages/php.md @@ -9,7 +9,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for PHP - PHP 8.1 or later. CI tests PHP 8.1 and the newest PHP release. - PHP 8.1 reached end of life on 31 Dec 2025, and PHP 8.2 reaches it on 31 Dec 2026. The 8.1 floor serves WordPress stores on older hosting. Under VER-1, ADR 0001 must record this trade-off with current WordPress usage data before the first release. If the data does not support 8.1, raise the floor. - PHPUnit 11 and later need PHP 8.2. CI runs PHPUnit 10 on PHP 8.1 and the newest PHPUnit on newer PHP versions. -- WordPress and WooCommerce: the plugin sub-plan sets the floors after it tests the Additional Checkout Fields API. +- WordPress and WooCommerce: the floors are not defined yet. The team sets them after it tests the Additional Checkout Fields API, and before the first release of a plugin. ## Part A: the SDK @@ -55,7 +55,7 @@ PHP-CS-Fixer does not limit line length, so `check-tells` enforces TELL-1. PHPMD ### Idiomatic additions (API-1) -The PHP SDK follows the spec's interface section. Its client constructor takes PSR-18, PSR-17 and PSR-16 objects. These are parameters, not new public symbols. +The PHP SDK follows the Interface section of `spec/grammar.md`. Its client constructor takes PSR-18, PSR-17 and PSR-16 objects. These are parameters, not new public symbols. ### Names diff --git a/standards/languages/typescript.md b/standards/languages/typescript.md index 1dad8a9..a2380f7 100644 --- a/standards/languages/typescript.md +++ b/standards/languages/typescript.md @@ -119,7 +119,7 @@ Prettier's `printWidth: 100` matches TELL-1. Prettier does not split long string ### Idiomatic additions (API-1) -The TypeScript SDK follows the spec's interface section exactly. It adds no idiomatic symbols. +The TypeScript SDK follows the Interface section of `spec/grammar.md` exactly. It adds no idiomatic symbols. ## Names diff --git a/vectors/README.md b/vectors/README.md index d0d0ea9..303a9da 100644 --- a/vectors/README.md +++ b/vectors/README.md @@ -29,6 +29,8 @@ Each case has these fields: | `truncate` | `{ "canonical": ... }`, or `{ "rejects": true }` when the call must fail with the language's programmer error | | `parent` | `{ "canonical": ... }`, where null means no parent | +The Interface section of `grammar.md` defines the fields of a parse result and of a parse error. + ## Runner rules - For `truncate`, `parent`, `contains` and `redact`, the inputs are canonical codes. Parse them with `allowPartial` before the call. From 06128824c9a10a8aafbb7aabfe3f458363659e7b Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 12:53:55 +0100 Subject: [PATCH 32/45] fix: make check-tells check what the standards say Most rules of check-tells had no test. A run of one-line changes to the script showed that 24 of 33 hand-written changes passed the tests. The tests now pin each emoji range, limit, name, folder, extension, debug call, skippable rule and marker. A run of 328 one-line changes leaves no survivor. The standards say that check-tells rejects emojis everywhere, and that it checks line length, debug calls and to-do comments in code and config files. The tool did less. It gave no check to phpunit.xml.dist, phpstan.neon and infection.json5, and none to style and markup files such as .css and .html. It gave no emoji check to files such as .txt and .lock. It accepted a to-do in lower case and a fix-me without an issue link. All of these now get the same checks as the files next to them. A code or config file that was not UTF-8 passed without a word, and a bad root or path ended in a traceback. check-tells now reports such a file as a TELL-14 break. It answers a bad root, a missing path or a path outside the root with one line and exit code 2. A variable that held a path object now says what it holds. The lockfile list no longer repeats the names that the .lock suffix already covers. Signed-off-by: MobileMage --- scripts/check_tells.py | 177 +++++-- scripts/tests/test_check_tells.py | 777 +++++++++++++++++++++++++++++- standards/CODING_STANDARDS.md | 18 +- 3 files changed, 918 insertions(+), 54 deletions(-) diff --git a/scripts/check_tells.py b/scripts/check_tells.py index a5fec3f..d091584 100644 --- a/scripts/check_tells.py +++ b/scripts/check_tells.py @@ -16,6 +16,9 @@ from pathlib import Path, PurePosixPath from typing import Literal +# Code and config files get the strict rules. The other kinds can use any script, so they get +# only the emoji check: prose (Markdown), catalogue (translations), exempt (lockfiles, data and +# images) and other (a file type that no rule names). Kind = Literal["code", "config", "prose", "catalogue", "exempt", "other"] NumberedLine = tuple[int, str] @@ -23,6 +26,7 @@ TAB_WIDTH = 4 MAX_SUBJECT = 72 LAST_ASCII = 0x7F +EXIT_BAD_INPUT = 2 GENERATED_MARKERS = ("@generated", "DO NOT EDIT", ">> and ...) start the examples in Python docstrings. DOCTEST_LINE = re.compile(r"^\s*(>>>|\.\.\.)") URL = re.compile(r"https?://") -TODO_FORM = re.compile(r"\bTODO\b(?!\(#\d+\))") +# The words are split, so that this line does not match its own pattern. +TODO_FORM = re.compile(r"\b(?:TO" r"DO|FIX" r"ME)\b(?!\(#\d+\))", re.IGNORECASE) SKIP = re.compile(r"check-tells: allow ([A-Z]+-\d+)(?: because (\S.*))?") EMOJI_RANGES = ( (0x1F000, 0x1FAFF), @@ -170,6 +192,10 @@ def __str__(self) -> str: return f"{self.path}:{self.line}:{self.column}: {self.rule} {self.message}" +class InputError(Exception): + """The command line names something that check-tells cannot read.""" + + @dataclass(frozen=True) class FileInfo: """What a file holds, which decides the rules that apply to it.""" @@ -192,41 +218,46 @@ def _split_lines(text: str) -> list[str]: def classify(path: str, text: str) -> FileInfo: """Decide the kind and the language of one file.""" - pure = PurePosixPath(path) + repo_path = _without_dist(PurePosixPath(path)) head = "\n".join(_split_lines(text)[:5]) return FileInfo( path=path, - kind=_kind(pure, text), - language=_language(pure, text), + kind=_kind(repo_path, text), + language=_language(repo_path, text), is_test=bool(TEST_PATH.search(path)), is_generated=any(marker in head for marker in GENERATED_MARKERS), ) -def _kind(pure: PurePosixPath, text: str) -> Kind: - suffix = pure.suffix.lower() - if pure.name in LOCKFILES or suffix in EXEMPT_EXTENSIONS: +def _without_dist(repo_path: PurePosixPath) -> PurePosixPath: + """Read a .dist file as the file that it copies, such as phpunit.xml.dist as phpunit.xml.""" + return repo_path.with_suffix("") if repo_path.suffix.lower() == ".dist" else repo_path + + +def _kind(repo_path: PurePosixPath, text: str) -> Kind: + suffix = repo_path.suffix.lower() + if repo_path.name in LOCKFILES or suffix in EXEMPT_EXTENSIONS: return "exempt" if suffix in CODE_EXTENSIONS or (not suffix and text.startswith("#!")): return "code" - if _is_catalogue(pure): + if _is_catalogue(repo_path): return "catalogue" if suffix in PROSE_EXTENSIONS: return "prose" - if suffix in CONFIG_EXTENSIONS or pure.name in CONFIG_NAMES: + if suffix in CONFIG_EXTENSIONS or repo_path.name in CONFIG_NAMES: return "config" return "other" -def _is_catalogue(pure: PurePosixPath) -> bool: - in_folder = any(part in CATALOGUE_FOLDERS for part in pure.parts[:-1]) - android = pure.name == "strings.xml" and pure.parent.name.startswith("values") - return in_folder or android or pure.suffix.lower() in CATALOGUE_EXTENSIONS +def _is_catalogue(repo_path: PurePosixPath) -> bool: + in_folder = any(part in CATALOGUE_FOLDERS for part in repo_path.parts[:-1]) + android = repo_path.name == "strings.xml" and repo_path.parent.name.startswith("values") + return in_folder or android or repo_path.suffix.lower() in CATALOGUE_EXTENSIONS -def _language(pure: PurePosixPath, text: str) -> str | None: - if pure.suffix: - return LANGUAGE_BY_EXTENSION.get(pure.suffix.lower()) +def _language(repo_path: PurePosixPath, text: str) -> str | None: + if repo_path.suffix: + return LANGUAGE_BY_EXTENSION.get(repo_path.suffix.lower()) first_line = _split_lines(text)[0] if not first_line.startswith("#!"): return None @@ -295,17 +326,22 @@ def _line_length(info: FileInfo, number: int, line: str) -> list[Violation]: def _is_data_json(path: str) -> bool: - pure = PurePosixPath(path) - in_folder = any(part in DATA_FOLDERS for part in pure.parts[:-1]) - return in_folder and pure.suffix.lower() == ".json" + repo_path = PurePosixPath(path) + in_folder = any(part in DATA_FOLDERS for part in repo_path.parts[:-1]) + return in_folder and repo_path.suffix.lower() == ".json" def _todo_form(info: FileInfo, number: int, line: str) -> list[Violation]: if info.kind not in ("code", "config"): return [] - message = "Link an issue in the form TODO(#123)." return [ - Violation(info.path, number, match.start() + 1, "CS-6", message) + Violation( + info.path, + number, + match.start() + 1, + "CS-6", + f"Link an issue in the form {match.group(0).upper()}(#123).", + ) for match in TODO_FORM.finditer(line) ] @@ -328,8 +364,6 @@ def _debug_calls(info: FileInfo, number: int, line: str) -> list[Violation]: def _characters(info: FileInfo, number: int, line: str) -> list[Violation]: - if info.kind == "exempt": - return [] found = [ Violation(info.path, number, index + 1, "TELL-14", f"Remove the emoji U+{ord(char):04X}.") for index, char in enumerate(line) @@ -400,37 +434,92 @@ def tracked_files(root: Path) -> list[Path]: "--others", "--exclude-standard", ] - names = subprocess.run(command, check=True, capture_output=True).stdout.decode().split("\0") + try: + output = subprocess.run(command, check=True, capture_output=True).stdout + except subprocess.CalledProcessError as error: + reason = " ".join(error.stderr.decode(errors="replace").split()) + raise InputError(f"Cannot list the files in {root}. Git says: {reason}") from error + names = output.decode().split("\0") return [root / name for name in names if name and (root / name).is_file()] +def _read(path: Path) -> bytes: + try: + return path.read_bytes() + except OSError as error: + raise InputError(f"Cannot read {path}: {error.strerror}.") from error + + +def _not_utf8(path: str, data: bytes, error: UnicodeDecodeError, advice: str) -> Violation: + """Point at the first byte that is not UTF-8. All the bytes before it are valid.""" + line_start = data.rfind(b"\n", 0, error.start) + 1 + line = data.count(b"\n", 0, error.start) + 1 + column = len(data[line_start : error.start].decode("utf-8")) + 1 + message = f"Byte 0x{data[error.start]:02X} is not valid UTF-8. {advice}" + return Violation(path, line, column, "TELL-14", message) + + +def _check_undecodable(path: str, data: bytes, error: UnicodeDecodeError) -> list[Violation]: + """Report a code or config file that is not UTF-8. Other files can be binary, so skip them.""" + # Only the suffix and the first bytes decide the kind, and bad bytes would stop the decoder. + kind = classify(path, data[:512].decode("utf-8", errors="replace")).kind + if kind not in ("code", "config"): + return [] + advice = "Save the file as UTF-8, and use ASCII in code and config." + return [_not_utf8(path, data, error, advice)] + + def check_files(root: Path, files: list[Path]) -> list[Violation]: - """Check each file. Skip files that are not UTF-8 text, because they have no lines.""" + """Check each file. Report a code or config file that is not UTF-8, and skip other files.""" violations: list[Violation] = [] for file in files: + if not file.is_relative_to(root): + raise InputError(f"{file} is outside the root {root}.") + path = file.relative_to(root).as_posix() + data = _read(file) try: - text = file.read_text(encoding="utf-8") - except UnicodeDecodeError: - continue - violations.extend(check_text(file.relative_to(root).as_posix(), text)) + text = data.decode("utf-8") + except UnicodeDecodeError as error: + violations.extend(_check_undecodable(path, data, error)) + else: + violations.extend(check_text(path, text)) return violations +def _check_commit_message_file(path: Path) -> list[Violation]: + data = _read(path) + try: + text = data.decode("utf-8") + except UnicodeDecodeError as error: + advice = "Save the message as UTF-8, and use ASCII." + return [_not_utf8("commit message", data, error, advice)] + return check_commit_message(text) + + +def _run(args: argparse.Namespace) -> list[Violation]: + if args.commit_msg is not None: + return _check_commit_message_file(args.commit_msg) + root = args.root.resolve() + files = [(root / path).resolve() for path in args.paths] or tracked_files(root) + return check_files(root, files) + + def main(argv: list[str] | None = None) -> int: - """Run the checks, print each violation and return 1 if any rule broke.""" + """Run the checks and print each violation. + + Return 0 if all is well, 1 if a rule broke and 2 if the input cannot be read. + """ parser = argparse.ArgumentParser( prog="check-tells", description="Find the agent tells that language tools miss." ) parser.add_argument("--root", type=Path, default=Path.cwd(), help="The repo root.") parser.add_argument("--commit-msg", type=Path, help="Check this commit message file.") parser.add_argument("paths", nargs="*", type=Path, help="Files to check. Default: all.") - args = parser.parse_args(argv) - if args.commit_msg is not None: - violations = check_commit_message(args.commit_msg.read_text(encoding="utf-8")) - else: - root = args.root.resolve() - files = [(root / path).resolve() for path in args.paths] or tracked_files(root) - violations = check_files(root, files) + try: + violations = _run(parser.parse_args(argv)) + except InputError as error: + sys.stderr.write(f"check-tells: {error}\n") + return EXIT_BAD_INPUT for violation in sorted(violations): sys.stdout.write(f"{violation}\n") return 1 if violations else 0 diff --git a/scripts/tests/test_check_tells.py b/scripts/tests/test_check_tells.py index 798f705..453b94c 100644 --- a/scripts/tests/test_check_tells.py +++ b/scripts/tests/test_check_tells.py @@ -4,13 +4,16 @@ import contextlib import io +import os import subprocess import tempfile import unittest +from collections.abc import Iterator from pathlib import Path +from unittest import mock import check_tells -from check_tells import Violation, check_commit_message, check_text +from check_tells import Violation, check_commit_message, check_text, classify EMOJI = chr(0x1F600) NO_BREAK_SPACE = chr(0x00A0) @@ -18,16 +21,294 @@ EM_DASH = chr(0x2014) LINE_SEPARATOR = chr(0x2028) O_DOT_BELOW = chr(0x1ECD) +LONG_LINE = "x" * 150 # Built from two parts, so that this file passes its own to-do check. TO_DO = "TO" + "DO" +FIX_ME = "FIX" + "ME" # Built from two parts, so that this file passes its own skip check. SKIP_MARKER = "check-tells: " + "allow" +# The ranges are typed out here, not read from check_tells, so that a change to them fails. +EMOJI_RANGES = ( + (0x1F000, 0x1FAFF), + (0x231A, 0x231B), + (0x23E9, 0x23F3), + (0x2600, 0x27BF), + (0x2B00, 0x2BFF), + (0xFE0F, 0xFE0F), +) +# The debug calls that each language file lists under TELL-13, one sample line for each call. +DEBUG_LINES = { + "src/a.ts": ["console.log(x);", "console.debug(x);", "console.dir(x);", "debugger;"], + "src/a.php": [ + "var_dump($x);", + "print_r($x);", + "var_export($x);", + "dd($x);", + "dump($x);", + "error_log($x);", + "debug_print_backtrace();", + ], + "pkg/a.py": ["print(x)", "pprint(x)", "breakpoint()", "pdb.set_trace()"], + "a.go": [ + "fmt.Print(x)", + "fmt.Printf(x)", + "fmt.Println(x)", + "log.Print(x)", + "log.Printf(x)", + "log.Println(x)", + "log.Fatal(x)", + "log.Fatalf(x)", + "log.Fatalln(x)", + "log.Panic(x)", + "log.Panicf(x)", + "log.Panicln(x)", + "print(x)", + "println(x)", + ], + "src/A.kt": [ + "println(x)", + "print(x)", + "System.out.println(x)", + "System.err.println(x)", + "error.printStackTrace()", + "Log.v(TAG, x)", + "Log.d(TAG, x)", + "Log.i(TAG, x)", + "Log.w(TAG, x)", + "Log.e(TAG, x)", + "Log.wtf(TAG, x)", + ], + "src/A.java": ["System.out.println(x);", "System.err.println(x);", "error.printStackTrace();"], + "src/A.cs": [ + "Console.Write(x);", + "Console.WriteLine(x);", + "Debug.WriteLine(x);", + "Trace.WriteLine(x);", + ], + "lib/a.dart": ["print(x);", "debugPrint(x);"], + "Sources/A.swift": ["print(x)", "debugPrint(x)", "dump(x)", "NSLog(x)"], +} +# One debug call for each file extension that has a language. +DEBUG_LINE_BY_EXTENSION = { + ".ts": "console.log(x);", + ".tsx": "console.log(x);", + ".mts": "console.log(x);", + ".cts": "console.log(x);", + ".js": "console.log(x);", + ".jsx": "console.log(x);", + ".mjs": "console.log(x);", + ".cjs": "console.log(x);", + ".vue": "console.log(x);", + ".svelte": "console.log(x);", + ".astro": "console.log(x);", + ".php": "var_dump($x);", + ".py": "print(x)", + ".go": "fmt.Println(x)", + ".kt": "println(x)", + ".kts": "println(x)", + ".java": "System.out.println(x);", + ".cs": "Console.WriteLine(x);", + ".dart": "debugPrint(x);", + ".swift": "NSLog(x)", +} +# Calls that look like a debug call but are not one. +HARMLESS_LINES = { + "pkg/a.py": ["report.print(x)", "pprint_table(x)", "self.breakpoint(x)", "sprint(x)"], + "src/a.ts": ["myconsole.log(x);", "console.logs(x);", "console.warn(x);"], + "src/a.php": ["$this->dump($x);", "Debug::dump($x);", "my_dump($x);", "$dd($x);"], + "a.go": ["report.Print(x)", "sprint(x)", "s.println(x)"], + "src/A.kt": ["report.print(x)", "logPrint(x)"], + "src/A.cs": ["Logger.WriteLine(x);"], + "lib/a.dart": ["report.print(x);", "sprint(x);"], + "Sources/A.swift": ["report.print(x)", "sprint(x)"], +} +# Every file name that check-tells must read as a test file, each with the language's debug call. +TEST_FILES = { + "test/a.ts": "console.log(x);", + "tests/a.ts": "console.log(x);", + "src/__tests__/a.ts": "console.log(x);", + "src/testdata/a.ts": "console.log(x);", + "testdata/a.go": "fmt.Println(x)", + "pkg/testdata/a.go": "fmt.Println(x)", + "Tests/A.cs": "Console.WriteLine(x);", + "a.test.ts": "console.log(x);", + "a.spec.ts": "console.log(x);", + "a.test.tsx": "console.log(x);", + "a.test.js": "console.log(x);", + "a.test.jsx": "console.log(x);", + "a.test.mjs": "console.log(x);", + "a.test.cjs": "console.log(x);", + "a.test.mts": "console.log(x);", + "a.test.cts": "console.log(x);", + "a_test.go": "fmt.Println(x)", + "a_test.dart": "debugPrint(x);", + "test_a.py": "print(x)", + "src/ATest.kt": "println(x)", + "src/ATests.kt": "println(x)", + "src/ATest.java": "System.out.println(x);", + "src/ATests.java": "System.out.println(x);", + "src/ATest.cs": "Console.WriteLine(x);", + "src/ATests.cs": "Console.WriteLine(x);", + "Sources/ATest.swift": "dump(x)", + "Sources/ATests.swift": "dump(x)", +} +# File names that look like test files but are not. +NON_TEST_FILES = { + "src/latest/a.ts": "console.log(x);", + "src/contest.ts": "console.log(x);", + "src/a_test.ts": "console.log(x);", + "src/a.testing.ts": "console.log(x);", + "src/Latest.kt": "println(x)", + "src/test.py": "print(x)", + "src/testdata.py": "print(x)", +} +# A line that breaks each rule that a skip comment can skip. +SKIP_SAMPLES = { + "TELL-1": ("src/a.ts", LONG_LINE), + "TELL-13": ("src/a.ts", "console.log(x);"), + "TELL-14": ("src/a.ts", f"const a = '{NO_BREAK_SPACE}';"), + "CS-6": ("src/a.ts", f"// {TO_DO}: x"), +} +KIND_BY_PATH = { + "code": [ + "a.ts", + "a.tsx", + "a.mts", + "a.cts", + "a.js", + "a.jsx", + "a.mjs", + "a.cjs", + "a.php", + "a.py", + "a.go", + "a.kt", + "a.kts", + "a.java", + "a.cs", + "a.dart", + "a.swift", + "a.sql", + "a.sh", + "a.bash", + "a.css", + "a.scss", + "a.html", + "a.astro", + "a.vue", + "a.svelte", + "A.TS", + "config.php.dist", + ], + "config": [ + "a.json", + "a.yml", + "a.yaml", + "a.toml", + "a.xml", + "a.ini", + "a.cfg", + "a.properties", + "a.gradle", + "a.neon", + "a.json5", + "a.jsonc", + "A.JSON", + "phpunit.xml.dist", + "phpstan.neon.dist", + "phpstan.dist.neon", + "Makefile", + "Dockerfile", + ".editorconfig", + ".gitignore", + ".gitattributes", + ".npmrc", + ], + "prose": ["a.md", "a.mdx", "A.MD"], + "catalogue": [ + "a.arb", + "a.po", + "a.xlf", + "a.xliff", + "a.strings", + "a.stringsdict", + "src/locales/en.json", + "src/i18n/en.json", + "src/l10n/en.json", + "src/messages/en.json", + "src/locales/yo/home.json", + "res/values/strings.xml", + "res/values-yo/strings.xml", + ], + "exempt": [ + "pnpm-lock.yaml", + "package-lock.json", + "yarn.lock", + "composer.lock", + "go.sum", + "uv.lock", + "poetry.lock", + "Cargo.lock", + "Package.resolved", + "pubspec.lock", + "gradle.lockfile", + "packages.lock.json", + "a.svg", + "a.lock", + "a.csv", + "a.txt", + "a.api", + "a.snap", + "a.png", + "a.jpg", + "a.ico", + "src/locales/yo.txt", + ], + "other": ["LICENSE", "CODEOWNERS", ".gitmodules", "a.woff", "res/layout/strings.xml.bak"], +} +LANGUAGE_BY_PATH = { + "a.ts": "javascript", + "a.vue": "javascript", + "a.astro": "javascript", + "a.svelte": "javascript", + "a.php": "php", + "config.php.dist": "php", + "a.py": "python", + "a.go": "go", + "a.kt": "jvm", + "a.java": "jvm", + "a.cs": "dotnet", + "a.dart": "dart", + "a.swift": "swift", + "a.css": None, + "a.html": None, + "a.sh": None, + "a.md": None, +} + def rules(violations: list[Violation]) -> list[str]: return [violation.rule for violation in violations] +@contextlib.contextmanager +def git_repo() -> Iterator[Path]: + """Make an empty git repo in a temporary folder and give its resolved path.""" + with tempfile.TemporaryDirectory() as folder: + root = Path(folder).resolve() + subprocess.run(["git", "init", "-q", str(root)], check=True) + yield root + + +def run_main(*args: str) -> tuple[int, str, str]: + """Run the command line and give the exit code, the standard output and the error output.""" + output, errors = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(errors): + status = check_tells.main(list(args)) + return status, output.getvalue(), errors.getvalue() + + class LineLengthTest(unittest.TestCase): def test_accepts_a_line_of_100_characters(self) -> None: self.assertEqual(check_text("src/a.ts", "x" * 100), []) @@ -38,10 +319,16 @@ def test_rejects_a_line_of_101_characters(self) -> None: self.assertEqual(found[0].column, 101) def test_counts_a_tab_as_4_columns(self) -> None: + self.assertEqual(check_text("Makefile", "\t" + "x" * 96), []) self.assertEqual(rules(check_text("Makefile", "\t" + "x" * 97)), ["TELL-1"]) + self.assertEqual(check_text("Makefile", "\t\t" + "x" * 92), []) + self.assertEqual(rules(check_text("Makefile", "\t\t" + "x" * 93)), ["TELL-1"]) def test_skips_a_line_with_a_url(self) -> None: - self.assertEqual(check_text("src/a.ts", "// https://example.com/" + "a" * 100), []) + for scheme in ("https", "http"): + with self.subTest(scheme=scheme): + line = f"// {scheme}://example.com/" + "a" * 100 + self.assertEqual(check_text("src/a.ts", line), []) def test_skips_markdown_lockfiles_and_vectors(self) -> None: self.assertEqual(check_text("README.md", "word " * 40), []) @@ -50,12 +337,15 @@ def test_skips_markdown_lockfiles_and_vectors(self) -> None: def test_skips_json_in_a_data_folder(self) -> None: self.assertEqual(check_text("data/states.json", "x" * 150), []) + self.assertEqual(check_text("vectors/parse.json", "x" * 150), []) self.assertEqual(check_text("spec/vectors/parse.json", "x" * 150), []) + self.assertEqual(check_text("spec/data/format.json", "x" * 150), []) def test_checks_other_files_in_a_data_folder(self) -> None: self.assertEqual(rules(check_text("data/rules.yml", "x" * 150)), ["TELL-1"]) self.assertEqual(rules(check_text("vectors/build.py", "x" * 150)), ["TELL-1"]) self.assertEqual(rules(check_text("package.json", "x" * 150)), ["TELL-1"]) + self.assertEqual(rules(check_text("database/rules.json", "x" * 150)), ["TELL-1"]) def test_skips_a_message_catalogue(self) -> None: line = '{"title": "' + "x" * 150 + '"}' @@ -78,6 +368,87 @@ def test_honours_a_skip_with_a_reason(self) -> None: line = "x" * 120 + " // check-tells: allow TELL-1 because the regex cannot be split" self.assertEqual(check_text("src/a.ts", line), []) + def test_checks_each_code_and_config_kind(self) -> None: + for kind in ("code", "config"): + for path in KIND_BY_PATH[kind]: + with self.subTest(path=path): + self.assertEqual(rules(check_text(path, LONG_LINE)), ["TELL-1"]) + + def test_leaves_every_other_kind_alone(self) -> None: + for kind in ("prose", "catalogue", "exempt", "other"): + for path in KIND_BY_PATH[kind]: + with self.subTest(path=path): + self.assertEqual(check_text(path, LONG_LINE), []) + + def test_reads_a_marker_in_the_first_5_lines_of_a_file(self) -> None: + for marker in ("@generated", "DO NOT EDIT", " None: + for marker in ("@generated", "DO NOT EDIT", " None: + text = "// @generated\nconsole.log(1);\n// " + TO_DO + ": x" + self.assertEqual(rules(check_text("src/a.ts", text)), ["TELL-13", "CS-6"]) + + +class ClassifyTest(unittest.TestCase): + def test_reads_the_kind_of_each_file_name(self) -> None: + for kind, paths in KIND_BY_PATH.items(): + for path in paths: + with self.subTest(path=path): + self.assertEqual(classify(path, "").kind, kind) + + def test_reads_the_language_of_each_file_name(self) -> None: + for path, language in LANGUAGE_BY_PATH.items(): + with self.subTest(path=path): + self.assertEqual(classify(path, "").language, language) + + def test_reads_a_file_without_a_suffix_from_its_first_line(self) -> None: + self.assertEqual(classify("scripts/run", "#!/usr/bin/env python3\n").kind, "code") + self.assertEqual(classify("scripts/run", "plain text\n").kind, "other") + self.assertEqual(classify("scripts/run", "").kind, "other") + + def test_reads_the_suffix_before_a_shebang(self) -> None: + cases = { + "notes.txt": "exempt", + "README.md": "prose", + "package.json": "config", + "font.woff": "other", + "src/locales/en.json": "catalogue", + } + for path, kind in cases.items(): + with self.subTest(path=path): + self.assertEqual(classify(path, "#!/usr/bin/env python3\n").kind, kind) + + def test_reads_the_interpreter_of_a_shebang(self) -> None: + cases = { + "#!/usr/bin/env python3": "python", + "#!/usr/bin/env python": "python", + "#!/usr/bin/python3": "python", + "#!/usr/bin/env node": "javascript", + "#!/usr/bin/node": "javascript", + "#!/bin/sh": None, + } + for first_line, language in cases.items(): + with self.subTest(first_line=first_line): + self.assertEqual(classify("scripts/run", first_line).language, language) + + def test_reads_a_dist_file_like_the_file_that_it_copies(self) -> None: + self.assertEqual(rules(check_text("phpunit.xml.dist", LONG_LINE)), ["TELL-1"]) + self.assertEqual(rules(check_text("config.php.dist", "var_dump($x);")), ["TELL-13"]) + self.assertEqual(rules(check_text("PHPUNIT.XML.DIST", LONG_LINE)), ["TELL-1"]) + + def test_keeps_the_folder_rules_for_a_dist_file(self) -> None: + self.assertEqual(check_text("src/locales/en.json.dist", LONG_LINE), []) + class FileNameTest(unittest.TestCase): def test_rejects_utils_helpers_common_and_misc(self) -> None: @@ -85,9 +456,18 @@ def test_rejects_utils_helpers_common_and_misc(self) -> None: with self.subTest(name=name): self.assertEqual(rules(check_text(name, "")), ["TELL-7"]) + def test_rejects_the_name_with_any_extension_or_none(self) -> None: + for name in ("utils", "helpers.test.ts", "common.d.ts", "src/MISC.md", "Utils.json"): + with self.subTest(name=name): + self.assertEqual(rules(check_text(name, "")), ["TELL-7"]) + def test_accepts_a_folder_named_common(self) -> None: self.assertEqual(check_text("common/parse.ts", ""), []) + def test_accepts_a_name_that_only_starts_with_one_of_the_words(self) -> None: + self.assertEqual(check_text("src/utilities.ts", ""), []) + self.assertEqual(check_text("src/commonly.ts", ""), []) + class TodoFormTest(unittest.TestCase): def test_rejects_a_to_do_without_an_issue(self) -> None: @@ -97,12 +477,41 @@ def test_rejects_a_to_do_in_a_config_file(self) -> None: found = check_text("Makefile", "# " + TO_DO + ": pin the version") self.assertEqual(rules(found), ["CS-6"]) + def test_rejects_a_to_do_or_a_fix_me_in_any_letter_case(self) -> None: + for word in (TO_DO, TO_DO.lower(), TO_DO.title(), FIX_ME, FIX_ME.lower(), FIX_ME.title()): + with self.subTest(word=word): + found = check_text("src/a.ts", f"// {word}: handle 00") + self.assertEqual(rules(found), ["CS-6"]) + self.assertEqual(found[0].column, 4) + self.assertEqual( + found[0].message, f"Link an issue in the form {word.upper()}(#123)." + ) + def test_accepts_a_to_do_with_an_issue(self) -> None: self.assertEqual(check_text("src/a.ts", f"// {TO_DO}(#12): handle 00"), []) + def test_accepts_a_to_do_or_a_fix_me_with_an_issue_in_any_letter_case(self) -> None: + for word in (TO_DO, TO_DO.lower(), FIX_ME, FIX_ME.title()): + with self.subTest(word=word): + self.assertEqual(check_text("src/a.ts", f"// {word}(#12): handle 00"), []) + + def test_rejects_a_to_do_with_a_bad_issue_reference(self) -> None: + for reference in ("(12)", "(#)", "(#x)", " (#12)"): + with self.subTest(reference=reference): + found = check_text("src/a.ts", f"// {TO_DO}{reference}: handle 00") + self.assertEqual(rules(found), ["CS-6"]) + + def test_accepts_a_word_that_only_contains_to_do(self) -> None: + line = "const todos = 1; const mastodon = 2; const fixmeLater = 3; // _todo_form" + self.assertEqual(check_text("src/a.ts", line), []) + def test_ignores_prose(self) -> None: self.assertEqual(check_text("docs/notes.md", f"{TO_DO}: write this"), []) + def test_checks_a_config_file_that_php_tools_use(self) -> None: + found = check_text("phpstan.dist.neon", "# " + TO_DO + ": raise the level") + self.assertEqual(rules(found), ["CS-6"]) + class DebugCallTest(unittest.TestCase): def test_rejects_a_debug_call_in_each_language(self) -> None: @@ -128,6 +537,22 @@ def test_rejects_a_debug_call_in_each_language(self) -> None: with self.subTest(path=path): self.assertEqual(rules(check_text(path, line)), ["TELL-13"]) + def test_rejects_each_debug_call_that_a_language_file_lists(self) -> None: + for path, lines in DEBUG_LINES.items(): + for line in lines: + with self.subTest(path=path, line=line): + self.assertEqual(rules(check_text(path, line)), ["TELL-13"]) + + def test_rejects_a_debug_call_in_each_file_extension_with_a_language(self) -> None: + for extension, line in DEBUG_LINE_BY_EXTENSION.items(): + with self.subTest(extension=extension): + self.assertEqual(rules(check_text("src/a" + extension, line)), ["TELL-13"]) + + def test_names_the_call_and_its_place(self) -> None: + found = check_text("src/a.ts", " console.log(x);") + self.assertEqual((found[0].line, found[0].column), (1, 3)) + self.assertEqual(found[0].message, "Remove 'console.log('.") + def test_checks_a_code_file_in_a_catalogue_folder(self) -> None: found = check_text("src/messages/format.ts", "console.log(1);") self.assertEqual(rules(found), ["TELL-13"]) @@ -136,6 +561,12 @@ def test_accepts_methods_with_similar_names(self) -> None: self.assertEqual(check_text("pkg/a.py", "report.print(code)"), []) self.assertEqual(check_text("pkg/a.py", "pprint_table(code)"), []) + def test_accepts_each_call_that_only_looks_like_a_debug_call(self) -> None: + for path, lines in HARMLESS_LINES.items(): + for line in lines: + with self.subTest(path=path, line=line): + self.assertEqual(check_text(path, line), []) + def test_ignores_test_files_and_comment_lines(self) -> None: self.assertEqual(check_text("example_test.go", 'fmt.Println("EK-01")'), []) self.assertEqual(check_text("src/a.ts", " * console.log(parse(code));"), []) @@ -155,6 +586,16 @@ def test_ignores_each_kind_of_test_path(self) -> None: with self.subTest(path=path): self.assertEqual(check_text(path, line), []) + def test_ignores_every_test_file_name_that_the_standards_allow(self) -> None: + for path, line in TEST_FILES.items(): + with self.subTest(path=path): + self.assertEqual(check_text(path, line), []) + + def test_checks_a_file_name_that_only_looks_like_a_test_file(self) -> None: + for path, line in NON_TEST_FILES.items(): + with self.subTest(path=path): + self.assertEqual(rules(check_text(path, line)), ["TELL-13"]) + def test_ignores_each_kind_of_comment_start(self) -> None: cases = [ ("src/a.ts", "// console.log(x)"), @@ -175,9 +616,22 @@ def test_checks_a_spread_line_outside_python(self) -> None: self.assertEqual(rules(check_text("src/a.ts", line)), ["TELL-13"]) def test_reads_the_language_from_a_shebang(self) -> None: - self.assertEqual( - rules(check_text("scripts/run", "#!/usr/bin/env python3\nprint(1)")), ["TELL-13"] - ) + cases = { + "#!/usr/bin/env python3\nprint(1)": ["TELL-13"], + "#!/usr/bin/env python\nprint(1)": ["TELL-13"], + "#!/usr/bin/python3\nprint(1)": ["TELL-13"], + "#!/usr/bin/env node\nconsole.log(1)": ["TELL-13"], + "#!/usr/bin/node\nconsole.log(1)": ["TELL-13"], + "#!/bin/sh\nprint(1)": [], + } + for text, expected in cases.items(): + with self.subTest(text=text): + self.assertEqual(rules(check_text("scripts/run", text)), expected) + + def test_does_not_check_markup_or_style_files_for_debug_calls(self) -> None: + for path in ("site/a.html", "src/a.css", "src/a.scss"): + with self.subTest(path=path): + self.assertEqual(check_text(path, "console.log(x);"), []) class CharacterTest(unittest.TestCase): @@ -188,15 +642,89 @@ def test_rejects_an_hourglass_in_markdown(self) -> None: self.assertEqual(rules(check_text("README.md", "Waiting " + chr(0x23F3))), ["TELL-14"]) self.assertEqual(rules(check_text("README.md", "Done " + chr(0x231B))), ["TELL-14"]) + def test_rejects_the_emoji_that_agents_write(self) -> None: + names = { + "check mark": 0x2705, + "cross mark": 0x274C, + "warning sign": 0x26A0, + "sparkles": 0x2728, + "rocket": 0x1F680, + "star": 0x2B50, + "arrow": 0x2B06, + } + for name, point in names.items(): + with self.subTest(name=name): + found = check_text("README.md", f"Done {chr(point)} here") + self.assertEqual(rules(found), ["TELL-14"]) + self.assertEqual(found[0].column, 6) + self.assertEqual(found[0].message, f"Remove the emoji U+{point:04X}.") + + def test_rejects_both_ends_of_each_emoji_range(self) -> None: + for low, high in EMOJI_RANGES: + for point in (low, high): + with self.subTest(point=f"U+{point:04X}"): + found = check_text("README.md", f"a {chr(point)} b") + self.assertEqual(rules(found), ["TELL-14"]) + + def test_accepts_the_character_next_to_each_emoji_range(self) -> None: + for low, high in EMOJI_RANGES: + for point in (low - 1, high + 1): + with self.subTest(point=f"U+{point:04X}"): + self.assertEqual(check_text("README.md", f"a {chr(point)} b"), []) + + def test_rejects_the_emoji_variation_selector_after_a_symbol(self) -> None: + found = check_text("README.md", "Heart " + chr(0x2764) + chr(0xFE0F)) + self.assertEqual(rules(found), ["TELL-14", "TELL-14"]) + self.assertEqual([violation.column for violation in found], [7, 8]) + def test_rejects_non_ascii_in_code(self) -> None: found = check_text("src/a.ts", f"const space = '{NO_BREAK_SPACE}';") self.assertEqual(rules(found), ["TELL-14"]) self.assertIn("U+00A0", found[0].message) + def test_rejects_non_ascii_in_each_code_and_config_kind(self) -> None: + for kind in ("code", "config"): + for path in KIND_BY_PATH[kind]: + with self.subTest(path=path): + found = check_text(path, f"a{NO_BREAK_SPACE}b") + self.assertEqual(rules(found), ["TELL-14"]) + self.assertEqual(found[0].column, 2) + + def test_asks_for_an_escape_sequence_for_non_ascii_and_not_for_an_emoji(self) -> None: + emoji = check_text("src/a.ts", f"const a = '{EMOJI}';") + other = check_text("src/a.ts", f"const a = '{E_ACUTE}';") + self.assertEqual(emoji[0].message, "Remove the emoji U+1F600.") + self.assertEqual( + other[0].message, + "Write U+00E9 as an escape sequence. Code and config use ASCII.", + ) + + def test_reports_one_non_ascii_character_for_each_line(self) -> None: + found = check_text("src/a.ts", f"{E_ACUTE}{E_ACUTE}\n{E_ACUTE}") + self.assertEqual([(v.line, v.column) for v in found], [(1, 1), (2, 1)]) + + def test_accepts_the_last_ascii_character_in_code(self) -> None: + self.assertEqual(check_text("src/a.ts", "// " + chr(0x7F)), []) + self.assertEqual(rules(check_text("src/a.ts", "// " + chr(0x80))), ["TELL-14"]) + def test_accepts_non_ascii_in_prose_and_catalogues(self) -> None: self.assertEqual(check_text("docs/guide.md", f"Caf{E_ACUTE}"), []) self.assertEqual(check_text("src/locales/yo.json", f'{{"title": "Caf{E_ACUTE}"}}'), []) + def test_accepts_non_ascii_in_every_file_that_is_not_code_or_config(self) -> None: + for kind in ("prose", "catalogue", "exempt", "other"): + for path in KIND_BY_PATH[kind]: + with self.subTest(path=path): + self.assertEqual(check_text(path, f"Caf{E_ACUTE}"), []) + + def test_rejects_an_emoji_in_every_kind_of_file(self) -> None: + for paths in KIND_BY_PATH.values(): + for path in paths: + with self.subTest(path=path): + found = check_text(path, f"a {EMOJI} b") + self.assertEqual(rules(found), ["TELL-14"]) + self.assertEqual(found[0].column, 3) + def test_holds_a_code_file_in_a_catalogue_folder_to_ascii(self) -> None: line = "export const title = '" + O_DOT_BELOW + "';" self.assertEqual(rules(check_text("src/i18n/yo.ts", line)), ["TELL-14"]) @@ -205,6 +733,32 @@ def test_rejects_an_emoji_in_a_catalogue(self) -> None: found = check_text("src/locales/en.json", f'{{"ok": "{EMOJI}"}}') self.assertEqual(rules(found), ["TELL-14"]) + def test_accepts_non_ascii_in_each_catalogue_folder_and_extension(self) -> None: + for path in KIND_BY_PATH["catalogue"]: + with self.subTest(path=path): + self.assertEqual(check_text(path, f"{O_DOT_BELOW}: {LONG_LINE}"), []) + + def test_holds_other_android_files_to_ascii(self) -> None: + for path in ("res/layout/strings.xml", "res/values/colors.xml", "strings.xml"): + with self.subTest(path=path): + self.assertEqual(rules(check_text(path, f"{E_ACUTE}")), ["TELL-14"]) + + def test_reads_a_smart_quote_in_a_style_file(self) -> None: + found = check_text("src/field.css", "content: " + chr(0x201C) + "x" + chr(0x201D) + ";") + self.assertEqual(rules(found), ["TELL-14"]) + + def test_checks_an_exempt_file_for_emoji_only(self) -> None: + for path in KIND_BY_PATH["exempt"]: + with self.subTest(path=path): + found = check_text(path, f"{LONG_LINE}{E_ACUTE}\n{TO_DO}\n{EMOJI}") + self.assertEqual([(v.rule, v.line) for v in found], [("TELL-14", 3)]) + + def test_checks_a_text_file_with_a_binary_looking_name_for_emoji(self) -> None: + self.assertEqual( + rules(check_text("readme.txt", f"Works with WooCommerce {EMOJI}")), ["TELL-14"] + ) + self.assertEqual(rules(check_text("src/locales/yo.txt", f"Ok {EMOJI}")), ["TELL-14"]) + class LineSplitTest(unittest.TestCase): def test_keeps_a_line_separator_inside_the_line(self) -> None: @@ -232,6 +786,34 @@ def test_rejects_a_skip_of_a_rule_that_cannot_be_skipped(self) -> None: line = f"const a = 1; // {SKIP_MARKER} TELL-7 because it is short" self.assertEqual(rules(check_text("src/a.ts", line)), ["SKIP"]) + def test_skips_each_rule_that_a_skip_comment_can_skip(self) -> None: + for rule, (path, line) in SKIP_SAMPLES.items(): + with self.subTest(rule=rule): + self.assertEqual(rules(check_text(path, line)), [rule]) + skipped = f"{line} // {SKIP_MARKER} {rule} because the line is a sample" + self.assertEqual(check_text(path, skipped), []) + + def test_rejects_a_skip_of_each_rule_that_cannot_be_skipped(self) -> None: + for rule in ("TELL-2", "TELL-7", "TELL-18", "CS-5", "API-3", "GIT-1"): + with self.subTest(rule=rule): + line = f"const a = 1; // {SKIP_MARKER} {rule} because it is short" + found = check_text("src/a.ts", line) + self.assertEqual(rules(found), ["SKIP"]) + self.assertEqual(found[0].column, 17) + + def test_skips_only_the_rule_that_the_comment_names(self) -> None: + line = f"console.log('{LONG_LINE}'); // {SKIP_MARKER} TELL-1 because the string is data" + self.assertEqual(rules(check_text("src/a.ts", line)), ["TELL-13"]) + + def test_skips_only_the_line_that_holds_the_comment(self) -> None: + text = f"console.log(1); // {SKIP_MARKER} TELL-13 because it is a sample\nconsole.log(2);" + found = check_text("src/a.ts", text) + self.assertEqual([(v.rule, v.line) for v in found], [("TELL-13", 2)]) + + def test_honours_a_skip_in_a_config_file(self) -> None: + line = f"{LONG_LINE} # {SKIP_MARKER} TELL-1 because the value is a hash" + self.assertEqual(check_text("Makefile", line), []) + def test_ignores_a_skip_marker_in_prose(self) -> None: line = "A line such as `" + SKIP_MARKER + " TELL-1` has no reason." self.assertEqual(check_text("README.md", line), []) @@ -245,6 +827,19 @@ def test_accepts_a_good_message(self) -> None: def test_rejects_a_long_subject(self) -> None: self.assertEqual(rules(check_commit_message("fix: " + "a" * 70)), ["TELL-18"]) + def test_accepts_a_subject_of_72_characters(self) -> None: + self.assertEqual(check_commit_message("a" * 72), []) + + def test_rejects_a_subject_of_73_characters(self) -> None: + found = check_commit_message("a" * 73) + self.assertEqual(rules(found), ["TELL-18"]) + self.assertEqual(found[0].column, 73) + self.assertEqual(found[0].message, "The subject has 73 characters. Use 72 or fewer.") + + def test_reads_the_subject_after_blank_lines(self) -> None: + found = check_commit_message("\n\n" + "a" * 73 + "\n\n" + "b" * 80) + self.assertEqual([(v.rule, v.line) for v in found], [("TELL-18", 3)]) + def test_rejects_an_emoji(self) -> None: self.assertEqual(rules(check_commit_message(f"fix: parse codes {EMOJI}")), ["TELL-14"]) @@ -255,6 +850,10 @@ def test_accepts_an_accented_name_in_a_sign_off(self) -> None: def test_ignores_git_comment_lines(self) -> None: self.assertEqual(check_commit_message(f"fix: parse codes\n# {EMOJI} from git\n"), []) + def test_accepts_the_last_ascii_character_in_a_commit_message(self) -> None: + self.assertEqual(check_commit_message("fix: a" + chr(0x7F)), []) + self.assertEqual(rules(check_commit_message("fix: a" + chr(0x80))), ["TELL-14"]) + def test_rejects_non_ascii_in_a_subject(self) -> None: self.assertEqual(rules(check_commit_message("fix: parse caf" + E_ACUTE)), ["TELL-14"]) @@ -292,6 +891,56 @@ def test_keeps_a_line_separator_in_a_commit_message(self) -> None: self.assertEqual([(v.rule, v.line) for v in found], [("TELL-14", 3)]) +class FileEncodingTest(unittest.TestCase): + def check(self, name: str, data: bytes) -> list[Violation]: + with tempfile.TemporaryDirectory() as folder: + root = Path(folder).resolve() + file = root / name + file.parent.mkdir(parents=True, exist_ok=True) + file.write_bytes(data) + return check_tells.check_files(root, [file]) + + def test_reports_a_code_file_that_is_not_utf_8(self) -> None: + data = b"const a = 1;\nconst b = \x93hi\x94;\nconsole.log(b);\n" + found = self.check("src/a.ts", data) + self.assertEqual(rules(found), ["TELL-14"]) + self.assertEqual((found[0].path, found[0].line, found[0].column), ("src/a.ts", 2, 11)) + self.assertEqual( + found[0].message, + "Byte 0x93 is not valid UTF-8. " + "Save the file as UTF-8, and use ASCII in code and config.", + ) + + def test_counts_the_column_of_the_bad_byte_in_characters(self) -> None: + found = self.check("src/a.ts", f"// caf{E_ACUTE} ".encode() + b"\xff") + self.assertEqual((found[0].line, found[0].column), (1, 9)) + + def test_reports_a_config_file_that_is_not_utf_8(self) -> None: + found = self.check("package.json", b'{"name": "caf\xe9"}\n') + self.assertEqual(rules(found), ["TELL-14"]) + self.assertEqual((found[0].line, found[0].column), (1, 14)) + + def test_reports_a_script_with_a_shebang_that_is_not_utf_8(self) -> None: + found = self.check("scripts/run", b'#!/usr/bin/env python3\nx = "caf\xe9"\n') + self.assertEqual(rules(found), ["TELL-14"]) + self.assertEqual(found[0].line, 2) + + def test_reports_a_file_that_starts_with_a_utf_16_byte_order_mark(self) -> None: + found = self.check("src/a.ts", b"\xff\xfec\x00") + self.assertEqual((found[0].line, found[0].column), (1, 1)) + + def test_skips_a_file_that_is_not_text(self) -> None: + names = ["logo.png", "docs/a.md", "src/locales/yo.po", "LICENSE", "font.woff", "go.sum"] + for name in names: + with self.subTest(name=name): + self.assertEqual(self.check(name, b"\x89PNG\r\n\x1a\n\xff\xfe\x00"), []) + + def test_reads_a_file_that_is_utf_8(self) -> None: + data = f"const a = '{NO_BREAK_SPACE}';\n".encode() + self.assertEqual(rules(self.check("src/a.ts", data)), ["TELL-14"]) + self.assertEqual(self.check("src/a.ts", b"export const a = 1;\n"), []) + + class CommandTest(unittest.TestCase): def test_checks_new_files_and_skips_ignored_ones(self) -> None: with tempfile.TemporaryDirectory() as folder: @@ -314,3 +963,121 @@ def test_exits_with_0_for_a_clean_repo(self) -> None: (root / "clean.ts").write_text("export const a = 1;\n", encoding="utf-8") command = [str(Path(__file__).resolve().parents[1] / "check-tells"), "--root", folder] self.assertEqual(subprocess.run(command, check=False).returncode, 0) + + def test_prints_a_violation_like_a_compiler_message(self) -> None: + violation = Violation("src/a.ts", 3, 101, "TELL-1", "The line is too wide.") + self.assertEqual(str(violation), "src/a.ts:3:101: TELL-1 The line is too wide.") + + def test_prints_the_violations_in_the_order_of_the_files_and_lines(self) -> None: + with git_repo() as root: + (root / "b.ts").write_text("console.log(1);\n\nconsole.log(2);\n", encoding="utf-8") + (root / "a.ts").write_text("export const a = 1;\nconsole.log(1);\n", encoding="utf-8") + status, output, errors = run_main("--root", str(root)) + self.assertEqual(status, 1) + self.assertEqual(errors, "") + self.assertEqual( + [line.split(": ")[0] for line in output.splitlines()], + ["a.ts:2:1", "b.ts:1:1", "b.ts:3:1"], + ) + + def test_skips_a_tracked_file_that_is_missing_from_the_work_tree(self) -> None: + with git_repo() as root: + (root / "gone.ts").write_text("console.log(1);\n", encoding="utf-8") + subprocess.run(["git", "-C", str(root), "add", "gone.ts"], check=True) + (root / "gone.ts").unlink() + status, output, errors = run_main("--root", str(root)) + self.assertEqual((status, output, errors), (0, "", "")) + + def test_checks_only_the_paths_that_it_is_given(self) -> None: + with git_repo() as root: + (root / "a.ts").write_text("console.log(1);\n", encoding="utf-8") + (root / "b.ts").write_text("console.log(1);\n", encoding="utf-8") + status, output, _ = run_main("--root", str(root), "b.ts") + self.assertEqual(status, 1) + self.assertEqual(output.splitlines(), ["b.ts:1:1: TELL-13 Remove 'console.log('."]) + + def test_exits_with_1_and_names_a_code_file_that_is_not_utf_8(self) -> None: + with git_repo() as root: + (root / "a.ts").write_bytes(b"const a = '\x93';\n") + (root / "logo.png").write_bytes(b"\x89PNG\r\n\x1a\n\xff") + status, output, errors = run_main("--root", str(root)) + self.assertEqual(status, 1) + self.assertEqual(errors, "") + self.assertEqual(len(output.splitlines()), 1) + self.assertIn("a.ts:1:12: TELL-14 Byte 0x93 is not valid UTF-8.", output) + + +class BadInputTest(unittest.TestCase): + def assert_one_line_on_stderr(self, result: tuple[int, str, str], *words: str) -> None: + status, output, errors = result + self.assertEqual(status, 2) + self.assertEqual(output, "") + self.assertEqual(len(errors.splitlines()), 1, errors) + self.assertTrue(errors.startswith("check-tells: "), errors) + for word in words: + self.assertIn(word, errors) + + def test_reports_a_root_that_is_not_a_git_repo(self) -> None: + with tempfile.TemporaryDirectory() as folder: + root = Path(folder).resolve() + # Stop git from finding a repo above the temporary folder. + with mock.patch.dict(os.environ, {"GIT_CEILING_DIRECTORIES": str(root.parent)}): + result = run_main("--root", str(root)) + self.assert_one_line_on_stderr(result, str(root), "not a git repository") + + def test_reports_a_root_that_does_not_exist(self) -> None: + with tempfile.TemporaryDirectory() as folder: + missing = Path(folder).resolve() / "missing" + result = run_main("--root", str(missing)) + self.assert_one_line_on_stderr(result, str(missing)) + + def test_reports_a_path_that_does_not_exist(self) -> None: + with git_repo() as root: + result = run_main("--root", str(root), "missing.ts") + self.assert_one_line_on_stderr(result, "missing.ts", "No such file") + + def test_reports_a_path_that_is_a_folder(self) -> None: + with git_repo() as root: + (root / "src").mkdir() + result = run_main("--root", str(root), "src") + self.assert_one_line_on_stderr(result, "src", "directory") + + def test_reports_a_path_outside_the_root(self) -> None: + with git_repo() as root, tempfile.TemporaryDirectory() as other: + outside = Path(other).resolve() / "a.ts" + outside.write_text("console.log(1);\n", encoding="utf-8") + absolute = run_main("--root", str(root), str(outside)) + relative = run_main("--root", str(root), "../" + outside.parent.name + "/a.ts") + self.assert_one_line_on_stderr(absolute, str(outside), "outside the root") + self.assert_one_line_on_stderr(relative, "outside the root") + + def test_reports_a_commit_message_file_that_does_not_exist(self) -> None: + with tempfile.TemporaryDirectory() as folder: + missing = Path(folder) / "message" + result = run_main("--commit-msg", str(missing)) + self.assert_one_line_on_stderr(result, str(missing), "No such file") + + def test_reports_a_commit_message_that_is_not_utf_8(self) -> None: + with tempfile.TemporaryDirectory() as folder: + message = Path(folder) / "message" + message.write_bytes(b"fix: parse caf\xe9\n") + status, output, errors = run_main("--commit-msg", str(message)) + self.assertEqual((status, errors), (1, "")) + self.assertIn("commit message:1:15: TELL-14 Byte 0xE9 is not valid UTF-8.", output) + + def test_prints_no_traceback_from_the_command(self) -> None: + command = [str(Path(__file__).resolve().parents[1] / "check-tells")] + with tempfile.TemporaryDirectory() as folder: + root = Path(folder).resolve() + environment = {**os.environ, "GIT_CEILING_DIRECTORIES": str(root.parent)} + run = subprocess.run( + [*command, "--root", str(root)], + check=False, + capture_output=True, + text=True, + env=environment, + ) + self.assertEqual(run.returncode, 2) + self.assertEqual(run.stdout, "") + self.assertNotIn("Traceback", run.stderr) + self.assertEqual(len(run.stderr.splitlines()), 1) diff --git a/standards/CODING_STANDARDS.md b/standards/CODING_STANDARDS.md index 13a23f9..33e70c0 100644 --- a/standards/CODING_STANDARDS.md +++ b/standards/CODING_STANDARDS.md @@ -67,7 +67,7 @@ In every language, the error code's wire value is the same string, for example ` - **CS-3 SHOULD.** A function fits on one screen, about 40 lines. A file stays under about 300 lines. - **CS-4 MUST.** Values from the spec, such as the GPS accuracy limits and the state codes, come from the spec's data files. Each language keeps them in one place. The vectors test each value, so a wrong copy fails CI. - **CS-5 MUST.** There is no dead code and no commented-out code. -- **CS-6 MUST.** Each TODO comment links an issue: `TODO(#123): ...` **(tool)** +- **CS-6 MUST.** Each `TODO` or `FIXME` comment links an issue: `TODO(#123): ...` **(tool)** - **CS-7 MUST.** Comments explain why. The code shows what. - **CS-8 MUST.** Add no option, hook or abstraction that the spec does not need. - **CS-9 SHOULD.** Prefer plain data and functions. Use a class when the language expects one, or when it holds configuration, like the client. @@ -101,17 +101,25 @@ Coding agents repeat some habits so often that readers spot them at once. Each r ### The `check-tells` script -`spec/scripts/check-tells` runs in the CI of every repo. It covers the tool rules that no language tool covers: +`spec/scripts/check-tells` runs in the CI of every repo. It sorts each file into a kind, and the kind decides which rules apply: -- **TELL-1.** It checks line length in source and config files, and it counts a tab as 4 columns. It skips lines with a URL, files marked as generated, lockfiles, message catalogues, JSON files in a folder named `data` or `vectors`, and Markdown prose. +- **Code files** are source files (`.ts`, `.js`, `.php`, `.py`, `.go`, `.kt`, `.java`, `.cs`, `.dart`, `.swift` and their variants), SQL and shell scripts, files with no extension that start with `#!`, and `.css`, `.scss`, `.html`, `.astro`, `.vue` and `.svelte` files. +- **Config files** are `.json`, `.json5`, `.jsonc`, `.yml`, `.yaml`, `.toml`, `.xml`, `.ini`, `.cfg`, `.properties`, `.gradle` and `.neon` files, and files such as `Makefile` and `Dockerfile`. A `.dist` file counts as the file that it copies, so `phpunit.xml.dist` is a config file. +- **Other files** can use any script: Markdown prose, message catalogues, lockfiles, data files, images and file types that no rule names. + +It covers the tool rules that no language tool covers: + +- **TELL-1.** It checks line length in code and config files, and it counts a tab as 4 columns. It skips lines with a URL, files that mark themselves as generated in their first 5 lines (`@generated`, `DO NOT EDIT`, `>>` or `...`. -- **TELL-14.** It rejects emojis everywhere, including Markdown. It rejects other non-ASCII characters in source files, config files and commit messages. Message catalogues, docs prose and the sign-off lines of commit messages are exempt from the non-ASCII check, because translations and contributors' names can need any script. A message catalogue is a data file: a file that is not code in a folder named `locales`, `i18n`, `l10n` or `messages`, a `.po`, `.arb`, `.xlf`, `.xliff`, `.strings` or `.stringsdict` file, or an Android `strings.xml`. A code file in such a folder is code, so translations live in data files. +- **TELL-14.** It rejects emojis in every file, including Markdown, message catalogues and lockfiles. It rejects other non-ASCII characters in code files, config files and commit messages. It also rejects a code or config file that is not valid UTF-8, because it cannot read the file. It skips any other file that is not valid UTF-8, because the file can be an image. Message catalogues, docs prose and the sign-off lines of commit messages are exempt from the non-ASCII check, because translations and contributors' names can need any script. A message catalogue is a data file: a file that is not code in a folder named `locales`, `i18n`, `l10n` or `messages`, a `.po`, `.arb`, `.xlf`, `.xliff`, `.strings` or `.stringsdict` file, or an Android `strings.xml`. A code file in such a folder is code, so translations live in data files. - **TELL-18.** With `--commit-msg `, it checks the length of a commit subject. Repos without commitlint use this mode. A line in a code or config file can skip one check with a comment that names the rule and gives a reason, for example `check-tells: allow TELL-1 because the regex cannot be split`. Reviewers check each skip. +The command exits with 0 when all files pass, with 1 when a rule is broken, and with 2 when its input is bad, for example a path that does not exist or a folder that is not a git repo. + ## Dependencies - **DEP-1 MUST.** A `core` package has no runtime dependencies. From 206ce995bd2f4dd4e726be7a0057c91cbef07e7d Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 13:28:39 +0100 Subject: [PATCH 33/45] ci: check the message that reaches main and pin the tools The repo squash-merges. The title of the pull request becomes the subject on main, and its body becomes the body. A message on a branch never reaches main, and CI checked no message at all. A new workflow now builds the squash message from the title, " (#N)" and the body, and runs check-tells on it. The message needs a Conventional Commits type with no scope, a Signed-off-by line, no text left over from the pull request template, and the ASCII and 72-character rules that already existed. A second job needs a Signed-off-by line in each commit of the pull request. No other message rule applies to a commit, because only the squash message reaches main. The jobs run again when someone edits the title or the body, because such an edit pushes no commit. They leave out the HTML comments of the body, which no reader sees and Renovate adds. The standards mark GIT-1, GIT-2, SEC-6 and DEP-5 as enforced by a tool. GIT-1 and GIT-2 now have the jobs above. zizmor, pinned, now runs in make check. It finds an action that is not pinned to a commit, and text of a pull request that reaches a script. renovate.json opens update pull requests each week, pins the digests of actions and moves the tool versions of the Makefile. Its commits and its pull request bodies pass the new jobs. Renovate runs only after the owner installs its GitHub app. A new Ruff release can add rules and turn CI red on a change that has nothing to do with it, and ubuntu-latest changes image on a date that nobody chooses. Each tool in the Makefile now has an exact version, each runner has a version, each job has a time limit, and the matrix no longer stops at the first failure. make lint and make types now check scripts/check-tells, which has no .py suffix and was skipped. Signed-off-by: MobileMage --- .github/workflows/ci.yml | 8 +- .github/workflows/pull-request.yml | 52 +++ Makefile | 33 +- mypy.ini | 3 +- renovate.json | 23 ++ scripts/check_tells.py | 254 +++++++++++--- scripts/tests/test_check_tells.py | 512 ++++++++++++++++++++++++++++- standards/CODING_STANDARDS.md | 4 +- 8 files changed, 840 insertions(+), 49 deletions(-) create mode 100644 .github/workflows/pull-request.yml create mode 100644 renovate.json diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0b3cb3b..2bf9d0c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -12,8 +12,11 @@ permissions: jobs: check: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + timeout-minutes: 15 strategy: + # A failure on one Python version must not hide the result on the other. + fail-fast: false matrix: # The oldest Python that the scripts support, and the newest release. python-version: ["3.11", "3.x"] @@ -31,7 +34,8 @@ jobs: - run: make check secrets: - runs-on: ubuntu-latest + runs-on: ubuntu-24.04 + timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml new file mode 100644 index 0000000..6af6488 --- /dev/null +++ b/.github/workflows/pull-request.yml @@ -0,0 +1,52 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +name: pull request + +# A squash merge puts the title and the body of the pull request on main as the commit message. +# An edit of the title or the body does not push a commit, so it must start these jobs again. +on: + pull_request: + types: [opened, edited, synchronize, reopened] + +permissions: + contents: read + +concurrency: + group: pull-request-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + sign-off: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + # The check reads each commit from the base to the head. + fetch-depth: 0 + persist-credentials: false + # Only the message that reaches main gets the other message rules, so a commit needs a + # sign-off and nothing more. + - name: Check that each commit has a sign-off + env: + BASE: ${{ github.event.pull_request.base.sha }} + HEAD: ${{ github.event.pull_request.head.sha }} + run: python3 scripts/check-tells --signed-off "$BASE..$HEAD" + + squash-message: + runs-on: ubuntu-24.04 + timeout-minutes: 10 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The title and the body come through env, so that no text of the pull request ends up + # inside the script. + - name: Check the message that the squash merge puts on main + env: + TITLE: ${{ github.event.pull_request.title }} + NUMBER: ${{ github.event.pull_request.number }} + BODY: ${{ github.event.pull_request.body }} + run: | + printf '%s (#%s)\n\n%s\n' "$TITLE" "$NUMBER" "$BODY" > "$RUNNER_TEMP/message" + python3 scripts/check-tells --squash-msg "$RUNNER_TEMP/message" --no-scope diff --git a/Makefile b/Makefile index f37f214..5130d37 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,24 @@ # SPDX-FileCopyrightText: 2026 The Gatepost authors # SPDX-License-Identifier: Apache-2.0 -.PHONY: check test vectors tells lint types licences +.PHONY: check test vectors tells lint types licences actions -check: test vectors tells lint types licences +# Each tool has an exact version. A new release can add rules and turn CI red on a change that +# has nothing to do with it. Renovate opens the pull requests that move these versions. +# renovate: datasource=pypi depName=ruff +RUFF_VERSION := 0.16.10 +# renovate: datasource=pypi depName=mypy +MYPY_VERSION := 2.4.0 +# renovate: datasource=pypi depName=reuse +REUSE_VERSION := 6.2.0 +# renovate: datasource=pypi depName=charset-normalizer +CHARSET_NORMALIZER_VERSION := 3.5.2 +# renovate: datasource=pypi depName=zizmor +ZIZMOR_VERSION := 1.30.1 + +# scripts/check-tells has no .py suffix, so the tools need its name. +SCRIPTS := scripts scripts/check-tells + +check: test vectors tells lint types licences actions test: python3 -m unittest discover -s scripts/tests -t scripts @@ -14,12 +30,17 @@ tells: python3 scripts/check-tells lint: - uvx ruff check scripts - uvx ruff format --check scripts + uvx ruff@$(RUFF_VERSION) check $(SCRIPTS) + uvx ruff@$(RUFF_VERSION) format --check $(SCRIPTS) types: - uvx mypy + uvx mypy@$(MYPY_VERSION) # reuse needs an encoding detector, and uvx installs reuse without one. licences: - uvx --with charset-normalizer reuse lint + uvx --with charset-normalizer==$(CHARSET_NORMALIZER_VERSION) reuse@$(REUSE_VERSION) lint + +# SEC-6: zizmor finds an action that is not pinned to a commit, and text of a pull request that +# reaches a script. Its other checks need a GitHub token, so this run is offline. +actions: + uvx zizmor@$(ZIZMOR_VERSION) --offline . diff --git a/mypy.ini b/mypy.ini index d9ed64d..725bca9 100644 --- a/mypy.ini +++ b/mypy.ini @@ -3,4 +3,5 @@ [mypy] strict = True mypy_path = scripts -files = scripts +# scripts/check-tells has no .py suffix, so the files list names it. +files = scripts, scripts/check-tells diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..af37877 --- /dev/null +++ b/renovate.json @@ -0,0 +1,23 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "config:recommended", + "helpers:pinGitHubActionDigests", + "customManagers:makefileVersions", + "schedule:weekly" + ], + "semanticCommits": "enabled", + "semanticCommitScope": "", + "commitTrailers": ["Signed-off-by: {{{gitAuthor}}}"], + "prBodyTemplate": "{{{notes}}}", + "prBodyNotes": [ + "Update a pinned tool or action.", + "Signed-off-by: Renovate Bot " + ], + "packageRules": [ + { + "matchManagers": ["github-actions"], + "semanticCommitType": "ci" + } + ] +} diff --git a/scripts/check_tells.py b/scripts/check_tells.py index d091584..6447705 100644 --- a/scripts/check_tells.py +++ b/scripts/check_tells.py @@ -3,7 +3,8 @@ """Find the agent tells that language tools miss. The rules come from standards/CODING_STANDARDS.md: TELL-1, TELL-7, TELL-13, TELL-14 -and TELL-18, and CS-6 for the form of to-do comments. +and TELL-18, and CS-6 for the form of to-do comments. For the message that a squash merge +puts on main, it also checks GIT-1 and GIT-2. """ from __future__ import annotations @@ -12,6 +13,7 @@ import re import subprocess import sys +from collections.abc import Callable from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import Literal @@ -30,6 +32,20 @@ GENERATED_MARKERS = ("@generated", "DO NOT EDIT", "[a-z]+)(?P\([^()\s]+\))?!?: \S") +SIGN_OFF = re.compile(r"signed-off-by:[ \t]+[^<>\s][^<>]*<[^<>\s]+@[^<>\s]+>[ \t]*", re.IGNORECASE) +# The address in the sign-off line of templates/PULL_REQUEST_TEMPLATE.md. +PLACEHOLDER_ADDRESS = "you@example.com" +# The start of the paragraph in that template that asks the author for prose. +TEMPLATE_PARAGRAPH = "Write two or three sentences of plain prose" +EMPTY_CLOSES_LINE = re.compile(r"closes #\s*", re.IGNORECASE) +HTML_COMMENT = re.compile(r"", re.DOTALL) +SQUASH_NOTE = " The subject is the title plus the ' (#N)' that the squash merge adds." EMOJI_RANGES = ( (0x1F000, 0x1FAFF), (0x231A, 0x231B), @@ -389,16 +416,19 @@ def _message_lines(text: str) -> list[NumberedLine]: return numbered -def _check_subject(lines: list[NumberedLine]) -> list[Violation]: - """Check the subject, which is the first line that has text.""" - for number, subject in lines: - if not subject.strip(): - continue - if len(subject) <= MAX_SUBJECT: - return [] - message = f"The subject has {len(subject)} characters. Use {MAX_SUBJECT} or fewer." - return [Violation("commit message", number, MAX_SUBJECT + 1, "TELL-18", message)] - return [] +def _subject(lines: list[NumberedLine]) -> NumberedLine | None: + """Find the subject, which is the first line that has text.""" + return next(((number, line) for number, line in lines if line.strip()), None) + + +def _check_subject(lines: list[NumberedLine], note: str = "") -> list[Violation]: + """Check the length of the subject. The note adds advice that fits the message.""" + subject = _subject(lines) + if subject is None or len(subject[1]) <= MAX_SUBJECT: + return [] + number, text = subject + message = f"The subject has {len(text)} characters. Use {MAX_SUBJECT} or fewer.{note}" + return [Violation("commit message", number, MAX_SUBJECT + 1, "TELL-18", message)] def _first_bad_character(number: int, line: str) -> Violation | None: @@ -411,10 +441,8 @@ def _first_bad_character(number: int, line: str) -> Violation | None: return None -def check_commit_message(text: str) -> list[Violation]: - """Check the subject length and the characters of a commit message.""" - lines = _message_lines(text) - violations = _check_subject(lines) +def _check_message(lines: list[NumberedLine], note: str = "") -> list[Violation]: + violations = _check_subject(lines, note) for number, line in lines: violation = _first_bad_character(number, line) if violation is not None: @@ -422,27 +450,143 @@ def check_commit_message(text: str) -> list[Violation]: return violations -def tracked_files(root: Path) -> list[Path]: - """List the files that git tracks, or would track, under root.""" - command = [ - "git", - "-C", - str(root), - "ls-files", - "-z", - "--cached", - "--others", - "--exclude-standard", +def _check_subject_form(lines: list[NumberedLine], *, no_scope: bool) -> list[Violation]: + """Check GIT-1: the subject has a type that Conventional Commits lists, and maybe a scope.""" + subject = _subject(lines) + if subject is None: + return [Violation("commit message", 1, 1, "GIT-1", "The message has no subject.")] + number, text = subject + match = SUBJECT_FORM.match(text) + if match is None: + message = ( + "The subject does not follow Conventional Commits. " + f"Write 'type: description', where type is {COMMIT_TYPE_LIST}." + ) + return [Violation("commit message", number, 1, "GIT-1", message)] + if match["type"] not in COMMIT_TYPES: + message = f"'{match['type']}' is not a Conventional Commits type. Use {COMMIT_TYPE_LIST}." + return [Violation("commit message", number, 1, "GIT-1", message)] + if no_scope and match["scope"]: + message = ( + f"Remove the scope {match['scope']}. " + "This repo has no packages, so a subject has no scope." + ) + column = match.start("scope") + 1 + return [Violation("commit message", number, column, "GIT-1", message)] + return [] + + +def _check_sign_off(lines: list[NumberedLine], path: str) -> list[Violation]: + """Check GIT-2: the message has a line 'Signed-off-by: Name
', not the template's.""" + sign_offs = [(number, line) for number, line in lines if SIGN_OFF.fullmatch(line)] + if not sign_offs: + message = ( + "The message has no Signed-off-by line. Add the line 'Signed-off-by: Name
'. " + "git commit --signoff adds it to a commit, and git rebase --signoff adds it to each " + "commit of a branch." + ) + return [Violation(path, 1, 1, "GIT-2", message)] + message = ( + f"The sign-off still holds the placeholder address {PLACEHOLDER_ADDRESS}. " + "Write your own name and address." + ) + return [ + Violation(path, number, 1, "GIT-2", message) + for number, line in sign_offs + if PLACEHOLDER_ADDRESS in line.lower() + ] + + +def _check_template_text(lines: list[NumberedLine]) -> list[Violation]: + """Find the text of the pull request template that the author did not replace.""" + violations: list[Violation] = [] + for number, line in lines: + text = line.strip() + if text.startswith(TEMPLATE_PARAGRAPH): + message = ( + "The body still holds the placeholder paragraph of the pull request template. " + "Replace it with two or three sentences that say what the change does and why." + ) + elif EMPTY_CLOSES_LINE.fullmatch(text): + message = ( + "The body still holds the line 'Closes #'. Add the issue number after it. " + "If the change closes no issue, remove the line." + ) + else: + continue + violations.append(Violation("commit message", number, 1, "TELL-18", message)) + return violations + + +def check_commit_message(text: str) -> list[Violation]: + """Check the subject length and the characters of a commit message.""" + return _check_message(_message_lines(text)) + + +def _blank_html_comments(body: str) -> str: + """Replace each HTML comment with spaces, and keep its line feeds, so that no place moves.""" + return HTML_COMMENT.sub(lambda comment: re.sub(r"[^\n]", " ", comment[0]), body) + + +def check_squash_message(text: str, *, no_scope: bool = False) -> list[Violation]: + """Check the message that a squash merge puts on main: the pull request title and body. + + The body is not a git message, so a line that starts with # is text and not a comment. No + reader sees an HTML comment, and Renovate ends each body with one, so the body check leaves + them out. The title is the first line, and no comment hides text there. + """ + subject, line_feed, body = text.partition("\n") + lines = list(enumerate(_split_lines(subject + line_feed + _blank_html_comments(body)), start=1)) + return [ + *_check_message(lines, SQUASH_NOTE), + *_check_subject_form(lines, no_scope=no_scope), + *_check_sign_off(lines, "commit message"), + *_check_template_text(lines), ] + + +def _git(root: Path, *arguments: str) -> bytes: + """Run git in root and give its output. A failure is bad input, such as no repo.""" try: - output = subprocess.run(command, check=True, capture_output=True).stdout + run = subprocess.run(["git", "-C", str(root), *arguments], check=True, capture_output=True) except subprocess.CalledProcessError as error: reason = " ".join(error.stderr.decode(errors="replace").split()) - raise InputError(f"Cannot list the files in {root}. Git says: {reason}") from error + raise InputError(f"Git failed in {root}: {reason}") from error + return run.stdout + + +def tracked_files(root: Path) -> list[Path]: + """List the files that git tracks, or would track, under root.""" + output = _git(root, "ls-files", "-z", "--cached", "--others", "--exclude-standard") names = output.decode().split("\0") return [root / name for name in names if name and (root / name).is_file()] +def _commits(root: Path, revision_range: str) -> list[tuple[str, str]]: + """Give the hash and the message of each commit in a range. Merge commits are left out.""" + output = _git( + root, + "log", + "--no-merges", + "--no-show-signature", + "--format=%H%n%B%x00", + "--end-of-options", + revision_range, + ) + records = [record.lstrip("\n") for record in output.decode(errors="replace").split("\0")] + pairs = (record.partition("\n") for record in records if record) + return [(commit, message) for commit, _, message in pairs] + + +def check_signed_off(root: Path, revision_range: str) -> list[Violation]: + """Check that each commit in a revision range has a sign-off. Merge commits are left out.""" + violations: list[Violation] = [] + for commit, message in _commits(root, revision_range): + lines = list(enumerate(_split_lines(message), start=1)) + violations.extend(_check_sign_off(lines, f"commit {commit[:12]}")) + return violations + + def _read(path: Path) -> bytes: try: return path.read_bytes() @@ -486,37 +630,71 @@ def check_files(root: Path, files: list[Path]) -> list[Violation]: return violations -def _check_commit_message_file(path: Path) -> list[Violation]: +def _check_message_file(path: Path, check: Callable[[str], list[Violation]]) -> list[Violation]: data = _read(path) try: text = data.decode("utf-8") except UnicodeDecodeError as error: advice = "Save the message as UTF-8, and use ASCII." return [_not_utf8("commit message", data, error, advice)] - return check_commit_message(text) + return check(text) def _run(args: argparse.Namespace) -> list[Violation]: if args.commit_msg is not None: - return _check_commit_message_file(args.commit_msg) + return _check_message_file(args.commit_msg, check_commit_message) + if args.squash_msg is not None: + return _check_message_file( + args.squash_msg, lambda text: check_squash_message(text, no_scope=args.no_scope) + ) root = args.root.resolve() + if args.signed_off is not None: + return check_signed_off(root, args.signed_off) files = [(root / path).resolve() for path in args.paths] or tracked_files(root) return check_files(root, files) -def main(argv: list[str] | None = None) -> int: - """Run the checks and print each violation. - - Return 0 if all is well, 1 if a rule broke and 2 if the input cannot be read. - """ +def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser( prog="check-tells", description="Find the agent tells that language tools miss." ) parser.add_argument("--root", type=Path, default=Path.cwd(), help="The repo root.") - parser.add_argument("--commit-msg", type=Path, help="Check this commit message file.") + messages = parser.add_mutually_exclusive_group() + messages.add_argument( + "--commit-msg", type=Path, help="Check this commit message file (TELL-14 and TELL-18)." + ) + messages.add_argument( + "--squash-msg", + type=Path, + help="Check this file as the message that a squash merge puts on main " + "(TELL-14, TELL-18, GIT-1, GIT-2 and the text of the pull request template).", + ) + messages.add_argument( + "--signed-off", + metavar="RANGE", + help="Check that each commit in this git revision range has a sign-off (GIT-2).", + ) + parser.add_argument( + "--no-scope", + action="store_true", + help="With --squash-msg, reject a scope in the subject " + "(GIT-1 for a repo without packages).", + ) parser.add_argument("paths", nargs="*", type=Path, help="Files to check. Default: all.") + return parser + + +def main(argv: list[str] | None = None) -> int: + """Run the checks and print each violation. + + Return 0 if all is well, 1 if a rule broke and 2 if the input cannot be read. + """ + parser = _parser() + args = parser.parse_args(argv) + if args.no_scope and args.squash_msg is None: + parser.error("--no-scope needs --squash-msg.") try: - violations = _run(parser.parse_args(argv)) + violations = _run(args) except InputError as error: sys.stderr.write(f"check-tells: {error}\n") return EXIT_BAD_INPUT diff --git a/scripts/tests/test_check_tells.py b/scripts/tests/test_check_tells.py index 453b94c..9979c98 100644 --- a/scripts/tests/test_check_tells.py +++ b/scripts/tests/test_check_tells.py @@ -5,6 +5,7 @@ import contextlib import io import os +import shutil import subprocess import tempfile import unittest @@ -13,7 +14,14 @@ from unittest import mock import check_tells -from check_tells import Violation, check_commit_message, check_text, classify +from check_tells import ( + Violation, + check_commit_message, + check_signed_off, + check_squash_message, + check_text, + classify, +) EMOJI = chr(0x1F600) NO_BREAK_SPACE = chr(0x00A0) @@ -288,6 +296,34 @@ } +SIGN_OFF_LINE = "Signed-off-by: Ada Bello " +PLACEHOLDER_PARAGRAPH = ( + "Write two or three sentences of plain prose that say what this change does and why." +) +COMMIT_TYPES = ( + "build", + "chore", + "ci", + "docs", + "feat", + "fix", + "perf", + "refactor", + "revert", + "style", + "test", +) +# A host with a signing key in its git config must not sign the commits of these tests. +GIT_ENVIRONMENT = { + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_AUTHOR_NAME": "Ada Bello", + "GIT_AUTHOR_EMAIL": "ada@example.org", + "GIT_COMMITTER_NAME": "Ada Bello", + "GIT_COMMITTER_EMAIL": "ada@example.org", +} + + def rules(violations: list[Violation]) -> list[str]: return [violation.rule for violation in violations] @@ -301,6 +337,31 @@ def git_repo() -> Iterator[Path]: yield root +def git(root: Path, *args: str) -> str: + """Run git in a test repo and give its standard output.""" + run = subprocess.run( + ["git", "-C", str(root), *args], + check=True, + capture_output=True, + text=True, + env={**os.environ, **GIT_ENVIRONMENT}, + ) + return run.stdout.strip() + + +def commit(root: Path, message: str) -> str: + """Make an empty commit and give its hash.""" + git(root, "-c", "commit.gpgsign=false", "commit", "--allow-empty", "-q", "-m", message) + return git(root, "rev-parse", "HEAD") + + +def squash( + subject: str, body: str = "Why the change is needed.", sign_off: str = SIGN_OFF_LINE +) -> str: + """Build a squash merge message from a title with its suffix, a body and a sign-off.""" + return f"{subject}\n\n{body}\n\n{sign_off}\n" + + def run_main(*args: str) -> tuple[int, str, str]: """Run the command line and give the exit code, the standard output and the error output.""" output, errors = io.StringIO(), io.StringIO() @@ -1081,3 +1142,452 @@ def test_prints_no_traceback_from_the_command(self) -> None: self.assertEqual(run.stdout, "") self.assertNotIn("Traceback", run.stderr) self.assertEqual(len(run.stderr.splitlines()), 1) + + +class ConventionalSubjectTest(unittest.TestCase): + def test_accepts_each_commit_type(self) -> None: + for kind in COMMIT_TYPES: + with self.subTest(kind=kind): + self.assertEqual(check_squash_message(squash(f"{kind}: reject a unit")), []) + + def test_accepts_a_mark_for_a_breaking_change(self) -> None: + self.assertEqual(check_squash_message(squash("feat!: drop the old name (#4)")), []) + + def test_rejects_a_subject_that_does_not_follow_the_form(self) -> None: + subjects = [ + "Foundation: standards and grammar (#1)", + "Fix: reject a unit", + "fix:reject a unit", + "fix : reject a unit", + "fix(): reject a unit", + "fix", + "fix: ", + "Update files", + ": reject a unit", + "fix(core:) reject a unit", + ] + for subject in subjects: + with self.subTest(subject=subject): + found = check_squash_message(squash(subject)) + self.assertEqual(rules(found), ["GIT-1"]) + self.assertEqual((found[0].line, found[0].column), (1, 1)) + self.assertIn("does not follow Conventional Commits", found[0].message) + + def test_rejects_a_type_that_conventional_commits_does_not_list(self) -> None: + for kind in ("update", "feature", "bugfix", "wip", "chores"): + with self.subTest(kind=kind): + found = check_squash_message(squash(f"{kind}: reject a unit")) + self.assertEqual(rules(found), ["GIT-1"]) + self.assertIn(f"'{kind}' is not a Conventional Commits type", found[0].message) + + def test_names_each_type_that_it_accepts(self) -> None: + found = check_squash_message(squash("update: reject a unit")) + self.assertIn( + "build, chore, ci, docs, feat, fix, perf, refactor, revert, style or test", + found[0].message, + ) + + def test_accepts_a_scope_unless_the_repo_has_none(self) -> None: + self.assertEqual(check_squash_message(squash("fix(core): reject a unit")), []) + self.assertEqual(check_squash_message(squash("fix(core)!: reject a unit")), []) + + def test_rejects_a_scope_in_a_repo_that_has_none(self) -> None: + for subject in ("fix(core): reject a unit", "feat(deps)!: drop a name"): + with self.subTest(subject=subject): + found = check_squash_message(squash(subject), no_scope=True) + self.assertEqual(rules(found), ["GIT-1"]) + self.assertEqual((found[0].line, found[0].column), (1, subject.index("(") + 1)) + self.assertIn("Remove the scope", found[0].message) + + def test_accepts_a_subject_without_a_scope_in_a_repo_that_has_none(self) -> None: + self.assertEqual(check_squash_message(squash("fix: reject a unit"), no_scope=True), []) + + def test_rejects_a_message_that_has_no_subject(self) -> None: + found = check_squash_message("\n \n") + self.assertEqual(sorted(rules(found)), ["GIT-1", "GIT-2"]) + self.assertEqual(found[0].message, "The message has no subject.") + + def test_rejects_the_default_squash_subject_of_a_pull_request_with_no_type(self) -> None: + title = "Foundation: standards, grammar, data, shared vectors and check-tells (#1)" + found = check_squash_message(squash(title), no_scope=True) + self.assertEqual(sorted(rules(found)), ["GIT-1", "TELL-18"]) + + def test_reads_the_first_line_that_has_text_as_the_subject(self) -> None: + found = check_squash_message("\n\nupdate: x\n\nbody\n\n" + SIGN_OFF_LINE) + self.assertEqual([(v.rule, v.line) for v in found], [("GIT-1", 3)]) + + +class SquashMessageTest(unittest.TestCase): + def test_accepts_a_good_message(self) -> None: + message = ( + "fix: reject a unit of 00 (#7)\n\n" + "The parser accepted a unit of 00. It now returns bad_segment.\n\n" + "Closes #5\n\n" + SIGN_OFF_LINE + "\n" + ) + self.assertEqual(check_squash_message(message, no_scope=True), []) + + def test_counts_the_suffix_that_the_merge_adds_to_the_title(self) -> None: + self.assertEqual(check_squash_message(squash("fix: " + "a" * 62 + " (#7)")), []) + found = check_squash_message(squash("fix: " + "a" * 63 + " (#7)")) + self.assertEqual(rules(found), ["TELL-18"]) + self.assertEqual(found[0].column, 73) + self.assertIn("The subject has 73 characters. Use 72 or fewer.", found[0].message) + self.assertIn("(#N)", found[0].message) + + def test_reads_a_line_that_starts_with_a_hash_as_text(self) -> None: + body = f"## Notes {EMOJI}\n# {EM_DASH} here" + found = check_squash_message(squash("fix: reject a unit", body=body)) + self.assertEqual([(v.rule, v.line) for v in found], [("TELL-14", 3), ("TELL-14", 4)]) + + def test_reads_a_line_that_starts_with_a_hash_as_a_comment_in_a_commit_file(self) -> None: + message = f"fix: reject a unit\n\n## Notes {EMOJI}\n" + self.assertEqual(check_commit_message(message), []) + + def test_rejects_non_ascii_in_the_body_but_not_in_a_sign_off(self) -> None: + found = check_squash_message(squash("fix: a unit", body=f"Use a dash {EM_DASH} here.")) + self.assertEqual([(v.rule, v.line) for v in found], [("TELL-14", 3)]) + sign_off = f"Signed-off-by: Ad{E_ACUTE} Bello " + self.assertEqual(check_squash_message(squash("fix: a unit", sign_off=sign_off)), []) + + def test_treats_crlf_like_lf(self) -> None: + message = squash("fix: reject a unit (#7)", body="Closes #5").replace("\n", "\r\n") + self.assertEqual(check_squash_message(message), []) + + def test_reports_every_problem_once(self) -> None: + message = "Update files\n\n" + PLACEHOLDER_PARAGRAPH + "\n\nCloses #\n" + found = check_squash_message(message) + self.assertEqual(sorted(rules(found)), ["GIT-1", "GIT-2", "TELL-18", "TELL-18"]) + + +class HtmlCommentTest(unittest.TestCase): + RENOVATE_COMMENT = "" + + def test_leaves_out_the_comment_that_renovate_ends_a_body_with(self) -> None: + message = squash("ci: update an action (#4)", body="Update a pinned tool or action.") + self.assertEqual(check_squash_message(message + self.RENOVATE_COMMENT + "\n"), []) + + def test_leaves_out_the_text_that_a_comment_holds(self) -> None: + hidden = [ + EMOJI, + f"a dash {EM_DASH} here", + "Closes #", + PLACEHOLDER_PARAGRAPH, + "Signed-off-by: Your Name ", + ] + for text in hidden: + with self.subTest(text=text): + body = f"Why the change is needed.\n\n" + self.assertEqual(check_squash_message(squash("fix: a unit", body=body)), []) + + def test_leaves_out_a_comment_that_spans_lines(self) -> None: + body = f"Why the change is needed.\n\n" + self.assertEqual(check_squash_message(squash("fix: a unit", body=body)), []) + + def test_keeps_the_place_of_the_text_around_a_comment(self) -> None: + body = f"Use a dash {EM_DASH} here." + found = check_squash_message(squash("fix: a unit", body=body)) + self.assertEqual([(v.rule, v.line, v.column) for v in found], [("TELL-14", 4, 7)]) + + def test_leaves_out_each_comment_of_a_line_and_not_the_text_between_them(self) -> None: + body = f"Fix the {EM_DASH} unit rule." + found = check_squash_message(squash("fix: a unit", body=body)) + self.assertEqual( + [(v.rule, v.line, v.column) for v in found], [("TELL-14", 3, body.index(EM_DASH) + 1)] + ) + + def test_does_not_count_a_sign_off_that_a_comment_holds(self) -> None: + body = f"Why the change is needed.\n\n" + found = check_squash_message(f"fix: a unit\n\n{body}\n") + self.assertEqual(rules(found), ["GIT-2"]) + self.assertIn("The message has no Signed-off-by line.", found[0].message) + + def test_reads_a_comment_that_does_not_end_as_text(self) -> None: + body = f"Why the change is needed.\n\n")) + self.assertEqual([(v.rule, v.line) for v in found], [("TELL-14", 1)]) + + def test_reads_a_comment_in_a_message_with_crlf(self) -> None: + body = f"Why the change is needed.\n\n" + message = squash("fix: a unit", body=body).replace("\n", "\r\n") + self.assertEqual(check_squash_message(message), []) + + def test_leaves_out_nothing_in_a_commit_message_file(self) -> None: + message = f"fix: a unit\n\n\n" + self.assertEqual(rules(check_commit_message(message)), ["TELL-14"]) + + +class SignOffTest(unittest.TestCase): + def test_requires_a_sign_off(self) -> None: + found = check_squash_message("fix: reject a unit\n\nWhy the change is needed.\n") + self.assertEqual(rules(found), ["GIT-2"]) + self.assertEqual((found[0].line, found[0].column), (1, 1)) + self.assertIn("The message has no Signed-off-by line.", found[0].message) + + def test_reads_each_form_of_a_sign_off(self) -> None: + forms = [ + "Signed-off-by: Ada Bello ", + "signed-off-by: Ada Bello ", + "SIGNED-OFF-BY: Ada Bello ", + "Signed-off-by: Ada Bello ", + "Signed-off-by: A ", + f"Signed-off-by: Ad{E_ACUTE} Bello ", + "Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>", + ] + for form in forms: + with self.subTest(form=form): + self.assertEqual(check_squash_message(squash("fix: a unit", sign_off=form)), []) + + def test_rejects_each_line_that_only_looks_like_a_sign_off(self) -> None: + forms = [ + "Signed-off-by:", + "Signed-off-by: Ada Bello", + "Signed-off-by: ", + "Signed-off-by: Ada Bello ", + "Signed-off-by: Ada Bello ada@example.org", + "Signed-off-by Ada Bello ", + "Reviewed-by: Ada Bello ", + "Co-authored-by: Ada Bello ", + "- Signed-off-by: Ada Bello ", + "Signed-off-by: Ada ", + ] + for form in forms: + with self.subTest(form=form): + found = check_squash_message(squash("fix: a unit", sign_off=form)) + self.assertEqual(rules(found), ["GIT-2"]) + + def test_rejects_the_placeholder_address_of_the_template(self) -> None: + sign_off = "Signed-off-by: Your Name " + found = check_squash_message(squash("fix: a unit", sign_off=sign_off)) + self.assertEqual(rules(found), ["GIT-2"]) + self.assertEqual((found[0].line, found[0].column), (5, 1)) + self.assertIn("placeholder address", found[0].message) + + def test_rejects_the_placeholder_address_in_any_letter_case(self) -> None: + sign_off = "Signed-off-by: Ada Bello " + found = check_squash_message(squash("fix: a unit", sign_off=sign_off)) + self.assertEqual(rules(found), ["GIT-2"]) + + def test_rejects_the_placeholder_address_next_to_a_real_sign_off(self) -> None: + body = "Signed-off-by: Your Name " + found = check_squash_message(squash("fix: a unit", body=body)) + self.assertEqual([(v.rule, v.line) for v in found], [("GIT-2", 3)]) + + def test_accepts_the_placeholder_address_in_prose(self) -> None: + body = "The template shows you@example.com in its sign-off line." + self.assertEqual(check_squash_message(squash("docs: fix a template", body=body)), []) + + +class TemplateTextTest(unittest.TestCase): + def test_rejects_the_placeholder_paragraph_of_the_template(self) -> None: + found = check_squash_message(squash("fix: a unit", body=PLACEHOLDER_PARAGRAPH)) + self.assertEqual(rules(found), ["TELL-18"]) + self.assertEqual((found[0].line, found[0].column), (3, 1)) + self.assertIn("placeholder paragraph", found[0].message) + + def test_rejects_the_paragraph_after_an_edit_of_its_end_or_its_layout(self) -> None: + bodies = [ + PLACEHOLDER_PARAGRAPH + " This change fixes a unit.", + " " + PLACEHOLDER_PARAGRAPH, + PLACEHOLDER_PARAGRAPH.removesuffix("."), + "Write two or three sentences of plain prose\nthat say what this change does.", + ] + for body in bodies: + with self.subTest(body=body): + found = check_squash_message(squash("fix: a unit", body=body)) + self.assertEqual(rules(found), ["TELL-18"]) + + def test_accepts_a_body_that_only_mentions_the_words(self) -> None: + bodies = [ + "The template asks the author to write two or three sentences of plain prose.", + "The template says: Write two or three sentences of plain prose.", + ] + for body in bodies: + with self.subTest(body=body): + found = check_squash_message(squash("docs: fix a template", body=body)) + self.assertEqual(found, []) + + def test_rejects_an_empty_closes_line(self) -> None: + for body in ("Closes #", "closes #", "Closes # ", " Closes #"): + with self.subTest(body=body): + found = check_squash_message(squash("fix: a unit", body=body)) + self.assertEqual(rules(found), ["TELL-18"]) + self.assertEqual(found[0].line, 3) + self.assertIn("Closes #", found[0].message) + + def test_accepts_a_closes_line_with_an_issue(self) -> None: + for body in ("Closes #5", "Closes #5, closes #6", "Fixes #12"): + with self.subTest(body=body): + self.assertEqual(check_squash_message(squash("fix: a unit", body=body)), []) + + +class SignedOffRangeTest(unittest.TestCase): + def test_reports_each_commit_without_a_sign_off(self) -> None: + with git_repo() as root: + base = commit(root, "chore: start") + signed = commit(root, "feat: a\n\n" + SIGN_OFF_LINE) + unsigned = commit(root, "fix: b") + found = check_signed_off(root, f"{base}..HEAD") + self.assertEqual([(v.path, v.rule) for v in found], [(f"commit {unsigned[:12]}", "GIT-2")]) + self.assertNotIn(signed[:12], str(found)) + + def test_accepts_a_range_where_each_commit_has_a_sign_off(self) -> None: + with git_repo() as root: + base = commit(root, "chore: start") + commit(root, "feat: a\n\n" + SIGN_OFF_LINE) + commit(root, "fix: b\n\nWhy.\n\n" + SIGN_OFF_LINE) + self.assertEqual(check_signed_off(root, f"{base}..HEAD"), []) + + def test_ignores_the_commits_before_the_range(self) -> None: + with git_repo() as root: + commit(root, "chore: start") + base = commit(root, "chore: second") + commit(root, "feat: a\n\n" + SIGN_OFF_LINE) + self.assertEqual(check_signed_off(root, f"{base}..HEAD"), []) + + def test_skips_a_merge_commit(self) -> None: + with git_repo() as root: + base = commit(root, "chore: start") + branch = git(root, "rev-parse", "--abbrev-ref", "HEAD") + git(root, "checkout", "-q", "-b", "topic") + commit(root, "feat: a\n\n" + SIGN_OFF_LINE) + git(root, "checkout", "-q", branch) + commit(root, "fix: b\n\n" + SIGN_OFF_LINE) + git( + root, "-c", "commit.gpgsign=false", "merge", "--no-ff", "-q", "-m", "Merge", "topic" + ) + self.assertEqual(check_signed_off(root, f"{base}..HEAD"), []) + + def test_reads_a_line_that_starts_with_a_hash_as_text(self) -> None: + with git_repo() as root: + base = commit(root, "chore: start") + commit(root, "fix: b\n\n# Signed-off-by: Ada Bello ") + self.assertEqual(rules(check_signed_off(root, f"{base}..HEAD")), ["GIT-2"]) + + @unittest.skipUnless(shutil.which("ssh-keygen"), "ssh-keygen makes the signing key") + def test_gives_the_hash_of_a_signed_commit_when_git_shows_signatures(self) -> None: + with git_repo() as root, tempfile.TemporaryDirectory() as keys: + key = Path(keys) / "key" + subprocess.run( + ["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(key)], check=True + ) + allowed = Path(keys) / "allowed" + allowed.write_text(f"ada@example.org {key.with_suffix('.pub').read_text()}") + settings = { + "gpg.format": "ssh", + "user.signingkey": str(key), + "gpg.ssh.allowedSignersFile": str(allowed), + "log.showSignature": "true", + } + for name, value in settings.items(): + git(root, "config", name, value) + base = commit(root, "chore: start") + git(root, "commit", "--allow-empty", "-q", "-S", "-m", "fix: b") + signed = git(root, "rev-parse", "HEAD") + found = check_signed_off(root, f"{base}..HEAD") + self.assertEqual([violation.path for violation in found], [f"commit {signed[:12]}"]) + + def test_does_not_reject_non_ascii_in_a_commit_message(self) -> None: + with git_repo() as root: + base = commit(root, "chore: start") + commit(root, f"fix: caf{E_ACUTE}\n\n" + SIGN_OFF_LINE) + self.assertEqual(check_signed_off(root, f"{base}..HEAD"), []) + + def test_does_not_run_the_other_message_rules_on_a_commit(self) -> None: + with git_repo() as root: + base = commit(root, "chore: start") + commit(root, "Update files " + EMOJI + "\n\n" + SIGN_OFF_LINE) + self.assertEqual(check_signed_off(root, f"{base}..HEAD"), []) + + def test_reads_the_range_from_the_command_line(self) -> None: + with git_repo() as root: + base = commit(root, "chore: start") + unsigned = commit(root, "fix: b") + status, output, errors = run_main("--root", str(root), "--signed-off", f"{base}..HEAD") + self.assertEqual((status, errors), (1, "")) + self.assertEqual(len(output.splitlines()), 1) + self.assertTrue(output.startswith(f"commit {unsigned[:12]}:1:1: GIT-2 "), output) + + def test_exits_with_0_when_each_commit_has_a_sign_off(self) -> None: + with git_repo() as root: + base = commit(root, "chore: start") + commit(root, "fix: b\n\n" + SIGN_OFF_LINE) + result = run_main("--root", str(root), "--signed-off", f"{base}..HEAD") + self.assertEqual(result, (0, "", "")) + + def test_reports_a_range_that_git_cannot_read(self) -> None: + with git_repo() as root: + commit(root, "chore: start") + status, output, errors = run_main("--root", str(root), "--signed-off", "nope..HEAD") + self.assertEqual((status, output), (2, "")) + self.assertEqual(len(errors.splitlines()), 1, errors) + self.assertIn("nope..HEAD", errors) + + def test_does_not_read_a_range_as_an_option(self) -> None: + with git_repo() as root: + commit(root, "chore: start") + status, output, errors = run_main("--root", str(root), "--signed-off=--all") + self.assertEqual((status, output), (2, "")) + self.assertEqual(len(errors.splitlines()), 1, errors) + + +class SquashCommandTest(unittest.TestCase): + def run_with_message(self, text: str, *options: str) -> tuple[int, str, str]: + with tempfile.TemporaryDirectory() as folder: + message = Path(folder) / "message" + message.write_text(text, encoding="utf-8") + return run_main("--squash-msg", str(message), *options) + + def test_exits_with_0_for_a_good_message(self) -> None: + result = self.run_with_message(squash("fix: reject a unit (#7)"), "--no-scope") + self.assertEqual(result, (0, "", "")) + + def test_exits_with_1_and_prints_each_problem(self) -> None: + status, output, errors = self.run_with_message("Update files\n\nBody\n") + self.assertEqual((status, errors), (1, "")) + self.assertEqual( + [line.split(" ")[:3] for line in output.splitlines()], + [["commit", "message:1:1:", "GIT-1"], ["commit", "message:1:1:", "GIT-2"]], + ) + + def test_rejects_a_scope_only_with_the_no_scope_option(self) -> None: + message = squash("fix(core): reject a unit (#7)") + self.assertEqual(self.run_with_message(message), (0, "", "")) + status, output, _ = self.run_with_message(message, "--no-scope") + self.assertEqual(status, 1) + self.assertIn("commit message:1:4: GIT-1 Remove the scope (core).", output) + + def test_reads_a_squash_message_that_is_not_utf_8(self) -> None: + with tempfile.TemporaryDirectory() as folder: + message = Path(folder) / "message" + message.write_bytes(b"fix: parse caf\xe9\n\n" + SIGN_OFF_LINE.encode() + b"\n") + status, output, _ = run_main("--squash-msg", str(message)) + self.assertEqual(status, 1) + self.assertIn("commit message:1:15: TELL-14 Byte 0xE9 is not valid UTF-8.", output) + + def test_reports_a_squash_message_file_that_does_not_exist(self) -> None: + status, output, errors = run_main("--squash-msg", "no-such-file") + self.assertEqual((status, output), (2, "")) + self.assertEqual(len(errors.splitlines()), 1) + + def test_asks_for_a_squash_message_when_it_gets_the_no_scope_option(self) -> None: + errors = io.StringIO() + with self.assertRaises(SystemExit) as caught, contextlib.redirect_stderr(errors): + check_tells.main(["--no-scope"]) + self.assertEqual(caught.exception.code, 2) + self.assertIn("--no-scope needs --squash-msg", errors.getvalue()) + + def test_reads_one_kind_of_message_at_a_time(self) -> None: + for options in ( + ["--commit-msg", "a", "--squash-msg", "b"], + ["--commit-msg", "a", "--signed-off", "b..c"], + ["--squash-msg", "a", "--signed-off", "b..c"], + ): + with self.subTest(options=options): + errors = io.StringIO() + with self.assertRaises(SystemExit) as caught, contextlib.redirect_stderr(errors): + check_tells.main(options) + self.assertEqual(caught.exception.code, 2) diff --git a/standards/CODING_STANDARDS.md b/standards/CODING_STANDARDS.md index 33e70c0..924f77e 100644 --- a/standards/CODING_STANDARDS.md +++ b/standards/CODING_STANDARDS.md @@ -115,6 +115,8 @@ It covers the tool rules that no language tool covers: - **TELL-13.** It rejects the debug calls that the language file lists. It skips test files, comment lines and, in Python, doctest lines that start with `>>>` or `...`. - **TELL-14.** It rejects emojis in every file, including Markdown, message catalogues and lockfiles. It rejects other non-ASCII characters in code files, config files and commit messages. It also rejects a code or config file that is not valid UTF-8, because it cannot read the file. It skips any other file that is not valid UTF-8, because the file can be an image. Message catalogues, docs prose and the sign-off lines of commit messages are exempt from the non-ASCII check, because translations and contributors' names can need any script. A message catalogue is a data file: a file that is not code in a folder named `locales`, `i18n`, `l10n` or `messages`, a `.po`, `.arb`, `.xlf`, `.xliff`, `.strings` or `.stringsdict` file, or an Android `strings.xml`. A code file in such a folder is code, so translations live in data files. - **TELL-18.** With `--commit-msg `, it checks the length of a commit subject. Repos without commitlint use this mode. +- **GIT-1.** With `--squash-msg `, it checks the message that a squash merge puts on `main`. That message is the pull request title, then ` (#N)`, then the pull request body. The subject must start with a type from the Conventional Commits list, then a colon and a space. TELL-14 and TELL-18 apply to the same message, and the ` (#N)` counts towards the 72 characters. The tool leaves out the HTML comments of the body, which no reader sees. Renovate ends each body with one. With `--no-scope`, the subject must also have no scope, as GIT-1 asks of a repo without packages. +- **GIT-2.** The squash message must have a line `Signed-off-by: Name
`, and the address must not be the placeholder of the pull request template. The message must not hold the placeholder paragraph or the line `Closes #` of that template. With `--signed-off `, the tool checks that each commit in a git revision range has a `Signed-off-by` line. It skips merge commits, and it applies no other message rule to a commit, because only the squash message reaches `main`. A line in a code or config file can skip one check with a comment that names the rule and gives a reason, for example `check-tells: allow TELL-1 because the regex cannot be split`. Reviewers check each skip. @@ -181,7 +183,7 @@ The command exits with 0 when all files pass, with 1 when a rule is broken, and - **GIT-3 MUST.** Commits do not carry co-author lines for AI tools. The person who opens the pull request owns the change. - **GIT-4 MUST.** Nobody force-pushes to `main` or to a shared branch. - **GIT-5 SHOULD.** A pull request changes at most about 400 lines. Tests, vectors and generated files do not count. -- **GIT-6 MUST.** A pull request needs one approval and green CI. Merges use squash. +- **GIT-6 MUST.** A pull request needs one approval and green CI. Merges use squash. The title of the pull request becomes the subject of the commit on `main`, and its body becomes the body, so both follow GIT-1, GIT-2 and TELL-18. - **GIT-7 MUST.** A pull request that changes the public interface also updates the API report, the docs and a change file. **(tool)** - **GIT-8 MUST.** Each package exposes a `SPEC_VERSION` constant. It names the spec version that the package implements. From 9a02477a7caa3ddc76e1dd4596bba2297e0af965 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 13:29:58 +0100 Subject: [PATCH 34/45] docs: state each rule in the standards and fix the templates Five rules, one review item and a table named tools that only the maintainers use, so an outside contributor could not meet them. Each rule now states the property that it needs. DOC-4 gives the writing rules. API-12, DOC-6, UI-5, NPG-8 and the review checklist give theirs. The table is gone. LIC-1 now says what the repos do. A file whose format has comments starts with the two SPDX lines, and REUSE.toml covers Markdown, JSON and YAML files. The text no longer quotes the licence tag, so it needs no ignore markers. DOC-5 allows the changelog format of the release tool, and it names Changesets. The definition of a message catalogue moves next to the kinds of file, and the two rules that use the term point to it. The check-tells section and the language files use the word code for the files that hold code. The pull request body is now the body of the squash commit on main, so the pull request template holds only that text: a paragraph of prose, a Closes line and a Signed-off-by line. The check refuses the placeholder text, and a test ties the template to the check. The checklist moves into CONTRIBUTING.md as plain sentences. CONTRIBUTING.md and AGENTS.md told a contributor to add a remediation commit after a DCO check. No DCO app runs, and the new job needs a sign-off in each commit, so a remediation commit does not help. Both files now say to run git rebase --signoff, and then to push to the contributor's own branch with --force-with-lease. CONTRIBUTING.md no longer points to a code of conduct that no repo holds. AGENTS.md says that spec/ holds the grammar with its Interface section. Signed-off-by: MobileMage --- scripts/tests/test_check_tells.py | 86 ++++++++++++++++++++++++++++++ standards/CODING_STANDARDS.md | 43 +++++---------- standards/languages/csharp.md | 2 +- standards/languages/dart.md | 2 +- standards/languages/go.md | 2 +- standards/languages/kotlin-java.md | 2 +- standards/languages/python.md | 2 +- standards/languages/swift.md | 2 +- templates/AGENTS.md | 4 +- templates/CONTRIBUTING.md | 24 ++++++--- templates/PULL_REQUEST_TEMPLATE.md | 18 ++----- 11 files changed, 127 insertions(+), 60 deletions(-) diff --git a/scripts/tests/test_check_tells.py b/scripts/tests/test_check_tells.py index 9979c98..eba20ed 100644 --- a/scripts/tests/test_check_tells.py +++ b/scripts/tests/test_check_tells.py @@ -1591,3 +1591,89 @@ def test_reads_one_kind_of_message_at_a_time(self) -> None: with self.assertRaises(SystemExit) as caught, contextlib.redirect_stderr(errors): check_tells.main(options) self.assertEqual(caught.exception.code, 2) + + +def read_template(name: str) -> str: + """Read a file of templates/, which the pull request check and the contributors share.""" + return (Path(__file__).resolve().parents[2] / "templates" / name).read_text(encoding="utf-8") + + +class PullRequestTemplateTest(unittest.TestCase): + TITLE = "fix: reject a unit of 00 (#7)" + + def test_holds_the_paragraph_the_closes_line_and_the_sign_off_line(self) -> None: + lines = read_template("PULL_REQUEST_TEMPLATE.md").split("\n") + self.assertEqual( + lines, + [ + PLACEHOLDER_PARAGRAPH, + "", + "Closes #", + "", + "Signed-off-by: Your Name ", + "", + ], + ) + + def test_has_no_heading_checklist_or_html_comment(self) -> None: + text = read_template("PULL_REQUEST_TEMPLATE.md") + starts = ("#", "- [", "* [", " -- **LIC-1 MUST.** Each file has licence information that `reuse lint` accepts. Where the file format allows comments, the file starts with `SPDX-License-Identifier: Apache-2.0`. Other files, such as JSON, get an entry in `REUSE.toml`. **(tool)** - +- **LIC-1 MUST.** Each file has licence information that `reuse lint` accepts. A file whose format has comments starts with two SPDX lines, one for the copyright and one for the licence `Apache-2.0`, unless `REUSE.toml` covers it. `REUSE.toml` covers Markdown, JSON and YAML files, and a few plain files such as `NOTICE`. **(tool)** - **LIC-2 MUST.** Each repo has `LICENSE` and `NOTICE`. - **LIC-3 MUST.** Each README and each package description says "Unofficial. Not made or endorsed by NIPOST." @@ -230,7 +228,7 @@ A new package can publish its first release only when all of these are true: - **NPG-5.** The repo holds the API report or API dump. - **NPG-6.** The README quickstart runs in CI. - **NPG-7.** The coverage floors pass. -- **NPG-8.** A second person reviewed the whole package with the `code-review` skill. Both axes, Standards and Spec, are clean. +- **NPG-8.** A second person reviewed the whole package twice, once against these standards and once against the spec. Neither review has an open finding. ### Release gate @@ -260,19 +258,4 @@ Tools check the **(tool)** rules. Reviewers check these by hand: 4. The tests: T-4 and T-6. 5. The docs: DOC-1, DOC-4 and DOC-7. 6. The tells that need a person: TELL-4, TELL-5, TELL-6, TELL-8, TELL-11, TELL-15, TELL-19 and TELL-20. -7. The smell baseline in the `code-review` skill. Treat each smell as a judgement call. - -## Skills for each stage - -| Stage | Skill | -|---|---| -| Design an interface | `codebase-design`, then `critic` on the result | -| Name a concept or record a decision | `domain-modeling` | -| Write code | `test-driven-development` | -| Write tests | `authoring-test-cases` | -| Run a plan | `subagent-driven-development` | -| Claim that work is done | `verification-before-completion` | -| Review a branch | `code-review` for both axes, `simplify` for cleanups, `security-review` for keys and input handling | -| Write prose | `asd-ste100`, then `unslop` | -| Design a screen | `frontend-design` | -| Write agent docs or skills | `writing-for-agents` | +7. Code smells that no rule names, such as a function that does several jobs, or the same logic in two places. Treat each smell as a judgement call. diff --git a/standards/languages/csharp.md b/standards/languages/csharp.md index fee5e70..0141c1a 100644 --- a/standards/languages/csharp.md +++ b/standards/languages/csharp.md @@ -30,7 +30,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for C# c | Stryker.NET | mutation tests | core types, weekly scheduled job | | SourceLink, deterministic builds and symbol packages | debugging support | `ContinuousIntegrationBuild` in CI, `.snupkg` files | | changie | change files and changelog | one change file per user-visible change | -| `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in source, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | +| `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in code, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | | REUSE | licence headers | `reuse lint` | | gitleaks | secret scan | each pull request and each push to `main` | diff --git a/standards/languages/dart.md b/standards/languages/dart.md index 69f1fda..89e858b 100644 --- a/standards/languages/dart.md +++ b/standards/languages/dart.md @@ -25,7 +25,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for Dart | dart_apitool | API check against the last release | `diff`, with `--old` set to the last release and `--new` set to the working copy | | pana | pub.dev score | 160 of 160 points | | changie | change files and changelog | one change file per user-visible change | -| `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in source, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | +| `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in code, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | | REUSE | licence headers | `reuse lint` | | gitleaks | secret scan | each pull request and each push to `main` | diff --git a/standards/languages/go.md b/standards/languages/go.md index 951196f..e877262 100644 --- a/standards/languages/go.md +++ b/standards/languages/go.md @@ -26,7 +26,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for Go c | govulncheck | vulnerability scan | each pull request and each push to `main` | | gorelease | API check against the last tag | enforces GIT-7 and API-13 | | changie | change files and changelog | one change file per user-visible change | -| `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in source, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | +| `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in code, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | | REUSE | licence headers | `reuse lint` | | gitleaks | secret scan | each pull request and each push to `main` | diff --git a/standards/languages/kotlin-java.md b/standards/languages/kotlin-java.md index 71d592b..711f045 100644 --- a/standards/languages/kotlin-java.md +++ b/standards/languages/kotlin-java.md @@ -31,7 +31,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for Kotl | Dokka | API docs | every public symbol | | Android Lint | Android checks | Android modules only, warnings as errors | | changie | change files and changelog | one change file per user-visible change | -| `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in source, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | +| `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in code, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | | REUSE | licence headers | `reuse lint` | | gitleaks | secret scan | each pull request and each push to `main` | diff --git a/standards/languages/python.md b/standards/languages/python.md index 830bb77..50b8508 100644 --- a/standards/languages/python.md +++ b/standards/languages/python.md @@ -26,7 +26,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for Pyth | mutmut | mutation tests | the core modules, weekly scheduled job | | griffe | API check against the last tag | `griffe check gatepost_postcode --against ` | | towncrier | change files and changelog | one fragment in `changes/` per user-visible change | -| `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in source, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | +| `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in code, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | | REUSE | licence headers | `reuse lint` | | gitleaks | secret scan | each pull request and each push to `main` | diff --git a/standards/languages/swift.md b/standards/languages/swift.md index dab4950..7f3abdb 100644 --- a/standards/languages/swift.md +++ b/standards/languages/swift.md @@ -24,7 +24,7 @@ This file and `standards/CODING_STANDARDS.md` together are the standard for Swif | `swift package diagnose-api-breaking-changes` | API check against the last tag | runs on a macOS runner | | DocC | API docs | `swift-docc-plugin` | | changie | change files and changelog | one change file per user-visible change | -| `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in source, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | +| `spec/scripts/check-tells` | agent-tell checks | line length, emojis, other non-ASCII in code, the file names `utils`, `helpers`, `common` and `misc`, the `TODO(#123)` form and debug output | | REUSE | licence headers | `reuse lint` | | gitleaks | secret scan | each pull request and each push to `main` | diff --git a/templates/AGENTS.md b/templates/AGENTS.md index 15bbb6e..570be8a 100644 --- a/templates/AGENTS.md +++ b/templates/AGENTS.md @@ -7,7 +7,7 @@ This repo is part of Gatepost, unofficial open-source developer tools for Nigeri - `CODING_STANDARDS.md` and the two files it names hold the rules. Cite rule IDs, such as `API-3`, in reviews and commit bodies. - `CONTEXT.md` is the glossary. Name things with its terms in code, tests, docs and commits. - `docs/adr/` holds the decisions. When a change contradicts an ADR, say so in the pull request. -- `spec/` holds the interfaces and the vectors. The spec wins over code and over these docs. +- `spec/` holds the grammar, with its Interface section, and the vectors. The spec wins over code and over these docs. ## Rules that no config file shows @@ -15,7 +15,7 @@ This repo is part of Gatepost, unofficial open-source developer tools for Nigeri - Commit only NIPOST's published test codes and synthetic values, such as `FC-01-Z99-ZZ-01`. - Sign off each commit with `git commit -s`. The person who opens the pull request is the only author. Leave out co-author lines for AI tools. - Change `main` only through pull requests. Keep the history of `main` and of other people's branches as it is: never force-push to them. -- If a commit lacks its sign-off, add a DCO remediation commit, as the DCO check explains. +- If a commit lacks its sign-off, run `git rebase --signoff` with the base branch. Then push to your own branch with `--force-with-lease`. - Send every call to the gateway through the client. The client limits how many calls run at once. - Write prose in plain British English, with short sentences in the active voice. - Some agent tools change a typed backslash-u escape, such as the one for an en dash, into the character itself. Build such characters with code, for example `chr(0x2013)` in Python or `String.fromCodePoint(0x2013)` in TypeScript. In data and prose, write `U+2013`. diff --git a/templates/CONTRIBUTING.md b/templates/CONTRIBUTING.md index c73adf4..b3edcb7 100644 --- a/templates/CONTRIBUTING.md +++ b/templates/CONTRIBUTING.md @@ -25,7 +25,7 @@ Gatepost is unofficial. For problems with NIPOST's own app, data or API, contact - Use Conventional Commits, for example `fix(core): reject a unit of 00` (GIT-1). - Keep the subject at 72 characters or fewer. Explain why in the body (TELL-18). - Sign off each commit with `git commit -s` (GIT-2). The sign-off states that you accept the Developer Certificate of Origin at https://developercertificate.org. -- If you forget a sign-off, add a remediation commit, as the DCO check explains. You do not need to rewrite history. +- If you forget a sign-off, run `git rebase --signoff` with the base branch of your pull request, for example `git rebase --signoff origin/main`. Then push to your own branch with `git push --force-with-lease`. GIT-4 forbids a force-push only to `main` and to shared branches. - You are the only author of your commits. Do not add co-author lines for AI tools (GIT-3). ## AI-assisted contributions @@ -34,9 +34,23 @@ You can use AI tools. You are responsible for each line that you submit. If an A ## Open a pull request +The title and the body of the pull request become the commit message on `main`, and CI checks both. + 1. Keep it to one topic. Aim for 400 changed lines or fewer, not counting tests, vectors and generated files (GIT-5). -2. Fill in the pull request template. -3. Wait for green CI and one approval. We merge with squash (GIT-6). +2. Write the title as a Conventional Commits subject, for example `fix(core): reject a unit of 00` (GIT-1). GitHub adds ` (#N)` to it, and the whole subject has 72 characters or fewer (TELL-18). +3. Fill in the pull request template. + - Replace the first paragraph with two or three sentences of plain prose that say what the change does and why. + - Replace `Closes #` with the number of the issue that the change closes, or remove the line. + - Replace the sign-off line with your own name and address (GIT-2). +4. Before you ask for a review, check these points. + - A failing test came first, and it passes now (T-1). + - The repo's check command passes on your machine. + - Each user-visible change has a change file (DOC-5). + - If the public interface changed, you updated the API report and the docs (GIT-7). + - Each new domain term is in `CONTEXT.md` (DOC-7). + - The diff touches only the lines that the task needs (TELL-15). + - Each commit has a sign-off from `git commit -s` (GIT-2). +5. Wait for green CI and one approval. We merge with squash (GIT-6). ## Rules from NIPOST @@ -46,10 +60,6 @@ Gatepost follows NIPOST's terms and acceptable use policy: - Commit only NIPOST's published test codes and synthetic values. Do not commit real API responses, NINs, email addresses or the postcodes of private homes. - Do not use NIPOST's name or logo as branding. -## Conduct - -Everyone in this project follows the Code of Conduct. Report a problem to {{CONDUCT_EMAIL}}. - ## Licence You submit each contribution under the Apache-2.0 licence. diff --git a/templates/PULL_REQUEST_TEMPLATE.md b/templates/PULL_REQUEST_TEMPLATE.md index e7110a0..497a866 100644 --- a/templates/PULL_REQUEST_TEMPLATE.md +++ b/templates/PULL_REQUEST_TEMPLATE.md @@ -1,17 +1,5 @@ -## What and why +Write two or three sentences of plain prose that say what this change does and why. - +Closes # -## Checklist - -- [ ] I wrote a failing test first, and it passes now (T-1). -- [ ] The repo's check command passes on my machine. -- [ ] Each user-visible change has a change file (DOC-5). -- [ ] If the public interface changed, I updated the API report and the docs (GIT-7). -- [ ] Each new domain term is in `CONTEXT.md` (DOC-7). -- [ ] The diff touches only the lines that the task needs (TELL-15). -- [ ] Each commit has a sign-off from `git commit -s` (GIT-2). - -## AI assistance - - +Signed-off-by: Your Name From 8ed85fa919aa9409d44532b2c66f4f09d6de75db Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 13:43:37 +0100 Subject: [PATCH 35/45] docs: name the interface and the grammar reading in the README The README lists what each folder holds. It now says that grammar.md ends with the public interface, and that scripts/ holds the plain reading of the grammar that checks each vector. Signed-off-by: MobileMage --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index fb23914..8efd4e6 100644 --- a/README.md +++ b/README.md @@ -8,13 +8,13 @@ The shared contract for every Gatepost SDK: the postcode grammar, the data files | Path | Holds | |---|---| -| `grammar.md` | how every SDK reads and writes postcodes | +| `grammar.md` | how every SDK reads and writes postcodes, and the public interface | | `data/` | state codes, GPS accuracy limits and the postcode format | | `vectors/` | shared test cases that every SDK must pass | | `standards/` | the coding standards and one file for each language | | `CONTEXT.md` | the glossary | | `templates/` | files for new repos | -| `scripts/` | `check-tells` and the vector builder | +| `scripts/` | `check-tells`, the vector builder, and a plain reading of the grammar that checks each vector | ## Check From 6ca79a0728f12c293b606dbd4720b2a1dfcc2bf5 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 14:15:27 +0100 Subject: [PATCH 36/45] refactor: split check-tells into modules CS-3 says that a file stays under about 300 lines. After the last fixes, check_tells.py had 707 lines and its test file had 1679 lines. The code is public, so it must read like a well-made tool and not like one long script. check-tells is now the package scripts/check_tells/. Each module has one job. The modules hold the result of a check, the lines and characters of a text, the kind of a file, the debug calls of TELL-13, the checks of one file, the checks of a message, the reading of files and commits, and the command line. Each table of rules sits in the module that reads it. The tests follow the same split. The sample input and the git helpers that several test modules need have modules of their own. Each new file has fewer than 300 lines. The tool does the same work as before. The command has the same options, exit codes and messages. The wrapper scripts/check-tells keeps its import, and only its docstring changes. The package still exports the names that the tests use. Each moved class and function keeps its syntax tree. Six helpers are now public, because they cross a module border. The number of tests is the same. Signed-off-by: MobileMage --- scripts/check-tells | 2 +- scripts/check_tells.py | 707 ------- scripts/check_tells/__init__.py | 48 + scripts/check_tells/command.py | 84 + scripts/check_tells/debug_calls.py | 63 + scripts/check_tells/file_checks.py | 112 ++ scripts/check_tells/file_kinds.py | 185 ++ scripts/check_tells/message_checks.py | 185 ++ scripts/check_tells/scan.py | 121 ++ scripts/check_tells/text.py | 29 + scripts/check_tells/violation.py | 22 + scripts/tests/check_tells_driver.py | 63 + scripts/tests/check_tells_samples.py | 126 ++ scripts/tests/test_check_tells.py | 1679 ----------------- scripts/tests/test_check_tells_command.py | 219 +++ scripts/tests/test_check_tells_debug_calls.py | 257 +++ scripts/tests/test_check_tells_file_checks.py | 220 +++ scripts/tests/test_check_tells_file_kinds.py | 81 + .../tests/test_check_tells_message_checks.py | 280 +++ .../tests/test_check_tells_pull_request.py | 202 ++ scripts/tests/test_check_tells_scan.py | 175 ++ scripts/tests/test_check_tells_text.py | 172 ++ 22 files changed, 2645 insertions(+), 2387 deletions(-) delete mode 100644 scripts/check_tells.py create mode 100644 scripts/check_tells/__init__.py create mode 100644 scripts/check_tells/command.py create mode 100644 scripts/check_tells/debug_calls.py create mode 100644 scripts/check_tells/file_checks.py create mode 100644 scripts/check_tells/file_kinds.py create mode 100644 scripts/check_tells/message_checks.py create mode 100644 scripts/check_tells/scan.py create mode 100644 scripts/check_tells/text.py create mode 100644 scripts/check_tells/violation.py create mode 100644 scripts/tests/check_tells_driver.py create mode 100644 scripts/tests/check_tells_samples.py delete mode 100644 scripts/tests/test_check_tells.py create mode 100644 scripts/tests/test_check_tells_command.py create mode 100644 scripts/tests/test_check_tells_debug_calls.py create mode 100644 scripts/tests/test_check_tells_file_checks.py create mode 100644 scripts/tests/test_check_tells_file_kinds.py create mode 100644 scripts/tests/test_check_tells_message_checks.py create mode 100644 scripts/tests/test_check_tells_pull_request.py create mode 100644 scripts/tests/test_check_tells_scan.py create mode 100644 scripts/tests/test_check_tells_text.py diff --git a/scripts/check-tells b/scripts/check-tells index a7dc92a..277fa02 100755 --- a/scripts/check-tells +++ b/scripts/check-tells @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # SPDX-FileCopyrightText: 2026 The Gatepost authors # SPDX-License-Identifier: Apache-2.0 -"""Run check_tells.py. The rules are in that file.""" +"""Run the check_tells package. The rules are in that package.""" import sys from pathlib import Path diff --git a/scripts/check_tells.py b/scripts/check_tells.py deleted file mode 100644 index 6447705..0000000 --- a/scripts/check_tells.py +++ /dev/null @@ -1,707 +0,0 @@ -# SPDX-FileCopyrightText: 2026 The Gatepost authors -# SPDX-License-Identifier: Apache-2.0 -"""Find the agent tells that language tools miss. - -The rules come from standards/CODING_STANDARDS.md: TELL-1, TELL-7, TELL-13, TELL-14 -and TELL-18, and CS-6 for the form of to-do comments. For the message that a squash merge -puts on main, it also checks GIT-1 and GIT-2. -""" - -from __future__ import annotations - -import argparse -import re -import subprocess -import sys -from collections.abc import Callable -from dataclasses import dataclass -from pathlib import Path, PurePosixPath -from typing import Literal - -# Code and config files get the strict rules. The other kinds can use any script, so they get -# only the emoji check: prose (Markdown), catalogue (translations), exempt (lockfiles, data and -# images) and other (a file type that no rule names). -Kind = Literal["code", "config", "prose", "catalogue", "exempt", "other"] -NumberedLine = tuple[int, str] - -MAX_LINE = 100 -TAB_WIDTH = 4 -MAX_SUBJECT = 72 -LAST_ASCII = 0x7F -EXIT_BAD_INPUT = 2 -GENERATED_MARKERS = ("@generated", "DO NOT EDIT", "$:])(var_dump|print_r|var_export|dd|dump|error_log|debug_print_backtrace)" - r"\s*\(" - ), - ), - "python": ( - re.compile(r"(?>> and ...) start the examples in Python docstrings. -DOCTEST_LINE = re.compile(r"^\s*(>>>|\.\.\.)") -URL = re.compile(r"https?://") -# The words are split, so that this line does not match its own pattern. -TODO_FORM = re.compile(r"\b(?:TO" r"DO|FIX" r"ME)\b(?!\(#\d+\))", re.IGNORECASE) -SKIP = re.compile(r"check-tells: allow ([A-Z]+-\d+)(?: because (\S.*))?") -# A type, a scope that can be missing, a mark for a breaking change that can be missing, a colon, -# a space and text. -SUBJECT_FORM = re.compile(r"(?P[a-z]+)(?P\([^()\s]+\))?!?: \S") -SIGN_OFF = re.compile(r"signed-off-by:[ \t]+[^<>\s][^<>]*<[^<>\s]+@[^<>\s]+>[ \t]*", re.IGNORECASE) -# The address in the sign-off line of templates/PULL_REQUEST_TEMPLATE.md. -PLACEHOLDER_ADDRESS = "you@example.com" -# The start of the paragraph in that template that asks the author for prose. -TEMPLATE_PARAGRAPH = "Write two or three sentences of plain prose" -EMPTY_CLOSES_LINE = re.compile(r"closes #\s*", re.IGNORECASE) -HTML_COMMENT = re.compile(r"", re.DOTALL) -SQUASH_NOTE = " The subject is the title plus the ' (#N)' that the squash merge adds." -EMOJI_RANGES = ( - (0x1F000, 0x1FAFF), - (0x231A, 0x231B), - (0x23E9, 0x23F3), - (0x2600, 0x27BF), - (0x2B00, 0x2BFF), - (0xFE0F, 0xFE0F), -) - - -@dataclass(frozen=True, order=True) -class Violation: - """One broken rule at one place in a file.""" - - path: str - line: int - column: int - rule: str - message: str - - def __str__(self) -> str: - """Format the violation like a compiler message.""" - return f"{self.path}:{self.line}:{self.column}: {self.rule} {self.message}" - - -class InputError(Exception): - """The command line names something that check-tells cannot read.""" - - -@dataclass(frozen=True) -class FileInfo: - """What a file holds, which decides the rules that apply to it.""" - - path: str - kind: Kind - language: str | None - is_test: bool - is_generated: bool - - -def _split_lines(text: str) -> list[str]: - """Split text at line feeds only, and remove one carriage return from each line. - - str.splitlines also splits at U+2028, U+2029, U+0085 and the form feed. That hides the - characters that TELL-14 looks for, and it moves the line numbers. - """ - return [line.removesuffix("\r") for line in text.split("\n")] - - -def classify(path: str, text: str) -> FileInfo: - """Decide the kind and the language of one file.""" - repo_path = _without_dist(PurePosixPath(path)) - head = "\n".join(_split_lines(text)[:5]) - return FileInfo( - path=path, - kind=_kind(repo_path, text), - language=_language(repo_path, text), - is_test=bool(TEST_PATH.search(path)), - is_generated=any(marker in head for marker in GENERATED_MARKERS), - ) - - -def _without_dist(repo_path: PurePosixPath) -> PurePosixPath: - """Read a .dist file as the file that it copies, such as phpunit.xml.dist as phpunit.xml.""" - return repo_path.with_suffix("") if repo_path.suffix.lower() == ".dist" else repo_path - - -def _kind(repo_path: PurePosixPath, text: str) -> Kind: - suffix = repo_path.suffix.lower() - if repo_path.name in LOCKFILES or suffix in EXEMPT_EXTENSIONS: - return "exempt" - if suffix in CODE_EXTENSIONS or (not suffix and text.startswith("#!")): - return "code" - if _is_catalogue(repo_path): - return "catalogue" - if suffix in PROSE_EXTENSIONS: - return "prose" - if suffix in CONFIG_EXTENSIONS or repo_path.name in CONFIG_NAMES: - return "config" - return "other" - - -def _is_catalogue(repo_path: PurePosixPath) -> bool: - in_folder = any(part in CATALOGUE_FOLDERS for part in repo_path.parts[:-1]) - android = repo_path.name == "strings.xml" and repo_path.parent.name.startswith("values") - return in_folder or android or repo_path.suffix.lower() in CATALOGUE_EXTENSIONS - - -def _language(repo_path: PurePosixPath, text: str) -> str | None: - if repo_path.suffix: - return LANGUAGE_BY_EXTENSION.get(repo_path.suffix.lower()) - first_line = _split_lines(text)[0] - if not first_line.startswith("#!"): - return None - interpreter = first_line.split()[-1].rsplit("/", 1)[-1] - return LANGUAGE_BY_INTERPRETER.get(interpreter) - - -def _is_emoji(character: str) -> bool: - return any(low <= ord(character) <= high for low, high in EMOJI_RANGES) - - -def check_text(path: str, text: str) -> list[Violation]: - """Check the name and every line of one file.""" - info = classify(path, text) - violations = _check_name(info) - for number, line in enumerate(_split_lines(text), start=1): - violations.extend(_check_line(info, number, line)) - return violations - - -def _check_name(info: FileInfo) -> list[Violation]: - stem = PurePosixPath(info.path).name.split(".")[0].lower() - if stem not in FORBIDDEN_STEMS: - return [] - message = "Name the file after the concept that it holds, not utils, helpers, common or misc." - return [Violation(info.path, 1, 1, "TELL-7", message)] - - -def _check_line(info: FileInfo, number: int, line: str) -> list[Violation]: - skipped: set[str] = set() - problems: list[Violation] = [] - if info.kind in ("code", "config"): - skipped, problems = _skips(info.path, number, line) - found = [ - violation - for check in (_line_length, _todo_form, _debug_calls, _characters) - for violation in check(info, number, line) - if violation.rule not in skipped - ] - return problems + found - - -def _skips(path: str, number: int, line: str) -> tuple[set[str], list[Violation]]: - skipped: set[str] = set() - problems: list[Violation] = [] - for match in SKIP.finditer(line): - rule, reason = match.group(1), match.group(2) - if rule in SKIPPABLE_RULES and reason: - skipped.add(rule) - continue - message = ( - f"Skip only {', '.join(sorted(SKIPPABLE_RULES))}, and give a reason after because." - ) - problems.append(Violation(path, number, match.start() + 1, "SKIP", message)) - return skipped, problems - - -def _line_length(info: FileInfo, number: int, line: str) -> list[Violation]: - if info.kind not in ("code", "config") or info.is_generated or _is_data_json(info.path): - return [] - width = len(line.expandtabs(TAB_WIDTH)) - if width <= MAX_LINE or URL.search(line): - return [] - message = f"The line is {width} characters wide. Keep lines at {MAX_LINE} or fewer." - return [Violation(info.path, number, MAX_LINE + 1, "TELL-1", message)] - - -def _is_data_json(path: str) -> bool: - repo_path = PurePosixPath(path) - in_folder = any(part in DATA_FOLDERS for part in repo_path.parts[:-1]) - return in_folder and repo_path.suffix.lower() == ".json" - - -def _todo_form(info: FileInfo, number: int, line: str) -> list[Violation]: - if info.kind not in ("code", "config"): - return [] - return [ - Violation( - info.path, - number, - match.start() + 1, - "CS-6", - f"Link an issue in the form {match.group(0).upper()}(#123).", - ) - for match in TODO_FORM.finditer(line) - ] - - -def _is_comment(info: FileInfo, line: str) -> bool: - if COMMENT_LINE.match(line): - return True - return info.language == "python" and bool(DOCTEST_LINE.match(line)) - - -def _debug_calls(info: FileInfo, number: int, line: str) -> list[Violation]: - if info.kind != "code" or info.is_test or info.language is None or _is_comment(info, line): - return [] - patterns = DEBUG_CALLS.get(info.language, ()) - return [ - Violation(info.path, number, match.start() + 1, "TELL-13", f"Remove {match.group(0)!r}.") - for pattern in patterns - for match in pattern.finditer(line) - ] - - -def _characters(info: FileInfo, number: int, line: str) -> list[Violation]: - found = [ - Violation(info.path, number, index + 1, "TELL-14", f"Remove the emoji U+{ord(char):04X}.") - for index, char in enumerate(line) - if _is_emoji(char) - ] - if found or info.kind not in ("code", "config"): - return found - for index, char in enumerate(line): - if ord(char) > LAST_ASCII: - message = f"Write U+{ord(char):04X} as an escape sequence. Code and config use ASCII." - return [Violation(info.path, number, index + 1, "TELL-14", message)] - return [] - - -def _message_lines(text: str) -> list[NumberedLine]: - """Pair each line with its number. Drop git's comment lines, and stop at its scissors line.""" - numbered: list[NumberedLine] = [] - for number, line in enumerate(_split_lines(text), start=1): - if line.startswith("# ") and ">8" in line: - break - if not line.startswith("#"): - numbered.append((number, line)) - return numbered - - -def _subject(lines: list[NumberedLine]) -> NumberedLine | None: - """Find the subject, which is the first line that has text.""" - return next(((number, line) for number, line in lines if line.strip()), None) - - -def _check_subject(lines: list[NumberedLine], note: str = "") -> list[Violation]: - """Check the length of the subject. The note adds advice that fits the message.""" - subject = _subject(lines) - if subject is None or len(subject[1]) <= MAX_SUBJECT: - return [] - number, text = subject - message = f"The subject has {len(text)} characters. Use {MAX_SUBJECT} or fewer.{note}" - return [Violation("commit message", number, MAX_SUBJECT + 1, "TELL-18", message)] - - -def _first_bad_character(number: int, line: str) -> Violation | None: - """Find the first emoji, or the first non-ASCII character outside a sign-off line.""" - allow_non_ascii = line.lower().startswith(SIGN_OFF_PREFIXES) - for index, char in enumerate(line): - if _is_emoji(char) or (ord(char) > LAST_ASCII and not allow_non_ascii): - message = f"Remove U+{ord(char):04X}. Commit messages use ASCII." - return Violation("commit message", number, index + 1, "TELL-14", message) - return None - - -def _check_message(lines: list[NumberedLine], note: str = "") -> list[Violation]: - violations = _check_subject(lines, note) - for number, line in lines: - violation = _first_bad_character(number, line) - if violation is not None: - violations.append(violation) - return violations - - -def _check_subject_form(lines: list[NumberedLine], *, no_scope: bool) -> list[Violation]: - """Check GIT-1: the subject has a type that Conventional Commits lists, and maybe a scope.""" - subject = _subject(lines) - if subject is None: - return [Violation("commit message", 1, 1, "GIT-1", "The message has no subject.")] - number, text = subject - match = SUBJECT_FORM.match(text) - if match is None: - message = ( - "The subject does not follow Conventional Commits. " - f"Write 'type: description', where type is {COMMIT_TYPE_LIST}." - ) - return [Violation("commit message", number, 1, "GIT-1", message)] - if match["type"] not in COMMIT_TYPES: - message = f"'{match['type']}' is not a Conventional Commits type. Use {COMMIT_TYPE_LIST}." - return [Violation("commit message", number, 1, "GIT-1", message)] - if no_scope and match["scope"]: - message = ( - f"Remove the scope {match['scope']}. " - "This repo has no packages, so a subject has no scope." - ) - column = match.start("scope") + 1 - return [Violation("commit message", number, column, "GIT-1", message)] - return [] - - -def _check_sign_off(lines: list[NumberedLine], path: str) -> list[Violation]: - """Check GIT-2: the message has a line 'Signed-off-by: Name
', not the template's.""" - sign_offs = [(number, line) for number, line in lines if SIGN_OFF.fullmatch(line)] - if not sign_offs: - message = ( - "The message has no Signed-off-by line. Add the line 'Signed-off-by: Name
'. " - "git commit --signoff adds it to a commit, and git rebase --signoff adds it to each " - "commit of a branch." - ) - return [Violation(path, 1, 1, "GIT-2", message)] - message = ( - f"The sign-off still holds the placeholder address {PLACEHOLDER_ADDRESS}. " - "Write your own name and address." - ) - return [ - Violation(path, number, 1, "GIT-2", message) - for number, line in sign_offs - if PLACEHOLDER_ADDRESS in line.lower() - ] - - -def _check_template_text(lines: list[NumberedLine]) -> list[Violation]: - """Find the text of the pull request template that the author did not replace.""" - violations: list[Violation] = [] - for number, line in lines: - text = line.strip() - if text.startswith(TEMPLATE_PARAGRAPH): - message = ( - "The body still holds the placeholder paragraph of the pull request template. " - "Replace it with two or three sentences that say what the change does and why." - ) - elif EMPTY_CLOSES_LINE.fullmatch(text): - message = ( - "The body still holds the line 'Closes #'. Add the issue number after it. " - "If the change closes no issue, remove the line." - ) - else: - continue - violations.append(Violation("commit message", number, 1, "TELL-18", message)) - return violations - - -def check_commit_message(text: str) -> list[Violation]: - """Check the subject length and the characters of a commit message.""" - return _check_message(_message_lines(text)) - - -def _blank_html_comments(body: str) -> str: - """Replace each HTML comment with spaces, and keep its line feeds, so that no place moves.""" - return HTML_COMMENT.sub(lambda comment: re.sub(r"[^\n]", " ", comment[0]), body) - - -def check_squash_message(text: str, *, no_scope: bool = False) -> list[Violation]: - """Check the message that a squash merge puts on main: the pull request title and body. - - The body is not a git message, so a line that starts with # is text and not a comment. No - reader sees an HTML comment, and Renovate ends each body with one, so the body check leaves - them out. The title is the first line, and no comment hides text there. - """ - subject, line_feed, body = text.partition("\n") - lines = list(enumerate(_split_lines(subject + line_feed + _blank_html_comments(body)), start=1)) - return [ - *_check_message(lines, SQUASH_NOTE), - *_check_subject_form(lines, no_scope=no_scope), - *_check_sign_off(lines, "commit message"), - *_check_template_text(lines), - ] - - -def _git(root: Path, *arguments: str) -> bytes: - """Run git in root and give its output. A failure is bad input, such as no repo.""" - try: - run = subprocess.run(["git", "-C", str(root), *arguments], check=True, capture_output=True) - except subprocess.CalledProcessError as error: - reason = " ".join(error.stderr.decode(errors="replace").split()) - raise InputError(f"Git failed in {root}: {reason}") from error - return run.stdout - - -def tracked_files(root: Path) -> list[Path]: - """List the files that git tracks, or would track, under root.""" - output = _git(root, "ls-files", "-z", "--cached", "--others", "--exclude-standard") - names = output.decode().split("\0") - return [root / name for name in names if name and (root / name).is_file()] - - -def _commits(root: Path, revision_range: str) -> list[tuple[str, str]]: - """Give the hash and the message of each commit in a range. Merge commits are left out.""" - output = _git( - root, - "log", - "--no-merges", - "--no-show-signature", - "--format=%H%n%B%x00", - "--end-of-options", - revision_range, - ) - records = [record.lstrip("\n") for record in output.decode(errors="replace").split("\0")] - pairs = (record.partition("\n") for record in records if record) - return [(commit, message) for commit, _, message in pairs] - - -def check_signed_off(root: Path, revision_range: str) -> list[Violation]: - """Check that each commit in a revision range has a sign-off. Merge commits are left out.""" - violations: list[Violation] = [] - for commit, message in _commits(root, revision_range): - lines = list(enumerate(_split_lines(message), start=1)) - violations.extend(_check_sign_off(lines, f"commit {commit[:12]}")) - return violations - - -def _read(path: Path) -> bytes: - try: - return path.read_bytes() - except OSError as error: - raise InputError(f"Cannot read {path}: {error.strerror}.") from error - - -def _not_utf8(path: str, data: bytes, error: UnicodeDecodeError, advice: str) -> Violation: - """Point at the first byte that is not UTF-8. All the bytes before it are valid.""" - line_start = data.rfind(b"\n", 0, error.start) + 1 - line = data.count(b"\n", 0, error.start) + 1 - column = len(data[line_start : error.start].decode("utf-8")) + 1 - message = f"Byte 0x{data[error.start]:02X} is not valid UTF-8. {advice}" - return Violation(path, line, column, "TELL-14", message) - - -def _check_undecodable(path: str, data: bytes, error: UnicodeDecodeError) -> list[Violation]: - """Report a code or config file that is not UTF-8. Other files can be binary, so skip them.""" - # Only the suffix and the first bytes decide the kind, and bad bytes would stop the decoder. - kind = classify(path, data[:512].decode("utf-8", errors="replace")).kind - if kind not in ("code", "config"): - return [] - advice = "Save the file as UTF-8, and use ASCII in code and config." - return [_not_utf8(path, data, error, advice)] - - -def check_files(root: Path, files: list[Path]) -> list[Violation]: - """Check each file. Report a code or config file that is not UTF-8, and skip other files.""" - violations: list[Violation] = [] - for file in files: - if not file.is_relative_to(root): - raise InputError(f"{file} is outside the root {root}.") - path = file.relative_to(root).as_posix() - data = _read(file) - try: - text = data.decode("utf-8") - except UnicodeDecodeError as error: - violations.extend(_check_undecodable(path, data, error)) - else: - violations.extend(check_text(path, text)) - return violations - - -def _check_message_file(path: Path, check: Callable[[str], list[Violation]]) -> list[Violation]: - data = _read(path) - try: - text = data.decode("utf-8") - except UnicodeDecodeError as error: - advice = "Save the message as UTF-8, and use ASCII." - return [_not_utf8("commit message", data, error, advice)] - return check(text) - - -def _run(args: argparse.Namespace) -> list[Violation]: - if args.commit_msg is not None: - return _check_message_file(args.commit_msg, check_commit_message) - if args.squash_msg is not None: - return _check_message_file( - args.squash_msg, lambda text: check_squash_message(text, no_scope=args.no_scope) - ) - root = args.root.resolve() - if args.signed_off is not None: - return check_signed_off(root, args.signed_off) - files = [(root / path).resolve() for path in args.paths] or tracked_files(root) - return check_files(root, files) - - -def _parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - prog="check-tells", description="Find the agent tells that language tools miss." - ) - parser.add_argument("--root", type=Path, default=Path.cwd(), help="The repo root.") - messages = parser.add_mutually_exclusive_group() - messages.add_argument( - "--commit-msg", type=Path, help="Check this commit message file (TELL-14 and TELL-18)." - ) - messages.add_argument( - "--squash-msg", - type=Path, - help="Check this file as the message that a squash merge puts on main " - "(TELL-14, TELL-18, GIT-1, GIT-2 and the text of the pull request template).", - ) - messages.add_argument( - "--signed-off", - metavar="RANGE", - help="Check that each commit in this git revision range has a sign-off (GIT-2).", - ) - parser.add_argument( - "--no-scope", - action="store_true", - help="With --squash-msg, reject a scope in the subject " - "(GIT-1 for a repo without packages).", - ) - parser.add_argument("paths", nargs="*", type=Path, help="Files to check. Default: all.") - return parser - - -def main(argv: list[str] | None = None) -> int: - """Run the checks and print each violation. - - Return 0 if all is well, 1 if a rule broke and 2 if the input cannot be read. - """ - parser = _parser() - args = parser.parse_args(argv) - if args.no_scope and args.squash_msg is None: - parser.error("--no-scope needs --squash-msg.") - try: - violations = _run(args) - except InputError as error: - sys.stderr.write(f"check-tells: {error}\n") - return EXIT_BAD_INPUT - for violation in sorted(violations): - sys.stdout.write(f"{violation}\n") - return 1 if violations else 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/check_tells/__init__.py b/scripts/check_tells/__init__.py new file mode 100644 index 0000000..0edff59 --- /dev/null +++ b/scripts/check_tells/__init__.py @@ -0,0 +1,48 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +"""Find the agent tells that language tools miss. + +The rules come from standards/CODING_STANDARDS.md: TELL-1, TELL-7, TELL-13, TELL-14 +and TELL-18, and CS-6 for the form of to-do comments. For the message that a squash merge +puts on main, it also checks GIT-1 and GIT-2. + +Each module has one job: + +- violation: the result of a check. +- text: the lines of a text, and the characters that TELL-14 rejects. +- file_kinds: the kind and the language of a file. +- debug_calls: the debug calls of TELL-13. +- file_checks: the checks of the name and the lines of one file. +- message_checks: the checks of commit messages and squash messages. +- scan: the reading of files, message files and commits. +- command: the command line. +""" + +from check_tells.command import main +from check_tells.file_checks import check_text +from check_tells.file_kinds import FileInfo, Kind, classify +from check_tells.message_checks import ( + PLACEHOLDER_ADDRESS, + TEMPLATE_PARAGRAPH, + check_commit_message, + check_squash_message, +) +from check_tells.scan import InputError, check_files, check_signed_off, tracked_files +from check_tells.violation import Violation + +__all__ = [ + "PLACEHOLDER_ADDRESS", + "TEMPLATE_PARAGRAPH", + "FileInfo", + "InputError", + "Kind", + "Violation", + "check_commit_message", + "check_files", + "check_signed_off", + "check_squash_message", + "check_text", + "classify", + "main", + "tracked_files", +] diff --git a/scripts/check_tells/command.py b/scripts/check_tells/command.py new file mode 100644 index 0000000..edf901c --- /dev/null +++ b/scripts/check_tells/command.py @@ -0,0 +1,84 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +"""The command line: the options, the exit codes and the output.""" + +from __future__ import annotations + +import argparse +import sys +from pathlib import Path + +from check_tells.message_checks import check_commit_message, check_squash_message +from check_tells.scan import ( + InputError, + check_files, + check_message_file, + check_signed_off, + tracked_files, +) +from check_tells.violation import Violation + +EXIT_BAD_INPUT = 2 + + +def _run(args: argparse.Namespace) -> list[Violation]: + if args.commit_msg is not None: + return check_message_file(args.commit_msg, check_commit_message) + if args.squash_msg is not None: + return check_message_file( + args.squash_msg, lambda text: check_squash_message(text, no_scope=args.no_scope) + ) + root = args.root.resolve() + if args.signed_off is not None: + return check_signed_off(root, args.signed_off) + files = [(root / path).resolve() for path in args.paths] or tracked_files(root) + return check_files(root, files) + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="check-tells", description="Find the agent tells that language tools miss." + ) + parser.add_argument("--root", type=Path, default=Path.cwd(), help="The repo root.") + messages = parser.add_mutually_exclusive_group() + messages.add_argument( + "--commit-msg", type=Path, help="Check this commit message file (TELL-14 and TELL-18)." + ) + messages.add_argument( + "--squash-msg", + type=Path, + help="Check this file as the message that a squash merge puts on main " + "(TELL-14, TELL-18, GIT-1, GIT-2 and the text of the pull request template).", + ) + messages.add_argument( + "--signed-off", + metavar="RANGE", + help="Check that each commit in this git revision range has a sign-off (GIT-2).", + ) + parser.add_argument( + "--no-scope", + action="store_true", + help="With --squash-msg, reject a scope in the subject " + "(GIT-1 for a repo without packages).", + ) + parser.add_argument("paths", nargs="*", type=Path, help="Files to check. Default: all.") + return parser + + +def main(argv: list[str] | None = None) -> int: + """Run the checks and print each violation. + + Return 0 if all is well, 1 if a rule broke and 2 if the input cannot be read. + """ + parser = _parser() + args = parser.parse_args(argv) + if args.no_scope and args.squash_msg is None: + parser.error("--no-scope needs --squash-msg.") + try: + violations = _run(args) + except InputError as error: + sys.stderr.write(f"check-tells: {error}\n") + return EXIT_BAD_INPUT + for violation in sorted(violations): + sys.stdout.write(f"{violation}\n") + return 1 if violations else 0 diff --git a/scripts/check_tells/debug_calls.py b/scripts/check_tells/debug_calls.py new file mode 100644 index 0000000..6350842 --- /dev/null +++ b/scripts/check_tells/debug_calls.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +"""TELL-13: the debug calls that each language file lists, and the check of one line.""" + +from __future__ import annotations + +import re + +from check_tells.file_kinds import FileInfo +from check_tells.violation import Violation + +DEBUG_CALLS: dict[str, tuple[re.Pattern[str], ...]] = { + "javascript": (re.compile(r"\bconsole\.(log|debug|dir)\s*\("), re.compile(r"\bdebugger\b")), + "php": ( + re.compile( + r"(?$:])(var_dump|print_r|var_export|dd|dump|error_log|debug_print_backtrace)" + r"\s*\(" + ), + ), + "python": ( + re.compile(r"(?>> and ...) start the examples in Python docstrings. +DOCTEST_LINE = re.compile(r"^\s*(>>>|\.\.\.)") + + +def _is_comment(info: FileInfo, line: str) -> bool: + if COMMENT_LINE.match(line): + return True + return info.language == "python" and bool(DOCTEST_LINE.match(line)) + + +def check_debug_calls(info: FileInfo, number: int, line: str) -> list[Violation]: + """Find the debug calls of the file's language in one line (TELL-13). + + A test file and a comment line have no debug calls. + """ + if info.kind != "code" or info.is_test or info.language is None or _is_comment(info, line): + return [] + patterns = DEBUG_CALLS.get(info.language, ()) + return [ + Violation(info.path, number, match.start() + 1, "TELL-13", f"Remove {match.group(0)!r}.") + for pattern in patterns + for match in pattern.finditer(line) + ] diff --git a/scripts/check_tells/file_checks.py b/scripts/check_tells/file_checks.py new file mode 100644 index 0000000..aa7c29e --- /dev/null +++ b/scripts/check_tells/file_checks.py @@ -0,0 +1,112 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +"""The checks of the name and the lines of one file. + +They cover TELL-1, TELL-7, TELL-13, TELL-14 and CS-6, and the comment that skips one of +them for one line. +""" + +from __future__ import annotations + +import re +from pathlib import PurePosixPath + +from check_tells.debug_calls import check_debug_calls +from check_tells.file_kinds import FileInfo, classify, is_data_json +from check_tells.text import LAST_ASCII, is_emoji, split_lines +from check_tells.violation import Violation + +MAX_LINE = 100 +TAB_WIDTH = 4 +FORBIDDEN_STEMS = frozenset({"utils", "helpers", "common", "misc"}) +SKIPPABLE_RULES = frozenset({"TELL-1", "TELL-13", "TELL-14", "CS-6"}) +URL = re.compile(r"https?://") +# The words are split, so that this line does not match its own pattern. +TODO_FORM = re.compile(r"\b(?:TO" r"DO|FIX" r"ME)\b(?!\(#\d+\))", re.IGNORECASE) +SKIP = re.compile(r"check-tells: allow ([A-Z]+-\d+)(?: because (\S.*))?") + + +def check_text(path: str, text: str) -> list[Violation]: + """Check the name and every line of one file.""" + info = classify(path, text) + violations = _check_name(info) + for number, line in enumerate(split_lines(text), start=1): + violations.extend(_check_line(info, number, line)) + return violations + + +def _check_name(info: FileInfo) -> list[Violation]: + stem = PurePosixPath(info.path).name.split(".")[0].lower() + if stem not in FORBIDDEN_STEMS: + return [] + message = "Name the file after the concept that it holds, not utils, helpers, common or misc." + return [Violation(info.path, 1, 1, "TELL-7", message)] + + +def _check_line(info: FileInfo, number: int, line: str) -> list[Violation]: + skipped: set[str] = set() + problems: list[Violation] = [] + if info.kind in ("code", "config"): + skipped, problems = _skips(info.path, number, line) + found = [ + violation + for check in (_line_length, _todo_form, check_debug_calls, _characters) + for violation in check(info, number, line) + if violation.rule not in skipped + ] + return problems + found + + +def _skips(path: str, number: int, line: str) -> tuple[set[str], list[Violation]]: + skipped: set[str] = set() + problems: list[Violation] = [] + for match in SKIP.finditer(line): + rule, reason = match.group(1), match.group(2) + if rule in SKIPPABLE_RULES and reason: + skipped.add(rule) + continue + message = ( + f"Skip only {', '.join(sorted(SKIPPABLE_RULES))}, and give a reason after because." + ) + problems.append(Violation(path, number, match.start() + 1, "SKIP", message)) + return skipped, problems + + +def _line_length(info: FileInfo, number: int, line: str) -> list[Violation]: + if info.kind not in ("code", "config") or info.is_generated or is_data_json(info.path): + return [] + width = len(line.expandtabs(TAB_WIDTH)) + if width <= MAX_LINE or URL.search(line): + return [] + message = f"The line is {width} characters wide. Keep lines at {MAX_LINE} or fewer." + return [Violation(info.path, number, MAX_LINE + 1, "TELL-1", message)] + + +def _todo_form(info: FileInfo, number: int, line: str) -> list[Violation]: + if info.kind not in ("code", "config"): + return [] + return [ + Violation( + info.path, + number, + match.start() + 1, + "CS-6", + f"Link an issue in the form {match.group(0).upper()}(#123).", + ) + for match in TODO_FORM.finditer(line) + ] + + +def _characters(info: FileInfo, number: int, line: str) -> list[Violation]: + found = [ + Violation(info.path, number, index + 1, "TELL-14", f"Remove the emoji U+{ord(char):04X}.") + for index, char in enumerate(line) + if is_emoji(char) + ] + if found or info.kind not in ("code", "config"): + return found + for index, char in enumerate(line): + if ord(char) > LAST_ASCII: + message = f"Write U+{ord(char):04X} as an escape sequence. Code and config use ASCII." + return [Violation(info.path, number, index + 1, "TELL-14", message)] + return [] diff --git a/scripts/check_tells/file_kinds.py b/scripts/check_tells/file_kinds.py new file mode 100644 index 0000000..a039333 --- /dev/null +++ b/scripts/check_tells/file_kinds.py @@ -0,0 +1,185 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +"""The kind and the language of a file, which decide the rules that apply to it.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import PurePosixPath +from typing import Literal + +from check_tells.text import split_lines + +# Code and config files get the strict rules. The other kinds can use any script, so they get +# only the emoji check: prose (Markdown), catalogue (translations), exempt (lockfiles, data and +# images) and other (a file type that no rule names). +Kind = Literal["code", "config", "prose", "catalogue", "exempt", "other"] + +CODE_EXTENSIONS = frozenset( + { + ".ts", + ".tsx", + ".mts", + ".cts", + ".js", + ".jsx", + ".mjs", + ".cjs", + ".php", + ".py", + ".go", + ".kt", + ".kts", + ".java", + ".cs", + ".dart", + ".swift", + ".sql", + ".sh", + ".bash", + ".css", + ".scss", + ".html", + ".astro", + ".vue", + ".svelte", + } +) +CONFIG_EXTENSIONS = frozenset( + { + ".json", + ".yml", + ".yaml", + ".toml", + ".xml", + ".ini", + ".cfg", + ".properties", + ".gradle", + ".neon", + ".json5", + ".jsonc", + } +) +CONFIG_NAMES = frozenset( + {"Makefile", "Dockerfile", ".editorconfig", ".gitignore", ".gitattributes", ".npmrc"} +) +PROSE_EXTENSIONS = frozenset({".md", ".mdx"}) +EXEMPT_EXTENSIONS = frozenset( + {".svg", ".lock", ".csv", ".txt", ".api", ".snap", ".png", ".jpg", ".ico"} +) +# A lockfile with the suffix .lock is exempt through EXEMPT_EXTENSIONS, so this names the others. +LOCKFILES = frozenset( + { + "pnpm-lock.yaml", + "package-lock.json", + "go.sum", + "Package.resolved", + "gradle.lockfile", + "packages.lock.json", + } +) +CATALOGUE_FOLDERS = frozenset({"locales", "i18n", "l10n", "messages"}) +DATA_FOLDERS = frozenset({"data", "vectors"}) +CATALOGUE_EXTENSIONS = frozenset({".arb", ".po", ".xlf", ".xliff", ".strings", ".stringsdict"}) + +LANGUAGE_BY_EXTENSION = { + ".ts": "javascript", + ".tsx": "javascript", + ".mts": "javascript", + ".cts": "javascript", + ".js": "javascript", + ".jsx": "javascript", + ".mjs": "javascript", + ".cjs": "javascript", + ".astro": "javascript", + ".vue": "javascript", + ".svelte": "javascript", + ".php": "php", + ".py": "python", + ".go": "go", + ".kt": "jvm", + ".kts": "jvm", + ".java": "jvm", + ".cs": "dotnet", + ".dart": "dart", + ".swift": "swift", +} +LANGUAGE_BY_INTERPRETER = {"python3": "python", "python": "python", "node": "javascript"} + +GENERATED_MARKERS = ("@generated", "DO NOT EDIT", " FileInfo: + """Decide the kind and the language of one file.""" + repo_path = _without_dist(PurePosixPath(path)) + head = "\n".join(split_lines(text)[:5]) + return FileInfo( + path=path, + kind=_kind(repo_path, text), + language=_language(repo_path, text), + is_test=bool(TEST_PATH.search(path)), + is_generated=any(marker in head for marker in GENERATED_MARKERS), + ) + + +def _without_dist(repo_path: PurePosixPath) -> PurePosixPath: + """Read a .dist file as the file that it copies, such as phpunit.xml.dist as phpunit.xml.""" + return repo_path.with_suffix("") if repo_path.suffix.lower() == ".dist" else repo_path + + +def _kind(repo_path: PurePosixPath, text: str) -> Kind: + suffix = repo_path.suffix.lower() + if repo_path.name in LOCKFILES or suffix in EXEMPT_EXTENSIONS: + return "exempt" + if suffix in CODE_EXTENSIONS or (not suffix and text.startswith("#!")): + return "code" + if _is_catalogue(repo_path): + return "catalogue" + if suffix in PROSE_EXTENSIONS: + return "prose" + if suffix in CONFIG_EXTENSIONS or repo_path.name in CONFIG_NAMES: + return "config" + return "other" + + +def _is_catalogue(repo_path: PurePosixPath) -> bool: + in_folder = any(part in CATALOGUE_FOLDERS for part in repo_path.parts[:-1]) + android = repo_path.name == "strings.xml" and repo_path.parent.name.startswith("values") + return in_folder or android or repo_path.suffix.lower() in CATALOGUE_EXTENSIONS + + +def _language(repo_path: PurePosixPath, text: str) -> str | None: + if repo_path.suffix: + return LANGUAGE_BY_EXTENSION.get(repo_path.suffix.lower()) + first_line = split_lines(text)[0] + if not first_line.startswith("#!"): + return None + interpreter = first_line.split()[-1].rsplit("/", 1)[-1] + return LANGUAGE_BY_INTERPRETER.get(interpreter) + + +def is_data_json(path: str) -> bool: + """Tell whether the path is a JSON file in a data or vectors folder, where TELL-1 is off.""" + repo_path = PurePosixPath(path) + in_folder = any(part in DATA_FOLDERS for part in repo_path.parts[:-1]) + return in_folder and repo_path.suffix.lower() == ".json" diff --git a/scripts/check_tells/message_checks.py b/scripts/check_tells/message_checks.py new file mode 100644 index 0000000..98c1e5f --- /dev/null +++ b/scripts/check_tells/message_checks.py @@ -0,0 +1,185 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +"""The checks of a commit message and of the message that a squash merge puts on main. + +TELL-14 and TELL-18 apply to both. GIT-1, GIT-2 and the text of the pull request template +apply to the squash message. +""" + +from __future__ import annotations + +import re + +from check_tells.text import LAST_ASCII, is_emoji, split_lines +from check_tells.violation import Violation + +NumberedLine = tuple[int, str] + +MAX_SUBJECT = 72 +# In lower case, because git reads the keys of trailer lines without regard to case. +SIGN_OFF_PREFIXES = ("signed-off-by:", "co-authored-by:") +COMMIT_TYPES = ( + "build", + "chore", + "ci", + "docs", + "feat", + "fix", + "perf", + "refactor", + "revert", + "style", + "test", +) +COMMIT_TYPE_LIST = f"{', '.join(COMMIT_TYPES[:-1])} or {COMMIT_TYPES[-1]}" +# A type, a scope that can be missing, a mark for a breaking change that can be missing, a colon, +# a space and text. +SUBJECT_FORM = re.compile(r"(?P[a-z]+)(?P\([^()\s]+\))?!?: \S") +SIGN_OFF = re.compile(r"signed-off-by:[ \t]+[^<>\s][^<>]*<[^<>\s]+@[^<>\s]+>[ \t]*", re.IGNORECASE) +# The address in the sign-off line of templates/PULL_REQUEST_TEMPLATE.md. +PLACEHOLDER_ADDRESS = "you@example.com" +# The start of the paragraph in that template that asks the author for prose. +TEMPLATE_PARAGRAPH = "Write two or three sentences of plain prose" +EMPTY_CLOSES_LINE = re.compile(r"closes #\s*", re.IGNORECASE) +HTML_COMMENT = re.compile(r"", re.DOTALL) +SQUASH_NOTE = " The subject is the title plus the ' (#N)' that the squash merge adds." + + +def _message_lines(text: str) -> list[NumberedLine]: + """Pair each line with its number. Drop git's comment lines, and stop at its scissors line.""" + numbered: list[NumberedLine] = [] + for number, line in enumerate(split_lines(text), start=1): + if line.startswith("# ") and ">8" in line: + break + if not line.startswith("#"): + numbered.append((number, line)) + return numbered + + +def _subject(lines: list[NumberedLine]) -> NumberedLine | None: + """Find the subject, which is the first line that has text.""" + return next(((number, line) for number, line in lines if line.strip()), None) + + +def _check_subject(lines: list[NumberedLine], note: str = "") -> list[Violation]: + """Check the length of the subject. The note adds advice that fits the message.""" + subject = _subject(lines) + if subject is None or len(subject[1]) <= MAX_SUBJECT: + return [] + number, text = subject + message = f"The subject has {len(text)} characters. Use {MAX_SUBJECT} or fewer.{note}" + return [Violation("commit message", number, MAX_SUBJECT + 1, "TELL-18", message)] + + +def _first_bad_character(number: int, line: str) -> Violation | None: + """Find the first emoji, or the first non-ASCII character outside a sign-off line.""" + allow_non_ascii = line.lower().startswith(SIGN_OFF_PREFIXES) + for index, char in enumerate(line): + if is_emoji(char) or (ord(char) > LAST_ASCII and not allow_non_ascii): + message = f"Remove U+{ord(char):04X}. Commit messages use ASCII." + return Violation("commit message", number, index + 1, "TELL-14", message) + return None + + +def _check_message(lines: list[NumberedLine], note: str = "") -> list[Violation]: + violations = _check_subject(lines, note) + for number, line in lines: + violation = _first_bad_character(number, line) + if violation is not None: + violations.append(violation) + return violations + + +def _check_subject_form(lines: list[NumberedLine], *, no_scope: bool) -> list[Violation]: + """Check GIT-1: the subject has a type that Conventional Commits lists, and maybe a scope.""" + subject = _subject(lines) + if subject is None: + return [Violation("commit message", 1, 1, "GIT-1", "The message has no subject.")] + number, text = subject + match = SUBJECT_FORM.match(text) + if match is None: + message = ( + "The subject does not follow Conventional Commits. " + f"Write 'type: description', where type is {COMMIT_TYPE_LIST}." + ) + return [Violation("commit message", number, 1, "GIT-1", message)] + if match["type"] not in COMMIT_TYPES: + message = f"'{match['type']}' is not a Conventional Commits type. Use {COMMIT_TYPE_LIST}." + return [Violation("commit message", number, 1, "GIT-1", message)] + if no_scope and match["scope"]: + message = ( + f"Remove the scope {match['scope']}. " + "This repo has no packages, so a subject has no scope." + ) + column = match.start("scope") + 1 + return [Violation("commit message", number, column, "GIT-1", message)] + return [] + + +def check_sign_off(lines: list[NumberedLine], path: str) -> list[Violation]: + """Check GIT-2: the message has a line 'Signed-off-by: Name
', not the template's.""" + sign_offs = [(number, line) for number, line in lines if SIGN_OFF.fullmatch(line)] + if not sign_offs: + message = ( + "The message has no Signed-off-by line. Add the line 'Signed-off-by: Name
'. " + "git commit --signoff adds it to a commit, and git rebase --signoff adds it to each " + "commit of a branch." + ) + return [Violation(path, 1, 1, "GIT-2", message)] + message = ( + f"The sign-off still holds the placeholder address {PLACEHOLDER_ADDRESS}. " + "Write your own name and address." + ) + return [ + Violation(path, number, 1, "GIT-2", message) + for number, line in sign_offs + if PLACEHOLDER_ADDRESS in line.lower() + ] + + +def _check_template_text(lines: list[NumberedLine]) -> list[Violation]: + """Find the text of the pull request template that the author did not replace.""" + violations: list[Violation] = [] + for number, line in lines: + text = line.strip() + if text.startswith(TEMPLATE_PARAGRAPH): + message = ( + "The body still holds the placeholder paragraph of the pull request template. " + "Replace it with two or three sentences that say what the change does and why." + ) + elif EMPTY_CLOSES_LINE.fullmatch(text): + message = ( + "The body still holds the line 'Closes #'. Add the issue number after it. " + "If the change closes no issue, remove the line." + ) + else: + continue + violations.append(Violation("commit message", number, 1, "TELL-18", message)) + return violations + + +def check_commit_message(text: str) -> list[Violation]: + """Check the subject length and the characters of a commit message.""" + return _check_message(_message_lines(text)) + + +def _blank_html_comments(body: str) -> str: + """Replace each HTML comment with spaces, and keep its line feeds, so that no place moves.""" + return HTML_COMMENT.sub(lambda comment: re.sub(r"[^\n]", " ", comment[0]), body) + + +def check_squash_message(text: str, *, no_scope: bool = False) -> list[Violation]: + """Check the message that a squash merge puts on main: the pull request title and body. + + The body is not a git message, so a line that starts with # is text and not a comment. No + reader sees an HTML comment, and Renovate ends each body with one, so the body check leaves + them out. The title is the first line, and no comment hides text there. + """ + subject, line_feed, body = text.partition("\n") + lines = list(enumerate(split_lines(subject + line_feed + _blank_html_comments(body)), start=1)) + return [ + *_check_message(lines, SQUASH_NOTE), + *_check_subject_form(lines, no_scope=no_scope), + *check_sign_off(lines, "commit message"), + *_check_template_text(lines), + ] diff --git a/scripts/check_tells/scan.py b/scripts/check_tells/scan.py new file mode 100644 index 0000000..d9753a4 --- /dev/null +++ b/scripts/check_tells/scan.py @@ -0,0 +1,121 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +"""Read the input of a run and check it: the files of a repo, a message file or a range of commits. + +Input that cannot be read raises InputError. +""" + +from __future__ import annotations + +import subprocess +from collections.abc import Callable +from pathlib import Path + +from check_tells.file_checks import check_text +from check_tells.file_kinds import classify +from check_tells.message_checks import check_sign_off +from check_tells.text import split_lines +from check_tells.violation import Violation + + +class InputError(Exception): + """The command line names something that check-tells cannot read.""" + + +def _git(root: Path, *arguments: str) -> bytes: + """Run git in root and give its output. A failure is bad input, such as no repo.""" + try: + run = subprocess.run(["git", "-C", str(root), *arguments], check=True, capture_output=True) + except subprocess.CalledProcessError as error: + reason = " ".join(error.stderr.decode(errors="replace").split()) + raise InputError(f"Git failed in {root}: {reason}") from error + return run.stdout + + +def tracked_files(root: Path) -> list[Path]: + """List the files that git tracks, or would track, under root.""" + output = _git(root, "ls-files", "-z", "--cached", "--others", "--exclude-standard") + names = output.decode().split("\0") + return [root / name for name in names if name and (root / name).is_file()] + + +def _commits(root: Path, revision_range: str) -> list[tuple[str, str]]: + """Give the hash and the message of each commit in a range. Merge commits are left out.""" + output = _git( + root, + "log", + "--no-merges", + "--no-show-signature", + "--format=%H%n%B%x00", + "--end-of-options", + revision_range, + ) + records = [record.lstrip("\n") for record in output.decode(errors="replace").split("\0")] + pairs = (record.partition("\n") for record in records if record) + return [(commit, message) for commit, _, message in pairs] + + +def check_signed_off(root: Path, revision_range: str) -> list[Violation]: + """Check that each commit in a revision range has a sign-off. Merge commits are left out.""" + violations: list[Violation] = [] + for commit, message in _commits(root, revision_range): + lines = list(enumerate(split_lines(message), start=1)) + violations.extend(check_sign_off(lines, f"commit {commit[:12]}")) + return violations + + +def _read(path: Path) -> bytes: + try: + return path.read_bytes() + except OSError as error: + raise InputError(f"Cannot read {path}: {error.strerror}.") from error + + +def _not_utf8(path: str, data: bytes, error: UnicodeDecodeError, advice: str) -> Violation: + """Point at the first byte that is not UTF-8. All the bytes before it are valid.""" + line_start = data.rfind(b"\n", 0, error.start) + 1 + line = data.count(b"\n", 0, error.start) + 1 + column = len(data[line_start : error.start].decode("utf-8")) + 1 + message = f"Byte 0x{data[error.start]:02X} is not valid UTF-8. {advice}" + return Violation(path, line, column, "TELL-14", message) + + +def _check_undecodable(path: str, data: bytes, error: UnicodeDecodeError) -> list[Violation]: + """Report a code or config file that is not UTF-8. Other files can be binary, so skip them.""" + # Only the suffix and the first bytes decide the kind, and bad bytes would stop the decoder. + kind = classify(path, data[:512].decode("utf-8", errors="replace")).kind + if kind not in ("code", "config"): + return [] + advice = "Save the file as UTF-8, and use ASCII in code and config." + return [_not_utf8(path, data, error, advice)] + + +def check_files(root: Path, files: list[Path]) -> list[Violation]: + """Check each file. Report a code or config file that is not UTF-8, and skip other files.""" + violations: list[Violation] = [] + for file in files: + if not file.is_relative_to(root): + raise InputError(f"{file} is outside the root {root}.") + path = file.relative_to(root).as_posix() + data = _read(file) + try: + text = data.decode("utf-8") + except UnicodeDecodeError as error: + violations.extend(_check_undecodable(path, data, error)) + else: + violations.extend(check_text(path, text)) + return violations + + +def check_message_file(path: Path, check: Callable[[str], list[Violation]]) -> list[Violation]: + """Read a message file and run a check on its text. + + A file that is not UTF-8 gives a TELL-14 violation at the first bad byte. + """ + data = _read(path) + try: + text = data.decode("utf-8") + except UnicodeDecodeError as error: + advice = "Save the message as UTF-8, and use ASCII." + return [_not_utf8("commit message", data, error, advice)] + return check(text) diff --git a/scripts/check_tells/text.py b/scripts/check_tells/text.py new file mode 100644 index 0000000..bd828b2 --- /dev/null +++ b/scripts/check_tells/text.py @@ -0,0 +1,29 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +"""How the checks read text: its lines, and the characters that TELL-14 rejects.""" + +from __future__ import annotations + +LAST_ASCII = 0x7F +EMOJI_RANGES = ( + (0x1F000, 0x1FAFF), + (0x231A, 0x231B), + (0x23E9, 0x23F3), + (0x2600, 0x27BF), + (0x2B00, 0x2BFF), + (0xFE0F, 0xFE0F), +) + + +def split_lines(text: str) -> list[str]: + """Split text at line feeds only, and remove one carriage return from each line. + + str.splitlines also splits at U+2028, U+2029, U+0085 and the form feed. That hides the + characters that TELL-14 looks for, and it moves the line numbers. + """ + return [line.removesuffix("\r") for line in text.split("\n")] + + +def is_emoji(character: str) -> bool: + """Tell whether the character is in an emoji range. The ranges include U+FE0F.""" + return any(low <= ord(character) <= high for low, high in EMOJI_RANGES) diff --git a/scripts/check_tells/violation.py b/scripts/check_tells/violation.py new file mode 100644 index 0000000..3cfe59d --- /dev/null +++ b/scripts/check_tells/violation.py @@ -0,0 +1,22 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +"""The result of a check: one broken rule at one place in a file or a message.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True, order=True) +class Violation: + """One broken rule at one place in a file.""" + + path: str + line: int + column: int + rule: str + message: str + + def __str__(self) -> str: + """Format the violation like a compiler message.""" + return f"{self.path}:{self.line}:{self.column}: {self.rule} {self.message}" diff --git a/scripts/tests/check_tells_driver.py b/scripts/tests/check_tells_driver.py new file mode 100644 index 0000000..9d98f22 --- /dev/null +++ b/scripts/tests/check_tells_driver.py @@ -0,0 +1,63 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +"""Run check_tells in a test: make git repos, run the command and read the rules.""" + +import contextlib +import io +import os +import subprocess +import tempfile +from collections.abc import Iterator +from pathlib import Path + +import check_tells +from check_tells import Violation + +# A host with a signing key in its git config must not sign the commits of these tests. +GIT_ENVIRONMENT = { + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_SYSTEM": os.devnull, + "GIT_AUTHOR_NAME": "Ada Bello", + "GIT_AUTHOR_EMAIL": "ada@example.org", + "GIT_COMMITTER_NAME": "Ada Bello", + "GIT_COMMITTER_EMAIL": "ada@example.org", +} + + +def rules(violations: list[Violation]) -> list[str]: + return [violation.rule for violation in violations] + + +@contextlib.contextmanager +def git_repo() -> Iterator[Path]: + """Make an empty git repo in a temporary folder and give its resolved path.""" + with tempfile.TemporaryDirectory() as folder: + root = Path(folder).resolve() + subprocess.run(["git", "init", "-q", str(root)], check=True) + yield root + + +def git(root: Path, *args: str) -> str: + """Run git in a test repo and give its standard output.""" + run = subprocess.run( + ["git", "-C", str(root), *args], + check=True, + capture_output=True, + text=True, + env={**os.environ, **GIT_ENVIRONMENT}, + ) + return run.stdout.strip() + + +def commit(root: Path, message: str) -> str: + """Make an empty commit and give its hash.""" + git(root, "-c", "commit.gpgsign=false", "commit", "--allow-empty", "-q", "-m", message) + return git(root, "rev-parse", "HEAD") + + +def run_main(*args: str) -> tuple[int, str, str]: + """Run the command line and give the exit code, the standard output and the error output.""" + output, errors = io.StringIO(), io.StringIO() + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(errors): + status = check_tells.main(list(args)) + return status, output.getvalue(), errors.getvalue() diff --git a/scripts/tests/check_tells_samples.py b/scripts/tests/check_tells_samples.py new file mode 100644 index 0000000..93c863c --- /dev/null +++ b/scripts/tests/check_tells_samples.py @@ -0,0 +1,126 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +"""Sample text, file names and messages that the check_tells tests share.""" + +EMOJI = chr(0x1F600) +NO_BREAK_SPACE = chr(0x00A0) +E_ACUTE = chr(0x00E9) +EM_DASH = chr(0x2014) +LINE_SEPARATOR = chr(0x2028) +O_DOT_BELOW = chr(0x1ECD) +LONG_LINE = "x" * 150 +# Built from two parts, so that this file passes its own to-do check. +TO_DO = "TO" + "DO" +FIX_ME = "FIX" + "ME" +# Built from two parts, so that this file passes its own skip check. +SKIP_MARKER = "check-tells: " + "allow" + +KIND_BY_PATH = { + "code": [ + "a.ts", + "a.tsx", + "a.mts", + "a.cts", + "a.js", + "a.jsx", + "a.mjs", + "a.cjs", + "a.php", + "a.py", + "a.go", + "a.kt", + "a.kts", + "a.java", + "a.cs", + "a.dart", + "a.swift", + "a.sql", + "a.sh", + "a.bash", + "a.css", + "a.scss", + "a.html", + "a.astro", + "a.vue", + "a.svelte", + "A.TS", + "config.php.dist", + ], + "config": [ + "a.json", + "a.yml", + "a.yaml", + "a.toml", + "a.xml", + "a.ini", + "a.cfg", + "a.properties", + "a.gradle", + "a.neon", + "a.json5", + "a.jsonc", + "A.JSON", + "phpunit.xml.dist", + "phpstan.neon.dist", + "phpstan.dist.neon", + "Makefile", + "Dockerfile", + ".editorconfig", + ".gitignore", + ".gitattributes", + ".npmrc", + ], + "prose": ["a.md", "a.mdx", "A.MD"], + "catalogue": [ + "a.arb", + "a.po", + "a.xlf", + "a.xliff", + "a.strings", + "a.stringsdict", + "src/locales/en.json", + "src/i18n/en.json", + "src/l10n/en.json", + "src/messages/en.json", + "src/locales/yo/home.json", + "res/values/strings.xml", + "res/values-yo/strings.xml", + ], + "exempt": [ + "pnpm-lock.yaml", + "package-lock.json", + "yarn.lock", + "composer.lock", + "go.sum", + "uv.lock", + "poetry.lock", + "Cargo.lock", + "Package.resolved", + "pubspec.lock", + "gradle.lockfile", + "packages.lock.json", + "a.svg", + "a.lock", + "a.csv", + "a.txt", + "a.api", + "a.snap", + "a.png", + "a.jpg", + "a.ico", + "src/locales/yo.txt", + ], + "other": ["LICENSE", "CODEOWNERS", ".gitmodules", "a.woff", "res/layout/strings.xml.bak"], +} + +SIGN_OFF_LINE = "Signed-off-by: Ada Bello " +PLACEHOLDER_PARAGRAPH = ( + "Write two or three sentences of plain prose that say what this change does and why." +) + + +def squash( + subject: str, body: str = "Why the change is needed.", sign_off: str = SIGN_OFF_LINE +) -> str: + """Build a squash merge message from a title with its suffix, a body and a sign-off.""" + return f"{subject}\n\n{body}\n\n{sign_off}\n" diff --git a/scripts/tests/test_check_tells.py b/scripts/tests/test_check_tells.py deleted file mode 100644 index eba20ed..0000000 --- a/scripts/tests/test_check_tells.py +++ /dev/null @@ -1,1679 +0,0 @@ -# SPDX-FileCopyrightText: 2026 The Gatepost authors -# SPDX-License-Identifier: Apache-2.0 -"""Tests for check_tells.""" - -import contextlib -import io -import os -import shutil -import subprocess -import tempfile -import unittest -from collections.abc import Iterator -from pathlib import Path -from unittest import mock - -import check_tells -from check_tells import ( - Violation, - check_commit_message, - check_signed_off, - check_squash_message, - check_text, - classify, -) - -EMOJI = chr(0x1F600) -NO_BREAK_SPACE = chr(0x00A0) -E_ACUTE = chr(0x00E9) -EM_DASH = chr(0x2014) -LINE_SEPARATOR = chr(0x2028) -O_DOT_BELOW = chr(0x1ECD) -LONG_LINE = "x" * 150 -# Built from two parts, so that this file passes its own to-do check. -TO_DO = "TO" + "DO" -FIX_ME = "FIX" + "ME" -# Built from two parts, so that this file passes its own skip check. -SKIP_MARKER = "check-tells: " + "allow" - -# The ranges are typed out here, not read from check_tells, so that a change to them fails. -EMOJI_RANGES = ( - (0x1F000, 0x1FAFF), - (0x231A, 0x231B), - (0x23E9, 0x23F3), - (0x2600, 0x27BF), - (0x2B00, 0x2BFF), - (0xFE0F, 0xFE0F), -) -# The debug calls that each language file lists under TELL-13, one sample line for each call. -DEBUG_LINES = { - "src/a.ts": ["console.log(x);", "console.debug(x);", "console.dir(x);", "debugger;"], - "src/a.php": [ - "var_dump($x);", - "print_r($x);", - "var_export($x);", - "dd($x);", - "dump($x);", - "error_log($x);", - "debug_print_backtrace();", - ], - "pkg/a.py": ["print(x)", "pprint(x)", "breakpoint()", "pdb.set_trace()"], - "a.go": [ - "fmt.Print(x)", - "fmt.Printf(x)", - "fmt.Println(x)", - "log.Print(x)", - "log.Printf(x)", - "log.Println(x)", - "log.Fatal(x)", - "log.Fatalf(x)", - "log.Fatalln(x)", - "log.Panic(x)", - "log.Panicf(x)", - "log.Panicln(x)", - "print(x)", - "println(x)", - ], - "src/A.kt": [ - "println(x)", - "print(x)", - "System.out.println(x)", - "System.err.println(x)", - "error.printStackTrace()", - "Log.v(TAG, x)", - "Log.d(TAG, x)", - "Log.i(TAG, x)", - "Log.w(TAG, x)", - "Log.e(TAG, x)", - "Log.wtf(TAG, x)", - ], - "src/A.java": ["System.out.println(x);", "System.err.println(x);", "error.printStackTrace();"], - "src/A.cs": [ - "Console.Write(x);", - "Console.WriteLine(x);", - "Debug.WriteLine(x);", - "Trace.WriteLine(x);", - ], - "lib/a.dart": ["print(x);", "debugPrint(x);"], - "Sources/A.swift": ["print(x)", "debugPrint(x)", "dump(x)", "NSLog(x)"], -} -# One debug call for each file extension that has a language. -DEBUG_LINE_BY_EXTENSION = { - ".ts": "console.log(x);", - ".tsx": "console.log(x);", - ".mts": "console.log(x);", - ".cts": "console.log(x);", - ".js": "console.log(x);", - ".jsx": "console.log(x);", - ".mjs": "console.log(x);", - ".cjs": "console.log(x);", - ".vue": "console.log(x);", - ".svelte": "console.log(x);", - ".astro": "console.log(x);", - ".php": "var_dump($x);", - ".py": "print(x)", - ".go": "fmt.Println(x)", - ".kt": "println(x)", - ".kts": "println(x)", - ".java": "System.out.println(x);", - ".cs": "Console.WriteLine(x);", - ".dart": "debugPrint(x);", - ".swift": "NSLog(x)", -} -# Calls that look like a debug call but are not one. -HARMLESS_LINES = { - "pkg/a.py": ["report.print(x)", "pprint_table(x)", "self.breakpoint(x)", "sprint(x)"], - "src/a.ts": ["myconsole.log(x);", "console.logs(x);", "console.warn(x);"], - "src/a.php": ["$this->dump($x);", "Debug::dump($x);", "my_dump($x);", "$dd($x);"], - "a.go": ["report.Print(x)", "sprint(x)", "s.println(x)"], - "src/A.kt": ["report.print(x)", "logPrint(x)"], - "src/A.cs": ["Logger.WriteLine(x);"], - "lib/a.dart": ["report.print(x);", "sprint(x);"], - "Sources/A.swift": ["report.print(x)", "sprint(x)"], -} -# Every file name that check-tells must read as a test file, each with the language's debug call. -TEST_FILES = { - "test/a.ts": "console.log(x);", - "tests/a.ts": "console.log(x);", - "src/__tests__/a.ts": "console.log(x);", - "src/testdata/a.ts": "console.log(x);", - "testdata/a.go": "fmt.Println(x)", - "pkg/testdata/a.go": "fmt.Println(x)", - "Tests/A.cs": "Console.WriteLine(x);", - "a.test.ts": "console.log(x);", - "a.spec.ts": "console.log(x);", - "a.test.tsx": "console.log(x);", - "a.test.js": "console.log(x);", - "a.test.jsx": "console.log(x);", - "a.test.mjs": "console.log(x);", - "a.test.cjs": "console.log(x);", - "a.test.mts": "console.log(x);", - "a.test.cts": "console.log(x);", - "a_test.go": "fmt.Println(x)", - "a_test.dart": "debugPrint(x);", - "test_a.py": "print(x)", - "src/ATest.kt": "println(x)", - "src/ATests.kt": "println(x)", - "src/ATest.java": "System.out.println(x);", - "src/ATests.java": "System.out.println(x);", - "src/ATest.cs": "Console.WriteLine(x);", - "src/ATests.cs": "Console.WriteLine(x);", - "Sources/ATest.swift": "dump(x)", - "Sources/ATests.swift": "dump(x)", -} -# File names that look like test files but are not. -NON_TEST_FILES = { - "src/latest/a.ts": "console.log(x);", - "src/contest.ts": "console.log(x);", - "src/a_test.ts": "console.log(x);", - "src/a.testing.ts": "console.log(x);", - "src/Latest.kt": "println(x)", - "src/test.py": "print(x)", - "src/testdata.py": "print(x)", -} -# A line that breaks each rule that a skip comment can skip. -SKIP_SAMPLES = { - "TELL-1": ("src/a.ts", LONG_LINE), - "TELL-13": ("src/a.ts", "console.log(x);"), - "TELL-14": ("src/a.ts", f"const a = '{NO_BREAK_SPACE}';"), - "CS-6": ("src/a.ts", f"// {TO_DO}: x"), -} -KIND_BY_PATH = { - "code": [ - "a.ts", - "a.tsx", - "a.mts", - "a.cts", - "a.js", - "a.jsx", - "a.mjs", - "a.cjs", - "a.php", - "a.py", - "a.go", - "a.kt", - "a.kts", - "a.java", - "a.cs", - "a.dart", - "a.swift", - "a.sql", - "a.sh", - "a.bash", - "a.css", - "a.scss", - "a.html", - "a.astro", - "a.vue", - "a.svelte", - "A.TS", - "config.php.dist", - ], - "config": [ - "a.json", - "a.yml", - "a.yaml", - "a.toml", - "a.xml", - "a.ini", - "a.cfg", - "a.properties", - "a.gradle", - "a.neon", - "a.json5", - "a.jsonc", - "A.JSON", - "phpunit.xml.dist", - "phpstan.neon.dist", - "phpstan.dist.neon", - "Makefile", - "Dockerfile", - ".editorconfig", - ".gitignore", - ".gitattributes", - ".npmrc", - ], - "prose": ["a.md", "a.mdx", "A.MD"], - "catalogue": [ - "a.arb", - "a.po", - "a.xlf", - "a.xliff", - "a.strings", - "a.stringsdict", - "src/locales/en.json", - "src/i18n/en.json", - "src/l10n/en.json", - "src/messages/en.json", - "src/locales/yo/home.json", - "res/values/strings.xml", - "res/values-yo/strings.xml", - ], - "exempt": [ - "pnpm-lock.yaml", - "package-lock.json", - "yarn.lock", - "composer.lock", - "go.sum", - "uv.lock", - "poetry.lock", - "Cargo.lock", - "Package.resolved", - "pubspec.lock", - "gradle.lockfile", - "packages.lock.json", - "a.svg", - "a.lock", - "a.csv", - "a.txt", - "a.api", - "a.snap", - "a.png", - "a.jpg", - "a.ico", - "src/locales/yo.txt", - ], - "other": ["LICENSE", "CODEOWNERS", ".gitmodules", "a.woff", "res/layout/strings.xml.bak"], -} -LANGUAGE_BY_PATH = { - "a.ts": "javascript", - "a.vue": "javascript", - "a.astro": "javascript", - "a.svelte": "javascript", - "a.php": "php", - "config.php.dist": "php", - "a.py": "python", - "a.go": "go", - "a.kt": "jvm", - "a.java": "jvm", - "a.cs": "dotnet", - "a.dart": "dart", - "a.swift": "swift", - "a.css": None, - "a.html": None, - "a.sh": None, - "a.md": None, -} - - -SIGN_OFF_LINE = "Signed-off-by: Ada Bello " -PLACEHOLDER_PARAGRAPH = ( - "Write two or three sentences of plain prose that say what this change does and why." -) -COMMIT_TYPES = ( - "build", - "chore", - "ci", - "docs", - "feat", - "fix", - "perf", - "refactor", - "revert", - "style", - "test", -) -# A host with a signing key in its git config must not sign the commits of these tests. -GIT_ENVIRONMENT = { - "GIT_CONFIG_GLOBAL": os.devnull, - "GIT_CONFIG_SYSTEM": os.devnull, - "GIT_AUTHOR_NAME": "Ada Bello", - "GIT_AUTHOR_EMAIL": "ada@example.org", - "GIT_COMMITTER_NAME": "Ada Bello", - "GIT_COMMITTER_EMAIL": "ada@example.org", -} - - -def rules(violations: list[Violation]) -> list[str]: - return [violation.rule for violation in violations] - - -@contextlib.contextmanager -def git_repo() -> Iterator[Path]: - """Make an empty git repo in a temporary folder and give its resolved path.""" - with tempfile.TemporaryDirectory() as folder: - root = Path(folder).resolve() - subprocess.run(["git", "init", "-q", str(root)], check=True) - yield root - - -def git(root: Path, *args: str) -> str: - """Run git in a test repo and give its standard output.""" - run = subprocess.run( - ["git", "-C", str(root), *args], - check=True, - capture_output=True, - text=True, - env={**os.environ, **GIT_ENVIRONMENT}, - ) - return run.stdout.strip() - - -def commit(root: Path, message: str) -> str: - """Make an empty commit and give its hash.""" - git(root, "-c", "commit.gpgsign=false", "commit", "--allow-empty", "-q", "-m", message) - return git(root, "rev-parse", "HEAD") - - -def squash( - subject: str, body: str = "Why the change is needed.", sign_off: str = SIGN_OFF_LINE -) -> str: - """Build a squash merge message from a title with its suffix, a body and a sign-off.""" - return f"{subject}\n\n{body}\n\n{sign_off}\n" - - -def run_main(*args: str) -> tuple[int, str, str]: - """Run the command line and give the exit code, the standard output and the error output.""" - output, errors = io.StringIO(), io.StringIO() - with contextlib.redirect_stdout(output), contextlib.redirect_stderr(errors): - status = check_tells.main(list(args)) - return status, output.getvalue(), errors.getvalue() - - -class LineLengthTest(unittest.TestCase): - def test_accepts_a_line_of_100_characters(self) -> None: - self.assertEqual(check_text("src/a.ts", "x" * 100), []) - - def test_rejects_a_line_of_101_characters(self) -> None: - found = check_text("src/a.ts", "x" * 101) - self.assertEqual(rules(found), ["TELL-1"]) - self.assertEqual(found[0].column, 101) - - def test_counts_a_tab_as_4_columns(self) -> None: - self.assertEqual(check_text("Makefile", "\t" + "x" * 96), []) - self.assertEqual(rules(check_text("Makefile", "\t" + "x" * 97)), ["TELL-1"]) - self.assertEqual(check_text("Makefile", "\t\t" + "x" * 92), []) - self.assertEqual(rules(check_text("Makefile", "\t\t" + "x" * 93)), ["TELL-1"]) - - def test_skips_a_line_with_a_url(self) -> None: - for scheme in ("https", "http"): - with self.subTest(scheme=scheme): - line = f"// {scheme}://example.com/" + "a" * 100 - self.assertEqual(check_text("src/a.ts", line), []) - - def test_skips_markdown_lockfiles_and_vectors(self) -> None: - self.assertEqual(check_text("README.md", "word " * 40), []) - self.assertEqual(check_text("pnpm-lock.yaml", "x" * 150), []) - self.assertEqual(check_text("vectors/parse.json", "x" * 150), []) - - def test_skips_json_in_a_data_folder(self) -> None: - self.assertEqual(check_text("data/states.json", "x" * 150), []) - self.assertEqual(check_text("vectors/parse.json", "x" * 150), []) - self.assertEqual(check_text("spec/vectors/parse.json", "x" * 150), []) - self.assertEqual(check_text("spec/data/format.json", "x" * 150), []) - - def test_checks_other_files_in_a_data_folder(self) -> None: - self.assertEqual(rules(check_text("data/rules.yml", "x" * 150)), ["TELL-1"]) - self.assertEqual(rules(check_text("vectors/build.py", "x" * 150)), ["TELL-1"]) - self.assertEqual(rules(check_text("package.json", "x" * 150)), ["TELL-1"]) - self.assertEqual(rules(check_text("database/rules.json", "x" * 150)), ["TELL-1"]) - - def test_skips_a_message_catalogue(self) -> None: - line = '{"title": "' + "x" * 150 + '"}' - self.assertEqual(check_text("src/locales/en.json", line), []) - - def test_skips_a_generated_file(self) -> None: - text = "// @generated by a script\n" + "x" * 150 - self.assertEqual(check_text("src/spec-data.ts", text), []) - - def test_skips_files_that_other_tools_generate(self) -> None: - cases = { - "src/A.cs": "// \n", - "lib/a.g.dart": "// GENERATED CODE - DO NOT MODIFY BY HAND\n", - } - for path, header in cases.items(): - with self.subTest(path=path): - self.assertEqual(check_text(path, header + "x" * 150), []) - - def test_honours_a_skip_with_a_reason(self) -> None: - line = "x" * 120 + " // check-tells: allow TELL-1 because the regex cannot be split" - self.assertEqual(check_text("src/a.ts", line), []) - - def test_checks_each_code_and_config_kind(self) -> None: - for kind in ("code", "config"): - for path in KIND_BY_PATH[kind]: - with self.subTest(path=path): - self.assertEqual(rules(check_text(path, LONG_LINE)), ["TELL-1"]) - - def test_leaves_every_other_kind_alone(self) -> None: - for kind in ("prose", "catalogue", "exempt", "other"): - for path in KIND_BY_PATH[kind]: - with self.subTest(path=path): - self.assertEqual(check_text(path, LONG_LINE), []) - - def test_reads_a_marker_in_the_first_5_lines_of_a_file(self) -> None: - for marker in ("@generated", "DO NOT EDIT", " None: - for marker in ("@generated", "DO NOT EDIT", " None: - text = "// @generated\nconsole.log(1);\n// " + TO_DO + ": x" - self.assertEqual(rules(check_text("src/a.ts", text)), ["TELL-13", "CS-6"]) - - -class ClassifyTest(unittest.TestCase): - def test_reads_the_kind_of_each_file_name(self) -> None: - for kind, paths in KIND_BY_PATH.items(): - for path in paths: - with self.subTest(path=path): - self.assertEqual(classify(path, "").kind, kind) - - def test_reads_the_language_of_each_file_name(self) -> None: - for path, language in LANGUAGE_BY_PATH.items(): - with self.subTest(path=path): - self.assertEqual(classify(path, "").language, language) - - def test_reads_a_file_without_a_suffix_from_its_first_line(self) -> None: - self.assertEqual(classify("scripts/run", "#!/usr/bin/env python3\n").kind, "code") - self.assertEqual(classify("scripts/run", "plain text\n").kind, "other") - self.assertEqual(classify("scripts/run", "").kind, "other") - - def test_reads_the_suffix_before_a_shebang(self) -> None: - cases = { - "notes.txt": "exempt", - "README.md": "prose", - "package.json": "config", - "font.woff": "other", - "src/locales/en.json": "catalogue", - } - for path, kind in cases.items(): - with self.subTest(path=path): - self.assertEqual(classify(path, "#!/usr/bin/env python3\n").kind, kind) - - def test_reads_the_interpreter_of_a_shebang(self) -> None: - cases = { - "#!/usr/bin/env python3": "python", - "#!/usr/bin/env python": "python", - "#!/usr/bin/python3": "python", - "#!/usr/bin/env node": "javascript", - "#!/usr/bin/node": "javascript", - "#!/bin/sh": None, - } - for first_line, language in cases.items(): - with self.subTest(first_line=first_line): - self.assertEqual(classify("scripts/run", first_line).language, language) - - def test_reads_a_dist_file_like_the_file_that_it_copies(self) -> None: - self.assertEqual(rules(check_text("phpunit.xml.dist", LONG_LINE)), ["TELL-1"]) - self.assertEqual(rules(check_text("config.php.dist", "var_dump($x);")), ["TELL-13"]) - self.assertEqual(rules(check_text("PHPUNIT.XML.DIST", LONG_LINE)), ["TELL-1"]) - - def test_keeps_the_folder_rules_for_a_dist_file(self) -> None: - self.assertEqual(check_text("src/locales/en.json.dist", LONG_LINE), []) - - -class FileNameTest(unittest.TestCase): - def test_rejects_utils_helpers_common_and_misc(self) -> None: - for name in ("src/utils.ts", "lib/helpers.py", "common.go", "pkg/Misc.kt"): - with self.subTest(name=name): - self.assertEqual(rules(check_text(name, "")), ["TELL-7"]) - - def test_rejects_the_name_with_any_extension_or_none(self) -> None: - for name in ("utils", "helpers.test.ts", "common.d.ts", "src/MISC.md", "Utils.json"): - with self.subTest(name=name): - self.assertEqual(rules(check_text(name, "")), ["TELL-7"]) - - def test_accepts_a_folder_named_common(self) -> None: - self.assertEqual(check_text("common/parse.ts", ""), []) - - def test_accepts_a_name_that_only_starts_with_one_of_the_words(self) -> None: - self.assertEqual(check_text("src/utilities.ts", ""), []) - self.assertEqual(check_text("src/commonly.ts", ""), []) - - -class TodoFormTest(unittest.TestCase): - def test_rejects_a_to_do_without_an_issue(self) -> None: - self.assertEqual(rules(check_text("src/a.ts", f"// {TO_DO}: handle 00")), ["CS-6"]) - - def test_rejects_a_to_do_in_a_config_file(self) -> None: - found = check_text("Makefile", "# " + TO_DO + ": pin the version") - self.assertEqual(rules(found), ["CS-6"]) - - def test_rejects_a_to_do_or_a_fix_me_in_any_letter_case(self) -> None: - for word in (TO_DO, TO_DO.lower(), TO_DO.title(), FIX_ME, FIX_ME.lower(), FIX_ME.title()): - with self.subTest(word=word): - found = check_text("src/a.ts", f"// {word}: handle 00") - self.assertEqual(rules(found), ["CS-6"]) - self.assertEqual(found[0].column, 4) - self.assertEqual( - found[0].message, f"Link an issue in the form {word.upper()}(#123)." - ) - - def test_accepts_a_to_do_with_an_issue(self) -> None: - self.assertEqual(check_text("src/a.ts", f"// {TO_DO}(#12): handle 00"), []) - - def test_accepts_a_to_do_or_a_fix_me_with_an_issue_in_any_letter_case(self) -> None: - for word in (TO_DO, TO_DO.lower(), FIX_ME, FIX_ME.title()): - with self.subTest(word=word): - self.assertEqual(check_text("src/a.ts", f"// {word}(#12): handle 00"), []) - - def test_rejects_a_to_do_with_a_bad_issue_reference(self) -> None: - for reference in ("(12)", "(#)", "(#x)", " (#12)"): - with self.subTest(reference=reference): - found = check_text("src/a.ts", f"// {TO_DO}{reference}: handle 00") - self.assertEqual(rules(found), ["CS-6"]) - - def test_accepts_a_word_that_only_contains_to_do(self) -> None: - line = "const todos = 1; const mastodon = 2; const fixmeLater = 3; // _todo_form" - self.assertEqual(check_text("src/a.ts", line), []) - - def test_ignores_prose(self) -> None: - self.assertEqual(check_text("docs/notes.md", f"{TO_DO}: write this"), []) - - def test_checks_a_config_file_that_php_tools_use(self) -> None: - found = check_text("phpstan.dist.neon", "# " + TO_DO + ": raise the level") - self.assertEqual(rules(found), ["CS-6"]) - - -class DebugCallTest(unittest.TestCase): - def test_rejects_a_debug_call_in_each_language(self) -> None: - cases = { - "src/a.ts": "console.log(code);", - "src/b.ts": "debugger;", - "src/a.php": "var_dump($code);", - "pkg/a.py": "print(code)", - "pkg/b.py": "pdb.set_trace()", - "pkg/c.py": "breakpoint()", - "a.go": "fmt.Println(code)", - "b.go": 'log.Printf("x")', - "src/A.kt": "println(code)", - "src/B.kt": 'Log.d(TAG, "x")', - "src/A.java": "System.out.println(x);", - "src/B.java": "e.printStackTrace();", - "src/A.cs": "Console.WriteLine(code);", - "lib/a.dart": "debugPrint(code);", - "Sources/A.swift": "dump(code)", - "Sources/B.swift": 'NSLog("x")', - } - for path, line in cases.items(): - with self.subTest(path=path): - self.assertEqual(rules(check_text(path, line)), ["TELL-13"]) - - def test_rejects_each_debug_call_that_a_language_file_lists(self) -> None: - for path, lines in DEBUG_LINES.items(): - for line in lines: - with self.subTest(path=path, line=line): - self.assertEqual(rules(check_text(path, line)), ["TELL-13"]) - - def test_rejects_a_debug_call_in_each_file_extension_with_a_language(self) -> None: - for extension, line in DEBUG_LINE_BY_EXTENSION.items(): - with self.subTest(extension=extension): - self.assertEqual(rules(check_text("src/a" + extension, line)), ["TELL-13"]) - - def test_names_the_call_and_its_place(self) -> None: - found = check_text("src/a.ts", " console.log(x);") - self.assertEqual((found[0].line, found[0].column), (1, 3)) - self.assertEqual(found[0].message, "Remove 'console.log('.") - - def test_checks_a_code_file_in_a_catalogue_folder(self) -> None: - found = check_text("src/messages/format.ts", "console.log(1);") - self.assertEqual(rules(found), ["TELL-13"]) - - def test_accepts_methods_with_similar_names(self) -> None: - self.assertEqual(check_text("pkg/a.py", "report.print(code)"), []) - self.assertEqual(check_text("pkg/a.py", "pprint_table(code)"), []) - - def test_accepts_each_call_that_only_looks_like_a_debug_call(self) -> None: - for path, lines in HARMLESS_LINES.items(): - for line in lines: - with self.subTest(path=path, line=line): - self.assertEqual(check_text(path, line), []) - - def test_ignores_test_files_and_comment_lines(self) -> None: - self.assertEqual(check_text("example_test.go", 'fmt.Println("EK-01")'), []) - self.assertEqual(check_text("src/a.ts", " * console.log(parse(code));"), []) - - def test_ignores_each_kind_of_test_path(self) -> None: - cases = { - "test/parse.ts": "console.log(code);", - "src/__tests__/parse.ts": "console.log(code);", - "lib/parse_test.dart": "debugPrint(code);", - "src/parse.test.ts": "console.log(code);", - "src/parse.spec.mjs": "console.log(code);", - "test_parse.py": "print(code)", - "src/ParseTest.kt": "println(code)", - "Sources/ParseTests.swift": "dump(code)", - } - for path, line in cases.items(): - with self.subTest(path=path): - self.assertEqual(check_text(path, line), []) - - def test_ignores_every_test_file_name_that_the_standards_allow(self) -> None: - for path, line in TEST_FILES.items(): - with self.subTest(path=path): - self.assertEqual(check_text(path, line), []) - - def test_checks_a_file_name_that_only_looks_like_a_test_file(self) -> None: - for path, line in NON_TEST_FILES.items(): - with self.subTest(path=path): - self.assertEqual(rules(check_text(path, line)), ["TELL-13"]) - - def test_ignores_each_kind_of_comment_start(self) -> None: - cases = [ - ("src/a.ts", "// console.log(x)"), - ("src/a.ts", "/* console.log(x)"), - ("src/a.ts", " * console.log(x)"), - ("pkg/a.py", "# print(x)"), - ] - for path, line in cases: - with self.subTest(line=line): - self.assertEqual(check_text(path, line), []) - - def test_ignores_doctest_examples(self) -> None: - self.assertEqual(check_text("pkg/a.py", ' >>> print(parse("EK01A03FK01"))'), []) - self.assertEqual(check_text("pkg/a.py", " ... print(code)"), []) - - def test_checks_a_spread_line_outside_python(self) -> None: - line = " ...items.map((item) => console.log(item))," - self.assertEqual(rules(check_text("src/a.ts", line)), ["TELL-13"]) - - def test_reads_the_language_from_a_shebang(self) -> None: - cases = { - "#!/usr/bin/env python3\nprint(1)": ["TELL-13"], - "#!/usr/bin/env python\nprint(1)": ["TELL-13"], - "#!/usr/bin/python3\nprint(1)": ["TELL-13"], - "#!/usr/bin/env node\nconsole.log(1)": ["TELL-13"], - "#!/usr/bin/node\nconsole.log(1)": ["TELL-13"], - "#!/bin/sh\nprint(1)": [], - } - for text, expected in cases.items(): - with self.subTest(text=text): - self.assertEqual(rules(check_text("scripts/run", text)), expected) - - def test_does_not_check_markup_or_style_files_for_debug_calls(self) -> None: - for path in ("site/a.html", "src/a.css", "src/a.scss"): - with self.subTest(path=path): - self.assertEqual(check_text(path, "console.log(x);"), []) - - -class CharacterTest(unittest.TestCase): - def test_rejects_an_emoji_in_markdown(self) -> None: - self.assertEqual(rules(check_text("README.md", f"Done {EMOJI}")), ["TELL-14"]) - - def test_rejects_an_hourglass_in_markdown(self) -> None: - self.assertEqual(rules(check_text("README.md", "Waiting " + chr(0x23F3))), ["TELL-14"]) - self.assertEqual(rules(check_text("README.md", "Done " + chr(0x231B))), ["TELL-14"]) - - def test_rejects_the_emoji_that_agents_write(self) -> None: - names = { - "check mark": 0x2705, - "cross mark": 0x274C, - "warning sign": 0x26A0, - "sparkles": 0x2728, - "rocket": 0x1F680, - "star": 0x2B50, - "arrow": 0x2B06, - } - for name, point in names.items(): - with self.subTest(name=name): - found = check_text("README.md", f"Done {chr(point)} here") - self.assertEqual(rules(found), ["TELL-14"]) - self.assertEqual(found[0].column, 6) - self.assertEqual(found[0].message, f"Remove the emoji U+{point:04X}.") - - def test_rejects_both_ends_of_each_emoji_range(self) -> None: - for low, high in EMOJI_RANGES: - for point in (low, high): - with self.subTest(point=f"U+{point:04X}"): - found = check_text("README.md", f"a {chr(point)} b") - self.assertEqual(rules(found), ["TELL-14"]) - - def test_accepts_the_character_next_to_each_emoji_range(self) -> None: - for low, high in EMOJI_RANGES: - for point in (low - 1, high + 1): - with self.subTest(point=f"U+{point:04X}"): - self.assertEqual(check_text("README.md", f"a {chr(point)} b"), []) - - def test_rejects_the_emoji_variation_selector_after_a_symbol(self) -> None: - found = check_text("README.md", "Heart " + chr(0x2764) + chr(0xFE0F)) - self.assertEqual(rules(found), ["TELL-14", "TELL-14"]) - self.assertEqual([violation.column for violation in found], [7, 8]) - - def test_rejects_non_ascii_in_code(self) -> None: - found = check_text("src/a.ts", f"const space = '{NO_BREAK_SPACE}';") - self.assertEqual(rules(found), ["TELL-14"]) - self.assertIn("U+00A0", found[0].message) - - def test_rejects_non_ascii_in_each_code_and_config_kind(self) -> None: - for kind in ("code", "config"): - for path in KIND_BY_PATH[kind]: - with self.subTest(path=path): - found = check_text(path, f"a{NO_BREAK_SPACE}b") - self.assertEqual(rules(found), ["TELL-14"]) - self.assertEqual(found[0].column, 2) - - def test_asks_for_an_escape_sequence_for_non_ascii_and_not_for_an_emoji(self) -> None: - emoji = check_text("src/a.ts", f"const a = '{EMOJI}';") - other = check_text("src/a.ts", f"const a = '{E_ACUTE}';") - self.assertEqual(emoji[0].message, "Remove the emoji U+1F600.") - self.assertEqual( - other[0].message, - "Write U+00E9 as an escape sequence. Code and config use ASCII.", - ) - - def test_reports_one_non_ascii_character_for_each_line(self) -> None: - found = check_text("src/a.ts", f"{E_ACUTE}{E_ACUTE}\n{E_ACUTE}") - self.assertEqual([(v.line, v.column) for v in found], [(1, 1), (2, 1)]) - - def test_accepts_the_last_ascii_character_in_code(self) -> None: - self.assertEqual(check_text("src/a.ts", "// " + chr(0x7F)), []) - self.assertEqual(rules(check_text("src/a.ts", "// " + chr(0x80))), ["TELL-14"]) - - def test_accepts_non_ascii_in_prose_and_catalogues(self) -> None: - self.assertEqual(check_text("docs/guide.md", f"Caf{E_ACUTE}"), []) - self.assertEqual(check_text("src/locales/yo.json", f'{{"title": "Caf{E_ACUTE}"}}'), []) - - def test_accepts_non_ascii_in_every_file_that_is_not_code_or_config(self) -> None: - for kind in ("prose", "catalogue", "exempt", "other"): - for path in KIND_BY_PATH[kind]: - with self.subTest(path=path): - self.assertEqual(check_text(path, f"Caf{E_ACUTE}"), []) - - def test_rejects_an_emoji_in_every_kind_of_file(self) -> None: - for paths in KIND_BY_PATH.values(): - for path in paths: - with self.subTest(path=path): - found = check_text(path, f"a {EMOJI} b") - self.assertEqual(rules(found), ["TELL-14"]) - self.assertEqual(found[0].column, 3) - - def test_holds_a_code_file_in_a_catalogue_folder_to_ascii(self) -> None: - line = "export const title = '" + O_DOT_BELOW + "';" - self.assertEqual(rules(check_text("src/i18n/yo.ts", line)), ["TELL-14"]) - - def test_rejects_an_emoji_in_a_catalogue(self) -> None: - found = check_text("src/locales/en.json", f'{{"ok": "{EMOJI}"}}') - self.assertEqual(rules(found), ["TELL-14"]) - - def test_accepts_non_ascii_in_each_catalogue_folder_and_extension(self) -> None: - for path in KIND_BY_PATH["catalogue"]: - with self.subTest(path=path): - self.assertEqual(check_text(path, f"{O_DOT_BELOW}: {LONG_LINE}"), []) - - def test_holds_other_android_files_to_ascii(self) -> None: - for path in ("res/layout/strings.xml", "res/values/colors.xml", "strings.xml"): - with self.subTest(path=path): - self.assertEqual(rules(check_text(path, f"{E_ACUTE}")), ["TELL-14"]) - - def test_reads_a_smart_quote_in_a_style_file(self) -> None: - found = check_text("src/field.css", "content: " + chr(0x201C) + "x" + chr(0x201D) + ";") - self.assertEqual(rules(found), ["TELL-14"]) - - def test_checks_an_exempt_file_for_emoji_only(self) -> None: - for path in KIND_BY_PATH["exempt"]: - with self.subTest(path=path): - found = check_text(path, f"{LONG_LINE}{E_ACUTE}\n{TO_DO}\n{EMOJI}") - self.assertEqual([(v.rule, v.line) for v in found], [("TELL-14", 3)]) - - def test_checks_a_text_file_with_a_binary_looking_name_for_emoji(self) -> None: - self.assertEqual( - rules(check_text("readme.txt", f"Works with WooCommerce {EMOJI}")), ["TELL-14"] - ) - self.assertEqual(rules(check_text("src/locales/yo.txt", f"Ok {EMOJI}")), ["TELL-14"]) - - -class LineSplitTest(unittest.TestCase): - def test_keeps_a_line_separator_inside_the_line(self) -> None: - text = "const a = 1;" + LINE_SEPARATOR + "const b = 2;" - self.assertEqual(rules(check_text("src/a.ts", text)), ["TELL-14"]) - - def test_keeps_the_line_numbers_after_a_form_feed(self) -> None: - found = check_text("src/a.ts", "const a = 1;" + chr(0x0C) + "\nconsole.log(1);") - self.assertEqual(rules(found), ["TELL-13"]) - self.assertEqual(found[0].line, 2) - - def test_treats_crlf_like_lf(self) -> None: - text = "x" * 100 + "\nconsole.log(1);\n" - found = check_text("src/a.ts", text.replace("\n", "\r\n")) - self.assertEqual(rules(found), ["TELL-13"]) - self.assertEqual(found, check_text("src/a.ts", text)) - - -class SkipTest(unittest.TestCase): - def test_rejects_a_skip_without_a_reason(self) -> None: - line = "x" * 120 + f" // {SKIP_MARKER} TELL-1" - self.assertEqual(sorted(rules(check_text("src/a.ts", line))), ["SKIP", "TELL-1"]) - - def test_rejects_a_skip_of_a_rule_that_cannot_be_skipped(self) -> None: - line = f"const a = 1; // {SKIP_MARKER} TELL-7 because it is short" - self.assertEqual(rules(check_text("src/a.ts", line)), ["SKIP"]) - - def test_skips_each_rule_that_a_skip_comment_can_skip(self) -> None: - for rule, (path, line) in SKIP_SAMPLES.items(): - with self.subTest(rule=rule): - self.assertEqual(rules(check_text(path, line)), [rule]) - skipped = f"{line} // {SKIP_MARKER} {rule} because the line is a sample" - self.assertEqual(check_text(path, skipped), []) - - def test_rejects_a_skip_of_each_rule_that_cannot_be_skipped(self) -> None: - for rule in ("TELL-2", "TELL-7", "TELL-18", "CS-5", "API-3", "GIT-1"): - with self.subTest(rule=rule): - line = f"const a = 1; // {SKIP_MARKER} {rule} because it is short" - found = check_text("src/a.ts", line) - self.assertEqual(rules(found), ["SKIP"]) - self.assertEqual(found[0].column, 17) - - def test_skips_only_the_rule_that_the_comment_names(self) -> None: - line = f"console.log('{LONG_LINE}'); // {SKIP_MARKER} TELL-1 because the string is data" - self.assertEqual(rules(check_text("src/a.ts", line)), ["TELL-13"]) - - def test_skips_only_the_line_that_holds_the_comment(self) -> None: - text = f"console.log(1); // {SKIP_MARKER} TELL-13 because it is a sample\nconsole.log(2);" - found = check_text("src/a.ts", text) - self.assertEqual([(v.rule, v.line) for v in found], [("TELL-13", 2)]) - - def test_honours_a_skip_in_a_config_file(self) -> None: - line = f"{LONG_LINE} # {SKIP_MARKER} TELL-1 because the value is a hash" - self.assertEqual(check_text("Makefile", line), []) - - def test_ignores_a_skip_marker_in_prose(self) -> None: - line = "A line such as `" + SKIP_MARKER + " TELL-1` has no reason." - self.assertEqual(check_text("README.md", line), []) - - -class CommitMessageTest(unittest.TestCase): - def test_accepts_a_good_message(self) -> None: - message = "fix(core): reject a unit of 00\n\nThe spec forbids 00.\n" - self.assertEqual(check_commit_message(message), []) - - def test_rejects_a_long_subject(self) -> None: - self.assertEqual(rules(check_commit_message("fix: " + "a" * 70)), ["TELL-18"]) - - def test_accepts_a_subject_of_72_characters(self) -> None: - self.assertEqual(check_commit_message("a" * 72), []) - - def test_rejects_a_subject_of_73_characters(self) -> None: - found = check_commit_message("a" * 73) - self.assertEqual(rules(found), ["TELL-18"]) - self.assertEqual(found[0].column, 73) - self.assertEqual(found[0].message, "The subject has 73 characters. Use 72 or fewer.") - - def test_reads_the_subject_after_blank_lines(self) -> None: - found = check_commit_message("\n\n" + "a" * 73 + "\n\n" + "b" * 80) - self.assertEqual([(v.rule, v.line) for v in found], [("TELL-18", 3)]) - - def test_rejects_an_emoji(self) -> None: - self.assertEqual(rules(check_commit_message(f"fix: parse codes {EMOJI}")), ["TELL-14"]) - - def test_accepts_an_accented_name_in_a_sign_off(self) -> None: - message = f"fix: parse codes\n\nSigned-off-by: Ad{E_ACUTE} Bello \n" - self.assertEqual(check_commit_message(message), []) - - def test_ignores_git_comment_lines(self) -> None: - self.assertEqual(check_commit_message(f"fix: parse codes\n# {EMOJI} from git\n"), []) - - def test_accepts_the_last_ascii_character_in_a_commit_message(self) -> None: - self.assertEqual(check_commit_message("fix: a" + chr(0x7F)), []) - self.assertEqual(rules(check_commit_message("fix: a" + chr(0x80))), ["TELL-14"]) - - def test_rejects_non_ascii_in_a_subject(self) -> None: - self.assertEqual(rules(check_commit_message("fix: parse caf" + E_ACUTE)), ["TELL-14"]) - - def test_rejects_non_ascii_in_a_body_but_not_in_a_sign_off(self) -> None: - message = ( - "fix: parse codes\n\n" - f"Use a dash {EM_DASH} here.\n\n" - f"Signed-off-by: Ad{E_ACUTE} Bello \n" - ) - found = check_commit_message(message) - self.assertEqual(rules(found), ["TELL-14"]) - self.assertEqual(found[0].line, 3) - - def test_reads_a_sign_off_without_regard_to_case(self) -> None: - message = f"fix: parse codes\n\nCo-Authored-By: Ad{E_ACUTE} Bello " - self.assertEqual(check_commit_message(message), []) - - def test_stops_at_the_scissors_line(self) -> None: - scissors = "# ------------------------ >8 ------------------------" - message = f"fix: parse codes\n{scissors}\n+ caf{E_ACUTE}" - self.assertEqual(check_commit_message(message), []) - - def test_reports_the_real_line_of_a_character(self) -> None: - found = check_commit_message(f"# c1\n# c2\nfix: caf{E_ACUTE}") - self.assertEqual(rules(found), ["TELL-14"]) - self.assertEqual(found[0].line, 3) - - def test_reports_the_real_line_of_a_long_subject(self) -> None: - found = check_commit_message("# note\nfix: " + "a" * 70) - self.assertEqual(rules(found), ["TELL-18"]) - self.assertEqual(found[0].line, 2) - - def test_keeps_a_line_separator_in_a_commit_message(self) -> None: - found = check_commit_message("fix: a\n\nb" + LINE_SEPARATOR + "c") - self.assertEqual([(v.rule, v.line) for v in found], [("TELL-14", 3)]) - - -class FileEncodingTest(unittest.TestCase): - def check(self, name: str, data: bytes) -> list[Violation]: - with tempfile.TemporaryDirectory() as folder: - root = Path(folder).resolve() - file = root / name - file.parent.mkdir(parents=True, exist_ok=True) - file.write_bytes(data) - return check_tells.check_files(root, [file]) - - def test_reports_a_code_file_that_is_not_utf_8(self) -> None: - data = b"const a = 1;\nconst b = \x93hi\x94;\nconsole.log(b);\n" - found = self.check("src/a.ts", data) - self.assertEqual(rules(found), ["TELL-14"]) - self.assertEqual((found[0].path, found[0].line, found[0].column), ("src/a.ts", 2, 11)) - self.assertEqual( - found[0].message, - "Byte 0x93 is not valid UTF-8. " - "Save the file as UTF-8, and use ASCII in code and config.", - ) - - def test_counts_the_column_of_the_bad_byte_in_characters(self) -> None: - found = self.check("src/a.ts", f"// caf{E_ACUTE} ".encode() + b"\xff") - self.assertEqual((found[0].line, found[0].column), (1, 9)) - - def test_reports_a_config_file_that_is_not_utf_8(self) -> None: - found = self.check("package.json", b'{"name": "caf\xe9"}\n') - self.assertEqual(rules(found), ["TELL-14"]) - self.assertEqual((found[0].line, found[0].column), (1, 14)) - - def test_reports_a_script_with_a_shebang_that_is_not_utf_8(self) -> None: - found = self.check("scripts/run", b'#!/usr/bin/env python3\nx = "caf\xe9"\n') - self.assertEqual(rules(found), ["TELL-14"]) - self.assertEqual(found[0].line, 2) - - def test_reports_a_file_that_starts_with_a_utf_16_byte_order_mark(self) -> None: - found = self.check("src/a.ts", b"\xff\xfec\x00") - self.assertEqual((found[0].line, found[0].column), (1, 1)) - - def test_skips_a_file_that_is_not_text(self) -> None: - names = ["logo.png", "docs/a.md", "src/locales/yo.po", "LICENSE", "font.woff", "go.sum"] - for name in names: - with self.subTest(name=name): - self.assertEqual(self.check(name, b"\x89PNG\r\n\x1a\n\xff\xfe\x00"), []) - - def test_reads_a_file_that_is_utf_8(self) -> None: - data = f"const a = '{NO_BREAK_SPACE}';\n".encode() - self.assertEqual(rules(self.check("src/a.ts", data)), ["TELL-14"]) - self.assertEqual(self.check("src/a.ts", b"export const a = 1;\n"), []) - - -class CommandTest(unittest.TestCase): - def test_checks_new_files_and_skips_ignored_ones(self) -> None: - with tempfile.TemporaryDirectory() as folder: - root = Path(folder) - subprocess.run(["git", "init", "-q", str(root)], check=True) - (root / ".gitignore").write_text("ignored.ts\n", encoding="utf-8") - (root / "ignored.ts").write_text("console.log(1);\n", encoding="utf-8") - (root / "new.ts").write_text("console.log(1);\n", encoding="utf-8") - output = io.StringIO() - with contextlib.redirect_stdout(output): - status = check_tells.main(["--root", str(root)]) - self.assertEqual(status, 1) - self.assertIn("new.ts:1:1: TELL-13", output.getvalue()) - self.assertNotIn("ignored.ts", output.getvalue()) - - def test_exits_with_0_for_a_clean_repo(self) -> None: - with tempfile.TemporaryDirectory() as folder: - root = Path(folder) - subprocess.run(["git", "init", "-q", str(root)], check=True) - (root / "clean.ts").write_text("export const a = 1;\n", encoding="utf-8") - command = [str(Path(__file__).resolve().parents[1] / "check-tells"), "--root", folder] - self.assertEqual(subprocess.run(command, check=False).returncode, 0) - - def test_prints_a_violation_like_a_compiler_message(self) -> None: - violation = Violation("src/a.ts", 3, 101, "TELL-1", "The line is too wide.") - self.assertEqual(str(violation), "src/a.ts:3:101: TELL-1 The line is too wide.") - - def test_prints_the_violations_in_the_order_of_the_files_and_lines(self) -> None: - with git_repo() as root: - (root / "b.ts").write_text("console.log(1);\n\nconsole.log(2);\n", encoding="utf-8") - (root / "a.ts").write_text("export const a = 1;\nconsole.log(1);\n", encoding="utf-8") - status, output, errors = run_main("--root", str(root)) - self.assertEqual(status, 1) - self.assertEqual(errors, "") - self.assertEqual( - [line.split(": ")[0] for line in output.splitlines()], - ["a.ts:2:1", "b.ts:1:1", "b.ts:3:1"], - ) - - def test_skips_a_tracked_file_that_is_missing_from_the_work_tree(self) -> None: - with git_repo() as root: - (root / "gone.ts").write_text("console.log(1);\n", encoding="utf-8") - subprocess.run(["git", "-C", str(root), "add", "gone.ts"], check=True) - (root / "gone.ts").unlink() - status, output, errors = run_main("--root", str(root)) - self.assertEqual((status, output, errors), (0, "", "")) - - def test_checks_only_the_paths_that_it_is_given(self) -> None: - with git_repo() as root: - (root / "a.ts").write_text("console.log(1);\n", encoding="utf-8") - (root / "b.ts").write_text("console.log(1);\n", encoding="utf-8") - status, output, _ = run_main("--root", str(root), "b.ts") - self.assertEqual(status, 1) - self.assertEqual(output.splitlines(), ["b.ts:1:1: TELL-13 Remove 'console.log('."]) - - def test_exits_with_1_and_names_a_code_file_that_is_not_utf_8(self) -> None: - with git_repo() as root: - (root / "a.ts").write_bytes(b"const a = '\x93';\n") - (root / "logo.png").write_bytes(b"\x89PNG\r\n\x1a\n\xff") - status, output, errors = run_main("--root", str(root)) - self.assertEqual(status, 1) - self.assertEqual(errors, "") - self.assertEqual(len(output.splitlines()), 1) - self.assertIn("a.ts:1:12: TELL-14 Byte 0x93 is not valid UTF-8.", output) - - -class BadInputTest(unittest.TestCase): - def assert_one_line_on_stderr(self, result: tuple[int, str, str], *words: str) -> None: - status, output, errors = result - self.assertEqual(status, 2) - self.assertEqual(output, "") - self.assertEqual(len(errors.splitlines()), 1, errors) - self.assertTrue(errors.startswith("check-tells: "), errors) - for word in words: - self.assertIn(word, errors) - - def test_reports_a_root_that_is_not_a_git_repo(self) -> None: - with tempfile.TemporaryDirectory() as folder: - root = Path(folder).resolve() - # Stop git from finding a repo above the temporary folder. - with mock.patch.dict(os.environ, {"GIT_CEILING_DIRECTORIES": str(root.parent)}): - result = run_main("--root", str(root)) - self.assert_one_line_on_stderr(result, str(root), "not a git repository") - - def test_reports_a_root_that_does_not_exist(self) -> None: - with tempfile.TemporaryDirectory() as folder: - missing = Path(folder).resolve() / "missing" - result = run_main("--root", str(missing)) - self.assert_one_line_on_stderr(result, str(missing)) - - def test_reports_a_path_that_does_not_exist(self) -> None: - with git_repo() as root: - result = run_main("--root", str(root), "missing.ts") - self.assert_one_line_on_stderr(result, "missing.ts", "No such file") - - def test_reports_a_path_that_is_a_folder(self) -> None: - with git_repo() as root: - (root / "src").mkdir() - result = run_main("--root", str(root), "src") - self.assert_one_line_on_stderr(result, "src", "directory") - - def test_reports_a_path_outside_the_root(self) -> None: - with git_repo() as root, tempfile.TemporaryDirectory() as other: - outside = Path(other).resolve() / "a.ts" - outside.write_text("console.log(1);\n", encoding="utf-8") - absolute = run_main("--root", str(root), str(outside)) - relative = run_main("--root", str(root), "../" + outside.parent.name + "/a.ts") - self.assert_one_line_on_stderr(absolute, str(outside), "outside the root") - self.assert_one_line_on_stderr(relative, "outside the root") - - def test_reports_a_commit_message_file_that_does_not_exist(self) -> None: - with tempfile.TemporaryDirectory() as folder: - missing = Path(folder) / "message" - result = run_main("--commit-msg", str(missing)) - self.assert_one_line_on_stderr(result, str(missing), "No such file") - - def test_reports_a_commit_message_that_is_not_utf_8(self) -> None: - with tempfile.TemporaryDirectory() as folder: - message = Path(folder) / "message" - message.write_bytes(b"fix: parse caf\xe9\n") - status, output, errors = run_main("--commit-msg", str(message)) - self.assertEqual((status, errors), (1, "")) - self.assertIn("commit message:1:15: TELL-14 Byte 0xE9 is not valid UTF-8.", output) - - def test_prints_no_traceback_from_the_command(self) -> None: - command = [str(Path(__file__).resolve().parents[1] / "check-tells")] - with tempfile.TemporaryDirectory() as folder: - root = Path(folder).resolve() - environment = {**os.environ, "GIT_CEILING_DIRECTORIES": str(root.parent)} - run = subprocess.run( - [*command, "--root", str(root)], - check=False, - capture_output=True, - text=True, - env=environment, - ) - self.assertEqual(run.returncode, 2) - self.assertEqual(run.stdout, "") - self.assertNotIn("Traceback", run.stderr) - self.assertEqual(len(run.stderr.splitlines()), 1) - - -class ConventionalSubjectTest(unittest.TestCase): - def test_accepts_each_commit_type(self) -> None: - for kind in COMMIT_TYPES: - with self.subTest(kind=kind): - self.assertEqual(check_squash_message(squash(f"{kind}: reject a unit")), []) - - def test_accepts_a_mark_for_a_breaking_change(self) -> None: - self.assertEqual(check_squash_message(squash("feat!: drop the old name (#4)")), []) - - def test_rejects_a_subject_that_does_not_follow_the_form(self) -> None: - subjects = [ - "Foundation: standards and grammar (#1)", - "Fix: reject a unit", - "fix:reject a unit", - "fix : reject a unit", - "fix(): reject a unit", - "fix", - "fix: ", - "Update files", - ": reject a unit", - "fix(core:) reject a unit", - ] - for subject in subjects: - with self.subTest(subject=subject): - found = check_squash_message(squash(subject)) - self.assertEqual(rules(found), ["GIT-1"]) - self.assertEqual((found[0].line, found[0].column), (1, 1)) - self.assertIn("does not follow Conventional Commits", found[0].message) - - def test_rejects_a_type_that_conventional_commits_does_not_list(self) -> None: - for kind in ("update", "feature", "bugfix", "wip", "chores"): - with self.subTest(kind=kind): - found = check_squash_message(squash(f"{kind}: reject a unit")) - self.assertEqual(rules(found), ["GIT-1"]) - self.assertIn(f"'{kind}' is not a Conventional Commits type", found[0].message) - - def test_names_each_type_that_it_accepts(self) -> None: - found = check_squash_message(squash("update: reject a unit")) - self.assertIn( - "build, chore, ci, docs, feat, fix, perf, refactor, revert, style or test", - found[0].message, - ) - - def test_accepts_a_scope_unless_the_repo_has_none(self) -> None: - self.assertEqual(check_squash_message(squash("fix(core): reject a unit")), []) - self.assertEqual(check_squash_message(squash("fix(core)!: reject a unit")), []) - - def test_rejects_a_scope_in_a_repo_that_has_none(self) -> None: - for subject in ("fix(core): reject a unit", "feat(deps)!: drop a name"): - with self.subTest(subject=subject): - found = check_squash_message(squash(subject), no_scope=True) - self.assertEqual(rules(found), ["GIT-1"]) - self.assertEqual((found[0].line, found[0].column), (1, subject.index("(") + 1)) - self.assertIn("Remove the scope", found[0].message) - - def test_accepts_a_subject_without_a_scope_in_a_repo_that_has_none(self) -> None: - self.assertEqual(check_squash_message(squash("fix: reject a unit"), no_scope=True), []) - - def test_rejects_a_message_that_has_no_subject(self) -> None: - found = check_squash_message("\n \n") - self.assertEqual(sorted(rules(found)), ["GIT-1", "GIT-2"]) - self.assertEqual(found[0].message, "The message has no subject.") - - def test_rejects_the_default_squash_subject_of_a_pull_request_with_no_type(self) -> None: - title = "Foundation: standards, grammar, data, shared vectors and check-tells (#1)" - found = check_squash_message(squash(title), no_scope=True) - self.assertEqual(sorted(rules(found)), ["GIT-1", "TELL-18"]) - - def test_reads_the_first_line_that_has_text_as_the_subject(self) -> None: - found = check_squash_message("\n\nupdate: x\n\nbody\n\n" + SIGN_OFF_LINE) - self.assertEqual([(v.rule, v.line) for v in found], [("GIT-1", 3)]) - - -class SquashMessageTest(unittest.TestCase): - def test_accepts_a_good_message(self) -> None: - message = ( - "fix: reject a unit of 00 (#7)\n\n" - "The parser accepted a unit of 00. It now returns bad_segment.\n\n" - "Closes #5\n\n" + SIGN_OFF_LINE + "\n" - ) - self.assertEqual(check_squash_message(message, no_scope=True), []) - - def test_counts_the_suffix_that_the_merge_adds_to_the_title(self) -> None: - self.assertEqual(check_squash_message(squash("fix: " + "a" * 62 + " (#7)")), []) - found = check_squash_message(squash("fix: " + "a" * 63 + " (#7)")) - self.assertEqual(rules(found), ["TELL-18"]) - self.assertEqual(found[0].column, 73) - self.assertIn("The subject has 73 characters. Use 72 or fewer.", found[0].message) - self.assertIn("(#N)", found[0].message) - - def test_reads_a_line_that_starts_with_a_hash_as_text(self) -> None: - body = f"## Notes {EMOJI}\n# {EM_DASH} here" - found = check_squash_message(squash("fix: reject a unit", body=body)) - self.assertEqual([(v.rule, v.line) for v in found], [("TELL-14", 3), ("TELL-14", 4)]) - - def test_reads_a_line_that_starts_with_a_hash_as_a_comment_in_a_commit_file(self) -> None: - message = f"fix: reject a unit\n\n## Notes {EMOJI}\n" - self.assertEqual(check_commit_message(message), []) - - def test_rejects_non_ascii_in_the_body_but_not_in_a_sign_off(self) -> None: - found = check_squash_message(squash("fix: a unit", body=f"Use a dash {EM_DASH} here.")) - self.assertEqual([(v.rule, v.line) for v in found], [("TELL-14", 3)]) - sign_off = f"Signed-off-by: Ad{E_ACUTE} Bello " - self.assertEqual(check_squash_message(squash("fix: a unit", sign_off=sign_off)), []) - - def test_treats_crlf_like_lf(self) -> None: - message = squash("fix: reject a unit (#7)", body="Closes #5").replace("\n", "\r\n") - self.assertEqual(check_squash_message(message), []) - - def test_reports_every_problem_once(self) -> None: - message = "Update files\n\n" + PLACEHOLDER_PARAGRAPH + "\n\nCloses #\n" - found = check_squash_message(message) - self.assertEqual(sorted(rules(found)), ["GIT-1", "GIT-2", "TELL-18", "TELL-18"]) - - -class HtmlCommentTest(unittest.TestCase): - RENOVATE_COMMENT = "" - - def test_leaves_out_the_comment_that_renovate_ends_a_body_with(self) -> None: - message = squash("ci: update an action (#4)", body="Update a pinned tool or action.") - self.assertEqual(check_squash_message(message + self.RENOVATE_COMMENT + "\n"), []) - - def test_leaves_out_the_text_that_a_comment_holds(self) -> None: - hidden = [ - EMOJI, - f"a dash {EM_DASH} here", - "Closes #", - PLACEHOLDER_PARAGRAPH, - "Signed-off-by: Your Name ", - ] - for text in hidden: - with self.subTest(text=text): - body = f"Why the change is needed.\n\n" - self.assertEqual(check_squash_message(squash("fix: a unit", body=body)), []) - - def test_leaves_out_a_comment_that_spans_lines(self) -> None: - body = f"Why the change is needed.\n\n" - self.assertEqual(check_squash_message(squash("fix: a unit", body=body)), []) - - def test_keeps_the_place_of_the_text_around_a_comment(self) -> None: - body = f"Use a dash {EM_DASH} here." - found = check_squash_message(squash("fix: a unit", body=body)) - self.assertEqual([(v.rule, v.line, v.column) for v in found], [("TELL-14", 4, 7)]) - - def test_leaves_out_each_comment_of_a_line_and_not_the_text_between_them(self) -> None: - body = f"Fix the {EM_DASH} unit rule." - found = check_squash_message(squash("fix: a unit", body=body)) - self.assertEqual( - [(v.rule, v.line, v.column) for v in found], [("TELL-14", 3, body.index(EM_DASH) + 1)] - ) - - def test_does_not_count_a_sign_off_that_a_comment_holds(self) -> None: - body = f"Why the change is needed.\n\n" - found = check_squash_message(f"fix: a unit\n\n{body}\n") - self.assertEqual(rules(found), ["GIT-2"]) - self.assertIn("The message has no Signed-off-by line.", found[0].message) - - def test_reads_a_comment_that_does_not_end_as_text(self) -> None: - body = f"Why the change is needed.\n\n")) - self.assertEqual([(v.rule, v.line) for v in found], [("TELL-14", 1)]) - - def test_reads_a_comment_in_a_message_with_crlf(self) -> None: - body = f"Why the change is needed.\n\n" - message = squash("fix: a unit", body=body).replace("\n", "\r\n") - self.assertEqual(check_squash_message(message), []) - - def test_leaves_out_nothing_in_a_commit_message_file(self) -> None: - message = f"fix: a unit\n\n\n" - self.assertEqual(rules(check_commit_message(message)), ["TELL-14"]) - - -class SignOffTest(unittest.TestCase): - def test_requires_a_sign_off(self) -> None: - found = check_squash_message("fix: reject a unit\n\nWhy the change is needed.\n") - self.assertEqual(rules(found), ["GIT-2"]) - self.assertEqual((found[0].line, found[0].column), (1, 1)) - self.assertIn("The message has no Signed-off-by line.", found[0].message) - - def test_reads_each_form_of_a_sign_off(self) -> None: - forms = [ - "Signed-off-by: Ada Bello ", - "signed-off-by: Ada Bello ", - "SIGNED-OFF-BY: Ada Bello ", - "Signed-off-by: Ada Bello ", - "Signed-off-by: A ", - f"Signed-off-by: Ad{E_ACUTE} Bello ", - "Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>", - ] - for form in forms: - with self.subTest(form=form): - self.assertEqual(check_squash_message(squash("fix: a unit", sign_off=form)), []) - - def test_rejects_each_line_that_only_looks_like_a_sign_off(self) -> None: - forms = [ - "Signed-off-by:", - "Signed-off-by: Ada Bello", - "Signed-off-by: ", - "Signed-off-by: Ada Bello ", - "Signed-off-by: Ada Bello ada@example.org", - "Signed-off-by Ada Bello ", - "Reviewed-by: Ada Bello ", - "Co-authored-by: Ada Bello ", - "- Signed-off-by: Ada Bello ", - "Signed-off-by: Ada ", - ] - for form in forms: - with self.subTest(form=form): - found = check_squash_message(squash("fix: a unit", sign_off=form)) - self.assertEqual(rules(found), ["GIT-2"]) - - def test_rejects_the_placeholder_address_of_the_template(self) -> None: - sign_off = "Signed-off-by: Your Name " - found = check_squash_message(squash("fix: a unit", sign_off=sign_off)) - self.assertEqual(rules(found), ["GIT-2"]) - self.assertEqual((found[0].line, found[0].column), (5, 1)) - self.assertIn("placeholder address", found[0].message) - - def test_rejects_the_placeholder_address_in_any_letter_case(self) -> None: - sign_off = "Signed-off-by: Ada Bello " - found = check_squash_message(squash("fix: a unit", sign_off=sign_off)) - self.assertEqual(rules(found), ["GIT-2"]) - - def test_rejects_the_placeholder_address_next_to_a_real_sign_off(self) -> None: - body = "Signed-off-by: Your Name " - found = check_squash_message(squash("fix: a unit", body=body)) - self.assertEqual([(v.rule, v.line) for v in found], [("GIT-2", 3)]) - - def test_accepts_the_placeholder_address_in_prose(self) -> None: - body = "The template shows you@example.com in its sign-off line." - self.assertEqual(check_squash_message(squash("docs: fix a template", body=body)), []) - - -class TemplateTextTest(unittest.TestCase): - def test_rejects_the_placeholder_paragraph_of_the_template(self) -> None: - found = check_squash_message(squash("fix: a unit", body=PLACEHOLDER_PARAGRAPH)) - self.assertEqual(rules(found), ["TELL-18"]) - self.assertEqual((found[0].line, found[0].column), (3, 1)) - self.assertIn("placeholder paragraph", found[0].message) - - def test_rejects_the_paragraph_after_an_edit_of_its_end_or_its_layout(self) -> None: - bodies = [ - PLACEHOLDER_PARAGRAPH + " This change fixes a unit.", - " " + PLACEHOLDER_PARAGRAPH, - PLACEHOLDER_PARAGRAPH.removesuffix("."), - "Write two or three sentences of plain prose\nthat say what this change does.", - ] - for body in bodies: - with self.subTest(body=body): - found = check_squash_message(squash("fix: a unit", body=body)) - self.assertEqual(rules(found), ["TELL-18"]) - - def test_accepts_a_body_that_only_mentions_the_words(self) -> None: - bodies = [ - "The template asks the author to write two or three sentences of plain prose.", - "The template says: Write two or three sentences of plain prose.", - ] - for body in bodies: - with self.subTest(body=body): - found = check_squash_message(squash("docs: fix a template", body=body)) - self.assertEqual(found, []) - - def test_rejects_an_empty_closes_line(self) -> None: - for body in ("Closes #", "closes #", "Closes # ", " Closes #"): - with self.subTest(body=body): - found = check_squash_message(squash("fix: a unit", body=body)) - self.assertEqual(rules(found), ["TELL-18"]) - self.assertEqual(found[0].line, 3) - self.assertIn("Closes #", found[0].message) - - def test_accepts_a_closes_line_with_an_issue(self) -> None: - for body in ("Closes #5", "Closes #5, closes #6", "Fixes #12"): - with self.subTest(body=body): - self.assertEqual(check_squash_message(squash("fix: a unit", body=body)), []) - - -class SignedOffRangeTest(unittest.TestCase): - def test_reports_each_commit_without_a_sign_off(self) -> None: - with git_repo() as root: - base = commit(root, "chore: start") - signed = commit(root, "feat: a\n\n" + SIGN_OFF_LINE) - unsigned = commit(root, "fix: b") - found = check_signed_off(root, f"{base}..HEAD") - self.assertEqual([(v.path, v.rule) for v in found], [(f"commit {unsigned[:12]}", "GIT-2")]) - self.assertNotIn(signed[:12], str(found)) - - def test_accepts_a_range_where_each_commit_has_a_sign_off(self) -> None: - with git_repo() as root: - base = commit(root, "chore: start") - commit(root, "feat: a\n\n" + SIGN_OFF_LINE) - commit(root, "fix: b\n\nWhy.\n\n" + SIGN_OFF_LINE) - self.assertEqual(check_signed_off(root, f"{base}..HEAD"), []) - - def test_ignores_the_commits_before_the_range(self) -> None: - with git_repo() as root: - commit(root, "chore: start") - base = commit(root, "chore: second") - commit(root, "feat: a\n\n" + SIGN_OFF_LINE) - self.assertEqual(check_signed_off(root, f"{base}..HEAD"), []) - - def test_skips_a_merge_commit(self) -> None: - with git_repo() as root: - base = commit(root, "chore: start") - branch = git(root, "rev-parse", "--abbrev-ref", "HEAD") - git(root, "checkout", "-q", "-b", "topic") - commit(root, "feat: a\n\n" + SIGN_OFF_LINE) - git(root, "checkout", "-q", branch) - commit(root, "fix: b\n\n" + SIGN_OFF_LINE) - git( - root, "-c", "commit.gpgsign=false", "merge", "--no-ff", "-q", "-m", "Merge", "topic" - ) - self.assertEqual(check_signed_off(root, f"{base}..HEAD"), []) - - def test_reads_a_line_that_starts_with_a_hash_as_text(self) -> None: - with git_repo() as root: - base = commit(root, "chore: start") - commit(root, "fix: b\n\n# Signed-off-by: Ada Bello ") - self.assertEqual(rules(check_signed_off(root, f"{base}..HEAD")), ["GIT-2"]) - - @unittest.skipUnless(shutil.which("ssh-keygen"), "ssh-keygen makes the signing key") - def test_gives_the_hash_of_a_signed_commit_when_git_shows_signatures(self) -> None: - with git_repo() as root, tempfile.TemporaryDirectory() as keys: - key = Path(keys) / "key" - subprocess.run( - ["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-f", str(key)], check=True - ) - allowed = Path(keys) / "allowed" - allowed.write_text(f"ada@example.org {key.with_suffix('.pub').read_text()}") - settings = { - "gpg.format": "ssh", - "user.signingkey": str(key), - "gpg.ssh.allowedSignersFile": str(allowed), - "log.showSignature": "true", - } - for name, value in settings.items(): - git(root, "config", name, value) - base = commit(root, "chore: start") - git(root, "commit", "--allow-empty", "-q", "-S", "-m", "fix: b") - signed = git(root, "rev-parse", "HEAD") - found = check_signed_off(root, f"{base}..HEAD") - self.assertEqual([violation.path for violation in found], [f"commit {signed[:12]}"]) - - def test_does_not_reject_non_ascii_in_a_commit_message(self) -> None: - with git_repo() as root: - base = commit(root, "chore: start") - commit(root, f"fix: caf{E_ACUTE}\n\n" + SIGN_OFF_LINE) - self.assertEqual(check_signed_off(root, f"{base}..HEAD"), []) - - def test_does_not_run_the_other_message_rules_on_a_commit(self) -> None: - with git_repo() as root: - base = commit(root, "chore: start") - commit(root, "Update files " + EMOJI + "\n\n" + SIGN_OFF_LINE) - self.assertEqual(check_signed_off(root, f"{base}..HEAD"), []) - - def test_reads_the_range_from_the_command_line(self) -> None: - with git_repo() as root: - base = commit(root, "chore: start") - unsigned = commit(root, "fix: b") - status, output, errors = run_main("--root", str(root), "--signed-off", f"{base}..HEAD") - self.assertEqual((status, errors), (1, "")) - self.assertEqual(len(output.splitlines()), 1) - self.assertTrue(output.startswith(f"commit {unsigned[:12]}:1:1: GIT-2 "), output) - - def test_exits_with_0_when_each_commit_has_a_sign_off(self) -> None: - with git_repo() as root: - base = commit(root, "chore: start") - commit(root, "fix: b\n\n" + SIGN_OFF_LINE) - result = run_main("--root", str(root), "--signed-off", f"{base}..HEAD") - self.assertEqual(result, (0, "", "")) - - def test_reports_a_range_that_git_cannot_read(self) -> None: - with git_repo() as root: - commit(root, "chore: start") - status, output, errors = run_main("--root", str(root), "--signed-off", "nope..HEAD") - self.assertEqual((status, output), (2, "")) - self.assertEqual(len(errors.splitlines()), 1, errors) - self.assertIn("nope..HEAD", errors) - - def test_does_not_read_a_range_as_an_option(self) -> None: - with git_repo() as root: - commit(root, "chore: start") - status, output, errors = run_main("--root", str(root), "--signed-off=--all") - self.assertEqual((status, output), (2, "")) - self.assertEqual(len(errors.splitlines()), 1, errors) - - -class SquashCommandTest(unittest.TestCase): - def run_with_message(self, text: str, *options: str) -> tuple[int, str, str]: - with tempfile.TemporaryDirectory() as folder: - message = Path(folder) / "message" - message.write_text(text, encoding="utf-8") - return run_main("--squash-msg", str(message), *options) - - def test_exits_with_0_for_a_good_message(self) -> None: - result = self.run_with_message(squash("fix: reject a unit (#7)"), "--no-scope") - self.assertEqual(result, (0, "", "")) - - def test_exits_with_1_and_prints_each_problem(self) -> None: - status, output, errors = self.run_with_message("Update files\n\nBody\n") - self.assertEqual((status, errors), (1, "")) - self.assertEqual( - [line.split(" ")[:3] for line in output.splitlines()], - [["commit", "message:1:1:", "GIT-1"], ["commit", "message:1:1:", "GIT-2"]], - ) - - def test_rejects_a_scope_only_with_the_no_scope_option(self) -> None: - message = squash("fix(core): reject a unit (#7)") - self.assertEqual(self.run_with_message(message), (0, "", "")) - status, output, _ = self.run_with_message(message, "--no-scope") - self.assertEqual(status, 1) - self.assertIn("commit message:1:4: GIT-1 Remove the scope (core).", output) - - def test_reads_a_squash_message_that_is_not_utf_8(self) -> None: - with tempfile.TemporaryDirectory() as folder: - message = Path(folder) / "message" - message.write_bytes(b"fix: parse caf\xe9\n\n" + SIGN_OFF_LINE.encode() + b"\n") - status, output, _ = run_main("--squash-msg", str(message)) - self.assertEqual(status, 1) - self.assertIn("commit message:1:15: TELL-14 Byte 0xE9 is not valid UTF-8.", output) - - def test_reports_a_squash_message_file_that_does_not_exist(self) -> None: - status, output, errors = run_main("--squash-msg", "no-such-file") - self.assertEqual((status, output), (2, "")) - self.assertEqual(len(errors.splitlines()), 1) - - def test_asks_for_a_squash_message_when_it_gets_the_no_scope_option(self) -> None: - errors = io.StringIO() - with self.assertRaises(SystemExit) as caught, contextlib.redirect_stderr(errors): - check_tells.main(["--no-scope"]) - self.assertEqual(caught.exception.code, 2) - self.assertIn("--no-scope needs --squash-msg", errors.getvalue()) - - def test_reads_one_kind_of_message_at_a_time(self) -> None: - for options in ( - ["--commit-msg", "a", "--squash-msg", "b"], - ["--commit-msg", "a", "--signed-off", "b..c"], - ["--squash-msg", "a", "--signed-off", "b..c"], - ): - with self.subTest(options=options): - errors = io.StringIO() - with self.assertRaises(SystemExit) as caught, contextlib.redirect_stderr(errors): - check_tells.main(options) - self.assertEqual(caught.exception.code, 2) - - -def read_template(name: str) -> str: - """Read a file of templates/, which the pull request check and the contributors share.""" - return (Path(__file__).resolve().parents[2] / "templates" / name).read_text(encoding="utf-8") - - -class PullRequestTemplateTest(unittest.TestCase): - TITLE = "fix: reject a unit of 00 (#7)" - - def test_holds_the_paragraph_the_closes_line_and_the_sign_off_line(self) -> None: - lines = read_template("PULL_REQUEST_TEMPLATE.md").split("\n") - self.assertEqual( - lines, - [ - PLACEHOLDER_PARAGRAPH, - "", - "Closes #", - "", - "Signed-off-by: Your Name ", - "", - ], - ) - - def test_has_no_heading_checklist_or_html_comment(self) -> None: - text = read_template("PULL_REQUEST_TEMPLATE.md") - starts = ("#", "- [", "* [", "" + + def test_leaves_out_the_comment_that_renovate_ends_a_body_with(self) -> None: + message = squash("ci: update an action (#4)", body="Update a pinned tool or action.") + self.assertEqual(check_squash_message(message + self.RENOVATE_COMMENT + "\n"), []) + + def test_leaves_out_the_text_that_a_comment_holds(self) -> None: + hidden = [ + EMOJI, + f"a dash {EM_DASH} here", + "Closes #", + PLACEHOLDER_PARAGRAPH, + "Signed-off-by: Your Name ", + ] + for text in hidden: + with self.subTest(text=text): + body = f"Why the change is needed.\n\n" + self.assertEqual(check_squash_message(squash("fix: a unit", body=body)), []) + + def test_leaves_out_a_comment_that_spans_lines(self) -> None: + body = f"Why the change is needed.\n\n" + self.assertEqual(check_squash_message(squash("fix: a unit", body=body)), []) + + def test_keeps_the_place_of_the_text_around_a_comment(self) -> None: + body = f"Use a dash {EM_DASH} here." + found = check_squash_message(squash("fix: a unit", body=body)) + self.assertEqual([(v.rule, v.line, v.column) for v in found], [("TELL-14", 4, 7)]) + + def test_leaves_out_each_comment_of_a_line_and_not_the_text_between_them(self) -> None: + body = f"Fix the {EM_DASH} unit rule." + found = check_squash_message(squash("fix: a unit", body=body)) + self.assertEqual( + [(v.rule, v.line, v.column) for v in found], [("TELL-14", 3, body.index(EM_DASH) + 1)] + ) + + def test_does_not_count_a_sign_off_that_a_comment_holds(self) -> None: + body = f"Why the change is needed.\n\n" + found = check_squash_message(f"fix: a unit\n\n{body}\n") + self.assertEqual(rules(found), ["GIT-2"]) + self.assertIn("The message has no Signed-off-by line.", found[0].message) + + def test_reads_a_comment_that_does_not_end_as_text(self) -> None: + body = f"Why the change is needed.\n\n")) + self.assertEqual([(v.rule, v.line) for v in found], [("TELL-14", 1)]) + + def test_reads_a_comment_in_a_message_with_crlf(self) -> None: + body = f"Why the change is needed.\n\n" + message = squash("fix: a unit", body=body).replace("\n", "\r\n") + self.assertEqual(check_squash_message(message), []) + + def test_leaves_out_nothing_in_a_commit_message_file(self) -> None: + message = f"fix: a unit\n\n\n" + self.assertEqual(rules(check_commit_message(message)), ["TELL-14"]) diff --git a/scripts/tests/test_check_tells_pull_request.py b/scripts/tests/test_check_tells_pull_request.py new file mode 100644 index 0000000..2f184e4 --- /dev/null +++ b/scripts/tests/test_check_tells_pull_request.py @@ -0,0 +1,202 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +"""Tests for check_tells.message_checks: the sign-off, the template text and the templates.""" + +import unittest +from pathlib import Path + +import check_tells +from check_tells import check_squash_message + +from tests.check_tells_driver import rules +from tests.check_tells_samples import E_ACUTE, PLACEHOLDER_PARAGRAPH, squash + + +class SignOffTest(unittest.TestCase): + def test_requires_a_sign_off(self) -> None: + found = check_squash_message("fix: reject a unit\n\nWhy the change is needed.\n") + self.assertEqual(rules(found), ["GIT-2"]) + self.assertEqual((found[0].line, found[0].column), (1, 1)) + self.assertIn("The message has no Signed-off-by line.", found[0].message) + + def test_reads_each_form_of_a_sign_off(self) -> None: + forms = [ + "Signed-off-by: Ada Bello ", + "signed-off-by: Ada Bello ", + "SIGNED-OFF-BY: Ada Bello ", + "Signed-off-by: Ada Bello ", + "Signed-off-by: A ", + f"Signed-off-by: Ad{E_ACUTE} Bello ", + "Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>", + ] + for form in forms: + with self.subTest(form=form): + self.assertEqual(check_squash_message(squash("fix: a unit", sign_off=form)), []) + + def test_rejects_each_line_that_only_looks_like_a_sign_off(self) -> None: + forms = [ + "Signed-off-by:", + "Signed-off-by: Ada Bello", + "Signed-off-by: ", + "Signed-off-by: Ada Bello ", + "Signed-off-by: Ada Bello ada@example.org", + "Signed-off-by Ada Bello ", + "Reviewed-by: Ada Bello ", + "Co-authored-by: Ada Bello ", + "- Signed-off-by: Ada Bello ", + "Signed-off-by: Ada ", + ] + for form in forms: + with self.subTest(form=form): + found = check_squash_message(squash("fix: a unit", sign_off=form)) + self.assertEqual(rules(found), ["GIT-2"]) + + def test_rejects_the_placeholder_address_of_the_template(self) -> None: + sign_off = "Signed-off-by: Your Name " + found = check_squash_message(squash("fix: a unit", sign_off=sign_off)) + self.assertEqual(rules(found), ["GIT-2"]) + self.assertEqual((found[0].line, found[0].column), (5, 1)) + self.assertIn("placeholder address", found[0].message) + + def test_rejects_the_placeholder_address_in_any_letter_case(self) -> None: + sign_off = "Signed-off-by: Ada Bello " + found = check_squash_message(squash("fix: a unit", sign_off=sign_off)) + self.assertEqual(rules(found), ["GIT-2"]) + + def test_rejects_the_placeholder_address_next_to_a_real_sign_off(self) -> None: + body = "Signed-off-by: Your Name " + found = check_squash_message(squash("fix: a unit", body=body)) + self.assertEqual([(v.rule, v.line) for v in found], [("GIT-2", 3)]) + + def test_accepts_the_placeholder_address_in_prose(self) -> None: + body = "The template shows you@example.com in its sign-off line." + self.assertEqual(check_squash_message(squash("docs: fix a template", body=body)), []) + + +class TemplateTextTest(unittest.TestCase): + def test_rejects_the_placeholder_paragraph_of_the_template(self) -> None: + found = check_squash_message(squash("fix: a unit", body=PLACEHOLDER_PARAGRAPH)) + self.assertEqual(rules(found), ["TELL-18"]) + self.assertEqual((found[0].line, found[0].column), (3, 1)) + self.assertIn("placeholder paragraph", found[0].message) + + def test_rejects_the_paragraph_after_an_edit_of_its_end_or_its_layout(self) -> None: + bodies = [ + PLACEHOLDER_PARAGRAPH + " This change fixes a unit.", + " " + PLACEHOLDER_PARAGRAPH, + PLACEHOLDER_PARAGRAPH.removesuffix("."), + "Write two or three sentences of plain prose\nthat say what this change does.", + ] + for body in bodies: + with self.subTest(body=body): + found = check_squash_message(squash("fix: a unit", body=body)) + self.assertEqual(rules(found), ["TELL-18"]) + + def test_accepts_a_body_that_only_mentions_the_words(self) -> None: + bodies = [ + "The template asks the author to write two or three sentences of plain prose.", + "The template says: Write two or three sentences of plain prose.", + ] + for body in bodies: + with self.subTest(body=body): + found = check_squash_message(squash("docs: fix a template", body=body)) + self.assertEqual(found, []) + + def test_rejects_an_empty_closes_line(self) -> None: + for body in ("Closes #", "closes #", "Closes # ", " Closes #"): + with self.subTest(body=body): + found = check_squash_message(squash("fix: a unit", body=body)) + self.assertEqual(rules(found), ["TELL-18"]) + self.assertEqual(found[0].line, 3) + self.assertIn("Closes #", found[0].message) + + def test_accepts_a_closes_line_with_an_issue(self) -> None: + for body in ("Closes #5", "Closes #5, closes #6", "Fixes #12"): + with self.subTest(body=body): + self.assertEqual(check_squash_message(squash("fix: a unit", body=body)), []) + + +def read_template(name: str) -> str: + """Read a file of templates/, which the pull request check and the contributors share.""" + return (Path(__file__).resolve().parents[2] / "templates" / name).read_text(encoding="utf-8") + + +class PullRequestTemplateTest(unittest.TestCase): + TITLE = "fix: reject a unit of 00 (#7)" + + def test_holds_the_paragraph_the_closes_line_and_the_sign_off_line(self) -> None: + lines = read_template("PULL_REQUEST_TEMPLATE.md").split("\n") + self.assertEqual( + lines, + [ + PLACEHOLDER_PARAGRAPH, + "", + "Closes #", + "", + "Signed-off-by: Your Name ", + "", + ], + ) + + def test_has_no_heading_checklist_or_html_comment(self) -> None: + text = read_template("PULL_REQUEST_TEMPLATE.md") + starts = ("#", "- [", "* [", ". A Renovate pull request still passes. The GIT-1 text of the standards describes the new rule. The body of a Renovate pull request holds a literal Signed-off-by line. It named a different identity than the commits do. The hosted app commits as renovate[bot], so renovate.json now writes that identity in the body. The commits keep their sign-off through commitTrailers. Signed-off-by: MobileMage --- renovate.json | 2 +- scripts/check_tells/message_checks.py | 18 +++-- scripts/tests/check_tells_samples.py | 6 ++ .../tests/test_check_tells_message_checks.py | 68 +++++++++++++------ .../tests/test_check_tells_pull_request.py | 25 ++++++- standards/CODING_STANDARDS.md | 2 +- 6 files changed, 87 insertions(+), 34 deletions(-) diff --git a/renovate.json b/renovate.json index af37877..923edf1 100644 --- a/renovate.json +++ b/renovate.json @@ -12,7 +12,7 @@ "prBodyTemplate": "{{{notes}}}", "prBodyNotes": [ "Update a pinned tool or action.", - "Signed-off-by: Renovate Bot " + "Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>" ], "packageRules": [ { diff --git a/scripts/check_tells/message_checks.py b/scripts/check_tells/message_checks.py index 98c1e5f..8539333 100644 --- a/scripts/check_tells/message_checks.py +++ b/scripts/check_tells/message_checks.py @@ -41,7 +41,9 @@ # The start of the paragraph in that template that asks the author for prose. TEMPLATE_PARAGRAPH = "Write two or three sentences of plain prose" EMPTY_CLOSES_LINE = re.compile(r"closes #\s*", re.IGNORECASE) -HTML_COMMENT = re.compile(r"", re.DOTALL) +# Renovate adds this comment to the very end of each pull request body. The author did not +# write it, and its payload is base64. +RENOVATE_COMMENT = re.compile(r"\n?\s*$") SQUASH_NOTE = " The subject is the title plus the ' (#N)' that the squash merge adds." @@ -163,20 +165,16 @@ def check_commit_message(text: str) -> list[Violation]: return _check_message(_message_lines(text)) -def _blank_html_comments(body: str) -> str: - """Replace each HTML comment with spaces, and keep its line feeds, so that no place moves.""" - return HTML_COMMENT.sub(lambda comment: re.sub(r"[^\n]", " ", comment[0]), body) - - def check_squash_message(text: str, *, no_scope: bool = False) -> list[Violation]: """Check the message that a squash merge puts on main: the pull request title and body. - The body is not a git message, so a line that starts with # is text and not a comment. No - reader sees an HTML comment, and Renovate ends each body with one, so the body check leaves - them out. The title is the first line, and no comment hides text there. + The body is not a git message, so a line that starts with # is text and not a comment. GitHub + keeps the raw body, so the text of an HTML comment reaches main like any other text. The + check leaves out only the comment that Renovate adds at the very end of its body. """ subject, line_feed, body = text.partition("\n") - lines = list(enumerate(split_lines(subject + line_feed + _blank_html_comments(body)), start=1)) + checked = subject + line_feed + RENOVATE_COMMENT.sub("", body) + lines = list(enumerate(split_lines(checked), start=1)) return [ *_check_message(lines, SQUASH_NOTE), *_check_subject_form(lines, no_scope=no_scope), diff --git a/scripts/tests/check_tells_samples.py b/scripts/tests/check_tells_samples.py index 93c863c..c29a2e2 100644 --- a/scripts/tests/check_tells_samples.py +++ b/scripts/tests/check_tells_samples.py @@ -2,6 +2,8 @@ # SPDX-License-Identifier: Apache-2.0 """Sample text, file names and messages that the check_tells tests share.""" +import string + EMOJI = chr(0x1F600) NO_BREAK_SPACE = chr(0x00A0) E_ACUTE = chr(0x00E9) @@ -117,6 +119,10 @@ PLACEHOLDER_PARAGRAPH = ( "Write two or three sentences of plain prose that say what this change does and why." ) +# The comment that Renovate adds at the very end of the body of its pull requests. The payload +# holds each character of base64. +BASE64 = string.ascii_letters + string.digits + "+/" +RENOVATE_COMMENT = f"" def squash( diff --git a/scripts/tests/test_check_tells_message_checks.py b/scripts/tests/test_check_tells_message_checks.py index 14c0ca4..7ad27e5 100644 --- a/scripts/tests/test_check_tells_message_checks.py +++ b/scripts/tests/test_check_tells_message_checks.py @@ -13,6 +13,7 @@ EMOJI, LINE_SEPARATOR, PLACEHOLDER_PARAGRAPH, + RENOVATE_COMMENT, SIGN_OFF_LINE, squash, ) @@ -220,39 +221,63 @@ def test_reports_every_problem_once(self) -> None: class HtmlCommentTest(unittest.TestCase): - RENOVATE_COMMENT = "" - def test_leaves_out_the_comment_that_renovate_ends_a_body_with(self) -> None: message = squash("ci: update an action (#4)", body="Update a pinned tool or action.") - self.assertEqual(check_squash_message(message + self.RENOVATE_COMMENT + "\n"), []) - - def test_leaves_out_the_text_that_a_comment_holds(self) -> None: - hidden = [ - EMOJI, - f"a dash {EM_DASH} here", - "Closes #", - PLACEHOLDER_PARAGRAPH, - "Signed-off-by: Your Name ", - ] - for text in hidden: - with self.subTest(text=text): - body = f"Why the change is needed.\n\n" - self.assertEqual(check_squash_message(squash("fix: a unit", body=body)), []) + self.assertEqual(check_squash_message(message + RENOVATE_COMMENT + "\n"), []) + + def test_reads_the_comment_of_renovate_when_text_follows_it(self) -> None: + body = f"{RENOVATE_COMMENT} {EM_DASH} here." + found = check_squash_message(squash("ci: update an action (#4)", body=body)) + self.assertEqual( + [(v.rule, v.line, v.column) for v in found], + [("TELL-14", 3, len(RENOVATE_COMMENT) + 2)], + ) - def test_leaves_out_a_comment_that_spans_lines(self) -> None: + def test_reads_a_comment_that_starts_like_the_one_of_renovate_but_holds_other_text( + self, + ) -> None: + payloads = ((EMOJI, EMOJI), (f"a dash {EM_DASH}", EM_DASH), (E_ACUTE, E_ACUTE)) + for payload, character in payloads: + with self.subTest(payload=payload): + comment = f"" + message = squash("ci: update an action (#4)") + comment + "\n" + found = check_squash_message(message) + self.assertEqual( + [(v.rule, v.line, v.column) for v in found], + [("TELL-14", 6, comment.index(character) + 1)], + ) + + def test_leaves_out_the_comment_of_renovate_so_that_no_text_is_left(self) -> None: + # The title line is empty, so the first line with text would be the comment. + found = check_squash_message("\n" + RENOVATE_COMMENT + "\n") + self.assertEqual(found[0].message, "The message has no subject.") + + def test_reads_the_text_that_a_comment_holds(self) -> None: + for text, character in ((EMOJI, EMOJI), (f"a dash {EM_DASH} here", EM_DASH)): + with self.subTest(text=text): + comment = f"" + body = f"Why the change is needed.\n\n{comment}" + found = check_squash_message(squash("fix: a unit", body=body)) + self.assertEqual( + [(v.rule, v.line, v.column) for v in found], + [("TELL-14", 5, comment.index(character) + 1)], + ) + + def test_reads_a_comment_that_spans_lines(self) -> None: body = f"Why the change is needed.\n\n" - self.assertEqual(check_squash_message(squash("fix: a unit", body=body)), []) + found = check_squash_message(squash("fix: a unit", body=body)) + self.assertEqual([(v.rule, v.line) for v in found], [("TELL-14", 7), ("TELL-18", 8)]) def test_keeps_the_place_of_the_text_around_a_comment(self) -> None: body = f"Use a dash {EM_DASH} here." found = check_squash_message(squash("fix: a unit", body=body)) self.assertEqual([(v.rule, v.line, v.column) for v in found], [("TELL-14", 4, 7)]) - def test_leaves_out_each_comment_of_a_line_and_not_the_text_between_them(self) -> None: + def test_reports_the_first_bad_character_of_a_line_with_comments(self) -> None: body = f"Fix the {EM_DASH} unit rule." found = check_squash_message(squash("fix: a unit", body=body)) self.assertEqual( - [(v.rule, v.line, v.column) for v in found], [("TELL-14", 3, body.index(EM_DASH) + 1)] + [(v.rule, v.line, v.column) for v in found], [("TELL-14", 3, body.index(EMOJI) + 1)] ) def test_does_not_count_a_sign_off_that_a_comment_holds(self) -> None: @@ -273,7 +298,8 @@ def test_reads_a_comment_in_the_title_as_text(self) -> None: def test_reads_a_comment_in_a_message_with_crlf(self) -> None: body = f"Why the change is needed.\n\n" message = squash("fix: a unit", body=body).replace("\n", "\r\n") - self.assertEqual(check_squash_message(message), []) + found = check_squash_message(message) + self.assertEqual([(v.rule, v.line, v.column) for v in found], [("TELL-14", 6, 1)]) def test_leaves_out_nothing_in_a_commit_message_file(self) -> None: message = f"fix: a unit\n\n\n" diff --git a/scripts/tests/test_check_tells_pull_request.py b/scripts/tests/test_check_tells_pull_request.py index 2f184e4..451807a 100644 --- a/scripts/tests/test_check_tells_pull_request.py +++ b/scripts/tests/test_check_tells_pull_request.py @@ -2,14 +2,16 @@ # SPDX-License-Identifier: Apache-2.0 """Tests for check_tells.message_checks: the sign-off, the template text and the templates.""" +import json import unittest from pathlib import Path +from typing import Any import check_tells from check_tells import check_squash_message from tests.check_tells_driver import rules -from tests.check_tells_samples import E_ACUTE, PLACEHOLDER_PARAGRAPH, squash +from tests.check_tells_samples import E_ACUTE, PLACEHOLDER_PARAGRAPH, RENOVATE_COMMENT, squash class SignOffTest(unittest.TestCase): @@ -200,3 +202,24 @@ def test_contributing_tells_the_author_what_the_title_and_the_body_become(self) def test_agents_names_the_interface_section_of_the_grammar(self) -> None: text = read_template("AGENTS.md") self.assertIn("grammar, with its Interface section, and the vectors", text) + + +class RenovateConfigTest(unittest.TestCase): + # The hosted Renovate app commits as this author, so the body must be signed off as it. + HOSTED_APP = "Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>" + + def config(self) -> dict[str, Any]: + path = Path(__file__).resolve().parents[2] / "renovate.json" + config: dict[str, Any] = json.loads(path.read_text(encoding="utf-8")) + return config + + def test_signs_off_each_commit_as_its_author(self) -> None: + self.assertEqual(self.config()["commitTrailers"], ["Signed-off-by: {{{gitAuthor}}}"]) + + def test_signs_off_the_body_as_the_hosted_app(self) -> None: + self.assertIn(self.HOSTED_APP, self.config()["prBodyNotes"]) + + def test_writes_a_body_that_passes_the_squash_check(self) -> None: + notes = "\n\n".join(self.config()["prBodyNotes"]) + message = f"ci: update an action (#4)\n\n{notes}\n\n{RENOVATE_COMMENT}\n" + self.assertEqual(check_squash_message(message, no_scope=True), []) diff --git a/standards/CODING_STANDARDS.md b/standards/CODING_STANDARDS.md index f0ecf09..88fedbc 100644 --- a/standards/CODING_STANDARDS.md +++ b/standards/CODING_STANDARDS.md @@ -115,7 +115,7 @@ It covers the tool rules that no language tool covers: - **TELL-13.** It rejects the debug calls that the language file lists. It skips test files, comment lines and, in Python, doctest lines that start with `>>>` or `...`. - **TELL-14.** It rejects emojis in every file, including Markdown, message catalogues and lockfiles. It rejects other non-ASCII characters in code files, config files and commit messages. It also rejects a code or config file that is not valid UTF-8, because it cannot read the file. It skips any other file that is not valid UTF-8, because the file can be an image. Message catalogues, docs prose and the sign-off lines of commit messages are exempt from the non-ASCII check, because translations and contributors' names can need any script. - **TELL-18.** With `--commit-msg `, it checks the length of a commit subject. Repos without commitlint use this mode. -- **GIT-1.** With `--squash-msg `, it checks the message that a squash merge puts on `main`. That message is the pull request title, then ` (#N)`, then the pull request body. The subject must start with a type from the Conventional Commits list, then a colon and a space. TELL-14 and TELL-18 apply to the same message, and the ` (#N)` counts towards the 72 characters. The tool leaves out the HTML comments of the body, which no reader sees. Renovate ends each body with one. With `--no-scope`, the subject must also have no scope, as GIT-1 asks of a repo without packages. +- **GIT-1.** With `--squash-msg `, it checks the message that a squash merge puts on `main`. That message is the pull request title, then ` (#N)`, then the pull request body. The subject must start with a type from the Conventional Commits list, then a colon and a space. TELL-14 and TELL-18 apply to the same message, and the ` (#N)` counts towards the 72 characters. GitHub keeps the body as written, so the tool reads each HTML comment of the body as text. It leaves out only the comment that Renovate adds at the end of its body, which has the form ``. With `--no-scope`, the subject must also have no scope, as GIT-1 asks of a repo without packages. - **GIT-2.** The squash message must have a line `Signed-off-by: Name
`, and the address must not be the placeholder of the pull request template. The message must not hold the placeholder paragraph or the line `Closes #` of that template. With `--signed-off `, the tool checks that each commit in a git revision range has a `Signed-off-by` line. It skips merge commits, and it applies no other message rule to a commit, because only the squash message reaches `main`. A line in a code or config file can skip one check with a comment that names the rule and gives a reason, for example `check-tells: allow TELL-1 because the regex cannot be split`. Reviewers check each skip. From 66b9c5f759bb36a759aea7433776e976917e9cf3 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 14:27:46 +0100 Subject: [PATCH 38/45] test: add vectors for a letter after a digit in the LGA and the unit A digit check without its end anchor reads the first digit of a segment and ignores the rest. Such a check accepted JI-9X-CHK-LS-18, and it passed every vector. The only earlier cases with a letter after a digit have a unit that starts with 0, which fails the minimum on its own. Two vectors close the gap. JI9XCHKLS18 has an LGA of 9X, and JI99CHKLS1X has a unit of 1X. Both give bad_segment. The letter X has no fix, so neither code gets a suggestion. A copy of the reference reading with the weak check passes the 306 earlier vectors and fails exactly these two. A test checks that the file keeps a case of each kind. The ids from parse-segments-015 on move up by two, as the vector format allows. Signed-off-by: MobileMage --- scripts/build_vectors.py | 12 +++++ scripts/tests/test_build_vectors.py | 16 +++++++ vectors/parse-segments.json | 74 ++++++++++++++++++++--------- 3 files changed, 79 insertions(+), 23 deletions(-) diff --git a/scripts/build_vectors.py b/scripts/build_vectors.py index 32ee9a3..f4f79c8 100755 --- a/scripts/build_vectors.py +++ b/scripts/build_vectors.py @@ -404,6 +404,18 @@ def segment_rows() -> list[Row]: ), row("rejects an LGA of 00", "EK00A03FK01", failed("bad_segment", "lga")), row("rejects a unit of 00", "EK01A03FK00", failed("bad_segment", "unit")), + # A digit check that has no end anchor reads the first digit of the segment and accepts + # these codes. The letter X has no fix, so neither code gets a suggestion. + row( + "rejects a letter after a digit in the LGA", + "JI9XCHKLS18", + failed("bad_segment", "lga"), + ), + row( + "rejects a letter after a digit in the unit", + "JI99CHKLS1X", + failed("bad_segment", "unit"), + ), row( "suggests 0 for a letter O in the LGA", "EKO1A03FK01", diff --git a/scripts/tests/test_build_vectors.py b/scripts/tests/test_build_vectors.py index d2c33c8..f56f72e 100644 --- a/scripts/tests/test_build_vectors.py +++ b/scripts/tests/test_build_vectors.py @@ -230,6 +230,22 @@ def test_parse_segments_rejects_a_digit_before_a_letter_in_the_area(self) -> Non # A letter check that is anchored only at the end accepts such an area. self.assertTrue(rejected) + def test_parse_segments_rejects_a_letter_after_a_digit_in_the_lga_and_the_unit(self) -> None: + # A digit check that has no end anchor reads the first digit and accepts such a segment. + # A digit of 0 fails the minimum alone, so the digit here is 1 to 9. + for segment, start in (("lga", 2), ("unit", 9)): + with self.subTest(segment=segment): + rejected = [ + case + for case in load("parse-segments")["cases"] + if len(case["input"]) == 11 + and case["input"][start] in string.digits[1:] + and case["input"][start + 1] in string.ascii_letters + and case["expect"].get("error", {}).get("code") == "bad_segment" + and case["expect"]["error"]["segment"] == segment + ] + self.assertTrue(rejected) + def test_precision_for_accuracy_has_a_case_for_negative_zero(self) -> None: # The JSON literal -0 loads as the integer 0, so only -0.0 keeps the sign. negative_zero = [ diff --git a/vectors/parse-segments.json b/vectors/parse-segments.json index ea15662..938063c 100644 --- a/vectors/parse-segments.json +++ b/vectors/parse-segments.json @@ -201,6 +201,34 @@ }, { "id": "parse-segments-015", + "description": "rejects a letter after a digit in the LGA", + "input": "JI9XCHKLS18", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "lga", + "suggestion": null + } + } + }, + { + "id": "parse-segments-016", + "description": "rejects a letter after a digit in the unit", + "input": "JI99CHKLS1X", + "options": {}, + "expect": { + "ok": false, + "error": { + "code": "bad_segment", + "segment": "unit", + "suggestion": null + } + } + }, + { + "id": "parse-segments-017", "description": "suggests 0 for a letter O in the LGA", "input": "EKO1A03FK01", "options": {}, @@ -214,7 +242,7 @@ } }, { - "id": "parse-segments-016", + "id": "parse-segments-018", "description": "suggests 1 for a letter I in the LGA", "input": "EKI1A03FK01", "options": {}, @@ -228,7 +256,7 @@ } }, { - "id": "parse-segments-017", + "id": "parse-segments-019", "description": "suggests 1 for a letter L in the LGA", "input": "EKL1A03FK01", "options": {}, @@ -242,7 +270,7 @@ } }, { - "id": "parse-segments-018", + "id": "parse-segments-020", "description": "suggests I for a one in the area", "input": "EK01A03F101", "options": {}, @@ -256,7 +284,7 @@ } }, { - "id": "parse-segments-019", + "id": "parse-segments-021", "description": "suggests O for a zero in the area", "input": "EK01A03F001", "options": {}, @@ -270,7 +298,7 @@ } }, { - "id": "parse-segments-020", + "id": "parse-segments-022", "description": "rejects a digit before a letter in the area", "input": "EK01A031K01", "options": {}, @@ -284,7 +312,7 @@ } }, { - "id": "parse-segments-021", + "id": "parse-segments-023", "description": "suggests 0 for a letter O in the unit", "input": "EK01A03FKO1", "options": {}, @@ -298,7 +326,7 @@ } }, { - "id": "parse-segments-022", + "id": "parse-segments-024", "description": "suggests 1 for a letter I in the unit", "input": "EK01A03FKI2", "options": {}, @@ -312,7 +340,7 @@ } }, { - "id": "parse-segments-023", + "id": "parse-segments-025", "description": "suggests 1 for a letter L in the unit", "input": "EK01A03FKL1", "options": {}, @@ -326,7 +354,7 @@ } }, { - "id": "parse-segments-024", + "id": "parse-segments-026", "description": "suggests 1 for a letter I after a zero in the unit", "input": "EK01A03FK0I", "options": {}, @@ -340,7 +368,7 @@ } }, { - "id": "parse-segments-025", + "id": "parse-segments-027", "description": "gives no suggestion when the fix makes a unit of 00", "input": "EK01A03FK0O", "options": {}, @@ -354,7 +382,7 @@ } }, { - "id": "parse-segments-026", + "id": "parse-segments-028", "description": "gives no suggestion when a segment cannot be fixed", "input": "EK01A03F2O1", "options": {}, @@ -368,7 +396,7 @@ } }, { - "id": "parse-segments-027", + "id": "parse-segments-029", "description": "reports the first bad segment and fixes them all", "input": "EKO1A03F1O1", "options": {}, @@ -382,7 +410,7 @@ } }, { - "id": "parse-segments-028", + "id": "parse-segments-030", "description": "suggests a fix after making letters upper case", "input": "eko1a03fk01", "options": {}, @@ -396,7 +424,7 @@ } }, { - "id": "parse-segments-029", + "id": "parse-segments-031", "description": "rejects an LGA of 00 in a partial code", "input": "EK00", "options": { @@ -412,7 +440,7 @@ } }, { - "id": "parse-segments-030", + "id": "parse-segments-032", "description": "suggests a partial code", "input": "EKO1", "options": { @@ -428,7 +456,7 @@ } }, { - "id": "parse-segments-031", + "id": "parse-segments-033", "description": "suggests a partial area code", "input": "EK01A03F1", "options": { @@ -444,7 +472,7 @@ } }, { - "id": "parse-segments-032", + "id": "parse-segments-034", "description": "rejects an unknown state when partial codes are allowed", "input": "XX", "options": { @@ -460,7 +488,7 @@ } }, { - "id": "parse-segments-033", + "id": "parse-segments-035", "description": "suggests a state for a partial code", "input": "0G", "options": { @@ -476,7 +504,7 @@ } }, { - "id": "parse-segments-034", + "id": "parse-segments-036", "description": "suggests an LGA fix for a district code", "input": "EKO1A03", "options": { @@ -492,7 +520,7 @@ } }, { - "id": "parse-segments-035", + "id": "parse-segments-037", "description": "rejects an LGA of 00 in a district code", "input": "EK00A03", "options": { @@ -508,7 +536,7 @@ } }, { - "id": "parse-segments-036", + "id": "parse-segments-038", "description": "keeps a letter O in the district when it fixes the LGA", "input": "EKO1AO3FK01", "options": {}, @@ -522,7 +550,7 @@ } }, { - "id": "parse-segments-037", + "id": "parse-segments-039", "description": "keeps letters I and L in the district when it fixes the LGA", "input": "EKO1AILFK01", "options": {}, @@ -536,7 +564,7 @@ } }, { - "id": "parse-segments-038", + "id": "parse-segments-040", "description": "accepts letters and digits in any district position", "input": "EK01A0OFK01", "options": {}, From 55cc84cf4af421b9af9fd794fbd6fc0232448a58 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 14:33:43 +0100 Subject: [PATCH 39/45] ci: end the Renovate pull request body with the sign-off Renovate can add its own notes to the notes section of the body, after the sign-off line, and those notes can hold characters that the squash message check rejects. The sign-off now sits in the header, and the body template ends with it, as in the js repo. Signed-off-by: MobileMage --- renovate.json | 11 ++++++----- scripts/tests/test_check_tells_pull_request.py | 12 ++++++++---- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/renovate.json b/renovate.json index 923edf1..60fb5d1 100644 --- a/renovate.json +++ b/renovate.json @@ -1,5 +1,9 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "description": [ + "Each pull request body ends with the sign-off line that the pull request job needs.", + "prHeader holds the line because a JSON string cannot wrap, and the template ends with it." + ], "extends": [ "config:recommended", "helpers:pinGitHubActionDigests", @@ -9,11 +13,8 @@ "semanticCommits": "enabled", "semanticCommitScope": "", "commitTrailers": ["Signed-off-by: {{{gitAuthor}}}"], - "prBodyTemplate": "{{{notes}}}", - "prBodyNotes": [ - "Update a pinned tool or action.", - "Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>" - ], + "prHeader": "Signed-off-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com>", + "prBodyTemplate": "Update a pinned tool or action.\n\n{{{header}}}", "packageRules": [ { "matchManagers": ["github-actions"], diff --git a/scripts/tests/test_check_tells_pull_request.py b/scripts/tests/test_check_tells_pull_request.py index 451807a..578ed57 100644 --- a/scripts/tests/test_check_tells_pull_request.py +++ b/scripts/tests/test_check_tells_pull_request.py @@ -216,10 +216,14 @@ def config(self) -> dict[str, Any]: def test_signs_off_each_commit_as_its_author(self) -> None: self.assertEqual(self.config()["commitTrailers"], ["Signed-off-by: {{{gitAuthor}}}"]) - def test_signs_off_the_body_as_the_hosted_app(self) -> None: - self.assertIn(self.HOSTED_APP, self.config()["prBodyNotes"]) + def body(self) -> str: + config = self.config() + body: str = config["prBodyTemplate"].replace("{{{header}}}", config["prHeader"]) + return body + + def test_ends_the_body_with_a_sign_off_as_the_hosted_app(self) -> None: + self.assertTrue(self.body().endswith(self.HOSTED_APP)) def test_writes_a_body_that_passes_the_squash_check(self) -> None: - notes = "\n\n".join(self.config()["prBodyNotes"]) - message = f"ci: update an action (#4)\n\n{notes}\n\n{RENOVATE_COMMENT}\n" + message = f"ci: update an action (#4)\n\n{self.body()}\n\n{RENOVATE_COMMENT}\n" self.assertEqual(check_squash_message(message, no_scope=True), []) From 94ded1b65b18f16b120219b60ad2875cbc42266c Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 14:33:43 +0100 Subject: [PATCH 40/45] docs: add require to the banned globals of the TypeScript rules The js repo bans require in package code, because an ES module has no require at run time and the Node types make tsc accept it. The tool table now lists it with the other banned globals. Signed-off-by: MobileMage --- standards/languages/typescript.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/standards/languages/typescript.md b/standards/languages/typescript.md index a2380f7..acf58ac 100644 --- a/standards/languages/typescript.md +++ b/standards/languages/typescript.md @@ -79,7 +79,7 @@ tsdown builds the packages, so `tsc` only checks types. Each package's `tsconfig | `@typescript-eslint/explicit-module-boundary-types` | error in `packages/*/src` | TS-8 | | `@typescript-eslint/no-non-null-assertion` | off in `packages/*/test` | TS-4 | | `no-restricted-imports` for every Node built-in module, by bare name and as `node:*` | error in `packages/*/src` | CS-2 | -| `no-restricted-globals` for `process`, `Buffer`, `__dirname`, `__filename` and `global` | error in `packages/*/src` | CS-2 | +| `no-restricted-globals` for `process`, `Buffer`, `__dirname`, `__filename`, `global` and `require` | error in `packages/*/src` | CS-2 | | `max-depth` | 3 | TELL-2 | | `complexity` | 10 | TELL-2 | | `max-params` | 4 | TELL-3 | From dc5a6572ff7791afa7c8374ad72f62c8b6a4371a Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 15:23:53 +0100 Subject: [PATCH 41/45] fix: match only ASCII white space after the Renovate comment The squash check leaves out the comment that Renovate adds to the end of its pull request body. The pattern for that comment ended with \s*$. In Python, \s matches Unicode white space. A body that ended with the comment and then U+3000, U+2028, U+00A0, U+0085 or U+2003 lost that character together with the comment. TELL-14 never saw it, and the character reached main. Any author can write the comment. The pattern now allows only a space, a tab, a carriage return or a line feed after the comment. Then the text must end. TELL-14 reports any other character. A test covers each of the five characters. A second test covers the ASCII white space that still passes. Signed-off-by: MobileMage --- scripts/check_tells/message_checks.py | 5 +++-- .../tests/test_check_tells_message_checks.py | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/scripts/check_tells/message_checks.py b/scripts/check_tells/message_checks.py index 8539333..ee30589 100644 --- a/scripts/check_tells/message_checks.py +++ b/scripts/check_tells/message_checks.py @@ -42,8 +42,9 @@ TEMPLATE_PARAGRAPH = "Write two or three sentences of plain prose" EMPTY_CLOSES_LINE = re.compile(r"closes #\s*", re.IGNORECASE) # Renovate adds this comment to the very end of each pull request body. The author did not -# write it, and its payload is base64. -RENOVATE_COMMENT = re.compile(r"\n?\s*$") +# write it, and its payload is base64. Only ASCII white space may follow it, because \s also +# matches characters such as U+3000, and TELL-14 must see those. +RENOVATE_COMMENT = re.compile(r"\n?[ \t\r\n]*\Z") SQUASH_NOTE = " The subject is the title plus the ' (#N)' that the squash merge adds." diff --git a/scripts/tests/test_check_tells_message_checks.py b/scripts/tests/test_check_tells_message_checks.py index 7ad27e5..8dd3dca 100644 --- a/scripts/tests/test_check_tells_message_checks.py +++ b/scripts/tests/test_check_tells_message_checks.py @@ -252,6 +252,23 @@ def test_leaves_out_the_comment_of_renovate_so_that_no_text_is_left(self) -> Non found = check_squash_message("\n" + RENOVATE_COMMENT + "\n") self.assertEqual(found[0].message, "The message has no subject.") + def test_leaves_out_the_comment_of_renovate_when_ascii_white_space_follows_it(self) -> None: + for tail in ("", " ", "\t", "\n", "\r\n", " \t\r\n\n"): + with self.subTest(tail=tail): + found = check_squash_message("\n" + RENOVATE_COMMENT + tail) + self.assertEqual(found[0].message, "The message has no subject.") + + def test_reads_the_comment_of_renovate_when_non_ascii_white_space_follows_it(self) -> None: + # \s matches each of these, so a pattern that ends in \s* hides them from TELL-14. + for code_point in (0x3000, 0x2028, 0x00A0, 0x0085, 0x2003): + with self.subTest(code_point=f"U+{code_point:04X}"): + message = squash("ci: update an action (#4)") + found = check_squash_message(message + RENOVATE_COMMENT + chr(code_point) + "\n") + self.assertEqual( + [(v.rule, v.line, v.column) for v in found], + [("TELL-14", 6, len(RENOVATE_COMMENT) + 1)], + ) + def test_reads_the_text_that_a_comment_holds(self) -> None: for text, character in ((EMOJI, EMOJI), (f"a dash {EM_DASH} here", EM_DASH)): with self.subTest(text=text): From ac80e349ccb3ab78e202cc11b09347175c942731 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 15:32:19 +0100 Subject: [PATCH 42/45] docs: make the text agree with the tools and the vectors The standards listed the placeholder paragraph and the empty "Closes #" line of the pull request template under GIT-2. The tool reports both as TELL-18, which asks for the reason in the body, in prose. The text now sits under TELL-18. The standards also said that data files can use any script. The tool holds a JSON data file, such as data/states.json, to ASCII, because a .json file is a config file. The text now says so, and it names .csv as a data file that can use any script. The contributing guide gave only titles with a scope. The spec repo has no packages, and its squash check rejects a scope, so a contributor who copied the example got a GIT-1 failure. The guide now shows the title that the spec repo uses. The README of the vectors said that a case that the grammar contradicts fails the check. A case fails when the plain reading in scripts/reference.py disagrees with it, and the README now says that. The grammar said that the vectors test every rule, with two exceptions. At least three more rules have no shared vector: normalize accepts input of any length, truncate fails for a value that is not a precision, and each SDK exposes SPEC_VERSION. The grammar now lists all five and says that each SDK tests them itself. New tests tie each of these statements to the tool, so that the text and the tool cannot drift apart again without a failing test. Signed-off-by: MobileMage --- grammar.md | 10 +++- .../tests/test_check_tells_pull_request.py | 10 ++++ scripts/tests/test_check_tells_standards.py | 50 +++++++++++++++++++ standards/CODING_STANDARDS.md | 8 +-- templates/CONTRIBUTING.md | 4 +- vectors/README.md | 2 +- 6 files changed, 76 insertions(+), 8 deletions(-) create mode 100644 scripts/tests/test_check_tells_standards.py diff --git a/grammar.md b/grammar.md index 63d2927..910a3ef 100644 --- a/grammar.md +++ b/grammar.md @@ -1,6 +1,14 @@ # Postcode grammar -Spec version 0.1.0. This file defines how every Gatepost SDK reads and writes Nigeria's digital postcodes. The values live in `data/`. The vectors in `vectors/` test every rule, with two exceptions. Two groups of rules have no shared vectors: the Unicode version (see Normalise) and text that is not well-formed (see Parse). JSON cannot carry invalid UTF-8, and PHP rejects an escape for a lone surrogate. Each SDK tests these rules itself. If this file and a vector disagree, this file wins. Such a disagreement is a bug in the spec. +Spec version 0.1.0. This file defines how every Gatepost SDK reads and writes Nigeria's digital postcodes. The values live in `data/`. The vectors in `vectors/` test every rule, except the rules in the list below. If this file and a vector disagree, this file wins. Such a disagreement is a bug in the spec. + +No shared vector tests these rules. Each SDK tests them itself. + +- The Unicode version of NFKC (see Normalise). +- That `normalize` accepts input of any length (see Normalise). +- Text that is not well-formed (see Parse). JSON cannot carry invalid UTF-8, and PHP rejects an escape for a lone surrogate. +- That `truncate` fails for a value that is not a precision (see Hierarchy). +- That each SDK exposes `SPEC_VERSION` (see Versions). ## Forms diff --git a/scripts/tests/test_check_tells_pull_request.py b/scripts/tests/test_check_tells_pull_request.py index 578ed57..b2cd3d0 100644 --- a/scripts/tests/test_check_tells_pull_request.py +++ b/scripts/tests/test_check_tells_pull_request.py @@ -199,6 +199,16 @@ def test_contributing_tells_the_author_what_the_title_and_the_body_become(self) self.assertIn("commit message on `main`", section) self.assertIn("Conventional Commits", section) + def test_contributing_gives_the_title_form_that_the_spec_repo_accepts(self) -> None: + text = read_template("CONTRIBUTING.md") + scoped, plain = "fix(core): reject a unit of 00", "fix: reject a unit of 00" + found = check_squash_message(squash(scoped + " (#7)"), no_scope=True) + self.assertEqual(rules(found), ["GIT-1"]) + self.assertEqual(check_squash_message(squash(plain + " (#7)"), no_scope=True), []) + # Each title with a scope needs the title that the spec repo uses, which has none. + self.assertEqual(text.count(f"`{plain}`"), text.count(f"`{scoped}`")) + self.assertIn("no scope", text) + def test_agents_names_the_interface_section_of_the_grammar(self) -> None: text = read_template("AGENTS.md") self.assertIn("grammar, with its Interface section, and the vectors", text) diff --git a/scripts/tests/test_check_tells_standards.py b/scripts/tests/test_check_tells_standards.py new file mode 100644 index 0000000..5f7b35a --- /dev/null +++ b/scripts/tests/test_check_tells_standards.py @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: 2026 The Gatepost authors +# SPDX-License-Identifier: Apache-2.0 +"""Tests that the check-tells section of the standards says what the tool does.""" + +import unittest +from pathlib import Path + +from check_tells import check_squash_message, check_text + +from tests.check_tells_driver import rules +from tests.check_tells_samples import E_ACUTE, PLACEHOLDER_PARAGRAPH, squash + +STANDARDS = Path(__file__).resolve().parents[2] / "standards" / "CODING_STANDARDS.md" + + +def bullet(label: str) -> str: + """Return the bullet of the check-tells section that starts with this bold label.""" + text = STANDARDS.read_text(encoding="utf-8") + section = text.partition("### The `check-tells` script")[2].partition("\n## ")[0] + found = [line for line in section.splitlines() if line.startswith(f"- **{label}")] + if len(found) != 1: + raise AssertionError(f"The section has {len(found)} bullets for {label!r}.") + return found[0] + + +class TemplateTextRuleTest(unittest.TestCase): + def test_names_each_template_text_check_under_the_rule_that_the_tool_reports(self) -> None: + checks = {PLACEHOLDER_PARAGRAPH: "placeholder paragraph", "Closes #": "`Closes #`"} + for body, phrase in checks.items(): + with self.subTest(phrase=phrase): + [rule] = rules(check_squash_message(squash("fix: a unit", body=body))) + self.assertIn(phrase, bullet(f"{rule}.")) + + def test_names_no_template_text_check_under_the_sign_off_rule(self) -> None: + sign_off = bullet("GIT-2.") + self.assertNotIn("placeholder paragraph", sign_off) + self.assertNotIn("Closes #", sign_off) + + +class DataFileTest(unittest.TestCase): + def test_holds_a_json_data_file_to_ascii(self) -> None: + found = check_text("data/states.json", f'{{"name": "Caf{E_ACUTE}"}}') + self.assertEqual(rules(found), ["TELL-14"]) + config = bullet("Config files") + self.assertIn("JSON data file", config) + self.assertIn("ASCII", config) + + def test_lets_a_data_file_of_another_type_use_any_script(self) -> None: + self.assertEqual(check_text("data/names.csv", f"Caf{E_ACUTE}"), []) + self.assertIn("`.csv`", bullet("Other files")) diff --git a/standards/CODING_STANDARDS.md b/standards/CODING_STANDARDS.md index 88fedbc..cf14654 100644 --- a/standards/CODING_STANDARDS.md +++ b/standards/CODING_STANDARDS.md @@ -104,8 +104,8 @@ Coding agents repeat some habits so often that readers spot them at once. Each r `spec/scripts/check-tells` runs in the CI of every repo. It sorts each file into a kind, and the kind decides which rules apply: - **Code files** are files in a programming language (`.ts`, `.js`, `.php`, `.py`, `.go`, `.kt`, `.java`, `.cs`, `.dart`, `.swift` and their variants), SQL and shell scripts, files with no extension that start with `#!`, and `.css`, `.scss`, `.html`, `.astro`, `.vue` and `.svelte` files. -- **Config files** are `.json`, `.json5`, `.jsonc`, `.yml`, `.yaml`, `.toml`, `.xml`, `.ini`, `.cfg`, `.properties`, `.gradle` and `.neon` files, and files such as `Makefile` and `Dockerfile`. A `.dist` file counts as the file that it copies, so `phpunit.xml.dist` is a config file. -- **Other files** can use any script: Markdown prose, message catalogues, lockfiles, data files, images and file types that no rule names. A **message catalogue** is a data file that holds translated strings. It is a file that is not code in a folder named `locales`, `i18n`, `l10n` or `messages`, a `.po`, `.arb`, `.xlf`, `.xliff`, `.strings` or `.stringsdict` file, or an Android `strings.xml`. Such a file is not a config file, even when its extension is a config extension, such as `src/locales/en.json`. A code file in such a folder is code, so translations live in data files. +- **Config files** are `.json`, `.json5`, `.jsonc`, `.yml`, `.yaml`, `.toml`, `.xml`, `.ini`, `.cfg`, `.properties`, `.gradle` and `.neon` files, and files such as `Makefile` and `Dockerfile`. A `.dist` file counts as the file that it copies, so `phpunit.xml.dist` is a config file. A JSON data file, such as `data/states.json`, is a config file too. It uses ASCII, and it writes other characters as escapes. +- **Other files** can use any script: Markdown prose, message catalogues, lockfiles, images and file types that no rule names, such as `.csv` data files. A **message catalogue** is a data file that holds translated strings. It is a file that is not code in a folder named `locales`, `i18n`, `l10n` or `messages`, a `.po`, `.arb`, `.xlf`, `.xliff`, `.strings` or `.stringsdict` file, or an Android `strings.xml`. Such a file is not a config file, even when its extension is a config extension, such as `src/locales/en.json`. A code file in such a folder is code, so translations live in data files. It covers the tool rules that no language tool covers: @@ -114,9 +114,9 @@ It covers the tool rules that no language tool covers: - **TELL-12 and CS-6.** It rejects each `TODO` and `FIXME`, in any letter case, in code and config files, unless it has the form `TODO(#123)` or `FIXME(#123)`. - **TELL-13.** It rejects the debug calls that the language file lists. It skips test files, comment lines and, in Python, doctest lines that start with `>>>` or `...`. - **TELL-14.** It rejects emojis in every file, including Markdown, message catalogues and lockfiles. It rejects other non-ASCII characters in code files, config files and commit messages. It also rejects a code or config file that is not valid UTF-8, because it cannot read the file. It skips any other file that is not valid UTF-8, because the file can be an image. Message catalogues, docs prose and the sign-off lines of commit messages are exempt from the non-ASCII check, because translations and contributors' names can need any script. -- **TELL-18.** With `--commit-msg `, it checks the length of a commit subject. Repos without commitlint use this mode. +- **TELL-18.** With `--commit-msg `, it checks the length of a commit subject. Repos without commitlint use this mode. With `--squash-msg `, it also rejects a body that still holds the placeholder paragraph or the line `Closes #` of the pull request template. The body must explain why, in prose. - **GIT-1.** With `--squash-msg `, it checks the message that a squash merge puts on `main`. That message is the pull request title, then ` (#N)`, then the pull request body. The subject must start with a type from the Conventional Commits list, then a colon and a space. TELL-14 and TELL-18 apply to the same message, and the ` (#N)` counts towards the 72 characters. GitHub keeps the body as written, so the tool reads each HTML comment of the body as text. It leaves out only the comment that Renovate adds at the end of its body, which has the form ``. With `--no-scope`, the subject must also have no scope, as GIT-1 asks of a repo without packages. -- **GIT-2.** The squash message must have a line `Signed-off-by: Name
`, and the address must not be the placeholder of the pull request template. The message must not hold the placeholder paragraph or the line `Closes #` of that template. With `--signed-off `, the tool checks that each commit in a git revision range has a `Signed-off-by` line. It skips merge commits, and it applies no other message rule to a commit, because only the squash message reaches `main`. +- **GIT-2.** The squash message must have a line `Signed-off-by: Name
`, and the address must not be the placeholder of the pull request template. With `--signed-off `, the tool checks that each commit in a git revision range has a `Signed-off-by` line. It skips merge commits, and it applies no other message rule to a commit, because only the squash message reaches `main`. A line in a code or config file can skip one check with a comment that names the rule and gives a reason, for example `check-tells: allow TELL-1 because the regex cannot be split`. Reviewers check each skip. diff --git a/templates/CONTRIBUTING.md b/templates/CONTRIBUTING.md index b3edcb7..d99a0fc 100644 --- a/templates/CONTRIBUTING.md +++ b/templates/CONTRIBUTING.md @@ -22,7 +22,7 @@ Gatepost is unofficial. For problems with NIPOST's own app, data or API, contact ## Commit -- Use Conventional Commits, for example `fix(core): reject a unit of 00` (GIT-1). +- Use Conventional Commits, for example `fix(core): reject a unit of 00` (GIT-1). The `spec` repo has no packages, so its commits use no scope, as in `fix: reject a unit of 00`. - Keep the subject at 72 characters or fewer. Explain why in the body (TELL-18). - Sign off each commit with `git commit -s` (GIT-2). The sign-off states that you accept the Developer Certificate of Origin at https://developercertificate.org. - If you forget a sign-off, run `git rebase --signoff` with the base branch of your pull request, for example `git rebase --signoff origin/main`. Then push to your own branch with `git push --force-with-lease`. GIT-4 forbids a force-push only to `main` and to shared branches. @@ -37,7 +37,7 @@ You can use AI tools. You are responsible for each line that you submit. If an A The title and the body of the pull request become the commit message on `main`, and CI checks both. 1. Keep it to one topic. Aim for 400 changed lines or fewer, not counting tests, vectors and generated files (GIT-5). -2. Write the title as a Conventional Commits subject, for example `fix(core): reject a unit of 00` (GIT-1). GitHub adds ` (#N)` to it, and the whole subject has 72 characters or fewer (TELL-18). +2. Write the title as a Conventional Commits subject, for example `fix(core): reject a unit of 00` (GIT-1). GitHub adds ` (#N)` to it, and the whole subject has 72 characters or fewer (TELL-18). The `spec` repo uses no scope, so its titles look like `fix: reject a unit of 00`. 3. Fill in the pull request template. - Replace the first paragraph with two or three sentences of plain prose that say what the change does and why. - Replace `Closes #` with the number of the issue that the change closes, or remove the line. diff --git a/vectors/README.md b/vectors/README.md index 303a9da..b88f01b 100644 --- a/vectors/README.md +++ b/vectors/README.md @@ -4,7 +4,7 @@ Each JSON file here holds shared test cases for one core function. Every SDK run `scripts/build_vectors.py` builds these files. Do not edit them by hand. Edit the builder, then run `python3 scripts/build_vectors.py`. -`scripts/reference.py` is a plain reading of `grammar.md` and `data/`. `make check` runs every case through it. A case that the grammar contradicts fails the check, and the grammar wins. +`scripts/reference.py` is a plain reading of `grammar.md` and `data/`. `make check` runs every case through it. A case fails the check when the reading disagrees with it. The grammar wins over both the case and the reading. ## Format From bbc0eed32bd98979a3b164816e14ba87dcc7ef19 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 15:35:59 +0100 Subject: [PATCH 43/45] test: add vectors for long input to normalize and for contains Two mistakes passed every shared vector. The grammar says that normalize has no input limit, but no normalize case had more than 64 code points. A reading whose normalize refused long text, or cut it at the limit, passed all 308 vectors. The new case normalize-075 puts 64 spaces on each side of a code, so the code starts after the limit. Both mistakes fail it. A contains that looks for the prefix anywhere in the code, and not at its start, passed every vector too. The new case contains-009 has the prefix FC and the code EK-01-FC0-FK-01. The district FC0 starts with FC, so such a contains says true, and the right answer is false. The grammar named the first rule as one that no vector tests. It now names only the rules that still have no shared vector. Tests plant each mistake in a copy of the reference reading and require that a vector catches it. Both cases come last in their files, so no earlier id moves. Signed-off-by: MobileMage --- grammar.md | 1 - scripts/build_vectors.py | 13 ++++++++ scripts/tests/test_reference.py | 56 +++++++++++++++++++++++++++++++++ vectors/contains.json | 12 +++++++ vectors/normalize.json | 9 ++++++ 5 files changed, 90 insertions(+), 1 deletion(-) diff --git a/grammar.md b/grammar.md index 910a3ef..8dabf3a 100644 --- a/grammar.md +++ b/grammar.md @@ -5,7 +5,6 @@ Spec version 0.1.0. This file defines how every Gatepost SDK reads and writes Ni No shared vector tests these rules. Each SDK tests them itself. - The Unicode version of NFKC (see Normalise). -- That `normalize` accepts input of any length (see Normalise). - Text that is not well-formed (see Parse). JSON cannot carry invalid UTF-8, and PHP rejects an escape for a lone surrogate. - That `truncate` fails for a value that is not a precision (see Hierarchy). - That each SDK exposes `SPEC_VERSION` (see Versions). diff --git a/scripts/build_vectors.py b/scripts/build_vectors.py index f4f79c8..38b2ff4 100755 --- a/scripts/build_vectors.py +++ b/scripts/build_vectors.py @@ -160,6 +160,7 @@ def load_input_limit() -> int: def normalize_rows() -> list[Row]: """Cases for normalize.""" + limit = load_input_limit() want = value(CODE) separator_rows = [ row(f"removes the separator U+{code_point:04X}", joined(chr(code_point)), want) @@ -222,6 +223,11 @@ def normalize_rows() -> list[Row]: row("returns an empty string for empty input", "", value("")), row("returns an empty string for separators only", " - . ", value("")), *separator_rows, + # normalize has no input limit. The code starts after the first `limit` code points, so a + # reading that refuses the input, or that cuts it at the limit, fails here. + row( + "removes spaces from input over the input limit", " " * limit + CODE + " " * limit, want + ), ] @@ -677,6 +683,13 @@ def contains_rows() -> list[Row]: value(False), ), row("rejects a different unit", {"prefix": full, "code": "EK-01-A03-FK-02"}, value(False)), + # The district FC0 holds the text FC. A check that looks for the prefix anywhere in the + # code finds it there. + row( + "rejects a code that holds the prefix after its start", + {"prefix": "FC", "code": "EK-01-FC0-FK-01"}, + value(False), + ), ] diff --git a/scripts/tests/test_reference.py b/scripts/tests/test_reference.py index dd2260f..63c5015 100644 --- a/scripts/tests/test_reference.py +++ b/scripts/tests/test_reference.py @@ -8,6 +8,7 @@ from collections.abc import Callable from pathlib import Path from typing import Any +from unittest import mock import reference @@ -83,6 +84,23 @@ def vector_documents() -> list[tuple[str, dict[str, Any]]]: return [(path.name, json.loads(path.read_text(encoding="utf-8"))) for path in paths] +def failing_cases(function: str, grammar: reference.Grammar) -> list[str]: + """Run the vectors of one function through a reading, and list the ids that it fails.""" + reader = reference.READERS[function] + failing: list[str] = [] + for _, document in vector_documents(): + if document["function"] != function: + continue + for case in document["cases"]: + try: + answer = reader(grammar, case) + except ValueError: + answer = {"raised": True} + if answer != case["expect"]: + failing.append(case["id"]) + return failing + + def load_with(edits: dict[str, Edit]) -> reference.Grammar: """Load the data files from a temporary folder, after the edits to the named files.""" with tempfile.TemporaryDirectory() as folder: @@ -134,6 +152,44 @@ def test_truncate_rejects_a_precision_that_the_code_lacks(self) -> None: reference.load().truncate(("EK", "01", "A03"), "area") +class ReadingMistakeTest(unittest.TestCase): + """Each test makes one mistake in the reading, and a vector must catch it.""" + + def test_a_normalize_that_refuses_input_over_the_input_limit_fails_a_vector(self) -> None: + original = reference.Grammar.normalize + + def refuse(grammar: reference.Grammar, text: str) -> str: + if len(text) > grammar.input_limit: + raise ValueError("The input is too long.") + return original(grammar, text) + + with mock.patch.object(reference.Grammar, "normalize", refuse): + self.assertNotEqual(failing_cases("normalize", reference.load()), []) + + def test_a_normalize_that_cuts_input_at_the_input_limit_fails_a_vector(self) -> None: + original = reference.Grammar.normalize + + def cut(grammar: reference.Grammar, text: str) -> str: + return original(grammar, text[: grammar.input_limit]) + + with mock.patch.object(reference.Grammar, "normalize", cut): + self.assertNotEqual(failing_cases("normalize", reference.load()), []) + + def test_a_contains_that_looks_for_the_prefix_anywhere_fails_a_vector(self) -> None: + def look_anywhere( + grammar: reference.Grammar, prefix: reference.Postcode, code: reference.Postcode + ) -> bool: + return "-".join(prefix) in "-".join(code) + + with mock.patch.object(reference.Grammar, "contains", look_anywhere): + self.assertNotEqual(failing_cases("contains", reference.load()), []) + + def test_the_reading_without_a_mistake_fails_no_vector(self) -> None: + for function in ("normalize", "contains"): + with self.subTest(function): + self.assertEqual(failing_cases(function, reference.load()), []) + + class InterfaceTest(unittest.TestCase): def test_the_interface_names_each_function_that_a_vector_file_tests(self) -> None: interface = grammar_section("Interface") diff --git a/vectors/contains.json b/vectors/contains.json index c97abb0..523b0bf 100644 --- a/vectors/contains.json +++ b/vectors/contains.json @@ -98,6 +98,18 @@ "expect": { "value": false } + }, + { + "id": "contains-009", + "description": "rejects a code that holds the prefix after its start", + "input": { + "prefix": "FC", + "code": "EK-01-FC0-FK-01" + }, + "options": {}, + "expect": { + "value": false + } } ] } diff --git a/vectors/normalize.json b/vectors/normalize.json index 048d380..81e2ea2 100644 --- a/vectors/normalize.json +++ b/vectors/normalize.json @@ -668,6 +668,15 @@ "expect": { "value": "EK01A03FK01" } + }, + { + "id": "normalize-075", + "description": "removes spaces from input over the input limit", + "input": " EK01A03FK01 ", + "options": {}, + "expect": { + "value": "EK01A03FK01" + } } ] } From 195999a76a604cdf87954138fcb65f3929c03178 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 15:42:48 +0100 Subject: [PATCH 44/45] test: add contains vectors for another LGA and another area A contains that skips the LGA or the area of the prefix passed every vector. Two cases now give a prefix that differs from the code only in that segment, so such a check finds the code inside it and fails. Signed-off-by: MobileMage --- scripts/build_vectors.py | 7 +++++++ vectors/contains.json | 24 ++++++++++++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/scripts/build_vectors.py b/scripts/build_vectors.py index 38b2ff4..3fe0ebb 100755 --- a/scripts/build_vectors.py +++ b/scripts/build_vectors.py @@ -690,6 +690,13 @@ def contains_rows() -> list[Row]: {"prefix": "FC", "code": "EK-01-FC0-FK-01"}, value(False), ), + # A check that skips one segment of the prefix finds these codes inside it. + row("rejects a code from another LGA", {"prefix": "EK-02", "code": full}, value(False)), + row( + "rejects a code from another area", + {"prefix": "EK-01-A03-FA", "code": full}, + value(False), + ), ] diff --git a/vectors/contains.json b/vectors/contains.json index 523b0bf..8e290b2 100644 --- a/vectors/contains.json +++ b/vectors/contains.json @@ -110,6 +110,30 @@ "expect": { "value": false } + }, + { + "id": "contains-010", + "description": "rejects a code from another LGA", + "input": { + "prefix": "EK-02", + "code": "EK-01-A03-FK-01" + }, + "options": {}, + "expect": { + "value": false + } + }, + { + "id": "contains-011", + "description": "rejects a code from another area", + "input": { + "prefix": "EK-01-A03-FA", + "code": "EK-01-A03-FK-01" + }, + "options": {}, + "expect": { + "value": false + } } ] } From d0429052c0aff299a82a13af5bbd270181ad1f94 Mon Sep 17 00:00:00 2001 From: MobileMage Date: Fri, 2 Oct 2026 16:41:13 +0100 Subject: [PATCH 45/45] test: add a normalize vector with each lower-case ASCII letter Ten lower-case letters, such as z and q, were in no vector input, so a case table that lacked one of them passed every vector. Za is the code of Zamfara, so such a table would reject real input. Signed-off-by: MobileMage --- scripts/build_vectors.py | 6 ++++++ vectors/normalize.json | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/scripts/build_vectors.py b/scripts/build_vectors.py index 3fe0ebb..9376f19 100755 --- a/scripts/build_vectors.py +++ b/scripts/build_vectors.py @@ -228,6 +228,12 @@ def normalize_rows() -> list[Row]: row( "removes spaces from input over the input limit", " " * limit + CODE + " " * limit, want ), + # A case table that lacks one letter still passes every postcode that does not hold it. + row( + "makes each ASCII letter upper case", + "abcdefghijklmnopqrstuvwxyz", + value("ABCDEFGHIJKLMNOPQRSTUVWXYZ"), + ), ] diff --git a/vectors/normalize.json b/vectors/normalize.json index 81e2ea2..6dba5aa 100644 --- a/vectors/normalize.json +++ b/vectors/normalize.json @@ -677,6 +677,15 @@ "expect": { "value": "EK01A03FK01" } + }, + { + "id": "normalize-076", + "description": "makes each ASCII letter upper case", + "input": "abcdefghijklmnopqrstuvwxyz", + "options": {}, + "expect": { + "value": "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + } } ] }