From f502cafdd5dd02b5970ef4965b1cfe69276063ad Mon Sep 17 00:00:00 2001 From: Scott Anderson <662325+scottanderson@users.noreply.github.com> Date: Tue, 1 Sep 2026 01:27:20 -0400 Subject: [PATCH 1/2] feat: Add a pluggable text codec for string patterns Add StringEncodeDecode: an interface with decode(), encode(), and encodeLossy(). The host application sets one on the Evaluator. With none set, a string pattern keeps its raw-byte behavior. decode() and encode() return std::optional. A nullopt result marks a byte sequence, or a character, the named encoding cannot represent. encodeLossy() never fails. It substitutes a replacement character for anything the encoding cannot represent. PatternString routes reads and writes through the codec. getValue() and getBytesOf() throw core::err::E0004 on a nullopt result. A script can catch this error with try/catch. setValueLossy() writes through encodeLossy(), and clears the cached bytes and the cached display string; getBytesOf() otherwise reflects a write only after the next pattern run. setValue() clears the cached bytes the same way, on every pattern type, not just PatternString. formatDisplayValue() decodes through decode(), throwing the same core::err::E0004 on a nullopt result. It reads a little past its display budget, and backs decode() off a few bytes at a time on failure, so a multi-byte codepoint straddling the read cutoff doesn't report the whole value as invalid. It then trims the decoded text itself, not the raw bytes, to the display budget, so a codepoint at that cutoff stays whole instead of splitting mid-sequence. getBytesOf() caps the encoded result to the pattern's own size. getEncodingName() reads the string's own encoding attribute. With none, it reads the evaluator's default encoding, set through Evaluator::setDefaultEncoding(). Add a libstd #pragma encoding. It sets the evaluator's default encoding to its value, unconditionally; the codec, not the pragma, knows which encoding names are valid. PatternWideString keeps its fixed UTF-16 behavior. The codec does not apply to it. Add PatternLanguage::clearFormatCaches(). It clears every placed pattern's cached display value, across every section. --- lib/include/pl/core/evaluator.hpp | 38 +++++ lib/include/pl/core/string_encode_decode.hpp | 53 +++++++ lib/include/pl/pattern_language.hpp | 8 ++ lib/include/pl/patterns/pattern.hpp | 1 + lib/include/pl/patterns/pattern_string.hpp | 143 ++++++++++++++++--- lib/source/pl/lib/std/pragmas.cpp | 5 + lib/source/pl/pattern_language.cpp | 7 + 7 files changed, 239 insertions(+), 16 deletions(-) create mode 100644 lib/include/pl/core/string_encode_decode.hpp diff --git a/lib/include/pl/core/evaluator.hpp b/lib/include/pl/core/evaluator.hpp index 8d383757..1428a1a5 100644 --- a/lib/include/pl/core/evaluator.hpp +++ b/lib/include/pl/core/evaluator.hpp @@ -13,6 +13,7 @@ #include #include +#include #include #include @@ -207,6 +208,41 @@ namespace pl::core { void setDataSource(u64 baseAddress, size_t dataSize, std::function readerFunction, std::optional> writerFunction = std::nullopt); + /** + * @brief Sets the text codec for string patterns + * @param codec Codec to set. Pass nullptr to go back to raw bytes. + */ + void setStringEncodeDecode(std::shared_ptr codec) { + this->m_stringEncodeDecode = std::move(codec); + } + + /** + * @brief Gets the text codec set for string patterns + * @return The codec, or nullptr when none is set + */ + [[nodiscard]] const std::shared_ptr& getStringEncodeDecode() const { + return this->m_stringEncodeDecode; + } + + /** + * @brief Sets the default encoding a string pattern with no [[encoding]] + * attribute of its own resolves to + * @param encoding Encoding name to set. Empty lets the codec pick its own + * default. + */ + void setDefaultEncoding(std::string encoding) { + this->m_defaultEncoding = std::move(encoding); + } + + /** + * @brief Gets the default encoding a string pattern with no [[encoding]] + * attribute of its own resolves to + * @return The encoding name, empty when none is set + */ + [[nodiscard]] const std::string& getDefaultEncoding() const { + return this->m_defaultEncoding; + } + void setDataBaseAddress(u64 baseAddress) { this->m_dataBaseAddress = baseAddress; } @@ -556,6 +592,8 @@ namespace pl::core { std::vector> m_currentTemplateArguments; std::function m_dangerousFunctionCalledCallback = []{ return false; }; + std::shared_ptr m_stringEncodeDecode; + std::string m_defaultEncoding; std::function m_breakpointHitCallback = []{ }; std::atomic m_allowDangerousFunctions = DangerousFunctionPermission::Ask; ControlFlowStatement m_currControlFlowStatement = ControlFlowStatement::None; diff --git a/lib/include/pl/core/string_encode_decode.hpp b/lib/include/pl/core/string_encode_decode.hpp new file mode 100644 index 00000000..26832282 --- /dev/null +++ b/lib/include/pl/core/string_encode_decode.hpp @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace pl::core { + + /** + * @brief A pluggable text codec for string patterns + * @note The host application sets this on the Evaluator. With none set, string + * patterns keep their old raw-byte behavior. + */ + class StringEncodeDecode { + public: + virtual ~StringEncodeDecode() = default; + + /** + * @brief Decodes `bytes` under `encoding` + * @param bytes Bytes to decode + * @param encoding Encoding to decode with; empty when the pattern names no + * encoding, in which case the codec picks its own default + * @return The decoded text, or std::nullopt when `bytes` is not valid under + * `encoding` + */ + [[nodiscard]] virtual std::optional decode(std::span bytes, std::string_view encoding) const = 0; + + /** + * @brief Encodes `text` under `encoding` + * @param text Text to encode + * @param encoding Encoding to encode with; empty when the pattern names no + * encoding, in which case the codec picks its own default + * @return The encoded bytes, or std::nullopt when `text` is not representable + * under `encoding` + */ + [[nodiscard]] virtual std::optional> encode(std::string_view text, std::string_view encoding) const = 0; + + /** + * @brief Encodes `text` under `encoding`, never failing + * @param text Text to encode + * @param encoding Encoding to encode with; empty when the pattern names no + * encoding, in which case the codec picks its own default + * @return The encoded bytes, substituting a replacement for anything `encoding` + * cannot represent + */ + [[nodiscard]] virtual std::vector encodeLossy(std::string_view text, std::string_view encoding) const = 0; + }; + +} diff --git a/lib/include/pl/pattern_language.hpp b/lib/include/pl/pattern_language.hpp index 2cffb219..ee36c4d4 100644 --- a/lib/include/pl/pattern_language.hpp +++ b/lib/include/pl/pattern_language.hpp @@ -294,6 +294,14 @@ namespace pl { */ void reset(); + /** + * @brief Clears every placed pattern's cached display value, across every section. + * Call this after changing something a pattern's formatted value depends on + * without re-running the pattern, such as the host application's declared + * string encoding. + */ + void clearFormatCaches(); + /** * @brief Checks whether the runtime is currently running * @return True if the runtime is running, false otherwise diff --git a/lib/include/pl/patterns/pattern.hpp b/lib/include/pl/patterns/pattern.hpp index 8fdeff5b..4710bbf7 100644 --- a/lib/include/pl/patterns/pattern.hpp +++ b/lib/include/pl/patterns/pattern.hpp @@ -484,6 +484,7 @@ namespace pl::ptrn { if (!result.empty()) { this->getEvaluator()->writeData(this->getOffset(), result.data(), result.size(), this->getSection()); this->clearFormatCache(); + this->clearByteCache(); } } diff --git a/lib/include/pl/patterns/pattern_string.hpp b/lib/include/pl/patterns/pattern_string.hpp index 1d42d350..9cbf8e40 100644 --- a/lib/include/pl/patterns/pattern_string.hpp +++ b/lib/include/pl/patterns/pattern_string.hpp @@ -3,6 +3,7 @@ #include #include +#include namespace pl::ptrn { @@ -32,23 +33,84 @@ namespace pl::ptrn { } + /** + * @brief Gets this string's own encoding + * @return This pattern's own [[encoding]] attribute, the evaluator's default + * encoding if none, or empty when neither is set. An empty result falls back + * to the codec's own default. + */ + [[nodiscard]] std::string getEncodingName() const { + if (const auto &arguments = this->getAttributeArguments("encoding"); !arguments.empty()) + return arguments[0].toString(true); + return this->getEvaluator()->getDefaultEncoding(); + } + std::string getValue(size_t size) const { if (size == 0) return ""; + auto *evaluator = this->getEvaluator(); + std::string buffer(size, '\x00'); - this->getEvaluator()->readData(this->getOffset(), buffer.data(), size, this->getSection()); + evaluator->readData(this->getOffset(), buffer.data(), size, this->getSection()); + + if (const auto &codec = evaluator->getStringEncodeDecode(); codec != nullptr) { + const auto encoding = this->getEncodingName(); + auto decoded = codec->decode({ reinterpret_cast(buffer.data()), buffer.size() }, encoding); + if (!decoded.has_value()) + core::err::E0004.throwError(fmt::format("invalid byte sequence for encoding '{}'", encoding)); + + return *decoded; + } return buffer; } std::vector getBytesOf(const core::Token::Literal &value) const override { - if (auto stringValue = std::get_if(&value); stringValue != nullptr) - return { stringValue->begin(), stringValue->end() }; - else + if (auto stringValue = std::get_if(&value); stringValue != nullptr) { + std::vector bytes; + + if (const auto &codec = this->getEvaluator()->getStringEncodeDecode(); codec != nullptr) { + const auto encoding = this->getEncodingName(); + auto encoded = codec->encode(*stringValue, encoding); + if (!encoded.has_value()) + core::err::E0004.throwError(fmt::format("text has no byte value in encoding '{}'", encoding)); + + bytes = *encoded; + } else { + bytes = { stringValue->begin(), stringValue->end() }; + } + + // This field owns a fixed number of bytes in the file. A longer write would + // overwrite whatever comes right after it. Truncate or pad with NUL to the + // field's own size, the same contract every other pattern type already has. + bytes.resize(this->getSize()); + return bytes; + } else return { }; } + /** + * @brief Force-writes `value`, substituting a replacement character for + * anything the pattern's encoding cannot represent + * @param value Value to write + * @note setValue() rejects such a value instead; an editor that offers an + * explicit lossy override calls this one directly. + */ + void setValueLossy(const std::string &value) { + std::vector bytes; + + if (const auto &codec = this->getEvaluator()->getStringEncodeDecode(); codec != nullptr) + bytes = codec->encodeLossy(value, this->getEncodingName()); + else + bytes = { value.begin(), value.end() }; + + bytes.resize(this->getSize()); + this->getEvaluator()->writeData(this->getOffset(), bytes.data(), bytes.size(), this->getSection()); + this->clearFormatCache(); + this->clearByteCache(); + } + [[nodiscard]] std::string getFormattedName() const override { return "String"; } @@ -67,23 +129,72 @@ namespace pl::ptrn { } std::string formatDisplayValue() override { - auto size = std::min(this->getSize(), 0x7F); + auto *evaluator = this->getEvaluator(); + const auto fullSize = this->getSize(); + + // Read a little past DisplayBudget, so a multi-byte codepoint at the + // cutoff usually has the bytes to decode whole. The decode below still + // backs off further on failure; this only keeps that the common case. + constexpr size_t DisplayBudget = 0x7F; + auto size = std::min(fullSize, DisplayBudget + 8); if (size == 0) return "\"\""; std::string buffer(size, 0x00); - this->getEvaluator()->readData(this->getOffset(), buffer.data(), size, this->getSection()); - - const auto pos = buffer.find_last_not_of('\x00'); - if (pos == std::string::npos) - return "\"\""; - - buffer.erase(pos + 1); - - auto displayString = hlp::encodeByteString({ buffer.begin(), buffer.end() }); - - return Pattern::callUserFormatFunc(buffer).value_or(fmt::format("\"{0}\" {1}", displayString, size > this->getSize() ? "(truncated)" : "")); + evaluator->readData(this->getOffset(), buffer.data(), size, this->getSection()); + + if (auto formatted = Pattern::callUserFormatFunc(buffer); formatted.has_value()) + return *formatted; + + const auto &codec = evaluator->getStringEncodeDecode(); + bool truncated = size < fullSize; + + // No codec configured keeps the old raw-byte display, unescaped by any + // encoding. A configured codec reports an undecodable buffer as invalid, + // rather than substituting a replacement character into the display. + if (codec == nullptr) { + if (buffer.size() > DisplayBudget) { + buffer.resize(DisplayBudget); + truncated = true; + } + + const auto pos = buffer.find_last_not_of('\x00'); + if (pos == std::string::npos) + return "\"\""; + buffer.erase(pos + 1); + + return fmt::format("\"{0}\" {1}", hlp::encodeByteString({ buffer.begin(), buffer.end() }), truncated ? "(truncated)" : ""); + } + + const auto encoding = this->getEncodingName(); + + // The read above can itself end mid-codepoint. decode() fails on the whole + // buffer in that case, not just its incomplete tail, so back off a few + // bytes and retry rather than reporting a merely-truncated read as invalid. + std::optional decoded; + for (size_t backoff = 0; backoff <= std::min(4, buffer.size()); ++backoff) { + decoded = codec->decode({ reinterpret_cast(buffer.data()), buffer.size() - backoff }, encoding); + if (decoded.has_value()) { + if (backoff > 0) + truncated = true; + break; + } + } + if (!decoded.has_value()) + core::err::E0004.throwError(fmt::format("invalid byte sequence for encoding '{}'", encoding)); + + // Trim the decoded text itself, not the raw bytes, so a multi-byte + // codepoint at the cutoff stays whole instead of splitting mid-sequence. + if (decoded->size() > DisplayBudget) { + auto cut = DisplayBudget; + while (cut > 0 && (static_cast((*decoded)[cut]) & 0xC0) == 0x80) + --cut; + decoded->resize(cut); + truncated = true; + } + + return fmt::format("\"{0}\" {1}", *decoded, truncated ? "(truncated)" : ""); } std::shared_ptr getEntry(size_t index) const override { diff --git a/lib/source/pl/lib/std/pragmas.cpp b/lib/source/pl/lib/std/pragmas.cpp index 233c4814..11d6791c 100644 --- a/lib/source/pl/lib/std/pragmas.cpp +++ b/lib/source/pl/lib/std/pragmas.cpp @@ -38,6 +38,11 @@ namespace pl::lib::libstd { return false; }); + runtime.addPragma("encoding", [](pl::PatternLanguage &runtime, const std::string &value) { + runtime.getInternals().evaluator->setDefaultEncoding(value); + return true; + }); + runtime.addPragma("eval_depth", [](pl::PatternLanguage &runtime, const std::string &value) { auto limit = parseLimit(value); if (!limit.has_value()) diff --git a/lib/source/pl/pattern_language.cpp b/lib/source/pl/pattern_language.cpp index f161e264..0b638f66 100644 --- a/lib/source/pl/pattern_language.cpp +++ b/lib/source/pl/pattern_language.cpp @@ -533,6 +533,13 @@ namespace pl { this->m_parserManager.setPatternLanguage(this); } + void PatternLanguage::clearFormatCaches() { + for (const auto &[section, patterns] : this->m_patterns) { + for (const auto &pattern : patterns) + pattern->clearFormatCache(); + } + } + void PatternLanguage::addFunction(const api::Namespace &ns, const std::string &name, api::FunctionParameterCount parameterCount, const api::FunctionCallback &func) { this->m_functions.emplace_back(ns, name, parameterCount, func, false); } From dc8ea09ec0500a2847953b771ac269299e134631 Mon Sep 17 00:00:00 2001 From: Scott Anderson <662325+scottanderson@users.noreply.github.com> Date: Thu, 3 Sep 2026 07:26:21 -0400 Subject: [PATCH 2/2] feat: Let the host validate and observe #pragma encoding Add Evaluator::setEncodingValidator(). #pragma encoding fails when this rejects its value instead of always succeeding; the codec, not the evaluator, knows which names mean anything. Add Evaluator::setOnDefaultEncodingChanged(), called after setDefaultEncoding() actually changes the value (validation included). A host observes the declared encoding this way instead of having to replace the "encoding" pragma outright to see it - addPragma() keys handlers by name, so a second registration under the same name replaced libstd's, forcing the replacement to re-implement setDefaultEncoding() itself just to keep doing what the first one did. setDefaultEncoding() now returns bool: false when a validator rejects the value. --- lib/include/pl/core/evaluator.hpp | 33 ++++++++++++++++++++++++++++++- lib/source/pl/lib/std/pragmas.cpp | 3 +-- 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/lib/include/pl/core/evaluator.hpp b/lib/include/pl/core/evaluator.hpp index 9c876bdd..b55c1a1b 100644 --- a/lib/include/pl/core/evaluator.hpp +++ b/lib/include/pl/core/evaluator.hpp @@ -233,9 +233,19 @@ namespace pl::core { * attribute of its own resolves to * @param encoding Encoding name to set. Empty lets the codec pick its own * default. + * @return false when a validator set through setEncodingValidator() rejects + * `encoding`; the default encoding is left unchanged in that case. */ - void setDefaultEncoding(std::string encoding) { + bool setDefaultEncoding(std::string encoding) { + if (this->m_encodingValidator != nullptr && !this->m_encodingValidator(encoding)) + return false; + this->m_defaultEncoding = std::move(encoding); + + if (this->m_onDefaultEncodingChanged != nullptr) + this->m_onDefaultEncodingChanged(this->m_defaultEncoding); + + return true; } /** @@ -247,6 +257,25 @@ namespace pl::core { return this->m_defaultEncoding; } + /** + * @brief Sets a host-supplied check for whether an encoding name means + * anything. #pragma encoding fails when this rejects its value. + * @param validator Called with a candidate encoding name. Pass nullptr + * (the default) to accept any name. + */ + void setEncodingValidator(std::function validator) { + this->m_encodingValidator = std::move(validator); + } + + /** + * @brief Sets a callback invoked whenever setDefaultEncoding() actually + * changes the default encoding, after it passes any validator + * @param callback Called with the new encoding name + */ + void setOnDefaultEncodingChanged(std::function callback) { + this->m_onDefaultEncodingChanged = std::move(callback); + } + void setDataBaseAddress(u64 baseAddress) { this->m_dataBaseAddress = baseAddress; } @@ -598,6 +627,8 @@ namespace pl::core { std::function m_dangerousFunctionCalledCallback = []{ return false; }; std::shared_ptr m_stringEncodeDecode; std::string m_defaultEncoding; + std::function m_encodingValidator; + std::function m_onDefaultEncodingChanged; std::function m_breakpointHitCallback = []{ }; std::atomic m_allowDangerousFunctions = DangerousFunctionPermission::Ask; ControlFlowStatement m_currControlFlowStatement = ControlFlowStatement::None; diff --git a/lib/source/pl/lib/std/pragmas.cpp b/lib/source/pl/lib/std/pragmas.cpp index 11d6791c..f1eea63e 100644 --- a/lib/source/pl/lib/std/pragmas.cpp +++ b/lib/source/pl/lib/std/pragmas.cpp @@ -39,8 +39,7 @@ namespace pl::lib::libstd { }); runtime.addPragma("encoding", [](pl::PatternLanguage &runtime, const std::string &value) { - runtime.getInternals().evaluator->setDefaultEncoding(value); - return true; + return runtime.getInternals().evaluator->setDefaultEncoding(value); }); runtime.addPragma("eval_depth", [](pl::PatternLanguage &runtime, const std::string &value) {