diff --git a/lib/include/pl/core/evaluator.hpp b/lib/include/pl/core/evaluator.hpp index b9afa888..b55c1a1b 100644 --- a/lib/include/pl/core/evaluator.hpp +++ b/lib/include/pl/core/evaluator.hpp @@ -13,6 +13,7 @@ #include #include +#include #include #include @@ -211,6 +212,70 @@ namespace pl::core { void setDataSource(u64 baseAddress, size_t dataSize, std::function readerFunction, std::optional> writerFunction = std::nullopt); + /** + * @brief Sets the text codec for string patterns + * @param codec Codec to set. Pass nullptr to go back to raw bytes. + */ + void setStringEncodeDecode(std::shared_ptr codec) { + this->m_stringEncodeDecode = std::move(codec); + } + + /** + * @brief Gets the text codec set for string patterns + * @return The codec, or nullptr when none is set + */ + [[nodiscard]] const std::shared_ptr& getStringEncodeDecode() const { + return this->m_stringEncodeDecode; + } + + /** + * @brief Sets the default encoding a string pattern with no [[encoding]] + * attribute of its own resolves to + * @param encoding Encoding name to set. Empty lets the codec pick its own + * default. + * @return false when a validator set through setEncodingValidator() rejects + * `encoding`; the default encoding is left unchanged in that case. + */ + bool setDefaultEncoding(std::string encoding) { + if (this->m_encodingValidator != nullptr && !this->m_encodingValidator(encoding)) + return false; + + this->m_defaultEncoding = std::move(encoding); + + if (this->m_onDefaultEncodingChanged != nullptr) + this->m_onDefaultEncodingChanged(this->m_defaultEncoding); + + return true; + } + + /** + * @brief Gets the default encoding a string pattern with no [[encoding]] + * attribute of its own resolves to + * @return The encoding name, empty when none is set + */ + [[nodiscard]] const std::string& getDefaultEncoding() const { + return this->m_defaultEncoding; + } + + /** + * @brief Sets a host-supplied check for whether an encoding name means + * anything. #pragma encoding fails when this rejects its value. + * @param validator Called with a candidate encoding name. Pass nullptr + * (the default) to accept any name. + */ + void setEncodingValidator(std::function validator) { + this->m_encodingValidator = std::move(validator); + } + + /** + * @brief Sets a callback invoked whenever setDefaultEncoding() actually + * changes the default encoding, after it passes any validator + * @param callback Called with the new encoding name + */ + void setOnDefaultEncodingChanged(std::function callback) { + this->m_onDefaultEncodingChanged = std::move(callback); + } + void setDataBaseAddress(u64 baseAddress) { this->m_dataBaseAddress = baseAddress; } @@ -560,6 +625,10 @@ namespace pl::core { std::vector> m_currentTemplateArguments; std::function m_dangerousFunctionCalledCallback = []{ return false; }; + std::shared_ptr m_stringEncodeDecode; + std::string m_defaultEncoding; + std::function m_encodingValidator; + std::function m_onDefaultEncodingChanged; std::function m_breakpointHitCallback = []{ }; std::atomic m_allowDangerousFunctions = DangerousFunctionPermission::Ask; ControlFlowStatement m_currControlFlowStatement = ControlFlowStatement::None; diff --git a/lib/include/pl/core/string_encode_decode.hpp b/lib/include/pl/core/string_encode_decode.hpp new file mode 100644 index 00000000..26832282 --- /dev/null +++ b/lib/include/pl/core/string_encode_decode.hpp @@ -0,0 +1,53 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace pl::core { + + /** + * @brief A pluggable text codec for string patterns + * @note The host application sets this on the Evaluator. With none set, string + * patterns keep their old raw-byte behavior. + */ + class StringEncodeDecode { + public: + virtual ~StringEncodeDecode() = default; + + /** + * @brief Decodes `bytes` under `encoding` + * @param bytes Bytes to decode + * @param encoding Encoding to decode with; empty when the pattern names no + * encoding, in which case the codec picks its own default + * @return The decoded text, or std::nullopt when `bytes` is not valid under + * `encoding` + */ + [[nodiscard]] virtual std::optional decode(std::span bytes, std::string_view encoding) const = 0; + + /** + * @brief Encodes `text` under `encoding` + * @param text Text to encode + * @param encoding Encoding to encode with; empty when the pattern names no + * encoding, in which case the codec picks its own default + * @return The encoded bytes, or std::nullopt when `text` is not representable + * under `encoding` + */ + [[nodiscard]] virtual std::optional> encode(std::string_view text, std::string_view encoding) const = 0; + + /** + * @brief Encodes `text` under `encoding`, never failing + * @param text Text to encode + * @param encoding Encoding to encode with; empty when the pattern names no + * encoding, in which case the codec picks its own default + * @return The encoded bytes, substituting a replacement for anything `encoding` + * cannot represent + */ + [[nodiscard]] virtual std::vector encodeLossy(std::string_view text, std::string_view encoding) const = 0; + }; + +} diff --git a/lib/include/pl/pattern_language.hpp b/lib/include/pl/pattern_language.hpp index 2cffb219..ee36c4d4 100644 --- a/lib/include/pl/pattern_language.hpp +++ b/lib/include/pl/pattern_language.hpp @@ -294,6 +294,14 @@ namespace pl { */ void reset(); + /** + * @brief Clears every placed pattern's cached display value, across every section. + * Call this after changing something a pattern's formatted value depends on + * without re-running the pattern, such as the host application's declared + * string encoding. + */ + void clearFormatCaches(); + /** * @brief Checks whether the runtime is currently running * @return True if the runtime is running, false otherwise diff --git a/lib/include/pl/patterns/pattern.hpp b/lib/include/pl/patterns/pattern.hpp index f3c474f8..e2574e05 100644 --- a/lib/include/pl/patterns/pattern.hpp +++ b/lib/include/pl/patterns/pattern.hpp @@ -484,6 +484,7 @@ namespace pl::ptrn { if (!result.empty()) { this->getEvaluator()->writeData(this->getOffset(), result.data(), result.size(), this->getSection()); this->clearFormatCache(); + this->clearByteCache(); } } diff --git a/lib/include/pl/patterns/pattern_string.hpp b/lib/include/pl/patterns/pattern_string.hpp index 1d42d350..9cbf8e40 100644 --- a/lib/include/pl/patterns/pattern_string.hpp +++ b/lib/include/pl/patterns/pattern_string.hpp @@ -3,6 +3,7 @@ #include #include +#include namespace pl::ptrn { @@ -32,23 +33,84 @@ namespace pl::ptrn { } + /** + * @brief Gets this string's own encoding + * @return This pattern's own [[encoding]] attribute, the evaluator's default + * encoding if none, or empty when neither is set. An empty result falls back + * to the codec's own default. + */ + [[nodiscard]] std::string getEncodingName() const { + if (const auto &arguments = this->getAttributeArguments("encoding"); !arguments.empty()) + return arguments[0].toString(true); + return this->getEvaluator()->getDefaultEncoding(); + } + std::string getValue(size_t size) const { if (size == 0) return ""; + auto *evaluator = this->getEvaluator(); + std::string buffer(size, '\x00'); - this->getEvaluator()->readData(this->getOffset(), buffer.data(), size, this->getSection()); + evaluator->readData(this->getOffset(), buffer.data(), size, this->getSection()); + + if (const auto &codec = evaluator->getStringEncodeDecode(); codec != nullptr) { + const auto encoding = this->getEncodingName(); + auto decoded = codec->decode({ reinterpret_cast(buffer.data()), buffer.size() }, encoding); + if (!decoded.has_value()) + core::err::E0004.throwError(fmt::format("invalid byte sequence for encoding '{}'", encoding)); + + return *decoded; + } return buffer; } std::vector getBytesOf(const core::Token::Literal &value) const override { - if (auto stringValue = std::get_if(&value); stringValue != nullptr) - return { stringValue->begin(), stringValue->end() }; - else + if (auto stringValue = std::get_if(&value); stringValue != nullptr) { + std::vector bytes; + + if (const auto &codec = this->getEvaluator()->getStringEncodeDecode(); codec != nullptr) { + const auto encoding = this->getEncodingName(); + auto encoded = codec->encode(*stringValue, encoding); + if (!encoded.has_value()) + core::err::E0004.throwError(fmt::format("text has no byte value in encoding '{}'", encoding)); + + bytes = *encoded; + } else { + bytes = { stringValue->begin(), stringValue->end() }; + } + + // This field owns a fixed number of bytes in the file. A longer write would + // overwrite whatever comes right after it. Truncate or pad with NUL to the + // field's own size, the same contract every other pattern type already has. + bytes.resize(this->getSize()); + return bytes; + } else return { }; } + /** + * @brief Force-writes `value`, substituting a replacement character for + * anything the pattern's encoding cannot represent + * @param value Value to write + * @note setValue() rejects such a value instead; an editor that offers an + * explicit lossy override calls this one directly. + */ + void setValueLossy(const std::string &value) { + std::vector bytes; + + if (const auto &codec = this->getEvaluator()->getStringEncodeDecode(); codec != nullptr) + bytes = codec->encodeLossy(value, this->getEncodingName()); + else + bytes = { value.begin(), value.end() }; + + bytes.resize(this->getSize()); + this->getEvaluator()->writeData(this->getOffset(), bytes.data(), bytes.size(), this->getSection()); + this->clearFormatCache(); + this->clearByteCache(); + } + [[nodiscard]] std::string getFormattedName() const override { return "String"; } @@ -67,23 +129,72 @@ namespace pl::ptrn { } std::string formatDisplayValue() override { - auto size = std::min(this->getSize(), 0x7F); + auto *evaluator = this->getEvaluator(); + const auto fullSize = this->getSize(); + + // Read a little past DisplayBudget, so a multi-byte codepoint at the + // cutoff usually has the bytes to decode whole. The decode below still + // backs off further on failure; this only keeps that the common case. + constexpr size_t DisplayBudget = 0x7F; + auto size = std::min(fullSize, DisplayBudget + 8); if (size == 0) return "\"\""; std::string buffer(size, 0x00); - this->getEvaluator()->readData(this->getOffset(), buffer.data(), size, this->getSection()); - - const auto pos = buffer.find_last_not_of('\x00'); - if (pos == std::string::npos) - return "\"\""; - - buffer.erase(pos + 1); - - auto displayString = hlp::encodeByteString({ buffer.begin(), buffer.end() }); - - return Pattern::callUserFormatFunc(buffer).value_or(fmt::format("\"{0}\" {1}", displayString, size > this->getSize() ? "(truncated)" : "")); + evaluator->readData(this->getOffset(), buffer.data(), size, this->getSection()); + + if (auto formatted = Pattern::callUserFormatFunc(buffer); formatted.has_value()) + return *formatted; + + const auto &codec = evaluator->getStringEncodeDecode(); + bool truncated = size < fullSize; + + // No codec configured keeps the old raw-byte display, unescaped by any + // encoding. A configured codec reports an undecodable buffer as invalid, + // rather than substituting a replacement character into the display. + if (codec == nullptr) { + if (buffer.size() > DisplayBudget) { + buffer.resize(DisplayBudget); + truncated = true; + } + + const auto pos = buffer.find_last_not_of('\x00'); + if (pos == std::string::npos) + return "\"\""; + buffer.erase(pos + 1); + + return fmt::format("\"{0}\" {1}", hlp::encodeByteString({ buffer.begin(), buffer.end() }), truncated ? "(truncated)" : ""); + } + + const auto encoding = this->getEncodingName(); + + // The read above can itself end mid-codepoint. decode() fails on the whole + // buffer in that case, not just its incomplete tail, so back off a few + // bytes and retry rather than reporting a merely-truncated read as invalid. + std::optional decoded; + for (size_t backoff = 0; backoff <= std::min(4, buffer.size()); ++backoff) { + decoded = codec->decode({ reinterpret_cast(buffer.data()), buffer.size() - backoff }, encoding); + if (decoded.has_value()) { + if (backoff > 0) + truncated = true; + break; + } + } + if (!decoded.has_value()) + core::err::E0004.throwError(fmt::format("invalid byte sequence for encoding '{}'", encoding)); + + // Trim the decoded text itself, not the raw bytes, so a multi-byte + // codepoint at the cutoff stays whole instead of splitting mid-sequence. + if (decoded->size() > DisplayBudget) { + auto cut = DisplayBudget; + while (cut > 0 && (static_cast((*decoded)[cut]) & 0xC0) == 0x80) + --cut; + decoded->resize(cut); + truncated = true; + } + + return fmt::format("\"{0}\" {1}", *decoded, truncated ? "(truncated)" : ""); } std::shared_ptr getEntry(size_t index) const override { diff --git a/lib/source/pl/lib/std/pragmas.cpp b/lib/source/pl/lib/std/pragmas.cpp index 233c4814..f1eea63e 100644 --- a/lib/source/pl/lib/std/pragmas.cpp +++ b/lib/source/pl/lib/std/pragmas.cpp @@ -38,6 +38,10 @@ namespace pl::lib::libstd { return false; }); + runtime.addPragma("encoding", [](pl::PatternLanguage &runtime, const std::string &value) { + return runtime.getInternals().evaluator->setDefaultEncoding(value); + }); + runtime.addPragma("eval_depth", [](pl::PatternLanguage &runtime, const std::string &value) { auto limit = parseLimit(value); if (!limit.has_value()) diff --git a/lib/source/pl/pattern_language.cpp b/lib/source/pl/pattern_language.cpp index f161e264..0b638f66 100644 --- a/lib/source/pl/pattern_language.cpp +++ b/lib/source/pl/pattern_language.cpp @@ -533,6 +533,13 @@ namespace pl { this->m_parserManager.setPatternLanguage(this); } + void PatternLanguage::clearFormatCaches() { + for (const auto &[section, patterns] : this->m_patterns) { + for (const auto &pattern : patterns) + pattern->clearFormatCache(); + } + } + void PatternLanguage::addFunction(const api::Namespace &ns, const std::string &name, api::FunctionParameterCount parameterCount, const api::FunctionCallback &func) { this->m_functions.emplace_back(ns, name, parameterCount, func, false); }