diff --git a/lib/include/pl/core/evaluator.hpp b/lib/include/pl/core/evaluator.hpp index 8d383757..4805b4b5 100644 --- a/lib/include/pl/core/evaluator.hpp +++ b/lib/include/pl/core/evaluator.hpp @@ -13,6 +13,7 @@ #include #include +#include #include #include @@ -207,6 +208,15 @@ namespace pl::core { void setDataSource(u64 baseAddress, size_t dataSize, std::function readerFunction, std::optional> writerFunction = std::nullopt); + // Sets the text codec for string patterns. Pass nullptr to go back to raw bytes. + void setStringEncodeDecode(std::shared_ptr codec) { + this->m_stringEncodeDecode = std::move(codec); + } + + [[nodiscard]] const std::shared_ptr& getStringEncodeDecode() const { + return this->m_stringEncodeDecode; + } + void setDataBaseAddress(u64 baseAddress) { this->m_dataBaseAddress = baseAddress; } @@ -556,6 +566,7 @@ namespace pl::core { std::vector> m_currentTemplateArguments; std::function m_dangerousFunctionCalledCallback = []{ return false; }; + std::shared_ptr m_stringEncodeDecode; std::function m_breakpointHitCallback = []{ }; std::atomic m_allowDangerousFunctions = DangerousFunctionPermission::Ask; ControlFlowStatement m_currControlFlowStatement = ControlFlowStatement::None; diff --git a/lib/include/pl/core/string_encode_decode.hpp b/lib/include/pl/core/string_encode_decode.hpp new file mode 100644 index 00000000..8c8e8fe5 --- /dev/null +++ b/lib/include/pl/core/string_encode_decode.hpp @@ -0,0 +1,31 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include + +namespace pl::core { + + // A pluggable text codec for string patterns. The host application sets this on the + // Evaluator. With none set, string patterns keep their old raw-byte behavior. + // + // decode() and encode() are fallible: std::nullopt when `bytes`/`text` is not + // valid or representable under the named encoding. encodeLossy() never fails, + // substituting a replacement for anything it cannot represent. + class StringEncodeDecode { + public: + virtual ~StringEncodeDecode() = default; + + // `encoding` is empty when the pattern names no encoding; the codec picks + // its own default then. + [[nodiscard]] virtual std::optional decode(std::span bytes, std::string_view encoding) const = 0; + [[nodiscard]] virtual std::optional> encode(std::string_view text, std::string_view encoding) const = 0; + + [[nodiscard]] virtual std::vector encodeLossy(std::string_view text, std::string_view encoding) const = 0; + }; + +} diff --git a/lib/include/pl/core/string_encoding_region.hpp b/lib/include/pl/core/string_encoding_region.hpp new file mode 100644 index 00000000..d8dfc8df --- /dev/null +++ b/lib/include/pl/core/string_encoding_region.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include + +#include + +namespace pl::core { + + // The encoding a string pattern resolved to on the last successful run. Plain data, not a + // reference to the pattern. See PatternLanguage::getStringEncodingRegions(). + struct StringEncodingRegion { + u64 section; + u64 address; + u64 size; + std::string encoding; + }; + +} diff --git a/lib/include/pl/pattern_language.hpp b/lib/include/pl/pattern_language.hpp index 2cffb219..14eaa4b7 100644 --- a/lib/include/pl/pattern_language.hpp +++ b/lib/include/pl/pattern_language.hpp @@ -18,6 +18,7 @@ #include #include #include +#include #include @@ -280,6 +281,22 @@ namespace pl { */ [[nodiscard]] std::vector getPatternsAtAddress(u64 address, u64 section = 0x00) const; + /** + * @brief Gets a snapshot of what encoding every string pattern resolved to on the last + * successful run. Unlike the pattern tree itself, this is safe to read from any thread. + * @return The snapshot. Never null; empty before the first successful run. + */ + [[nodiscard]] std::shared_ptr> getStringEncodingRegions() const; + + /** + * @brief Looks up the encoding a string pattern at an exact address and size resolved to + * @param address Address to check + * @param size Size to check + * @param section Section id + * @return The encoding name, or std::nullopt if no string pattern matches exactly + */ + [[nodiscard]] std::optional findStringEncoding(u64 address, u64 size, u64 section = 0x00) const; + /** * @brief Get the colors of all patterns that overlap with the given address @@ -294,6 +311,14 @@ namespace pl { */ void reset(); + /** + * @brief Clears every placed pattern's cached display value, across every section. + * Call this after changing something a pattern's formatted value depends on + * without re-running the pattern, such as the host application's declared + * string encoding. + */ + void clearFormatCaches(); + /** * @brief Checks whether the runtime is currently running * @return True if the runtime is running, false otherwise @@ -413,6 +438,7 @@ namespace pl { private: void flattenPatterns(); + void buildStringEncodingRegions(); private: Internals m_internals; @@ -430,6 +456,8 @@ namespace pl { std::atomic m_flattenedPatternsValid = false; std::map> m_flattenedPatterns; std::thread m_flattenThread; + std::atomic>> m_stringEncodingRegions + = std::make_shared>(); std::vector> m_cleanupCallbacks; std::vector> m_currAST; diff --git a/lib/include/pl/patterns/pattern.hpp b/lib/include/pl/patterns/pattern.hpp index 8fdeff5b..4710bbf7 100644 --- a/lib/include/pl/patterns/pattern.hpp +++ b/lib/include/pl/patterns/pattern.hpp @@ -484,6 +484,7 @@ namespace pl::ptrn { if (!result.empty()) { this->getEvaluator()->writeData(this->getOffset(), result.data(), result.size(), this->getSection()); this->clearFormatCache(); + this->clearByteCache(); } } diff --git a/lib/include/pl/patterns/pattern_string.hpp b/lib/include/pl/patterns/pattern_string.hpp index 1d42d350..f499deab 100644 --- a/lib/include/pl/patterns/pattern_string.hpp +++ b/lib/include/pl/patterns/pattern_string.hpp @@ -3,6 +3,9 @@ #include #include +#include + +#include namespace pl::ptrn { @@ -32,23 +35,76 @@ namespace pl::ptrn { } + // Empty when this pattern names no encoding. The codec then falls back to its own + // default. + [[nodiscard]] std::string getEncodingName() const { + if (const auto &arguments = this->getAttributeArguments("encoding"); !arguments.empty()) + return arguments[0].toString(true); + return ""; + } + std::string getValue(size_t size) const { if (size == 0) return ""; + auto *evaluator = this->getEvaluator(); + std::string buffer(size, '\x00'); - this->getEvaluator()->readData(this->getOffset(), buffer.data(), size, this->getSection()); + evaluator->readData(this->getOffset(), buffer.data(), size, this->getSection()); + + if (const auto &codec = evaluator->getStringEncodeDecode(); codec != nullptr) { + const auto encoding = this->getEncodingName(); + auto decoded = codec->decode({ reinterpret_cast(buffer.data()), buffer.size() }, encoding); + if (!decoded.has_value()) + core::err::E0004.throwError(fmt::format("invalid byte sequence for encoding '{}'", encoding)); + + return *decoded; + } return buffer; } std::vector getBytesOf(const core::Token::Literal &value) const override { - if (auto stringValue = std::get_if(&value); stringValue != nullptr) - return { stringValue->begin(), stringValue->end() }; - else + if (auto stringValue = std::get_if(&value); stringValue != nullptr) { + std::vector bytes; + + if (const auto &codec = this->getEvaluator()->getStringEncodeDecode(); codec != nullptr) { + const auto encoding = this->getEncodingName(); + auto encoded = codec->encode(*stringValue, encoding); + if (!encoded.has_value()) + core::err::E0004.throwError(fmt::format("text has no byte value in encoding '{}'", encoding)); + + bytes = *encoded; + } else { + bytes = { stringValue->begin(), stringValue->end() }; + } + + // This field owns a fixed number of bytes in the file. A longer write would + // overwrite whatever comes right after it. Truncate or pad with NUL to the + // field's own size, the same contract every other pattern type already has. + bytes.resize(this->getSize()); + return bytes; + } else return { }; } + // Force-writes `value`, substituting a replacement character for anything the + // pattern's encoding cannot represent. setValue() rejects such a value instead; + // an editor that offers an explicit lossy override calls this one directly. + void setValueLossy(const std::string &value) { + std::vector bytes; + + if (const auto &codec = this->getEvaluator()->getStringEncodeDecode(); codec != nullptr) + bytes = codec->encodeLossy(value, this->getEncodingName()); + else + bytes = { value.begin(), value.end() }; + + bytes.resize(this->getSize()); + this->getEvaluator()->writeData(this->getOffset(), bytes.data(), bytes.size(), this->getSection()); + this->clearFormatCache(); + this->clearByteCache(); + } + [[nodiscard]] std::string getFormattedName() const override { return "String"; } @@ -67,23 +123,39 @@ namespace pl::ptrn { } std::string formatDisplayValue() override { - auto size = std::min(this->getSize(), 0x7F); + auto *evaluator = this->getEvaluator(); + const auto fullSize = this->getSize(); + auto size = std::min(fullSize, 0x7F); if (size == 0) return "\"\""; std::string buffer(size, 0x00); - this->getEvaluator()->readData(this->getOffset(), buffer.data(), size, this->getSection()); + evaluator->readData(this->getOffset(), buffer.data(), size, this->getSection()); - const auto pos = buffer.find_last_not_of('\x00'); - if (pos == std::string::npos) - return "\"\""; + if (auto formatted = Pattern::callUserFormatFunc(buffer); formatted.has_value()) + return *formatted; + + const auto &codec = evaluator->getStringEncodeDecode(); + const auto truncatedSuffix = size > fullSize ? "(truncated)" : ""; + + // No codec configured keeps the old raw-byte display, unescaped by any + // encoding. A configured codec reports an undecodable buffer as invalid, + // rather than substituting a replacement character into the display. + if (codec == nullptr) { + const auto pos = buffer.find_last_not_of('\x00'); + if (pos == std::string::npos) + return "\"\""; + buffer.erase(pos + 1); - buffer.erase(pos + 1); + return fmt::format("\"{0}\" {1}", hlp::encodeByteString({ buffer.begin(), buffer.end() }), truncatedSuffix); + } - auto displayString = hlp::encodeByteString({ buffer.begin(), buffer.end() }); + auto decoded = codec->decode({ reinterpret_cast(buffer.data()), buffer.size() }, this->getEncodingName()); + if (!decoded.has_value()) + throw std::runtime_error("Invalid"); - return Pattern::callUserFormatFunc(buffer).value_or(fmt::format("\"{0}\" {1}", displayString, size > this->getSize() ? "(truncated)" : "")); + return fmt::format("\"{0}\" {1}", *decoded, truncatedSuffix); } std::shared_ptr getEntry(size_t index) const override { diff --git a/lib/source/pl/pattern_language.cpp b/lib/source/pl/pattern_language.cpp index f161e264..ed5d68ef 100644 --- a/lib/source/pl/pattern_language.cpp +++ b/lib/source/pl/pattern_language.cpp @@ -11,6 +11,7 @@ #include #include +#include #include @@ -18,6 +19,9 @@ #include #include +#include +#include + namespace pl { static std::string getFunctionName(const api::Namespace &ns, const std::string &name) { @@ -81,6 +85,7 @@ namespace pl { this->m_patterns = std::move(other.m_patterns); this->m_flattenedPatterns = std::move(other.m_flattenedPatterns); + this->m_stringEncodingRegions.store(other.m_stringEncodingRegions.load()); this->m_cleanupCallbacks = std::move(other.m_cleanupCallbacks); this->m_currAST = std::move(other.m_currAST); @@ -314,6 +319,7 @@ namespace pl { this->reset(); } else { this->flattenPatterns(); + this->buildStringEncodingRegions(); this->m_patternsValid = true; } @@ -501,6 +507,7 @@ namespace pl { this->m_patterns.clear(); this->m_flattenedPatterns.clear(); this->m_flattenedPatternsValid = false; + this->m_stringEncodingRegions.store(std::make_shared>()); this->m_currError.reset(); this->m_compileErrors.clear(); @@ -533,6 +540,13 @@ namespace pl { this->m_parserManager.setPatternLanguage(this); } + void PatternLanguage::clearFormatCaches() { + for (const auto &[section, patterns] : this->m_patterns) { + for (const auto &pattern : patterns) + pattern->clearFormatCache(); + } + } + void PatternLanguage::addFunction(const api::Namespace &ns, const std::string &name, api::FunctionParameterCount parameterCount, const api::FunctionCallback &func) { this->m_functions.emplace_back(ns, name, parameterCount, func, false); } @@ -587,6 +601,57 @@ namespace pl { } } + void PatternLanguage::buildStringEncodingRegions() { + auto regions = std::make_shared>(); + + std::function walk = [&](ptrn::Pattern *pattern) { + if (this->m_aborted) + return; + + if (auto string = dynamic_cast(pattern)) { + if (auto encoding = string->getEncodingName(); !encoding.empty()) + regions->push_back({ pattern->getSection(), pattern->getOffset(), pattern->getSize(), std::move(encoding) }); + } + + if (auto iterable = dynamic_cast(pattern)) { + for (const auto &child : iterable->getEntries()) + walk(child.get()); + } + }; + + for (const auto &[section, patterns] : this->m_patterns) { + for (const auto &pattern : patterns) { + if (this->m_aborted) + return; + + walk(pattern.get()); + } + } + + std::ranges::sort(*regions, {}, [](const core::StringEncodingRegion ®ion) { + return std::pair{ region.section, region.address }; + }); + + this->m_stringEncodingRegions.store(std::move(regions)); + } + + std::shared_ptr> PatternLanguage::getStringEncodingRegions() const { + return this->m_stringEncodingRegions.load(); + } + + std::optional PatternLanguage::findStringEncoding(u64 address, u64 size, u64 section) const { + auto regions = this->getStringEncodingRegions(); + + auto it = std::ranges::lower_bound(*regions, std::pair{ section, address }, {}, [](const core::StringEncodingRegion ®ion) { + return std::pair{ region.section, region.address }; + }); + + if (it != regions->end() && it->section == section && it->address == address && it->size == size) + return it->encoding; + + return std::nullopt; + } + std::vector PatternLanguage::getPatternsAtAddress(u64 address, u64 section) const { if (this->m_flattenedPatterns.empty() || !this->m_flattenedPatterns.contains(section)) return { };