feat: Publish a per-run snapshot of string pattern encodings

lib/include/pl/core/evaluator.hpp

@@ -13,6 +13,7 @@
 
 #include <pl/core/log_console.hpp>
 #include <pl/core/token.hpp>
+#include <pl/core/string_encode_decode.hpp>
 #include <pl/api.hpp>
 
 #include <pl/core/errors/runtime_errors.hpp>
@@ -207,6 +208,15 @@ namespace pl::core {
 
         void setDataSource(u64 baseAddress, size_t dataSize, std::function<void(u64, u8*, size_t)> readerFunction, std::optional<std::function<void(u64, const u8*, size_t)>> writerFunction = std::nullopt);
 
+        // Sets the text codec for string patterns. Pass nullptr to go back to raw bytes.
+        void setStringEncodeDecode(std::shared_ptr<StringEncodeDecode> codec) {
+            this->m_stringEncodeDecode = std::move(codec);
+        }
+
+        [[nodiscard]] const std::shared_ptr<StringEncodeDecode>& getStringEncodeDecode() const {
+            return this->m_stringEncodeDecode;
+        }
+
         void setDataBaseAddress(u64 baseAddress) {
             this->m_dataBaseAddress = baseAddress;
         }
@@ -556,6 +566,7 @@ namespace pl::core {
         std::vector<std::unique_ptr<ast::ASTNode>> m_currentTemplateArguments;
 
         std::function<bool()> m_dangerousFunctionCalledCallback = []{ return false; };
+        std::shared_ptr<StringEncodeDecode> m_stringEncodeDecode;
         std::function<void()> m_breakpointHitCallback = []{ };
         std::atomic<DangerousFunctionPermission> m_allowDangerousFunctions = DangerousFunctionPermission::Ask;
         ControlFlowStatement m_currControlFlowStatement = ControlFlowStatement::None;

lib/include/pl/core/string_encode_decode.hpp

@@ -0,0 +1,31 @@
+#pragma once
+
+#include <optional>
+#include <span>
+#include <string>
+#include <string_view>
+#include <vector>
+
+#include <pl/helpers/types.hpp>
+
+namespace pl::core {
+
+    // A pluggable text codec for string patterns. The host application sets this on the
+    // Evaluator. With none set, string patterns keep their old raw-byte behavior.
+    //
+    // decode() and encode() are fallible: std::nullopt when `bytes`/`text` is not
+    // valid or representable under the named encoding. encodeLossy() never fails,
+    // substituting a replacement for anything it cannot represent.
+    class StringEncodeDecode {
+    public:
+        virtual ~StringEncodeDecode() = default;
+
+        // `encoding` is empty when the pattern names no encoding; the codec picks
+        // its own default then.
+        [[nodiscard]] virtual std::optional<std::string> decode(std::span<const u8> bytes, std::string_view encoding) const = 0;
+        [[nodiscard]] virtual std::optional<std::vector<u8>> encode(std::string_view text, std::string_view encoding) const = 0;
+
+        [[nodiscard]] virtual std::vector<u8> encodeLossy(std::string_view text, std::string_view encoding) const = 0;
+    };
+
+}

lib/include/pl/core/string_encoding_region.hpp

@@ -0,0 +1,18 @@
+#pragma once
+
+#include <string>
+
+#include <pl/helpers/types.hpp>
+
+namespace pl::core {
+
+    // The encoding a string pattern resolved to on the last successful run. Plain data, not a
+    // reference to the pattern. See PatternLanguage::getStringEncodingRegions().
+    struct StringEncodingRegion {
+        u64 section;
+        u64 address;
+        u64 size;
+        std::string encoding;
+    };
+
+}

lib/include/pl/pattern_language.hpp

@@ -18,6 +18,7 @@
 #include <pl/core/resolver.hpp>
 #include <pl/core/resolvers.hpp>
 #include <pl/core/parser_manager.hpp>
+#include <pl/core/string_encoding_region.hpp>
 
 #include <pl/helpers/types.hpp>
 
@@ -280,6 +281,22 @@ namespace pl {
          */
         [[nodiscard]] std::vector<ptrn::Pattern *> getPatternsAtAddress(u64 address, u64 section = 0x00) const;
 
+        /**
+         * @brief Gets a snapshot of what encoding every string pattern resolved to on the last
+         * successful run. Unlike the pattern tree itself, this is safe to read from any thread.
+         * @return The snapshot. Never null; empty before the first successful run.
+         */
+        [[nodiscard]] std::shared_ptr<const std::vector<core::StringEncodingRegion>> getStringEncodingRegions() const;
+
+        /**
+         * @brief Looks up the encoding a string pattern at an exact address and size resolved to
+         * @param address Address to check
+         * @param size Size to check
+         * @param section Section id
+         * @return The encoding name, or std::nullopt if no string pattern matches exactly
+         */
+        [[nodiscard]] std::optional<std::string> findStringEncoding(u64 address, u64 size, u64 section = 0x00) const;
+
 
         /**
          * @brief Get the colors of all patterns that overlap with the given address
@@ -294,6 +311,14 @@ namespace pl {
          */
         void reset();
 
+        /**
+         * @brief Clears every placed pattern's cached display value, across every section.
+         * Call this after changing something a pattern's formatted value depends on
+         * without re-running the pattern, such as the host application's declared
+         * string encoding.
+         */
+        void clearFormatCaches();
+
         /**
          * @brief Checks whether the runtime is currently running
          * @return True if the runtime is running, false otherwise
@@ -413,6 +438,7 @@ namespace pl {
 
     private:
         void flattenPatterns();
+        void buildStringEncodingRegions();
 
     private:
         Internals m_internals;
@@ -430,6 +456,8 @@ namespace pl {
         std::atomic<bool> m_flattenedPatternsValid = false;
         std::map<u64, wolv::container::IntervalTree<ptrn::Pattern*, u64, 8>> m_flattenedPatterns;
         std::thread m_flattenThread;
+        std::atomic<std::shared_ptr<const std::vector<core::StringEncodingRegion>>> m_stringEncodingRegions
+            = std::make_shared<const std::vector<core::StringEncodingRegion>>();
         std::vector<std::function<void(PatternLanguage&)>> m_cleanupCallbacks;
         std::vector<std::shared_ptr<core::ast::ASTNode>> m_currAST;
 

lib/include/pl/patterns/pattern.hpp

@@ -484,6 +484,7 @@ namespace pl::ptrn {
             if (!result.empty()) {
                 this->getEvaluator()->writeData(this->getOffset(), result.data(), result.size(), this->getSection());
                 this->clearFormatCache();
+                this->clearByteCache();
             }
         }
 

lib/include/pl/patterns/pattern_string.hpp

@@ -3,6 +3,9 @@
 #include <pl/patterns/pattern.hpp>
 
 #include <pl/patterns/pattern_character.hpp>
+#include <pl/core/errors/runtime_errors.hpp>
+
+#include <stdexcept>
 
 namespace pl::ptrn {
 
@@ -32,23 +35,76 @@ namespace pl::ptrn {
 
         }
 
+        // Empty when this pattern names no encoding. The codec then falls back to its own
+        // default.
+        [[nodiscard]] std::string getEncodingName() const {
+            if (const auto &arguments = this->getAttributeArguments("encoding"); !arguments.empty())
+                return arguments[0].toString(true);
+            return "";
+        }
+
         std::string getValue(size_t size) const {
             if (size == 0)
                 return "";
 
+            auto *evaluator = this->getEvaluator();
+
             std::string buffer(size, '\x00');
-            this->getEvaluator()->readData(this->getOffset(), buffer.data(), size, this->getSection());
+            evaluator->readData(this->getOffset(), buffer.data(), size, this->getSection());
+
+            if (const auto &codec = evaluator->getStringEncodeDecode(); codec != nullptr) {
+                const auto encoding = this->getEncodingName();
+                auto decoded = codec->decode({ reinterpret_cast<const u8*>(buffer.data()), buffer.size() }, encoding);
+                if (!decoded.has_value())
+                    core::err::E0004.throwError(fmt::format("invalid byte sequence for encoding '{}'", encoding));
+
+                return *decoded;
+            }
 
             return buffer;
         }
 
         std::vector<u8> getBytesOf(const core::Token::Literal &value) const override {
-            if (auto stringValue = std::get_if<std::string>(&value); stringValue != nullptr)
-                return { stringValue->begin(), stringValue->end() };
-            else
+            if (auto stringValue = std::get_if<std::string>(&value); stringValue != nullptr) {
+                std::vector<u8> bytes;
+
+                if (const auto &codec = this->getEvaluator()->getStringEncodeDecode(); codec != nullptr) {
+                    const auto encoding = this->getEncodingName();
+                    auto encoded = codec->encode(*stringValue, encoding);
+                    if (!encoded.has_value())
+                        core::err::E0004.throwError(fmt::format("text has no byte value in encoding '{}'", encoding));
+
+                    bytes = *encoded;
+                } else {
+                    bytes = { stringValue->begin(), stringValue->end() };
+                }
+
+                // This field owns a fixed number of bytes in the file. A longer write would
+                // overwrite whatever comes right after it. Truncate or pad with NUL to the
+                // field's own size, the same contract every other pattern type already has.
+                bytes.resize(this->getSize());
+                return bytes;
+            } else
                 return { };
         }
 
+        // Force-writes `value`, substituting a replacement character for anything the
+        // pattern's encoding cannot represent. setValue() rejects such a value instead;
+        // an editor that offers an explicit lossy override calls this one directly.
+        void setValueLossy(const std::string &value) {
+            std::vector<u8> bytes;
+
+            if (const auto &codec = this->getEvaluator()->getStringEncodeDecode(); codec != nullptr)
+                bytes = codec->encodeLossy(value, this->getEncodingName());
+            else
+                bytes = { value.begin(), value.end() };
+
+            bytes.resize(this->getSize());
+            this->getEvaluator()->writeData(this->getOffset(), bytes.data(), bytes.size(), this->getSection());
+            this->clearFormatCache();
+            this->clearByteCache();
+        }
+
         [[nodiscard]] std::string getFormattedName() const override {
             return "String";
         }
@@ -67,23 +123,39 @@ namespace pl::ptrn {
         }
 
         std::string formatDisplayValue() override {
-            auto size = std::min<size_t>(this->getSize(), 0x7F);
+            auto *evaluator = this->getEvaluator();
+            const auto fullSize = this->getSize();
+            auto size = std::min<size_t>(fullSize, 0x7F);
 
             if (size == 0)
                 return "\"\"";
 
             std::string buffer(size, 0x00);
-            this->getEvaluator()->readData(this->getOffset(), buffer.data(), size, this->getSection());
+            evaluator->readData(this->getOffset(), buffer.data(), size, this->getSection());
 
-            const auto pos = buffer.find_last_not_of('\x00');
-            if (pos == std::string::npos)
-                return "\"\"";
+            if (auto formatted = Pattern::callUserFormatFunc(buffer); formatted.has_value())
+                return *formatted;
+
+            const auto &codec = evaluator->getStringEncodeDecode();
+            const auto truncatedSuffix = size > fullSize ? "(truncated)" : "";
+
+            // No codec configured keeps the old raw-byte display, unescaped by any
+            // encoding. A configured codec reports an undecodable buffer as invalid,
+            // rather than substituting a replacement character into the display.
+            if (codec == nullptr) {
+                const auto pos = buffer.find_last_not_of('\x00');
+                if (pos == std::string::npos)
+                    return "\"\"";
+                buffer.erase(pos + 1);
 
-            buffer.erase(pos + 1);
+                return fmt::format("\"{0}\" {1}", hlp::encodeByteString({ buffer.begin(), buffer.end() }), truncatedSuffix);
+            }
 
-            auto displayString = hlp::encodeByteString({ buffer.begin(), buffer.end() });
+            auto decoded = codec->decode({ reinterpret_cast<const u8*>(buffer.data()), buffer.size() }, this->getEncodingName());
+            if (!decoded.has_value())
+                throw std::runtime_error("Invalid");
 
-            return Pattern::callUserFormatFunc(buffer).value_or(fmt::format("\"{0}\" {1}", displayString, size > this->getSize() ? "(truncated)" : ""));
+            return fmt::format("\"{0}\" {1}", *decoded, truncatedSuffix);
         }
 
         std::shared_ptr<Pattern> getEntry(size_t index) const override {